From e27b88d5bd09eb111d2936f3c6214baf1839a194 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Tue, 29 Jul 2014 16:31:39 -0700 Subject: [PATCH] remove seek from std::io::MemWriter, add SeekableMemWriter to librustc Not all users of MemWriter need to seek, but having MemWriter seekable adds between 3-29% in overhead in certain circumstances. This fixes that performance gap by making a non-seekable MemWriter, and creating a new SeekableMemWriter for those circumstances when that functionality is actually needed. ``` test io::mem::test::bench_buf_reader ... bench: 682 ns/iter (+/- 85) test io::mem::test::bench_buf_writer ... bench: 580 ns/iter (+/- 57) test io::mem::test::bench_mem_reader ... bench: 793 ns/iter (+/- 99) test io::mem::test::bench_mem_writer_001_0000 ... bench: 48 ns/iter (+/- 27) test io::mem::test::bench_mem_writer_001_0010 ... bench: 65 ns/iter (+/- 27) = 153 MB/s test io::mem::test::bench_mem_writer_001_0100 ... bench: 132 ns/iter (+/- 12) = 757 MB/s test io::mem::test::bench_mem_writer_001_1000 ... bench: 802 ns/iter (+/- 151) = 1246 MB/s test io::mem::test::bench_mem_writer_100_0000 ... bench: 481 ns/iter (+/- 28) test io::mem::test::bench_mem_writer_100_0010 ... bench: 1957 ns/iter (+/- 126) = 510 MB/s test io::mem::test::bench_mem_writer_100_0100 ... bench: 8222 ns/iter (+/- 434) = 1216 MB/s test io::mem::test::bench_mem_writer_100_1000 ... bench: 82496 ns/iter (+/- 11191) = 1212 MB/s test io::mem::test::bench_seekable_mem_writer_001_0000 ... bench: 48 ns/iter (+/- 2) test io::mem::test::bench_seekable_mem_writer_001_0010 ... bench: 64 ns/iter (+/- 2) = 156 MB/s test io::mem::test::bench_seekable_mem_writer_001_0100 ... bench: 129 ns/iter (+/- 7) = 775 MB/s test io::mem::test::bench_seekable_mem_writer_001_1000 ... bench: 801 ns/iter (+/- 159) = 1248 MB/s test io::mem::test::bench_seekable_mem_writer_100_0000 ... bench: 711 ns/iter (+/- 51) test io::mem::test::bench_seekable_mem_writer_100_0010 ... bench: 2532 ns/iter (+/- 227) = 394 MB/s test io::mem::test::bench_seekable_mem_writer_100_0100 ... bench: 8962 ns/iter (+/- 947) = 1115 MB/s test io::mem::test::bench_seekable_mem_writer_100_1000 ... bench: 85086 ns/iter (+/- 11555) = 1175 MB/s ``` [breaking-change] --- src/librustc/lib.rs | 4 + src/librustc/metadata/encoder.rs | 24 +- src/librustc/metadata/tyencode.rs | 45 ++-- src/librustc/middle/astencode.rs | 8 +- src/librustc/util/io.rs | 232 ++++++++++++++++++ src/libserialize/ebml.rs | 120 ++++++++- src/libstd/io/fs.rs | 2 +- src/libstd/io/mem.rs | 77 +----- src/libsyntax/print/pprust.rs | 2 +- src/libuuid/lib.rs | 10 +- ...riving-encodable-decodable-cell-refcell.rs | 20 +- src/test/run-pass/issue-11881.rs | 127 +++++++++- 12 files changed, 528 insertions(+), 143 deletions(-) create mode 100644 src/librustc/util/io.rs diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index 2a7f7e8c54d..82040f14159 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -48,6 +48,9 @@ extern crate time; #[phase(plugin, link)] extern crate log; #[phase(plugin, link)] extern crate syntax; +#[cfg(test)] +extern crate test; + mod diagnostics; pub mod back { @@ -129,6 +132,7 @@ pub mod util { pub mod common; pub mod ppaux; + pub mod io; pub mod nodemap; } diff --git a/src/librustc/metadata/encoder.rs b/src/librustc/metadata/encoder.rs index 6665e913e19..dbda4f58d96 100644 --- a/src/librustc/metadata/encoder.rs +++ b/src/librustc/metadata/encoder.rs @@ -26,6 +26,7 @@ use middle::ty; use middle::typeck; use middle::stability; use middle; +use util::io::SeekableMemWriter; use util::nodemap::{NodeMap, NodeSet}; use serialize::Encodable; @@ -33,7 +34,6 @@ use std::cell::RefCell; use std::gc::Gc; use std::hash::Hash; use std::hash; -use std::io::MemWriter; use std::mem; use std::collections::HashMap; use syntax::abi; @@ -61,7 +61,7 @@ pub enum InlinedItemRef<'a> { IIForeignRef(&'a ast::ForeignItem) } -pub type Encoder<'a> = writer::Encoder<'a, MemWriter>; +pub type Encoder<'a> = writer::Encoder<'a, SeekableMemWriter>; pub type EncodeInlinedItem<'a> = |ecx: &EncodeContext, ebml_w: &mut Encoder, @@ -1407,7 +1407,7 @@ fn encode_info_for_items(ecx: &EncodeContext, // Path and definition ID indexing fn encode_index(ebml_w: &mut Encoder, index: Vec>, - write_fn: |&mut MemWriter, &T|) { + write_fn: |&mut SeekableMemWriter, &T|) { let mut buckets: Vec>> = Vec::from_fn(256, |_| Vec::new()); for elt in index.move_iter() { let h = hash::hash(&elt.val) as uint; @@ -1424,7 +1424,7 @@ fn encode_index(ebml_w: &mut Encoder, index: Vec>, ebml_w.start_tag(tag_index_buckets_bucket_elt); assert!(elt.pos < 0xffff_ffff); { - let wr: &mut MemWriter = ebml_w.writer; + let wr: &mut SeekableMemWriter = ebml_w.writer; wr.write_be_u32(elt.pos as u32); } write_fn(ebml_w.writer, &elt.val); @@ -1436,15 +1436,15 @@ fn encode_index(ebml_w: &mut Encoder, index: Vec>, ebml_w.start_tag(tag_index_table); for pos in bucket_locs.iter() { assert!(*pos < 0xffff_ffff); - let wr: &mut MemWriter = ebml_w.writer; + let wr: &mut SeekableMemWriter = ebml_w.writer; wr.write_be_u32(*pos as u32); } ebml_w.end_tag(); ebml_w.end_tag(); } -fn write_i64(writer: &mut MemWriter, &n: &i64) { - let wr: &mut MemWriter = writer; +fn write_i64(writer: &mut SeekableMemWriter, &n: &i64) { + let wr: &mut SeekableMemWriter = writer; assert!(n < 0x7fff_ffff); wr.write_be_u32(n as u32); } @@ -1545,14 +1545,14 @@ fn encode_lang_items(ecx: &EncodeContext, ebml_w: &mut Encoder) { ebml_w.start_tag(tag_lang_items_item_id); { - let wr: &mut MemWriter = ebml_w.writer; + let wr: &mut SeekableMemWriter = ebml_w.writer; wr.write_be_u32(i as u32); } ebml_w.end_tag(); // tag_lang_items_item_id ebml_w.start_tag(tag_lang_items_item_node_id); { - let wr: &mut MemWriter = ebml_w.writer; + let wr: &mut SeekableMemWriter = ebml_w.writer; wr.write_be_u32(id.node as u32); } ebml_w.end_tag(); // tag_lang_items_item_node_id @@ -1824,12 +1824,12 @@ pub static metadata_encoding_version : &'static [u8] = 0, 0, 0, 1 ]; pub fn encode_metadata(parms: EncodeParams, krate: &Crate) -> Vec { - let mut wr = MemWriter::new(); + let mut wr = SeekableMemWriter::new(); encode_metadata_inner(&mut wr, parms, krate); wr.unwrap().move_iter().collect() } -fn encode_metadata_inner(wr: &mut MemWriter, parms: EncodeParams, krate: &Crate) { +fn encode_metadata_inner(wr: &mut SeekableMemWriter, parms: EncodeParams, krate: &Crate) { struct Stats { attr_bytes: u64, dep_bytes: u64, @@ -1982,7 +1982,7 @@ fn encode_metadata_inner(wr: &mut MemWriter, parms: EncodeParams, krate: &Crate) // Get the encoded string for a type pub fn encoded_ty(tcx: &ty::ctxt, t: ty::t) -> String { - let mut wr = MemWriter::new(); + let mut wr = SeekableMemWriter::new(); tyencode::enc_ty(&mut wr, &tyencode::ctxt { diag: tcx.sess.diagnostic(), ds: def_to_string, diff --git a/src/librustc/metadata/tyencode.rs b/src/librustc/metadata/tyencode.rs index 15e4e85ddb7..f301dc3760d 100644 --- a/src/librustc/metadata/tyencode.rs +++ b/src/librustc/metadata/tyencode.rs @@ -15,7 +15,6 @@ use std::cell::RefCell; use std::collections::HashMap; -use std::io::MemWriter; use middle::subst; use middle::subst::VecPerParamSpace; @@ -28,6 +27,8 @@ use syntax::ast::*; use syntax::diagnostic::SpanHandler; use syntax::parse::token; +use util::io::SeekableMemWriter; + macro_rules! mywrite( ($($arg:tt)*) => ({ write!($($arg)*); }) ) pub struct ctxt<'a> { @@ -48,7 +49,7 @@ pub struct ty_abbrev { pub type abbrev_map = RefCell>; -pub fn enc_ty(w: &mut MemWriter, cx: &ctxt, t: ty::t) { +pub fn enc_ty(w: &mut SeekableMemWriter, cx: &ctxt, t: ty::t) { match cx.abbrevs.borrow_mut().find(&t) { Some(a) => { w.write(a.s.as_bytes()); return; } None => {} @@ -72,19 +73,19 @@ pub fn enc_ty(w: &mut MemWriter, cx: &ctxt, t: ty::t) { } } -fn enc_mutability(w: &mut MemWriter, mt: ast::Mutability) { +fn enc_mutability(w: &mut SeekableMemWriter, mt: ast::Mutability) { match mt { MutImmutable => (), MutMutable => mywrite!(w, "m"), } } -fn enc_mt(w: &mut MemWriter, cx: &ctxt, mt: ty::mt) { +fn enc_mt(w: &mut SeekableMemWriter, cx: &ctxt, mt: ty::mt) { enc_mutability(w, mt.mutbl); enc_ty(w, cx, mt.ty); } -fn enc_opt(w: &mut MemWriter, t: Option, enc_f: |&mut MemWriter, T|) { +fn enc_opt(w: &mut SeekableMemWriter, t: Option, enc_f: |&mut SeekableMemWriter, T|) { match t { None => mywrite!(w, "n"), Some(v) => { @@ -94,10 +95,10 @@ fn enc_opt(w: &mut MemWriter, t: Option, enc_f: |&mut MemWriter, T|) { } } -fn enc_vec_per_param_space(w: &mut MemWriter, +fn enc_vec_per_param_space(w: &mut SeekableMemWriter, cx: &ctxt, v: &VecPerParamSpace, - op: |&mut MemWriter, &ctxt, &T|) { + op: |&mut SeekableMemWriter, &ctxt, &T|) { for &space in subst::ParamSpace::all().iter() { mywrite!(w, "["); for t in v.get_slice(space).iter() { @@ -107,13 +108,13 @@ fn enc_vec_per_param_space(w: &mut MemWriter, } } -pub fn enc_substs(w: &mut MemWriter, cx: &ctxt, substs: &subst::Substs) { +pub fn enc_substs(w: &mut SeekableMemWriter, cx: &ctxt, substs: &subst::Substs) { enc_region_substs(w, cx, &substs.regions); enc_vec_per_param_space(w, cx, &substs.types, |w, cx, &ty| enc_ty(w, cx, ty)); } -fn enc_region_substs(w: &mut MemWriter, cx: &ctxt, substs: &subst::RegionSubsts) { +fn enc_region_substs(w: &mut SeekableMemWriter, cx: &ctxt, substs: &subst::RegionSubsts) { match *substs { subst::ErasedRegions => { mywrite!(w, "e"); @@ -126,7 +127,7 @@ fn enc_region_substs(w: &mut MemWriter, cx: &ctxt, substs: &subst::RegionSubsts) } } -fn enc_region(w: &mut MemWriter, cx: &ctxt, r: ty::Region) { +fn enc_region(w: &mut SeekableMemWriter, cx: &ctxt, r: ty::Region) { match r { ty::ReLateBound(id, br) => { mywrite!(w, "b[{}|", id); @@ -161,7 +162,7 @@ fn enc_region(w: &mut MemWriter, cx: &ctxt, r: ty::Region) { } } -fn enc_bound_region(w: &mut MemWriter, cx: &ctxt, br: ty::BoundRegion) { +fn enc_bound_region(w: &mut SeekableMemWriter, cx: &ctxt, br: ty::BoundRegion) { match br { ty::BrAnon(idx) => { mywrite!(w, "a{}|", idx); @@ -177,12 +178,12 @@ fn enc_bound_region(w: &mut MemWriter, cx: &ctxt, br: ty::BoundRegion) { } } -pub fn enc_trait_ref(w: &mut MemWriter, cx: &ctxt, s: &ty::TraitRef) { +pub fn enc_trait_ref(w: &mut SeekableMemWriter, cx: &ctxt, s: &ty::TraitRef) { mywrite!(w, "{}|", (cx.ds)(s.def_id)); enc_substs(w, cx, &s.substs); } -pub fn enc_trait_store(w: &mut MemWriter, cx: &ctxt, s: ty::TraitStore) { +pub fn enc_trait_store(w: &mut SeekableMemWriter, cx: &ctxt, s: ty::TraitStore) { match s { ty::UniqTraitStore => mywrite!(w, "~"), ty::RegionTraitStore(re, m) => { @@ -193,7 +194,7 @@ pub fn enc_trait_store(w: &mut MemWriter, cx: &ctxt, s: ty::TraitStore) { } } -fn enc_sty(w: &mut MemWriter, cx: &ctxt, st: &ty::sty) { +fn enc_sty(w: &mut SeekableMemWriter, cx: &ctxt, st: &ty::sty) { match *st { ty::ty_nil => mywrite!(w, "n"), ty::ty_bot => mywrite!(w, "z"), @@ -293,33 +294,33 @@ fn enc_sty(w: &mut MemWriter, cx: &ctxt, st: &ty::sty) { } } -fn enc_fn_style(w: &mut MemWriter, p: FnStyle) { +fn enc_fn_style(w: &mut SeekableMemWriter, p: FnStyle) { match p { NormalFn => mywrite!(w, "n"), UnsafeFn => mywrite!(w, "u"), } } -fn enc_abi(w: &mut MemWriter, abi: Abi) { +fn enc_abi(w: &mut SeekableMemWriter, abi: Abi) { mywrite!(w, "["); mywrite!(w, "{}", abi.name()); mywrite!(w, "]") } -fn enc_onceness(w: &mut MemWriter, o: Onceness) { +fn enc_onceness(w: &mut SeekableMemWriter, o: Onceness) { match o { Once => mywrite!(w, "o"), Many => mywrite!(w, "m") } } -pub fn enc_bare_fn_ty(w: &mut MemWriter, cx: &ctxt, ft: &ty::BareFnTy) { +pub fn enc_bare_fn_ty(w: &mut SeekableMemWriter, cx: &ctxt, ft: &ty::BareFnTy) { enc_fn_style(w, ft.fn_style); enc_abi(w, ft.abi); enc_fn_sig(w, cx, &ft.sig); } -pub fn enc_closure_ty(w: &mut MemWriter, cx: &ctxt, ft: &ty::ClosureTy) { +pub fn enc_closure_ty(w: &mut SeekableMemWriter, cx: &ctxt, ft: &ty::ClosureTy) { enc_fn_style(w, ft.fn_style); enc_onceness(w, ft.onceness); enc_trait_store(w, cx, ft.store); @@ -330,7 +331,7 @@ pub fn enc_closure_ty(w: &mut MemWriter, cx: &ctxt, ft: &ty::ClosureTy) { enc_abi(w, ft.abi); } -fn enc_fn_sig(w: &mut MemWriter, cx: &ctxt, fsig: &ty::FnSig) { +fn enc_fn_sig(w: &mut SeekableMemWriter, cx: &ctxt, fsig: &ty::FnSig) { mywrite!(w, "[{}|", fsig.binder_id); for ty in fsig.inputs.iter() { enc_ty(w, cx, *ty); @@ -344,7 +345,7 @@ fn enc_fn_sig(w: &mut MemWriter, cx: &ctxt, fsig: &ty::FnSig) { enc_ty(w, cx, fsig.output); } -fn enc_bounds(w: &mut MemWriter, cx: &ctxt, bs: &ty::ParamBounds) { +fn enc_bounds(w: &mut SeekableMemWriter, cx: &ctxt, bs: &ty::ParamBounds) { for bound in bs.builtin_bounds.iter() { match bound { ty::BoundSend => mywrite!(w, "S"), @@ -363,7 +364,7 @@ fn enc_bounds(w: &mut MemWriter, cx: &ctxt, bs: &ty::ParamBounds) { mywrite!(w, "."); } -pub fn enc_type_param_def(w: &mut MemWriter, cx: &ctxt, v: &ty::TypeParameterDef) { +pub fn enc_type_param_def(w: &mut SeekableMemWriter, cx: &ctxt, v: &ty::TypeParameterDef) { mywrite!(w, "{}:{}|{}|{}|", token::get_ident(v.ident), (cx.ds)(v.def_id), v.space.to_uint(), v.index); diff --git a/src/librustc/middle/astencode.rs b/src/librustc/middle/astencode.rs index 722715405bc..9a587dd7741 100644 --- a/src/librustc/middle/astencode.rs +++ b/src/librustc/middle/astencode.rs @@ -28,6 +28,7 @@ use middle::subst; use middle::subst::VecPerParamSpace; use middle::typeck::{MethodCall, MethodCallee, MethodOrigin}; use middle::{ty, typeck}; +use util::io::SeekableMemWriter; use util::ppaux::ty_to_string; use syntax::{ast, ast_map, ast_util, codemap, fold}; @@ -39,7 +40,6 @@ use syntax; use libc; use std::io::Seek; -use std::io::MemWriter; use std::mem; use std::gc::GC; @@ -73,7 +73,7 @@ trait tr_intern { fn tr_intern(&self, xcx: &ExtendedDecodeContext) -> ast::DefId; } -pub type Encoder<'a> = writer::Encoder<'a, MemWriter>; +pub type Encoder<'a> = writer::Encoder<'a, SeekableMemWriter>; // ______________________________________________________________________ // Top-level methods. @@ -1573,10 +1573,8 @@ fn mk_ctxt() -> parse::ParseSess { #[cfg(test)] fn roundtrip(in_item: Option>) { - use std::io::MemWriter; - let in_item = in_item.unwrap(); - let mut wr = MemWriter::new(); + let mut wr = SeekableMemWriter::new(); { let mut ebml_w = writer::Encoder::new(&mut wr); encode_item_ast(&mut ebml_w, in_item); diff --git a/src/librustc/util/io.rs b/src/librustc/util/io.rs new file mode 100644 index 00000000000..f153aca7fac --- /dev/null +++ b/src/librustc/util/io.rs @@ -0,0 +1,232 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::io::{IoError, IoResult, SeekStyle}; +use std::io; +use std::slice; + +static BUF_CAPACITY: uint = 128; + +fn combine(seek: SeekStyle, cur: uint, end: uint, offset: i64) -> IoResult { + // compute offset as signed and clamp to prevent overflow + let pos = match seek { + io::SeekSet => 0, + io::SeekEnd => end, + io::SeekCur => cur, + } as i64; + + if offset + pos < 0 { + Err(IoError { + kind: io::InvalidInput, + desc: "invalid seek to a negative offset", + detail: None + }) + } else { + Ok((offset + pos) as u64) + } +} + +/// Writes to an owned, growable byte vector that supports seeking. +/// +/// # Example +/// +/// ```rust +/// # #![allow(unused_must_use)] +/// use std::io::SeekableMemWriter; +/// +/// let mut w = SeekableMemWriter::new(); +/// w.write([0, 1, 2]); +/// +/// assert_eq!(w.unwrap(), vec!(0, 1, 2)); +/// ``` +pub struct SeekableMemWriter { + buf: Vec, + pos: uint, +} + +impl SeekableMemWriter { + /// Create a new `SeekableMemWriter`. + #[inline] + pub fn new() -> SeekableMemWriter { + SeekableMemWriter::with_capacity(BUF_CAPACITY) + } + /// Create a new `SeekableMemWriter`, allocating at least `n` bytes for + /// the internal buffer. + #[inline] + pub fn with_capacity(n: uint) -> SeekableMemWriter { + SeekableMemWriter { buf: Vec::with_capacity(n), pos: 0 } + } + + /// Acquires an immutable reference to the underlying buffer of this + /// `SeekableMemWriter`. + /// + /// No method is exposed for acquiring a mutable reference to the buffer + /// because it could corrupt the state of this `MemWriter`. + #[inline] + pub fn get_ref<'a>(&'a self) -> &'a [u8] { self.buf.as_slice() } + + /// Unwraps this `SeekableMemWriter`, returning the underlying buffer + #[inline] + pub fn unwrap(self) -> Vec { self.buf } +} + +impl Writer for SeekableMemWriter { + #[inline] + fn write(&mut self, buf: &[u8]) -> IoResult<()> { + if self.pos == self.buf.len() { + self.buf.push_all(buf) + } else { + // Make sure the internal buffer is as least as big as where we + // currently are + let difference = self.pos as i64 - self.buf.len() as i64; + if difference > 0 { + self.buf.grow(difference as uint, &0); + } + + // Figure out what bytes will be used to overwrite what's currently + // there (left), and what will be appended on the end (right) + let cap = self.buf.len() - self.pos; + let (left, right) = if cap <= buf.len() { + (buf.slice_to(cap), buf.slice_from(cap)) + } else { + (buf, &[]) + }; + + // Do the necessary writes + if left.len() > 0 { + slice::bytes::copy_memory(self.buf.mut_slice_from(self.pos), left); + } + if right.len() > 0 { + self.buf.push_all(right); + } + } + + // Bump us forward + self.pos += buf.len(); + Ok(()) + } +} + +impl Seek for SeekableMemWriter { + #[inline] + fn tell(&self) -> IoResult { Ok(self.pos as u64) } + + #[inline] + fn seek(&mut self, pos: i64, style: SeekStyle) -> IoResult<()> { + let new = try!(combine(style, self.pos, self.buf.len(), pos)); + self.pos = new as uint; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::SeekableMemWriter; + use std::io; + use test::Bencher; + + #[test] + fn test_seekable_mem_writer() { + let mut writer = SeekableMemWriter::new(); + assert_eq!(writer.tell(), Ok(0)); + writer.write([0]).unwrap(); + assert_eq!(writer.tell(), Ok(1)); + writer.write([1, 2, 3]).unwrap(); + writer.write([4, 5, 6, 7]).unwrap(); + assert_eq!(writer.tell(), Ok(8)); + assert_eq!(writer.get_ref(), &[0, 1, 2, 3, 4, 5, 6, 7]); + + writer.seek(0, io::SeekSet).unwrap(); + assert_eq!(writer.tell(), Ok(0)); + writer.write([3, 4]).unwrap(); + assert_eq!(writer.get_ref(), &[3, 4, 2, 3, 4, 5, 6, 7]); + + writer.seek(1, io::SeekCur).unwrap(); + writer.write([0, 1]).unwrap(); + assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 7]); + + writer.seek(-1, io::SeekEnd).unwrap(); + writer.write([1, 2]).unwrap(); + assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 1, 2]); + + writer.seek(1, io::SeekEnd).unwrap(); + writer.write([1]).unwrap(); + assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 1, 2, 0, 1]); + } + + #[test] + fn seek_past_end() { + let mut r = SeekableMemWriter::new(); + r.seek(10, io::SeekSet).unwrap(); + assert!(r.write([3]).is_ok()); + } + + #[test] + fn seek_before_0() { + let mut r = SeekableMemWriter::new(); + assert!(r.seek(-1, io::SeekSet).is_err()); + } + + fn do_bench_seekable_mem_writer(b: &mut Bencher, times: uint, len: uint) { + let src: Vec = Vec::from_elem(len, 5); + + b.bytes = (times * len) as u64; + b.iter(|| { + let mut wr = SeekableMemWriter::new(); + for _ in range(0, times) { + wr.write(src.as_slice()).unwrap(); + } + + let v = wr.unwrap(); + assert_eq!(v.len(), times * len); + assert!(v.iter().all(|x| *x == 5)); + }); + } + + #[bench] + fn bench_seekable_mem_writer_001_0000(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 1, 0) + } + + #[bench] + fn bench_seekable_mem_writer_001_0010(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 1, 10) + } + + #[bench] + fn bench_seekable_mem_writer_001_0100(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 1, 100) + } + + #[bench] + fn bench_seekable_mem_writer_001_1000(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 1, 1000) + } + + #[bench] + fn bench_seekable_mem_writer_100_0000(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 100, 0) + } + + #[bench] + fn bench_seekable_mem_writer_100_0010(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 100, 10) + } + + #[bench] + fn bench_seekable_mem_writer_100_0100(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 100, 100) + } + + #[bench] + fn bench_seekable_mem_writer_100_1000(b: &mut Bencher) { + do_bench_seekable_mem_writer(b, 100, 1000) + } +} diff --git a/src/libserialize/ebml.rs b/src/libserialize/ebml.rs index dfce1eeb832..c3434466836 100644 --- a/src/libserialize/ebml.rs +++ b/src/libserialize/ebml.rs @@ -1023,8 +1023,124 @@ mod tests { use ebml::writer; use {Encodable, Decodable}; - use std::io::MemWriter; + use std::io::{IoError, IoResult, SeekStyle}; + use std::io; use std::option::{None, Option, Some}; + use std::slice; + + static BUF_CAPACITY: uint = 128; + + fn combine(seek: SeekStyle, cur: uint, end: uint, offset: i64) -> IoResult { + // compute offset as signed and clamp to prevent overflow + let pos = match seek { + io::SeekSet => 0, + io::SeekEnd => end, + io::SeekCur => cur, + } as i64; + + if offset + pos < 0 { + Err(IoError { + kind: io::InvalidInput, + desc: "invalid seek to a negative offset", + detail: None + }) + } else { + Ok((offset + pos) as u64) + } + } + + /// Writes to an owned, growable byte vector that supports seeking. + /// + /// # Example + /// + /// ```rust + /// # #![allow(unused_must_use)] + /// use std::io::SeekableMemWriter; + /// + /// let mut w = SeekableMemWriter::new(); + /// w.write([0, 1, 2]); + /// + /// assert_eq!(w.unwrap(), vec!(0, 1, 2)); + /// ``` + pub struct SeekableMemWriter { + buf: Vec, + pos: uint, + } + + impl SeekableMemWriter { + /// Create a new `SeekableMemWriter`. + #[inline] + pub fn new() -> SeekableMemWriter { + SeekableMemWriter::with_capacity(BUF_CAPACITY) + } + /// Create a new `SeekableMemWriter`, allocating at least `n` bytes for + /// the internal buffer. + #[inline] + pub fn with_capacity(n: uint) -> SeekableMemWriter { + SeekableMemWriter { buf: Vec::with_capacity(n), pos: 0 } + } + + /// Acquires an immutable reference to the underlying buffer of this + /// `SeekableMemWriter`. + /// + /// No method is exposed for acquiring a mutable reference to the buffer + /// because it could corrupt the state of this `MemWriter`. + #[inline] + pub fn get_ref<'a>(&'a self) -> &'a [u8] { self.buf.as_slice() } + + /// Unwraps this `SeekableMemWriter`, returning the underlying buffer + #[inline] + pub fn unwrap(self) -> Vec { self.buf } + } + + impl Writer for SeekableMemWriter { + #[inline] + fn write(&mut self, buf: &[u8]) -> IoResult<()> { + if self.pos == self.buf.len() { + self.buf.push_all(buf) + } else { + // Make sure the internal buffer is as least as big as where we + // currently are + let difference = self.pos as i64 - self.buf.len() as i64; + if difference > 0 { + self.buf.grow(difference as uint, &0); + } + + // Figure out what bytes will be used to overwrite what's currently + // there (left), and what will be appended on the end (right) + let cap = self.buf.len() - self.pos; + let (left, right) = if cap <= buf.len() { + (buf.slice_to(cap), buf.slice_from(cap)) + } else { + (buf, &[]) + }; + + // Do the necessary writes + if left.len() > 0 { + slice::bytes::copy_memory(self.buf.mut_slice_from(self.pos), left); + } + if right.len() > 0 { + self.buf.push_all(right); + } + } + + // Bump us forward + self.pos += buf.len(); + Ok(()) + } + } + + impl Seek for SeekableMemWriter { + #[inline] + fn tell(&self) -> IoResult { Ok(self.pos as u64) } + + #[inline] + fn seek(&mut self, pos: i64, style: SeekStyle) -> IoResult<()> { + let new = try!(combine(style, self.pos, self.buf.len(), pos)); + self.pos = new as uint; + Ok(()) + } + } #[test] fn test_vuint_at() { @@ -1078,7 +1194,7 @@ mod tests { fn test_option_int() { fn test_v(v: Option) { debug!("v == {}", v); - let mut wr = MemWriter::new(); + let mut wr = SeekableMemWriter::new(); { let mut ebml_w = writer::Encoder::new(&mut wr); let _ = v.encode(&mut ebml_w); diff --git a/src/libstd/io/fs.rs b/src/libstd/io/fs.rs index 790fe6cb8b9..f9f8ce377ec 100644 --- a/src/libstd/io/fs.rs +++ b/src/libstd/io/fs.rs @@ -955,7 +955,7 @@ mod test { } ) ) - struct TempDir(Path); + pub struct TempDir(Path); impl TempDir { fn join(&self, path: &str) -> Path { diff --git a/src/libstd/io/mem.rs b/src/libstd/io/mem.rs index b93b84b7d63..8a60233cb5c 100644 --- a/src/libstd/io/mem.rs +++ b/src/libstd/io/mem.rs @@ -22,6 +22,8 @@ use slice; use slice::{Vector, ImmutableVector, MutableVector}; use vec::Vec; +static BUF_CAPACITY: uint = 128; + fn combine(seek: SeekStyle, cur: uint, end: uint, offset: i64) -> IoResult { // compute offset as signed and clamp to prevent overflow let pos = match seek { @@ -56,27 +58,23 @@ fn combine(seek: SeekStyle, cur: uint, end: uint, offset: i64) -> IoResult /// ``` pub struct MemWriter { buf: Vec, - pos: uint, } impl MemWriter { /// Create a new `MemWriter`. #[inline] pub fn new() -> MemWriter { - MemWriter::with_capacity(128) + MemWriter::with_capacity(BUF_CAPACITY) } /// Create a new `MemWriter`, allocating at least `n` bytes for /// the internal buffer. #[inline] pub fn with_capacity(n: uint) -> MemWriter { - MemWriter { buf: Vec::with_capacity(n), pos: 0 } + MemWriter { buf: Vec::with_capacity(n) } } /// Acquires an immutable reference to the underlying buffer of this /// `MemWriter`. - /// - /// No method is exposed for acquiring a mutable reference to the buffer - /// because it could corrupt the state of this `MemWriter`. #[inline] pub fn get_ref<'a>(&'a self) -> &'a [u8] { self.buf.as_slice() } @@ -88,44 +86,7 @@ impl MemWriter { impl Writer for MemWriter { #[inline] fn write(&mut self, buf: &[u8]) -> IoResult<()> { - // Make sure the internal buffer is as least as big as where we - // currently are - let difference = self.pos as i64 - self.buf.len() as i64; - if difference > 0 { - self.buf.grow(difference as uint, &0); - } - - // Figure out what bytes will be used to overwrite what's currently - // there (left), and what will be appended on the end (right) - let cap = self.buf.len() - self.pos; - let (left, right) = if cap <= buf.len() { - (buf.slice_to(cap), buf.slice_from(cap)) - } else { - (buf, &[]) - }; - - // Do the necessary writes - if left.len() > 0 { - slice::bytes::copy_memory(self.buf.mut_slice_from(self.pos), left); - } - if right.len() > 0 { - self.buf.push_all(right); - } - - // Bump us forward - self.pos += buf.len(); - Ok(()) - } -} - -impl Seek for MemWriter { - #[inline] - fn tell(&self) -> IoResult { Ok(self.pos as u64) } - - #[inline] - fn seek(&mut self, pos: i64, style: SeekStyle) -> IoResult<()> { - let new = try!(combine(style, self.pos, self.buf.len(), pos)); - self.pos = new as uint; + self.buf.push_all(buf); Ok(()) } } @@ -381,30 +342,10 @@ mod test { #[test] fn test_mem_writer() { let mut writer = MemWriter::new(); - assert_eq!(writer.tell(), Ok(0)); writer.write([0]).unwrap(); - assert_eq!(writer.tell(), Ok(1)); writer.write([1, 2, 3]).unwrap(); writer.write([4, 5, 6, 7]).unwrap(); - assert_eq!(writer.tell(), Ok(8)); assert_eq!(writer.get_ref(), &[0, 1, 2, 3, 4, 5, 6, 7]); - - writer.seek(0, SeekSet).unwrap(); - assert_eq!(writer.tell(), Ok(0)); - writer.write([3, 4]).unwrap(); - assert_eq!(writer.get_ref(), &[3, 4, 2, 3, 4, 5, 6, 7]); - - writer.seek(1, SeekCur).unwrap(); - writer.write([0, 1]).unwrap(); - assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 7]); - - writer.seek(-1, SeekEnd).unwrap(); - writer.write([1, 2]).unwrap(); - assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 1, 2]); - - writer.seek(1, SeekEnd).unwrap(); - writer.write([1]).unwrap(); - assert_eq!(writer.get_ref(), &[3, 4, 2, 0, 1, 5, 6, 1, 2, 0, 1]); } #[test] @@ -570,10 +511,6 @@ mod test { r.seek(10, SeekSet).unwrap(); assert!(r.read(&mut []).is_err()); - let mut r = MemWriter::new(); - r.seek(10, SeekSet).unwrap(); - assert!(r.write([3]).is_ok()); - let mut buf = [0]; let mut r = BufWriter::new(buf); r.seek(10, SeekSet).unwrap(); @@ -589,9 +526,6 @@ mod test { let mut r = MemReader::new(vec!(10)); assert!(r.seek(-1, SeekSet).is_err()); - let mut r = MemWriter::new(); - assert!(r.seek(-1, SeekSet).is_err()); - let mut buf = [0]; let mut r = BufWriter::new(buf); assert!(r.seek(-1, SeekSet).is_err()); @@ -614,6 +548,7 @@ mod test { fn do_bench_mem_writer(b: &mut Bencher, times: uint, len: uint) { let src: Vec = Vec::from_elem(len, 5); + b.bytes = (times * len) as u64; b.iter(|| { let mut wr = MemWriter::new(); for _ in range(0, times) { diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index ac835565191..f35885c0ae2 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -137,7 +137,7 @@ pub fn to_string(f: |&mut State| -> IoResult<()>) -> String { // downcasts. let (_, wr): (uint, Box) = mem::transmute_copy(&s.s.out); let result = - String::from_utf8(Vec::from_slice(wr.get_ref())).unwrap(); + String::from_utf8(Vec::from_slice(wr.get_ref().as_slice())).unwrap(); mem::forget(wr); result.to_string() } diff --git a/src/libuuid/lib.rs b/src/libuuid/lib.rs index 0e29e621503..037afce9b16 100644 --- a/src/libuuid/lib.rs +++ b/src/libuuid/lib.rs @@ -790,16 +790,12 @@ mod test { #[test] fn test_serialize_round_trip() { - use serialize::ebml::Doc; - use serialize::ebml::writer::Encoder; - use serialize::ebml::reader::Decoder; + use serialize::json; use serialize::{Encodable, Decodable}; let u = Uuid::new_v4(); - let mut wr = MemWriter::new(); - let _ = u.encode(&mut Encoder::new(&mut wr)); - let doc = Doc::new(wr.get_ref()); - let u2 = Decodable::decode(&mut Decoder::new(doc)).unwrap(); + let s = json::encode(&u); + let u2 = json::decode(s.as_slice()).unwrap(); assert_eq!(u, u2); } diff --git a/src/test/run-pass/deriving-encodable-decodable-cell-refcell.rs b/src/test/run-pass/deriving-encodable-decodable-cell-refcell.rs index a7738bb803c..9eef83184e1 100644 --- a/src/test/run-pass/deriving-encodable-decodable-cell-refcell.rs +++ b/src/test/run-pass/deriving-encodable-decodable-cell-refcell.rs @@ -16,9 +16,7 @@ extern crate serialize; use std::cell::{Cell, RefCell}; use std::io::MemWriter; use serialize::{Encodable, Decodable}; -use serialize::ebml; -use serialize::ebml::writer::Encoder; -use serialize::ebml::reader::Decoder; +use serialize::json; #[deriving(Encodable, Decodable)] struct A { @@ -36,20 +34,8 @@ fn main() { foo: Cell::new(true), bar: RefCell::new( A { baz: 2 } ) }; - let mut w = MemWriter::new(); - { - let mut e = Encoder::new(&mut w); - match obj.encode(&mut e) { - Ok(()) => (), - Err(e) => fail!("Failed to encode: {}", e) - }; - } - let doc = ebml::Doc::new(w.get_ref()); - let mut dec = Decoder::new(doc); - let obj2: B = match Decodable::decode(&mut dec) { - Ok(v) => v, - Err(e) => fail!("Failed to decode: {}", e) - }; + let s = json::encode(&obj); + let obj2: B = json::decode(s.as_slice()).unwrap(); assert!(obj.foo.get() == obj2.foo.get()); assert!(obj.bar.borrow().baz == obj2.bar.borrow().baz); } diff --git a/src/test/run-pass/issue-11881.rs b/src/test/run-pass/issue-11881.rs index e0cd91adb1c..d7f487b629b 100644 --- a/src/test/run-pass/issue-11881.rs +++ b/src/test/run-pass/issue-11881.rs @@ -10,10 +10,127 @@ extern crate serialize; +use std::io; +use std::io::{IoError, IoResult, SeekStyle}; +use std::slice; + use serialize::{Encodable, Encoder}; use serialize::json; use serialize::ebml::writer; -use std::io::MemWriter; + +static BUF_CAPACITY: uint = 128; + +fn combine(seek: SeekStyle, cur: uint, end: uint, offset: i64) -> IoResult { + // compute offset as signed and clamp to prevent overflow + let pos = match seek { + io::SeekSet => 0, + io::SeekEnd => end, + io::SeekCur => cur, + } as i64; + + if offset + pos < 0 { + Err(IoError { + kind: io::InvalidInput, + desc: "invalid seek to a negative offset", + detail: None + }) + } else { + Ok((offset + pos) as u64) + } +} + +/// Writes to an owned, growable byte vector that supports seeking. +/// +/// # Example +/// +/// ```rust +/// # #![allow(unused_must_use)] +/// use std::io::SeekableMemWriter; +/// +/// let mut w = SeekableMemWriter::new(); +/// w.write([0, 1, 2]); +/// +/// assert_eq!(w.unwrap(), vec!(0, 1, 2)); +/// ``` +pub struct SeekableMemWriter { + buf: Vec, + pos: uint, +} + +impl SeekableMemWriter { + /// Create a new `SeekableMemWriter`. + #[inline] + pub fn new() -> SeekableMemWriter { + SeekableMemWriter::with_capacity(BUF_CAPACITY) + } + /// Create a new `SeekableMemWriter`, allocating at least `n` bytes for + /// the internal buffer. + #[inline] + pub fn with_capacity(n: uint) -> SeekableMemWriter { + SeekableMemWriter { buf: Vec::with_capacity(n), pos: 0 } + } + + /// Acquires an immutable reference to the underlying buffer of this + /// `SeekableMemWriter`. + /// + /// No method is exposed for acquiring a mutable reference to the buffer + /// because it could corrupt the state of this `MemWriter`. + #[inline] + pub fn get_ref<'a>(&'a self) -> &'a [u8] { self.buf.as_slice() } + + /// Unwraps this `SeekableMemWriter`, returning the underlying buffer + #[inline] + pub fn unwrap(self) -> Vec { self.buf } +} + +impl Writer for SeekableMemWriter { + #[inline] + fn write(&mut self, buf: &[u8]) -> IoResult<()> { + if self.pos == self.buf.len() { + self.buf.push_all(buf) + } else { + // Make sure the internal buffer is as least as big as where we + // currently are + let difference = self.pos as i64 - self.buf.len() as i64; + if difference > 0 { + self.buf.grow(difference as uint, &0); + } + + // Figure out what bytes will be used to overwrite what's currently + // there (left), and what will be appended on the end (right) + let cap = self.buf.len() - self.pos; + let (left, right) = if cap <= buf.len() { + (buf.slice_to(cap), buf.slice_from(cap)) + } else { + (buf, &[]) + }; + + // Do the necessary writes + if left.len() > 0 { + slice::bytes::copy_memory(self.buf.mut_slice_from(self.pos), left); + } + if right.len() > 0 { + self.buf.push_all(right); + } + } + + // Bump us forward + self.pos += buf.len(); + Ok(()) + } +} + +impl Seek for SeekableMemWriter { + #[inline] + fn tell(&self) -> IoResult { Ok(self.pos as u64) } + + #[inline] + fn seek(&mut self, pos: i64, style: SeekStyle) -> IoResult<()> { + let new = try!(combine(style, self.pos, self.buf.len(), pos)); + self.pos = new as uint; + Ok(()) + } +} #[deriving(Encodable)] struct Foo { @@ -34,21 +151,21 @@ enum WireProtocol { fn encode_json<'a, T: Encodable, std::io::IoError>>(val: &T, - wr: &'a mut MemWriter) { + wr: &'a mut SeekableMemWriter) { let mut encoder = json::Encoder::new(wr); val.encode(&mut encoder); } fn encode_ebml<'a, - T: Encodable, + T: Encodable, std::io::IoError>>(val: &T, - wr: &'a mut MemWriter) { + wr: &'a mut SeekableMemWriter) { let mut encoder = writer::Encoder::new(wr); val.encode(&mut encoder); } pub fn main() { let target = Foo{baz: false,}; - let mut wr = MemWriter::new(); + let mut wr = SeekableMemWriter::new(); let proto = JSON; match proto { JSON => encode_json(&target, &mut wr),