nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-22 14:55:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove in-place allocation and revert to separate methods for zeroed allocations

Browse Source 
Fix docs
This commit is contained in:
Tim Diekmann 2020-07-28 12:41:18 +02:00
parent 1f5d69dacc
commit 076ef66ba2
14 changed files with 397 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
library/alloc/src/alloc.rs
View File

@ -164,16 +164,26 @@ pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
#[unstable(feature = "allocator_api", issue = "32838")]
unsafe impl AllocRef for Global {
#[inline]
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
unsafe {
let size = layout.size();
if size == 0 {
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
let raw_ptr = match init {
AllocInit::Uninitialized => alloc(layout),
AllocInit::Zeroed => alloc_zeroed(layout),
};
let raw_ptr = alloc(layout);
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
Ok(MemoryBlock { ptr, size })
}
}
}
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
unsafe {
let size = layout.size();
if size == 0 {
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
let raw_ptr = alloc_zeroed(layout);
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
Ok(MemoryBlock { ptr, size })
}
@ -193,8 +203,6 @@ unsafe impl AllocRef for Global {
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
init: AllocInit,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
@ -206,26 +214,49 @@ unsafe impl AllocRef for Global {
return Ok(MemoryBlock { ptr, size });
}
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove if layout.size() == 0 => {
let new_layout =
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc(new_layout, init)
}
ReallocPlacement::MayMove => {
// `realloc` probably checks for `new_size > size` or something similar.
let ptr = unsafe {
intrinsics::assume(new_size > size);
realloc(ptr.as_ptr(), layout, new_size)
};
let memory =
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
unsafe {
init.init_offset(memory, size);
}
Ok(memory)
if layout.size() == 0 {
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc(new_layout)
} else {
// `realloc` probably checks for `new_size > size` or something similar.
let ptr = unsafe {
intrinsics::assume(new_size > size);
realloc(ptr.as_ptr(), layout, new_size)
};
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
}
}
unsafe fn grow_zeroed(
&mut self,
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
new_size >= size,
"`new_size` must be greater than or equal to `memory.size()`"
);
if size == new_size {
return Ok(MemoryBlock { ptr, size });
}
if layout.size() == 0 {
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc(new_layout)
} else {
// `realloc` probably checks for `new_size > size` or something similar.
let ptr = unsafe {
intrinsics::assume(new_size > size);
realloc(ptr.as_ptr(), layout, new_size)
};
let memory = MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
unsafe {
memory.ptr.as_ptr().add(size).write_bytes(0, memory.size - size);
}
Ok(memory)
}
}
@ -235,7 +266,6 @@ unsafe impl AllocRef for Global {
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
@ -247,22 +277,18 @@ unsafe impl AllocRef for Global {
return Ok(MemoryBlock { ptr, size });
}
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove if new_size == 0 => {
unsafe {
self.dealloc(ptr, layout);
}
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
}
ReallocPlacement::MayMove => {
// `realloc` probably checks for `new_size < size` or something similar.
let ptr = unsafe {
intrinsics::assume(new_size < size);
realloc(ptr.as_ptr(), layout, new_size)
};
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
if new_size == 0 {
unsafe {
self.dealloc(ptr, layout);
}
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
// `realloc` probably checks for `new_size < size` or something similar.
let ptr = unsafe {
intrinsics::assume(new_size < size);
realloc(ptr.as_ptr(), layout, new_size)
};
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
}
}
}
@ -274,7 +300,7 @@ unsafe impl AllocRef for Global {
#[inline]
unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
match Global.alloc(layout, AllocInit::Uninitialized) {
match Global.alloc(layout) {
Ok(memory) => memory.ptr.as_ptr(),
Err(_) => handle_alloc_error(layout),
}

5
library/alloc/src/alloc/tests.rs
View File

@ -8,9 +8,8 @@ use test::Bencher;
fn allocate_zeroed() {
unsafe {
let layout = Layout::from_size_align(1024, 1).unwrap();
let memory = Global
.alloc(layout.clone(), AllocInit::Zeroed)
.unwrap_or_else(|_| handle_alloc_error(layout));
let memory =
Global.alloc_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout));
let mut i = memory.ptr.cast::<u8>().as_ptr();
let end = i.add(layout.size());

11
library/alloc/src/boxed.rs
View File

@ -146,7 +146,7 @@ use core::pin::Pin;
use core::ptr::{self, NonNull, Unique};
use core::task::{Context, Poll};
use crate::alloc::{self, AllocInit, AllocRef, Global};
use crate::alloc::{self, AllocRef, Global};
use crate::borrow::Cow;
use crate::raw_vec::RawVec;
use crate::str::from_boxed_utf8_unchecked;
@ -197,11 +197,8 @@ impl<T> Box<T> {
#[unstable(feature = "new_uninit", issue = "63291")]
pub fn new_uninit() -> Box<mem::MaybeUninit<T>> {
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
let ptr = Global
.alloc(layout, AllocInit::Uninitialized)
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
.ptr
.cast();
let ptr =
Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).ptr.cast();
unsafe { Box::from_raw(ptr.as_ptr()) }
}
@ -227,7 +224,7 @@ impl<T> Box<T> {
pub fn new_zeroed() -> Box<mem::MaybeUninit<T>> {
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
let ptr = Global
.alloc(layout, AllocInit::Zeroed)
.alloc_zeroed(layout)
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
.ptr
.cast();

44
library/alloc/src/raw_vec.rs
View File

@ -8,18 +8,20 @@ use core::ops::Drop;
use core::ptr::{NonNull, Unique};
use core::slice;
use crate::alloc::{
handle_alloc_error,
AllocInit::{self, *},
AllocRef, Global, Layout,
ReallocPlacement::{self, *},
};
use crate::alloc::{handle_alloc_error, AllocRef, Global, Layout};
use crate::boxed::Box;
use crate::collections::TryReserveError::{self, *};
#[cfg(test)]
mod tests;
enum AllocInit {
/// The contents of the new memory are uninitialized.
Uninitialized,
/// The new memory is guaranteed to be zeroed.
Zeroed,
}
/// A low-level utility for more ergonomically allocating, reallocating, and deallocating
/// a buffer of memory on the heap without having to worry about all the corner cases
/// involved. This type is excellent for building your own data structures like Vec and VecDeque.
@ -156,14 +158,14 @@ impl<T, A: AllocRef> RawVec<T, A> {
/// allocator for the returned `RawVec`.
#[inline]
pub fn with_capacity_in(capacity: usize, alloc: A) -> Self {
Self::allocate_in(capacity, Uninitialized, alloc)
Self::allocate_in(capacity, AllocInit::Uninitialized, alloc)
}
/// Like `with_capacity_zeroed`, but parameterized over the choice
/// of allocator for the returned `RawVec`.
#[inline]
pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self {
Self::allocate_in(capacity, Zeroed, alloc)
Self::allocate_in(capacity, AllocInit::Zeroed, alloc)
}
fn allocate_in(capacity: usize, init: AllocInit, mut alloc: A) -> Self {
@ -180,7 +182,11 @@ impl<T, A: AllocRef> RawVec<T, A> {
Ok(_) => {}
Err(_) => capacity_overflow(),
}
let memory = match alloc.alloc(layout, init) {
let result = match init {
AllocInit::Uninitialized => alloc.alloc(layout),
AllocInit::Zeroed => alloc.alloc_zeroed(layout),
};
let memory = match result {
Ok(memory) => memory,
Err(_) => handle_alloc_error(layout),
};
@ -358,7 +364,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
///
/// Aborts on OOM.
pub fn shrink_to_fit(&mut self, amount: usize) {
match self.shrink(amount, MayMove) {
match self.shrink(amount) {
Err(CapacityOverflow) => capacity_overflow(),
Err(AllocError { layout, .. }) => handle_alloc_error(layout),
Ok(()) => { /* yay */ }
@ -450,22 +456,16 @@ impl<T, A: AllocRef> RawVec<T, A> {
Ok(())
}
fn shrink(
&mut self,
amount: usize,
placement: ReallocPlacement,
) -> Result<(), TryReserveError> {
fn shrink(&mut self, amount: usize) -> Result<(), TryReserveError> {
assert!(amount <= self.capacity(), "Tried to shrink to a larger capacity");
let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) };
let new_size = amount * mem::size_of::<T>();
let memory = unsafe {
self.alloc.shrink(ptr, layout, new_size, placement).map_err(|_| {
TryReserveError::AllocError {
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
non_exhaustive: (),
}
self.alloc.shrink(ptr, layout, new_size).map_err(|_| TryReserveError::AllocError {
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
non_exhaustive: (),
})?
};
self.set_memory(memory);
@ -492,9 +492,9 @@ where
let memory = if let Some((ptr, old_layout)) = current_memory {
debug_assert_eq!(old_layout.align(), new_layout.align());
unsafe { alloc.grow(ptr, old_layout, new_layout.size(), MayMove, Uninitialized) }
unsafe { alloc.grow(ptr, old_layout, new_layout.size()) }
} else {
alloc.alloc(new_layout, Uninitialized)
alloc.alloc(new_layout)
}
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?;

4
library/alloc/src/raw_vec/tests.rs
View File

@ -20,12 +20,12 @@ fn allocator_param() {
fuel: usize,
}
unsafe impl AllocRef for BoundedAlloc {
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
if size > self.fuel {
return Err(AllocErr);
}
match Global.alloc(layout, init) {
match Global.alloc(layout) {
ok @ Ok(_) => {
self.fuel -= size;
ok

6
library/alloc/src/rc.rs
View File

@ -250,7 +250,7 @@ use core::pin::Pin;
use core::ptr::{self, NonNull};
use core::slice::from_raw_parts_mut;
use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout};
use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout};
use crate::borrow::{Cow, ToOwned};
use crate::string::String;
use crate::vec::Vec;
@ -951,9 +951,7 @@ impl<T: ?Sized> Rc<T> {
let layout = Layout::new::<RcBox<()>>().extend(value_layout).unwrap().0.pad_to_align();
// Allocate for the layout.
let mem = Global
.alloc(layout, AllocInit::Uninitialized)
.unwrap_or_else(|_| handle_alloc_error(layout));
let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
// Initialize the RcBox
let inner = mem_to_rcbox(mem.ptr.as_ptr());

6
library/alloc/src/sync.rs
View File

@ -23,7 +23,7 @@ use core::slice::from_raw_parts_mut;
use core::sync::atomic;
use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst};
use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout};
use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout};
use crate::borrow::{Cow, ToOwned};
use crate::boxed::Box;
use crate::rc::is_dangling;
@ -906,9 +906,7 @@ impl<T: ?Sized> Arc<T> {
// reference (see #54908).
let layout = Layout::new::<ArcInner<()>>().extend(value_layout).unwrap().0.pad_to_align();
let mem = Global
.alloc(layout, AllocInit::Uninitialized)
.unwrap_or_else(|_| handle_alloc_error(layout));
let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
// Initialize the ArcInner
let inner = mem_to_arcinner(mem.ptr.as_ptr());

10
library/alloc/tests/heap.rs
View File

@ -1,4 +1,4 @@
use std::alloc::{AllocInit, AllocRef, Global, Layout, System};
use std::alloc::{AllocRef, Global, Layout, System};
/// Issue #45955 and #62251.
#[test]
@ -20,13 +20,7 @@ fn check_overalign_requests<T: AllocRef>(mut allocator: T) {
unsafe {
let pointers: Vec<_> = (0..iterations)
.map(|_| {
allocator
.alloc(
Layout::from_size_align(size, align).unwrap(),
AllocInit::Uninitialized,
)
.unwrap()
.ptr
allocator.alloc(Layout::from_size_align(size, align).unwrap()).unwrap().ptr
})
.collect();
for &ptr in &pointers {

344
library/core/src/alloc/mod.rs
View File

@ -29,66 +29,6 @@ impl fmt::Display for AllocErr {
}
}
/// A desired initial state for allocated memory.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[unstable(feature = "allocator_api", issue = "32838")]
pub enum AllocInit {
/// The contents of the new memory are uninitialized.
Uninitialized,
/// The new memory is guaranteed to be zeroed.
Zeroed,
}
impl AllocInit {
/// Initialize the specified memory block.
///
/// This behaves like calling [`AllocInit::init_offset(memory, 0)`][off].
///
/// [off]: AllocInit::init_offset
///
/// # Safety
///
/// * `memory.ptr` must be [valid] for writes of `memory.size` bytes.
///
/// [valid]: ../../core/ptr/index.html#safety
#[inline]
#[unstable(feature = "allocator_api", issue = "32838")]
pub unsafe fn init(self, memory: MemoryBlock) {
// SAFETY: the safety contract for `init_offset` must be
// upheld by the caller.
unsafe { self.init_offset(memory, 0) }
}
/// Initialize the memory block like specified by `init` at the specified `offset`.
///
/// This is a no-op for [`AllocInit::Uninitialized`][] and writes zeroes for
/// [`AllocInit::Zeroed`][] at `ptr + offset` until `ptr + layout.size()`.
///
/// # Safety
///
/// * `memory.ptr` must be [valid] for writes of `memory.size` bytes.
/// * `offset` must be smaller than or equal to `memory.size`
///
/// [valid]: ../../core/ptr/index.html#safety
#[inline]
#[unstable(feature = "allocator_api", issue = "32838")]
pub unsafe fn init_offset(self, memory: MemoryBlock, offset: usize) {
debug_assert!(
offset <= memory.size,
"`offset` must be smaller than or equal to `memory.size`"
);
match self {
AllocInit::Uninitialized => (),
AllocInit::Zeroed => {
// SAFETY: the caller must guarantee that `offset` is smaller than or equal to `memory.size`,
// so the memory from `memory.ptr + offset` of length `memory.size - offset`
// is guaranteed to be contaned in `memory` and thus valid for writes.
unsafe { memory.ptr.as_ptr().add(offset).write_bytes(0, memory.size - offset) }
}
}
}
}
/// Represents a block of allocated memory returned by an allocator.
#[derive(Debug, Copy, Clone)]
#[unstable(feature = "allocator_api", issue = "32838")]
@ -97,24 +37,6 @@ pub struct MemoryBlock {
pub size: usize,
}
/// A placement constraint when growing or shrinking an existing allocation.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[unstable(feature = "allocator_api", issue = "32838")]
pub enum ReallocPlacement {
/// The allocator is allowed to move the allocation to a different memory address.
// FIXME(wg-allocators#46): Add a section to the module documentation "What is a legal
// allocator" and link it at "valid location".
///
/// If the allocation _does_ move, it's the responsibility of the allocator
/// to also move the data from the previous location to the new location.
MayMove,
/// The address of the new memory must not change.
///
/// If the allocation would have to be moved to a new location to fit, the
/// reallocation request will fail.
InPlace,
}
/// An implementation of `AllocRef` can allocate, grow, shrink, and deallocate arbitrary blocks of
/// data described via [`Layout`][].
///
@ -177,10 +99,8 @@ pub unsafe trait AllocRef {
///
/// On success, returns a [`MemoryBlock`][] meeting the size and alignment guarantees of `layout`.
///
/// The returned block may have a larger size than specified by `layout.size()` and is
/// initialized as specified by [`init`], all the way up to the returned size of the block.
///
/// [`init`]: AllocInit
/// The returned block may have a larger size than specified by `layout.size()`, and may or may
/// not have its contents initialized.
///
/// # Errors
///
@ -195,7 +115,29 @@ pub unsafe trait AllocRef {
/// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar.
///
/// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr>;
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr>;
/// Behaves like `alloc`, but also ensures that the contents are set to zero before being returned.
///
/// # Errors
///
/// Returning `Err` indicates that either memory is exhausted or `layout` does not meet
/// allocator's size or alignment constraints.
///
/// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or
/// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement
/// this trait atop an underlying native allocation library that aborts on memory exhaustion.)
///
/// Clients wishing to abort computation in response to an allocation error are encouraged to
/// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar.
///
/// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
let memory = self.alloc(layout)?;
// SAFETY: `alloc` returns a valid memory block
unsafe { memory.ptr.as_ptr().write_bytes(0, memory.size) }
Ok(memory)
}
/// Deallocates the memory referenced by `ptr`.
///
@ -213,31 +155,11 @@ pub unsafe trait AllocRef {
/// Returns a new [`MemoryBlock`][] containing a pointer and the actual size of the allocated
/// memory. The pointer is suitable for holding data described by a new layout with `layout`’s
/// alignment and a size given by `new_size`. To accomplish this, the allocator may extend the
/// allocation referenced by `ptr` to fit the new layout. If the [`placement`] is
/// [`InPlace`], the returned pointer is guaranteed to be the same as the passed `ptr`.
///
/// If [`MayMove`] is used then ownership of the memory block referenced by `ptr`
/// is transferred to this allocator. The memory may or may not be freed, and should be
/// considered unusable (unless of course it is transferred back to the caller again via the
/// return value of this method).
///
/// allocation referenced by `ptr` to fit the new layout.
///~
/// If this method returns `Err`, then ownership of the memory block has not been transferred to
/// this allocator, and the contents of the memory block are unaltered.
///
/// The memory block will contain the following contents after a successful call to `grow`:
/// * Bytes `0..layout.size()` are preserved from the original allocation.
/// * Bytes `layout.size()..old_size` will either be preserved or initialized according to
/// [`init`], depending on the allocator implementation. `old_size` refers to the size of
/// the `MemoryBlock` prior to the `grow` call, which may be larger than the size
/// that was originally requested when it was allocated.
/// * Bytes `old_size..new_size` are initialized according to [`init`]. `new_size` refers to
/// the size of the `MemoryBlock` returned by the `grow` call.
///
/// [`InPlace`]: ReallocPlacement::InPlace
/// [`MayMove`]: ReallocPlacement::MayMove
/// [`placement`]: ReallocPlacement
/// [`init`]: AllocInit
///
/// # Safety
///
/// * `ptr` must denote a block of memory [*currently allocated*] via this allocator,
@ -270,40 +192,105 @@ pub unsafe trait AllocRef {
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
init: AllocInit,
) -> Result<MemoryBlock, AllocErr> {
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove => {
let size = layout.size();
debug_assert!(
new_size >= size,
"`new_size` must be greater than or equal to `layout.size()`"
);
let size = layout.size();
debug_assert!(
new_size >= size,
"`new_size` must be greater than or equal to `layout.size()`"
);
if new_size == size {
return Ok(MemoryBlock { ptr, size });
}
if new_size == size {
return Ok(MemoryBlock { ptr, size });
}
let new_layout =
// SAFETY: the caller must ensure that the `new_size` does not overflow.
// `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
// The caller must ensure that `new_size` is greater than zero.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
let new_memory = self.alloc(new_layout, init)?;
let new_layout =
// SAFETY: the caller must ensure that the `new_size` does not overflow.
// `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
// The caller must ensure that `new_size` is greater than or equal to zero. If it's equal
// to zero, it's catched beforehand.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
let new_memory = self.alloc(new_layout)?;
// SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new
// memory allocation are valid for reads and writes for `size` bytes. Also, because the old
// allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to
// `copy_nonoverlapping` is safe.
// The safety contract for `dealloc` must be upheld by the caller.
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size);
self.dealloc(ptr, layout);
Ok(new_memory)
}
}
// SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new
// memory allocation are valid for reads and writes for `size` bytes. Also, because the old
// allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to
// `copy_nonoverlapping` is safe.
// The safety contract for `dealloc` must be upheld by the caller.
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size);
self.dealloc(ptr, layout);
Ok(new_memory)
}
}
/// Behaves like `grow`, but also ensures that the new contents are set to zero before being
/// returned.
///
/// The memory block will contain the following contents after a successful call to `grow`:
/// * Bytes `0..layout.size()` are preserved from the original allocation.
/// * Bytes `layout.size()..new_size` are zeroed. `new_size` refers to
/// the size of the `MemoryBlock` returned by the `grow` call.
///
/// # Safety
///
/// * `ptr` must denote a block of memory [*currently allocated*] via this allocator,
/// * `layout` must [*fit*] that block of memory (The `new_size` argument need not fit it.),
// We can't require that `new_size` is strictly greater than `memory.size` because of ZSTs.
// An alternative would be
// * `new_size must be strictly greater than `memory.size` or both are zero
/// * `new_size` must be greater than or equal to `layout.size()`, and
/// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, must not overflow
/// (i.e., the rounded value must be less than or equal to `usize::MAX`).
///
/// [*currently allocated*]: #currently-allocated-memory
/// [*fit*]: #memory-fitting
///
/// # Errors
///
/// Returns `Err` if the new layout does not meet the allocator's size and alignment
/// constraints of the allocator, or if growing otherwise fails.
///
/// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or
/// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement
/// this trait atop an underlying native allocation library that aborts on memory exhaustion.)
///
/// Clients wishing to abort computation in response to an allocation error are encouraged to
/// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar.
///
/// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html
unsafe fn grow_zeroed(
&mut self,
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
new_size >= size,
"`new_size` must be greater than or equal to `layout.size()`"
);
if new_size == size {
return Ok(MemoryBlock { ptr, size });
}
let new_layout =
// SAFETY: the caller must ensure that the `new_size` does not overflow.
// `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
// The caller must ensure that `new_size` is greater than or equal to zero. If it's equal
// to zero, it's catched beforehand.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
let new_memory = self.alloc_zeroed(new_layout)?;
// SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new
// memory allocation are valid for reads and writes for `size` bytes. Also, because the old
// allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to
// `copy_nonoverlapping` is safe.
// The safety contract for `dealloc` must be upheld by the caller.
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size);
self.dealloc(ptr, layout);
Ok(new_memory)
}
}
@ -312,8 +299,7 @@ pub unsafe trait AllocRef {
/// Returns a new [`MemoryBlock`][] containing a pointer and the actual size of the allocated
/// memory. The pointer is suitable for holding data described by a new layout with `layout`’s
/// alignment and a size given by `new_size`. To accomplish this, the allocator may shrink the
/// allocation referenced by `ptr` to fit the new layout. If the [`placement`] is
/// [`InPlace`], the returned pointer is guaranteed to be the same as the passed `ptr`.
/// allocation referenced by `ptr` to fit the new layout.
///
/// If this returns `Ok`, then ownership of the memory block referenced by `ptr` has been
/// transferred to this allocator. The memory may or may not have been freed, and should be
@ -323,11 +309,6 @@ pub unsafe trait AllocRef {
/// If this method returns `Err`, then ownership of the memory block has not been transferred to
/// this allocator, and the contents of the memory block are unaltered.
///
/// The behavior of how the allocator tries to shrink the memory is specified by [`placement`].
///
/// [`InPlace`]: ReallocPlacement::InPlace
/// [`placement`]: ReallocPlacement
///
/// # Safety
///
/// * `ptr` must denote a block of memory [*currently allocated*] via this allocator,
@ -358,39 +339,33 @@ pub unsafe trait AllocRef {
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
) -> Result<MemoryBlock, AllocErr> {
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove => {
let size = layout.size();
debug_assert!(
new_size <= size,
"`new_size` must be smaller than or equal to `layout.size()`"
);
let size = layout.size();
debug_assert!(
new_size <= size,
"`new_size` must be smaller than or equal to `layout.size()`"
);
if new_size == size {
return Ok(MemoryBlock { ptr, size });
}
if new_size == size {
return Ok(MemoryBlock { ptr, size });
}
let new_layout =
// SAFETY: the caller must ensure that the `new_size` does not overflow.
// `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
// The caller must ensure that `new_size` is greater than zero.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
let new_memory = self.alloc(new_layout, AllocInit::Uninitialized)?;
let new_layout =
// SAFETY: the caller must ensure that the `new_size` does not overflow.
// `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
// The caller must ensure that `new_size` is greater than zero.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
let new_memory = self.alloc(new_layout)?;
// SAFETY: because `new_size` must be lower than or equal to `size`, both the old and new
// memory allocation are valid for reads and writes for `new_size` bytes. Also, because the
// old allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to
// `copy_nonoverlapping` is safe.
// The safety contract for `dealloc` must be upheld by the caller.
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), new_size);
self.dealloc(ptr, layout);
Ok(new_memory)
}
}
// SAFETY: because `new_size` must be lower than or equal to `size`, both the old and new
// memory allocation are valid for reads and writes for `new_size` bytes. Also, because the
// old allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to
// `copy_nonoverlapping` is safe.
// The safety contract for `dealloc` must be upheld by the caller.
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), new_size);
self.dealloc(ptr, layout);
Ok(new_memory)
}
}
@ -409,8 +384,13 @@ where
A: AllocRef + ?Sized,
{
#[inline]
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
(**self).alloc(layout, init)
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
(**self).alloc(layout)
}
#[inline]
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
(**self).alloc_zeroed(layout)
}
#[inline]
@ -425,11 +405,20 @@ where
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
init: AllocInit,
) -> Result<MemoryBlock, AllocErr> {
// SAFETY: the safety contract must be upheld by the caller
unsafe { (**self).grow(ptr, layout, new_size, placement, init) }
unsafe { (**self).grow(ptr, layout, new_size) }
}
#[inline]
unsafe fn grow_zeroed(
&mut self,
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
) -> Result<MemoryBlock, AllocErr> {
// SAFETY: the safety contract must be upheld by the caller
unsafe { (**self).grow_zeroed(ptr, layout, new_size) }
}
#[inline]
@ -438,9 +427,8 @@ where
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
) -> Result<MemoryBlock, AllocErr> {
// SAFETY: the safety contract must be upheld by the caller
unsafe { (**self).shrink(ptr, layout, new_size, placement) }
unsafe { (**self).shrink(ptr, layout, new_size) }
}
}

181
library/std/src/alloc.rs
View File

@ -140,16 +140,27 @@ pub struct System;
#[unstable(feature = "allocator_api", issue = "32838")]
unsafe impl AllocRef for System {
#[inline]
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
unsafe {
let size = layout.size();
if size == 0 {
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
let raw_ptr = match init {
AllocInit::Uninitialized => GlobalAlloc::alloc(self, layout),
AllocInit::Zeroed => GlobalAlloc::alloc_zeroed(self, layout),
};
let raw_ptr = GlobalAlloc::alloc(self, layout);
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
Ok(MemoryBlock { ptr, size })
}
}
}
#[inline]
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
unsafe {
let size = layout.size();
if size == 0 {
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
let raw_ptr = GlobalAlloc::alloc_zeroed(self, layout);
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
Ok(MemoryBlock { ptr, size })
}
@ -171,8 +182,6 @@ unsafe impl AllocRef for System {
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
init: AllocInit,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
@ -184,51 +193,95 @@ unsafe impl AllocRef for System {
return Ok(MemoryBlock { ptr, size });
}
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove if layout.size() == 0 => {
let new_layout =
// SAFETY: The new size and layout alignement guarantees
// are transfered to the caller (they come from parameters).
//
// See the preconditions for `Layout::from_size_align` to
// see what must be checked.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc(new_layout, init)
}
ReallocPlacement::MayMove => {
// SAFETY:
if layout.size() == 0 {
let new_layout =
// SAFETY: The new size and layout alignement guarantees
// are transfered to the caller (they come from parameters).
//
// The safety guarantees are explained in the documentation
// for the `GlobalAlloc` trait and its `dealloc` method.
//
// `realloc` probably checks for `new_size > size` or something
// similar.
//
// For the guarantees about `init_offset`, see its documentation:
// `ptr` is assumed valid (and checked for non-NUL) and
// `memory.size` is set to `new_size` so the offset being `size`
// is valid.
let memory = unsafe {
intrinsics::assume(new_size > size);
let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size);
let memory =
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
init.init_offset(memory, size);
memory
};
// See the preconditions for `Layout::from_size_align` to
// see what must be checked.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc(new_layout)
} else {
// SAFETY:
//
// The safety guarantees are explained in the documentation
// for the `GlobalAlloc` trait and its `dealloc` method.
//
// `realloc` probably checks for `new_size > size` or something
// similar.
//
// For the guarantees about `init_offset`, see its documentation:
// `ptr` is assumed valid (and checked for non-NUL) and
// `memory.size` is set to `new_size` so the offset being `size`
// is valid.
unsafe {
intrinsics::assume(new_size > size);
let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size);
let memory =
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
Ok(memory)
}
}
}
#[inline]
unsafe fn grow_zeroed(
&mut self,
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
new_size >= size,
"`new_size` must be greater than or equal to `memory.size()`"
);
if size == new_size {
return Ok(MemoryBlock { ptr, size });
}
if layout.size() == 0 {
let new_layout =
// SAFETY: The new size and layout alignement guarantees
// are transfered to the caller (they come from parameters).
//
// See the preconditions for `Layout::from_size_align` to
// see what must be checked.
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
self.alloc_zeroed(new_layout)
} else {
// SAFETY:
//
// The safety guarantees are explained in the documentation
// for the `GlobalAlloc` trait and its `dealloc` method.
//
// `realloc` probably checks for `new_size > size` or something
// similar.
//
// For the guarantees about `init_offset`, see its documentation:
// `ptr` is assumed valid (and checked for non-NUL) and
// `memory.size` is set to `new_size` so the offset being `size`
// is valid.
let memory = unsafe {
intrinsics::assume(new_size > size);
let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size);
let memory =
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
memory.ptr.as_ptr().add(size).write_bytes(0, memory.size - size);
memory
};
Ok(memory)
}
}
#[inline]
unsafe fn shrink(
&mut self,
ptr: NonNull<u8>,
layout: Layout,
new_size: usize,
placement: ReallocPlacement,
) -> Result<MemoryBlock, AllocErr> {
let size = layout.size();
debug_assert!(
@ -240,32 +293,28 @@ unsafe impl AllocRef for System {
return Ok(MemoryBlock { ptr, size });
}
match placement {
ReallocPlacement::InPlace => Err(AllocErr),
ReallocPlacement::MayMove if new_size == 0 => {
// SAFETY: see `GlobalAlloc::dealloc` for the guarantees that
// must be respected. `ptr` and `layout` are parameters and so
// those guarantees must be checked by the caller.
unsafe { self.dealloc(ptr, layout) };
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
}
ReallocPlacement::MayMove => {
// SAFETY:
//
// See `GlobalAlloc::realloc` for more informations about the
// guarantees expected by this method. `ptr`, `layout` and
// `new_size` are parameters and the responsability for their
// correctness is left to the caller.
//
// `realloc` probably checks for `new_size < size` or something
// similar.
let memory = unsafe {
intrinsics::assume(new_size < size);
let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size);
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }
};
Ok(memory)
}
if new_size == 0 {
// SAFETY: see `GlobalAlloc::dealloc` for the guarantees that
// must be respected. `ptr` and `layout` are parameters and so
// those guarantees must be checked by the caller.
unsafe { self.dealloc(ptr, layout) };
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
} else {
// SAFETY:
//
// See `GlobalAlloc::realloc` for more informations about the
// guarantees expected by this method. `ptr`, `layout` and
// `new_size` are parameters and the responsability for their
// correctness is left to the caller.
//
// `realloc` probably checks for `new_size < size` or something
// similar.
let memory = unsafe {
intrinsics::assume(new_size < size);
let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size);
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }
};
Ok(memory)
}
}
}

6
src/test/ui/allocator/custom.rs
View File

@ -7,7 +7,7 @@
extern crate helper;
use std::alloc::{self, AllocInit, AllocRef, Global, Layout, System};
use std::alloc::{self, AllocRef, Global, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering};
static HITS: AtomicUsize = AtomicUsize::new(0);
@ -37,7 +37,7 @@ fn main() {
unsafe {
let layout = Layout::from_size_align(4, 2).unwrap();
let memory = Global.alloc(layout.clone(), AllocInit::Uninitialized).unwrap();
let memory = Global.alloc(layout.clone()).unwrap();
helper::work_with(&memory.ptr);
assert_eq!(HITS.load(Ordering::SeqCst), n + 1);
Global.dealloc(memory.ptr, layout);
@ -49,7 +49,7 @@ fn main() {
drop(s);
assert_eq!(HITS.load(Ordering::SeqCst), n + 4);
let memory = System.alloc(layout.clone(), AllocInit::Uninitialized).unwrap();
let memory = System.alloc(layout.clone()).unwrap();
assert_eq!(HITS.load(Ordering::SeqCst), n + 4);
helper::work_with(&memory.ptr);
System.dealloc(memory.ptr, layout);

6
src/test/ui/allocator/xcrate-use.rs
View File

@ -9,7 +9,7 @@
extern crate custom;
extern crate helper;
use std::alloc::{AllocInit, AllocRef, Global, Layout, System};
use std::alloc::{AllocRef, Global, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering};
#[global_allocator]
@ -20,13 +20,13 @@ fn main() {
let n = GLOBAL.0.load(Ordering::SeqCst);
let layout = Layout::from_size_align(4, 2).unwrap();
let memory = Global.alloc(layout.clone(), AllocInit::Uninitialized).unwrap();
let memory = Global.alloc(layout.clone()).unwrap();
helper::work_with(&memory.ptr);
assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 1);
Global.dealloc(memory.ptr, layout);
assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 2);
let memory = System.alloc(layout.clone(), AllocInit::Uninitialized).unwrap();
let memory = System.alloc(layout.clone()).unwrap();
assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 2);
helper::work_with(&memory.ptr);
System.dealloc(memory.ptr, layout);

10
src/test/ui/realloc-16687.rs
View File

@ -6,7 +6,7 @@
#![feature(allocator_api)]
use std::alloc::{handle_alloc_error, AllocInit, AllocRef, Global, Layout, ReallocPlacement};
use std::alloc::{handle_alloc_error, AllocRef, Global, Layout};
use std::ptr::{self, NonNull};
fn main() {
@ -41,9 +41,7 @@ unsafe fn test_triangle() -> bool {
println!("allocate({:?})", layout);
}
let memory = Global
.alloc(layout, AllocInit::Uninitialized)
.unwrap_or_else(|_| handle_alloc_error(layout));
let memory = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
if PRINT {
println!("allocate({:?}) = {:?}", layout, memory.ptr);
@ -70,11 +68,9 @@ unsafe fn test_triangle() -> bool {
NonNull::new_unchecked(ptr),
old,
new.size(),
ReallocPlacement::MayMove,
AllocInit::Uninitialized,
)
} else {
Global.shrink(NonNull::new_unchecked(ptr), old, new.size(), ReallocPlacement::MayMove)
Global.shrink(NonNull::new_unchecked(ptr), old, new.size())
};
let memory = memory.unwrap_or_else(|_| {

6
src/test/ui/regions/regions-mock-codegen.rs
View File

@ -4,7 +4,7 @@
// pretty-expanded FIXME #23616
#![feature(allocator_api)]
use std::alloc::{handle_alloc_error, AllocInit, AllocRef, Global, Layout};
use std::alloc::{handle_alloc_error, AllocRef, Global, Layout};
use std::ptr::NonNull;
struct arena(());
@ -25,9 +25,7 @@ struct Ccx {
fn alloc(_bcx: &arena) -> &Bcx<'_> {
unsafe {
let layout = Layout::new::<Bcx>();
let memory = Global
.alloc(layout, AllocInit::Uninitialized)
.unwrap_or_else(|_| handle_alloc_error(layout));
let memory = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
&*(memory.ptr.as_ptr() as *const _)
}
}