nordic-dev.net/vulkano
nordic-dev.net/vulkano
2
0
Fork 0
mirror of https://github.com/vulkano-rs/vulkano.git synced 2024-11-22 06:45:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make GenericMemoryAllocatorCreateInfo::block_sizes more flexible (#2319)

Browse Source
This commit is contained in:
marc0246 2023-09-13 16:17:54 +02:00 committed by GitHub
parent 28224138f2
commit aecb7a476a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 59 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
vulkano/src/memory/allocator/mod.rs
View File

@ -238,7 +238,7 @@ use crate::{
DeviceSize, Requires, RequiresAllOf, RequiresOneOf, Validated, ValidationError, Version, DeviceSize, Requires, RequiresAllOf, RequiresOneOf, Validated, ValidationError, Version,
VulkanError, VulkanError,
}; };
use ash::vk::{MAX_MEMORY_HEAPS, MAX_MEMORY_TYPES}; use ash::vk::MAX_MEMORY_TYPES;
use parking_lot::Mutex; use parking_lot::Mutex;
use std::{ use std::{
error::Error, error::Error,
@ -249,11 +249,6 @@ use std::{
sync::Arc, sync::Arc,
}; };
const B: DeviceSize = 1;
const K: DeviceSize = 1024 * B;
const M: DeviceSize = 1024 * K;
const G: DeviceSize = 1024 * M;
/// General-purpose memory allocators which allocate from any memory type dynamically as needed. /// General-purpose memory allocators which allocate from any memory type dynamically as needed.
/// ///
/// # Safety /// # Safety
@ -821,12 +816,26 @@ pub type StandardMemoryAllocator = GenericMemoryAllocator<FreeListAllocator>;
impl StandardMemoryAllocator { impl StandardMemoryAllocator {
/// Creates a new `StandardMemoryAllocator` with default configuration. /// Creates a new `StandardMemoryAllocator` with default configuration.
pub fn new_default(device: Arc<Device>) -> Self { pub fn new_default(device: Arc<Device>) -> Self {
let memory_types = &device.physical_device().memory_properties().memory_types; let MemoryProperties {
memory_types,
memory_heaps,
} = device.physical_device().memory_properties();
let mut block_sizes = [0; MAX_MEMORY_TYPES];
let mut memory_type_bits = u32::MAX; let mut memory_type_bits = u32::MAX;
for (index, MemoryType { property_flags, .. }) in memory_types.iter().enumerate() { for (index, memory_type) in memory_types.iter().enumerate() {
if property_flags.intersects( const LARGE_HEAP_THRESHOLD: DeviceSize = 1024 * 1024 * 1024;
let heap_size = memory_heaps[memory_type.heap_index as usize].size;
block_sizes[index] = if heap_size >= LARGE_HEAP_THRESHOLD {
256 * 1024 * 1024
} else {
64 * 1024 * 1024
};
if memory_type.property_flags.intersects(
MemoryPropertyFlags::LAZILY_ALLOCATED MemoryPropertyFlags::LAZILY_ALLOCATED
| MemoryPropertyFlags::PROTECTED | MemoryPropertyFlags::PROTECTED
| MemoryPropertyFlags::DEVICE_COHERENT | MemoryPropertyFlags::DEVICE_COHERENT
@ -839,13 +848,8 @@ impl StandardMemoryAllocator {
} }
} }
#[allow(clippy::erasing_op, clippy::identity_op)]
let create_info = GenericMemoryAllocatorCreateInfo { let create_info = GenericMemoryAllocatorCreateInfo {
#[rustfmt::skip] block_sizes: &block_sizes,
block_sizes: &[
(0 * B, 64 * M),
(1 * G, 256 * M),
],
memory_type_bits, memory_type_bits,
..Default::default() ..Default::default()
}; };
@ -894,8 +898,6 @@ pub struct GenericMemoryAllocator<S> {
buffer_image_granularity: DeviceAlignment, buffer_image_granularity: DeviceAlignment,
// Each memory type has a pool of `DeviceMemory` blocks. // Each memory type has a pool of `DeviceMemory` blocks.
pools: ArrayVec<Pool<S>, MAX_MEMORY_TYPES>, pools: ArrayVec<Pool<S>, MAX_MEMORY_TYPES>,
// Each memory heap has its own block size.
block_sizes: ArrayVec<DeviceSize, MAX_MEMORY_HEAPS>,
// Global mask of memory types. // Global mask of memory types.
memory_type_bits: u32, memory_type_bits: u32,
dedicated_allocation: bool, dedicated_allocation: bool,
@ -909,8 +911,9 @@ pub struct GenericMemoryAllocator<S> {
struct Pool<S> { struct Pool<S> {
blocks: Mutex<Vec<Box<Block<S>>>>, blocks: Mutex<Vec<Box<Block<S>>>>,
// This is cached here for faster access, so we don't need to hop through 3 pointers. // This is cached here for faster access, so we don't need to hop through 3 pointers.
memory_type: ash::vk::MemoryType, property_flags: MemoryPropertyFlags,
atom_size: DeviceAlignment, atom_size: DeviceAlignment,
block_size: DeviceSize,
} }
impl<S> GenericMemoryAllocator<S> { impl<S> GenericMemoryAllocator<S> {
@ -918,11 +921,9 @@ impl<S> GenericMemoryAllocator<S> {
#[allow(clippy::declare_interior_mutable_const)] #[allow(clippy::declare_interior_mutable_const)]
const EMPTY_POOL: Pool<S> = Pool { const EMPTY_POOL: Pool<S> = Pool {
blocks: Mutex::new(Vec::new()), blocks: Mutex::new(Vec::new()),
memory_type: ash::vk::MemoryType { property_flags: MemoryPropertyFlags::empty(),
property_flags: ash::vk::MemoryPropertyFlags::empty(),
heap_index: 0,
},
atom_size: DeviceAlignment::MIN, atom_size: DeviceAlignment::MIN,
block_size: 0,
}; };
/// Creates a new `GenericMemoryAllocator<S>` using the provided suballocator `S` for /// Creates a new `GenericMemoryAllocator<S>` using the provided suballocator `S` for
@ -930,12 +931,13 @@ impl<S> GenericMemoryAllocator<S> {
/// ///
/// # Panics /// # Panics
/// ///
/// - Panics if `create_info.block_sizes` is not sorted by threshold. /// - Panics if `create_info.block_sizes` doesn't contain as many elements as the number of
/// - Panics if `create_info.block_sizes` contains duplicate thresholds. /// memory types.
/// - Panics if `create_info.block_sizes` does not contain a baseline threshold of `0`. /// - Panics if `create_info.export_handle_types` is non-empty and doesn't contain as many
/// elements as the number of memory types.
pub fn new( pub fn new(
device: Arc<Device>, device: Arc<Device>,
create_info: GenericMemoryAllocatorCreateInfo<'_, '_>, create_info: GenericMemoryAllocatorCreateInfo<'_>,
) -> Result<Self, Box<ValidationError>> { ) -> Result<Self, Box<ValidationError>> {
Self::validate_new(&device, &create_info)?; Self::validate_new(&device, &create_info)?;
@ -944,7 +946,7 @@ impl<S> GenericMemoryAllocator<S> {
fn validate_new( fn validate_new(
device: &Device, device: &Device,
create_info: &GenericMemoryAllocatorCreateInfo<'_, '_>, create_info: &GenericMemoryAllocatorCreateInfo<'_>,
) -> Result<(), Box<ValidationError>> { ) -> Result<(), Box<ValidationError>> {
create_info create_info
.validate(device) .validate(device)
@ -956,7 +958,7 @@ impl<S> GenericMemoryAllocator<S> {
#[cfg_attr(not(feature = "document_unchecked"), doc(hidden))] #[cfg_attr(not(feature = "document_unchecked"), doc(hidden))]
pub unsafe fn new_unchecked( pub unsafe fn new_unchecked(
device: Arc<Device>, device: Arc<Device>,
create_info: GenericMemoryAllocatorCreateInfo<'_, '_>, create_info: GenericMemoryAllocatorCreateInfo<'_>,
) -> Self { ) -> Self {
let GenericMemoryAllocatorCreateInfo { let GenericMemoryAllocatorCreateInfo {
block_sizes, block_sizes,
@ -972,47 +974,23 @@ impl<S> GenericMemoryAllocator<S> {
.properties() .properties()
.buffer_image_granularity; .buffer_image_granularity;
let MemoryProperties { let memory_types = &device.physical_device().memory_properties().memory_types;
memory_types,
memory_heaps,
} = device.physical_device().memory_properties();
let mut pools = ArrayVec::new(memory_types.len(), [Self::EMPTY_POOL; MAX_MEMORY_TYPES]); let mut pools = ArrayVec::new(memory_types.len(), [Self::EMPTY_POOL; MAX_MEMORY_TYPES]);
for ( for (index, &MemoryType { property_flags, .. }) in memory_types.iter().enumerate() {
i, pools[index].property_flags = property_flags;
&MemoryType {
property_flags,
heap_index,
},
) in memory_types.iter().enumerate()
{
pools[i].memory_type = ash::vk::MemoryType {
property_flags: property_flags.into(),
heap_index,
};
if property_flags.intersects(MemoryPropertyFlags::HOST_VISIBLE) if property_flags.intersects(MemoryPropertyFlags::HOST_VISIBLE)
&& !property_flags.intersects(MemoryPropertyFlags::HOST_COHERENT) && !property_flags.intersects(MemoryPropertyFlags::HOST_COHERENT)
{ {
pools[i].atom_size = device.physical_device().properties().non_coherent_atom_size; pools[index].atom_size =
device.physical_device().properties().non_coherent_atom_size;
} }
pools[index].block_size = block_sizes[index];
} }
let block_sizes = {
let mut sizes = ArrayVec::new(memory_heaps.len(), [0; MAX_MEMORY_HEAPS]);
for (i, memory_heap) in memory_heaps.iter().enumerate() {
let idx = match block_sizes.binary_search_by_key(&memory_heap.size, |&(t, _)| t) {
Ok(idx) => idx,
Err(idx) => idx.saturating_sub(1),
};
sizes[i] = block_sizes[idx].1;
}
sizes
};
let export_handle_types = { let export_handle_types = {
let mut types = ArrayVec::new( let mut types = ArrayVec::new(
export_handle_types.len(), export_handle_types.len(),
@ -1046,7 +1024,6 @@ impl<S> GenericMemoryAllocator<S> {
device: InstanceOwnedDebugWrapper(device), device: InstanceOwnedDebugWrapper(device),
buffer_image_granularity, buffer_image_granularity,
pools, pools,
block_sizes,
dedicated_allocation, dedicated_allocation,
export_handle_types, export_handle_types,
flags, flags,
@ -1076,9 +1053,8 @@ impl<S> GenericMemoryAllocator<S> {
)?; )?;
if self.pools[memory_type_index as usize] if self.pools[memory_type_index as usize]
.memory_type
.property_flags .property_flags
.intersects(ash::vk::MemoryPropertyFlags::HOST_VISIBLE) .intersects(MemoryPropertyFlags::HOST_VISIBLE)
{ {
// SAFETY: // SAFETY:
// - We checked that the memory is host-visible. // - We checked that the memory is host-visible.
@ -1109,7 +1085,7 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
self.pools self.pools
.iter() .iter()
.map(|pool| pool.memory_type.property_flags) .map(|pool| ash::vk::MemoryPropertyFlags::from(pool.property_flags))
.enumerate() .enumerate()
// Filter out memory types which are supported by the memory type bits and have the // Filter out memory types which are supported by the memory type bits and have the
// required flags set. // required flags set.
@ -1163,9 +1139,8 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
) -> Result<MemoryAlloc, MemoryAllocatorError> { ) -> Result<MemoryAlloc, MemoryAllocatorError> {
let size = layout.size(); let size = layout.size();
let pool = &self.pools[memory_type_index as usize]; let pool = &self.pools[memory_type_index as usize];
let block_size = self.block_sizes[pool.memory_type.heap_index as usize];
if size > block_size { if size > pool.block_size {
return Err(MemoryAllocatorError::BlockSizeExceeded); return Err(MemoryAllocatorError::BlockSizeExceeded);
} }
@ -1199,7 +1174,7 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
let mut i = 0; let mut i = 0;
loop { loop {
let allocation_size = block_size >> i; let allocation_size = pool.block_size >> i;
match self.allocate_device_memory( match self.allocate_device_memory(
memory_type_index, memory_type_index,
@ -1214,7 +1189,7 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
// resulting size is still large enough. // resulting size is still large enough.
Err(Validated::Error( Err(Validated::Error(
VulkanError::OutOfHostMemory | VulkanError::OutOfDeviceMemory, VulkanError::OutOfHostMemory | VulkanError::OutOfDeviceMemory,
)) if i < 3 && block_size >> (i + 1) >= size => { )) if i < 3 && pool.block_size >> (i + 1) >= size => {
i += 1; i += 1;
} }
Err(err) => return Err(MemoryAllocatorError::AllocateDeviceMemory(err)), Err(err) => return Err(MemoryAllocatorError::AllocateDeviceMemory(err)),
@ -1330,8 +1305,7 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
}; };
loop { loop {
let memory_type = self.pools[memory_type_index as usize].memory_type; let pool = &self.pools[memory_type_index as usize];
let block_size = self.block_sizes[memory_type.heap_index as usize];
let res = match allocate_preference { let res = match allocate_preference {
MemoryAllocatePreference::Unknown => { MemoryAllocatePreference::Unknown => {
@ -1345,11 +1319,11 @@ unsafe impl<S: Suballocator + Send + 'static> MemoryAllocator for GenericMemoryA
export_handle_types, export_handle_types,
) )
} else { } else {
if size > block_size / 2 { if size > pool.block_size / 2 {
prefers_dedicated_allocation = true; prefers_dedicated_allocation = true;
} }
if self.device.allocation_count() > self.max_allocations if self.device.allocation_count() > self.max_allocations
&& size <= block_size && size <= pool.block_size
{ {
prefers_dedicated_allocation = false; prefers_dedicated_allocation = false;
} }
@ -1589,17 +1563,12 @@ impl<S: Suballocator> Block<S> {
/// Parameters to create a new [`GenericMemoryAllocator`]. /// Parameters to create a new [`GenericMemoryAllocator`].
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct GenericMemoryAllocatorCreateInfo<'b, 'e> { pub struct GenericMemoryAllocatorCreateInfo<'a> {
/// Lets you configure the block sizes for various heap size classes. /// Lets you configure the block sizes for each memory type.
/// ///
/// Each entry is a pair of the threshold for the heap size and the block size that should be /// Must contain one entry for each memory type. The allocator keeps a pool of [`DeviceMemory`]
/// used for that heap. Must be sorted by threshold and all thresholds must be unique. Must /// blocks for each memory type, and every time a new block is allocated, the block size
/// contain a baseline threshold of 0. /// corresponding to the memory type index is looked up here and used for the allocation.
///
/// The allocator keeps a pool of [`DeviceMemory`] blocks for each memory type, so each memory
/// type that resides in a heap whose size crosses one of the thresholds will use the
/// corresponding block size. If multiple thresholds apply to a given heap, the block size
/// corresponding to the largest threshold is chosen.
/// ///
/// The block size is going to be the maximum size of a `DeviceMemory` block that is tried. If /// The block size is going to be the maximum size of a `DeviceMemory` block that is tried. If
/// allocating a block with the size fails, the allocator tries 1/2, 1/4 and 1/8 of the block /// allocating a block with the size fails, the allocator tries 1/2, 1/4 and 1/8 of the block
@ -1609,7 +1578,7 @@ pub struct GenericMemoryAllocatorCreateInfo<'b, 'e> {
/// [`MemoryAllocatePreference::NeverAllocate`] however. /// [`MemoryAllocatePreference::NeverAllocate`] however.
/// ///
/// The default value is `&[]`, which must be overridden. /// The default value is `&[]`, which must be overridden.
pub block_sizes: &'b [(Threshold, BlockSize)], pub block_sizes: &'a [DeviceSize],
/// Lets you configure the allocator's global mask of memory type indices. Only the memory type /// Lets you configure the allocator's global mask of memory type indices. Only the memory type
/// indices that have a corresponding bit at the same index set will be allocated from when /// indices that have a corresponding bit at the same index set will be allocated from when
@ -1654,7 +1623,7 @@ pub struct GenericMemoryAllocatorCreateInfo<'b, 'e> {
/// here and used for the allocation. /// here and used for the allocation.
/// ///
/// The default value is `&[]`. /// The default value is `&[]`.
pub export_handle_types: &'e [ExternalMemoryHandleTypes], pub export_handle_types: &'a [ExternalMemoryHandleTypes],
/// Whether the allocator should allocate the [`DeviceMemory`] blocks with the /// Whether the allocator should allocate the [`DeviceMemory`] blocks with the
/// [`DEVICE_ADDRESS`] flag set. /// [`DEVICE_ADDRESS`] flag set.
@ -1679,11 +1648,7 @@ pub struct GenericMemoryAllocatorCreateInfo<'b, 'e> {
pub _ne: crate::NonExhaustive, pub _ne: crate::NonExhaustive,
} }
pub type Threshold = DeviceSize; impl GenericMemoryAllocatorCreateInfo<'_> {
pub type BlockSize = DeviceSize;
impl GenericMemoryAllocatorCreateInfo<'_, '_> {
pub(crate) fn validate(&self, device: &Device) -> Result<(), Box<ValidationError>> { pub(crate) fn validate(&self, device: &Device) -> Result<(), Box<ValidationError>> {
let &Self { let &Self {
block_sizes, block_sizes,
@ -1694,13 +1659,11 @@ impl GenericMemoryAllocatorCreateInfo<'_, '_> {
_ne: _, _ne: _,
} = self; } = self;
let memory_types = &device.physical_device().memory_properties().memory_types;
assert!( assert!(
block_sizes.windows(2).all(|win| win[0].0 < win[1].0), block_sizes.len() == memory_types.len(),
"`create_info.block_sizes` must be sorted by threshold without duplicates", "`create_info.block_sizes` must contain as many elements as the number of memory types",
);
assert!(
matches!(block_sizes.first(), Some((0, _))),
"`create_info.block_sizes` must contain a baseline threshold `0`",
); );
if !export_handle_types.is_empty() { if !export_handle_types.is_empty() {
@ -1719,12 +1682,7 @@ impl GenericMemoryAllocatorCreateInfo<'_, '_> {
} }
assert!( assert!(
export_handle_types.len() export_handle_types.len() == memory_types.len(),
== device
.physical_device()
.memory_properties()
.memory_types
.len(),
"`create_info.export_handle_types` must contain as many elements as the number of \ "`create_info.export_handle_types` must contain as many elements as the number of \
memory types if not empty", memory types if not empty",
); );
@ -1740,7 +1698,7 @@ impl GenericMemoryAllocatorCreateInfo<'_, '_> {
} }
} }
impl Default for GenericMemoryAllocatorCreateInfo<'_, '_> { impl Default for GenericMemoryAllocatorCreateInfo<'_> {
#[inline] #[inline]
fn default() -> Self { fn default() -> Self {
GenericMemoryAllocatorCreateInfo { GenericMemoryAllocatorCreateInfo {