From 127b3eae612c4aa52d45d428396296a416b819ce Mon Sep 17 00:00:00 2001 From: marc0246 <40955683+marc0246@users.noreply.github.com> Date: Wed, 15 Mar 2023 22:21:44 +0100 Subject: [PATCH] Generate device properties that are alignment as `DeviceAlignment` (#2155) * Move `DeviceAlignment` * Generate alignment-properties with the type `DeviceAlignment` * Small fix * Another small fix * Switch to using `is_aligned` where possible * Oopsie --- examples/src/bin/dynamic-buffers.rs | 3 +- vulkano/autogen/properties.rs | 13 + vulkano/src/buffer/allocator.rs | 10 +- vulkano/src/buffer/mod.rs | 13 +- vulkano/src/buffer/subbuffer.rs | 4 +- vulkano/src/buffer/sys.rs | 20 +- vulkano/src/buffer/view.rs | 14 +- .../src/command_buffer/commands/bind_push.rs | 11 +- .../standard/builder/bind_push.rs | 7 +- vulkano/src/device/properties.rs | 22 +- vulkano/src/image/attachment.rs | 6 +- vulkano/src/image/immutable.rs | 4 +- vulkano/src/image/storage.rs | 6 +- vulkano/src/image/sys.rs | 10 +- vulkano/src/memory/alignment.rs | 249 ++++++++++++++++++ vulkano/src/memory/allocator/layout.rs | 241 +---------------- vulkano/src/memory/allocator/mod.rs | 7 +- vulkano/src/memory/allocator/suballocator.rs | 83 +++--- vulkano/src/memory/device_memory.rs | 20 +- vulkano/src/memory/mod.rs | 4 +- 20 files changed, 384 insertions(+), 363 deletions(-) create mode 100644 vulkano/src/memory/alignment.rs diff --git a/examples/src/bin/dynamic-buffers.rs b/examples/src/bin/dynamic-buffers.rs index a2cf6df5..e981d32d 100644 --- a/examples/src/bin/dynamic-buffers.rs +++ b/examples/src/bin/dynamic-buffers.rs @@ -143,7 +143,8 @@ fn main() { let min_dynamic_align = device .physical_device() .properties() - .min_uniform_buffer_offset_alignment as usize; + .min_uniform_buffer_offset_alignment + .as_devicesize() as usize; println!("Minimum uniform buffer offset alignment: {min_dynamic_align}"); println!("Input: {data:?}"); diff --git a/vulkano/autogen/properties.rs b/vulkano/autogen/properties.rs index ff91b3a9..1212dbaf 100644 --- a/vulkano/autogen/properties.rs +++ b/vulkano/autogen/properties.rs @@ -196,6 +196,19 @@ fn properties_members(types: &HashMap<&str, (&Type, Vec<&str>)>) -> Vec quote! { Version }, + "bufferImageGranularity" + | "minStorageBufferOffsetAlignment" + | "minTexelBufferOffsetAlignment" + | "minUniformBufferOffsetAlignment" + | "nonCoherentAtomSize" + | "optimalBufferCopyOffsetAlignment" + | "optimalBufferCopyRowPitchAlignment" + | "robustStorageBufferAccessSizeAlignment" + | "robustUniformBufferAccessSizeAlignment" + | "storageTexelBufferOffsetAlignmentBytes" + | "uniformTexelBufferOffsetAlignmentBytes" => { + quote! { DeviceAlignment } + } _ => vulkano_type(ty, len), }; match properties.entry(vulkano_member.clone()) { diff --git a/vulkano/src/buffer/allocator.rs b/vulkano/src/buffer/allocator.rs index 0645093c..5f342a0d 100644 --- a/vulkano/src/buffer/allocator.rs +++ b/vulkano/src/buffer/allocator.rs @@ -13,9 +13,12 @@ use super::{Buffer, BufferContents, BufferError, BufferMemory, BufferUsage, Subb use crate::{ buffer::BufferAllocateInfo, device::{Device, DeviceOwned}, - memory::allocator::{ - align_up, AllocationCreationError, DeviceAlignment, DeviceLayout, MemoryAllocator, - MemoryUsage, StandardMemoryAllocator, + memory::{ + allocator::{ + align_up, AllocationCreationError, DeviceLayout, MemoryAllocator, MemoryUsage, + StandardMemoryAllocator, + }, + DeviceAlignment, }, DeviceSize, NonZeroDeviceSize, }; @@ -145,7 +148,6 @@ where .into_iter() .flatten() .max() - .map(|alignment| DeviceAlignment::new(alignment).unwrap()) .unwrap_or(DeviceAlignment::MIN); SubbufferAllocator { diff --git a/vulkano/src/buffer/mod.rs b/vulkano/src/buffer/mod.rs index 4f4b9f6b..a2ba48ea 100644 --- a/vulkano/src/buffer/mod.rs +++ b/vulkano/src/buffer/mod.rs @@ -111,11 +111,11 @@ use crate::{ macros::vulkan_bitflags, memory::{ allocator::{ - AllocationCreateInfo, AllocationCreationError, AllocationType, DeviceAlignment, - DeviceLayout, MemoryAlloc, MemoryAllocatePreference, MemoryAllocator, MemoryUsage, + AllocationCreateInfo, AllocationCreationError, AllocationType, DeviceLayout, + MemoryAlloc, MemoryAllocatePreference, MemoryAllocator, MemoryUsage, }, - DedicatedAllocation, ExternalMemoryHandleType, ExternalMemoryHandleTypes, - ExternalMemoryProperties, MemoryRequirements, + is_aligned, DedicatedAllocation, DeviceAlignment, ExternalMemoryHandleType, + ExternalMemoryHandleTypes, ExternalMemoryProperties, MemoryRequirements, }, range_map::RangeMap, sync::{future::AccessError, CurrentAccess, Sharing}, @@ -394,7 +394,10 @@ impl Buffer { }; let mut allocation = unsafe { allocator.allocate_unchecked(create_info) }?; - debug_assert!(allocation.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned( + allocation.offset(), + requirements.layout.alignment(), + )); debug_assert!(allocation.size() == requirements.layout.size()); // The implementation might require a larger size than we wanted. With this it is easier to diff --git a/vulkano/src/buffer/subbuffer.rs b/vulkano/src/buffer/subbuffer.rs index 49706722..7d4c5fb6 100644 --- a/vulkano/src/buffer/subbuffer.rs +++ b/vulkano/src/buffer/subbuffer.rs @@ -15,8 +15,8 @@ use crate::{ macros::try_opt, memory::{ self, - allocator::{align_down, align_up, DeviceAlignment, DeviceLayout}, - is_aligned, + allocator::{align_down, align_up, DeviceLayout}, + is_aligned, DeviceAlignment, }, DeviceSize, NonZeroDeviceSize, }; diff --git a/vulkano/src/buffer/sys.rs b/vulkano/src/buffer/sys.rs index 6c7985fb..4821e9ab 100644 --- a/vulkano/src/buffer/sys.rs +++ b/vulkano/src/buffer/sys.rs @@ -17,9 +17,9 @@ use super::{Buffer, BufferCreateFlags, BufferError, BufferMemory, BufferUsage}; use crate::{ device::{Device, DeviceOwned}, memory::{ - allocator::{AllocationType, DeviceAlignment, DeviceLayout, MemoryAlloc}, - DedicatedTo, ExternalMemoryHandleTypes, MemoryAllocateFlags, MemoryPropertyFlags, - MemoryRequirements, + allocator::{AllocationType, DeviceLayout, MemoryAlloc}, + is_aligned, DedicatedTo, ExternalMemoryHandleTypes, MemoryAllocateFlags, + MemoryPropertyFlags, MemoryRequirements, }, sync::Sharing, DeviceSize, RequiresOneOf, Version, VulkanError, VulkanObject, @@ -290,27 +290,21 @@ impl RawBuffer { if usage.intersects(BufferUsage::UNIFORM_TEXEL_BUFFER | BufferUsage::STORAGE_TEXEL_BUFFER) { memory_requirements.layout = memory_requirements .layout - .align_to( - DeviceAlignment::new(properties.min_texel_buffer_offset_alignment).unwrap(), - ) + .align_to(properties.min_texel_buffer_offset_alignment) .unwrap(); } if usage.intersects(BufferUsage::STORAGE_BUFFER) { memory_requirements.layout = memory_requirements .layout - .align_to( - DeviceAlignment::new(properties.min_storage_buffer_offset_alignment).unwrap(), - ) + .align_to(properties.min_storage_buffer_offset_alignment) .unwrap(); } if usage.intersects(BufferUsage::UNIFORM_BUFFER) { memory_requirements.layout = memory_requirements .layout - .align_to( - DeviceAlignment::new(properties.min_uniform_buffer_offset_alignment).unwrap(), - ) + .align_to(properties.min_uniform_buffer_offset_alignment) .unwrap(); } @@ -445,7 +439,7 @@ impl RawBuffer { } // VUID-VkBindBufferMemoryInfo-memoryOffset-01036 - if memory_offset % memory_requirements.layout.alignment().as_nonzero() != 0 { + if !is_aligned(memory_offset, memory_requirements.layout.alignment()) { return Err(BufferError::MemoryAllocationNotAligned { allocation_offset: memory_offset, required_alignment: memory_requirements.layout.alignment(), diff --git a/vulkano/src/buffer/view.rs b/vulkano/src/buffer/view.rs index a3beb09f..4dbed9e7 100644 --- a/vulkano/src/buffer/view.rs +++ b/vulkano/src/buffer/view.rs @@ -49,6 +49,7 @@ use super::{BufferUsage, Subbuffer}; use crate::{ device::{Device, DeviceOwned}, format::{Format, FormatFeatures}, + memory::{is_aligned, DeviceAlignment}, DeviceSize, OomError, RequirementNotMet, RequiresOneOf, Version, VulkanError, VulkanObject, }; use std::{ @@ -152,11 +153,12 @@ impl BufferView { if device.api_version() >= Version::V1_3 || device.enabled_features().texel_buffer_alignment { - let element_size = if block_size % 3 == 0 { + let element_size = DeviceAlignment::new(if block_size % 3 == 0 { block_size / 3 } else { block_size - }; + }) + .unwrap(); if buffer.usage().intersects(BufferUsage::STORAGE_TEXEL_BUFFER) { let mut required_alignment = properties @@ -171,7 +173,7 @@ impl BufferView { } // VUID-VkBufferViewCreateInfo-buffer-02750 - if offset % required_alignment != 0 { + if !is_aligned(offset, required_alignment) { return Err(BufferViewCreationError::OffsetNotAligned { offset, required_alignment, @@ -192,7 +194,7 @@ impl BufferView { } // VUID-VkBufferViewCreateInfo-buffer-02751 - if offset % required_alignment != 0 { + if !is_aligned(offset, required_alignment) { return Err(BufferViewCreationError::OffsetNotAligned { offset, required_alignment, @@ -203,7 +205,7 @@ impl BufferView { let required_alignment = properties.min_texel_buffer_offset_alignment; // VUID-VkBufferViewCreateInfo-offset-02749 - if offset % required_alignment != 0 { + if !is_aligned(offset, required_alignment) { return Err(BufferViewCreationError::OffsetNotAligned { offset, required_alignment, @@ -340,7 +342,7 @@ pub enum BufferViewCreationError { /// The offset within the buffer is not a multiple of the required alignment. OffsetNotAligned { offset: DeviceSize, - required_alignment: DeviceSize, + required_alignment: DeviceAlignment, }, /// The range within the buffer is not a multiple of the required alignment. diff --git a/vulkano/src/command_buffer/commands/bind_push.rs b/vulkano/src/command_buffer/commands/bind_push.rs index d238cc9c..2261096c 100644 --- a/vulkano/src/command_buffer/commands/bind_push.rs +++ b/vulkano/src/command_buffer/commands/bind_push.rs @@ -23,6 +23,7 @@ use crate::{ WriteDescriptorSet, }, device::{DeviceOwned, QueueFlags}, + memory::{is_aligned, DeviceAlignment}, pipeline::{ graphics::{ input_assembly::{Index, IndexType}, @@ -132,8 +133,8 @@ where } let properties = self.device().physical_device().properties(); - let uniform_alignment = properties.min_uniform_buffer_offset_alignment as u32; - let storage_alignment = properties.min_storage_buffer_offset_alignment as u32; + let uniform_alignment = properties.min_uniform_buffer_offset_alignment; + let storage_alignment = properties.min_storage_buffer_offset_alignment; for (i, set) in descriptor_sets.iter().enumerate() { let set_num = first_set + i as u32; @@ -183,7 +184,7 @@ where { // VUID-vkCmdBindDescriptorSets-pDynamicOffsets-01971 // VUID-vkCmdBindDescriptorSets-pDynamicOffsets-01972 - if offset % required_alignment != 0 { + if !is_aligned(offset as DeviceSize, required_alignment) { return Err(BindPushError::DynamicOffsetNotAligned { set_num, binding_num, @@ -1341,7 +1342,7 @@ pub(in super::super) enum BindPushError { binding_num: u32, index: u32, offset: u32, - required_alignment: u32, + required_alignment: DeviceAlignment, }, /// In an element of `descriptor_sets`, a provided dynamic offset, when added to the end of the @@ -1459,7 +1460,7 @@ impl Display for BindPushError { "in the element of `descriptor_sets` being bound to slot {}, the dynamic offset \ provided for binding {} index {} ({}) is not a multiple of the value of the \ `min_uniform_buffer_offset_alignment` or `min_storage_buffer_offset_alignment` \ - property ({})", + property ({:?})", set_num, binding_num, index, offset, required_alignment, ), Self::DynamicOffsetOutOfBufferBounds { diff --git a/vulkano/src/command_buffer/standard/builder/bind_push.rs b/vulkano/src/command_buffer/standard/builder/bind_push.rs index afafb3d8..7fe5d7ba 100644 --- a/vulkano/src/command_buffer/standard/builder/bind_push.rs +++ b/vulkano/src/command_buffer/standard/builder/bind_push.rs @@ -17,6 +17,7 @@ use crate::{ DescriptorWriteInfo, WriteDescriptorSet, }, device::{DeviceOwned, QueueFlags}, + memory::is_aligned, pipeline::{ graphics::{ input_assembly::{Index, IndexType}, @@ -110,8 +111,8 @@ where } let properties = self.device().physical_device().properties(); - let uniform_alignment = properties.min_uniform_buffer_offset_alignment as u32; - let storage_alignment = properties.min_storage_buffer_offset_alignment as u32; + let uniform_alignment = properties.min_uniform_buffer_offset_alignment; + let storage_alignment = properties.min_storage_buffer_offset_alignment; for (i, set) in descriptor_sets.iter().enumerate() { let set_num = first_set + i as u32; @@ -161,7 +162,7 @@ where { // VUID-vkCmdBindDescriptorSets-pDynamicOffsets-01971 // VUID-vkCmdBindDescriptorSets-pDynamicOffsets-01972 - if offset % required_alignment != 0 { + if !is_aligned(offset as DeviceSize, required_alignment) { return Err(BindPushError::DynamicOffsetNotAligned { set_num, binding_num, diff --git a/vulkano/src/device/properties.rs b/vulkano/src/device/properties.rs index 7cdb6e59..85e189ce 100644 --- a/vulkano/src/device/properties.rs +++ b/vulkano/src/device/properties.rs @@ -1,17 +1,14 @@ use super::physical::{ - MemoryDecompressionMethods, OpticalFlowGridSizes, PipelineRobustnessBufferBehavior, - PipelineRobustnessImageBehavior, RayTracingInvocationReorderMode, + ConformanceVersion, DriverId, MemoryDecompressionMethods, OpticalFlowGridSizes, + PhysicalDeviceType, PipelineRobustnessBufferBehavior, PipelineRobustnessImageBehavior, + PointClippingBehavior, RayTracingInvocationReorderMode, ShaderCoreProperties, + ShaderFloatControlsIndependence, SubgroupFeatures, }; use crate::{ - device::{ - physical::{ - ConformanceVersion, DriverId, PhysicalDeviceType, PointClippingBehavior, - ShaderCoreProperties, ShaderFloatControlsIndependence, SubgroupFeatures, - }, - DeviceExtensions, QueueFlags, - }, + device::{DeviceExtensions, QueueFlags}, image::{SampleCount, SampleCounts}, instance::InstanceExtensions, + memory::DeviceAlignment, render_pass::ResolveModes, shader::ShaderStages, DeviceSize, Version, @@ -65,6 +62,13 @@ impl FromVulkan for u64 { } } +impl FromVulkan for DeviceAlignment { + #[inline] + fn from_vulkan(val: u64) -> Option { + DeviceAlignment::new(val) + } +} + impl FromVulkan for usize { #[inline] fn from_vulkan(val: usize) -> Option { diff --git a/vulkano/src/image/attachment.rs b/vulkano/src/image/attachment.rs index 1f4e3b0f..b4577b2c 100644 --- a/vulkano/src/image/attachment.rs +++ b/vulkano/src/image/attachment.rs @@ -22,7 +22,7 @@ use crate::{ AllocationCreateInfo, AllocationType, MemoryAllocatePreference, MemoryAllocator, MemoryUsage, }, - DedicatedAllocation, DeviceMemoryError, ExternalMemoryHandleType, + is_aligned, DedicatedAllocation, DeviceMemoryError, ExternalMemoryHandleType, ExternalMemoryHandleTypes, }, DeviceSize, @@ -427,7 +427,7 @@ impl AttachmentImage { match unsafe { allocator.allocate_unchecked(create_info) } { Ok(alloc) => { - debug_assert!(alloc.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned(alloc.offset(), requirements.layout.alignment())); debug_assert!(alloc.size() == requirements.layout.size()); let inner = Arc::new(unsafe { @@ -532,7 +532,7 @@ impl AttachmentImage { ) } { Ok(alloc) => { - debug_assert!(alloc.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned(alloc.offset(), requirements.layout.alignment())); debug_assert!(alloc.size() == requirements.layout.size()); let inner = Arc::new(unsafe { diff --git a/vulkano/src/image/immutable.rs b/vulkano/src/image/immutable.rs index 258a4820..b8eff730 100644 --- a/vulkano/src/image/immutable.rs +++ b/vulkano/src/image/immutable.rs @@ -27,7 +27,7 @@ use crate::{ AllocationCreateInfo, AllocationCreationError, AllocationType, MemoryAllocatePreference, MemoryAllocator, MemoryUsage, }, - DedicatedAllocation, + is_aligned, DedicatedAllocation, }, sampler::Filter, sync::Sharing, @@ -149,7 +149,7 @@ impl ImmutableImage { match unsafe { allocator.allocate_unchecked(create_info) } { Ok(alloc) => { - debug_assert!(alloc.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned(alloc.offset(), requirements.layout.alignment())); debug_assert!(alloc.size() == requirements.layout.size()); let inner = Arc::new(unsafe { diff --git a/vulkano/src/image/storage.rs b/vulkano/src/image/storage.rs index c98af74c..42ff167a 100644 --- a/vulkano/src/image/storage.rs +++ b/vulkano/src/image/storage.rs @@ -22,7 +22,7 @@ use crate::{ AllocationCreateInfo, AllocationType, MemoryAllocatePreference, MemoryAllocator, MemoryUsage, }, - DedicatedAllocation, DeviceMemoryError, ExternalMemoryHandleType, + is_aligned, DedicatedAllocation, DeviceMemoryError, ExternalMemoryHandleType, ExternalMemoryHandleTypes, }, sync::Sharing, @@ -125,7 +125,7 @@ impl StorageImage { match unsafe { allocator.allocate_unchecked(create_info) } { Ok(alloc) => { - debug_assert!(alloc.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned(alloc.offset(), requirements.layout.alignment())); debug_assert!(alloc.size() == requirements.layout.size()); let inner = Arc::new(unsafe { @@ -205,7 +205,7 @@ impl StorageImage { ) } { Ok(alloc) => { - debug_assert!(alloc.offset() % requirements.layout.alignment().as_nonzero() == 0); + debug_assert!(is_aligned(alloc.offset(), requirements.layout.alignment())); debug_assert!(alloc.size() == requirements.layout.size()); let inner = Arc::new(unsafe { diff --git a/vulkano/src/image/sys.rs b/vulkano/src/image/sys.rs index f95ed5c7..a18d0c22 100644 --- a/vulkano/src/image/sys.rs +++ b/vulkano/src/image/sys.rs @@ -28,11 +28,9 @@ use crate::{ SparseImageFormatProperties, }, memory::{ - allocator::{ - AllocationCreationError, AllocationType, DeviceAlignment, DeviceLayout, MemoryAlloc, - }, - DedicatedTo, ExternalMemoryHandleType, ExternalMemoryHandleTypes, MemoryPropertyFlags, - MemoryRequirements, + allocator::{AllocationCreationError, AllocationType, DeviceLayout, MemoryAlloc}, + is_aligned, DedicatedTo, DeviceAlignment, ExternalMemoryHandleType, + ExternalMemoryHandleTypes, MemoryPropertyFlags, MemoryRequirements, }, range_map::RangeMap, swapchain::Swapchain, @@ -1479,7 +1477,7 @@ impl RawImage { // VUID-VkBindImageMemoryInfo-pNext-01616 // VUID-VkBindImageMemoryInfo-pNext-01620 - if memory_offset % memory_requirements.layout.alignment().as_nonzero() != 0 { + if !is_aligned(memory_offset, memory_requirements.layout.alignment()) { return Err(ImageError::MemoryAllocationNotAligned { allocations_index, allocation_offset: memory_offset, diff --git a/vulkano/src/memory/alignment.rs b/vulkano/src/memory/alignment.rs new file mode 100644 index 00000000..ec5afacb --- /dev/null +++ b/vulkano/src/memory/alignment.rs @@ -0,0 +1,249 @@ +use crate::{DeviceSize, NonZeroDeviceSize}; +use std::{ + cmp::Ordering, + error::Error, + fmt::{Debug, Display, Formatter, Result as FmtResult}, + hash::{Hash, Hasher}, + mem::{self, align_of, size_of}, +}; + +/// Vulkan analog of std's [`Alignment`], stored as a [`DeviceSize`] that is guaranteed to be a +/// valid Vulkan alignment. +/// +/// [`Alignment`]: std::ptr::Alignment +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct DeviceAlignment(AlignmentEnum); + +const _: () = assert!(size_of::() == size_of::()); +const _: () = assert!(align_of::() == align_of::()); + +impl DeviceAlignment { + /// The smallest possible alignment, 1. + pub const MIN: Self = Self(AlignmentEnum::_Align1Shl0); + + /// The largest possible alignment, 263. + pub const MAX: Self = Self(AlignmentEnum::_Align1Shl63); + + /// Returns the alignment for a type. + #[inline] + pub const fn of() -> Self { + #[cfg(any( + target_pointer_width = "64", + target_pointer_width = "32", + target_pointer_width = "16", + ))] + { + const _: () = assert!(size_of::() >= size_of::()); + + // SAFETY: rustc guarantees that the alignment of types is a power of two. + unsafe { DeviceAlignment::new_unchecked(align_of::() as DeviceSize) } + } + } + + /// Tries to create a `DeviceAlignment` from a [`DeviceSize`], returning [`None`] if it's not a + /// power of two. + #[inline] + pub const fn new(alignment: DeviceSize) -> Option { + if alignment.is_power_of_two() { + Some(unsafe { DeviceAlignment::new_unchecked(alignment) }) + } else { + None + } + } + + /// Creates a `DeviceAlignment` from a [`DeviceSize`] without checking if it's a power of two. + /// + /// # Safety + /// + /// - `alignment` must be a power of two, which also means it must be non-zero. + #[inline] + pub const unsafe fn new_unchecked(alignment: DeviceSize) -> Self { + debug_assert!(alignment.is_power_of_two()); + + unsafe { mem::transmute::(alignment) } + } + + /// Returns the alignment as a [`DeviceSize`]. + #[inline] + pub const fn as_devicesize(self) -> DeviceSize { + self.0 as DeviceSize + } + + /// Returns the alignment as a [`NonZeroDeviceSize`]. + #[inline] + pub const fn as_nonzero(self) -> NonZeroDeviceSize { + // SAFETY: All the discriminants are non-zero. + unsafe { NonZeroDeviceSize::new_unchecked(self.as_devicesize()) } + } + + /// Returns the base-2 logarithm of the alignment. + #[inline] + pub const fn log2(self) -> u32 { + self.as_nonzero().trailing_zeros() + } + + // TODO: Replace with `Ord::max` once its constness is stabilized. + #[inline(always)] + pub(crate) const fn max(self, other: Self) -> Self { + if self.as_devicesize() >= other.as_devicesize() { + self + } else { + other + } + } +} + +impl Debug for DeviceAlignment { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + write!(f, "{:?} (1 << {:?})", self.as_nonzero(), self.log2()) + } +} + +impl Default for DeviceAlignment { + #[inline] + fn default() -> Self { + DeviceAlignment::MIN + } +} + +impl TryFrom for DeviceAlignment { + type Error = TryFromIntError; + + #[inline] + fn try_from(alignment: NonZeroDeviceSize) -> Result { + if alignment.is_power_of_two() { + Ok(unsafe { DeviceAlignment::new_unchecked(alignment.get()) }) + } else { + Err(TryFromIntError) + } + } +} + +impl TryFrom for DeviceAlignment { + type Error = TryFromIntError; + + #[inline] + fn try_from(alignment: DeviceSize) -> Result { + DeviceAlignment::new(alignment).ok_or(TryFromIntError) + } +} + +impl From for NonZeroDeviceSize { + #[inline] + fn from(alignment: DeviceAlignment) -> Self { + alignment.as_nonzero() + } +} + +impl From for DeviceSize { + #[inline] + fn from(alignment: DeviceAlignment) -> Self { + alignment.as_devicesize() + } +} + +// This is a false-positive, the underlying values that this impl and the derived `PartialEq` work +// with are the same. +#[allow(clippy::derive_hash_xor_eq)] +impl Hash for DeviceAlignment { + #[inline] + fn hash(&self, state: &mut H) { + self.as_nonzero().hash(state); + } +} + +impl PartialOrd for DeviceAlignment { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + self.as_nonzero().partial_cmp(&other.as_nonzero()) + } +} + +impl Ord for DeviceAlignment { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + self.as_nonzero().cmp(&other.as_nonzero()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u64)] +enum AlignmentEnum { + _Align1Shl0 = 1 << 0, + _Align1Shl1 = 1 << 1, + _Align1Shl2 = 1 << 2, + _Align1Shl3 = 1 << 3, + _Align1Shl4 = 1 << 4, + _Align1Shl5 = 1 << 5, + _Align1Shl6 = 1 << 6, + _Align1Shl7 = 1 << 7, + _Align1Shl8 = 1 << 8, + _Align1Shl9 = 1 << 9, + _Align1Shl10 = 1 << 10, + _Align1Shl11 = 1 << 11, + _Align1Shl12 = 1 << 12, + _Align1Shl13 = 1 << 13, + _Align1Shl14 = 1 << 14, + _Align1Shl15 = 1 << 15, + _Align1Shl16 = 1 << 16, + _Align1Shl17 = 1 << 17, + _Align1Shl18 = 1 << 18, + _Align1Shl19 = 1 << 19, + _Align1Shl20 = 1 << 20, + _Align1Shl21 = 1 << 21, + _Align1Shl22 = 1 << 22, + _Align1Shl23 = 1 << 23, + _Align1Shl24 = 1 << 24, + _Align1Shl25 = 1 << 25, + _Align1Shl26 = 1 << 26, + _Align1Shl27 = 1 << 27, + _Align1Shl28 = 1 << 28, + _Align1Shl29 = 1 << 29, + _Align1Shl30 = 1 << 30, + _Align1Shl31 = 1 << 31, + _Align1Shl32 = 1 << 32, + _Align1Shl33 = 1 << 33, + _Align1Shl34 = 1 << 34, + _Align1Shl35 = 1 << 35, + _Align1Shl36 = 1 << 36, + _Align1Shl37 = 1 << 37, + _Align1Shl38 = 1 << 38, + _Align1Shl39 = 1 << 39, + _Align1Shl40 = 1 << 40, + _Align1Shl41 = 1 << 41, + _Align1Shl42 = 1 << 42, + _Align1Shl43 = 1 << 43, + _Align1Shl44 = 1 << 44, + _Align1Shl45 = 1 << 45, + _Align1Shl46 = 1 << 46, + _Align1Shl47 = 1 << 47, + _Align1Shl48 = 1 << 48, + _Align1Shl49 = 1 << 49, + _Align1Shl50 = 1 << 50, + _Align1Shl51 = 1 << 51, + _Align1Shl52 = 1 << 52, + _Align1Shl53 = 1 << 53, + _Align1Shl54 = 1 << 54, + _Align1Shl55 = 1 << 55, + _Align1Shl56 = 1 << 56, + _Align1Shl57 = 1 << 57, + _Align1Shl58 = 1 << 58, + _Align1Shl59 = 1 << 59, + _Align1Shl60 = 1 << 60, + _Align1Shl61 = 1 << 61, + _Align1Shl62 = 1 << 62, + _Align1Shl63 = 1 << 63, +} + +/// Error that can happen when trying to convert an integer to a `DeviceAlignment`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TryFromIntError; + +impl Error for TryFromIntError {} + +impl Display for TryFromIntError { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.write_str("attempted to convert a non-power-of-two integer to a `DeviceAlignment`") + } +} diff --git a/vulkano/src/memory/allocator/layout.rs b/vulkano/src/memory/allocator/layout.rs index e5b7b488..bca396aa 100644 --- a/vulkano/src/memory/allocator/layout.rs +++ b/vulkano/src/memory/allocator/layout.rs @@ -8,14 +8,13 @@ // according to those terms. use super::align_up; -use crate::{macros::try_opt, DeviceSize, NonZeroDeviceSize}; +use crate::{macros::try_opt, memory::DeviceAlignment, DeviceSize, NonZeroDeviceSize}; use std::{ alloc::Layout, - cmp::Ordering, error::Error, fmt::{Debug, Display, Formatter, Result as FmtResult}, - hash::{Hash, Hasher}, - mem::{self, align_of, size_of}, + hash::Hash, + mem::size_of, }; /// Vulkan analog of std's [`Layout`], represented using [`DeviceSize`]s. @@ -351,237 +350,3 @@ impl Display for TryFromDeviceLayoutError { ) } } - -/// Vulkan analog of std's [`Alignment`], stored as a [`DeviceSize`] that is guaranteed to be a -/// valid Vulkan alignment. -/// -/// [`Alignment`]: std::ptr::Alignment -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(transparent)] -pub struct DeviceAlignment(AlignmentEnum); - -const _: () = assert!(size_of::() == size_of::()); -const _: () = assert!(align_of::() == align_of::()); - -impl DeviceAlignment { - /// The smallest possible alignment, 1. - pub const MIN: Self = Self(AlignmentEnum::_Align1Shl0); - - /// The largest possible alignment, 263. - pub const MAX: Self = Self(AlignmentEnum::_Align1Shl63); - - /// Returns the alignment for a type. - #[inline] - pub const fn of() -> Self { - #[cfg(any( - target_pointer_width = "64", - target_pointer_width = "32", - target_pointer_width = "16", - ))] - { - const _: () = assert!(size_of::() >= size_of::()); - - // SAFETY: rustc guarantees that the alignment of types is a power of two. - unsafe { DeviceAlignment::new_unchecked(align_of::() as DeviceSize) } - } - } - - /// Tries to create a `DeviceAlignment` from a [`DeviceSize`], returning [`None`] if it's not a - /// power of two. - #[inline] - pub const fn new(alignment: DeviceSize) -> Option { - if alignment.is_power_of_two() { - Some(unsafe { DeviceAlignment::new_unchecked(alignment) }) - } else { - None - } - } - - /// Creates a `DeviceAlignment` from a [`DeviceSize`] without checking if it's a power of two. - /// - /// # Safety - /// - /// - `alignment` must be a power of two, which also means it must be non-zero. - #[inline] - pub const unsafe fn new_unchecked(alignment: DeviceSize) -> Self { - debug_assert!(alignment.is_power_of_two()); - - unsafe { mem::transmute::(alignment) } - } - - /// Returns the alignment as a [`DeviceSize`]. - #[inline] - pub const fn as_devicesize(self) -> DeviceSize { - self.0 as DeviceSize - } - - /// Returns the alignment as a [`NonZeroDeviceSize`]. - #[inline] - pub const fn as_nonzero(self) -> NonZeroDeviceSize { - // SAFETY: All the discriminants are non-zero. - unsafe { NonZeroDeviceSize::new_unchecked(self.as_devicesize()) } - } - - /// Returns the base-2 logarithm of the alignment. - #[inline] - pub const fn log2(self) -> u32 { - self.as_nonzero().trailing_zeros() - } - - // TODO: Replace with `Ord::max` once its constness is stabilized. - #[inline(always)] - pub(crate) const fn max(self, other: Self) -> Self { - if self.as_devicesize() >= other.as_devicesize() { - self - } else { - other - } - } -} - -impl Debug for DeviceAlignment { - fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, "{:?} (1 << {:?})", self.as_nonzero(), self.log2()) - } -} - -impl TryFrom for DeviceAlignment { - type Error = TryFromIntError; - - #[inline] - fn try_from(alignment: NonZeroDeviceSize) -> Result { - if alignment.is_power_of_two() { - Ok(unsafe { DeviceAlignment::new_unchecked(alignment.get()) }) - } else { - Err(TryFromIntError) - } - } -} - -impl TryFrom for DeviceAlignment { - type Error = TryFromIntError; - - #[inline] - fn try_from(alignment: DeviceSize) -> Result { - DeviceAlignment::new(alignment).ok_or(TryFromIntError) - } -} - -impl From for NonZeroDeviceSize { - #[inline] - fn from(alignment: DeviceAlignment) -> Self { - alignment.as_nonzero() - } -} - -impl From for DeviceSize { - #[inline] - fn from(alignment: DeviceAlignment) -> Self { - alignment.as_devicesize() - } -} - -// This is a false-positive, the underlying values that this impl and the derived `PartialEq` work -// with are the same. -#[allow(clippy::derive_hash_xor_eq)] -impl Hash for DeviceAlignment { - #[inline] - fn hash(&self, state: &mut H) { - self.as_nonzero().hash(state); - } -} - -impl PartialOrd for DeviceAlignment { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.as_nonzero().partial_cmp(&other.as_nonzero()) - } -} - -impl Ord for DeviceAlignment { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - self.as_nonzero().cmp(&other.as_nonzero()) - } -} - -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(u64)] -enum AlignmentEnum { - _Align1Shl0 = 1 << 0, - _Align1Shl1 = 1 << 1, - _Align1Shl2 = 1 << 2, - _Align1Shl3 = 1 << 3, - _Align1Shl4 = 1 << 4, - _Align1Shl5 = 1 << 5, - _Align1Shl6 = 1 << 6, - _Align1Shl7 = 1 << 7, - _Align1Shl8 = 1 << 8, - _Align1Shl9 = 1 << 9, - _Align1Shl10 = 1 << 10, - _Align1Shl11 = 1 << 11, - _Align1Shl12 = 1 << 12, - _Align1Shl13 = 1 << 13, - _Align1Shl14 = 1 << 14, - _Align1Shl15 = 1 << 15, - _Align1Shl16 = 1 << 16, - _Align1Shl17 = 1 << 17, - _Align1Shl18 = 1 << 18, - _Align1Shl19 = 1 << 19, - _Align1Shl20 = 1 << 20, - _Align1Shl21 = 1 << 21, - _Align1Shl22 = 1 << 22, - _Align1Shl23 = 1 << 23, - _Align1Shl24 = 1 << 24, - _Align1Shl25 = 1 << 25, - _Align1Shl26 = 1 << 26, - _Align1Shl27 = 1 << 27, - _Align1Shl28 = 1 << 28, - _Align1Shl29 = 1 << 29, - _Align1Shl30 = 1 << 30, - _Align1Shl31 = 1 << 31, - _Align1Shl32 = 1 << 32, - _Align1Shl33 = 1 << 33, - _Align1Shl34 = 1 << 34, - _Align1Shl35 = 1 << 35, - _Align1Shl36 = 1 << 36, - _Align1Shl37 = 1 << 37, - _Align1Shl38 = 1 << 38, - _Align1Shl39 = 1 << 39, - _Align1Shl40 = 1 << 40, - _Align1Shl41 = 1 << 41, - _Align1Shl42 = 1 << 42, - _Align1Shl43 = 1 << 43, - _Align1Shl44 = 1 << 44, - _Align1Shl45 = 1 << 45, - _Align1Shl46 = 1 << 46, - _Align1Shl47 = 1 << 47, - _Align1Shl48 = 1 << 48, - _Align1Shl49 = 1 << 49, - _Align1Shl50 = 1 << 50, - _Align1Shl51 = 1 << 51, - _Align1Shl52 = 1 << 52, - _Align1Shl53 = 1 << 53, - _Align1Shl54 = 1 << 54, - _Align1Shl55 = 1 << 55, - _Align1Shl56 = 1 << 56, - _Align1Shl57 = 1 << 57, - _Align1Shl58 = 1 << 58, - _Align1Shl59 = 1 << 59, - _Align1Shl60 = 1 << 60, - _Align1Shl61 = 1 << 61, - _Align1Shl62 = 1 << 62, - _Align1Shl63 = 1 << 63, -} - -/// Error that can happen when trying to convert an integer to a `DeviceAlignment`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TryFromIntError; - -impl Error for TryFromIntError {} - -impl Display for TryFromIntError { - fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - f.write_str("attempted to convert a non-power-of-two integer to a `DeviceAlignment`") - } -} diff --git a/vulkano/src/memory/allocator/mod.rs b/vulkano/src/memory/allocator/mod.rs index 746a2e60..554f6f51 100644 --- a/vulkano/src/memory/allocator/mod.rs +++ b/vulkano/src/memory/allocator/mod.rs @@ -221,15 +221,16 @@ pub mod suballocator; use self::array_vec::ArrayVec; pub use self::{ - layout::{DeviceAlignment, DeviceLayout}, + layout::DeviceLayout, suballocator::{ AllocationType, BuddyAllocator, BumpAllocator, FreeListAllocator, MemoryAlloc, PoolAllocator, SuballocationCreateInfo, SuballocationCreationError, Suballocator, }, }; use super::{ - DedicatedAllocation, DeviceMemory, ExternalMemoryHandleTypes, MemoryAllocateFlags, - MemoryAllocateInfo, MemoryProperties, MemoryPropertyFlags, MemoryRequirements, MemoryType, + DedicatedAllocation, DeviceAlignment, DeviceMemory, ExternalMemoryHandleTypes, + MemoryAllocateFlags, MemoryAllocateInfo, MemoryProperties, MemoryPropertyFlags, + MemoryRequirements, MemoryType, }; use crate::{ device::{Device, DeviceOwned}, diff --git a/vulkano/src/memory/allocator/suballocator.rs b/vulkano/src/memory/allocator/suballocator.rs index 16e9f41a..4a1269af 100644 --- a/vulkano/src/memory/allocator/suballocator.rs +++ b/vulkano/src/memory/allocator/suballocator.rs @@ -21,7 +21,7 @@ use super::{ use crate::{ device::{Device, DeviceOwned}, image::ImageTiling, - memory::{DeviceMemory, MemoryPropertyFlags}, + memory::{is_aligned, DeviceMemory, MemoryPropertyFlags}, DeviceSize, NonZeroDeviceSize, VulkanError, VulkanObject, }; use crossbeam_queue::ArrayQueue; @@ -135,9 +135,7 @@ impl MemoryAlloc { let atom_size = (property_flags.intersects(MemoryPropertyFlags::HOST_VISIBLE) && !property_flags.intersects(MemoryPropertyFlags::HOST_COHERENT)) - .then_some( - DeviceAlignment::new(physical_device.properties().non_coherent_atom_size).unwrap(), - ); + .then_some(physical_device.properties().non_coherent_atom_size); Ok(MemoryAlloc { offset: 0, @@ -301,8 +299,8 @@ impl MemoryAlloc { // VUID-VkMappedMemoryRange-offset-00687 // VUID-VkMappedMemoryRange-size-01390 assert!( - range.start % atom_size.as_nonzero() == 0 - && (range.end % atom_size.as_nonzero() == 0 || range.end == self.size) + is_aligned(range.start, atom_size) + && (is_aligned(range.end, atom_size) || range.end == self.size) ); // VUID-VkMappedMemoryRange-offset-00687 @@ -334,17 +332,15 @@ impl MemoryAlloc { /// to be host-coherent. fn debug_validate_memory_range(&self, range: &Range) { debug_assert!(!range.is_empty() && range.end <= self.size); - debug_assert!( - { - let atom_size = self - .device() - .physical_device() - .properties() - .non_coherent_atom_size; - range.start % atom_size == 0 - && (range.end % atom_size == 0 || range.end == self.size) - }, + let atom_size = self + .device() + .physical_device() + .properties() + .non_coherent_atom_size; + debug_assert!( + is_aligned(range.start, atom_size) + && (is_aligned(range.end, atom_size) || range.end == self.size), "attempted to invalidate or flush a memory range that is not aligned to the \ non-coherent atom size", ); @@ -1002,14 +998,12 @@ impl FreeListAllocator { .root() .expect("dedicated allocations can't be suballocated") .clone(); - let buffer_image_granularity = DeviceAlignment::new( - device_memory - .device() - .physical_device() - .properties() - .buffer_image_granularity, - ) - .unwrap(); + let buffer_image_granularity = device_memory + .device() + .physical_device() + .properties() + .buffer_image_granularity; + let atom_size = region.atom_size.unwrap_or(DeviceAlignment::MIN); let free_size = AtomicU64::new(region.size); @@ -1600,14 +1594,11 @@ impl BuddyAllocator { .root() .expect("dedicated allocations can't be suballocated") .clone(); - let buffer_image_granularity = DeviceAlignment::new( - device_memory - .device() - .physical_device() - .properties() - .buffer_image_granularity, - ) - .unwrap(); + let buffer_image_granularity = device_memory + .device() + .physical_device() + .properties() + .buffer_image_granularity; let atom_size = region.atom_size.unwrap_or(DeviceAlignment::MIN); let free_size = AtomicU64::new(region.size); @@ -1742,7 +1733,7 @@ unsafe impl Suballocator for Arc { // Start searching at the lowest possible order going up. for (order, free_list) in state.free_list.iter_mut().enumerate().skip(min_order) { for (index, &offset) in free_list.iter().enumerate() { - if offset % alignment.as_nonzero() == 0 { + if is_aligned(offset, alignment) { free_list.remove(index); // Go in the opposite direction, splitting nodes from higher orders. The lowest @@ -2117,14 +2108,11 @@ impl PoolAllocatorInner { .expect("dedicated allocations can't be suballocated") .clone(); #[cfg(not(test))] - let buffer_image_granularity = DeviceAlignment::new( - device_memory - .device() - .physical_device() - .properties() - .buffer_image_granularity, - ) - .unwrap(); + let buffer_image_granularity = device_memory + .device() + .physical_device() + .properties() + .buffer_image_granularity; let atom_size = region.atom_size.unwrap_or(DeviceAlignment::MIN); if region.allocation_type == AllocationType::Unknown { block_size = align_up(block_size, buffer_image_granularity); @@ -2294,14 +2282,11 @@ impl BumpAllocator { .root() .expect("dedicated allocations can't be suballocated") .clone(); - let buffer_image_granularity = DeviceAlignment::new( - device_memory - .device() - .physical_device() - .properties() - .buffer_image_granularity, - ) - .unwrap(); + let buffer_image_granularity = device_memory + .device() + .physical_device() + .properties() + .buffer_image_granularity; let atom_size = region.atom_size.unwrap_or(DeviceAlignment::MIN); let state = AtomicU64::new(region.allocation_type as DeviceSize); diff --git a/vulkano/src/memory/device_memory.rs b/vulkano/src/memory/device_memory.rs index b921d30f..1fdf802e 100644 --- a/vulkano/src/memory/device_memory.rs +++ b/vulkano/src/memory/device_memory.rs @@ -7,11 +7,11 @@ // notice may not be copied, modified, or distributed except // according to those terms. -use super::{DedicatedAllocation, DedicatedTo}; +use super::{DedicatedAllocation, DedicatedTo, DeviceAlignment}; use crate::{ device::{Device, DeviceOwned}, macros::{vulkan_bitflags, vulkan_bitflags_enum}, - memory::MemoryPropertyFlags, + memory::{is_aligned, MemoryPropertyFlags}, DeviceSize, OomError, RequirementNotMet, RequiresOneOf, Version, VulkanError, VulkanObject, }; use std::{ @@ -1188,7 +1188,7 @@ pub struct MappedDeviceMemory { pointer: *mut c_void, // points to `range.start` range: Range, - atom_size: DeviceSize, + atom_size: DeviceAlignment, coherent: bool, } @@ -1249,8 +1249,8 @@ impl MappedDeviceMemory { // parts of the mapped memory at the start and end that they're not able to // invalidate/flush, which is probably unintended. if !coherent - && (range.start % atom_size != 0 - || (range.end % atom_size != 0 && range.end != memory.allocation_size)) + && (!is_aligned(range.start, atom_size) + || (!is_aligned(range.end, atom_size) && range.end != memory.allocation_size)) { return Err(MemoryMapError::RangeNotAlignedToAtomSize { range, atom_size }); } @@ -1275,7 +1275,6 @@ impl MappedDeviceMemory { memory, pointer, range, - atom_size, coherent, }) @@ -1460,8 +1459,9 @@ impl MappedDeviceMemory { if !self.coherent { // VUID-VkMappedMemoryRange-offset-00687 // VUID-VkMappedMemoryRange-size-01390 - if range.start % self.atom_size != 0 - || (range.end % self.atom_size != 0 && range.end != self.memory.allocation_size) + if !is_aligned(range.start, self.atom_size) + || (!is_aligned(range.end, self.atom_size) + && range.end != self.memory.allocation_size) { return Err(MemoryMapError::RangeNotAlignedToAtomSize { range, @@ -1521,7 +1521,7 @@ pub enum MemoryMapError { /// property. RangeNotAlignedToAtomSize { range: Range, - atom_size: DeviceSize, + atom_size: DeviceAlignment, }, } @@ -1554,7 +1554,7 @@ impl Display for MemoryMapError { Self::RangeNotAlignedToAtomSize { range, atom_size } => write!( f, "the memory is not host-coherent, and the specified `range` bounds ({:?}) are not \ - a multiple of the `non_coherent_atom_size` device property ({})", + a multiple of the `non_coherent_atom_size` device property ({:?})", range, atom_size, ), } diff --git a/vulkano/src/memory/mod.rs b/vulkano/src/memory/mod.rs index a5e802cd..94b6a1ae 100644 --- a/vulkano/src/memory/mod.rs +++ b/vulkano/src/memory/mod.rs @@ -91,7 +91,8 @@ //! get memory from that pool. By default if you don't specify any pool when creating a buffer or //! an image, an instance of `StandardMemoryPool` that is shared by the `Device` object is used. -use self::allocator::{DeviceAlignment, DeviceLayout}; +pub use self::alignment::DeviceAlignment; +use self::allocator::DeviceLayout; pub use self::device_memory::{ DeviceMemory, DeviceMemoryError, ExternalMemoryHandleType, ExternalMemoryHandleTypes, MappedDeviceMemory, MemoryAllocateFlags, MemoryAllocateInfo, MemoryImportInfo, MemoryMapError, @@ -109,6 +110,7 @@ use std::{ sync::Arc, }; +mod alignment; pub mod allocator; mod device_memory;