From 5782c1a2a64cf679b5bad2157c41157d657deefd Mon Sep 17 00:00:00 2001 From: marc0246 <40955683+marc0246@users.noreply.github.com> Date: Sun, 28 Jul 2024 11:35:58 +0200 Subject: [PATCH] Task graph [3/10]: execution (#2548) --- Cargo.lock | 2 +- Cargo.toml | 2 +- vulkano-taskgraph/src/graph/execute.rs | 1787 ++++++++++++++++++++++++ vulkano-taskgraph/src/graph/mod.rs | 22 +- vulkano-taskgraph/src/lib.rs | 46 +- vulkano-taskgraph/src/resource.rs | 63 +- 6 files changed, 1890 insertions(+), 32 deletions(-) create mode 100644 vulkano-taskgraph/src/graph/execute.rs diff --git a/Cargo.lock b/Cargo.lock index a1aa6c3b..60d44b81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "concurrent-slotmap" version = "0.1.0" -source = "git+https://github.com/vulkano-rs/concurrent-slotmap?rev=a65c7642f8a647739973157d0c04d07e4474ebec#a65c7642f8a647739973157d0c04d07e4474ebec" +source = "git+https://github.com/vulkano-rs/concurrent-slotmap?rev=bf52f0a55713bb29dde3e38bc3497b03473d1628#bf52f0a55713bb29dde3e38bc3497b03473d1628" dependencies = [ "virtual-buffer", ] diff --git a/Cargo.toml b/Cargo.toml index 8a296f94..1667bacc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ ahash = "0.8" # https://github.com/KhronosGroup/Vulkan-Headers/commits/main/registry/vk.xml ash = "0.38.0" bytemuck = "1.9" -concurrent-slotmap = { git = "https://github.com/vulkano-rs/concurrent-slotmap", rev = "a65c7642f8a647739973157d0c04d07e4474ebec" } +concurrent-slotmap = { git = "https://github.com/vulkano-rs/concurrent-slotmap", rev = "bf52f0a55713bb29dde3e38bc3497b03473d1628" } core-graphics-types = "0.1" crossbeam-queue = "0.3" half = "2.0" diff --git a/vulkano-taskgraph/src/graph/execute.rs b/vulkano-taskgraph/src/graph/execute.rs new file mode 100644 index 00000000..88565bd7 --- /dev/null +++ b/vulkano-taskgraph/src/graph/execute.rs @@ -0,0 +1,1787 @@ +use super::{ + BarrierIndex, ExecutableTaskGraph, ImageReference, Instruction, InstructionIndex, NodeIndex, + ResourceAccess, SemaphoreIndex, EXCLUSIVE_BIT, +}; +use crate::{ + resource::{ + BufferAccess, BufferState, DeathRow, Flight, FlightState, ImageAccess, ImageState, + Resources, SwapchainState, + }, + Id, InvalidSlotError, TaskContext, TaskError, +}; +use ash::vk; +use concurrent_slotmap::epoch; +use smallvec::SmallVec; +use std::{ + cell::Cell, + error::Error, + fmt, mem, + ops::Range, + ptr, + sync::{atomic::Ordering, Arc}, +}; +use vulkano::{ + buffer::Buffer, + command_buffer::{ + sys::{RawCommandBuffer, RawRecordingCommandBuffer}, + CommandBufferBeginInfo, CommandBufferLevel, CommandBufferUsage, + }, + device::{Device, DeviceOwned, Queue}, + image::Image, + swapchain::Swapchain, + sync::{fence::Fence, semaphore::Semaphore, AccessFlags, PipelineStages}, + Validated, Version, VulkanError, VulkanObject, +}; + +impl ExecutableTaskGraph { + /// Executes the next frame of the [flight] given by `flight_id`. + /// + /// # Safety + /// + /// - There must be no other task graphs executing that access any of the same subresources as + /// `self`. + /// - A subresource in flight must not be accessed in more than one frame in flight. + /// + /// # Panics + /// + /// - Panics if `resource_map` doesn't map the virtual resources of `self` exhaustively. + /// - Panics if `flight_id` is invalid. + /// - Panics if another thread is already executing a task graph using the flight. + /// - Panics if the [current fence] of the flight wasn't waited on. + /// - Panics if `resource_map` maps to any swapchain that isn't owned by the flight. + /// + /// [current fence]: Flight::current_fence + pub unsafe fn execute( + &self, + resource_map: ResourceMap<'_>, + flight_id: Id, + world: &W, + ) -> Result { + assert!(ptr::eq( + resource_map.virtual_resources, + &self.graph.resources, + )); + assert!(resource_map.is_exhaustive()); + + // SAFETY: `resource_map` owns an `epoch::Guard`. + let flight = unsafe { resource_map.resources.flight_unprotected(flight_id) } + .expect("invalid flight"); + + let mut flight_state = flight.state.try_lock().unwrap_or_else(|| { + panic!( + "another thread is already executing a task graph using the flight {flight_id:?}", + ); + }); + + let current_fence = flight.current_fence(); + + assert!( + current_fence.is_signaled()?, + "you must wait on the fence for the current frame before submitting more work", + ); + + for &swapchain_id in &self.swapchains { + // SAFETY: We checked that `resource_map` maps the virtual IDs exhaustively. + let swapchain_state = unsafe { resource_map.swapchain_unchecked(swapchain_id) }; + + assert_eq!( + swapchain_state.flight_id(), + flight_id, + "`resource_map` must not map to any swapchain not owned by the flight \ + corresponding to `flight_id`", + ); + } + + let current_frame = flight.current_frame(); + + for object in flight_state.death_rows[current_frame as usize].drain(..) { + // FIXME: + drop(object); + } + + // SAFETY: We checked that `resource_map` maps the virtual IDs exhaustively. + unsafe { self.acquire_images_khr(&resource_map, current_frame) }?; + + // SAFETY: We checked that the fence has been signalled. + unsafe { current_fence.reset_unchecked() }?; + + let mut state_guard = StateGuard { + executable: self, + resource_map: &resource_map, + submission_count: 0, + }; + + let execute_instructions = if resource_map.device().enabled_features().synchronization2 { + Self::execute_instructions2 + } else { + Self::execute_instructions + }; + + // SAFETY: We checked that `resource_map` maps the virtual IDs exhaustively. + unsafe { + execute_instructions( + self, + &resource_map, + &mut flight_state, + current_frame, + current_fence, + &mut state_guard.submission_count, + world, + ) + }?; + + mem::forget(state_guard); + + unsafe { flight.next_frame() }; + + // SAFETY: We checked that `resource_map` maps the virtual IDs exhaustively. + let res = unsafe { self.present_images_khr(&resource_map, current_frame) }; + + // SAFETY: We checked that `resource_map` maps the virtual IDs exhaustively. + unsafe { self.update_resource_state(&resource_map, 0..self.instructions.len()) }; + + resource_map + .resources + .try_advance_global_and_collect(&resource_map.guard); + + res + } + + unsafe fn acquire_images_khr( + &self, + resource_map: &ResourceMap<'_>, + current_frame: u32, + ) -> Result { + let fns = resource_map.device().fns(); + let acquire_next_image_khr = fns.khr_swapchain.acquire_next_image_khr; + + for &swapchain_id in &self.swapchains { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs exhaustively. + let swapchain_state = unsafe { resource_map.swapchain_unchecked(swapchain_id) }; + let semaphore = + &swapchain_state.semaphores[current_frame as usize].image_available_semaphore; + + // Make sure to not acquire another image index if we already acquired one. This can + // happen when using multiple swapchains, if one acquire succeeds and another fails, or + // when executing a submission or presenting an image fails. + if swapchain_state.current_image_index.load(Ordering::Relaxed) != u32::MAX { + continue; + } + + let mut current_image_index = u32::MAX; + let result = unsafe { + acquire_next_image_khr( + resource_map.device().handle(), + swapchain_state.swapchain().handle(), + u64::MAX, + semaphore.handle(), + vk::Fence::null(), + &mut current_image_index, + ) + }; + + // If an error occurred, this will set the index to `u32::MAX`. + swapchain_state + .current_image_index + .store(current_image_index, Ordering::Relaxed); + + // These are the only possible success codes because we set the timeout to `u64::MAX`. + if !matches!(result, vk::Result::SUCCESS | vk::Result::SUBOPTIMAL_KHR) { + return Err(ExecuteError::Swapchain { + swapchain_id, + error: result.into(), + }); + } + } + + Ok(()) + } + + unsafe fn execute_instructions2( + &self, + resource_map: &ResourceMap<'_>, + flight_state: &mut FlightState, + current_frame: u32, + current_fence: &Fence, + submission_count: &mut usize, + world: &W, + ) -> Result { + let death_row = &mut flight_state.death_rows[current_frame as usize]; + let mut state = ExecuteState2::new( + self, + resource_map, + death_row, + current_frame, + current_fence, + submission_count, + world, + )?; + let mut execute_initial_barriers = true; + + for instruction in self.instructions.iter().cloned() { + if execute_initial_barriers { + let submission = &state.executable.submissions[*state.submission_count]; + state.initial_pipeline_barrier( + submission.initial_buffer_barrier_range.clone(), + submission.initial_image_barrier_range.clone(), + ); + execute_initial_barriers = false; + } + + match instruction { + Instruction::WaitAcquire { + swapchain_id, + stage_mask, + } => { + state.wait_acquire(swapchain_id, stage_mask); + } + Instruction::WaitSemaphore { + semaphore_index, + stage_mask, + } => { + state.wait_semaphore(semaphore_index, stage_mask); + } + Instruction::ExecuteTask { node_index } => { + state.execute_task(node_index)?; + } + Instruction::PipelineBarrier { + buffer_barrier_range, + image_barrier_range, + } => { + state.pipeline_barrier(buffer_barrier_range, image_barrier_range); + } + Instruction::SignalSemaphore { + semaphore_index, + stage_mask, + } => { + state.signal_semaphore(semaphore_index, stage_mask); + } + Instruction::SignalPresent { + swapchain_id, + stage_mask, + } => { + state.signal_present(swapchain_id, stage_mask); + } + Instruction::FlushSubmit => { + state.flush_submit()?; + } + Instruction::Submit => { + state.submit()?; + execute_initial_barriers = true; + } + } + } + + Ok(()) + } + + unsafe fn execute_instructions( + &self, + resource_map: &ResourceMap<'_>, + flight_state: &mut FlightState, + current_frame: u32, + current_fence: &Fence, + submission_count: &mut usize, + world: &W, + ) -> Result { + let death_row = &mut flight_state.death_rows[current_frame as usize]; + let mut state = ExecuteState::new( + self, + resource_map, + death_row, + current_frame, + current_fence, + submission_count, + world, + )?; + let mut execute_initial_barriers = true; + + for instruction in self.instructions.iter().cloned() { + if execute_initial_barriers { + let submission = &state.executable.submissions[*state.submission_count]; + state.initial_pipeline_barrier( + submission.initial_buffer_barrier_range.clone(), + submission.initial_image_barrier_range.clone(), + ); + execute_initial_barriers = false; + } + + match instruction { + Instruction::WaitAcquire { + swapchain_id, + stage_mask, + } => { + state.wait_acquire(swapchain_id, stage_mask); + } + Instruction::WaitSemaphore { + semaphore_index, + stage_mask, + } => { + state.wait_semaphore(semaphore_index, stage_mask); + } + Instruction::ExecuteTask { node_index } => { + state.execute_task(node_index)?; + } + Instruction::PipelineBarrier { + buffer_barrier_range, + image_barrier_range, + } => { + state.pipeline_barrier(buffer_barrier_range, image_barrier_range); + } + Instruction::SignalSemaphore { + semaphore_index, + stage_mask, + } => { + state.signal_semaphore(semaphore_index, stage_mask); + } + Instruction::SignalPresent { + swapchain_id, + stage_mask, + } => { + state.signal_present(swapchain_id, stage_mask); + } + Instruction::FlushSubmit => { + state.flush_submit()?; + } + Instruction::Submit => { + state.submit()?; + execute_initial_barriers = true; + } + } + } + + Ok(()) + } + + unsafe fn present_images_khr( + &self, + resource_map: &ResourceMap<'_>, + current_frame: u32, + ) -> Result { + let Some(present_queue) = &self.present_queue else { + return Ok(()); + }; + + let swapchain_count = self.swapchains.len(); + let mut semaphores = SmallVec::<[_; 1]>::with_capacity(swapchain_count); + let mut swapchains = SmallVec::<[_; 1]>::with_capacity(swapchain_count); + let mut image_indices = SmallVec::<[_; 1]>::with_capacity(swapchain_count); + let mut results = SmallVec::<[_; 1]>::with_capacity(swapchain_count); + + for &swapchain_id in &self.swapchains { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs exhaustively. + let swapchain_state = unsafe { resource_map.swapchain_unchecked(swapchain_id) }; + semaphores.push( + swapchain_state.semaphores[current_frame as usize] + .tasks_complete_semaphore + .handle(), + ); + swapchains.push(swapchain_state.swapchain().handle()); + image_indices.push(swapchain_state.current_image_index().unwrap()); + results.push(vk::Result::SUCCESS); + } + + let present_info = vk::PresentInfoKHR::default() + .wait_semaphores(&semaphores) + .swapchains(&swapchains) + .image_indices(&image_indices) + .results(&mut results); + + let fns = resource_map.device().fns(); + let queue_present_khr = fns.khr_swapchain.queue_present_khr; + let _ = unsafe { queue_present_khr(present_queue.handle(), &present_info) }; + + let mut res = Ok(()); + + for (&result, &swapchain_id) in results.iter().zip(&self.swapchains) { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs exhaustively. + let swapchain_state = unsafe { resource_map.swapchain_unchecked(swapchain_id) }; + + unsafe { + swapchain_state.set_access( + 0..swapchain_state.swapchain().image_array_layers(), + // TODO: Could there be a use case for keeping the old image contents? + ImageAccess::NONE, + ) + }; + + // In case of these error codes, the semaphore wait operation is not executed. + if !matches!( + result, + vk::Result::ERROR_OUT_OF_HOST_MEMORY + | vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + | vk::Result::ERROR_DEVICE_LOST + ) { + swapchain_state + .current_image_index + .store(u32::MAX, Ordering::Relaxed); + } + + if !matches!(result, vk::Result::SUCCESS | vk::Result::SUBOPTIMAL_KHR) { + // Return the first error for consistency with the acquisition logic. + if res.is_ok() { + res = Err(ExecuteError::Swapchain { + swapchain_id, + error: result.into(), + }); + } + } + } + + res + } + + unsafe fn update_resource_state( + &self, + resource_map: &ResourceMap<'_>, + instruction_range: Range, + ) { + // TODO: This isn't particularly efficient. + for instruction in &self.instructions[instruction_range] { + let Instruction::ExecuteTask { node_index } = instruction else { + continue; + }; + let task_node = unsafe { self.graph.nodes.task_node_unchecked(*node_index) }; + let queue_family_index = task_node.queue_family_index; + + for resource_access in task_node.accesses.iter().cloned() { + match resource_access { + ResourceAccess::Buffer(a) => { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs + // exhaustively. + let state = unsafe { resource_map.buffer_unchecked(a.id) }; + let access = BufferAccess::new(a.access_type, queue_family_index); + unsafe { state.set_access(a.range, access) }; + } + ResourceAccess::Image(a) => { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs + // exhaustively. + let state = unsafe { resource_map.image_unchecked(a.id) }; + let access = + ImageAccess::new(a.access_type, a.layout_type, queue_family_index); + unsafe { state.set_access(a.subresource_range, access) }; + } + ResourceAccess::Swapchain(a) => { + // SAFETY: The caller must ensure that `resource_map` maps the virtual IDs + // exhaustively. + let state = unsafe { resource_map.swapchain_unchecked(a.id) }; + let access = + ImageAccess::new(a.access_type, a.layout_type, queue_family_index); + unsafe { state.set_access(a.array_layers, access) }; + } + } + } + } + } +} + +struct ExecuteState2<'a, W: ?Sized + 'static> { + executable: &'a ExecutableTaskGraph, + resource_map: &'a ResourceMap<'a>, + death_row: &'a mut DeathRow, + current_frame: u32, + current_fence: &'a Fence, + submission_count: &'a mut usize, + world: &'a W, + cmd_pipeline_barrier2: vk::PFN_vkCmdPipelineBarrier2, + queue_submit2: vk::PFN_vkQueueSubmit2, + per_submits: SmallVec<[PerSubmitInfo2; 4]>, + current_per_submit: PerSubmitInfo2, + current_command_buffer: Option, + command_buffers: Vec>, + current_buffer_barriers: Vec>, + current_image_barriers: Vec>, +} + +#[derive(Default)] +struct PerSubmitInfo2 { + wait_semaphore_infos: SmallVec<[vk::SemaphoreSubmitInfo<'static>; 4]>, + command_buffer_infos: SmallVec<[vk::CommandBufferSubmitInfo<'static>; 1]>, + signal_semaphore_infos: SmallVec<[vk::SemaphoreSubmitInfo<'static>; 4]>, +} + +impl<'a, W: ?Sized + 'static> ExecuteState2<'a, W> { + fn new( + executable: &'a ExecutableTaskGraph, + resource_map: &'a ResourceMap<'a>, + death_row: &'a mut DeathRow, + current_frame: u32, + current_fence: &'a Fence, + submission_count: &'a mut usize, + world: &'a W, + ) -> Result { + let fns = resource_map.device().fns(); + let (cmd_pipeline_barrier2, queue_submit2); + + if resource_map.device().api_version() >= Version::V1_3 { + cmd_pipeline_barrier2 = fns.v1_3.cmd_pipeline_barrier2; + queue_submit2 = fns.v1_3.queue_submit2; + } else { + cmd_pipeline_barrier2 = fns.khr_synchronization2.cmd_pipeline_barrier2_khr; + queue_submit2 = fns.khr_synchronization2.queue_submit2_khr; + } + + let current_command_buffer = + create_command_buffer(resource_map, &executable.submissions[0].queue)?; + + Ok(ExecuteState2 { + executable, + resource_map, + death_row, + current_frame, + current_fence, + submission_count, + world, + cmd_pipeline_barrier2, + queue_submit2, + per_submits: SmallVec::new(), + current_per_submit: PerSubmitInfo2::default(), + current_command_buffer: Some(current_command_buffer), + command_buffers: Vec::new(), + current_buffer_barriers: Vec::new(), + current_image_barriers: Vec::new(), + }) + } + + fn initial_pipeline_barrier( + &mut self, + buffer_barrier_range: Range, + image_barrier_range: Range, + ) { + self.convert_initial_buffer_barriers(buffer_barrier_range); + self.convert_initial_image_barriers(image_barrier_range); + + unsafe { + (self.cmd_pipeline_barrier2)( + self.current_command_buffer.as_mut().unwrap().handle(), + &vk::DependencyInfo::default() + .buffer_memory_barriers(&self.current_buffer_barriers) + .image_memory_barriers(&self.current_image_barriers), + ) + }; + + self.current_buffer_barriers.clear(); + self.current_image_barriers.clear(); + } + + fn convert_initial_buffer_barriers(&mut self, barrier_range: Range) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.buffer_barriers[barrier_range] { + let state = unsafe { self.resource_map.buffer_unchecked(barrier.buffer) }; + + for (range, access) in state.accesses(barrier.range.clone()) { + self.current_buffer_barriers.push( + vk::BufferMemoryBarrier2::default() + .src_stage_mask(access.stage_mask().into()) + .src_access_mask(access.access_mask().into()) + .dst_stage_mask(barrier.dst_stage_mask.into()) + .dst_access_mask(barrier.dst_access_mask.into()) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .buffer(state.buffer().handle()) + .offset(range.start) + .size(range.end - range.start), + ); + } + } + } + + fn convert_initial_image_barriers(&mut self, barrier_range: Range) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.image_barriers[barrier_range] { + match barrier.image { + ImageReference::Normal(image) => { + let state = unsafe { self.resource_map.image_unchecked(image) }; + + for (subresource_range, access) in + state.accesses(barrier.subresource_range.clone()) + { + self.current_image_barriers.push( + vk::ImageMemoryBarrier2::default() + .src_stage_mask(access.stage_mask().into()) + .src_access_mask(access.access_mask().into()) + .dst_stage_mask(barrier.dst_stage_mask.into()) + .dst_access_mask(barrier.dst_access_mask.into()) + .old_layout(access.image_layout().into()) + .new_layout(barrier.new_layout.into()) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(state.image().handle()) + .subresource_range(subresource_range.into()), + ); + } + } + ImageReference::Swapchain(swapchain) => { + let state = unsafe { self.resource_map.swapchain_unchecked(swapchain) }; + + for (subresource_range, access) in + state.accesses(barrier.subresource_range.array_layers.clone()) + { + self.current_image_barriers.push( + vk::ImageMemoryBarrier2::default() + .src_stage_mask(access.stage_mask().into()) + .src_access_mask(access.access_mask().into()) + .dst_stage_mask(barrier.dst_stage_mask.into()) + .dst_access_mask(barrier.dst_access_mask.into()) + .old_layout(access.image_layout().into()) + .new_layout(barrier.new_layout.into()) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(state.current_image().handle()) + .subresource_range(subresource_range.into()), + ); + } + } + } + } + } + + fn wait_acquire(&mut self, swapchain_id: Id, stage_mask: PipelineStages) { + let swapchain_state = unsafe { self.resource_map.swapchain_unchecked(swapchain_id) }; + let semaphore = + &swapchain_state.semaphores[self.current_frame as usize].image_available_semaphore; + + self.current_per_submit.wait_semaphore_infos.push( + vk::SemaphoreSubmitInfo::default() + .semaphore(semaphore.handle()) + .stage_mask(stage_mask.into()), + ); + } + + fn wait_semaphore(&mut self, semaphore_index: SemaphoreIndex, stage_mask: PipelineStages) { + self.current_per_submit.wait_semaphore_infos.push( + vk::SemaphoreSubmitInfo::default() + .semaphore(self.executable.semaphores.borrow()[semaphore_index].handle()) + .stage_mask(stage_mask.into()), + ); + } + + fn execute_task(&mut self, node_index: NodeIndex) -> Result { + let task_node = unsafe { self.executable.graph.nodes.task_node_unchecked(node_index) }; + let mut context = TaskContext { + resource_map: self.resource_map, + death_row: Cell::new(Some(self.death_row)), + current_command_buffer: Cell::new(Some(self.current_command_buffer.as_mut().unwrap())), + command_buffers: Cell::new(Some(&mut self.command_buffers)), + accesses: &task_node.accesses, + }; + + unsafe { task_node.task.execute(&mut context, self.world) } + .map_err(|error| ExecuteError::Task { node_index, error })?; + + if !self.command_buffers.is_empty() { + unsafe { self.flush_current_command_buffer() }?; + + for command_buffer in self.command_buffers.drain(..) { + self.current_per_submit.command_buffer_infos.push( + vk::CommandBufferSubmitInfo::default().command_buffer(command_buffer.handle()), + ); + self.death_row.push(command_buffer); + } + } + + Ok(()) + } + + fn pipeline_barrier( + &mut self, + buffer_barrier_range: Range, + image_barrier_range: Range, + ) { + self.convert_buffer_barriers(buffer_barrier_range); + self.convert_image_barriers(image_barrier_range); + + unsafe { + (self.cmd_pipeline_barrier2)( + self.current_command_buffer.as_mut().unwrap().handle(), + &vk::DependencyInfo::default() + .buffer_memory_barriers(&self.current_buffer_barriers) + .image_memory_barriers(&self.current_image_barriers), + ) + }; + + self.current_buffer_barriers.clear(); + self.current_image_barriers.clear(); + } + + fn convert_buffer_barriers(&mut self, barrier_range: Range) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.buffer_barriers[barrier_range] { + let state = unsafe { self.resource_map.buffer_unchecked(barrier.buffer) }; + + self.current_buffer_barriers.push( + vk::BufferMemoryBarrier2::default() + .src_stage_mask(barrier.src_stage_mask.into()) + .src_access_mask(barrier.src_access_mask.into()) + .dst_stage_mask(barrier.dst_stage_mask.into()) + .dst_access_mask(barrier.dst_access_mask.into()) + .src_queue_family_index(barrier.src_queue_family_index) + .dst_queue_family_index(barrier.dst_queue_family_index) + .buffer(state.buffer().handle()) + .offset(barrier.range.start) + .size(barrier.range.end - barrier.range.start), + ); + } + } + + fn convert_image_barriers(&mut self, barrier_range: Range) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.image_barriers[barrier_range] { + let image = match barrier.image { + ImageReference::Normal(image) => { + unsafe { self.resource_map.image_unchecked(image) }.image() + } + ImageReference::Swapchain(swapchain) => { + unsafe { self.resource_map.swapchain_unchecked(swapchain) }.current_image() + } + }; + + self.current_image_barriers.push( + vk::ImageMemoryBarrier2::default() + .src_stage_mask(barrier.src_stage_mask.into()) + .src_access_mask(barrier.src_access_mask.into()) + .dst_stage_mask(barrier.dst_stage_mask.into()) + .dst_access_mask(barrier.dst_access_mask.into()) + .old_layout(barrier.old_layout.into()) + .new_layout(barrier.new_layout.into()) + .src_queue_family_index(barrier.src_queue_family_index) + .dst_queue_family_index(barrier.dst_queue_family_index) + .image(image.handle()) + .subresource_range(barrier.subresource_range.clone().into()), + ); + } + } + + fn signal_semaphore(&mut self, semaphore_index: SemaphoreIndex, stage_mask: PipelineStages) { + self.current_per_submit.signal_semaphore_infos.push( + vk::SemaphoreSubmitInfo::default() + .semaphore(self.executable.semaphores.borrow()[semaphore_index].handle()) + .stage_mask(stage_mask.into()), + ); + } + + fn signal_present(&mut self, swapchain_id: Id, stage_mask: PipelineStages) { + let swapchain_state = unsafe { self.resource_map.swapchain_unchecked(swapchain_id) }; + let semaphore = + &swapchain_state.semaphores[self.current_frame as usize].tasks_complete_semaphore; + + self.current_per_submit.signal_semaphore_infos.push( + vk::SemaphoreSubmitInfo::default() + .semaphore(semaphore.handle()) + .stage_mask(stage_mask.into()), + ); + } + + fn flush_submit(&mut self) -> Result { + unsafe { self.flush_current_command_buffer() }?; + + self.per_submits + .push(mem::take(&mut self.current_per_submit)); + + Ok(()) + } + + fn submit(&mut self) -> Result { + let submission = &self.executable.submissions[*self.submission_count]; + + let mut submit_infos = SmallVec::<[_; 4]>::with_capacity(self.per_submits.len()); + submit_infos.extend(self.per_submits.iter().map(|per_submit| { + vk::SubmitInfo2::default() + .wait_semaphore_infos(&per_submit.wait_semaphore_infos) + .command_buffer_infos(&per_submit.command_buffer_infos) + .signal_semaphore_infos(&per_submit.signal_semaphore_infos) + })); + + let max_submission_index = self.executable.submissions.len() - 1; + let fence_handle = if *self.submission_count == max_submission_index { + self.current_fence.handle() + } else { + vk::Fence::null() + }; + + unsafe { + (self.queue_submit2)( + submission.queue.handle(), + submit_infos.len() as u32, + submit_infos.as_ptr(), + fence_handle, + ) + } + .result() + .map_err(VulkanError::from)?; + + *self.submission_count += 1; + + Ok(()) + } + + unsafe fn flush_current_command_buffer(&mut self) -> Result { + if let Some(command_buffer) = self.current_command_buffer.take() { + let command_buffer = unsafe { command_buffer.end() }?; + self.current_per_submit.command_buffer_infos.push( + vk::CommandBufferSubmitInfo::default().command_buffer(command_buffer.handle()), + ); + self.death_row.push(Arc::new(command_buffer)); + self.current_command_buffer = Some(create_command_buffer( + self.resource_map, + &self.executable.submissions[*self.submission_count].queue, + )?); + } + + Ok(()) + } +} + +struct ExecuteState<'a, W: ?Sized + 'static> { + executable: &'a ExecutableTaskGraph, + resource_map: &'a ResourceMap<'a>, + death_row: &'a mut DeathRow, + current_frame: u32, + current_fence: &'a Fence, + submission_count: &'a mut usize, + world: &'a W, + cmd_pipeline_barrier: vk::PFN_vkCmdPipelineBarrier, + queue_submit: vk::PFN_vkQueueSubmit, + per_submits: SmallVec<[PerSubmitInfo; 4]>, + current_per_submit: PerSubmitInfo, + current_command_buffer: Option, + command_buffers: Vec>, + current_buffer_barriers: Vec>, + current_image_barriers: Vec>, +} + +#[derive(Default)] +struct PerSubmitInfo { + wait_semaphores: SmallVec<[vk::Semaphore; 4]>, + wait_dst_stage_mask: SmallVec<[vk::PipelineStageFlags; 4]>, + command_buffers: SmallVec<[vk::CommandBuffer; 1]>, + signal_semaphores: SmallVec<[vk::Semaphore; 4]>, +} + +impl<'a, W: ?Sized + 'static> ExecuteState<'a, W> { + fn new( + executable: &'a ExecutableTaskGraph, + resource_map: &'a ResourceMap<'a>, + death_row: &'a mut DeathRow, + current_frame: u32, + current_fence: &'a Fence, + submission_count: &'a mut usize, + world: &'a W, + ) -> Result { + let fns = resource_map.device().fns(); + let cmd_pipeline_barrier = fns.v1_0.cmd_pipeline_barrier; + let queue_submit = fns.v1_0.queue_submit; + + let current_command_buffer = + create_command_buffer(resource_map, &executable.submissions[0].queue)?; + + Ok(ExecuteState { + executable, + resource_map, + death_row, + current_frame, + current_fence, + submission_count, + world, + cmd_pipeline_barrier, + queue_submit, + per_submits: SmallVec::new(), + current_per_submit: PerSubmitInfo::default(), + current_command_buffer: Some(current_command_buffer), + command_buffers: Vec::new(), + current_buffer_barriers: Vec::new(), + current_image_barriers: Vec::new(), + }) + } + + fn initial_pipeline_barrier( + &mut self, + buffer_barrier_range: Range, + image_barrier_range: Range, + ) { + let mut src_stage_mask = vk::PipelineStageFlags::empty(); + let mut dst_stage_mask = vk::PipelineStageFlags::empty(); + + self.convert_initial_buffer_barriers( + buffer_barrier_range, + &mut src_stage_mask, + &mut dst_stage_mask, + ); + self.convert_initial_image_barriers( + image_barrier_range, + &mut src_stage_mask, + &mut dst_stage_mask, + ); + + if src_stage_mask.is_empty() { + src_stage_mask = vk::PipelineStageFlags::TOP_OF_PIPE; + } + + if dst_stage_mask.is_empty() { + dst_stage_mask = vk::PipelineStageFlags::BOTTOM_OF_PIPE; + } + + unsafe { + (self.cmd_pipeline_barrier)( + self.current_command_buffer.as_mut().unwrap().handle(), + src_stage_mask, + dst_stage_mask, + vk::DependencyFlags::empty(), + 0, + ptr::null(), + self.current_buffer_barriers.len() as u32, + self.current_buffer_barriers.as_ptr(), + self.current_image_barriers.len() as u32, + self.current_image_barriers.as_ptr(), + ) + }; + + self.current_buffer_barriers.clear(); + self.current_image_barriers.clear(); + } + + fn convert_initial_buffer_barriers( + &mut self, + barrier_range: Range, + src_stage_mask: &mut vk::PipelineStageFlags, + dst_stage_mask: &mut vk::PipelineStageFlags, + ) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.buffer_barriers[barrier_range] { + let state = unsafe { self.resource_map.buffer_unchecked(barrier.buffer) }; + + for (range, access) in state.accesses(barrier.range.clone()) { + self.current_buffer_barriers.push( + vk::BufferMemoryBarrier::default() + .src_access_mask(convert_access_mask(access.access_mask())) + .dst_access_mask(convert_access_mask(barrier.dst_access_mask)) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .buffer(state.buffer().handle()) + .offset(range.start) + .size(range.end - range.start), + ); + + *src_stage_mask |= convert_stage_mask(access.stage_mask()); + } + + *dst_stage_mask |= convert_stage_mask(barrier.dst_stage_mask); + } + } + + fn convert_initial_image_barriers( + &mut self, + barrier_range: Range, + src_stage_mask: &mut vk::PipelineStageFlags, + dst_stage_mask: &mut vk::PipelineStageFlags, + ) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.image_barriers[barrier_range] { + match barrier.image { + ImageReference::Normal(image) => { + let state = unsafe { self.resource_map.image_unchecked(image) }; + + for (subresource_range, access) in + state.accesses(barrier.subresource_range.clone()) + { + self.current_image_barriers.push( + vk::ImageMemoryBarrier::default() + .src_access_mask(convert_access_mask(access.access_mask())) + .dst_access_mask(convert_access_mask(barrier.dst_access_mask)) + .old_layout(access.image_layout().into()) + .new_layout(barrier.new_layout.into()) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(state.image().handle()) + .subresource_range(subresource_range.into()), + ); + + *src_stage_mask |= convert_stage_mask(access.stage_mask()); + } + } + ImageReference::Swapchain(swapchain) => { + let state = unsafe { self.resource_map.swapchain_unchecked(swapchain) }; + + for (subresource_range, access) in + state.accesses(barrier.subresource_range.array_layers.clone()) + { + self.current_image_barriers.push( + vk::ImageMemoryBarrier::default() + .src_access_mask(convert_access_mask(access.access_mask())) + .dst_access_mask(convert_access_mask(barrier.dst_access_mask)) + .old_layout(access.image_layout().into()) + .new_layout(barrier.new_layout.into()) + // FIXME: + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(state.current_image().handle()) + .subresource_range(subresource_range.into()), + ); + + *src_stage_mask |= convert_stage_mask(access.stage_mask()); + } + } + } + + *dst_stage_mask |= convert_stage_mask(barrier.dst_stage_mask); + } + } + + fn wait_acquire(&mut self, swapchain_id: Id, stage_mask: PipelineStages) { + let swapchain_state = unsafe { self.resource_map.swapchain_unchecked(swapchain_id) }; + let semaphore = + &swapchain_state.semaphores[self.current_frame as usize].image_available_semaphore; + + self.current_per_submit + .wait_semaphores + .push(semaphore.handle()); + self.current_per_submit + .wait_dst_stage_mask + .push(convert_stage_mask(stage_mask)); + } + + fn wait_semaphore(&mut self, semaphore_index: SemaphoreIndex, stage_mask: PipelineStages) { + self.current_per_submit + .wait_semaphores + .push(self.executable.semaphores.borrow()[semaphore_index].handle()); + self.current_per_submit + .wait_dst_stage_mask + .push(convert_stage_mask(stage_mask)); + } + + fn execute_task(&mut self, node_index: NodeIndex) -> Result { + let task_node = unsafe { self.executable.graph.nodes.task_node_unchecked(node_index) }; + let mut context = TaskContext { + resource_map: self.resource_map, + death_row: Cell::new(Some(self.death_row)), + current_command_buffer: Cell::new(Some(self.current_command_buffer.as_mut().unwrap())), + command_buffers: Cell::new(Some(&mut self.command_buffers)), + accesses: &task_node.accesses, + }; + + unsafe { task_node.task.execute(&mut context, self.world) } + .map_err(|error| ExecuteError::Task { node_index, error })?; + + if !self.command_buffers.is_empty() { + unsafe { self.flush_current_command_buffer() }?; + + for command_buffer in self.command_buffers.drain(..) { + self.current_per_submit + .command_buffers + .push(command_buffer.handle()); + self.death_row.push(command_buffer); + } + } + + Ok(()) + } + + fn pipeline_barrier( + &mut self, + buffer_barrier_range: Range, + image_barrier_range: Range, + ) { + let mut src_stage_mask = vk::PipelineStageFlags::empty(); + let mut dst_stage_mask = vk::PipelineStageFlags::empty(); + + self.convert_buffer_barriers( + buffer_barrier_range, + &mut src_stage_mask, + &mut dst_stage_mask, + ); + self.convert_image_barriers( + image_barrier_range, + &mut src_stage_mask, + &mut dst_stage_mask, + ); + + if src_stage_mask.is_empty() { + src_stage_mask = vk::PipelineStageFlags::TOP_OF_PIPE; + } + + if dst_stage_mask.is_empty() { + dst_stage_mask = vk::PipelineStageFlags::BOTTOM_OF_PIPE; + } + + unsafe { + (self.cmd_pipeline_barrier)( + self.current_command_buffer.as_mut().unwrap().handle(), + src_stage_mask, + dst_stage_mask, + vk::DependencyFlags::empty(), + 0, + ptr::null(), + self.current_buffer_barriers.len() as u32, + self.current_buffer_barriers.as_ptr(), + self.current_image_barriers.len() as u32, + self.current_image_barriers.as_ptr(), + ) + }; + + self.current_buffer_barriers.clear(); + self.current_image_barriers.clear(); + } + + fn convert_buffer_barriers( + &mut self, + barrier_range: Range, + src_stage_mask: &mut vk::PipelineStageFlags, + dst_stage_mask: &mut vk::PipelineStageFlags, + ) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.buffer_barriers[barrier_range] { + let state = unsafe { self.resource_map.buffer_unchecked(barrier.buffer) }; + + self.current_buffer_barriers.push( + vk::BufferMemoryBarrier::default() + .src_access_mask(convert_access_mask(barrier.src_access_mask)) + .dst_access_mask(convert_access_mask(barrier.dst_access_mask)) + .src_queue_family_index(barrier.src_queue_family_index) + .dst_queue_family_index(barrier.dst_queue_family_index) + .buffer(state.buffer().handle()) + .offset(barrier.range.start) + .size(barrier.range.end - barrier.range.start), + ); + + *src_stage_mask |= convert_stage_mask(barrier.src_stage_mask); + *dst_stage_mask |= convert_stage_mask(barrier.dst_stage_mask); + } + } + + fn convert_image_barriers( + &mut self, + barrier_range: Range, + src_stage_mask: &mut vk::PipelineStageFlags, + dst_stage_mask: &mut vk::PipelineStageFlags, + ) { + let barrier_range = barrier_range.start as usize..barrier_range.end as usize; + + for barrier in &self.executable.image_barriers[barrier_range] { + let image = match barrier.image { + ImageReference::Normal(image) => { + unsafe { self.resource_map.image_unchecked(image) }.image() + } + ImageReference::Swapchain(swapchain) => { + unsafe { self.resource_map.swapchain_unchecked(swapchain) }.current_image() + } + }; + + self.current_image_barriers.push( + vk::ImageMemoryBarrier::default() + .src_access_mask(convert_access_mask(barrier.src_access_mask)) + .dst_access_mask(convert_access_mask(barrier.dst_access_mask)) + .old_layout(barrier.old_layout.into()) + .new_layout(barrier.new_layout.into()) + .src_queue_family_index(barrier.src_queue_family_index) + .dst_queue_family_index(barrier.dst_queue_family_index) + .image(image.handle()) + .subresource_range(barrier.subresource_range.clone().into()), + ); + + *src_stage_mask |= convert_stage_mask(barrier.src_stage_mask); + *dst_stage_mask |= convert_stage_mask(barrier.dst_stage_mask); + } + } + + fn signal_semaphore(&mut self, semaphore_index: SemaphoreIndex, _stage_mask: PipelineStages) { + self.current_per_submit + .signal_semaphores + .push(self.executable.semaphores.borrow()[semaphore_index].handle()); + } + + fn signal_present(&mut self, swapchain_id: Id, _stage_mask: PipelineStages) { + let swapchain_state = unsafe { self.resource_map.swapchain_unchecked(swapchain_id) }; + let semaphore = + &swapchain_state.semaphores[self.current_frame as usize].tasks_complete_semaphore; + + self.current_per_submit + .signal_semaphores + .push(semaphore.handle()); + } + + fn flush_submit(&mut self) -> Result { + unsafe { self.flush_current_command_buffer() }?; + + self.per_submits + .push(mem::take(&mut self.current_per_submit)); + + Ok(()) + } + + fn submit(&mut self) -> Result { + let submission = &self.executable.submissions[*self.submission_count]; + + let mut submit_infos = SmallVec::<[_; 4]>::with_capacity(self.per_submits.len()); + submit_infos.extend(self.per_submits.iter().map(|per_submit| { + vk::SubmitInfo::default() + .wait_semaphores(&per_submit.wait_semaphores) + .wait_dst_stage_mask(&per_submit.wait_dst_stage_mask) + .command_buffers(&per_submit.command_buffers) + .signal_semaphores(&per_submit.signal_semaphores) + })); + + let max_submission_index = self.executable.submissions.len() - 1; + let fence_handle = if *self.submission_count == max_submission_index { + self.current_fence.handle() + } else { + vk::Fence::null() + }; + + unsafe { + (self.queue_submit)( + submission.queue.handle(), + submit_infos.len() as u32, + submit_infos.as_ptr(), + fence_handle, + ) + } + .result() + .map_err(VulkanError::from)?; + + *self.submission_count += 1; + + Ok(()) + } + + unsafe fn flush_current_command_buffer(&mut self) -> Result { + if let Some(command_buffer) = self.current_command_buffer.take() { + let command_buffer = unsafe { command_buffer.end() }?; + self.current_per_submit + .command_buffers + .push(command_buffer.handle()); + self.death_row.push(Arc::new(command_buffer)); + self.current_command_buffer = Some(create_command_buffer( + self.resource_map, + &self.executable.submissions[*self.submission_count].queue, + )?); + } + + Ok(()) + } +} + +fn create_command_buffer( + resource_map: &ResourceMap<'_>, + queue: &Queue, +) -> Result { + // SAFETY: The parameters are valid. + unsafe { + RawRecordingCommandBuffer::new_unchecked( + resource_map.resources.command_buffer_allocator().clone(), + queue.queue_family_index(), + CommandBufferLevel::Primary, + CommandBufferBeginInfo { + usage: CommandBufferUsage::OneTimeSubmit, + inheritance_info: None, + ..Default::default() + }, + ) + } + // This can't panic because we know that the queue family index is active on the device, + // otherwise we wouldn't have a reference to the `Queue`. + .map_err(Validated::unwrap) +} + +fn convert_stage_mask(mut stage_mask: PipelineStages) -> vk::PipelineStageFlags { + const VERTEX_INPUT_FLAGS: PipelineStages = + PipelineStages::INDEX_INPUT.union(PipelineStages::VERTEX_ATTRIBUTE_INPUT); + const TRANSFER_FLAGS: PipelineStages = PipelineStages::COPY + .union(PipelineStages::RESOLVE) + .union(PipelineStages::BLIT) + .union(PipelineStages::CLEAR); + + if stage_mask.intersects(VERTEX_INPUT_FLAGS) { + stage_mask -= VERTEX_INPUT_FLAGS; + stage_mask |= PipelineStages::VERTEX_INPUT; + } + + if stage_mask.intersects(TRANSFER_FLAGS) { + stage_mask -= TRANSFER_FLAGS; + stage_mask |= PipelineStages::ALL_TRANSFER; + } + + stage_mask.into() +} + +fn convert_access_mask(mut access_mask: AccessFlags) -> vk::AccessFlags { + const READ_FLAGS: AccessFlags = + AccessFlags::SHADER_SAMPLED_READ.union(AccessFlags::SHADER_STORAGE_READ); + const WRITE_FLAGS: AccessFlags = AccessFlags::SHADER_STORAGE_WRITE; + + if access_mask.intersects(READ_FLAGS) { + access_mask -= READ_FLAGS; + access_mask |= AccessFlags::SHADER_READ; + } + + if access_mask.intersects(WRITE_FLAGS) { + access_mask -= WRITE_FLAGS; + access_mask |= AccessFlags::SHADER_WRITE; + } + + access_mask.into() +} + +struct StateGuard<'a, W: ?Sized + 'static> { + executable: &'a ExecutableTaskGraph, + resource_map: &'a ResourceMap<'a>, + submission_count: usize, +} + +impl Drop for StateGuard<'_, W> { + #[cold] + fn drop(&mut self) { + if self.submission_count == 0 { + return; + } + + let submissions = &self.executable.submissions; + + // We must make sure that invalid state cannot be observed, because if at least one + // submission succeeded while one failed, that means that there are pending semaphore + // signal operations. + for submission in &submissions[0..self.submission_count] { + if let Err(err) = submission.queue.with(|mut guard| guard.wait_idle()) { + // Device loss is already a form of poisoning built into Vulkan. There's no + // invalid state that can be observed by design. + if err == VulkanError::DeviceLost { + return; + } + + eprintln!( + "failed to wait on queue idle after partly failed submissions rendering \ + recovery impossible: {err}; aborting", + ); + std::process::abort(); + } + } + + let device = submissions[0].queue.device(); + + // But even after waiting for idle, the state of the graph is invalid because some + // semaphores are still signalled, so we have to recreate them. + for semaphore in self.executable.semaphores.borrow_mut().iter_mut() { + // SAFETY: The parameters are valid. + match unsafe { Semaphore::new_unchecked(device.clone(), Default::default()) } { + Ok(new_semaphore) => { + let _ = mem::replace(semaphore, new_semaphore); + } + Err(err) => { + if err == VulkanError::DeviceLost { + return; + } + + eprintln!( + "failed to recreate semaphores after partly failed submissions rendering \ + recovery impossible: {err}; aborting", + ); + std::process::abort(); + } + } + } + + unsafe { + self.executable.update_resource_state( + self.resource_map, + 0..submissions[self.submission_count - 1].instruction_range.end, + ) + }; + } +} + +/// Maps [virtual resources] to physical resources. +pub struct ResourceMap<'a> { + virtual_resources: &'a super::Resources, + resources: &'a Resources, + map: Vec<*const ()>, + len: u32, + guard: epoch::Guard<'a>, +} + +impl<'a> ResourceMap<'a> { + /// Creates a new `ResourceMap` mapping the virtual resources of the given `executable` to + /// physical resources from the given `resources` collection. + /// + /// # Panics + /// + /// - Panics if the device of `executable` is not the same as that of `resources`. + pub fn new(executable: &'a ExecutableTaskGraph, resources: &'a Resources) -> Self { + assert_eq!(executable.device(), resources.device()); + + let virtual_resources = &executable.graph.resources; + let map = vec![ptr::null(); virtual_resources.capacity() as usize]; + + ResourceMap { + virtual_resources, + resources, + map, + len: 0, + guard: resources.pin(), + } + } + + /// Inserts a mapping from the [virtual buffer resource] corresponding to `virtual_id` to the + /// physical resource corresponding to `physical_id`. + /// + /// # Panics + /// + /// - Panics if the physical resource doesn't match the virtual resource. + /// - Panics if the physical resource already has a mapping from another virtual resource. + #[inline] + pub fn insert_buffer( + &mut self, + virtual_id: Id, + physical_id: Id, + ) -> Result<(), InvalidSlotError> { + let virtual_buffer = self.virtual_resources.buffer(virtual_id)?; + + // SAFETY: We own an `epoch::Guard`. + let state = unsafe { self.resources.buffer_unprotected(physical_id) }?; + + assert!(state.buffer().size() >= virtual_buffer.size); + assert_eq!( + state.buffer().sharing().is_exclusive(), + virtual_id.tag() & EXCLUSIVE_BIT != 0, + ); + + let ptr = <*const _>::cast(state); + let is_duplicate = self.map.iter().any(|&p| p == ptr); + + // SAFETY: We checked that `virtual_id` is present in `self.virtual_resources` above, and + // since we initialized `self.map` with a length at least that of `self.virtual_resources`, + // the index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if *slot != ptr { + assert!(!is_duplicate); + } + + if slot.is_null() { + self.len += 1; + } + + *slot = ptr; + + Ok(()) + } + + /// Inserts a mapping from the [virtual buffer resource] corresponding to `virtual_id` to the + /// physical resource corresponding to `physical_id` without doing any checks. + /// + /// # Safety + /// + /// - `virtual_id` must be a valid virtual resource ID. + /// - `physical_id` must be a valid physical resource ID. + /// - The physical resource must match the virtual resource. + /// - The physical resource must not have a mapping from another virtual resource. + #[inline] + pub unsafe fn insert_buffer_unchecked( + &mut self, + virtual_id: Id, + physical_id: Id, + ) { + // SAFETY: + // * The caller must ensure that `physical_id` is a valid ID. + // * We own an `epoch::Guard`. + let state = unsafe { self.resources.buffer_unchecked_unprotected(physical_id) }; + + // SAFETY: The caller must ensure that `virtual_id` is a valid virtual ID, and since we + // initialized `self.map` with a length at least that of `self.virtual_resources`, the + // index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if slot.is_null() { + self.len += 1; + } + + *slot = <*const _>::cast(state); + } + + /// Inserts a mapping from the [virtual image resource] corresponding to `virtual_id` to the + /// physical resource corresponding to `physical_id`. + /// + /// # Panics + /// + /// - Panics if the physical resource doesn't match the virtual resource. + /// - Panics if the physical resource already has a mapping from another virtual resource. + #[inline] + pub fn insert_image( + &mut self, + virtual_id: Id, + physical_id: Id, + ) -> Result<(), InvalidSlotError> { + let virtual_image = self.virtual_resources.image(virtual_id)?; + + // SAFETY: We own an `epoch::Guard`. + let state = unsafe { self.resources.image_unprotected(physical_id) }?; + + assert_eq!(state.image().flags(), virtual_image.flags); + assert_eq!(state.image().format(), virtual_image.format); + assert!(state.image().array_layers() >= virtual_image.array_layers); + assert!(state.image().mip_levels() >= virtual_image.mip_levels); + assert_eq!( + state.image().sharing().is_exclusive(), + virtual_id.tag() & EXCLUSIVE_BIT != 0, + ); + + let ptr = <*const _>::cast(state); + let is_duplicate = self.map.iter().any(|&p| p == ptr); + + // SAFETY: We checked that `virtual_id` is present in `self.virtual_resources` above, and + // since we initialized `self.map` with a length at least that of `self.virtual_resources`, + // the index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if *slot != ptr { + assert!(!is_duplicate); + } + + if slot.is_null() { + self.len += 1; + } + + *slot = ptr; + + Ok(()) + } + + /// Inserts a mapping from the [virtual image resource] corresponding to `virtual_id` to the + /// physical resource corresponding to `physical_id` without doing any checks. + /// + /// # Safety + /// + /// - `virtual_id` must be a valid virtual resource ID. + /// - `physical_id` must be a valid physical resource ID. + /// - The physical resource must match the virtual resource. + /// - The physical resource must not have a mapping from another virtual resource. + #[inline] + pub unsafe fn insert_image_unchecked(&mut self, virtual_id: Id, physical_id: Id) { + // SAFETY: + // * The caller must ensure that `physical_id` is a valid ID. + // * We own an `epoch::Guard`. + let state = unsafe { self.resources.image_unchecked_unprotected(physical_id) }; + + // SAFETY: The caller must ensure that `virtual_id` is a valid virtual ID, and since we + // initialized `self.map` with a length at least that of `self.virtual_resources`, the + // index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if slot.is_null() { + self.len += 1; + } + + *slot = <*const _>::cast(state); + } + + /// Inserts a mapping from the [virtual swapchain resource] corresponding to `virtual_id` to + /// the physical resource corresponding to `physical_id`. + /// + /// # Panics + /// + /// - Panics if the physical resource doesn't match the virtual resource. + /// - Panics if the physical resource already has a mapping from another virtual resource. + #[inline] + pub fn insert_swapchain( + &mut self, + virtual_id: Id, + physical_id: Id, + ) -> Result<(), InvalidSlotError> { + let virtual_swapchain = self.virtual_resources.swapchain(virtual_id)?; + + // SAFETY: We own an `epoch::Guard`. + let state = unsafe { self.resources.swapchain_unprotected(physical_id) }?; + + assert!(state.swapchain().image_array_layers() >= virtual_swapchain.image_array_layers); + + let ptr = <*const _>::cast(state); + let is_duplicate = self.map.iter().any(|&p| p == ptr); + + // SAFETY: We checked that `virtual_id` is present in `self.virtual_resources` above, and + // since we initialized `self.map` with a length at least that of `self.virtual_resources`, + // the index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if *slot != ptr { + assert!(!is_duplicate); + } + + if slot.is_null() { + self.len += 1; + } + + *slot = ptr; + + Ok(()) + } + + /// Inserts a mapping from the [virtual swapchain resource] corresponding to `virtual_id` to + /// the physical resource corresponding to `physical_id` without doing any checks. + /// + /// # Safety + /// + /// - `virtual_id` must be a valid virtual resource ID. + /// - `physical_id` must be a valid physical resource ID. + /// - The physical resource must match the virtual resource. + /// - The physical resource must not have a mapping from another virtual resource. + #[inline] + pub unsafe fn insert_swapchain_unchecked( + &mut self, + virtual_id: Id, + physical_id: Id, + ) { + // SAFETY: + // * The caller must ensure that `physical_id` is a valid ID. + // * We own an `epoch::Guard`. + let state = unsafe { self.resources.swapchain_unchecked_unprotected(physical_id) }; + + // SAFETY: The caller must ensure that `virtual_id` is a valid virtual ID, and since we + // initialized `self.map` with a length at least that of `self.virtual_resources`, the + // index must be in bounds. + let slot = unsafe { self.map.get_unchecked_mut(virtual_id.index() as usize) }; + + if slot.is_null() { + self.len += 1; + } + + *slot = <*const _>::cast(state); + } + + /// Returns the `Resources` collection. + #[inline] + #[must_use] + pub fn resources(&self) -> &'a Resources { + self.resources + } + + /// Returns the number of mappings in the map. + #[inline] + #[must_use] + pub fn len(&self) -> u32 { + self.len + } + + /// Returns `true` if the map maps every virtual resource. + #[inline] + #[must_use] + pub fn is_exhaustive(&self) -> bool { + // By our own invariant, the map can only contain mappings for virtual resources that are + // present in `self.virtual_resources`. It follows then, that when the length of `self` is + // that of `self.virtual_resources`, that the virtual resources are mapped exhaustively. + self.len() == self.virtual_resources.len() + } + + pub(crate) unsafe fn buffer(&self, id: Id) -> Result<&BufferState, InvalidSlotError> { + self.virtual_resources.buffer(id)?; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + Ok(unsafe { self.buffer_unchecked(id) }) + } + + pub(crate) unsafe fn buffer_unchecked(&self, id: Id) -> &BufferState { + // SAFETY: The caller must ensure that `id` is a valid virtual ID. + let &slot = unsafe { self.map.get_unchecked(id.index() as usize) }; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + unsafe { &*slot.cast::() } + } + + pub(crate) unsafe fn image(&self, id: Id) -> Result<&ImageState, InvalidSlotError> { + self.virtual_resources.image(id)?; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + Ok(unsafe { self.image_unchecked(id) }) + } + + pub(crate) unsafe fn image_unchecked(&self, id: Id) -> &ImageState { + // SAFETY: The caller must ensure that `id` is a valid virtual ID. + let &slot = unsafe { self.map.get_unchecked(id.index() as usize) }; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + unsafe { &*slot.cast::() } + } + + pub(crate) unsafe fn swapchain( + &self, + id: Id, + ) -> Result<&SwapchainState, InvalidSlotError> { + self.virtual_resources.swapchain(id)?; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + Ok(unsafe { self.swapchain_unchecked(id) }) + } + + pub(crate) unsafe fn swapchain_unchecked(&self, id: Id) -> &SwapchainState { + // SAFETY: The caller must ensure that `id` is a valid virtual ID. + let &slot = unsafe { self.map.get_unchecked(id.index() as usize) }; + + // SAFETY: The caller must ensure that a mapping for `id` has been inserted. + unsafe { &*slot.cast::() } + } +} + +unsafe impl DeviceOwned for ResourceMap<'_> { + #[inline] + fn device(&self) -> &Arc { + self.resources.device() + } +} + +type Result = ::std::result::Result; + +/// Error that can happen when [executing] an [`ExecutableTaskGraph`]. +/// +/// [executing]: ExecutableTaskGraph::execute +#[derive(Debug)] +pub enum ExecuteError { + Task { + node_index: NodeIndex, + error: TaskError, + }, + Swapchain { + swapchain_id: Id, + error: VulkanError, + }, + VulkanError(VulkanError), +} + +impl From for ExecuteError { + fn from(err: VulkanError) -> Self { + Self::VulkanError(err) + } +} + +impl fmt::Display for ExecuteError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Task { node_index, .. } => { + write!(f, "an error occurred while executing task {node_index:?}") + } + Self::Swapchain { swapchain_id, .. } => write!( + f, + "an error occurred while using swapchain {swapchain_id:?}", + ), + Self::VulkanError(_) => f.write_str("a runtime error occurred"), + } + } +} + +impl Error for ExecuteError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Task { error, .. } => Some(error), + Self::Swapchain { error, .. } => Some(error), + Self::VulkanError(err) => Some(err), + } + } +} diff --git a/vulkano-taskgraph/src/graph/mod.rs b/vulkano-taskgraph/src/graph/mod.rs index 22b3ad87..e315a21a 100644 --- a/vulkano-taskgraph/src/graph/mod.rs +++ b/vulkano-taskgraph/src/graph/mod.rs @@ -1,12 +1,16 @@ //! The task graph data structure and associated types. +pub use self::execute::{ExecuteError, ResourceMap}; use crate::{ resource::{AccessType, BufferRange, ImageLayoutType}, Id, InvalidSlotError, QueueFamilyType, Task, BUFFER_TAG, IMAGE_TAG, SWAPCHAIN_TAG, }; use concurrent_slotmap::{IterMut, IterUnprotected, SlotId, SlotMap}; use smallvec::SmallVec; -use std::{borrow::Cow, error::Error, fmt, hint, iter::FusedIterator, ops::Range, sync::Arc}; +use std::{ + borrow::Cow, cell::RefCell, error::Error, fmt, hint, iter::FusedIterator, ops::Range, slice, + sync::Arc, +}; use vulkano::{ buffer::{Buffer, BufferCreateInfo}, device::{Device, DeviceOwned, Queue}, @@ -19,6 +23,8 @@ use vulkano::{ DeviceSize, }; +mod execute; + const EXCLUSIVE_BIT: u32 = 1 << 6; const VIRTUAL_BIT: u32 = 1 << 7; @@ -647,6 +653,10 @@ impl TaskNode { } impl ResourceAccesses { + fn iter(&self) -> slice::Iter<'_, ResourceAccess> { + self.inner.iter() + } + pub(crate) fn contains_buffer_access( &self, id: Id, @@ -655,7 +665,7 @@ impl ResourceAccesses { ) -> bool { debug_assert!(!range.is_empty()); - self.inner.iter().any(|resource_access| { + self.iter().any(|resource_access| { matches!(resource_access, ResourceAccess::Buffer(a) if a.id == id && a.access_type == access_type && a.range.start <= range.start @@ -674,7 +684,7 @@ impl ResourceAccesses { debug_assert!(!subresource_range.mip_levels.is_empty()); debug_assert!(!subresource_range.array_layers.is_empty()); - self.inner.iter().any(|resource_access| { + self.iter().any(|resource_access| { matches!(resource_access, ResourceAccess::Image(a) if a.id == id && a.access_type == access_type && a.layout_type == layout_type @@ -695,7 +705,7 @@ impl ResourceAccesses { ) -> bool { debug_assert!(!array_layers.is_empty()); - self.inner.iter().any(|resource_access| { + self.iter().any(|resource_access| { matches!(resource_access, ResourceAccess::Swapchain(a) if a.id == id && a.access_type == access_type && a.layout_type == layout_type @@ -814,7 +824,7 @@ impl TaskNodeBuilder<'_, W> { pub unsafe fn image_access_unchecked( &mut self, id: Id, - mut subresource_range: ImageSubresourceRange, + subresource_range: ImageSubresourceRange, access_type: AccessType, mut layout_type: ImageLayoutType, ) -> &mut Self { @@ -908,7 +918,7 @@ pub struct ExecutableTaskGraph { submissions: Vec, buffer_barriers: Vec, image_barriers: Vec, - semaphores: Vec, + semaphores: RefCell>, swapchains: SmallVec<[Id; 1]>, present_queue: Option>, } diff --git a/vulkano-taskgraph/src/lib.rs b/vulkano-taskgraph/src/lib.rs index 30402e42..c3b4fdb7 100644 --- a/vulkano-taskgraph/src/lib.rs +++ b/vulkano-taskgraph/src/lib.rs @@ -3,10 +3,8 @@ #![forbid(unsafe_op_in_unsafe_fn)] use concurrent_slotmap::SlotId; -use graph::ResourceAccesses; -use resource::{ - AccessType, BufferRange, BufferState, DeathRow, ImageState, Resources, SwapchainState, -}; +use graph::{ResourceAccesses, ResourceMap}; +use resource::{AccessType, BufferRange, BufferState, DeathRow, ImageState, SwapchainState}; use std::{ any::{Any, TypeId}, cell::Cell, @@ -16,6 +14,7 @@ use std::{ hash::{Hash, Hasher}, marker::PhantomData, ops::{Deref, DerefMut, Range, RangeBounds}, + sync::Arc, thread, }; use vulkano::{ @@ -117,10 +116,10 @@ impl fmt::Debug for dyn Task { /// /// This gives you access to the current command buffer, resources, as well as resource cleanup. pub struct TaskContext<'a> { - resources: &'a Resources, + resource_map: &'a ResourceMap<'a>, death_row: Cell>, current_command_buffer: Cell>, - command_buffers: Cell>>, + command_buffers: Cell>>>, accesses: &'a ResourceAccesses, } @@ -160,7 +159,7 @@ impl<'a> TaskContext<'a> { /// /// [`raw_command_buffer`]: Self::raw_command_buffer #[inline] - pub unsafe fn push_command_buffer(&self, command_buffer: RawCommandBuffer) { + pub unsafe fn push_command_buffer(&self, command_buffer: Arc) { let vec = self.command_buffers.take().unwrap(); vec.push(command_buffer); self.command_buffers.set(Some(vec)); @@ -179,7 +178,7 @@ impl<'a> TaskContext<'a> { #[inline] pub unsafe fn extend_command_buffers( &self, - command_buffers: impl IntoIterator, + command_buffers: impl IntoIterator>, ) { let vec = self.command_buffers.take().unwrap(); vec.extend(command_buffers); @@ -189,28 +188,31 @@ impl<'a> TaskContext<'a> { /// Returns the buffer corresponding to `id`, or returns an error if it isn't present. #[inline] pub fn buffer(&self, id: Id) -> TaskResult<&'a BufferState> { - // SAFETY: Ensured by the caller of `Task::execute`. - Ok(unsafe { self.resources.buffer_unprotected(id) }?) + // SAFETY: The caller of `Task::execute` must ensure that `self.resource_map` maps the + // virtual IDs of the graph exhaustively. + Ok(unsafe { self.resource_map.buffer(id) }?) } /// Returns the image corresponding to `id`, or returns an error if it isn't present. #[inline] pub fn image(&self, id: Id) -> TaskResult<&'a ImageState> { - // SAFETY: Ensured by the caller of `Task::execute`. - Ok(unsafe { self.resources.image_unprotected(id) }?) + // SAFETY: The caller of `Task::execute` must ensure that `self.resource_map` maps the + // virtual IDs of the graph exhaustively. + Ok(unsafe { self.resource_map.image(id) }?) } /// Returns the swapchain corresponding to `id`, or returns an error if it isn't present. #[inline] pub fn swapchain(&self, id: Id) -> TaskResult<&'a SwapchainState> { - // SAFETY: Ensured by the caller of `Task::execute`. - Ok(unsafe { self.resources.swapchain_unprotected(id) }?) + // SAFETY: The caller of `Task::execute` must ensure that `self.resource_map` maps the + // virtual IDs of the graph exhaustively. + Ok(unsafe { self.resource_map.swapchain(id) }?) } - /// Returns the `Resources` collection. + /// Returns the `ResourceMap`. #[inline] - pub fn resources(&self) -> &'a Resources { - self.resources + pub fn resource_map(&self) -> &'a ResourceMap<'a> { + self.resource_map } /// Tries to get read access to a portion of the buffer corresponding to `id`. @@ -624,7 +626,7 @@ impl<'a> TaskContext<'a> { // FIXME: unsafe #[inline] pub unsafe fn destroy_buffer(&self, id: Id) -> TaskResult { - let state = unsafe { self.resources.remove_buffer(id) }?; + let state = unsafe { self.resource_map.resources().remove_buffer(id) }?; let death_row = self.death_row.take().unwrap(); // FIXME: death_row.push(state.buffer().clone()); @@ -638,7 +640,7 @@ impl<'a> TaskContext<'a> { // FIXME: unsafe #[inline] pub unsafe fn destroy_image(&self, id: Id) -> TaskResult { - let state = unsafe { self.resources.remove_image(id) }?; + let state = unsafe { self.resource_map.resources().remove_image(id) }?; let death_row = self.death_row.take().unwrap(); // FIXME: death_row.push(state.image().clone()); @@ -652,7 +654,7 @@ impl<'a> TaskContext<'a> { // FIXME: unsafe #[inline] pub unsafe fn destroy_swapchain(&self, id: Id) -> TaskResult { - let state = unsafe { self.resources.remove_swapchain(id) }?; + let state = unsafe { self.resource_map.resources().remove_swapchain(id) }?; let death_row = self.death_row.take().unwrap(); // FIXME: death_row.push(state.swapchain().clone()); @@ -905,6 +907,10 @@ impl Id { fn index(self) -> u32 { self.slot.index() } + + fn tag(self) -> u32 { + self.slot.tag() + } } impl Clone for Id { diff --git a/vulkano-taskgraph/src/resource.rs b/vulkano-taskgraph/src/resource.rs index 0e22597d..f6b78bba 100644 --- a/vulkano-taskgraph/src/resource.rs +++ b/vulkano-taskgraph/src/resource.rs @@ -8,6 +8,7 @@ use rangemap::RangeMap; use smallvec::SmallVec; use std::{ any::Any, + cmp, hash::Hash, iter::FusedIterator, mem, @@ -21,6 +22,7 @@ use std::{ use thread_local::ThreadLocal; use vulkano::{ buffer::{AllocateBufferError, Buffer, BufferCreateInfo}, + command_buffer::allocator::StandardCommandBufferAllocator, device::{Device, DeviceOwned}, image::{ AllocateImageError, Image, ImageAspects, ImageCreateFlags, ImageCreateInfo, ImageLayout, @@ -48,6 +50,7 @@ static REGISTERED_DEVICES: Mutex> = Mutex::new(Vec::new()); #[derive(Debug)] pub struct Resources { memory_allocator: Arc, + command_buffer_allocator: Arc, global: epoch::GlobalHandle, locals: ThreadLocal, @@ -140,10 +143,16 @@ impl Resources { registered_devices.push(device_addr); + let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new( + device.clone(), + Default::default(), + )); + let global = epoch::GlobalHandle::new(); Resources { memory_allocator, + command_buffer_allocator, locals: ThreadLocal::new(), buffers: SlotMap::with_global(create_info.max_buffers, global.clone()), images: SlotMap::with_global(create_info.max_images, global.clone()), @@ -520,6 +529,12 @@ impl Resources { unsafe { self.buffers.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) } + #[inline] + pub(crate) unsafe fn buffer_unchecked_unprotected(&self, id: Id) -> &BufferState { + // SAFETY: Enforced by the caller. + unsafe { self.buffers.index_unchecked_unprotected(id.index()) } + } + /// Returns the image corresponding to `id`. #[inline] pub fn image(&self, id: Id) -> Result> { @@ -535,6 +550,12 @@ impl Resources { unsafe { self.images.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) } + #[inline] + pub(crate) unsafe fn image_unchecked_unprotected(&self, id: Id) -> &ImageState { + // SAFETY: Enforced by the caller. + unsafe { self.images.index_unchecked_unprotected(id.index()) } + } + /// Returns the swapchain corresponding to `id`. #[inline] pub fn swapchain(&self, id: Id) -> Result> { @@ -553,6 +574,15 @@ impl Resources { unsafe { self.swapchains.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) } + #[inline] + pub(crate) unsafe fn swapchain_unchecked_unprotected( + &self, + id: Id, + ) -> &SwapchainState { + // SAFETY: Enforced by the caller. + unsafe { self.swapchains.index_unchecked_unprotected(id.index()) } + } + /// Returns the [flight] corresponding to `id`. #[inline] pub fn flight(&self, id: Id) -> Result> { @@ -573,6 +603,10 @@ impl Resources { self.locals.get_or(|| self.global.register_local()).pin() } + pub(crate) fn command_buffer_allocator(&self) -> &Arc { + &self.command_buffer_allocator + } + pub(crate) fn try_advance_global_and_collect(&self, guard: &epoch::Guard<'_>) { if guard.try_advance_global() { self.buffers.try_collect(guard); @@ -585,6 +619,9 @@ impl Resources { impl Drop for Resources { fn drop(&mut self) { + // FIXME: + let _ = unsafe { self.device().wait_idle() }; + let mut registered_devices = REGISTERED_DEVICES.lock(); // This can't panic because there's no way to construct this type without the device's @@ -624,6 +661,7 @@ impl BufferState { assert!(!range.is_empty()); BufferAccesses { + range: range.clone(), overlapping: MutexGuard::leak(self.last_accesses.lock()).overlapping(range), // SAFETY: We locked the mutex above. _guard: unsafe { AccessesGuard::new(&self.last_accesses) }, @@ -701,6 +739,7 @@ impl ImageState { mip_levels: self.image.mip_levels(), array_layers: self.image.array_layers(), subresource_ranges, + range: 0..0, overlapping: last_accesses.overlapping(0..0), last_accesses, // SAFETY: We locked the mutex above. @@ -809,6 +848,13 @@ impl SwapchainState { &self.images } + /// Returns the ID of the [flight] which owns this swapchain. + #[inline] + #[must_use] + pub fn flight_id(&self) -> Id { + self.flight_id + } + /// Returns the image index that's acquired in the current frame, or returns `None` if no image /// index is acquired. #[inline] @@ -841,6 +887,7 @@ impl SwapchainState { mip_levels: 1, array_layers: self.swapchain.image_array_layers(), subresource_ranges, + range: 0..0, overlapping: last_accesses.overlapping(0..0), last_accesses, // SAFETY: We locked the mutex above. @@ -931,6 +978,7 @@ pub type BufferRange = Range; /// /// [`accesses`]: BufferState::accesses pub struct BufferAccesses<'a> { + range: BufferRange, overlapping: rangemap::map::Overlapping<'a, DeviceSize, BufferAccess, Range>, _guard: AccessesGuard<'a, BufferAccess>, } @@ -940,9 +988,12 @@ impl<'a> Iterator for BufferAccesses<'a> { #[inline] fn next(&mut self) -> Option { - self.overlapping - .next() - .map(|(range, access)| (range.clone(), access)) + self.overlapping.next().map(|(range, access)| { + let start = cmp::max(range.start, self.range.start); + let end = cmp::min(range.end, self.range.end); + + (start..end, access) + }) } } @@ -957,6 +1008,7 @@ pub struct ImageAccesses<'a> { mip_levels: u32, array_layers: u32, subresource_ranges: SubresourceRanges, + range: Range, overlapping: rangemap::map::Overlapping<'a, DeviceSize, ImageAccess, Range>, last_accesses: &'a RangeMap, _guard: AccessesGuard<'a, ImageAccess>, @@ -969,11 +1021,14 @@ impl<'a> Iterator for ImageAccesses<'a> { fn next(&mut self) -> Option { loop { if let Some((range, access)) = self.overlapping.next() { + let start = cmp::max(range.start, self.range.start); + let end = cmp::min(range.end, self.range.end); let subresource_range = - range_to_subresources(range.clone(), self.mip_levels, self.array_layers); + range_to_subresources(start..end, self.mip_levels, self.array_layers); break Some((subresource_range, access)); } else if let Some(range) = self.subresource_ranges.next() { + self.range = range.clone(); self.overlapping = self.last_accesses.overlapping(range); } else { break None;