diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 1dba81434..82207d510 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -43,4 +43,4 @@ pollster.workspace = true profiling.workspace = true rayon.workspace = true tracy-client = { workspace = true, optional = true } -wgpu = { workspace = true, features = ["wgsl"] } +wgpu = { workspace = true, features = ["wgsl", "metal", "dx12"] } diff --git a/wgpu-core/src/device/global.rs b/wgpu-core/src/device/global.rs index b0003e035..5ebd7c7de 100644 --- a/wgpu-core/src/device/global.rs +++ b/wgpu-core/src/device/global.rs @@ -19,7 +19,6 @@ use crate::{ present, resource::{ self, BufferAccessError, BufferAccessResult, BufferMapOperation, CreateBufferError, - Trackable, }, storage::Storage, Label, @@ -260,15 +259,25 @@ impl Global { ) -> Result<(), WaitIdleError> { let hub = A::hub(self); - let last_submission = match hub.buffers.read().get(buffer_id) { - Ok(buffer) => buffer.submission_index(), + let device = hub + .devices + .get(device_id) + .map_err(|_| DeviceError::InvalidDeviceId)?; + + let buffer = match hub.buffers.get(buffer_id) { + Ok(buffer) => buffer, Err(_) => return Ok(()), }; - hub.devices - .get(device_id) - .map_err(|_| DeviceError::InvalidDeviceId)? - .wait_for_submit(last_submission) + let last_submission = device + .lock_life() + .get_buffer_latest_submission_index(&buffer); + + if let Some(last_submission) = last_submission { + device.wait_for_submit(last_submission) + } else { + Ok(()) + } } #[doc(hidden)] @@ -424,7 +433,13 @@ impl Global { ); if wait { - let last_submit_index = buffer.submission_index(); + let Some(last_submit_index) = buffer + .device + .lock_life() + .get_buffer_latest_submission_index(&buffer) + else { + return; + }; match buffer.device.wait_for_submit(last_submit_index) { Ok(()) => (), Err(e) => log::error!("Failed to wait for buffer {:?}: {}", buffer_id, e), @@ -599,7 +614,13 @@ impl Global { } if wait { - let last_submit_index = texture.submission_index(); + let Some(last_submit_index) = texture + .device + .lock_life() + .get_texture_latest_submission_index(&texture) + else { + return; + }; match texture.device.wait_for_submit(last_submit_index) { Ok(()) => (), Err(e) => log::error!("Failed to wait for texture {texture_id:?}: {e}"), @@ -672,7 +693,13 @@ impl Global { } if wait { - let last_submit_index = view.submission_index(); + let Some(last_submit_index) = view + .device + .lock_life() + .get_texture_latest_submission_index(&view.parent) + else { + return Ok(()); + }; match view.device.wait_for_submit(last_submit_index) { Ok(()) => (), Err(e) => { diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 118e1498b..3696d8abe 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -5,7 +5,7 @@ use crate::{ }, hal_api::HalApi, id, - resource::{self, Buffer, Labeled, Trackable}, + resource::{self, Buffer, Labeled, Texture, Trackable}, snatch::SnatchGuard, SubmissionIndex, }; @@ -55,6 +55,58 @@ struct ActiveSubmission { work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>, } +impl ActiveSubmission { + /// Returns true if this submission contains the given buffer. + /// + /// This only uses constant-time operations. + pub fn contains_buffer(&self, buffer: &Buffer) -> bool { + for encoder in &self.encoders { + // The ownership location of buffers depends on where the command encoder + // came from. If it is the staging command encoder on the queue, it is + // in the pending buffer list. If it came from a user command encoder, + // it is in the tracker. + + if encoder.trackers.buffers.contains(buffer) { + return true; + } + + if encoder + .pending_buffers + .contains_key(&buffer.tracker_index()) + { + return true; + } + } + + false + } + + /// Returns true if this submission contains the given texture. + /// + /// This only uses constant-time operations. + pub fn contains_texture(&self, texture: &Texture) -> bool { + for encoder in &self.encoders { + // The ownership location of textures depends on where the command encoder + // came from. If it is the staging command encoder on the queue, it is + // in the pending buffer list. If it came from a user command encoder, + // it is in the tracker. + + if encoder.trackers.textures.contains(texture) { + return true; + } + + if encoder + .pending_textures + .contains_key(&texture.tracker_index()) + { + return true; + } + } + + false + } +} + #[derive(Clone, Debug, Error)] #[non_exhaustive] pub enum WaitIdleError { @@ -165,6 +217,40 @@ impl LifetimeTracker { self.mapped.push(value.clone()); } + /// Returns the submission index of the most recent submission that uses the + /// given buffer. + pub fn get_buffer_latest_submission_index( + &self, + buffer: &Buffer, + ) -> Option { + // We iterate in reverse order, so that we can bail out early as soon + // as we find a hit. + self.active.iter().rev().find_map(|submission| { + if submission.contains_buffer(buffer) { + Some(submission.index) + } else { + None + } + }) + } + + /// Returns the submission index of the most recent submission that uses the + /// given texture. + pub fn get_texture_latest_submission_index( + &self, + texture: &Texture, + ) -> Option { + // We iterate in reverse order, so that we can bail out early as soon + // as we find a hit. + self.active.iter().rev().find_map(|submission| { + if submission.contains_texture(texture) { + Some(submission.index) + } else { + None + } + }) + } + /// Sort out the consequences of completed submissions. /// /// Assume that all submissions up through `last_done` have completed. @@ -236,9 +322,7 @@ impl LifetimeTracker { } } } -} -impl LifetimeTracker { /// Determine which buffers are ready to map, and which must wait for the /// GPU. /// @@ -249,17 +333,19 @@ impl LifetimeTracker { } for buffer in self.mapped.drain(..) { - let submit_index = buffer.submission_index(); + let submission = self + .active + .iter_mut() + .rev() + .find(|a| a.contains_buffer(&buffer)); + log::trace!( - "Mapping of {} at submission {:?} gets assigned to active {:?}", + "Mapping of {} at submission {:?}", buffer.error_ident(), - submit_index, - self.active.iter().position(|a| a.index == submit_index) + submission.as_deref().map(|s| s.index) ); - self.active - .iter_mut() - .find(|a| a.index == submit_index) + submission .map_or(&mut self.ready_to_map, |a| &mut a.mapped) .push(buffer); } diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 9f138594d..220085f8f 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -149,12 +149,12 @@ pub enum TempResource { pub(crate) struct EncoderInFlight { raw: A::CommandEncoder, cmd_buffers: Vec, - trackers: Tracker, + pub(crate) trackers: Tracker, /// These are the buffers that have been tracked by `PendingWrites`. - pending_buffers: Vec>>, + pub(crate) pending_buffers: FastHashMap>>, /// These are the textures that have been tracked by `PendingWrites`. - pending_textures: Vec>>, + pub(crate) pending_textures: FastHashMap>>, } impl EncoderInFlight { @@ -268,8 +268,8 @@ impl PendingWrites { queue: &A::Queue, ) -> Result>, DeviceError> { if self.is_recording { - let pending_buffers = self.dst_buffers.drain().map(|(_, b)| b).collect(); - let pending_textures = self.dst_textures.drain().map(|(_, t)| t).collect(); + let pending_buffers = mem::take(&mut self.dst_buffers); + let pending_textures = mem::take(&mut self.dst_textures); let cmd_buf = unsafe { self.command_encoder.end_encoding()? }; self.is_recording = false; @@ -570,8 +570,6 @@ impl Global { self.queue_validate_write_buffer_impl(&dst, buffer_offset, staging_buffer.size)?; - dst.use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); - let region = hal::BufferCopy { src_offset: 0, dst_offset: buffer_offset, @@ -762,7 +760,6 @@ impl Global { // call above. Since we've held `texture_guard` the whole time, we know // the texture hasn't gone away in the mean time, so we can unwrap. let dst = hub.textures.get(destination.texture).unwrap(); - dst.use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); let dst_raw = dst.try_raw(&snatch_guard)?; @@ -1007,7 +1004,6 @@ impl Global { .drain(init_layer_range); } } - dst.use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); let snatch_guard = device.snatchable_lock.read(); let dst_raw = dst.try_raw(&snatch_guard)?; @@ -1126,7 +1122,7 @@ impl Global { } { - profiling::scope!("update submission ids"); + profiling::scope!("check resource state"); let cmd_buf_data = cmdbuf.data.lock(); let cmd_buf_trackers = &cmd_buf_data.as_ref().unwrap().trackers; @@ -1136,7 +1132,6 @@ impl Global { profiling::scope!("buffers"); for buffer in cmd_buf_trackers.buffers.used_resources() { buffer.check_destroyed(&snatch_guard)?; - buffer.use_at(submit_index); match *buffer.map_state.lock() { BufferMapState::Idle => (), @@ -1163,7 +1158,6 @@ impl Global { true } }; - texture.use_at(submit_index); if should_extend { unsafe { used_surface_textures @@ -1177,69 +1171,6 @@ impl Global { } } } - { - profiling::scope!("views"); - for texture_view in cmd_buf_trackers.views.used_resources() { - texture_view.use_at(submit_index); - } - } - { - profiling::scope!("bind groups (+ referenced views/samplers)"); - for bg in cmd_buf_trackers.bind_groups.used_resources() { - bg.use_at(submit_index); - // We need to update the submission indices for the contained - // state-less (!) resources as well, so that they don't get - // deleted too early if the parent bind group goes out of scope. - for view in bg.used.views.used_resources() { - view.use_at(submit_index); - } - for sampler in bg.used.samplers.used_resources() { - sampler.use_at(submit_index); - } - } - } - { - profiling::scope!("compute pipelines"); - for compute_pipeline in - cmd_buf_trackers.compute_pipelines.used_resources() - { - compute_pipeline.use_at(submit_index); - } - } - { - profiling::scope!("render pipelines"); - for render_pipeline in - cmd_buf_trackers.render_pipelines.used_resources() - { - render_pipeline.use_at(submit_index); - } - } - { - profiling::scope!("query sets"); - for query_set in cmd_buf_trackers.query_sets.used_resources() { - query_set.use_at(submit_index); - } - } - { - profiling::scope!( - "render bundles (+ referenced pipelines/query sets)" - ); - for bundle in cmd_buf_trackers.bundles.used_resources() { - bundle.use_at(submit_index); - // We need to update the submission indices for the contained - // state-less (!) resources as well, excluding the bind groups. - // They don't get deleted too early if the bundle goes out of scope. - for render_pipeline in - bundle.used.render_pipelines.read().used_resources() - { - render_pipeline.use_at(submit_index); - } - for query_set in bundle.used.query_sets.read().used_resources() - { - query_set.use_at(submit_index); - } - } - } } let mut baked = cmdbuf.from_arc_into_baked(); @@ -1303,8 +1234,8 @@ impl Global { raw: baked.encoder, cmd_buffers: baked.list, trackers: baked.trackers, - pending_buffers: Vec::new(), - pending_textures: Vec::new(), + pending_buffers: FastHashMap::default(), + pending_textures: FastHashMap::default(), }); } diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 6070089e2..4e94f1731 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -14,7 +14,7 @@ use crate::{ resource_log, snatch::{ExclusiveSnatchGuard, SnatchGuard, Snatchable}, track::{SharedTrackerIndexAllocator, TextureSelector, TrackerIndex}, - Label, LabelHelpers, SubmissionIndex, + Label, LabelHelpers, }; use hal::CommandEncoder; @@ -28,7 +28,7 @@ use std::{ mem::{self, ManuallyDrop}, ops::Range, ptr::NonNull, - sync::{atomic::Ordering, Arc, Weak}, + sync::{Arc, Weak}, }; /// Information about the wgpu-core resource. @@ -54,14 +54,6 @@ use std::{ pub(crate) struct TrackingData { tracker_index: TrackerIndex, tracker_indices: Arc, - /// The index of the last queue submission in which the resource - /// was used. - /// - /// Each queue submission is fenced and assigned an index number - /// sequentially. Thus, when a queue submission completes, we know any - /// resources used in that submission and any lower-numbered submissions are - /// no longer in use by the GPU. - submission_index: hal::AtomicFenceValue, } impl Drop for TrackingData { @@ -75,23 +67,12 @@ impl TrackingData { Self { tracker_index: tracker_indices.alloc(), tracker_indices, - submission_index: hal::AtomicFenceValue::new(0), } } pub(crate) fn tracker_index(&self) -> TrackerIndex { self.tracker_index } - - /// Record that this resource will be used by the queue submission with the - /// given index. - pub(crate) fn use_at(&self, submit_index: SubmissionIndex) { - self.submission_index.store(submit_index, Ordering::Release); - } - - pub(crate) fn submission_index(&self) -> SubmissionIndex { - self.submission_index.load(Ordering::Acquire) - } } #[derive(Clone, Debug)] @@ -193,10 +174,6 @@ macro_rules! impl_labeled { pub(crate) trait Trackable: Labeled { fn tracker_index(&self) -> TrackerIndex; - /// Record that this resource will be used by the queue submission with the - /// given index. - fn use_at(&self, submit_index: SubmissionIndex); - fn submission_index(&self) -> SubmissionIndex; } #[macro_export] @@ -206,12 +183,6 @@ macro_rules! impl_trackable { fn tracker_index(&self) -> $crate::track::TrackerIndex { self.tracking_data.tracker_index() } - fn use_at(&self, submit_index: $crate::SubmissionIndex) { - self.tracking_data.use_at(submit_index) - } - fn submission_index(&self) -> $crate::SubmissionIndex { - self.tracking_data.submission_index() - } } }; } @@ -660,7 +631,6 @@ impl Buffer { let staging_buffer = staging_buffer.flush(); - self.use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); let region = wgt::BufferSize::new(self.size).map(|size| hal::BufferCopy { src_offset: 0, dst_offset: 0, @@ -748,10 +718,11 @@ impl Buffer { if pending_writes.contains_buffer(self) { pending_writes.consume_temp(temp); } else { - let last_submit_index = self.submission_index(); - device - .lock_life() - .schedule_resource_destruction(temp, last_submit_index); + let mut life_lock = device.lock_life(); + let last_submit_index = life_lock.get_buffer_latest_submission_index(self); + if let Some(last_submit_index) = last_submit_index { + life_lock.schedule_resource_destruction(temp, last_submit_index); + } } Ok(()) @@ -1211,10 +1182,11 @@ impl Texture { if pending_writes.contains_texture(self) { pending_writes.consume_temp(temp); } else { - let last_submit_index = self.submission_index(); - device - .lock_life() - .schedule_resource_destruction(temp, last_submit_index); + let mut life_lock = device.lock_life(); + let last_submit_index = life_lock.get_texture_latest_submission_index(self); + if let Some(last_submit_index) = last_submit_index { + life_lock.schedule_resource_destruction(temp, last_submit_index); + } } Ok(()) diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index a7ec8201f..afb20e149 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -277,6 +277,11 @@ impl BufferTracker { } } + /// Returns true if the given buffer is tracked. + pub fn contains(&self, buffer: &Buffer) -> bool { + self.metadata.contains(buffer.tracker_index().as_usize()) + } + /// Returns a list of all buffers tracked. pub fn used_resources(&self) -> impl Iterator>> + '_ { self.metadata.owned_resources() diff --git a/wgpu-core/src/track/metadata.rs b/wgpu-core/src/track/metadata.rs index d7d63f04f..855282d72 100644 --- a/wgpu-core/src/track/metadata.rs +++ b/wgpu-core/src/track/metadata.rs @@ -67,7 +67,7 @@ impl ResourceMetadata { /// Returns true if the set contains the resource with the given index. pub(super) fn contains(&self, index: usize) -> bool { - self.owned[index] + self.owned.get(index).unwrap_or(false) } /// Returns true if the set contains the resource with the given index. diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 06779540d..7d8d904d2 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -34,12 +34,6 @@ impl StatelessBindGroupState { resources.sort_unstable_by_key(|resource| resource.tracker_index()); } - /// Returns a list of all resources tracked. May contain duplicates. - pub fn used_resources(&self) -> impl Iterator> + '_ { - let resources = self.resources.lock(); - resources.iter().cloned().collect::>().into_iter() - } - /// Adds the given resource. pub fn add_single(&self, resource: &Arc) { let mut resources = self.resources.lock(); @@ -79,11 +73,6 @@ impl StatelessTracker { } } - /// Returns a list of all resources tracked. - pub fn used_resources(&self) -> impl Iterator> + '_ { - self.metadata.owned_resources() - } - /// Inserts a single resource into the resource tracker. /// /// If the resource already exists in the tracker, it will be overwritten. diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index d34f47e12..bad216db1 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -446,6 +446,11 @@ impl TextureTracker { } } + /// Returns true if the tracker owns the given texture. + pub fn contains(&self, texture: &Texture) -> bool { + self.metadata.contains(texture.tracker_index().as_usize()) + } + /// Returns a list of all textures tracked. pub fn used_resources(&self) -> impl Iterator>> + '_ { self.metadata.owned_resources()