Add image comparison to all examples

.gitignore

@@ -17,4 +17,4 @@
 wgpu/red.png
 
 # Output from invalid comparison tests
-**/screenshot-difference.png
+**/*-difference.png

wgpu/Cargo.toml

@@ -87,6 +87,26 @@ git = "https://github.com/gfx-rs/naga"
 tag = "gfx-26"
 features = ["wgsl-in", "spv-out"]
 
+[[example]]
+name="boids"
+path="examples/boids/main.rs"
+test = true
+
+[[example]]
+name="bunnymark"
+path="examples/bunnymark/main.rs"
+test = true
+
+[[example]]
+name="conservative-raster"
+path="examples/conservative-raster/main.rs"
+test = true
+
+[[example]]
+name="cube"
+path="examples/cube/main.rs"
+test = true
+
 [[example]]
 name="hello-compute"
 test = true
@@ -99,6 +119,31 @@ required-features = ["spirv"]
 name="mipmap"
 test = true
 
+[[example]]
+name="msaa-line"
+path="examples/msaa-line/main.rs"
+test = true
+
+[[example]]
+name="shadow"
+path="examples/shadow/main.rs"
+test = true
+
+[[example]]
+name="skybox"
+path="examples/skybox/main.rs"
+test = true
+
+[[example]]
+name="texture-arrays"
+path="examples/texture-arrays/main.rs"
+test = true
+
+[[example]]
+name="water"
+path="examples/water/main.rs"
+test = true
+
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen = "0.2.73" # remember to change version in wiki as well
 web-sys = { version = "=0.3.50", features = [

wgpu/examples/boids/main.rs

@@ -1,7 +1,10 @@
 // Flocking boids example with gpu compute update pass
 // adapted from https://github.com/austinEng/webgpu-samples/blob/master/src/examples/computeBoids.ts
 
-use rand::distributions::{Distribution, Uniform};
+use rand::{
+    distributions::{Distribution, Uniform},
+    SeedableRng,
+};
 use std::{borrow::Cow, mem};
 use wgpu::util::DeviceExt;
 
@@ -168,7 +171,7 @@ impl framework::Example for Example {
         // buffer for all particles data of type [(posx,posy,velx,vely),...]
 
         let mut initial_particle_data = vec![0.0f32; (4 * NUM_PARTICLES) as usize];
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rngs::StdRng::seed_from_u64(42);
         let unif = Uniform::new_inclusive(-1.0, 1.0);
         for particle_instance_chunk in initial_particle_data.chunks_mut(4) {
             particle_instance_chunk[0] = unif.sample(&mut rng); // posx
@@ -314,3 +317,16 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("boids");
 }
+
+#[test]
+fn boids() {
+    framework::test::<Example>(
+        concat!(env!("CARGO_MANIFEST_DIR"), "/examples/boids/screenshot.png"),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        0,
+        50,
+    );
+}

wgpu/examples/bunnymark/main.rs

@@ -349,3 +349,19 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("bunnymark");
 }
+
+#[test]
+fn bunnymark() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/bunnymark/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        0,
+        50,
+    );
+}

wgpu/examples/conservative-raster/main.rs

@@ -73,7 +73,7 @@ impl framework::Example for Example {
     }
     fn init(
         sc_desc: &wgpu::SwapChainDescriptor,
-        adapter: &wgpu::Adapter,
+        _adapter: &wgpu::Adapter,
         device: &wgpu::Device,
         _queue: &wgpu::Queue,
     ) -> Self {
@@ -133,7 +133,7 @@ impl framework::Example for Example {
                 multisample: wgpu::MultisampleState::default(),
             });
 
-        let pipeline_lines = if adapter
+        let pipeline_lines = if device
             .features()
             .contains(wgpu::Features::NON_FILL_POLYGON_MODE)
         {
@@ -312,3 +312,19 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("conservative-raster");
 }
+
+#[test]
+fn conservative_raster() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/conservative-raster/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        0,
+        0,
+    );
+}

wgpu/examples/cube/main.rs

@@ -379,3 +379,32 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("cube");
 }
+
+#[test]
+fn cube() {
+    framework::test::<Example>(
+        concat!(env!("CARGO_MANIFEST_DIR"), "/examples/cube/screenshot.png"),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        1,
+        3,
+    );
+}
+
+#[test]
+fn cube_lines() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/cube/screenshot-lines.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::NON_FILL_POLYGON_MODE,
+        framework::test_common::TestParameters::default(),
+        1,
+        3,
+    );
+}

wgpu/examples/framework.rs

@@ -1,13 +1,13 @@
-use std::future::Future;
 #[cfg(not(target_arch = "wasm32"))]
 use std::time::{Duration, Instant};
+use std::{future::Future};
 use winit::{
     event::{self, WindowEvent},
     event_loop::{ControlFlow, EventLoop},
 };
 
 #[path = "../tests/common/mod.rs"]
-mod test_common;
+pub mod test_common;
 
 #[rustfmt::skip]
 #[allow(unused)]
@@ -365,20 +365,28 @@ pub fn run<E: Example>(title: &str) {
 }
 
 #[cfg(test)]
-pub fn test<E: Example>(image_path: &str, width: u32, height: u32, tollerance: u8, max_outliers: usize) {
+#[allow(dead_code)]
+pub fn test<E: Example>(
+    image_path: &str,
+    width: u32,
+    height: u32,
+    optional_features: wgpu::Features,
+    base_test_parameters: test_common::TestParameters,
+    tollerance: u8,
+    max_outliers: usize,
+) {
     use std::num::NonZeroU32;
 
     assert_eq!(width % 64, 0, "width needs to be aligned 64");
 
-    let _optional = E::optional_features();
+    let features = E::required_features() | optional_features;
-    let features = E::required_features();
     let mut limits = E::required_limits();
     if limits == wgpu::Limits::default() {
         limits = test_common::lowest_reasonable_limits();
     }
 
     test_common::initialize_test(
-        test_common::TestParameters::default()
+        base_test_parameters
             .features(features)
             .limits(limits),
         |ctx| {

wgpu/examples/mipmap/main.rs

@@ -484,7 +484,9 @@ fn mipmap() {
         ),
         1024,
         768,
-        10, // Mipmap sampling is highly variant between impls.
+        wgpu::Features::default(),
-        10,
+        framework::test_common::TestParameters::default(),
+        15, // Mipmap sampling is highly variant between impls. This is currently bounded by the M1
+        40,
     );
 }

wgpu/examples/msaa-line/main.rs

@@ -282,3 +282,19 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("msaa-line");
 }
+
+#[test]
+fn msaa_line() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/msaa-line/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        0,
+        1 << 15, // MSAA is comically different between vendors, 32k is a decent limit
+    );
+}

wgpu/examples/shadow/main.rs

@@ -821,3 +821,19 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("shadow");
 }
+
+#[test]
+fn shadow() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/shadow/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        1,
+        1,
+    );
+}

wgpu/examples/skybox/main.rs

@@ -459,3 +459,67 @@ impl framework::Example for Skybox {
 fn main() {
     framework::run::<Skybox>("skybox");
 }
+
+#[test]
+fn skybox() {
+    framework::test::<Skybox>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/skybox/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        2,
+        3,
+    );
+}
+
+#[test]
+fn skybox_bc1() {
+    framework::test::<Skybox>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/skybox/screenshot-bc1.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::TEXTURE_COMPRESSION_BC,
+        framework::test_common::TestParameters::default(),
+        4,
+        0,
+    );
+}
+
+#[test]
+fn skybox_etc2() {
+    framework::test::<Skybox>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/skybox/screenshot-etc2.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::TEXTURE_COMPRESSION_ETC2,
+        framework::test_common::TestParameters::default(),
+        1, // TODO
+        1, // TODO
+    );
+}
+
+#[test]
+fn skybox_astc() {
+    framework::test::<Skybox>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/skybox/screenshot-astc.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::TEXTURE_COMPRESSION_ASTC_LDR,
+        framework::test_common::TestParameters::default(),
+        1, // TODO
+        1, // TODO
+    );
+}

wgpu/examples/skybox/shader.wgsl

@@ -73,6 +73,6 @@ fn fs_entity(in: EntityOutput) -> [[location(0)]] vec4<f32> {
     let normal = normalize(in.normal);
     let reflected = incident - 2.0 * dot(normal, incident) * normal;
 
-    let reflected_color = textureSample(r_texture, r_sampler, reflected);
+    let reflected_color = textureSample(r_texture, r_sampler, reflected).rgb;
-    return vec4<f32>(0.1, 0.1, 0.1, 0.1) + 0.5 * reflected_color;
+    return vec4<f32>(vec3<f32>(0.1) + 0.5 * reflected_color, 1.0);
 }

wgpu/examples/texture-arrays/main.rs

@@ -100,7 +100,7 @@ impl framework::Example for Example {
             f if f.contains(wgpu::Features::SAMPLED_TEXTURE_ARRAY_NON_UNIFORM_INDEXING) => {
                 wgpu::include_spirv_raw!("non-uniform.frag.spv")
             }
-            f if f.contains(wgpu::Features::SAMPLED_TEXTURE_ARRAY_DYNAMIC_INDEXING) => {
+            f if f.contains(wgpu::Features::SAMPLED_TEXTURE_ARRAY_DYNAMIC_INDEXING | wgpu::Features::PUSH_CONSTANTS) => {
                 uniform_workaround = true;
                 wgpu::include_spirv_raw!("uniform.frag.spv")
             }
@@ -322,3 +322,73 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("texture-arrays");
 }
+
+// This fails due to an issue with naga https://github.com/gfx-rs/wgpu/issues/1532
+#[test]
+fn texture_arrays_constant() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/texture-arrays/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default().failure(),
+        0,
+        0,
+    );
+}
+
+// This fails due to an issue with naga https://github.com/gfx-rs/wgpu/issues/1532
+#[test]
+fn texture_arrays_uniform() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/texture-arrays/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::SAMPLED_TEXTURE_ARRAY_DYNAMIC_INDEXING | wgpu::Features::PUSH_CONSTANTS,
+        framework::test_common::TestParameters::default().failure(),
+        0,
+        0,
+    );
+}
+
+
+// This fails due to an issue with naga https://github.com/gfx-rs/wgpu/issues/1532
+#[test]
+fn texture_arrays_non_uniform() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/texture-arrays/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::SAMPLED_TEXTURE_ARRAY_NON_UNIFORM_INDEXING,
+        framework::test_common::TestParameters::default().failure(),
+        0,
+        0,
+    );
+}
+
+
+// This fails due to an issue with naga https://github.com/gfx-rs/wgpu/issues/1532
+#[test]
+fn texture_arrays_unsized_non_uniform() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/texture-arrays/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::SAMPLED_TEXTURE_ARRAY_NON_UNIFORM_INDEXING | wgpu::Features::UNSIZED_BINDING_ARRAY,
+        framework::test_common::TestParameters::default().failure(),
+        0,
+        0,
+    );
+}

wgpu/examples/water/main.rs

@@ -5,6 +5,7 @@ mod point_gen;
 
 use bytemuck::{Pod, Zeroable};
 use cgmath::Point3;
+use rand::SeedableRng;
 use std::{borrow::Cow, iter, mem};
 use wgpu::util::DeviceExt;
 
@@ -279,7 +280,7 @@ impl framework::Example for Example {
         let terrain_noise = noise::OpenSimplex::new();
 
         // Random colouration
-        let mut terrain_random = rand::thread_rng();
+        let mut terrain_random = rand::rngs::StdRng::seed_from_u64(42);
 
         // Generate terrain. The closure determines what each hexagon will look like.
         let terrain =
@@ -788,3 +789,19 @@ impl framework::Example for Example {
 fn main() {
     framework::run::<Example>("water");
 }
+
+#[test]
+fn shadow() {
+    framework::test::<Example>(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/examples/water/screenshot.png"
+        ),
+        1024,
+        768,
+        wgpu::Features::default(),
+        framework::test_common::TestParameters::default(),
+        2,
+        2,
+    );
+}

wgpu/tests/common/image.rs

@@ -100,7 +100,7 @@ pub fn compare_image_output(
             .max()
             .unwrap();
 
-        if outliers >= max_outliers {
+        if outliers > max_outliers {
             // Because the deta is mismatched, lets output the difference to a file.
             let old_path = Path::new(&path);
             let difference_path = Path::new(&path).with_file_name(

wgpu/tests/common/mod.rs

@@ -46,18 +46,18 @@ pub fn lowest_reasonable_limits() -> Limits {
         max_texture_dimension_2d: 1024,
         max_texture_dimension_3d: 32,
         max_texture_array_layers: 32,
-        max_bind_groups: 1,
+        max_bind_groups: 2,
-        max_dynamic_uniform_buffers_per_pipeline_layout: 1,
+        max_dynamic_uniform_buffers_per_pipeline_layout: 2,
-        max_dynamic_storage_buffers_per_pipeline_layout: 1,
+        max_dynamic_storage_buffers_per_pipeline_layout: 2,
-        max_sampled_textures_per_shader_stage: 1,
+        max_sampled_textures_per_shader_stage:2,
-        max_samplers_per_shader_stage: 1,
+        max_samplers_per_shader_stage: 2,
-        max_storage_buffers_per_shader_stage: 1,
+        max_storage_buffers_per_shader_stage: 2,
-        max_storage_textures_per_shader_stage: 1,
+        max_storage_textures_per_shader_stage: 2,
-        max_uniform_buffers_per_shader_stage: 1,
+        max_uniform_buffers_per_shader_stage: 2,
         max_uniform_buffer_binding_size: 256,
-        max_storage_buffer_binding_size: 256,
+        max_storage_buffer_binding_size: 1 << 16,
-        max_vertex_buffers: 1,
+        max_vertex_buffers: 4,
-        max_vertex_attributes: 2,
+        max_vertex_attributes: 4,
         max_vertex_buffer_array_stride: 32,
         max_push_constant_size: 0,
     }