From 6bff395680424f0ff86e22007987c5a48dc309e1 Mon Sep 17 00:00:00 2001 From: Ashley Hauck Date: Fri, 21 May 2021 14:09:56 +0200 Subject: [PATCH] Build the wgpu shader at runtime (#627) * Build the wgpu shader at runtime --- .github/workflows/ci.yaml | 1 + Cargo.lock | 7 +++ Cargo.toml | 1 + crates/rustc_codegen_spirv/build.rs | 8 ---- crates/rustc_codegen_spirv/src/lib.rs | 11 +++++ examples/runners/wgpu/Cargo.toml | 2 +- examples/runners/wgpu/build.rs | 57 +++++++++++++++++++---- examples/runners/wgpu/builder/Cargo.toml | 16 +++++++ examples/runners/wgpu/builder/src/main.rs | 15 ++++++ examples/runners/wgpu/src/compute.rs | 4 +- examples/runners/wgpu/src/graphics.rs | 9 ++-- examples/runners/wgpu/src/lib.rs | 49 +++++++++++++++++++ 12 files changed, 157 insertions(+), 23 deletions(-) delete mode 100644 crates/rustc_codegen_spirv/build.rs create mode 100644 examples/runners/wgpu/builder/Cargo.toml create mode 100644 examples/runners/wgpu/builder/src/main.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 899f6b0445..44706649d1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,6 +23,7 @@ jobs: runs-on: ${{ matrix.os }} env: spirv_tools_version: "20200928" + RUSTUP_UNPACK_RAM: "104857600" steps: - uses: actions/checkout@v2 # Ubuntu does have `brew install spirv-tools`, but it installs from diff --git a/Cargo.lock b/Cargo.lock index 1c182eb9aa..e4af13a098 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,6 +783,13 @@ dependencies = [ "winit", ] +[[package]] +name = "example-runner-wgpu-builder" +version = "0.4.0-alpha.7" +dependencies = [ + "spirv-builder", +] + [[package]] name = "filetime" version = "0.2.14" diff --git a/Cargo.toml b/Cargo.toml index 5ad386ddeb..8fa5f448c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "examples/runners/cpu", "examples/runners/ash", "examples/runners/wgpu", + "examples/runners/wgpu/builder", "examples/shaders/sky-shader", "examples/shaders/simplest-shader", "examples/shaders/compute-shader", diff --git a/crates/rustc_codegen_spirv/build.rs b/crates/rustc_codegen_spirv/build.rs deleted file mode 100644 index 14d3a9efad..0000000000 --- a/crates/rustc_codegen_spirv/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -// Putting this check here causes compilation failure seconds into the build, -// putting it in lib.rs fails after minutes because spirv-tools gets compiled first. -#[cfg(all(feature = "use-compiled-tools", feature = "use-installed-tools"))] -compile_error!( - "Either \"use-compiled-tools\" (enabled by default) or \"use-installed-tools\" may be enabled." -); - -fn main() {} diff --git a/crates/rustc_codegen_spirv/src/lib.rs b/crates/rustc_codegen_spirv/src/lib.rs index 0f3204923a..59606c40ce 100644 --- a/crates/rustc_codegen_spirv/src/lib.rs +++ b/crates/rustc_codegen_spirv/src/lib.rs @@ -80,6 +80,17 @@ )] #![deny(clippy::unimplemented, clippy::ok_expect)] +// Unfortunately, this will not fail fast when compiling, but rather will wait for +// rustc_codegen_spirv to be compiled. Putting this in build.rs will solve that problem, however, +// that creates the much worse problem that then running `cargo check` will cause +// rustc_codegen_spirv to be *compiled* instead of merely checked, something that takes +// significantly longer. So, the trade-off between detecting a bad configuration slower for a +// faster `cargo check` is worth it. +#[cfg(all(feature = "use-compiled-tools", feature = "use-installed-tools"))] +compile_error!( + "Either \"use-compiled-tools\" (enabled by default) or \"use-installed-tools\" may be enabled." +); + extern crate rustc_ast; extern crate rustc_attr; extern crate rustc_codegen_ssa; diff --git a/examples/runners/wgpu/Cargo.toml b/examples/runners/wgpu/Cargo.toml index 4ba9ec5d3a..06b3b0810f 100644 --- a/examples/runners/wgpu/Cargo.toml +++ b/examples/runners/wgpu/Cargo.toml @@ -24,7 +24,7 @@ winit = { version = "0.24", features = ["web-sys"] } clap = "3.0.0-beta.2" strum = { version = "0.20", default_features = false, features = ["derive"] } -[build-dependencies] +[target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] spirv-builder = { path = "../../../crates/spirv-builder", default-features = false } [target.'cfg(target_os = "android")'.dependencies] diff --git a/examples/runners/wgpu/build.rs b/examples/runners/wgpu/build.rs index da88425a60..e0d7c97cae 100644 --- a/examples/runners/wgpu/build.rs +++ b/examples/runners/wgpu/build.rs @@ -1,15 +1,52 @@ -use spirv_builder::SpirvBuilder; +use std::env; use std::error::Error; - -fn build_shader(path_to_create: &str) -> Result<(), Box> { - SpirvBuilder::new(path_to_create, "spirv-unknown-vulkan1.0").build()?; - Ok(()) -} +use std::path::PathBuf; fn main() -> Result<(), Box> { - build_shader("../../shaders/sky-shader")?; - build_shader("../../shaders/simplest-shader")?; - build_shader("../../shaders/compute-shader")?; - build_shader("../../shaders/mouse-shader")?; + let target_os = std::env::var("CARGO_CFG_TARGET_OS")?; + let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH")?; + // Always build on CI to make sure the shaders can still build + let is_on_ci = std::env::var("CI"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS"); + println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_ARCH"); + println!("cargo:rerun-if-env-changed=CI"); + // While OUT_DIR is set for both build.rs and compiling the crate, PROFILE is only set in + // build.rs. So, export it to crate compilation as well. + let profile = env::var("PROFILE").unwrap(); + println!("cargo:rustc-env=PROFILE={}", profile); + if target_os != "android" && target_arch != "wasm32" && is_on_ci.is_err() { + return Ok(()); + } + let mut dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + // Strip `$profile/build/*/out`. + let ok = dir.ends_with("out") + && dir.pop() + && dir.pop() + && dir.ends_with("build") + && dir.pop() + && dir.ends_with(profile) + && dir.pop(); + assert!(ok); + let dir = dir.join("spirv-builder"); + let status = std::process::Command::new("cargo") + .args([ + "run", + "--release", + "-p", + "example-runner-wgpu-builder", + "--target-dir", + ]) + .arg(dir) + .stderr(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .status()?; + if !status.success() { + if let Some(code) = status.code() { + std::process::exit(code); + } else { + std::process::exit(1); + } + } Ok(()) } diff --git a/examples/runners/wgpu/builder/Cargo.toml b/examples/runners/wgpu/builder/Cargo.toml new file mode 100644 index 0000000000..cce2720add --- /dev/null +++ b/examples/runners/wgpu/builder/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "example-runner-wgpu-builder" +version = "0.4.0-alpha.7" +authors = ["Embark "] +edition = "2018" +license = "MIT OR Apache-2.0" +publish = false + +# See rustc_codegen_spirv/Cargo.toml for details on these features +[features] +default = ["use-compiled-tools"] +use-installed-tools = ["spirv-builder/use-installed-tools"] +use-compiled-tools = ["spirv-builder/use-compiled-tools"] + +[dependencies] +spirv-builder = { path = "../../../../crates/spirv-builder", default-features = false } diff --git a/examples/runners/wgpu/builder/src/main.rs b/examples/runners/wgpu/builder/src/main.rs new file mode 100644 index 0000000000..da88425a60 --- /dev/null +++ b/examples/runners/wgpu/builder/src/main.rs @@ -0,0 +1,15 @@ +use spirv_builder::SpirvBuilder; +use std::error::Error; + +fn build_shader(path_to_create: &str) -> Result<(), Box> { + SpirvBuilder::new(path_to_create, "spirv-unknown-vulkan1.0").build()?; + Ok(()) +} + +fn main() -> Result<(), Box> { + build_shader("../../shaders/sky-shader")?; + build_shader("../../shaders/simplest-shader")?; + build_shader("../../shaders/compute-shader")?; + build_shader("../../shaders/mouse-shader")?; + Ok(()) +} diff --git a/examples/runners/wgpu/src/compute.rs b/examples/runners/wgpu/src/compute.rs index 308463ab31..947d81ef25 100644 --- a/examples/runners/wgpu/src/compute.rs +++ b/examples/runners/wgpu/src/compute.rs @@ -34,10 +34,12 @@ fn create_device_queue() -> (wgpu::Device, wgpu::Queue) { } pub fn start(options: &Options) { + let shader_binary = shader_module(options.shader); + let (device, queue) = create_device_queue(); // Load the shaders from disk - let module = device.create_shader_module(&shader_module(options.shader)); + let module = device.create_shader_module(&shader_binary); let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: None, diff --git a/examples/runners/wgpu/src/graphics.rs b/examples/runners/wgpu/src/graphics.rs index d76794cff1..7a84017205 100644 --- a/examples/runners/wgpu/src/graphics.rs +++ b/examples/runners/wgpu/src/graphics.rs @@ -20,10 +20,10 @@ fn mouse_button_index(button: MouseButton) -> usize { } async fn run( - options: &Options, event_loop: EventLoop<()>, window: Window, swapchain_format: wgpu::TextureFormat, + shader_binary: wgpu::ShaderModuleDescriptor<'_>, ) { let size = window.inner_size(); let instance = wgpu::Instance::new(wgpu::BackendBit::VULKAN | wgpu::BackendBit::METAL); @@ -65,7 +65,7 @@ async fn run( .expect("Failed to create device"); // Load the shaders from disk - let module = device.create_shader_module(&shader_module(options.shader)); + let module = device.create_shader_module(&shader_binary); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, @@ -273,6 +273,9 @@ async fn run( } pub fn start(options: &Options) { + // Build the shader before we pop open a window, since it might take a while. + let shader_binary = shader_module(options.shader); + let event_loop = EventLoop::new(); let window = winit::window::WindowBuilder::new() .with_title("Rust GPU - wgpu") @@ -303,7 +306,6 @@ pub fn start(options: &Options) { } else { wgpu_subscriber::initialize_default_subscriber(None); futures::executor::block_on(run( - options, event_loop, window, if cfg!(target_os = "android") { @@ -311,6 +313,7 @@ pub fn start(options: &Options) { } else { wgpu::TextureFormat::Bgra8UnormSrgb }, + shader_binary, )); } } diff --git a/examples/runners/wgpu/src/lib.rs b/examples/runners/wgpu/src/lib.rs index a6675fffc1..40d3e9e7bb 100644 --- a/examples/runners/wgpu/src/lib.rs +++ b/examples/runners/wgpu/src/lib.rs @@ -57,6 +57,55 @@ pub enum RustGPUShader { } fn shader_module(shader: RustGPUShader) -> wgpu::ShaderModuleDescriptor<'static> { + #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] + { + use spirv_builder::SpirvBuilder; + use std::borrow::Cow; + use std::path::{Path, PathBuf}; + // Hack: spirv_builder builds into a custom directory if running under cargo, to not + // deadlock, and the default target directory if not. However, packages like `proc-macro2` + // have different configurations when being built here vs. when building + // rustc_codegen_spirv normally, so we *want* to build into a separate target directory, to + // not have to rebuild half the crate graph every time we run. So, pretend we're running + // under cargo by setting these environment variables. + std::env::set_var("OUT_DIR", env!("OUT_DIR")); + std::env::set_var("PROFILE", env!("PROFILE")); + let crate_name = match shader { + RustGPUShader::Simplest => "sky-shader", + RustGPUShader::Sky => "simplest-shader", + RustGPUShader::Compute => "compute-shader", + RustGPUShader::Mouse => "mouse-shader", + }; + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let crate_path = [ + Path::new(manifest_dir), + Path::new(".."), + Path::new(".."), + Path::new("shaders"), + Path::new(crate_name), + ] + .iter() + .copied() + .collect::(); + let result = SpirvBuilder::new(crate_path, "spirv-unknown-vulkan1.0") + .print_metadata(false) + .build() + .unwrap(); + let data = std::fs::read(result).unwrap(); + let spirv = wgpu::util::make_spirv(&data); + let spirv = match spirv { + wgpu::ShaderSource::Wgsl(cow) => wgpu::ShaderSource::Wgsl(Cow::Owned(cow.into_owned())), + wgpu::ShaderSource::SpirV(cow) => { + wgpu::ShaderSource::SpirV(Cow::Owned(cow.into_owned())) + } + }; + wgpu::ShaderModuleDescriptor { + label: None, + source: spirv, + flags: wgpu::ShaderFlags::default(), + } + } + #[cfg(any(target_os = "android", target_arch = "wasm32"))] match shader { RustGPUShader::Simplest => wgpu::include_spirv!(env!("simplest_shader.spv")), RustGPUShader::Sky => wgpu::include_spirv!(env!("sky_shader.spv")),