render/vulkan: add dummy 3d lookup table to output shader

Later commits will add shader options that use a real 3d
lookup table.

include/render/vulkan.h

@@ -243,10 +243,22 @@ struct wlr_vk_renderer {
 
 	// for blend->output subpass
 	VkPipelineLayout output_pipe_layout;
-	VkDescriptorSetLayout output_ds_layout;
+	VkDescriptorSetLayout output_ds_srgb_layout;
+	VkDescriptorSetLayout output_ds_lut3d_layout;
+	VkSampler output_sampler_lut3d;
+	// descriptor set indicating dummy 1x1x1 image, for use in the lut3d slot
+	VkDescriptorSet output_ds_lut3d_dummy;
+	struct wlr_vk_descriptor_pool *output_ds_lut3d_dummy_pool;
+
 	size_t last_output_pool_size;
 	struct wl_list output_descriptor_pools; // wlr_vk_descriptor_pool.link
 
+	// dummy sampler to bind when output shader is not using a lookup table
+	VkImage dummy3d_image;
+	VkDeviceMemory dummy3d_mem;
+	VkImageView dummy3d_image_view;
+	bool dummy3d_image_transitioned;
+
 	VkSemaphore timeline_semaphore;
 	uint64_t timeline_point;
 
@@ -286,6 +298,11 @@ struct wlr_vk_vert_pcr_data {
 	float uv_size[2];
 };
 
+struct wlr_vk_frag_output_pcr_data {
+	float lut_3d_offset;
+	float lut_3d_scale;
+};
+
 struct wlr_vk_texture_view {
 	struct wl_list link; // struct wlr_vk_texture.views
 	const struct wlr_vk_pipeline_layout *layout;

render/vulkan/pass.c

@@ -103,14 +103,26 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 			.uv_off = { 0, 0 },
 			.uv_size = { 1, 1 },
 		};
+		struct wlr_vk_frag_output_pcr_data frag_pcr_data = {
+			.lut_3d_offset = 0.5f / 1,
+			.lut_3d_scale = (float)(1 - 1) / 1,
+		};
 		mat3_to_mat4(final_matrix, vert_pcr_data.mat4);
 
 		bind_pipeline(pass, render_buffer->render_setup->output_pipe);
 		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
 			VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
+		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
+			VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data),
+			sizeof(frag_pcr_data), &frag_pcr_data);
+		VkDescriptorSet ds[] = {
+			render_buffer->blend_descriptor_set, // set 0
+			renderer->output_ds_lut3d_dummy, // set 1
+		};
+		size_t ds_len = sizeof(ds) / sizeof(ds[0]);
 		vkCmdBindDescriptorSets(render_cb->vk,
 			VK_PIPELINE_BIND_POINT_GRAPHICS, renderer->output_pipe_layout,
-			0, 1, &render_buffer->blend_descriptor_set, 0, NULL);
+			0, ds_len, ds, 0, NULL);
 
 		const pixman_region32_t *clip = rect_union_evaluate(&pass->updated_region);
 		int clip_rects_len;
@@ -686,6 +698,14 @@ struct wlr_vk_render_pass *vulkan_begin_render_pass(struct wlr_vk_renderer *rend
 		return NULL;
 	}
 
+	if (!renderer->dummy3d_image_transitioned) {
+		renderer->dummy3d_image_transitioned = true;
+		vulkan_change_layout(cb->vk, renderer->dummy3d_image,
+			VK_IMAGE_LAYOUT_UNDEFINED, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+			0, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
+			VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT, VK_ACCESS_SHADER_READ_BIT);
+	}
+
 	int width = buffer->wlr_buffer->width;
 	int height = buffer->wlr_buffer->height;
 	VkRect2D rect = { .extent = { width, height } };

render/vulkan/renderer.c

@@ -138,7 +138,7 @@ struct wlr_vk_descriptor_pool *vulkan_alloc_texture_ds(
 struct wlr_vk_descriptor_pool *vulkan_alloc_blend_ds(
 	struct wlr_vk_renderer *renderer, VkDescriptorSet *ds) {
 	return alloc_ds(renderer, ds, VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT,
-		&renderer->output_ds_layout, &renderer->output_descriptor_pools,
+		&renderer->output_ds_srgb_layout, &renderer->output_descriptor_pools,
 		&renderer->last_output_pool_size);
 }
 
@@ -1045,10 +1045,20 @@ static void vulkan_destroy(struct wlr_renderer *wlr_renderer) {
 		vkDestroySamplerYcbcrConversion(dev->dev, pipeline_layout->ycbcr.conversion, NULL);
 	}
 
+	if (renderer->output_ds_lut3d_dummy_pool) {
+		vulkan_free_ds(renderer, renderer->output_ds_lut3d_dummy_pool,
+			renderer->output_ds_lut3d_dummy);
+	}
+	vkDestroyImageView(dev->dev, renderer->dummy3d_image_view, NULL);
+	vkDestroyImage(dev->dev, renderer->dummy3d_image, NULL);
+	vkFreeMemory(dev->dev, renderer->dummy3d_mem, NULL);
+
 	vkDestroySemaphore(dev->dev, renderer->timeline_semaphore, NULL);
 	vkDestroyPipelineLayout(dev->dev, renderer->output_pipe_layout, NULL);
-	vkDestroyDescriptorSetLayout(dev->dev, renderer->output_ds_layout, NULL);
+	vkDestroyDescriptorSetLayout(dev->dev, renderer->output_ds_srgb_layout, NULL);
+	vkDestroyDescriptorSetLayout(dev->dev, renderer->output_ds_lut3d_layout, NULL);
 	vkDestroyCommandPool(dev->dev, renderer->command_pool, NULL);
+	vkDestroySampler(dev->dev, renderer->output_sampler_lut3d, NULL);
 
 	if (renderer->read_pixels_cache.initialized) {
 		vkFreeMemory(dev->dev, renderer->read_pixels_cache.dst_img_memory, NULL);
@@ -1384,13 +1394,11 @@ static bool init_tex_layouts(struct wlr_vk_renderer *renderer,
 	return true;
 }
 
-static bool init_blend_to_output_layouts(struct wlr_vk_renderer *renderer,
+static bool init_blend_to_output_layouts(struct wlr_vk_renderer *renderer) {
-		VkDescriptorSetLayout *out_ds_layout,
-		VkPipelineLayout *out_pipe_layout) {
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
-	VkDescriptorSetLayoutBinding ds_binding = {
+	VkDescriptorSetLayoutBinding ds_binding_input = {
 		.binding = 0,
 		.descriptorType = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT,
 		.descriptorCount = 1,
@@ -1401,32 +1409,83 @@ static bool init_blend_to_output_layouts(struct wlr_vk_renderer *renderer,
 	VkDescriptorSetLayoutCreateInfo ds_info = {
 		.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 		.bindingCount = 1,
-		.pBindings = &ds_binding,
+		.pBindings = &ds_binding_input,
 	};
 
-	res = vkCreateDescriptorSetLayout(dev, &ds_info, NULL, out_ds_layout);
+	res = vkCreateDescriptorSetLayout(dev, &ds_info, NULL, &renderer->output_ds_srgb_layout);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateDescriptorSetLayout", res);
+		return false;
+	}
+
+	VkSamplerCreateInfo sampler_create_info = {
+		.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
+		.magFilter = VK_FILTER_LINEAR,
+		.minFilter = VK_FILTER_LINEAR,
+		.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
+		.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.minLod = 0.f,
+		.maxLod = 0.25f,
+	};
+
+	res = vkCreateSampler(renderer->dev->dev, &sampler_create_info, NULL,
+		&renderer->output_sampler_lut3d);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateSampler", res);
+		return false;
+	}
+
+	VkDescriptorSetLayoutBinding ds_binding_lut3d = {
+		.binding = 0,
+		.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+		.descriptorCount = 1,
+		.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
+		.pImmutableSamplers = &renderer->output_sampler_lut3d,
+	};
+
+	VkDescriptorSetLayoutCreateInfo ds_lut3d_info = {
+		.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+		.bindingCount = 1,
+		.pBindings = &ds_binding_lut3d,
+	};
+
+	res = vkCreateDescriptorSetLayout(dev, &ds_lut3d_info, NULL,
+		&renderer->output_ds_lut3d_layout);
 	if (res != VK_SUCCESS) {
 		wlr_vk_error("vkCreateDescriptorSetLayout", res);
 		return false;
 	}
 
 	// pipeline layout -- standard vertex uniforms, no shader uniforms
-	VkPushConstantRange pc_ranges[1] = {
+	VkPushConstantRange pc_ranges[2] = {
 		{
+			.offset = 0,
 			.size = sizeof(struct wlr_vk_vert_pcr_data),
 			.stageFlags = VK_SHADER_STAGE_VERTEX_BIT,
 		},
+		{
+			.offset = sizeof(struct wlr_vk_vert_pcr_data),
+			.size = sizeof(struct wlr_vk_frag_output_pcr_data),
+			.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
+		},
+	};
+
+	VkDescriptorSetLayout out_ds_layouts[2] = {
+		renderer->output_ds_srgb_layout,
+		renderer->output_ds_lut3d_layout,
 	};
 
 	VkPipelineLayoutCreateInfo pl_info = {
 		.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
-		.setLayoutCount = 1,
+		.setLayoutCount = 2,
-		.pSetLayouts = out_ds_layout,
+		.pSetLayouts = out_ds_layouts,
-		.pushConstantRangeCount = 1,
+		.pushConstantRangeCount = 2,
 		.pPushConstantRanges = pc_ranges,
 	};
 
-	res = vkCreatePipelineLayout(dev, &pl_info, NULL, out_pipe_layout);
+	res = vkCreatePipelineLayout(dev, &pl_info, NULL, &renderer->output_pipe_layout);
 	if (res != VK_SUCCESS) {
 		wlr_vk_error("vkCreatePipelineLayout", res);
 		return false;
@@ -1820,6 +1879,106 @@ struct wlr_vk_pipeline_layout *get_or_create_pipeline_layout(
 	return pipeline_layout;
 }
 
+
+/* The fragment shader for the blend->image subpass can be configured to either
+ * use or not a sampler3d lookup table; however, even if the shader does not use
+ * the sampler, a valid descriptor set should be bound. Create that here, linked to
+ * a 1x1x1 image.
+ */
+static bool init_dummy_images(struct wlr_vk_renderer *renderer) {
+	VkResult res;
+	VkDevice dev = renderer->dev->dev;
+
+	VkFormat format = VK_FORMAT_R32G32B32A32_SFLOAT;
+
+	VkImageCreateInfo img_info = {
+		.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+		.imageType = VK_IMAGE_TYPE_3D,
+		.format = format,
+		.mipLevels = 1,
+		.arrayLayers = 1,
+		.samples = VK_SAMPLE_COUNT_1_BIT,
+		.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+		.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+		.extent = (VkExtent3D) { 1, 1, 1 },
+		.tiling = VK_IMAGE_TILING_OPTIMAL,
+		.usage = VK_IMAGE_USAGE_SAMPLED_BIT,
+	};
+	res = vkCreateImage(dev, &img_info, NULL, &renderer->dummy3d_image);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateImage failed", res);
+		return false;
+	}
+
+	VkMemoryRequirements mem_reqs = {0};
+	vkGetImageMemoryRequirements(dev, renderer->dummy3d_image, &mem_reqs);
+	int mem_type_index = vulkan_find_mem_type(renderer->dev,
+		VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, mem_reqs.memoryTypeBits);
+	if (mem_type_index == -1) {
+		wlr_log(WLR_ERROR, "Failed to find suitable memory type");
+		return false;
+	}
+	VkMemoryAllocateInfo mem_info = {
+		.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+		.allocationSize = mem_reqs.size,
+		.memoryTypeIndex = mem_type_index,
+	};
+	res = vkAllocateMemory(dev, &mem_info, NULL, &renderer->dummy3d_mem);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkAllocateMemory failed", res);
+		return false;
+	}
+	res = vkBindImageMemory(dev, renderer->dummy3d_image, renderer->dummy3d_mem, 0);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkBindMemory failed", res);
+		return false;
+	}
+
+	VkImageViewCreateInfo view_info = {
+		.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
+		.viewType = VK_IMAGE_VIEW_TYPE_3D,
+		.format = format,
+		.components.r = VK_COMPONENT_SWIZZLE_IDENTITY,
+		.components.g = VK_COMPONENT_SWIZZLE_IDENTITY,
+		.components.b = VK_COMPONENT_SWIZZLE_IDENTITY,
+		.components.a = VK_COMPONENT_SWIZZLE_IDENTITY,
+		.subresourceRange = (VkImageSubresourceRange) {
+			.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+			.baseMipLevel = 0,
+			.levelCount = 1,
+			.baseArrayLayer = 0,
+			.layerCount = 1,
+		},
+		.image = renderer->dummy3d_image,
+	};
+	res = vkCreateImageView(dev, &view_info, NULL, &renderer->dummy3d_image_view);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateImageView failed", res);
+		return false;
+	}
+
+	renderer->output_ds_lut3d_dummy_pool = vulkan_alloc_texture_ds(renderer,
+		renderer->output_ds_lut3d_layout, &renderer->output_ds_lut3d_dummy);
+	if (!renderer->output_ds_lut3d_dummy_pool) {
+		wlr_log(WLR_ERROR, "Failed to allocate descriptor");
+		return false;
+	}
+	VkDescriptorImageInfo ds_img_info = {
+		.imageView = renderer->dummy3d_image_view,
+		.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
+	};
+	VkWriteDescriptorSet ds_write = {
+		.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
+		.descriptorCount = 1,
+		.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+		.dstSet = renderer->output_ds_lut3d_dummy,
+		.pImageInfo = &ds_img_info,
+	};
+	vkUpdateDescriptorSets(dev, 1, &ds_write, 0, NULL);
+
+	return true;
+}
+
 // Creates static render data, such as sampler, layouts and shader modules
 // for the given renderer.
 // Cleanup is done by destroying the renderer.
@@ -1827,8 +1986,11 @@ static bool init_static_render_data(struct wlr_vk_renderer *renderer) {
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
-	if (!init_blend_to_output_layouts(renderer, &renderer->output_ds_layout,
+	if (!init_blend_to_output_layouts(renderer)) {
-			&renderer->output_pipe_layout)) {
+		return false;
+	}
+
+	if (!init_dummy_images(renderer)) {
 		return false;
 	}
 

render/vulkan/shaders/output.frag

@@ -2,9 +2,17 @@
 
 layout (input_attachment_index = 0, binding = 0) uniform subpassInput in_color;
 
+layout(set = 1, binding = 0) uniform sampler3D lut_3d;
+
 layout(location = 0) in vec2 uv;
 layout(location = 0) out vec4 out_color;
 
+/* struct wlr_vk_frag_output_pcr_data */
+layout(push_constant) uniform UBO {
+	layout(offset = 80) float lut_3d_offset;
+	float lut_3d_scale;
+} data;
+
 float linear_channel_to_srgb(float x) {
 	return max(min(x * 12.92, 0.04045), 1.055 * pow(x, 1. / 2.4) - 0.055);
 }
@@ -25,5 +33,10 @@ vec4 linear_color_to_srgb(vec4 color) {
 
 void main() {
 	vec4 val = subpassLoad(in_color).rgba;
+
+	// temporary code to use the 3d look up table; to be dropped next commit
+	vec3 pos = data.lut_3d_offset + vec3(0.5,0.5,0.5) * data.lut_3d_scale;
+	val.rgb *= texture(lut_3d, pos).rgb;
+
 	out_color = linear_color_to_srgb(val);
 }