Support direct scanout with src crop and dst boxes

Enable scene-tree direct scanout of a single buffer with various options
for scaling and source crop. This is intended to support direct scanout
for fullscreen video with/without scaling, letterboxing/pillarboxing
(e.g. 4:3 content on a 16:9 display), and source crop (e.g. when
1920x1088 planes are used for 1920x1080 video).

This works by explicitly specifying the source crop and destination box
for the primary buffer in the output state.  DRM atomic and libliftoff
backends will turn this into a crop and scale of the plane (assuming the
hardware supports that).  For the Wayland/X11/DRM-legacy backends I just
reject this so scanout will be disabled.

The previous behaviour is preserved if buffer_src_box and buffer_dst_box
are unset: the buffer is displayed at native size at the top-left of the
output with no crop.

The change to `struct wlr_output_state` makes this a binary breaking
change (but this works transparently for scene-tree compositors like
labwc after a recompile).

backend/drm/atomic.c

@@ -3,6 +3,7 @@
 #include <stdio.h>
 #include <unistd.h>
 #include <wlr/render/drm_syncobj.h>
+#include <wlr/util/box.h>
 #include <wlr/util/log.h>
 #include <xf86drm.h>
 #include <xf86drmMode.h>
@@ -10,6 +11,7 @@
 #include "backend/drm/fb.h"
 #include "backend/drm/iface.h"
 #include "backend/drm/util.h"
+#include "types/wlr_output.h"
 
 static char *atomic_commit_flags_str(uint32_t flags) {
 	const char *const l[] = {
@@ -354,7 +356,8 @@ static void plane_disable(struct atomic *atom, struct wlr_drm_plane *plane) {
 
 static void set_plane_props(struct atomic *atom, struct wlr_drm_backend *drm,
 		struct wlr_drm_plane *plane, struct wlr_drm_fb *fb, uint32_t crtc_id,
-		int32_t x, int32_t y) {
+		const struct wlr_box *dst_box,
+		const struct wlr_fbox *src_box) {
 	uint32_t id = plane->id;
 	const struct wlr_drm_plane_props *props = &plane->props;
 
@@ -364,20 +367,17 @@ static void set_plane_props(struct atomic *atom, struct wlr_drm_backend *drm,
 		return;
 	}
 
-	uint32_t width = fb->wlr_buf->width;
-	uint32_t height = fb->wlr_buf->height;
-
 	// The src_* properties are in 16.16 fixed point
-	atomic_add(atom, id, props->src_x, 0);
+	atomic_add(atom, id, props->src_x, src_box->x * (1 << 16));
-	atomic_add(atom, id, props->src_y, 0);
+	atomic_add(atom, id, props->src_y, src_box->y * (1 << 16));
-	atomic_add(atom, id, props->src_w, (uint64_t)width << 16);
+	atomic_add(atom, id, props->src_w, src_box->width * (1 << 16));
-	atomic_add(atom, id, props->src_h, (uint64_t)height << 16);
+	atomic_add(atom, id, props->src_h, src_box->height * (1 << 16));
-	atomic_add(atom, id, props->crtc_w, width);
-	atomic_add(atom, id, props->crtc_h, height);
 	atomic_add(atom, id, props->fb_id, fb->id);
 	atomic_add(atom, id, props->crtc_id, crtc_id);
-	atomic_add(atom, id, props->crtc_x, (uint64_t)x);
+	atomic_add(atom, id, props->crtc_x, dst_box->x);
-	atomic_add(atom, id, props->crtc_y, (uint64_t)y);
+	atomic_add(atom, id, props->crtc_y, dst_box->y);
+	atomic_add(atom, id, props->crtc_w, dst_box->width);
+	atomic_add(atom, id, props->crtc_h, dst_box->height);
 }
 
 static bool supports_cursor_hotspots(const struct wlr_drm_plane *plane) {
@@ -437,8 +437,14 @@ static void atomic_connector_add(struct atomic *atom,
 		if (crtc->props.vrr_enabled != 0) {
 			atomic_add(atom, crtc->id, crtc->props.vrr_enabled, state->vrr_enabled);
 		}
+
+		struct wlr_fbox src_box;
+		struct wlr_box dst_box;
+		output_state_get_buffer_src_box(state->base, &src_box);
+		output_state_get_buffer_dst_box(state->base, &dst_box);
+
 		set_plane_props(atom, drm, crtc->primary, state->primary_fb, crtc->id,
-			0, 0);
+			&dst_box, &src_box);
 		if (crtc->primary->props.fb_damage_clips != 0) {
 			atomic_add(atom, crtc->primary->id,
 				crtc->primary->props.fb_damage_clips, state->fb_damage_clips);
@@ -451,8 +457,18 @@ static void atomic_connector_add(struct atomic *atom,
 		}
 		if (crtc->cursor) {
 			if (drm_connector_is_cursor_visible(conn)) {
+				struct wlr_fbox cursor_src = {
+					.width = state->cursor_fb->wlr_buf->width,
+					.height = state->cursor_fb->wlr_buf->height,
+				};
+				struct wlr_box cursor_dst = {
+					.x = conn->cursor_x,
+					.y = conn->cursor_y,
+					.width = state->cursor_fb->wlr_buf->width,
+					.height = state->cursor_fb->wlr_buf->height,
+				};
 				set_plane_props(atom, drm, crtc->cursor, state->cursor_fb,
-					crtc->id, conn->cursor_x, conn->cursor_y);
+					crtc->id, &cursor_dst, &cursor_src);
 				if (supports_cursor_hotspots(crtc->cursor)) {
 					atomic_add(atom, crtc->cursor->id,
 						crtc->cursor->props.hotspot_x, conn->cursor_hotspot_x);

backend/drm/legacy.c

@@ -7,6 +7,7 @@
 #include "backend/drm/fb.h"
 #include "backend/drm/iface.h"
 #include "backend/drm/util.h"
+#include "types/wlr_output.h"
 
 static bool legacy_fb_props_match(struct wlr_drm_fb *fb1,
 		struct wlr_drm_fb *fb2) {
@@ -39,7 +40,27 @@ static bool legacy_crtc_test(const struct wlr_drm_connector_state *state,
 	struct wlr_drm_connector *conn = state->connector;
 	struct wlr_drm_crtc *crtc = conn->crtc;
 
-	if ((state->base->committed & WLR_OUTPUT_STATE_BUFFER) && !modeset) {
+	if (state->base->committed & WLR_OUTPUT_STATE_BUFFER) {
+		// If the size doesn't match, reject buffer (scaling is not supported)
+		int pending_width, pending_height;
+		output_pending_resolution(&state->connector->output, state->base,
+			&pending_width, &pending_height);
+		if (state->base->buffer->width != pending_width ||
+				state->base->buffer->height != pending_height) {
+			wlr_log(WLR_DEBUG, "Primary buffer size mismatch");
+			return false;
+		}
+		// Source crop is also not supported
+		struct wlr_fbox src_box;
+		output_state_get_buffer_src_box(state->base, &src_box);
+		if (src_box.x != 0.0 || src_box.y != 0.0 ||
+				src_box.width != (double)state->base->buffer->width ||
+				src_box.height != (double)state->base->buffer->height) {
+			wlr_log(WLR_DEBUG, "Source crop not supported in DRM-legacy output");
+			return false;
+		}
+
+		if (!modeset) {
 			struct wlr_drm_fb *pending_fb = state->primary_fb;
 
 			struct wlr_drm_fb *prev_fb = crtc->primary->queued_fb;
@@ -55,6 +76,7 @@ static bool legacy_crtc_test(const struct wlr_drm_connector_state *state,
 				return false;
 			}
 		}
+	}
 
 	return true;
 }

backend/drm/libliftoff.c

@@ -4,12 +4,14 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <wlr/util/box.h>
 #include <wlr/util/log.h>
 
 #include "backend/drm/drm.h"
 #include "backend/drm/fb.h"
 #include "backend/drm/iface.h"
 #include "config.h"
+#include "types/wlr_output.h"
 
 static void log_handler(enum liftoff_log_priority priority, const char *fmt, va_list args) {
 	enum wlr_log_importance importance = WLR_SILENT;
@@ -149,25 +151,23 @@ static bool add_prop(drmModeAtomicReq *req, uint32_t obj,
 }
 
 static bool set_plane_props(struct wlr_drm_plane *plane,
-		struct liftoff_layer *layer, struct wlr_drm_fb *fb, int32_t x, int32_t y, uint64_t zpos) {
+		struct liftoff_layer *layer, struct wlr_drm_fb *fb, uint64_t zpos,
+		const struct wlr_box *dst_box, const struct wlr_fbox *src_box) {
 	if (fb == NULL) {
 		wlr_log(WLR_ERROR, "Failed to acquire FB for plane %"PRIu32, plane->id);
 		return false;
 	}
 
-	uint32_t width = fb->wlr_buf->width;
+	// The src_* properties are in 16.16 fixed point
-	uint32_t height = fb->wlr_buf->height;
-
-	// The SRC_* properties are in 16.16 fixed point
 	return liftoff_layer_set_property(layer, "zpos", zpos) == 0 &&
-		liftoff_layer_set_property(layer, "SRC_X", 0) == 0 &&
+		liftoff_layer_set_property(layer, "SRC_X", src_box->x * (1 << 16)) == 0 &&
-		liftoff_layer_set_property(layer, "SRC_Y", 0) == 0 &&
+		liftoff_layer_set_property(layer, "SRC_Y", src_box->y * (1 << 16)) == 0 &&
-		liftoff_layer_set_property(layer, "SRC_W", (uint64_t)width << 16) == 0 &&
+		liftoff_layer_set_property(layer, "SRC_W", src_box->width * (1 << 16)) == 0 &&
-		liftoff_layer_set_property(layer, "SRC_H", (uint64_t)height << 16) == 0 &&
+		liftoff_layer_set_property(layer, "SRC_H", src_box->height * (1 << 16)) == 0 &&
-		liftoff_layer_set_property(layer, "CRTC_X", (uint64_t)x) == 0 &&
+		liftoff_layer_set_property(layer, "CRTC_X", dst_box->x) == 0 &&
-		liftoff_layer_set_property(layer, "CRTC_Y", (uint64_t)y) == 0 &&
+		liftoff_layer_set_property(layer, "CRTC_Y", dst_box->y) == 0 &&
-		liftoff_layer_set_property(layer, "CRTC_W", width) == 0 &&
+		liftoff_layer_set_property(layer, "CRTC_W", dst_box->width) == 0 &&
-		liftoff_layer_set_property(layer, "CRTC_H", height) == 0 &&
+		liftoff_layer_set_property(layer, "CRTC_H", dst_box->height) == 0 &&
 		liftoff_layer_set_property(layer, "FB_ID", fb->id) == 0;
 }
 
@@ -331,9 +331,17 @@ static bool add_connector(drmModeAtomicReq *req,
 		if (crtc->props.vrr_enabled != 0) {
 			ok = ok && add_prop(req, crtc->id, crtc->props.vrr_enabled, state->vrr_enabled);
 		}
+		struct wlr_fbox src_box;
+		struct wlr_box dst_box;
+		output_state_get_buffer_src_box(state->base, &src_box);
+		output_state_get_buffer_dst_box(state->base, &dst_box);
+
 		ok = ok &&
-			set_plane_props(crtc->primary, crtc->primary->liftoff_layer, state->primary_fb, 0, 0, 0) &&
+			set_plane_props(crtc->primary, crtc->primary->liftoff_layer,
-			set_plane_props(crtc->primary, crtc->liftoff_composition_layer, state->primary_fb, 0, 0, 0);
+				state->primary_fb, 0, &dst_box, &src_box);
+		ok = ok &&
+			set_plane_props(crtc->primary, crtc->liftoff_composition_layer,
+				state->primary_fb, 0, &dst_box, &src_box);
 		liftoff_layer_set_property(crtc->primary->liftoff_layer,
 			"FB_DAMAGE_CLIPS", state->fb_damage_clips);
 		liftoff_layer_set_property(crtc->liftoff_composition_layer,
@@ -360,9 +368,19 @@ static bool add_connector(drmModeAtomicReq *req,
 
 		if (crtc->cursor) {
 			if (drm_connector_is_cursor_visible(conn)) {
+				struct wlr_fbox cursor_src = {
+					.width = state->cursor_fb->wlr_buf->width,
+					.height = state->cursor_fb->wlr_buf->height,
+				};
+				struct wlr_box cursor_dst = {
+					.x = conn->cursor_x,
+					.y = conn->cursor_y,
+					.width = state->cursor_fb->wlr_buf->width,
+					.height = state->cursor_fb->wlr_buf->height,
+				};
 				ok = ok && set_plane_props(crtc->cursor, crtc->cursor->liftoff_layer,
-					state->cursor_fb, conn->cursor_x, conn->cursor_y,
+					state->cursor_fb, wl_list_length(&crtc->layers) + 1,
-					wl_list_length(&crtc->layers) + 1);
+					&cursor_dst, &cursor_src);
 			} else {
 				ok = ok && disable_plane(crtc->cursor);
 			}

backend/wayland/output.c

@@ -300,6 +300,28 @@ static bool output_test(struct wlr_output *wlr_output,
 	struct wlr_wl_output *output =
 		get_wl_output_from_output(wlr_output);
 
+	if (state->committed & WLR_OUTPUT_STATE_BUFFER) {
+		// If the size doesn't match, reject buffer (scaling is not currently
+		// supported but could be implemented with viewporter)
+		int pending_width, pending_height;
+		output_pending_resolution(wlr_output, state,
+			&pending_width, &pending_height);
+		if (state->buffer->width != pending_width ||
+				state->buffer->height != pending_height) {
+			wlr_log(WLR_DEBUG, "Primary buffer size mismatch");
+			return false;
+		}
+		// Source crop is also not currently supported
+		struct wlr_fbox src_box;
+		output_state_get_buffer_src_box(state, &src_box);
+		if (src_box.x != 0.0 || src_box.y != 0.0 ||
+				src_box.width != (double)state->buffer->width ||
+				src_box.height != (double)state->buffer->height) {
+			wlr_log(WLR_DEBUG, "Source crop not supported in wayland output");
+			return false;
+		}
+	}
+
 	uint32_t unsupported = state->committed & ~SUPPORTED_OUTPUT_STATE;
 	if (unsupported != 0) {
 		wlr_log(WLR_DEBUG, "Unsupported output state fields: 0x%"PRIx32,

backend/x11/output.c

@@ -129,6 +129,27 @@ static bool output_test(struct wlr_output *wlr_output,
 		return false;
 	}
 
+	if (state->committed & WLR_OUTPUT_STATE_BUFFER) {
+		// If the size doesn't match, reject buffer (scaling is not supported)
+		int pending_width, pending_height;
+		output_pending_resolution(wlr_output, state,
+			&pending_width, &pending_height);
+		if (state->buffer->width != pending_width ||
+				state->buffer->height != pending_height) {
+			wlr_log(WLR_DEBUG, "Primary buffer size mismatch");
+			return false;
+		}
+		// Source crop is not supported
+		struct wlr_fbox src_box;
+		output_state_get_buffer_src_box(state, &src_box);
+		if (src_box.x != 0.0 || src_box.y != 0.0 ||
+				src_box.width != (double)state->buffer->width ||
+				src_box.height != (double)state->buffer->height) {
+			wlr_log(WLR_DEBUG, "Source crop not supported in X11 output");
+			return false;
+		}
+	}
+
 	// All we can do to influence adaptive sync on the X11 backend is set the
 	// _VARIABLE_REFRESH window property like mesa automatically does. We don't
 	// have any control beyond that, so we set the state to enabled on creating

include/types/wlr_output.h

@@ -26,4 +26,9 @@ void output_defer_present(struct wlr_output *output, struct wlr_output_event_pre
 bool output_prepare_commit(struct wlr_output *output, const struct wlr_output_state *state);
 void output_apply_commit(struct wlr_output *output, const struct wlr_output_state *state);
 
+void output_state_get_buffer_src_box(const struct wlr_output_state *state,
+	struct wlr_fbox *out);
+void output_state_get_buffer_dst_box(const struct wlr_output_state *state,
+	struct wlr_box *out);
+
 #endif

include/wlr/types/wlr_output.h

@@ -17,6 +17,7 @@
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_buffer.h>
 #include <wlr/util/addon.h>
+#include <wlr/util/box.h>
 
 enum wlr_output_mode_aspect_ratio {
 	WLR_OUTPUT_MODE_ASPECT_RATIO_NONE,
@@ -94,6 +95,14 @@ struct wlr_output_state {
 	enum wl_output_subpixel subpixel;
 
 	struct wlr_buffer *buffer;
+	// Source crop for the buffer.  If all zeros then no crop is applied.
+	// Double-buffered by WLR_OUTPUT_STATE_BUFFER along with `buffer`.
+	struct wlr_fbox buffer_src_box;
+	// Destination rect to scale the buffer to (after source crop).  If width
+	// and height are zero then the buffer is displayed at native size.
+	// Double-buffered by WLR_OUTPUT_STATE_BUFFER along with `buffer`.
+	struct wlr_box buffer_dst_box;
+
 	/* Request a tearing page-flip. When enabled, this may cause the output to
 	 * display a part of the previous buffer and a part of the current buffer at
 	 * the same time. The backend may reject the commit if a tearing page-flip

types/output/output.c

@@ -548,14 +548,37 @@ static uint32_t output_compare_state(struct wlr_output *output,
 static bool output_basic_test(struct wlr_output *output,
 		const struct wlr_output_state *state) {
 	if (state->committed & WLR_OUTPUT_STATE_BUFFER) {
-		// If the size doesn't match, reject buffer (scaling is not
+		struct wlr_fbox src_box;
-		// supported)
+		output_state_get_buffer_src_box(state, &src_box);
+
+		// Source box must be contained within the buffer
+		if (src_box.x < 0.0 || src_box.y < 0.0 ||
+				src_box.x + src_box.width > state->buffer->width ||
+				src_box.y + src_box.height > state->buffer->height) {
+			wlr_log(WLR_ERROR, "Tried to commit with invalid buffer_src_box");
+			return false;
+		}
+
+		// Source box must not be empty (but it can be smaller than 1 pixel,
+		// some DRM devices support sub-pixel crops)
+		if (wlr_fbox_empty(&src_box)) {
+			wlr_log(WLR_ERROR, "Tried to commit with an empty buffer_src_box");
+			return false;
+		}
+
+		// Destination box cannot be entirely off-screen (but it also doesn't
+		// have to be entirely on-screen).  This also checks the dst box is
+		// not empty.
 		int pending_width, pending_height;
-		output_pending_resolution(output, state,
+		output_pending_resolution(output, state, &pending_width, &pending_height);
-			&pending_width, &pending_height);
+		struct wlr_box output_box = {
-		if (state->buffer->width != pending_width ||
+			.width = pending_width,
-				state->buffer->height != pending_height) {
+			.height = pending_height
-			wlr_log(WLR_DEBUG, "Primary buffer size mismatch");
+		};
+		struct wlr_box dst_box;
+		output_state_get_buffer_dst_box(state, &dst_box);
+		if (!wlr_box_intersection(&output_box, &output_box, &dst_box)) {
+			wlr_log(WLR_ERROR, "Primary buffer is entirely off-screen or 0-sized");
 			return false;
 		}
 	} else {
@@ -845,6 +868,39 @@ void output_defer_present(struct wlr_output *output, struct wlr_output_event_pre
 		deferred_present_event_handle_idle, deferred);
 }
 
+void output_state_get_buffer_src_box(const struct wlr_output_state *state,
+		struct wlr_fbox *out) {
+	out->x = state->buffer_src_box.x;
+	out->y = state->buffer_src_box.y;
+	// If the source box is unset then default to the whole buffer.
+	if (state->buffer_src_box.width == 0.0 &&
+			state->buffer_src_box.height == 0.0) {
+		out->width = (double)state->buffer->width;
+		out->height = (double)state->buffer->height;
+	} else {
+		out->width = state->buffer_src_box.width;
+		out->height = state->buffer_src_box.height;
+	}
+}
+
+void output_state_get_buffer_dst_box(const struct wlr_output_state *state,
+		struct wlr_box *out) {
+	out->x = state->buffer_dst_box.x;
+	out->y = state->buffer_dst_box.y;
+	// If the dst box is unset then default to source crop size (which itself
+	// defaults to the whole buffer size if unset)
+	if (state->buffer_dst_box.width == 0 &&
+			state->buffer_dst_box.height == 0) {
+		struct wlr_fbox src_box;
+		output_state_get_buffer_src_box(state, &src_box);
+		out->width = (int)src_box.width;
+		out->height = (int)src_box.height;
+	} else {
+		out->width = state->buffer_dst_box.width;
+		out->height = state->buffer_dst_box.height;
+	}
+}
+
 void wlr_output_send_request_state(struct wlr_output *output,
 		const struct wlr_output_state *state) {
 	uint32_t unchanged = output_compare_state(output, state);

types/output/state.c

@@ -142,6 +142,8 @@ bool wlr_output_state_copy(struct wlr_output_state *dst,
 		WLR_OUTPUT_STATE_WAIT_TIMELINE |
 		WLR_OUTPUT_STATE_SIGNAL_TIMELINE);
 	copy.buffer = NULL;
+	copy.buffer_src_box = (struct wlr_fbox){0};
+	copy.buffer_dst_box = (struct wlr_box){0};
 	pixman_region32_init(&copy.damage);
 	copy.gamma_lut = NULL;
 	copy.gamma_lut_size = 0;
@@ -150,6 +152,8 @@ bool wlr_output_state_copy(struct wlr_output_state *dst,
 
 	if (src->committed & WLR_OUTPUT_STATE_BUFFER) {
 		wlr_output_state_set_buffer(&copy, src->buffer);
+		copy.buffer_src_box = src->buffer_src_box;
+		copy.buffer_dst_box = src->buffer_dst_box;
 	}
 
 	if (src->committed & WLR_OUTPUT_STATE_DAMAGE) {

types/scene/wlr_scene.c

@@ -1831,6 +1831,7 @@ static bool scene_entry_try_direct_scanout(struct render_list_entry *entry,
 		return false;
 	}
 
+	// The native size of the buffer after any transform is applied
 	int default_width = buffer->buffer->width;
 	int default_height = buffer->buffer->height;
 	wlr_output_transform_coords(buffer->transform, &default_width, &default_height);
@@ -1839,11 +1840,6 @@ static bool scene_entry_try_direct_scanout(struct render_list_entry *entry,
 		.height = default_height,
 	};
 
-	if (!wlr_fbox_empty(&buffer->src_box) &&
-			!wlr_fbox_equal(&buffer->src_box, &default_box)) {
-		return false;
-	}
-
 	if (buffer->transform != data->transform) {
 		return false;
 	}
@@ -1851,10 +1847,6 @@ static bool scene_entry_try_direct_scanout(struct render_list_entry *entry,
 	struct wlr_box node_box = { .x = entry->x, .y = entry->y };
 	scene_node_get_size(node, &node_box.width, &node_box.height);
 
-	if (!wlr_box_equal(&data->logical, &node_box)) {
-		return false;
-	}
-
 	if (buffer->primary_output == scene_output) {
 		struct wlr_linux_dmabuf_feedback_v1_init_options options = {
 			.main_renderer = scene_output->output->renderer,
@@ -1871,6 +1863,12 @@ static bool scene_entry_try_direct_scanout(struct render_list_entry *entry,
 		return false;
 	}
 
+	if (!wlr_fbox_empty(&buffer->src_box) &&
+			!wlr_fbox_equal(&buffer->src_box, &default_box)) {
+		pending.buffer_src_box = buffer->src_box;
+	}
+	pending.buffer_dst_box = node_box;
+
 	wlr_output_state_set_buffer(&pending, buffer->buffer);
 	if (buffer->wait_timeline != NULL) {
 		wlr_output_state_set_wait_timeline(&pending, buffer->wait_timeline, buffer->wait_point);