From c682d97841235cc328f044a4e2f2272f7a32b226 Mon Sep 17 00:00:00 2001
From: Isaac Freund <ifreund@ifreund.xyz>
Date: Mon, 23 Mar 2020 14:32:27 +0100
Subject: [PATCH] Return failure of wlr_renderer_init_wl_display()

This makes it easier for the user of this library to properly handle
failure of this function.

The signature of wlr_renderer_impl.init_wl_display was also modified to
allow for proper error propagation.
---
 include/wlr/render/interface.h    |  2 +-
 include/wlr/render/wlr_renderer.h |  7 ++++++-
 render/gles2/renderer.c           |  4 +++-
 render/wlr_renderer.c             | 16 +++++++++++-----
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/include/wlr/render/interface.h b/include/wlr/render/interface.h
index 4e9920777..6936afa33 100644
--- a/include/wlr/render/interface.h
+++ b/include/wlr/render/interface.h
@@ -62,7 +62,7 @@ struct wlr_renderer_impl {
 	struct wlr_texture *(*texture_from_dmabuf)(struct wlr_renderer *renderer,
 		struct wlr_dmabuf_attributes *attribs);
 	void (*destroy)(struct wlr_renderer *renderer);
-	void (*init_wl_display)(struct wlr_renderer *renderer,
+	bool (*init_wl_display)(struct wlr_renderer *renderer,
 		struct wl_display *wl_display);
 };
 
diff --git a/include/wlr/render/wlr_renderer.h b/include/wlr/render/wlr_renderer.h
index 54e6506e6..db543bef2 100644
--- a/include/wlr/render/wlr_renderer.h
+++ b/include/wlr/render/wlr_renderer.h
@@ -109,7 +109,12 @@ bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
  */
 bool wlr_renderer_format_supported(struct wlr_renderer *r,
 	enum wl_shm_format fmt);
-void wlr_renderer_init_wl_display(struct wlr_renderer *r,
+/**
+ * Creates necessary shm and invokes the initialization of the implementation.
+ *
+ * Returns false on failure.
+ */
+bool wlr_renderer_init_wl_display(struct wlr_renderer *r,
 	struct wl_display *wl_display);
 
 /**
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index 65c2b8012..9df21fccb 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -351,13 +351,15 @@ static struct wlr_texture *gles2_texture_from_dmabuf(
 	return wlr_gles2_texture_from_dmabuf(renderer->egl, attribs);
 }
 
-static void gles2_init_wl_display(struct wlr_renderer *wlr_renderer,
+static bool gles2_init_wl_display(struct wlr_renderer *wlr_renderer,
 		struct wl_display *wl_display) {
 	struct wlr_gles2_renderer *renderer =
 		gles2_get_renderer(wlr_renderer);
 	if (!wlr_egl_bind_display(renderer->egl, wl_display)) {
 		wlr_log(WLR_INFO, "failed to bind wl_display to EGL");
+		return false;
 	}
+	return true;
 }
 
 struct wlr_egl *wlr_gles2_renderer_get_egl(struct wlr_renderer *wlr_renderer) {
diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c
index eaf630c37..b12f4b3a3 100644
--- a/render/wlr_renderer.c
+++ b/render/wlr_renderer.c
@@ -160,18 +160,18 @@ bool wlr_renderer_format_supported(struct wlr_renderer *r,
 	return r->impl->format_supported(r, fmt);
 }
 
-void wlr_renderer_init_wl_display(struct wlr_renderer *r,
+bool wlr_renderer_init_wl_display(struct wlr_renderer *r,
 		struct wl_display *wl_display) {
 	if (wl_display_init_shm(wl_display)) {
 		wlr_log(WLR_ERROR, "Failed to initialize shm");
-		return;
+		return false;
 	}
 
 	size_t len;
 	const enum wl_shm_format *formats = wlr_renderer_get_formats(r, &len);
 	if (formats == NULL) {
 		wlr_log(WLR_ERROR, "Failed to initialize shm: cannot get formats");
-		return;
+		return false;
 	}
 
 	for (size_t i = 0; i < len; ++i) {
@@ -183,12 +183,18 @@ void wlr_renderer_init_wl_display(struct wlr_renderer *r,
 	}
 
 	if (r->impl->texture_from_dmabuf) {
-		wlr_linux_dmabuf_v1_create(wl_display, r);
+		if (wlr_linux_dmabuf_v1_create(wl_display, r) == NULL) {
+			return false;
+		}
 	}
 
 	if (r->impl->init_wl_display) {
-		r->impl->init_wl_display(r, wl_display);
+		if (!r->impl->init_wl_display(r, wl_display)) {
+			return false;
+		}
 	}
+
+	return true;
 }
 
 struct wlr_renderer *wlr_renderer_autocreate(struct wlr_egl *egl,