diff --git a/backend/drm/backend.c b/backend/drm/backend.c
index 73ccf4139..2840f17f3 100644
--- a/backend/drm/backend.c
+++ b/backend/drm/backend.c
@@ -29,6 +29,7 @@ static void wlr_drm_backend_destroy(struct wlr_backend_state *state) {
 		struct wlr_output_state *output = state->outputs->items[i];
 		wlr_output_destroy(output->wlr_output);
 	}
+	wlr_udev_signal_remove(state->udev, &state->drm_invalidated);
 	wlr_drm_renderer_free(&state->renderer);
 	wlr_session_close_file(state->session, state->fd);
 	wl_event_source_remove(state->drm_event);
@@ -75,15 +76,15 @@ static void device_resumed(struct wl_listener *listener, void *data) {
 
 static void drm_invalidated(struct wl_listener *listener, void *data) {
 	struct wlr_backend_state *drm = wl_container_of(listener, drm, drm_invalidated);
-	dev_t *dev = data;
+	struct wlr_udev *udev = data;
 
-	if (drm->dev == *dev) {
-		char *name = drmGetDeviceNameFromFd2(drm->fd);
-		wlr_log(L_DEBUG, "%s invalidated", name);
-		free(name);
+	(void)udev;
 
-		wlr_drm_scan_connectors(drm);
-	}
+	char *name = drmGetDeviceNameFromFd2(drm->fd);
+	wlr_log(L_DEBUG, "%s invalidated", name);
+	free(name);
+
+	wlr_drm_scan_connectors(drm);
 }
 
 struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
@@ -112,6 +113,7 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 
 	state->backend = backend;
 	state->session = session;
+	state->udev = udev;
 	state->outputs = list_create();
 	if (!state->outputs) {
 		wlr_log(L_ERROR, "Failed to allocate list");
@@ -119,9 +121,6 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 	}
 
 	state->fd = gpu_fd;
-	wl_list_init(&state->drm_invalidated.link);
-	state->drm_invalidated.notify = drm_invalidated;
-	wl_signal_add(&udev->invalidate_drm, &state->drm_invalidated);
 
 	struct stat st;
 	if (fstat(state->fd, &st) < 0) {
@@ -129,6 +128,9 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 	}
 	state->dev = st.st_rdev;
 
+	state->drm_invalidated.notify = drm_invalidated;
+	wlr_udev_signal_add(udev, state->dev, &state->drm_invalidated);
+
 	struct wl_event_loop *event_loop = wl_display_get_event_loop(display);
 
 	state->drm_event = wl_event_loop_add_fd(event_loop, state->fd,
diff --git a/backend/udev.c b/backend/udev.c
index f9577c125..fb2133394 100644
--- a/backend/udev.c
+++ b/backend/udev.c
@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdint.h>
+#include <errno.h>
 #include <xf86drm.h>
 #include <xf86drmMode.h>
 #include <wayland-server.h>
@@ -147,7 +148,14 @@ static int udev_event(int fd, uint32_t mask, void *data) {
 	}
 
 	dev_t devnum = udev_device_get_devnum(dev);
-	wl_signal_emit(&udev->invalidate_drm, &devnum);
+	struct wlr_udev_dev *signal;
+
+	wl_list_for_each(signal, &udev->devices, link) {
+		if (signal->dev == devnum) {
+			wl_signal_emit(&signal->invalidate, udev);
+			break;
+		}
+	}
 
 out:
 	udev_device_unref(dev);
@@ -164,7 +172,6 @@ struct wlr_udev *wlr_udev_create(struct wl_display *display) {
 		wlr_log(L_ERROR, "Failed to create udev context");
 		goto error;
 	}
-	wl_signal_init(&udev->invalidate_drm);
 
 	udev->mon = udev_monitor_new_from_netlink(udev->udev, "udev");
 	if (!udev->mon) {
@@ -185,6 +192,8 @@ struct wlr_udev *wlr_udev_create(struct wl_display *display) {
 		goto error_mon;
 	}
 	
+	wl_list_init(&udev->devices);
+
 	wlr_log(L_DEBUG, "Successfully initialized udev");
 	return udev;
 
@@ -202,8 +211,43 @@ void wlr_udev_destroy(struct wlr_udev *udev) {
 		return;
 	}
 
-	wl_event_source_remove(udev->event);
+	struct wlr_udev_dev *dev, *tmp;
+	wl_list_for_each_safe(dev, tmp, &udev->devices, link) {
+		free(dev);
+	}
 
+	wl_event_source_remove(udev->event);
 	udev_monitor_unref(udev->mon);
 	udev_unref(udev->udev);
 }
+
+bool wlr_udev_signal_add(struct wlr_udev *udev, dev_t dev, struct wl_listener *listener) {
+	struct wlr_udev_dev *device = malloc(sizeof(*device));
+	if (!device) {
+		wlr_log(L_ERROR, "Allocation failed: %s", strerror(errno));
+		return false;
+	}
+
+	device->dev = dev;
+	wl_signal_init(&device->invalidate);
+	wl_signal_add(&device->invalidate, listener);
+	wl_list_insert(&udev->devices, &device->link);
+
+	return true;
+}
+
+void wlr_udev_signal_remove(struct wlr_udev *udev, struct wl_listener *listener) {
+	if (!udev || !listener) {
+		return;
+	}
+
+	struct wlr_udev_dev *dev, *tmp;
+	wl_list_for_each_safe(dev, tmp, &udev->devices, link) {
+		// The signal should only have a single listener
+		if (wl_signal_get(&dev->invalidate, listener->notify) != NULL) {
+			wl_list_remove(&dev->link);
+			free(dev);
+			return;
+		}
+	}
+}
diff --git a/include/backend/udev.h b/include/backend/udev.h
index 0ff35bfa8..a2834063a 100644
--- a/include/backend/udev.h
+++ b/include/backend/udev.h
@@ -1,18 +1,29 @@
 #ifndef _WLR_INTERNAL_UDEV_H
 #define _WLR_INTERNAL_UDEV_H
 
+#include <sys/types.h>
 #include <libudev.h>
 #include <wlr/session.h>
 #include <wayland-server.h>
 #include <wlr/backend/udev.h>
 
+struct wlr_udev_dev {
+	dev_t dev;
+	struct wl_signal invalidate;
+
+	struct wl_list link;
+};
+
 struct wlr_udev {
 	struct udev *udev;
 	struct udev_monitor *mon;
 	struct wl_event_source *event;
-	struct wl_signal invalidate_drm;
+
+	struct wl_list devices;
 };
 
 int wlr_udev_find_gpu(struct wlr_udev *udev, struct wlr_session *session);
+bool wlr_udev_signal_add(struct wlr_udev *udev, dev_t dev, struct wl_listener *listener);
+void wlr_udev_signal_remove(struct wlr_udev *udev, struct wl_listener *listener);
 
 #endif