diff --git a/backend/libinput/backend.c b/backend/libinput/backend.c
index 8023807fd..50726ec5a 100644
--- a/backend/libinput/backend.c
+++ b/backend/libinput/backend.c
@@ -110,9 +110,11 @@ static bool backend_start(struct wlr_backend *wlr_backend) {
 			no_devs = NULL;
 		}
 	}
-	if (!no_devs && backend->wlr_device_lists.size == 0) {
+	if (!no_devs && (backend->wlr_device_lists.size == 0
+			|| wl_list_empty(&backend->devices))) {
 		handle_libinput_readable(libinput_fd, WL_EVENT_READABLE, backend);
-		if (backend->wlr_device_lists.size == 0) {
+		if (backend->wlr_device_lists.size == 0
+				&& wl_list_empty(&backend->devices)) {
 			wlr_log(WLR_ERROR, "libinput initialization failed, no input devices");
 			wlr_log(WLR_ERROR, "Set WLR_LIBINPUT_NO_DEVICES=1 to suppress this check");
 			return false;
@@ -150,6 +152,11 @@ static void backend_destroy(struct wlr_backend *wlr_backend) {
 		free(*wlr_devices_ptr);
 	}
 
+	struct wlr_libinput_input_device *dev, *tmp;
+	wl_list_for_each_safe(dev, tmp, &backend->devices, link) {
+		destroy_libinput_input_device(dev);
+	}
+
 	wlr_backend_finish(wlr_backend);
 
 	wl_list_remove(&backend->display_destroy.link);
@@ -212,6 +219,7 @@ struct wlr_backend *wlr_libinput_backend_create(struct wl_display *display,
 	wlr_backend_init(&backend->backend, &backend_impl);
 
 	wl_array_init(&backend->wlr_device_lists);
+	wl_list_init(&backend->devices);
 
 	backend->session = session;
 	backend->display = display;
diff --git a/backend/libinput/events.c b/backend/libinput/events.c
index e8d99f251..39b3fc000 100644
--- a/backend/libinput/events.c
+++ b/backend/libinput/events.c
@@ -224,6 +224,16 @@ static void handle_device_removed(struct wlr_libinput_backend *backend,
 	int product = libinput_device_get_id_product(libinput_dev);
 	const char *name = libinput_device_get_name(libinput_dev);
 	wlr_log(WLR_DEBUG, "Removing %s [%d:%d]", name, vendor, product);
+
+	if (!wl_list_empty(&backend->devices)) {
+		struct wlr_libinput_input_device *dev, *tmp_dev;
+		wl_list_for_each_safe(dev, tmp_dev, &backend->devices, link) {
+			if (dev->handle == libinput_dev) {
+				destroy_libinput_input_device(dev);
+			}
+		}
+	}
+
 	if (!wlr_devices) {
 		return;
 	}
diff --git a/include/backend/libinput.h b/include/backend/libinput.h
index 39e6dfe5a..e443c0ec1 100644
--- a/include/backend/libinput.h
+++ b/include/backend/libinput.h
@@ -27,6 +27,7 @@ struct wlr_libinput_backend {
 	struct wl_listener session_signal;
 
 	struct wl_array wlr_device_lists; // struct wl_list *
+	struct wl_list devices; // wlr_libinput_device::link
 };
 
 struct wlr_libinput_input_device {