diff --git a/sway/input/keyboard.c b/sway/input/keyboard.c
index e925c00d4..a42ce9111 100644
--- a/sway/input/keyboard.c
+++ b/sway/input/keyboard.c
@@ -918,7 +918,9 @@ void sway_keyboard_destroy(struct sway_keyboard *keyboard) {
 	if (!keyboard) {
 		return;
 	}
-	sway_keyboard_group_remove(keyboard);
+	if (keyboard->seat_device->input_device->wlr_device->keyboard->group) {
+		sway_keyboard_group_remove(keyboard);
+	}
 	if (keyboard->keymap) {
 		xkb_keymap_unref(keyboard->keymap);
 	}