diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index a07bc53b4..60d4bf5d2 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -1220,6 +1220,9 @@ static void handle_request_set_cursor(struct wl_listener *listener,
 
 void cursor_set_image(struct sway_cursor *cursor, const char *image,
 		struct wl_client *client) {
+	if (!(cursor->seat->wlr_seat->capabilities & WL_SEAT_CAPABILITY_POINTER)) {
+		return;
+	}
 	if (!image) {
 		wlr_cursor_set_image(cursor->cursor, NULL, 0, 0, 0, 0, 0, 0);
 	} else if (!cursor->image || strcmp(cursor->image, image) != 0) {
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 16acc8a59..89d841bbb 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -389,12 +389,15 @@ static void seat_update_capabilities(struct sway_seat *seat) {
 			break;
 		}
 	}
-	wlr_seat_set_capabilities(seat->wlr_seat, caps);
 
-	// Hide cursor if seat doesn't have pointer capability
+	// Hide cursor if seat doesn't have pointer capability.
+	// We must call cursor_set_image while the wlr_seat has the capabilities
+	// otherwise it's a no op.
 	if ((caps & WL_SEAT_CAPABILITY_POINTER) == 0) {
 		cursor_set_image(seat->cursor, NULL, NULL);
+		wlr_seat_set_capabilities(seat->wlr_seat, caps);
 	} else {
+		wlr_seat_set_capabilities(seat->wlr_seat, caps);
 		cursor_set_image(seat->cursor, "left_ptr", NULL);
 	}
 }