diff --git a/include/sway/input/cursor.h b/include/sway/input/cursor.h
index 774921c85..c2e2228af 100644
--- a/include/sway/input/cursor.h
+++ b/include/sway/input/cursor.h
@@ -91,7 +91,7 @@ void cursor_rebase_all(void);
 void cursor_update_image(struct sway_cursor *cursor, struct sway_node *node);
 
 void cursor_handle_activity(struct sway_cursor *cursor,
-	enum sway_input_idle_source idle_source);
+	struct wlr_input_device *device);
 void cursor_unhide(struct sway_cursor *cursor);
 int cursor_get_timeout(struct sway_cursor *cursor);
 
diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index d6c50acc3..f4ac74b4f 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -253,11 +253,32 @@ int cursor_get_timeout(struct sway_cursor *cursor) {
 	return timeout;
 }
 
+static enum sway_input_idle_source idle_source_from_device(
+		struct wlr_input_device *device) {
+	switch (device->type) {
+		case WLR_INPUT_DEVICE_KEYBOARD:
+			return IDLE_SOURCE_KEYBOARD;
+		case WLR_INPUT_DEVICE_POINTER:
+			return IDLE_SOURCE_POINTER;
+		case WLR_INPUT_DEVICE_TOUCH:
+			return IDLE_SOURCE_TOUCH;
+		case WLR_INPUT_DEVICE_TABLET_TOOL:
+			return IDLE_SOURCE_TABLET_TOOL;
+		case WLR_INPUT_DEVICE_TABLET_PAD:
+			return IDLE_SOURCE_TABLET_PAD;
+		case WLR_INPUT_DEVICE_SWITCH:
+			return IDLE_SOURCE_SWITCH;
+	}
+
+	assert(false);
+}
+
 void cursor_handle_activity(struct sway_cursor *cursor,
-		enum sway_input_idle_source idle_source) {
+		struct wlr_input_device *device) {
 	wl_event_source_timer_update(
 			cursor->hide_source, cursor_get_timeout(cursor));
 
+	enum sway_input_idle_source idle_source = idle_source_from_device(device);
 	seat_idle_notify_activity(cursor->seat, idle_source);
 	if (cursor->hidden && idle_source != IDLE_SOURCE_TOUCH) {
 		cursor_unhide(cursor);
@@ -283,10 +304,6 @@ void cursor_unhide(struct sway_cursor *cursor) {
 static void pointer_motion(struct sway_cursor *cursor, uint32_t time_msec,
 		struct wlr_input_device *device, double dx, double dy,
 		double dx_unaccel, double dy_unaccel) {
-	if (device->type == WLR_INPUT_DEVICE_POINTER) {
-		cursor_handle_activity(cursor, IDLE_SOURCE_POINTER);
-	}
-
 	wlr_relative_pointer_manager_v1_send_relative_motion(
 		server.relative_pointer_manager,
 		cursor->seat->wlr_seat, (uint64_t)time_msec * 1000,
@@ -322,6 +339,7 @@ static void handle_pointer_motion_relative(
 		struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, motion);
 	struct wlr_event_pointer_motion *e = data;
+	cursor_handle_activity(cursor, e->device);
 
 	pointer_motion(cursor, e->time_msec, e->device, e->delta_x, e->delta_y,
 			e->unaccel_dx, e->unaccel_dy);
@@ -333,6 +351,7 @@ static void handle_pointer_motion_absolute(
 	struct sway_cursor *cursor =
 		wl_container_of(listener, cursor, motion_absolute);
 	struct wlr_event_pointer_motion_absolute *event = data;
+	cursor_handle_activity(cursor, event->device);
 
 	double lx, ly;
 	wlr_cursor_absolute_to_layout_coords(cursor->cursor, event->device,
@@ -358,6 +377,7 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 static void handle_pointer_button(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, button);
 	struct wlr_event_pointer_button *event = data;
+	cursor_handle_activity(cursor, event->device);
 
 	if (event->state == WLR_BUTTON_PRESSED) {
 		cursor->pressed_button_count++;
@@ -369,7 +389,6 @@ static void handle_pointer_button(struct wl_listener *listener, void *data) {
 		}
 	}
 
-	cursor_handle_activity(cursor, IDLE_SOURCE_POINTER);
 	dispatch_cursor_button(cursor, event->device,
 			event->time_msec, event->button, event->state);
 	transaction_commit_dirty();
@@ -383,20 +402,21 @@ void dispatch_cursor_axis(struct sway_cursor *cursor,
 static void handle_pointer_axis(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, axis);
 	struct wlr_event_pointer_axis *event = data;
-	cursor_handle_activity(cursor, IDLE_SOURCE_POINTER);
+	cursor_handle_activity(cursor, event->device);
 	dispatch_cursor_axis(cursor, event);
 	transaction_commit_dirty();
 }
 
 static void handle_pointer_frame(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, frame);
-	cursor_handle_activity(cursor, IDLE_SOURCE_POINTER);
 	wlr_seat_pointer_notify_frame(cursor->seat->wlr_seat);
 }
 
 static void handle_touch_down(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, touch_down);
 	struct wlr_event_touch_down *event = data;
+	cursor_handle_activity(cursor, event->device);
+	cursor_hide(cursor);
 
 	struct sway_seat *seat = cursor->seat;
 	struct wlr_seat *wlr_seat = seat->wlr_seat;
@@ -414,8 +434,6 @@ static void handle_touch_down(struct wl_listener *listener, void *data) {
 
 	if (surface && wlr_surface_accepts_touch(wlr_seat, surface)) {
 		if (seat_is_input_allowed(seat, surface)) {
-			cursor_hide(cursor);
-			cursor_handle_activity(cursor, IDLE_SOURCE_TOUCH);
 			wlr_seat_touch_notify_down(wlr_seat, surface, event->time_msec,
 					event->touch_id, sx, sy);
 
@@ -444,19 +462,19 @@ static void handle_touch_down(struct wl_listener *listener, void *data) {
 static void handle_touch_up(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, touch_up);
 	struct wlr_event_touch_up *event = data;
+	cursor_handle_activity(cursor, event->device);
+
 	struct wlr_seat *wlr_seat = cursor->seat->wlr_seat;
 
 	if (cursor->simulating_pointer_from_touch) {
 		if (cursor->pointer_touch_id == cursor->seat->touch_id) {
 			cursor->simulating_pointer_from_touch = false;
-			cursor_handle_activity(cursor, IDLE_SOURCE_POINTER);
 			dispatch_cursor_button(cursor, event->device, event->time_msec,
 					BTN_LEFT, WLR_BUTTON_RELEASED);
 			wlr_seat_pointer_notify_frame(wlr_seat);
 			transaction_commit_dirty();
 		}
 	} else {
-		cursor_handle_activity(cursor, IDLE_SOURCE_TOUCH);
 		wlr_seat_touch_notify_up(wlr_seat, event->time_msec, event->touch_id);
 	}
 }
@@ -465,6 +483,7 @@ static void handle_touch_motion(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor =
 		wl_container_of(listener, cursor, touch_motion);
 	struct wlr_event_touch_motion *event = data;
+	cursor_handle_activity(cursor, event->device);
 
 	struct sway_seat *seat = cursor->seat;
 	struct wlr_seat *wlr_seat = seat->wlr_seat;
@@ -497,7 +516,6 @@ static void handle_touch_motion(struct wl_listener *listener, void *data) {
 			transaction_commit_dirty();
 		}
 	} else if (surface) {
-		cursor_handle_activity(cursor, IDLE_SOURCE_TOUCH);
 		wlr_seat_touch_notify_motion(wlr_seat, event->time_msec,
 			event->touch_id, sx, sy);
 	}
@@ -535,7 +553,6 @@ static void handle_tablet_tool_position(struct sway_cursor *cursor,
 		bool change_x, bool change_y,
 		double x, double y, double dx, double dy,
 		int32_t time_msec) {
-	cursor_handle_activity(cursor, IDLE_SOURCE_TABLET_TOOL);
 
 	if (!change_x && !change_y) {
 		return;
@@ -587,10 +604,10 @@ static void handle_tablet_tool_position(struct sway_cursor *cursor,
 
 static void handle_tool_axis(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, tool_axis);
-	cursor_handle_activity(cursor, IDLE_SOURCE_TABLET_TOOL);
 	struct wlr_event_tablet_tool_axis *event = data;
-	struct sway_tablet_tool *sway_tool = event->tool->data;
+	cursor_handle_activity(cursor, event->device);
 
+	struct sway_tablet_tool *sway_tool = event->tool->data;
 	if (!sway_tool) {
 		sway_log(SWAY_DEBUG, "tool axis before proximity");
 		return;
@@ -643,12 +660,14 @@ static void handle_tool_axis(struct wl_listener *listener, void *data) {
 
 static void handle_tool_tip(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, tool_tip);
-	cursor_handle_activity(cursor, IDLE_SOURCE_TABLET_TOOL);
 	struct wlr_event_tablet_tool_tip *event = data;
+	cursor_handle_activity(cursor, event->device);
+
 	struct sway_tablet_tool *sway_tool = event->tool->data;
 	struct wlr_tablet_v2_tablet *tablet_v2 = sway_tool->tablet->tablet_v2;
 	struct sway_seat *seat = cursor->seat;
 
+
 	double sx, sy;
 	struct wlr_surface *surface = NULL;
 	node_at_coords(seat, cursor->cursor->x, cursor->cursor->y,
@@ -692,9 +711,10 @@ static struct sway_tablet *get_tablet_for_device(struct sway_cursor *cursor,
 }
 
 static void handle_tool_proximity(struct wl_listener *listener, void *data) {
-	struct sway_cursor *cursor = wl_container_of(listener, cursor, tool_proximity);
-	cursor_handle_activity(cursor, IDLE_SOURCE_TABLET_TOOL);
+	struct sway_cursor *cursor =
+		wl_container_of(listener, cursor, tool_proximity);
 	struct wlr_event_tablet_tool_proximity *event = data;
+	cursor_handle_activity(cursor, event->device);
 
 	struct wlr_tablet_tool *tool = event->tool;
 	if (!tool->data) {
@@ -723,10 +743,9 @@ static void handle_tool_proximity(struct wl_listener *listener, void *data) {
 
 static void handle_tool_button(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, tool_button);
-	struct sway_seat *seat = cursor->seat;
-	cursor_handle_activity(cursor, IDLE_SOURCE_TABLET_TOOL);
-
 	struct wlr_event_tablet_tool_button *event = data;
+	cursor_handle_activity(cursor, event->device);
+
 	struct sway_tablet_tool *sway_tool = event->tool->data;
 	if (!sway_tool) {
 		sway_log(SWAY_DEBUG, "tool button before proximity");
@@ -737,7 +756,7 @@ static void handle_tool_button(struct wl_listener *listener, void *data) {
 	double sx, sy;
 	struct wlr_surface *surface = NULL;
 
-	node_at_coords(seat, cursor->cursor->x, cursor->cursor->y,
+	node_at_coords(cursor->seat, cursor->cursor->x, cursor->cursor->y,
 		&surface, &sx, &sy);
 
 	if (!surface || !wlr_surface_accepts_tablet_v2(tablet_v2, surface)) {
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 73a8deac3..1be8d5527 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -1380,9 +1380,9 @@ void seat_apply_config(struct sway_seat *seat,
 
 	wl_list_for_each(seat_device, &seat->devices, link) {
 		seat_configure_device(seat, seat_device->input_device);
+		cursor_handle_activity(seat->cursor,
+			seat_device->input_device->wlr_device);
 	}
-
-	cursor_handle_activity(seat->cursor, IDLE_SOURCE_POINTER);
 }
 
 struct seat_config *seat_get_config(struct sway_seat *seat) {