Implement popup_during_fullscreen

This introduces a new view_impl function: is_transient_for. Similar to
container_has_ancestor but works using the surface parents rather than
the tree.

This patch modifies view_is_visible, container_at and so on to allow
transient views to function normally when they're in front of a
fullscreen view.

include/sway/commands.h

@@ -154,6 +154,7 @@ sway_cmd cmd_new_window;
 sway_cmd cmd_no_focus;
 sway_cmd cmd_output;
 sway_cmd cmd_permit;
+sway_cmd cmd_popup_during_fullscreen;
 sway_cmd cmd_reject;
 sway_cmd cmd_reload;
 sway_cmd cmd_rename;

include/sway/config.h

@@ -256,6 +256,12 @@ enum edge_border_types {
 	E_SMART_NO_GAPS, /**< hide both if one window and gaps to edge is zero */
 };
 
+enum sway_popup_during_fullscreen {
+	POPUP_SMART,
+	POPUP_IGNORE,
+	POPUP_LEAVE,
+};
+
 enum command_context {
 	CONTEXT_CONFIG = 1,
 	CONTEXT_BINDING = 2,
@@ -355,6 +361,7 @@ struct sway_config {
 	bool pango_markup;
 	size_t urgent_timeout;
 	enum sway_fowa focus_on_window_activation;
+	enum sway_popup_during_fullscreen popup_during_fullscreen;
 
 	// Flags
 	bool focus_follows_mouse;

include/sway/tree/view.h

@@ -49,6 +49,8 @@ struct sway_view_impl {
 		wlr_surface_iterator_func_t iterator, void *user_data);
 	void (*for_each_popup)(struct sway_view *view,
 		wlr_surface_iterator_func_t iterator, void *user_data);
+	bool (*is_transient_for)(struct sway_view *child,
+			struct sway_view *ancestor);
 	void (*close)(struct sway_view *view);
 	void (*close_popups)(struct sway_view *view);
 	void (*destroy)(struct sway_view *view);
@@ -396,4 +398,6 @@ void view_remove_saved_buffer(struct sway_view *view);
 
 void view_save_buffer(struct sway_view *view);
 
+bool view_is_transient_for(struct sway_view *child, struct sway_view *ancestor);
+
 #endif

sway/commands.c

@@ -109,6 +109,7 @@ static struct cmd_handler handlers[] = {
 	{ "new_window", cmd_default_border },
 	{ "no_focus", cmd_no_focus },
 	{ "output", cmd_output },
+	{ "popup_during_fullscreen", cmd_popup_during_fullscreen },
 	{ "raise_floating", cmd_raise_floating },
 	{ "seat", cmd_seat },
 	{ "set", cmd_set },

sway/commands/popup_during_fullscreen.c

@@ -0,0 +1,25 @@
+#include <strings.h>
+#include "sway/commands.h"
+#include "sway/config.h"
+
+struct cmd_results *cmd_popup_during_fullscreen(int argc, char **argv) {
+	struct cmd_results *error = NULL;
+	if ((error = checkarg(argc, "popup_during_fullscreen",
+					EXPECTED_EQUAL_TO, 1))) {
+		return error;
+	}
+
+	if (strcasecmp(argv[0], "smart") == 0) {
+		config->popup_during_fullscreen = POPUP_SMART;
+	} else if (strcasecmp(argv[0], "ignore") == 0) {
+		config->popup_during_fullscreen = POPUP_IGNORE;
+	} else if (strcasecmp(argv[0], "leave_fullscreen") == 0) {
+		config->popup_during_fullscreen = POPUP_LEAVE;
+	} else {
+		return cmd_results_new(CMD_INVALID, "popup_during_fullscreen",
+				"Expected "
+				"'popup_during_fullscreen smart|ignore|leave_fullscreen'");
+	}
+
+	return cmd_results_new(CMD_SUCCESS, NULL, NULL);
+}

sway/config.c

@@ -212,6 +212,7 @@ static void config_defaults(struct sway_config *config) {
 	if (!(config->font = strdup("monospace 10"))) goto cleanup;
 	config->font_height = 17; // height of monospace 10
 	config->urgent_timeout = 500;
+	config->popup_during_fullscreen = POPUP_SMART;
 
 	// floating view
 	config->floating_maximum_width = 0;

sway/desktop/output.c

@@ -329,6 +329,17 @@ static void send_frame_done(struct sway_output *output, struct timespec *when) {
 				workspace->current.fullscreen, &data);
 		container_for_each_child(workspace->current.fullscreen,
 				send_frame_done_container_iterator, &data);
+		if (config->popup_during_fullscreen == POPUP_SMART &&
+				workspace->current.fullscreen->view) {
+			for (int i = 0; i < workspace->current.floating->length; ++i) {
+				struct sway_container *floater =
+					workspace->current.floating->items[i];
+				if (floater->view && view_is_transient_for(floater->view,
+							workspace->current.fullscreen->view)) {
+					send_frame_done_container_iterator(floater, &data);
+				}
+			}
+		}
 #ifdef HAVE_XWAYLAND
 		send_frame_done_unmanaged(output, &root->xwayland_unmanaged, when);
 #endif

sway/desktop/render.c

@@ -961,6 +961,17 @@ void output_render(struct sway_output *output, struct timespec *when,
 			render_container(output, damage, fullscreen_con,
 					fullscreen_con->current.focused);
 		}
+
+		if (config->popup_during_fullscreen == POPUP_SMART &&
+				fullscreen_con->view) {
+			for (int i = 0; i < workspace->floating->length; ++i) {
+				struct sway_container *floater = workspace->floating->items[i];
+				if (floater->view && view_is_transient_for(
+							floater->view, fullscreen_con->view)) {
+					render_floating_container(output, damage, floater);
+				}
+			}
+		}
 #ifdef HAVE_XWAYLAND
 		render_unmanaged(output, damage, &root->xwayland_unmanaged);
 #endif

sway/desktop/xdg_shell.c

@@ -192,6 +192,21 @@ static void for_each_popup(struct sway_view *view,
 	wlr_xdg_surface_for_each_popup(view->wlr_xdg_surface, iterator, user_data);
 }
 
+static bool is_transient_for(struct sway_view *child,
+		struct sway_view *ancestor) {
+	if (xdg_shell_view_from_view(child) == NULL) {
+		return false;
+	}
+	struct wlr_xdg_surface *surface = child->wlr_xdg_surface;
+	while (surface) {
+		if (surface->toplevel->parent == ancestor->wlr_xdg_surface) {
+			return true;
+		}
+		surface = surface->toplevel->parent;
+	}
+	return false;
+}
+
 static void _close(struct sway_view *view) {
 	if (xdg_shell_view_from_view(view) == NULL) {
 		return;
@@ -233,6 +248,7 @@ static const struct sway_view_impl view_impl = {
 	.wants_floating = wants_floating,
 	.for_each_surface = for_each_surface,
 	.for_each_popup = for_each_popup,
+	.is_transient_for = is_transient_for,
 	.close = _close,
 	.close_popups = close_popups,
 	.destroy = destroy,
@@ -385,6 +401,18 @@ static void handle_map(struct wl_listener *listener, void *data) {
 		view_update_csd_from_client(view, csd);
 	}
 
+	if (config->popup_during_fullscreen == POPUP_LEAVE &&
+			view->container->workspace &&
+			view->container->workspace->fullscreen &&
+			xdg_surface->toplevel->parent) {
+		struct wlr_xdg_surface *psurface = xdg_surface->toplevel->parent;
+		struct sway_xdg_shell_view *parent = psurface->data;
+		struct sway_view *sway_view = &parent->view;
+		if (sway_view->container && sway_view->container->is_fullscreen) {
+			container_set_fullscreen(sway_view->container, false);
+		}
+	}
+
 	if (xdg_surface->toplevel->client_pending.fullscreen) {
 		container_set_fullscreen(view->container, true);
 		arrange_workspace(view->container->workspace);
@@ -395,6 +423,7 @@ static void handle_map(struct wl_listener *listener, void *data) {
 			arrange_workspace(view->container->workspace);
 		}
 	}
+
 	transaction_commit_dirty();
 
 	xdg_shell_view->commit.notify = handle_commit;

sway/desktop/xdg_shell_v6.c

@@ -189,6 +189,21 @@ static void for_each_popup(struct sway_view *view,
 		user_data);
 }
 
+static bool is_transient_for(struct sway_view *child,
+		struct sway_view *ancestor) {
+	if (xdg_shell_v6_view_from_view(child) == NULL) {
+		return false;
+	}
+	struct wlr_xdg_surface_v6 *surface = child->wlr_xdg_surface_v6;
+	while (surface) {
+		if (surface->toplevel->parent == ancestor->wlr_xdg_surface_v6) {
+			return true;
+		}
+		surface = surface->toplevel->parent;
+	}
+	return false;
+}
+
 static void _close(struct sway_view *view) {
 	if (xdg_shell_v6_view_from_view(view) == NULL) {
 		return;
@@ -230,6 +245,7 @@ static const struct sway_view_impl view_impl = {
 	.wants_floating = wants_floating,
 	.for_each_surface = for_each_surface,
 	.for_each_popup = for_each_popup,
+	.is_transient_for = is_transient_for,
 	.close = _close,
 	.close_popups = close_popups,
 	.destroy = destroy,
@@ -380,6 +396,18 @@ static void handle_map(struct wl_listener *listener, void *data) {
 			WLR_SERVER_DECORATION_MANAGER_MODE_CLIENT;
 	view_update_csd_from_client(view, csd);
 
+	if (config->popup_during_fullscreen == POPUP_LEAVE &&
+			view->container->workspace &&
+			view->container->workspace->fullscreen &&
+			xdg_surface->toplevel->parent) {
+		struct wlr_xdg_surface_v6 *psurface = xdg_surface->toplevel->parent;
+		struct sway_xdg_shell_v6_view *parent = psurface->data;
+		struct sway_view *sway_view = &parent->view;
+		if (sway_view->container && sway_view->container->is_fullscreen) {
+			container_set_fullscreen(sway_view->container, false);
+		}
+	}
+
 	if (xdg_surface->toplevel->client_pending.fullscreen) {
 		container_set_fullscreen(view->container, true);
 		arrange_workspace(view->container->workspace);

sway/desktop/xwayland.c

@@ -15,6 +15,7 @@
 #include "sway/tree/arrange.h"
 #include "sway/tree/container.h"
 #include "sway/tree/view.h"
+#include "sway/tree/workspace.h"
 
 static const char *atom_map[ATOM_LAST] = {
 	"_NET_WM_WINDOW_TYPE_NORMAL",
@@ -253,6 +254,21 @@ static void handle_set_decorations(struct wl_listener *listener, void *data) {
 	view_update_csd_from_client(view, csd);
 }
 
+static bool is_transient_for(struct sway_view *child,
+		struct sway_view *ancestor) {
+	if (xwayland_view_from_view(child) == NULL) {
+		return false;
+	}
+	struct wlr_xwayland_surface *surface = child->wlr_xwayland_surface;
+	while (surface) {
+		if (surface->parent == ancestor->wlr_xwayland_surface) {
+			return true;
+		}
+		surface = surface->parent;
+	}
+	return false;
+}
+
 static void _close(struct sway_view *view) {
 	if (xwayland_view_from_view(view) == NULL) {
 		return;
@@ -276,6 +292,7 @@ static const struct sway_view_impl view_impl = {
 	.set_tiled = set_tiled,
 	.set_fullscreen = set_fullscreen,
 	.wants_floating = wants_floating,
+	.is_transient_for = is_transient_for,
 	.close = _close,
 	.destroy = destroy,
 };
@@ -390,6 +407,18 @@ static void handle_map(struct wl_listener *listener, void *data) {
 	// Put it back into the tree
 	view_map(view, xsurface->surface);
 
+	if (config->popup_during_fullscreen == POPUP_LEAVE &&
+			view->container->workspace &&
+			view->container->workspace->fullscreen &&
+			xsurface->parent) {
+		struct wlr_xwayland_surface *psurface = xsurface->parent;
+		struct sway_xwayland_view *parent = psurface->data;
+		struct sway_view *sway_view = &parent->view;
+		if (sway_view->container && sway_view->container->is_fullscreen) {
+			container_set_fullscreen(sway_view->container, false);
+		}
+	}
+
 	if (xsurface->fullscreen) {
 		container_set_fullscreen(view->container, true);
 		arrange_workspace(view->container->workspace);

sway/input/cursor.c

@@ -98,6 +98,22 @@ static struct sway_node *node_at_coords(
 		return NULL;
 	}
 	if (ws->fullscreen) {
+		// Try transient containers
+		if (config->popup_during_fullscreen == POPUP_SMART &&
+				ws->fullscreen->view) {
+			for (int i = 0; i < ws->floating->length; ++i) {
+				struct sway_container *floater = ws->floating->items[i];
+				if (floater->view && view_is_transient_for(
+							floater->view, ws->fullscreen->view)) {
+					struct sway_container *con = tiling_container_at(
+							&floater->node, lx, ly, surface, sx, sy);
+					if (con) {
+						return &con->node;
+					}
+				}
+			}
+		}
+		// Try fullscreen container
 		struct sway_container *con =
 			tiling_container_at(&ws->fullscreen->node, lx, ly, surface, sx, sy);
 		if (con) {

sway/input/seat.c

@@ -655,7 +655,14 @@ void seat_set_focus_warp(struct sway_seat *seat, struct sway_node *node,
 	// Deny setting focus to a view which is hidden by a fullscreen container
 	if (new_workspace && new_workspace->fullscreen && container &&
 			!container_is_fullscreen_or_child(container)) {
-		return;
+		// Unless it's a transient container
+		bool is_transient = new_workspace->fullscreen->view &&
+			config->popup_during_fullscreen == POPUP_SMART &&
+			container->view && view_is_transient_for(
+					container->view, new_workspace->fullscreen->view);
+		if (!is_transient) {
+			return;
+		}
 	}
 
 	struct sway_output *last_output = last_workspace ?

sway/meson.build

@@ -70,6 +70,7 @@ sway_sources = files(
 	'commands/no_focus.c',
 	'commands/nop.c',
 	'commands/output.c',
+	'commands/popup_during_fullscreen.c',
 	'commands/reload.c',
 	'commands/rename.c',
 	'commands/resize.c',

sway/sway.5.scd

@@ -549,6 +549,12 @@ You may combine output commands into one, like so:
 You can get a list of output names with *swaymsg -t get\_outputs*. You may also
 match any output by using the output name "\*".
 
+*popup\_during\_fullscreen* smart|ignore|leave\_fullscreen
+	Determines what to do when a fullscreen view opens a dialog.
+	If _smart_ (the default), the dialog will be displayed. If _ignore_, the
+	dialog will not be rendered. If _leave\_fullscreen_, the view will exit
+	fullscreen mode and the dialog will be rendered.
+
 *set* $<name> <value>
 	Sets variable $_name_ to _value_. You can use the new variable in the
 	arguments of future commands. When the variable is used, it can be escaped

sway/tree/view.c

@@ -1042,7 +1042,14 @@ bool view_is_visible(struct sway_view *view) {
 	// Check view isn't hidden by another fullscreen view
 	if (workspace->fullscreen &&
 			!container_is_fullscreen_or_child(view->container)) {
-		return false;
+		// However, if we're transient for the fullscreen view and we allow
+		// "popups" during fullscreen then it might be visible
+		bool is_transient = config->popup_during_fullscreen == POPUP_SMART &&
+				workspace->fullscreen->view &&
+				view_is_transient_for(view, workspace->fullscreen->view);
+		if (!is_transient) {
+			return false;
+		}
 	}
 	return true;
 }
@@ -1095,3 +1102,9 @@ void view_save_buffer(struct sway_view *view) {
 		view->saved_buffer_height = view->surface->current.height;
 	}
 }
+
+bool view_is_transient_for(struct sway_view *child,
+		struct sway_view *ancestor) {
+	return child->impl->is_transient_for &&
+		child->impl->is_transient_for(child, ancestor);
+}