Bump RLIMIT_NOFILE

Wayland compositors handle many file descriptors: client
connections, DMA-BUFs, sync_files, wl_data_device pipes, and so
on. Bump the limit to the max.

Closes: https://github.com/swaywm/sway/issues/6285

include/sway/server.h

@@ -137,6 +137,8 @@ void server_fini(struct sway_server *server);
 bool server_start(struct sway_server *server);
 void server_run(struct sway_server *server);
 
+void restore_nofile_limit(void);
+
 void handle_compositor_new_surface(struct wl_listener *listener, void *data);
 void handle_new_output(struct wl_listener *listener, void *data);
 

sway/commands/exec_always.c

@@ -7,6 +7,7 @@
 #include <signal.h>
 #include "sway/commands.h"
 #include "sway/config.h"
+#include "sway/server.h"
 #include "sway/tree/container.h"
 #include "sway/tree/root.h"
 #include "sway/tree/workspace.h"
@@ -53,6 +54,7 @@ struct cmd_results *cmd_exec_process(int argc, char **argv) {
 	// Fork process
 	if ((pid = fork()) == 0) {
 		// Fork child process again
+		restore_nofile_limit();
 		setsid();
 		sigset_t set;
 		sigemptyset(&set);

sway/config/bar.c

@@ -219,6 +219,8 @@ static void invoke_swaybar(struct bar_config *bar) {
 		sigprocmask(SIG_SETMASK, &set, NULL);
 		signal(SIGPIPE, SIG_DFL);
 
+		restore_nofile_limit();
+
 		pid = fork();
 		if (pid < 0) {
 			sway_log_errno(SWAY_ERROR, "fork failed");

sway/config/output.c

@@ -750,6 +750,8 @@ static bool _spawn_swaybg(char **command) {
 		sway_log_errno(SWAY_ERROR, "fork failed");
 		return false;
 	} else if (pid == 0) {
+		restore_nofile_limit();
+
 		pid = fork();
 		if (pid < 0) {
 			sway_log_errno(SWAY_ERROR, "fork failed");

sway/main.c

@@ -6,6 +6,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -27,6 +28,7 @@
 
 static bool terminate_request = false;
 static int exit_value = 0;
+static struct rlimit original_nofile_rlimit = {0};
 struct sway_server server = {0};
 struct sway_debug debug = {0};
 
@@ -169,6 +171,33 @@ static bool drop_permissions(void) {
 	return true;
 }
 
+static void increase_nofile_limit(void) {
+	if (getrlimit(RLIMIT_NOFILE, &original_nofile_rlimit) != 0) {
+		sway_log_errno(SWAY_ERROR, "Failed to bump max open files limit: "
+			"getrlimit(NOFILE) failed");
+		return;
+	}
+
+	struct rlimit new_rlimit = original_nofile_rlimit;
+	new_rlimit.rlim_cur = new_rlimit.rlim_max;
+	if (setrlimit(RLIMIT_NOFILE, &new_rlimit) != 0) {
+		sway_log_errno(SWAY_ERROR, "Failed to bump max open files limit: "
+			"setrlimit(NOFILE) failed");
+		sway_log(SWAY_INFO, "Running with %d max open files",
+			(int)original_nofile_rlimit.rlim_cur);
+	}
+}
+
+void restore_nofile_limit(void) {
+	if (original_nofile_rlimit.rlim_cur == 0) {
+		return;
+	}
+	if (setrlimit(RLIMIT_NOFILE, &original_nofile_rlimit) != 0) {
+		sway_log_errno(SWAY_ERROR, "Failed to restore max open files limit: "
+			"setrlimit(NOFILE) failed");
+	}
+}
+
 void enable_debug_flag(const char *flag) {
 	if (strcmp(flag, "damage=highlight") == 0) {
 		debug.damage = DAMAGE_HIGHLIGHT;
@@ -349,6 +378,8 @@ int main(int argc, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 
+	increase_nofile_limit();
+
 	// handle SIGTERM signals
 	signal(SIGTERM, sig_handler);
 	signal(SIGINT, sig_handler);

sway/swaynag.c

@@ -64,6 +64,8 @@ bool swaynag_spawn(const char *swaynag_command,
 		sway_log(SWAY_ERROR, "Failed to create fork for swaynag");
 		goto failed;
 	} else if (pid == 0) {
+		restore_nofile_limit();
+
 		pid = fork();
 		if (pid < 0) {
 			sway_log_errno(SWAY_ERROR, "fork failed");