diff --git a/sway/server.c b/sway/server.c
index c036396f3..ff848450d 100644
--- a/sway/server.c
+++ b/sway/server.c
@@ -202,6 +202,7 @@ bool server_start(struct sway_server *server) {
 					config->xwayland == XWAYLAND_MODE_LAZY);
 		if (!server->xwayland.wlr_xwayland) {
 			sway_log(SWAY_ERROR, "Failed to start Xwayland");
+			unsetenv("DISPLAY");
 		} else {
 			wl_signal_add(&server->xwayland.wlr_xwayland->events.new_surface,
 				&server->xwayland_surface);