diff --git a/sway/desktop/layer_shell.c b/sway/desktop/layer_shell.c
index 2d355b74d..3accdefb0 100644
--- a/sway/desktop/layer_shell.c
+++ b/sway/desktop/layer_shell.c
@@ -252,7 +252,7 @@ static void unmap(struct sway_layer_surface *sway_layer) {
 		return;
 	}
 	struct sway_output *output = wlr_output->data;
-	if (output == NULL) {
+	if (output == NULL || output->swayc == NULL) {
 		return;
 	}
 	output_damage_surface(output, sway_layer->geo.x, sway_layer->geo.y,
@@ -279,7 +279,7 @@ static void handle_destroy(struct wl_listener *listener, void *data) {
 	wl_list_remove(&sway_layer->surface_commit.link);
 	if (sway_layer->layer_surface->output != NULL) {
 		struct sway_output *output = sway_layer->layer_surface->output->data;
-		if (output != NULL) {
+		if (output != NULL && output->swayc != NULL) {
 			arrange_layers(output);
 		}
 		wl_list_remove(&sway_layer->output_destroy.link);
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index 5d02f5ebc..cb53a9807 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -1221,6 +1221,8 @@ void handle_new_output(struct wl_listener *listener, void *data) {
 	output->server = server;
 	list_add(root_container.sway_root->outputs, output);
 
+	output->damage = wlr_output_damage_create(wlr_output);
+
 	if (!wl_list_empty(&wlr_output->modes)) {
 		struct wlr_output_mode *mode =
 			wl_container_of(wlr_output->modes.prev, mode, link);
@@ -1233,14 +1235,6 @@ void handle_new_output(struct wl_listener *listener, void *data) {
 void output_enable(struct sway_output *output) {
 	struct wlr_output *wlr_output = output->wlr_output;
 
-	if (!wlr_output->data) {
-		wlr_output->data = output;
-	}
-
-	if (!output->damage) {
-		output->damage = wlr_output_damage_create(wlr_output);
-	}
-
 	output->swayc = output_create(output);
 	if (!output->swayc) {
 		// Output is disabled
diff --git a/sway/tree/container.c b/sway/tree/container.c
index af965857d..ca993c412 100644
--- a/sway/tree/container.c
+++ b/sway/tree/container.c
@@ -262,8 +262,6 @@ static struct sway_container *container_output_destroy(
 	wl_list_remove(&output->sway_output->damage_destroy.link);
 	wl_list_remove(&output->sway_output->damage_frame.link);
 
-	// clear the wlr_output reference to this container
-	output->sway_output->wlr_output->data = NULL;
 	output->sway_output->swayc = NULL;
 
 	wlr_log(L_DEBUG, "OUTPUT: Destroying output '%s'", output->name);