Streamlined pipelines

- Make UberShaderComponent#build NotNull
- Move index update and key creation logic to PipelineCompiler
- Always update index when a resource location is requested to fix
  MaterialEncoder misses
- Indices trigger pipeline compiler deletion when updated

common/src/backend/java/dev/engine_room/flywheel/backend/MaterialShaderIndices.java

@@ -6,6 +6,7 @@ import org.jetbrains.annotations.Unmodifiable;
 
 import dev.engine_room.flywheel.api.material.CutoutShader;
 import dev.engine_room.flywheel.api.material.FogShader;
+import dev.engine_room.flywheel.backend.compile.PipelineCompiler;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
@@ -45,23 +46,31 @@ public final class MaterialShaderIndices {
 			this.sources = new ObjectArrayList<>();
 		}
 
-		public void add(ResourceLocation source) {
+		public ResourceLocation get(int index) {
-			if (sources2Index.putIfAbsent(source, sources.size()) == -1) {
+			return sources.get(index);
-				sources.add(source);
-			}
 		}
 
 		public int index(ResourceLocation source) {
-			return sources2Index.getInt(source);
+			var out = sources2Index.getInt(source);
-		}
 
-		public ResourceLocation get(int index) {
+			if (out == -1) {
-			return sources.get(index);
+				add(source);
+				PipelineCompiler.deleteAll();
+				return sources2Index.getInt(source);
+			}
+
+			return out;
 		}
 
 		@Unmodifiable
 		public List<ResourceLocation> all() {
 			return sources;
 		}
+
+		private void add(ResourceLocation source) {
+			if (sources2Index.putIfAbsent(source, sources.size()) == -1) {
+				sources.add(source);
+			}
+		}
 	}
 }

common/src/backend/java/dev/engine_room/flywheel/backend/compile/IndirectPrograms.java

@@ -9,12 +9,10 @@ import com.google.common.collect.ImmutableList;
 import dev.engine_room.flywheel.api.Flywheel;
 import dev.engine_room.flywheel.api.instance.InstanceType;
 import dev.engine_room.flywheel.api.material.Material;
-import dev.engine_room.flywheel.backend.MaterialShaderIndices;
 import dev.engine_room.flywheel.backend.compile.component.InstanceStructComponent;
 import dev.engine_room.flywheel.backend.compile.component.SsboInstanceComponent;
 import dev.engine_room.flywheel.backend.compile.core.CompilationHarness;
 import dev.engine_room.flywheel.backend.compile.core.Compile;
-import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms;
 import dev.engine_room.flywheel.backend.engine.uniform.Uniforms;
 import dev.engine_room.flywheel.backend.gl.GlCompat;
 import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
@@ -44,11 +42,11 @@ public class IndirectPrograms extends AtomicReferenceCounted {
 	@Nullable
 	private static IndirectPrograms instance;
 
-	private final CompilationHarness<PipelineProgramKey> pipeline;
+	private final PipelineCompiler pipeline;
 	private final CompilationHarness<InstanceType<?>> culling;
 	private final CompilationHarness<ResourceLocation> utils;
 
-	private IndirectPrograms(CompilationHarness<PipelineProgramKey> pipeline, CompilationHarness<InstanceType<?>> culling, CompilationHarness<ResourceLocation> utils) {
+	private IndirectPrograms(PipelineCompiler pipeline, CompilationHarness<InstanceType<?>> culling, CompilationHarness<ResourceLocation> utils) {
 		this.pipeline = pipeline;
 		this.culling = culling;
 		this.utils = utils;
@@ -151,19 +149,7 @@ public class IndirectPrograms extends AtomicReferenceCounted {
 	}
 
 	public GlProgram getIndirectProgram(InstanceType<?> instanceType, ContextShader contextShader, Material material) {
-		var light = material.light();
+		return pipeline.get(instanceType, contextShader, material);
-		var cutout = material.cutout();
-		var shaders = material.shaders();
-		var fog = material.fog();
-
-		var fogIndex = MaterialShaderIndices.fogSources();
-		if (fogIndex.index(fog.source()) == -1) {
-			fogIndex.add(fog.source());
-			pipeline.delete();
-			PipelineCompiler.createFogComponent();
-		}
-
-		return pipeline.get(new PipelineProgramKey(instanceType, contextShader, light, cutout, shaders, FrameUniforms.debugOn()));
 	}
 
 	public GlProgram getCullingProgram(InstanceType<?> instanceType) {

common/src/backend/java/dev/engine_room/flywheel/backend/compile/InstancingPrograms.java

@@ -8,9 +8,6 @@ import com.google.common.collect.ImmutableList;
 
 import dev.engine_room.flywheel.api.instance.InstanceType;
 import dev.engine_room.flywheel.api.material.Material;
-import dev.engine_room.flywheel.backend.MaterialShaderIndices;
-import dev.engine_room.flywheel.backend.compile.core.CompilationHarness;
-import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms;
 import dev.engine_room.flywheel.backend.gl.GlCompat;
 import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
 import dev.engine_room.flywheel.backend.glsl.GlslVersion;
@@ -24,9 +21,9 @@ public class InstancingPrograms extends AtomicReferenceCounted {
 	@Nullable
 	private static InstancingPrograms instance;
 
-	private final CompilationHarness<PipelineProgramKey> pipeline;
+	private final PipelineCompiler pipeline;
 
-	private InstancingPrograms(CompilationHarness<PipelineProgramKey> pipeline) {
+	private InstancingPrograms(PipelineCompiler pipeline) {
 		this.pipeline = pipeline;
 	}
 
@@ -73,20 +70,7 @@ public class InstancingPrograms extends AtomicReferenceCounted {
 	}
 
 	public GlProgram get(InstanceType<?> instanceType, ContextShader contextShader, Material material) {
-		var light = material.light();
+		return pipeline.get(instanceType, contextShader, material);
-		var cutout = material.cutout();
-		var materialShaders = material.shaders();
-
-		var fog = material.fog();
-
-		var fogIndex = MaterialShaderIndices.fogSources();
-		if (fogIndex.index(fog.source()) == -1) {
-			fogIndex.add(fog.source());
-			pipeline.delete();
-			PipelineCompiler.createFogComponent();
-		}
-
-		return pipeline.get(new PipelineProgramKey(instanceType, contextShader, light, cutout, materialShaders, FrameUniforms.debugOn()));
 	}
 
 	@Override

common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineCompiler.java

@@ -4,6 +4,10 @@ import java.util.Collection;
 import java.util.List;
 
 import dev.engine_room.flywheel.api.Flywheel;
+import dev.engine_room.flywheel.api.instance.InstanceType;
+import dev.engine_room.flywheel.api.material.LightShader;
+import dev.engine_room.flywheel.api.material.Material;
+import dev.engine_room.flywheel.api.material.MaterialShaders;
 import dev.engine_room.flywheel.backend.BackendConfig;
 import dev.engine_room.flywheel.backend.InternalVertex;
 import dev.engine_room.flywheel.backend.MaterialShaderIndices;
@@ -12,6 +16,7 @@ import dev.engine_room.flywheel.backend.compile.component.InstanceStructComponen
 import dev.engine_room.flywheel.backend.compile.component.UberShaderComponent;
 import dev.engine_room.flywheel.backend.compile.core.CompilationHarness;
 import dev.engine_room.flywheel.backend.compile.core.Compile;
+import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms;
 import dev.engine_room.flywheel.backend.engine.uniform.Uniforms;
 import dev.engine_room.flywheel.backend.gl.GlCompat;
 import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
@@ -25,6 +30,8 @@ import dev.engine_room.flywheel.lib.util.ResourceUtil;
 import net.minecraft.resources.ResourceLocation;
 
 public final class PipelineCompiler {
+	private static final List<PipelineCompiler> ALL = List.of();
+
 	private static final Compile<PipelineProgramKey> PIPELINE = new Compile<>();
 
 	private static UberShaderComponent FOG;
@@ -33,11 +40,48 @@ public final class PipelineCompiler {
 	private static final ResourceLocation API_IMPL_VERT = Flywheel.rl("internal/api_impl.vert");
 	private static final ResourceLocation API_IMPL_FRAG = Flywheel.rl("internal/api_impl.frag");
 
-	static CompilationHarness<PipelineProgramKey> create(ShaderSources sources, Pipeline pipeline, List<SourceComponent> vertexComponents, List<SourceComponent> fragmentComponents, Collection<String> extensions) {
+	private final CompilationHarness<PipelineProgramKey> harness;
+
+	public PipelineCompiler(CompilationHarness<PipelineProgramKey> harness) {
+		this.harness = harness;
+	}
+
+	public GlProgram get(InstanceType<?> instanceType, ContextShader contextShader, Material material) {
+		var light = material.light();
+		var cutout = material.cutout();
+		var shaders = material.shaders();
+		var fog = material.fog();
+
+		// Tell fogSources to index the fog shader if we haven't seen it before.
+		// If it is new, this will trigger a deletion of all programs.
+		MaterialShaderIndices.fogSources()
+				.index(fog.source());
+
+		boolean useCutout = cutout != CutoutShaders.OFF;
+
+		if (useCutout) {
+			// Same thing for cutout.
+			MaterialShaderIndices.cutoutSources()
+					.index(cutout.source());
+		}
+
+		return harness.get(new PipelineProgramKey(instanceType, contextShader, light, shaders, useCutout, FrameUniforms.debugOn()));
+	}
+
+	public void delete() {
+		harness.delete();
+	}
+
+	public static void deleteAll() {
+		createFogComponent();
+		createCutoutComponent();
+		ALL.forEach(PipelineCompiler::delete);
+	}
+
+	static PipelineCompiler create(ShaderSources sources, Pipeline pipeline, List<SourceComponent> vertexComponents, List<SourceComponent> fragmentComponents, Collection<String> extensions) {
 		// We could technically compile every version of light smoothness ahead of time,
 		// but that seems unnecessary as I doubt most folks will be changing this option often.
-		var lightSmoothness = BackendConfig.INSTANCE.lightSmoothness();
+		var harness = PIPELINE.program()
-		return PIPELINE.program()
 				.link(PIPELINE.shader(GlCompat.MAX_GLSL_VERSION, ShaderType.VERTEX)
 						.nameMapper(key -> {
 							var instance = ResourceUtil.toDebugFileNameNoExtension(key.instanceType()
@@ -53,7 +97,8 @@ public final class PipelineCompiler {
 						.requireExtensions(extensions)
 						.onCompile((key, comp) -> key.contextShader()
 								.onCompile(comp))
-						.onCompile((key, comp) -> lightSmoothness.onCompile(comp))
+						.onCompile((key, comp) -> BackendConfig.INSTANCE.lightSmoothness()
+								.onCompile(comp))
 						.onCompile((key, comp) -> {
 							if (key.debugEnabled()) {
 								comp.define("_FLW_DEBUG");
@@ -78,26 +123,25 @@ public final class PipelineCompiler {
 							var material = ResourceUtil.toDebugFileNameNoExtension(key.materialShaders()
 									.fragmentSource());
 
-							var cutout = ResourceUtil.toDebugFileNameNoExtension(key.cutout()
-									.source());
-
 							var light = ResourceUtil.toDebugFileNameNoExtension(key.light()
 									.source());
 							var debug = key.debugEnabled() ? "_debug" : "";
-							return "pipeline/" + pipeline.compilerMarker() + "/frag/" + material + "/" + light + "_" + cutout + "_" + context + debug;
+							var cutout = key.useCutout() ? "_cutout" : "";
+							return "pipeline/" + pipeline.compilerMarker() + "/frag/" + material + "/" + light + "_" + context + cutout + debug;
 						})
 						.requireExtensions(extensions)
 						.enableExtension("GL_ARB_conservative_depth")
 						.onCompile((key, comp) -> key.contextShader()
 								.onCompile(comp))
-						.onCompile((key, comp) -> lightSmoothness.onCompile(comp))
+						.onCompile((key, comp) -> BackendConfig.INSTANCE.lightSmoothness()
+								.onCompile(comp))
 						.onCompile((key, comp) -> {
 							if (key.debugEnabled()) {
 								comp.define("_FLW_DEBUG");
 							}
 						})
 						.onCompile((key, comp) -> {
-							if (key.cutout() != CutoutShaders.OFF) {
+							if (key.useCutout()) {
 								comp.define("_FLW_USE_DISCARD");
 							}
 						})
@@ -108,8 +152,7 @@ public final class PipelineCompiler {
 						.withComponent(key -> FOG)
 						.withResource(key -> key.light()
 								.source())
-						.withResource(key -> key.cutout()
+						.with((key, fetcher) -> (key.useCutout() ? CUTOUT : fetcher.get(CutoutShaders.OFF.source())))
-								.source())
 						.withResource(pipeline.fragmentMain()))
 				.preLink((key, program) -> {
 					program.bindAttribLocation("_flw_aPos", 0);
@@ -135,6 +178,8 @@ public final class PipelineCompiler {
 					GlProgram.unbind();
 				})
 				.harness(pipeline.compilerMarker(), sources);
+
+		return new PipelineCompiler(harness);
 	}
 
 	public static void createFogComponent() {
@@ -162,4 +207,15 @@ public final class PipelineCompiler {
 				.switchOn(GlslExpr.variable("_flw_uberCutoutIndex"))
 				.build(FlwPrograms.SOURCES);
 	}
+
+	/**
+	 * Represents the entire context of a program's usage.
+	 *
+	 * @param instanceType  The instance shader to use.
+	 * @param contextShader The context shader to use.
+	 * @param light         The light shader to use.
+	 */
+	public record PipelineProgramKey(InstanceType<?> instanceType, ContextShader contextShader, LightShader light,
+									 MaterialShaders materialShaders, boolean useCutout, boolean debugEnabled) {
+	}
 }

common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineProgramKey.java

@@ -1,17 +0,0 @@
-package dev.engine_room.flywheel.backend.compile;
-
-import dev.engine_room.flywheel.api.instance.InstanceType;
-import dev.engine_room.flywheel.api.material.CutoutShader;
-import dev.engine_room.flywheel.api.material.LightShader;
-import dev.engine_room.flywheel.api.material.MaterialShaders;
-
-/**
- * Represents the entire context of a program's usage.
- *
- * @param instanceType  The instance shader to use.
- * @param contextShader The context shader to use.
- * @param light The light shader to use.
- */
-public record PipelineProgramKey(InstanceType<?> instanceType, ContextShader contextShader, LightShader light,
-								 CutoutShader cutout, MaterialShaders materialShaders, boolean debugEnabled) {
-}

common/src/backend/java/dev/engine_room/flywheel/backend/compile/component/UberShaderComponent.java

@@ -136,7 +136,6 @@ public class UberShaderComponent implements SourceComponent {
 			return this;
 		}
 
-		@Nullable
 		public UberShaderComponent build(ShaderSources sources) {
 			if (switchArg == null) {
 				throw new NullPointerException("Switch argument must be set");
@@ -144,24 +143,15 @@ public class UberShaderComponent implements SourceComponent {
 
 			var transformed = ImmutableList.<StringSubstitutionComponent>builder();
 
-			boolean errored = false;
 			int index = 0;
 			for (var rl : materialSources) {
 				SourceFile sourceFile = sources.get(rl);
-				if (sourceFile != null) {
+				final int finalIndex = index;
-					final int finalIndex = index;
+				var adapterMap = createAdapterMap(adaptedFunctions, fnName -> "_" + fnName + "_" + finalIndex);
-					var adapterMap = createAdapterMap(adaptedFunctions, fnName -> "_" + fnName + "_" + finalIndex);
+				transformed.add(new StringSubstitutionComponent(sourceFile, adapterMap));
-					transformed.add(new StringSubstitutionComponent(sourceFile, adapterMap));
-				} else {
-					errored = true;
-				}
 				index++;
 			}
 
-			if (errored) {
-				return null;
-			}
-
 			return new UberShaderComponent(name, switchArg, adaptedFunctions, transformed.build());
 		}