Stop playing hide and seek

- Dump stitched shaders to meaningful paths.
- Simplify dataflow to Compilation.
- Rename some compiler classes to better reflect what they do.
- Inline some utility methods on shader enums.

src/main/java/com/jozufozu/flywheel/backend/compile/IndirectPrograms.java

@@ -18,6 +18,7 @@ import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.backend.glsl.GlslVersion;
 import com.jozufozu.flywheel.backend.glsl.ShaderSources;
 import com.jozufozu.flywheel.backend.glsl.SourceComponent;
+import com.jozufozu.flywheel.lib.util.ResourceUtil;
 
 import net.minecraft.resources.ResourceLocation;
 
@@ -74,6 +75,7 @@ public class IndirectPrograms extends AbstractPrograms {
 	private static CompilationHarness<InstanceType<?>> createCullingCompiler(ShaderSources sources) {
 		return CULL.program()
 				.link(CULL.shader(GlslVersion.V460, ShaderType.COMPUTE)
+						.nameMapper(instanceType -> "culling/" + ResourceUtil.toDebugFileNameNoExtension(instanceType.cullShader()))
 						.define("_FLW_SUBGROUP_SIZE", GlCompat.SUBGROUP_SIZE)
 						.withResource(CULL_SHADER_HEADER)
 						.withComponent(IndirectComponent::create)
@@ -86,6 +88,7 @@ public class IndirectPrograms extends AbstractPrograms {
 	private static CompilationHarness<ResourceLocation> createUtilCompiler(ShaderSources sources) {
 		return UTIL.program()
 				.link(UTIL.shader(GlslVersion.V460, ShaderType.COMPUTE)
+						.nameMapper(resourceLocation -> "utilities/" + ResourceUtil.toDebugFileNameNoExtension(resourceLocation))
 						.define("_FLW_SUBGROUP_SIZE", GlCompat.SUBGROUP_SIZE)
 						.withResource(s -> s))
 				.harness("utilities", sources);

src/main/java/com/jozufozu/flywheel/backend/compile/PipelineCompiler.java

@@ -8,6 +8,7 @@ import com.jozufozu.flywheel.backend.compile.core.Compile;
 import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.backend.glsl.ShaderSources;
 import com.jozufozu.flywheel.backend.glsl.SourceComponent;
+import com.jozufozu.flywheel.lib.util.ResourceUtil;
 
 public class PipelineCompiler {
 	private static final Compile<PipelineProgramKey> PIPELINE = new Compile<>();
@@ -15,6 +16,14 @@ public class PipelineCompiler {
 	static CompilationHarness<PipelineProgramKey> create(ShaderSources sources, Pipeline pipeline, List<SourceComponent> vertexComponents, List<SourceComponent> fragmentComponents) {
 		return PIPELINE.program()
 				.link(PIPELINE.shader(pipeline.glslVersion(), ShaderType.VERTEX)
+						.nameMapper(key -> {
+							var instance = ResourceUtil.toDebugFileNameNoExtension(key.instanceType()
+									.vertexShader());
+
+							var context = ResourceUtil.toDebugFileNameNoExtension(key.contextShader()
+									.vertexShader());
+							return "pipeline/" + pipeline.compilerMarker() + "/" + instance + "_" + context;
+						})
 						.withResource(pipeline.vertexApiImpl())
 						.withResource(InternalVertex.LAYOUT_SHADER)
 						.withComponent(key -> pipeline.assembler()
@@ -26,6 +35,14 @@ public class PipelineCompiler {
 								.vertexShader())
 						.withResource(pipeline.vertexMain()))
 				.link(PIPELINE.shader(pipeline.glslVersion(), ShaderType.FRAGMENT)
+						.nameMapper(key -> {
+							var instance = ResourceUtil.toDebugFileNameNoExtension(key.instanceType()
+									.vertexShader());
+
+							var context = ResourceUtil.toDebugFileNameNoExtension(key.contextShader()
+									.fragmentShader());
+							return "pipeline/" + pipeline.compilerMarker() + "/" + instance + "_" + context;
+						})
 						.enableExtension("GL_ARB_conservative_depth")
 						.withResource(pipeline.fragmentApiImpl())
 						.withComponents(fragmentComponents)

src/main/java/com/jozufozu/flywheel/backend/compile/core/Compilation.java

@@ -29,30 +29,20 @@ public class Compilation {
 	public static final boolean DUMP_SHADER_SOURCE = System.getProperty("flw.dumpShaderSource") != null;
 
 	private final List<SourceFile> files = new ArrayList<>();
-	private final StringBuilder generatedSource;
-	private final StringBuilder fullSource;
-	private final GlslVersion glslVersion;
-	private final ShaderType shaderType;
+	private final StringBuilder generatedSource = new StringBuilder();
+	private final StringBuilder fullSource = new StringBuilder();
 	private int generatedLines = 0;
 
-	public Compilation(GlslVersion glslVersion, ShaderType shaderType) {
-		this.glslVersion = glslVersion;
-		this.shaderType = shaderType;
-
-		generatedSource = new StringBuilder();
-		fullSource = new StringBuilder(glslVersion.getVersionLine()).append(shaderType.getDefineStatement()).append('\n');
-	}
-
 	@NotNull
-	public ShaderResult compile() {
+	public ShaderResult compile(ShaderType shaderType, String name) {
 		int handle = GL20.glCreateShader(shaderType.glEnum);
 		var source = fullSource.toString();
 
 		GlCompat.safeShaderSource(handle, source);
 		GL20.glCompileShader(handle);
 
-		var shaderName = shaderType.name + glslVersion + '_' + Integer.toUnsignedString(source.hashCode());
-		dumpSource(source, shaderType.getFileName(shaderName));
+		var shaderName = name + "." + shaderType.extension;
+		dumpSource(source, shaderName);
 
 		var infoLog = GL20.glGetShaderInfoLog(handle);
 
@@ -64,6 +54,12 @@ public class Compilation {
 		return ShaderResult.failure(new FailedCompilation(shaderName, files, generatedSource.toString(), source, infoLog));
 	}
 
+	public void version(GlslVersion version) {
+		fullSource.append("#version ")
+				.append(version.version)
+				.append('\n');
+	}
+
 	public void enableExtension(String ext) {
 		fullSource.append("#extension ")
 				.append(ext)
@@ -78,6 +74,12 @@ public class Compilation {
 				.append('\n');
 	}
 
+	public void define(String key) {
+		fullSource.append("#define ")
+				.append(key)
+				.append('\n');
+	}
+
 	public void appendComponent(SourceComponent component) {
 		var source = component.source();
 
@@ -117,9 +119,10 @@ public class Compilation {
 			return;
 		}
 
-		File dir = new File(Minecraft.getInstance().gameDirectory, "flywheel_sources");
-		dir.mkdirs();
-		File file = new File(dir, fileName);
+		File file = new File(new File(Minecraft.getInstance().gameDirectory, "flywheel_sources"), fileName);
+		// mkdirs of the parent so we don't create a directory named by the leaf file we want to write
+		file.getParentFile()
+				.mkdirs();
 		try (FileWriter writer = new FileWriter(file)) {
 			writer.write(source);
 		} catch (Exception e) {

src/main/java/com/jozufozu/flywheel/backend/compile/core/CompilationHarness.java

@@ -12,7 +12,7 @@ import com.jozufozu.flywheel.backend.glsl.ShaderSources;
 public class CompilationHarness<K> {
 	private final KeyCompiler<K> compiler;
 	private final SourceLoader sourceLoader;
-	private final ShaderCompiler shaderCompiler;
+	private final ShaderCache shaderCache;
 	private final ProgramLinker programLinker;
 	private final CompilerStats stats;
 
@@ -20,7 +20,7 @@ public class CompilationHarness<K> {
 		this.compiler = compiler;
 		stats = new CompilerStats(marker);
 		sourceLoader = new SourceLoader(sources, stats);
-		shaderCompiler = new ShaderCompiler(stats);
+		shaderCache = new ShaderCache(stats);
 		programLinker = new ProgramLinker(stats);
 	}
 
@@ -29,7 +29,7 @@ public class CompilationHarness<K> {
 		stats.start();
 		Map<K, GlProgram> out = new HashMap<>();
 		for (var key : keys) {
-			GlProgram glProgram = compiler.compile(key, sourceLoader, shaderCompiler, programLinker);
+			GlProgram glProgram = compiler.compile(key, sourceLoader, shaderCache, programLinker);
 			if (out != null && glProgram != null) {
 				out.put(key, glProgram);
 			} else {
@@ -47,10 +47,10 @@ public class CompilationHarness<K> {
 	}
 
 	public void delete() {
-		shaderCompiler.delete();
+		shaderCache.delete();
 	}
 
 	public interface KeyCompiler<K> {
-		@Nullable GlProgram compile(K key, SourceLoader loader, ShaderCompiler shaderCompiler, ProgramLinker programLinker);
+		@Nullable GlProgram compile(K key, SourceLoader loader, ShaderCache shaderCache, ProgramLinker programLinker);
 	}
 }

src/main/java/com/jozufozu/flywheel/backend/compile/core/Compile.java

@@ -33,16 +33,16 @@ import net.minecraft.resources.ResourceLocation;
  * @param <K> The type of the key used to compile shaders.
  */
 public class Compile<K> {
-	public ShaderCompilerBuilder<K> shader(GlslVersion glslVersion, ShaderType shaderType) {
-		return new ShaderCompilerBuilder<>(glslVersion, shaderType);
+	public ShaderCompiler<K> shader(GlslVersion glslVersion, ShaderType shaderType) {
+		return new ShaderCompiler<>(glslVersion, shaderType);
 	}
 
-	public ProgramLinkBuilder<K> program() {
-		return new ProgramLinkBuilder<>();
+	public ProgramStitcher<K> program() {
+		return new ProgramStitcher<>();
 	}
 
-	public static class ProgramLinkBuilder<K> implements CompilationHarness.KeyCompiler<K> {
-		private final Map<ShaderType, ShaderCompilerBuilder<K>> compilers = new EnumMap<>(ShaderType.class);
+	public static class ProgramStitcher<K> implements CompilationHarness.KeyCompiler<K> {
+		private final Map<ShaderType, ShaderCompiler<K>> compilers = new EnumMap<>(ShaderType.class);
 		private BiConsumer<K, GlProgram> onLink = (k, p) -> {
 		};
 
@@ -50,7 +50,7 @@ public class Compile<K> {
 			return new CompilationHarness<>(marker, sources, this);
 		}
 
-		public ProgramLinkBuilder<K> link(ShaderCompilerBuilder<K> compilerBuilder) {
+		public ProgramStitcher<K> link(ShaderCompiler<K> compilerBuilder) {
 			if (compilers.containsKey(compilerBuilder.shaderType)) {
 				throw new IllegalArgumentException("Duplicate shader type: " + compilerBuilder.shaderType);
 			}
@@ -58,14 +58,14 @@ public class Compile<K> {
 			return this;
 		}
 
-		public ProgramLinkBuilder<K> then(BiConsumer<K, GlProgram> onLink) {
+		public ProgramStitcher<K> then(BiConsumer<K, GlProgram> onLink) {
 			this.onLink = onLink;
 			return this;
 		}
 
 		@Override
 		@Nullable
-		public GlProgram compile(K key, SourceLoader loader, ShaderCompiler shaderCompiler, ProgramLinker programLinker) {
+		public GlProgram compile(K key, SourceLoader loader, ShaderCache shaderCache, ProgramLinker programLinker) {
 			if (compilers.isEmpty()) {
 				throw new IllegalStateException("No shader compilers were added!");
 			}
@@ -73,8 +73,8 @@ public class Compile<K> {
 			List<GlShader> shaders = new ArrayList<>();
 
 			boolean ok = true;
-			for (ShaderCompilerBuilder<K> compiler : compilers.values()) {
-				var shader = compiler.compile(key, shaderCompiler, loader);
+			for (ShaderCompiler<K> compiler : compilers.values()) {
+				var shader = compiler.compile(key, shaderCache, loader);
 				if (shader == null) {
 					ok = false;
 				}
@@ -95,59 +95,65 @@ public class Compile<K> {
 		}
 	}
 
-	public static class ShaderCompilerBuilder<K> {
+	public static class ShaderCompiler<K> {
 		private final GlslVersion glslVersion;
 		private final ShaderType shaderType;
+		private final List<BiFunction<K, SourceLoader, SourceComponent>> fetchers = new ArrayList<>();
 		private Consumer<Compilation> compilationCallbacks = $ -> {
 		};
-		private final List<BiFunction<K, SourceLoader, SourceComponent>> fetchers = new ArrayList<>();
+		private Function<K, String> nameMapper = Object::toString;
 
-		public ShaderCompilerBuilder(GlslVersion glslVersion, ShaderType shaderType) {
+		public ShaderCompiler(GlslVersion glslVersion, ShaderType shaderType) {
 			this.glslVersion = glslVersion;
 			this.shaderType = shaderType;
 		}
 
-		public ShaderCompilerBuilder<K> with(BiFunction<K, SourceLoader, SourceComponent> fetch) {
+		public ShaderCompiler<K> nameMapper(Function<K, String> nameMapper) {
+			this.nameMapper = nameMapper;
+			return this;
+		}
+
+		public ShaderCompiler<K> with(BiFunction<K, SourceLoader, SourceComponent> fetch) {
 			fetchers.add(fetch);
 			return this;
 		}
 
-		public ShaderCompilerBuilder<K> withComponents(Collection<SourceComponent> components) {
+		public ShaderCompiler<K> withComponents(Collection<SourceComponent> components) {
 			components.forEach(this::withComponent);
 			return this;
 		}
 
-		public ShaderCompilerBuilder<K> withComponent(SourceComponent component) {
+		public ShaderCompiler<K> withComponent(SourceComponent component) {
 			return withComponent($ -> component);
 		}
 
-		public ShaderCompilerBuilder<K> withComponent(Function<K, @NotNull SourceComponent> sourceFetcher) {
+		public ShaderCompiler<K> withComponent(Function<K, @NotNull SourceComponent> sourceFetcher) {
 			return with((key, $) -> sourceFetcher.apply(key));
 		}
 
-		public ShaderCompilerBuilder<K> withResource(Function<K, @NotNull ResourceLocation> sourceFetcher) {
+		public ShaderCompiler<K> withResource(Function<K, @NotNull ResourceLocation> sourceFetcher) {
 			return with((key, loader) -> loader.find(sourceFetcher.apply(key)));
 		}
 
-		public ShaderCompilerBuilder<K> withResource(ResourceLocation resourceLocation) {
+		public ShaderCompiler<K> withResource(ResourceLocation resourceLocation) {
 			return withResource($ -> resourceLocation);
 		}
 
-		public ShaderCompilerBuilder<K> onCompile(Consumer<Compilation> cb) {
+		public ShaderCompiler<K> onCompile(Consumer<Compilation> cb) {
 			compilationCallbacks = compilationCallbacks.andThen(cb);
 			return this;
 		}
 
-		public ShaderCompilerBuilder<K> define(String def, int value) {
+		public ShaderCompiler<K> define(String def, int value) {
 			return onCompile(ctx -> ctx.define(def, String.valueOf(value)));
 		}
 
-		public ShaderCompilerBuilder<K> enableExtension(String extension) {
+		public ShaderCompiler<K> enableExtension(String extension) {
 			return onCompile(ctx -> ctx.enableExtension(extension));
 		}
 
 		@Nullable
-		private GlShader compile(K key, ShaderCompiler compiler, SourceLoader loader) {
+		private GlShader compile(K key, ShaderCache compiler, SourceLoader loader) {
 			var components = new ArrayList<SourceComponent>();
 			boolean ok = true;
 			for (var fetcher : fetchers) {
@@ -162,7 +168,7 @@ public class Compile<K> {
 				return null;
 			}
 
-			return compiler.compile(glslVersion, shaderType, compilationCallbacks, components);
+			return compiler.compile(glslVersion, shaderType, nameMapper.apply(key), compilationCallbacks, components);
 		}
 	}
 }

src/main/java/com/jozufozu/flywheel/backend/compile/core/ShaderCache.java

@@ -15,36 +15,38 @@ import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.backend.glsl.GlslVersion;
 import com.jozufozu.flywheel.backend.glsl.SourceComponent;
 
-public class ShaderCompiler {
-	private final Map<ShaderKey, ShaderResult> shaderCache = new HashMap<>();
+public class ShaderCache {
+	private final Map<ShaderKey, ShaderResult> inner = new HashMap<>();
 	private final CompilerStats stats;
 
-	public ShaderCompiler(CompilerStats stats) {
+	public ShaderCache(CompilerStats stats) {
 		this.stats = stats;
 	}
 
 	@Nullable
-	public GlShader compile(GlslVersion glslVersion, ShaderType shaderType, Consumer<Compilation> callback, List<SourceComponent> sourceComponents) {
+	public GlShader compile(GlslVersion glslVersion, ShaderType shaderType, String name, Consumer<Compilation> callback, List<SourceComponent> sourceComponents) {
 		var key = new ShaderKey(glslVersion, shaderType, sourceComponents);
-		var cached = shaderCache.get(key);
+		var cached = inner.get(key);
 		if (cached != null) {
 			return cached.unwrap();
 		}
 
-		Compilation ctx = new Compilation(glslVersion, shaderType);
+		Compilation ctx = new Compilation();
+		ctx.version(glslVersion);
+		ctx.define(shaderType.define);
 
 		callback.accept(ctx);
 
 		expand(sourceComponents, ctx::appendComponent);
 
-		ShaderResult out = ctx.compile();
-		shaderCache.put(key, out);
+		ShaderResult out = ctx.compile(shaderType, name);
+		inner.put(key, out);
 		stats.shaderResult(out);
 		return out.unwrap();
 	}
 
 	public void delete() {
-		shaderCache.values()
+		inner.values()
 				.stream()
 				.map(ShaderResult::unwrap)
 				.filter(Objects::nonNull)

src/main/java/com/jozufozu/flywheel/backend/gl/shader/ShaderType.java

@@ -20,12 +20,4 @@ public enum ShaderType {
 		this.extension = extension;
 		this.glEnum = glEnum;
 	}
-
-	public String getDefineStatement() {
-		return "#define " + define + "\n";
-	}
-
-	public String getFileName(String baseName) {
-		return baseName + "." + extension;
-	}
 }

src/main/java/com/jozufozu/flywheel/backend/glsl/GlslVersion.java

@@ -27,7 +27,4 @@ public enum GlslVersion {
 		return Integer.toString(version);
 	}
 
-	public String getVersionLine() {
-		return "#version " + version + '\n';
-	}
 }

src/main/java/com/jozufozu/flywheel/lib/util/ResourceUtil.java

@@ -1,5 +1,7 @@
 package com.jozufozu.flywheel.lib.util;
 
+import org.jetbrains.annotations.NotNull;
+
 import com.jozufozu.flywheel.Flywheel;
 import com.mojang.brigadier.StringReader;
 import com.mojang.brigadier.exceptions.CommandSyntaxException;
@@ -52,4 +54,12 @@ public final class ResourceUtil {
 		   throw ERROR_INVALID.createWithContext(reader);
 		}
 	}
+
+	@NotNull
+	public static String toDebugFileNameNoExtension(ResourceLocation resourceLocation) {
+		var stringLoc = resourceLocation.toString();
+		return stringLoc.substring(0, stringLoc.lastIndexOf('.'))
+				.replace('/', '_')
+				.replace(':', '_');
+	}
 }