Errors galore

- Make everything in the compiler chain's results not null
- Throw errors immediately when encountered
- Log error messages when falling back
- Do not eagerly grab utility programs in IndirectDrawManager so we can
  actually catch errors and fall back
- Remove CompilerStats
This commit is contained in:
Jozufozu 2024-09-28 16:00:23 -07:00
parent 11ce4ac185
commit ef05f7d3fd
16 changed files with 103 additions and 216 deletions

View File

@ -6,7 +6,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import dev.engine_room.flywheel.api.Flywheel;
import dev.engine_room.flywheel.backend.compile.core.CompilerStats;
import dev.engine_room.flywheel.backend.glsl.ShaderSources;
import dev.engine_room.flywheel.backend.glsl.SourceComponent;
import net.minecraft.resources.ResourceLocation;
@ -29,17 +28,9 @@ public final class FlwPrograms {
var sources = new ShaderSources(resourceManager);
SOURCES = sources;
var stats = new CompilerStats("ubershaders");
var fragmentComponentsHeader = sources.get(COMPONENTS_HEADER_FRAG);
// TODO: separate compilation for cutout OFF, but keep the rest uber'd?
if (stats.errored() || fragmentComponentsHeader == null) {
// Probably means the shader sources are missing.
stats.emitErrorLog();
return;
}
List<SourceComponent> vertexComponents = List.of();
List<SourceComponent> fragmentComponents = List.of(fragmentComponentsHeader);

View File

@ -43,7 +43,6 @@ public class InstancingPrograms extends AtomicReferenceCounted {
return;
}
var pipelineCompiler = PipelineCompiler.create(sources, Pipelines.INSTANCING, vertexComponents, fragmentComponents, EXTENSIONS);
InstancingPrograms newInstance = new InstancingPrograms(pipelineCompiler);

View File

@ -3,8 +3,6 @@ package dev.engine_room.flywheel.backend.compile.core;
import java.util.HashMap;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.gl.GlObject;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
import dev.engine_room.flywheel.backend.glsl.ShaderSources;
@ -14,16 +12,14 @@ public class CompilationHarness<K> {
private final KeyCompiler<K> compiler;
private final ShaderCache shaderCache;
private final ProgramLinker programLinker;
private final CompilerStats stats;
private final Map<K, GlProgram> programs = new HashMap<>();
public CompilationHarness(String marker, ShaderSources sources, KeyCompiler<K> compiler) {
this.sources = sources;
this.compiler = compiler;
stats = new CompilerStats(marker);
shaderCache = new ShaderCache(stats);
programLinker = new ProgramLinker(stats);
shaderCache = new ShaderCache();
programLinker = new ProgramLinker();
}
public GlProgram get(K key) {
@ -31,14 +27,7 @@ public class CompilationHarness<K> {
}
private GlProgram compile(K key) {
var out = compiler.compile(key, sources, shaderCache, programLinker);
if (out == null) {
// TODO: populate exception with error details
throw new ShaderException();
}
return out;
return compiler.compile(key, sources, shaderCache, programLinker);
}
public void delete() {
@ -51,6 +40,6 @@ public class CompilationHarness<K> {
}
public interface KeyCompiler<K> {
@Nullable GlProgram compile(K key, ShaderSources loader, ShaderCache shaderCache, ProgramLinker programLinker);
GlProgram compile(K key, ShaderSources loader, ShaderCache shaderCache, ProgramLinker programLinker);
}
}

View File

@ -10,8 +10,6 @@ import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.compile.FlwPrograms;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
import dev.engine_room.flywheel.backend.gl.shader.GlShader;
@ -44,7 +42,7 @@ public class Compile<K> {
public static class ShaderCompiler<K> {
private final GlslVersion glslVersion;
private final ShaderType shaderType;
private final List<BiFunction<K, ShaderSources, @Nullable SourceComponent>> fetchers = new ArrayList<>();
private final List<BiFunction<K, ShaderSources, SourceComponent>> fetchers = new ArrayList<>();
private BiConsumer<K, Compilation> compilationCallbacks = ($, $$) -> {
};
private Function<K, String> nameMapper = Object::toString;
@ -59,21 +57,21 @@ public class Compile<K> {
return this;
}
public ShaderCompiler<K> with(BiFunction<K, ShaderSources, @Nullable SourceComponent> fetch) {
public ShaderCompiler<K> with(BiFunction<K, ShaderSources, SourceComponent> fetch) {
fetchers.add(fetch);
return this;
}
public ShaderCompiler<K> withComponents(Collection<@Nullable SourceComponent> components) {
public ShaderCompiler<K> withComponents(Collection<SourceComponent> components) {
components.forEach(this::withComponent);
return this;
}
public ShaderCompiler<K> withComponent(@Nullable SourceComponent component) {
public ShaderCompiler<K> withComponent(SourceComponent component) {
return withComponent($ -> component);
}
public ShaderCompiler<K> withComponent(Function<K, @Nullable SourceComponent> sourceFetcher) {
public ShaderCompiler<K> withComponent(Function<K, SourceComponent> sourceFetcher) {
return with((key, $) -> sourceFetcher.apply(key));
}
@ -122,22 +120,12 @@ public class Compile<K> {
});
}
@Nullable
private GlShader compile(K key, ShaderCache compiler, ShaderSources loader) {
long start = System.nanoTime();
var components = new ArrayList<SourceComponent>();
boolean ok = true;
for (var fetcher : fetchers) {
SourceComponent apply = fetcher.apply(key, loader);
if (apply == null) {
ok = false;
}
components.add(apply);
}
if (!ok) {
return null;
components.add(fetcher.apply(key, loader));
}
Consumer<Compilation> cb = ctx -> compilationCallbacks.accept(key, ctx);
@ -182,7 +170,6 @@ public class Compile<K> {
}
@Override
@Nullable
public GlProgram compile(K key, ShaderSources loader, ShaderCache shaderCache, ProgramLinker programLinker) {
if (compilers.isEmpty()) {
throw new IllegalStateException("No shader compilers were added!");
@ -192,24 +179,13 @@ public class Compile<K> {
List<GlShader> shaders = new ArrayList<>();
boolean ok = true;
for (ShaderCompiler<K> compiler : compilers.values()) {
var shader = compiler.compile(key, shaderCache, loader);
if (shader == null) {
ok = false;
}
shaders.add(shader);
}
if (!ok) {
return null;
shaders.add(compiler.compile(key, shaderCache, loader));
}
var out = programLinker.link(shaders, p -> preLink.accept(key, p));
if (out != null) {
postLink.accept(key, out);
}
postLink.accept(key, out);
long end = System.nanoTime();

View File

@ -1,106 +0,0 @@
package dev.engine_room.flywheel.backend.compile.core;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.slf4j.Marker;
import org.slf4j.MarkerFactory;
import dev.engine_room.flywheel.backend.compile.FlwPrograms;
import dev.engine_room.flywheel.backend.glsl.LoadError;
import dev.engine_room.flywheel.backend.glsl.LoadResult;
import dev.engine_room.flywheel.backend.glsl.error.ErrorBuilder;
import dev.engine_room.flywheel.lib.util.StringUtil;
public class CompilerStats {
private final Marker marker;
private long compileStart;
private final Set<LoadError> loadErrors = new HashSet<>();
private final List<FailedCompilation> shaderErrors = new ArrayList<>();
private final List<String> programErrors = new ArrayList<>();
private boolean errored = false;
private int shaderCount = 0;
private int programCount = 0;
public CompilerStats(String marker) {
this.marker = MarkerFactory.getMarker(marker);
}
public void start() {
compileStart = System.nanoTime();
}
public void finish() {
long compileEnd = System.nanoTime();
var elapsed = StringUtil.formatTime(compileEnd - compileStart);
FlwPrograms.LOGGER.info(marker, "Compiled %d programs (with %d link errors) and %d shaders (with %d compile errors) in %s".formatted(programCount, programErrors.size(), shaderCount, shaderErrors.size(), elapsed));
}
public boolean errored() {
return errored;
}
public void emitErrorLog() {
String out = "";
if (!loadErrors.isEmpty()) {
out += "\nErrors loading sources:\n" + loadErrors();
}
if (!shaderErrors.isEmpty()) {
out += "\nShader compilation errors:\n" + compileErrors();
}
if (!programErrors.isEmpty()) {
out += "\nProgram link errors:\n" + linkErrors();
}
FlwPrograms.LOGGER.error(marker, out);
}
private String compileErrors() {
return shaderErrors.stream()
.map(FailedCompilation::generateMessage)
.collect(Collectors.joining("\n"));
}
private String linkErrors() {
return String.join("\n", programErrors);
}
private String loadErrors() {
return loadErrors.stream()
.map(LoadError::generateMessage)
.map(ErrorBuilder::build)
.collect(Collectors.joining("\n"));
}
public void shaderResult(ShaderResult result) {
if (result instanceof ShaderResult.Failure f) {
shaderErrors.add(f.failure());
errored = true;
}
shaderCount++;
}
public void linkResult(LinkResult linkResult) {
if (linkResult instanceof LinkResult.Failure f) {
programErrors.add(f.failure());
errored = true;
}
programCount++;
}
public void loadResult(LoadResult loadResult) {
if (loadResult instanceof LoadResult.Failure f) {
loadErrors.add(f.error());
errored = true;
}
}
}

View File

@ -1,15 +1,11 @@
package dev.engine_room.flywheel.backend.compile.core;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
public sealed interface LinkResult {
@Nullable
default GlProgram unwrap() {
return null;
}
GlProgram unwrap();
record Success(GlProgram program, String log) implements LinkResult {
@Override
@ -20,6 +16,10 @@ public sealed interface LinkResult {
}
record Failure(String failure) implements LinkResult {
@Override
public GlProgram unwrap() {
throw new ShaderException.Link(failure);
}
}
static LinkResult success(GlProgram program, String log) {

View File

@ -11,24 +11,17 @@ import static org.lwjgl.opengl.GL20.glLinkProgram;
import java.util.List;
import java.util.function.Consumer;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
import dev.engine_room.flywheel.backend.gl.shader.GlShader;
public class ProgramLinker {
private final CompilerStats stats;
public ProgramLinker(CompilerStats stats) {
this.stats = stats;
public ProgramLinker() {
}
@Nullable
public GlProgram link(List<GlShader> shaders, Consumer<GlProgram> preLink) {
// this probably doesn't need caching
var linkResult = linkInternal(shaders, preLink);
stats.linkResult(linkResult);
return linkResult.unwrap();
return linkInternal(shaders, preLink).unwrap();
}
private LinkResult linkInternal(List<GlShader> shaders, Consumer<GlProgram> preLink) {

View File

@ -4,12 +4,9 @@ import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.gl.shader.GlShader;
import dev.engine_room.flywheel.backend.gl.shader.ShaderType;
import dev.engine_room.flywheel.backend.glsl.GlslVersion;
@ -17,13 +14,10 @@ import dev.engine_room.flywheel.backend.glsl.SourceComponent;
public class ShaderCache {
private final Map<ShaderKey, ShaderResult> inner = new HashMap<>();
private final CompilerStats stats;
public ShaderCache(CompilerStats stats) {
this.stats = stats;
public ShaderCache() {
}
@Nullable
public GlShader compile(GlslVersion glslVersion, ShaderType shaderType, String name, Consumer<Compilation> callback, List<SourceComponent> sourceComponents) {
var key = new ShaderKey(glslVersion, shaderType, name);
var cached = inner.get(key);
@ -41,15 +35,14 @@ public class ShaderCache {
ShaderResult out = ctx.compile(shaderType, name);
inner.put(key, out);
stats.shaderResult(out);
return out.unwrap();
}
public void delete() {
inner.values()
.stream()
.filter(r -> r instanceof ShaderResult.Success)
.map(ShaderResult::unwrap)
.filter(Objects::nonNull)
.forEach(GlShader::delete);
inner.clear();
}

View File

@ -1,4 +1,57 @@
package dev.engine_room.flywheel.backend.compile.core;
public class ShaderException extends RuntimeException {
public ShaderException(String message) {
super(message);
}
public ShaderException(String message, Throwable cause) {
super(message, cause);
}
public ShaderException(Throwable cause) {
super(cause);
}
public static class Link extends ShaderException {
public Link(String message) {
super(message);
}
public Link(String message, Throwable cause) {
super(message, cause);
}
public Link(Throwable cause) {
super(cause);
}
}
public static class Compile extends ShaderException {
public Compile(String message) {
super(message);
}
public Compile(String message, Throwable cause) {
super(message, cause);
}
public Compile(Throwable cause) {
super(cause);
}
}
public static class Load extends ShaderException {
public Load(String message) {
super(message);
}
public Load(String message, Throwable cause) {
super(message, cause);
}
public Load(Throwable cause) {
super(cause);
}
}
}

View File

@ -1,25 +1,22 @@
package dev.engine_room.flywheel.backend.compile.core;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.gl.shader.GlShader;
public sealed interface ShaderResult {
@Nullable
default GlShader unwrap() {
return null;
}
GlShader unwrap();
record Success(GlShader shader, String infoLog) implements ShaderResult {
@Override
@NotNull
public GlShader unwrap() {
return shader;
}
}
record Failure(FailedCompilation failure) implements ShaderResult {
@Override
public GlShader unwrap() {
throw new ShaderException.Compile(failure.generateMessage());
}
}
static ShaderResult success(GlShader program, String infoLog) {

View File

@ -13,6 +13,7 @@ import dev.engine_room.flywheel.api.task.Plan;
import dev.engine_room.flywheel.api.visualization.VisualEmbedding;
import dev.engine_room.flywheel.api.visualization.VisualType;
import dev.engine_room.flywheel.api.visualization.VisualizationContext;
import dev.engine_room.flywheel.backend.FlwBackend;
import dev.engine_room.flywheel.backend.compile.core.ShaderException;
import dev.engine_room.flywheel.backend.engine.embed.EmbeddedEnvironment;
import dev.engine_room.flywheel.backend.engine.embed.Environment;
@ -93,6 +94,7 @@ public class EngineImpl implements Engine {
environmentStorage.flush();
drawManager.flush(lightStorage, environmentStorage);
} catch (ShaderException e) {
FlwBackend.LOGGER.error("Falling back", e);
triggerFallback();
}
}
@ -102,6 +104,7 @@ public class EngineImpl implements Engine {
try (var state = GlStateTracker.getRestoreState()) {
drawManager.render(visualType);
} catch (ShaderException e) {
FlwBackend.LOGGER.error("Falling back", e);
triggerFallback();
}
}
@ -111,6 +114,7 @@ public class EngineImpl implements Engine {
try (var state = GlStateTracker.getRestoreState()) {
drawManager.renderCrumbling(crumblingBlocks);
} catch (ShaderException e) {
FlwBackend.LOGGER.error("Falling back", e);
triggerFallback();
}
}

View File

@ -5,23 +5,21 @@ import org.lwjgl.opengl.GL46;
import com.mojang.blaze3d.platform.GlStateManager;
import dev.engine_room.flywheel.backend.compile.IndirectPrograms;
import dev.engine_room.flywheel.backend.gl.GlTextureUnit;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
import dev.engine_room.flywheel.lib.math.MoreMath;
import net.minecraft.client.Minecraft;
public class DepthPyramid {
private final GlProgram downsampleFirstProgram;
private final GlProgram downsampleSecondProgram;
private final IndirectPrograms programs;
public int pyramidTextureId = -1;
private int lastWidth = -1;
private int lastHeight = -1;
public DepthPyramid(GlProgram downsampleFirstProgram, GlProgram downsampleSecondProgram) {
this.downsampleFirstProgram = downsampleFirstProgram;
this.downsampleSecondProgram = downsampleSecondProgram;
public DepthPyramid(IndirectPrograms programs) {
this.programs = programs;
}
public void generate() {
@ -42,6 +40,7 @@ public class DepthPyramid {
GlTextureUnit.T0.makeActive();
GlStateManager._bindTexture(depthBufferId);
var downsampleFirstProgram = programs.getDownsampleFirstProgram();
downsampleFirstProgram.bind();
downsampleFirstProgram.setUInt("max_mip_level", mipLevels);
@ -59,6 +58,7 @@ public class DepthPyramid {
GL46.glMemoryBarrier(GL46.GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
var downsampleSecondProgram = programs.getDownsampleSecondProgram();
downsampleSecondProgram.bind();
downsampleSecondProgram.setUInt("max_mip_level", mipLevels);

View File

@ -56,6 +56,8 @@ public class IndirectDrawManager extends DrawManager<IndirectInstancer<?>> {
this.programs = programs;
programs.acquire();
// WARN: We should avoid eagerly grabbing GlPrograms here as catching compile
// errors and falling back during construction is a bit more complicated.
stagingBuffer = new StagingBuffer(this.programs);
meshPool = new MeshPool();
vertexArray = GlVertexArray.create();
@ -63,7 +65,7 @@ public class IndirectDrawManager extends DrawManager<IndirectInstancer<?>> {
lightBuffers = new LightBuffers();
matrixBuffer = new MatrixBuffer();
depthPyramid = new DepthPyramid(programs.getDownsampleFirstProgram(), programs.getDownsampleSecondProgram());
depthPyramid = new DepthPyramid(programs);
}
@Override

View File

@ -10,7 +10,6 @@ import org.lwjgl.system.MemoryUtil;
import dev.engine_room.flywheel.backend.compile.IndirectPrograms;
import dev.engine_room.flywheel.backend.gl.GlFence;
import dev.engine_room.flywheel.backend.gl.buffer.GlBuffer;
import dev.engine_room.flywheel.backend.gl.shader.GlProgram;
import dev.engine_room.flywheel.lib.memory.FlwMemoryTracker;
import dev.engine_room.flywheel.lib.memory.MemoryBlock;
import it.unimi.dsi.fastutil.PriorityQueue;
@ -26,6 +25,7 @@ public class StagingBuffer {
private final int vbo;
private final long map;
private final long capacity;
private final IndirectPrograms programs;
private final OverflowStagingBuffer overflow = new OverflowStagingBuffer();
private final TransferList transfers = new TransferList();
@ -33,8 +33,6 @@ public class StagingBuffer {
private final GlBuffer scatterBuffer = new GlBuffer();
private final ScatterList scatterList = new ScatterList();
private final GlProgram scatterProgram;
/**
* The position in the buffer at the time of the last flush.
*/
@ -70,6 +68,7 @@ public class StagingBuffer {
public StagingBuffer(long capacity, IndirectPrograms programs) {
this.capacity = capacity;
this.programs = programs;
vbo = GL45C.glCreateBuffers();
GL45C.glNamedBufferStorage(vbo, capacity, STORAGE_FLAGS);
@ -78,8 +77,6 @@ public class StagingBuffer {
totalAvailable = capacity;
FlwMemoryTracker._allocCpuMemory(capacity);
scatterProgram = programs.getScatterProgram();
}
/**
@ -251,7 +248,8 @@ public class StagingBuffer {
* <a href=https://on-demand.gputechconf.com/gtc/2016/presentation/s6138-christoph-kubisch-pierre-boudier-gpu-driven-rendering.pdf>this presentation</a>
*/
private void dispatchComputeCopies() {
scatterProgram.bind();
programs.getScatterProgram()
.bind();
// These bindings don't change between dstVbos.
GL45.glBindBufferBase(GL45C.GL_SHADER_STORAGE_BUFFER, 0, scatterBuffer.handle());

View File

@ -1,22 +1,22 @@
package dev.engine_room.flywheel.backend.glsl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import dev.engine_room.flywheel.backend.compile.core.ShaderException;
public sealed interface LoadResult {
@Nullable
default SourceFile unwrap() {
return null;
}
SourceFile unwrap();
record Success(SourceFile source) implements LoadResult {
@Override
@NotNull
public SourceFile unwrap() {
return source;
}
}
record Failure(LoadError error) implements LoadResult {
@Override
public SourceFile unwrap() {
throw new ShaderException.Load(error.generateMessage()
.build());
}
}
}

View File

@ -9,7 +9,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
import dev.engine_room.flywheel.backend.compile.FlwPrograms;
@ -50,7 +49,6 @@ public class ShaderSources {
return cache.computeIfAbsent(location, loc -> new LoadResult.Failure(new LoadError.ResourceError(loc)));
}
@Nullable
public SourceFile get(ResourceLocation location) {
return find(location).unwrap();
}