Almost there

- WorldContext sort of uses a shader pipeline interface
 - Smarter import resolution
 - Reorganize shader sources
 - Rewritten shader templating
 - Still need builtin support

src/main/java/com/jozufozu/flywheel/backend/FileResolution.java

@@ -0,0 +1,46 @@
+package com.jozufozu.flywheel.backend;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import com.jozufozu.flywheel.backend.pipeline.SourceFile;
+import com.jozufozu.flywheel.backend.pipeline.error.ErrorReporter;
+import com.jozufozu.flywheel.backend.pipeline.span.Span;
+
+import net.minecraft.util.ResourceLocation;
+
+public class FileResolution {
+
+	private final List<Span> foundSpans = new ArrayList<>();
+	private final ResourceLocation fileLoc;
+	private SourceFile file;
+
+
+	public FileResolution(ResourceLocation fileLoc) {
+		this.fileLoc = fileLoc;
+	}
+
+	public ResourceLocation getFileLoc() {
+		return fileLoc;
+	}
+
+	@Nullable
+	public SourceFile getFile() {
+		return file;
+	}
+
+	public void addSpan(Span span) {
+		foundSpans.add(span);
+	}
+
+	public void resolve(ShaderSources sources) {
+
+		try {
+			file = sources.source(fileLoc);
+		} catch (RuntimeException error) {
+			ErrorReporter.generateSpanError(foundSpans.get(0), "could not find source");
+		}
+	}
+}

src/main/java/com/jozufozu/flywheel/backend/ShaderContext.java

@@ -14,6 +14,7 @@ import com.jozufozu.flywheel.backend.gl.shader.GlShader;
 import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.backend.loading.Program;
 import com.jozufozu.flywheel.backend.loading.Shader;
+import com.jozufozu.flywheel.backend.pipeline.SourceFile;
 import com.jozufozu.flywheel.core.shader.IMultiProgram;
 import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
 import com.jozufozu.flywheel.core.shader.spec.ProgramState;
@@ -35,65 +36,10 @@ public abstract class ShaderContext<P extends GlProgram> implements IShaderConte
 		return programs.get(spec);
 	}
 
-	public Program loadAndLink(ProgramSpec spec, @Nullable ProgramState state) {
-		Shader vertexFile = getSource(ShaderType.VERTEX, spec.vert);
-		Shader fragmentFile = getSource(ShaderType.FRAGMENT, spec.frag);
-
-		if (state != null) {
-			vertexFile.defineAll(state.getDefines());
-			fragmentFile.defineAll(state.getDefines());
-		}
-
-		Program linked = link(buildProgram(spec.name, vertexFile, fragmentFile));
-
-		String descriptor = linked.program + ": " + spec.name;
-
-		if (state != null)
-			descriptor += "#" + state;
-
-		Backend.log.debug(descriptor);
-
-		return linked;
-	}
-
-	protected Shader getSource(ShaderType type, ResourceLocation name) {
-		return backend.sources.source(name, type);
-	}
-
-	protected Program link(Program program) {
-		return program.link().deleteLinkedShaders();
-	}
-
 	@Override
 	public void delete() {
 		programs.values()
 				.forEach(IMultiProgram::delete);
 		programs.clear();
 	}
-
-	/**
-	 * Ingests the given shaders, compiling them and linking them together after applying the transformer to the source.
-	 *
-	 * @param name    What should we call this program if something goes wrong?
-	 * @param shaders What are the different shader stages that should be linked together?
-	 * @return A program with all provided shaders attached
-	 */
-	protected static Program buildProgram(ResourceLocation name, Shader... shaders) {
-		List<GlShader> compiled = new ArrayList<>(shaders.length);
-		try {
-			Program builder = new Program(name);
-
-			for (Shader shader : shaders) {
-				GlShader sh = new GlShader(shader);
-				compiled.add(sh);
-
-				builder.attachShader(shader, sh);
-			}
-
-			return builder;
-		} finally {
-			compiled.forEach(GlObject::delete);
-		}
-	}
-
 }

src/main/java/com/jozufozu/flywheel/backend/ShaderSources.java

@@ -1,27 +1,16 @@
 package com.jozufozu.flywheel.backend;
 
-import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringReader;
-import java.nio.Buffer;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.channels.ReadableByteChannel;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Predicate;
-import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
 
-import org.lwjgl.system.MemoryUtil;
-
 import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -35,9 +24,10 @@ import com.jozufozu.flywheel.backend.pipeline.SourceFile;
 import com.jozufozu.flywheel.backend.pipeline.WorldShaderPipeline;
 import com.jozufozu.flywheel.core.shader.WorldProgram;
 import com.jozufozu.flywheel.core.crumbling.CrumblingRenderer;
+import com.jozufozu.flywheel.core.shader.extension.IProgramExtension;
 import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
 import com.jozufozu.flywheel.event.GatherContextEvent;
-import com.mojang.blaze3d.systems.RenderSystem;
+import com.jozufozu.flywheel.util.StreamUtil;
 import com.mojang.datafixers.util.Pair;
 import com.mojang.serialization.DataResult;
 import com.mojang.serialization.JsonOps;
@@ -62,6 +52,8 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 	private final Map<ResourceLocation, String> shaderSource = new HashMap<>();
 	private final Map<ResourceLocation, SourceFile> shaderSources = new HashMap<>();
 
+	private final Map<ResourceLocation, FileResolution> resolutions = new HashMap<>();
+
 	private boolean shouldCrash;
 	private final Backend backend;
 
@@ -73,6 +65,42 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 		}
 	}
 
+	public SourceFile source(ResourceLocation name) {
+		SourceFile source = shaderSources.get(name);
+
+		if (source == null) {
+			throw new ShaderLoadingException(String.format("shader '%s' does not exist", name));
+		}
+
+		return source;
+	}
+
+	public FileResolution resolveFile(ResourceLocation fileLoc) {
+		return resolutions.computeIfAbsent(fileLoc, FileResolution::new);
+	}
+
+	@Deprecated
+	public Shader source(ResourceLocation name, ShaderType type) {
+		return new Shader(this, type, name, getShaderSource(name));
+	}
+
+	@Deprecated
+	public void notifyError() {
+		shouldCrash = true;
+	}
+
+	@Deprecated
+	@Nonnull
+	public String getShaderSource(ResourceLocation loc) {
+		String source = shaderSource.get(loc);
+
+		if (source == null) {
+			throw new ShaderLoadingException(String.format("shader '%s' does not exist", loc));
+		}
+
+		return source;
+	}
+
 	@Override
 	public void onResourceManagerReload(IResourceManager manager, Predicate<IResourceType> predicate) {
 		if (predicate.test(VanillaResourceType.SHADERS)) {
@@ -87,15 +115,20 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 				ModLoader.get()
 						.postEvent(new GatherContextEvent(backend));
 
+				resolutions.clear();
+
 				loadProgramSpecs(manager);
 				loadShaderSources(manager);
 
-				shaderSources.values().forEach(SourceFile::resolveIncludes);
+				for (FileResolution resolution : resolutions.values()) {
+					resolution.resolve(this);
+				}
 
-				WorldShaderPipeline<WorldProgram> pl = new WorldShaderPipeline<>(this);
+				WorldShaderPipeline<WorldProgram> pl = new WorldShaderPipeline<>(this, WorldProgram::new);
 
-				SourceFile source = source(new ResourceLocation(Flywheel.ID, "model.glsl"));
+//				ResourceLocation name = new ResourceLocation(Flywheel.ID, "model.glsl");
-				pl.compile(source, Collections.emptyList());
+//				SourceFile source = source(name);
+//				pl.compile(name, source, Collections.emptyList());
 
 				for (IShaderContext<?> context : backend.allContexts()) {
 					context.load();
@@ -120,6 +153,31 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 		}
 	}
 
+	private void loadShaderSources(IResourceManager manager) {
+		Collection<ResourceLocation> allShaders = manager.listResources(SHADER_DIR, s -> {
+			for (String ext : EXTENSIONS) {
+				if (s.endsWith(ext)) return true;
+			}
+			return false;
+		});
+
+		for (ResourceLocation location : allShaders) {
+			try {
+				IResource resource = manager.getResource(location);
+
+				String source = StreamUtil.readToString(resource.getInputStream());
+
+				ResourceLocation name = ResourceUtil.removePrefixUnchecked(location, SHADER_DIR);
+
+				shaderSource.put(name, source);
+				shaderSources.put(name, new SourceFile(this, name, source));
+			} catch (IOException e) {
+
+			}
+		}
+	}
+
+
 	private void loadProgramSpecs(IResourceManager manager) {
 		Collection<ResourceLocation> programSpecs = manager.listResources(PROGRAM_DIR, s -> s.endsWith(".json"));
 
@@ -127,7 +185,7 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 			try {
 				IResource file = manager.getResource(location);
 
-				String s = readToString(file.getInputStream());
+				String s = StreamUtil.readToString(file.getInputStream());
 
 				ResourceLocation specName = ResourceUtil.trim(location, PROGRAM_DIR, ".json");
 
@@ -145,104 +203,4 @@ public class ShaderSources implements ISelectiveResourceReloadListener {
 			}
 		}
 	}
-
-	@Deprecated
-	public void notifyError() {
-		shouldCrash = true;
-	}
-
-	@Deprecated
-	@Nonnull
-	public String getShaderSource(ResourceLocation loc) {
-		String source = shaderSource.get(loc);
-
-		if (source == null) {
-			throw new ShaderLoadingException(String.format("shader '%s' does not exist", loc));
-		}
-
-		return source;
-	}
-
-	private void loadShaderSources(IResourceManager manager) {
-		Collection<ResourceLocation> allShaders = manager.listResources(SHADER_DIR, s -> {
-			for (String ext : EXTENSIONS) {
-				if (s.endsWith(ext)) return true;
-			}
-			return false;
-		});
-
-		for (ResourceLocation location : allShaders) {
-			try {
-				IResource resource = manager.getResource(location);
-
-				String source = readToString(resource.getInputStream());
-
-				ResourceLocation name = ResourceUtil.removePrefixUnchecked(location, SHADER_DIR);
-
-				shaderSource.put(name, source);
-				shaderSources.put(name, new SourceFile(this, name, source));
-			} catch (IOException e) {
-
-			}
-		}
-	}
-
-	@Deprecated
-	public Shader source(ResourceLocation name, ShaderType type) {
-		return new Shader(this, type, name, getShaderSource(name));
-	}
-
-	public SourceFile source(ResourceLocation name) {
-		SourceFile source = shaderSources.get(name);
-
-		if (source == null) {
-			throw new ShaderLoadingException(String.format("shader '%s' does not exist", name));
-		}
-
-		return source;
-	}
-
-	public String readToString(InputStream is) {
-		RenderSystem.assertThread(RenderSystem::isOnRenderThread);
-		ByteBuffer bytebuffer = null;
-
-		try {
-			bytebuffer = readToBuffer(is);
-			int i = bytebuffer.position();
-			((Buffer) bytebuffer).rewind();
-			return MemoryUtil.memASCII(bytebuffer, i);
-		} catch (IOException e) {
-
-		} finally {
-			if (bytebuffer != null) {
-				MemoryUtil.memFree(bytebuffer);
-			}
-
-		}
-
-		return null;
-	}
-
-	public ByteBuffer readToBuffer(InputStream is) throws IOException {
-		ByteBuffer bytebuffer;
-		if (is instanceof FileInputStream) {
-			FileInputStream fileinputstream = (FileInputStream) is;
-			FileChannel filechannel = fileinputstream.getChannel();
-			bytebuffer = MemoryUtil.memAlloc((int) filechannel.size() + 1);
-
-			while (filechannel.read(bytebuffer) != -1) {
-			}
-		} else {
-			bytebuffer = MemoryUtil.memAlloc(8192);
-			ReadableByteChannel readablebytechannel = Channels.newChannel(is);
-
-			while (readablebytechannel.read(bytebuffer) != -1) {
-				if (bytebuffer.remaining() == 0) {
-					bytebuffer = MemoryUtil.memRealloc(bytebuffer, bytebuffer.capacity() * 2);
-				}
-			}
-		}
-
-		return bytebuffer;
-	}
 }

src/main/java/com/jozufozu/flywheel/backend/gl/shader/GlProgram.java

@@ -18,9 +18,9 @@ public abstract class GlProgram extends GlObject {
 
 	public final ResourceLocation name;
 
-	protected GlProgram(Program program) {
+	protected GlProgram(ResourceLocation name, int handle) {
-		setHandle(program.program);
+		this.name = name;
-		this.name = program.name;
+		setHandle(handle);
 	}
 
 	public void bind() {

src/main/java/com/jozufozu/flywheel/backend/gl/shader/GlShader.java

@@ -14,24 +14,24 @@ public class GlShader extends GlObject {
 	public final ResourceLocation name;
 	public final ShaderType type;
 
-	public GlShader(Shader shader) {
+	public GlShader(ResourceLocation name, ShaderType type, CharSequence source) {
-		this.type = shader.type;
+		this.name = name;
-		this.name = shader.name;
+		this.type = type;
-		int handle = GL20.glCreateShader(shader.type.glEnum);
+		int handle = GL20.glCreateShader(type.glEnum);
 
-		GlCompat.safeShaderSource(handle, shader.getSource());
+		GlCompat.safeShaderSource(handle, source);
 		GL20.glCompileShader(handle);
 
 		String log = GL20.glGetShaderInfoLog(handle);
 
 		if (!log.isEmpty()) {
-			Backend.log.error("Shader compilation log for " + shader.name + ": " + log);
+			Backend.log.error("Shader compilation log for " + name + ": " + log);
-			Backend.log.error(shader.printSource());
+			Backend.log.error(source);
 		}
 		//Backend.log.debug(shader.printSource());
 
 		if (GL20.glGetShaderi(handle, GL20.GL_COMPILE_STATUS) != GL20.GL_TRUE) {
-			throw new RuntimeException("Could not compile " + shader.name + ". See log for details.");
+			throw new RuntimeException("Could not compile " + name + ". See log for details.");
 		}
 
 		setHandle(handle);

src/main/java/com/jozufozu/flywheel/backend/loading/Program.java

@@ -24,17 +24,16 @@ import it.unimi.dsi.fastutil.ints.IntList;
 import net.minecraft.util.ResourceLocation;
 
 public class Program {
-	public final ResourceLocation name;
 	public final int program;
 
+	public ResourceLocation name;
 	private int attributeIndex;
 
 	public final Map<ShaderType, Shader> attached;
 
 	private final IntList shaders;
 
-	public Program(ResourceLocation name) {
+	public Program() {
-		this.name = name;
 		this.program = glCreateProgram();
 		attached = new EnumMap<>(ShaderType.class);
 		shaders = new IntArrayList(2);
@@ -57,13 +56,14 @@ public class Program {
 	/**
 	 * Links the attached shaders to this program.
 	 */
-	public Program link() {
+	public Program link(ResourceLocation name) {
+		this.name = name;
 		glLinkProgram(this.program);
 
 		String log = glGetProgramInfoLog(this.program);
 
 		if (!log.isEmpty()) {
-			Backend.log.debug("Program link log for " + this.name + ": " + log);
+			Backend.log.debug("Program link log for " + name + ": " + log);
 		}
 
 		int result = glGetProgrami(this.program, GL_LINK_STATUS);

src/main/java/com/jozufozu/flywheel/backend/pipeline/GlShader.java

@@ -1,39 +0,0 @@
-package com.jozufozu.flywheel.backend.pipeline;
-
-import org.lwjgl.opengl.GL20;
-
-import com.jozufozu.flywheel.backend.Backend;
-import com.jozufozu.flywheel.backend.gl.GlObject;
-import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
-import com.jozufozu.flywheel.backend.gl.versioned.GlCompat;
-
-public class GlShader extends GlObject {
-
-	public GlShader(ShaderType type, CharSequence source) {
-		int handle = GL20.glCreateShader(type.glEnum);
-
-		GlCompat.safeShaderSource(handle, source);
-
-		GL20.glCompileShader(handle);
-
-		String log = GL20.glGetShaderInfoLog(handle);
-
-		if (!log.isEmpty()) {
-			Backend.log.error("Shader compilation log for " + "DUBUG"+ ": " + log);
-			Backend.log.error(source);
-		}
-		//Backend.log.debug(shader.printSource());
-
-		if (GL20.glGetShaderi(handle, GL20.GL_COMPILE_STATUS) != GL20.GL_TRUE) {
-			throw new RuntimeException("Could not compile " + "DEBUG" + ". See log for details.");
-		}
-
-
-		setHandle(handle);
-	}
-
-	@Override
-	protected void deleteInternal(int handle) {
-
-	}
-}

src/main/java/com/jozufozu/flywheel/backend/pipeline/IShaderPipeline.java

@@ -0,0 +1,11 @@
+package com.jozufozu.flywheel.backend.pipeline;
+
+import com.jozufozu.flywheel.core.shader.IMultiProgram;
+import com.jozufozu.flywheel.core.shader.WorldProgram;
+import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
+
+public interface IShaderPipeline<P extends WorldProgram> {
+
+	IMultiProgram<P> compile(ProgramSpec spec);
+
+}

src/main/java/com/jozufozu/flywheel/backend/pipeline/Includer.java

@@ -4,11 +4,9 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.Stack;
 
 import com.google.common.collect.ImmutableList;
-import com.jozufozu.flywheel.backend.ShaderSources;
+import com.jozufozu.flywheel.backend.pipeline.parse.Import;
-import com.jozufozu.flywheel.backend.pipeline.parse.Include;
 
 import net.minecraft.util.ResourceLocation;
 
@@ -27,11 +25,11 @@ public class Includer {
 	}
 
 	private static void process(Set<ResourceLocation> seen, List<SourceFile> out, SourceFile source) {
-		ImmutableList<Include> includes = source.getIncludes();
+		ImmutableList<Import> imports = source.getIncludes();
 
-		for (Include include : includes) {
+		for (Import use : imports) {
 
-			SourceFile file = include.getTarget();
+			SourceFile file = use.getFile();
 			if (file != null && seen.add(file.name)) {
 				process(seen, out, file);
 

src/main/java/com/jozufozu/flywheel/backend/pipeline/LegacyPipeline.java

@@ -0,0 +1,137 @@
+package com.jozufozu.flywheel.backend.pipeline;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+import com.jozufozu.flywheel.backend.Backend;
+import com.jozufozu.flywheel.backend.ShaderSources;
+import com.jozufozu.flywheel.backend.gl.GlObject;
+import com.jozufozu.flywheel.backend.gl.shader.GlShader;
+import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
+import com.jozufozu.flywheel.backend.loading.Program;
+import com.jozufozu.flywheel.backend.loading.ProgramTemplate;
+import com.jozufozu.flywheel.backend.loading.Shader;
+import com.jozufozu.flywheel.backend.loading.ShaderLoadingException;
+import com.jozufozu.flywheel.backend.loading.ShaderTransformer;
+import com.jozufozu.flywheel.core.shader.ExtensibleGlProgram;
+import com.jozufozu.flywheel.core.shader.GameStateProgram;
+import com.jozufozu.flywheel.core.shader.IMultiProgram;
+import com.jozufozu.flywheel.core.shader.WorldProgram;
+import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
+import com.jozufozu.flywheel.core.shader.spec.ProgramState;
+
+public class LegacyPipeline<P extends WorldProgram> implements IShaderPipeline<P> {
+
+	private static final String declaration = "#flwbuiltins";
+	private static final Pattern builtinPattern = Pattern.compile(declaration);
+
+
+	private final ShaderSources sources;
+	protected ShaderTransformer transformer;
+	private final ProgramTemplate template;
+	private final ExtensibleGlProgram.Factory<P> factory;
+	private final Map<ShaderType, String> builtinSources;
+
+
+	public LegacyPipeline(ShaderSources sources, ProgramTemplate template, ExtensibleGlProgram.Factory<P> factory, Map<ShaderType, String> builtinSources) {
+		this.sources = sources;
+		this.template = template;
+		this.factory = factory;
+
+		transformer = new ShaderTransformer().pushStage(this::injectBuiltins)
+				.pushStage(Shader::processIncludes)
+				.pushStage(template)
+				.pushStage(Shader::processIncludes);
+		this.builtinSources = builtinSources;
+	}
+
+	@Override
+	public IMultiProgram<P> compile(ProgramSpec spec) {
+		GameStateProgram.Builder<P> builder = GameStateProgram.builder(compile(spec, null));
+
+		for (ProgramState state : spec.states) {
+
+			builder.withVariant(state.getContext(), compile(spec, state));
+		}
+
+		return builder.build();
+	}
+
+	/**
+	 * Ingests the given shaders, compiling them and linking them together after applying the transformer to the source.
+	 *
+	 * @param shaders What are the different shader stages that should be linked together?
+	 * @return A program with all provided shaders attached
+	 */
+	protected static Program buildProgram(Shader... shaders) {
+		List<GlShader> compiled = new ArrayList<>(shaders.length);
+		try {
+			Program builder = new Program();
+
+			for (Shader shader : shaders) {
+				GlShader sh = new GlShader(shader.name, shader.type, shader.getSource());
+				compiled.add(sh);
+
+				builder.attachShader(shader, sh);
+			}
+
+			return builder;
+		} finally {
+			compiled.forEach(GlObject::delete);
+		}
+	}
+
+	public Program loadAndLink(ProgramSpec spec, @Nullable ProgramState state) {
+		Shader vertexFile = sources.source(spec.vert, ShaderType.VERTEX);
+		Shader fragmentFile = sources.source(spec.frag, ShaderType.FRAGMENT);
+
+		transformer.transformSource(vertexFile);
+		transformer.transformSource(fragmentFile);
+
+		if (state != null) {
+			vertexFile.defineAll(state.getDefines());
+			fragmentFile.defineAll(state.getDefines());
+		}
+
+		Program program = buildProgram(vertexFile, fragmentFile);
+		template.attachAttributes(program);
+
+		program.link(spec.name).deleteLinkedShaders();
+
+		String descriptor = program.program + ": " + spec.name;
+
+		if (state != null)
+			descriptor += "#" + state;
+
+		Backend.log.debug(descriptor);
+
+		return program;
+	}
+
+	private P compile(ProgramSpec spec, @Nullable ProgramState state) {
+		if (state != null) {
+			Program program = loadAndLink(spec, state);
+			return factory.create(program.name, program.program, state.getExtensions());
+		} else {
+			Program program = loadAndLink(spec, null);
+			return factory.create(program.name, program.program, null);
+		}
+	}
+
+	/**
+	 * Replace #flwbuiltins with whatever expansion this context provides for the given shader.
+	 */
+	public void injectBuiltins(Shader shader) {
+		Matcher matcher = builtinPattern.matcher(shader.getSource());
+
+		if (matcher.find()) shader.setSource(matcher.replaceFirst(builtinSources.get(shader.type)));
+		else
+			throw new ShaderLoadingException(String.format("%s is missing %s, cannot use in World Context", shader.type.name, declaration));
+	}
+}

src/main/java/com/jozufozu/flywheel/backend/pipeline/PipelineProgramBuilder.java

@@ -1,34 +0,0 @@
-package com.jozufozu.flywheel.backend.pipeline;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-public class PipelineProgramBuilder {
-
-	private final List<SourceFile> sources = new ArrayList<>();
-
-	public PipelineProgramBuilder() {
-
-	}
-
-	public PipelineProgramBuilder include(SourceFile file) {
-		sources.add(file);
-		return this;
-	}
-
-	public PipelineProgramBuilder includeAll(Collection<? extends SourceFile> files) {
-		sources.addAll(files);
-		return this;
-	}
-
-	public CharSequence build() {
-		StringBuilder builder = new StringBuilder();
-
-		for (SourceFile source : sources) {
-			builder.append(source.getElidedSource());
-		}
-
-		return builder;
-	}
-}

src/main/java/com/jozufozu/flywheel/backend/pipeline/Shader.java

@@ -1,31 +1,11 @@
 package com.jozufozu.flywheel.backend.pipeline;
 
+import com.jozufozu.flywheel.backend.gl.shader.GlShader;
 import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 
+import net.minecraft.util.ResourceLocation;
+
 public class Shader {
 
-	private final GLSLVersion version;
-	private final CharSequence source;
 
-	public Shader(GLSLVersion version, CharSequence source) {
-		this.version = version;
-		this.source = source;
-	}
-
-	public GlShader create(ShaderType type) {
-
-		StringBuilder source = new StringBuilder();
-
-		source.append("#version ")
-				.append(version.version)
-				.append('\n');
-
-		source.append("#define ")
-				.append(type.define)
-				.append('\n');
-
-		source.append(this.source);
-
-		return new GlShader(type, source);
-	}
 }

src/main/java/com/jozufozu/flywheel/backend/pipeline/ShaderBuilder.java

@@ -0,0 +1,79 @@
+package com.jozufozu.flywheel.backend.pipeline;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.jozufozu.flywheel.backend.gl.shader.GlShader;
+import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
+
+import net.minecraft.util.ResourceLocation;
+
+public class ShaderBuilder {
+
+	public final ResourceLocation name;
+	public final Template template;
+
+	private SourceFile mainFile;
+	private GLSLVersion version;
+
+	private StringBuilder source;
+	private StringBuilder defines;
+	private CharSequence footer;
+
+	public ShaderBuilder(ResourceLocation name, Template template) {
+		this.name = name;
+		this.template = template;
+	}
+
+	public ShaderBuilder setVersion(GLSLVersion version) {
+		this.version = version;
+		return this;
+	}
+
+	public ShaderBuilder setDefines(List<String> defs) {
+		defines = new StringBuilder();
+
+		for (String def : defs) {
+			defines.append("#define ")
+					.append(def)
+					.append('\n');
+		}
+		return this;
+	}
+
+	public ShaderBuilder setFooter(CharSequence footer) {
+		this.footer = footer;
+		return this;
+	}
+
+	public ShaderBuilder setMainSource(SourceFile file) {
+		if (mainFile == file) return this;
+
+		mainFile = file;
+		source = new StringBuilder();
+
+		for (SourceFile includeFile : Includer.recurseIncludes(file)) {
+			source.append(includeFile.getElidedSource());
+		}
+		source.append(file.getElidedSource());
+
+		return this;
+	}
+
+	public GlShader compile(ResourceLocation name, ShaderType type) {
+
+		StringBuilder finalSource = new StringBuilder();
+
+		finalSource.append("#version ")
+				.append(version)
+				.append('\n')
+				.append("#define ")
+				.append(type.define)
+				.append('\n')
+				.append(defines != null ? defines : "")
+				.append(source)
+				.append(template.footer(type, mainFile));
+
+		return new GlShader(name, type, finalSource);
+	}
+}

src/main/java/com/jozufozu/flywheel/backend/pipeline/SourceFile.java

@@ -10,9 +10,7 @@ import java.util.regex.Pattern;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.jozufozu.flywheel.backend.ShaderSources;
-import com.jozufozu.flywheel.backend.pipeline.error.ErrorReporter;
+import com.jozufozu.flywheel.backend.pipeline.parse.Import;
-import com.jozufozu.flywheel.backend.pipeline.parse.AbstractShaderElement;
-import com.jozufozu.flywheel.backend.pipeline.parse.Include;
 import com.jozufozu.flywheel.backend.pipeline.parse.ShaderFunction;
 import com.jozufozu.flywheel.backend.pipeline.parse.ShaderStruct;
 import com.jozufozu.flywheel.backend.pipeline.span.CharPos;
@@ -47,7 +45,7 @@ public class SourceFile {
 	private final ImmutableMap<String, ShaderStruct> structs;
 
 	// Includes ordered as defined in the source
-	private final ImmutableList<Include> includes;
+	private final ImmutableList<Import> imports;
 
 	// Sections of the source that must be trimmed for compilation.
 	private final List<Span> elisions = new ArrayList<>();
@@ -60,7 +58,7 @@ public class SourceFile {
 		this.lineStarts = createLineStarts();
 		this.lines = createLineList(lineStarts);
 
-		this.includes = parseIncludes();
+		this.imports = parseImports();
 		this.functions = parseFunctions();
 		this.structs = parseStructs();
 
@@ -83,14 +81,12 @@ public class SourceFile {
 		return functions;
 	}
 
-	public ImmutableList<Include> getIncludes() {
+	public ImmutableMap<String, ShaderStruct> getStructs() {
-		return includes;
+		return structs;
 	}
 
-	public void resolveIncludes() {
+	public ImmutableList<Import> getIncludes() {
-		for (Include include : includes) {
+		return imports;
-			include.resolve();
-		}
 	}
 
 	public CharPos getCharPos(int charPos) {
@@ -210,7 +206,7 @@ public class SourceFile {
 	private ImmutableMap<String, ShaderStruct> parseStructs() {
 		Matcher matcher = ShaderStruct.struct.matcher(source);
 
-		ImmutableMap.Builder<String, ShaderStruct> functions = ImmutableMap.builder();
+		ImmutableMap.Builder<String, ShaderStruct> structs = ImmutableMap.builder();
 		while (matcher.find()) {
 			Span self = Span.fromMatcher(this, matcher);
 			Span name = Span.fromMatcher(this, matcher, 1);
@@ -218,31 +214,31 @@ public class SourceFile {
 
 			ShaderStruct shaderStruct = new ShaderStruct(self, name, body);
 
-			functions.put(body.get(), shaderStruct);
+			structs.put(name.get(), shaderStruct);
 		}
 
-		return functions.build();
+		return structs.build();
 	}
 
 	/**
 	 * Scan the source for <code>#use "..."</code> directives.
-	 * Records the contents of the directive into an {@link Include} object, and marks the directive for elision.
+	 * Records the contents of the directive into an {@link Import} object, and marks the directive for elision.
 	 */
-	private ImmutableList<Include> parseIncludes() {
+	private ImmutableList<Import> parseImports() {
 		Matcher uses = includePattern.matcher(source);
 
-		List<Include> includes = new ArrayList<>();
+		List<Import> imports = new ArrayList<>();
 
 		while (uses.find()) {
 			Span use = Span.fromMatcher(this, uses);
 			Span file = Span.fromMatcher(this, uses, 1);
 
-			includes.add(new Include(parent, use, file));
+			imports.add(new Import(parent, use, file));
 
 			elisions.add(use); // we have to trim that later
 		}
 
-		return ImmutableList.copyOf(includes);
+		return ImmutableList.copyOf(imports);
 	}
 
 	/**

src/main/java/com/jozufozu/flywheel/backend/pipeline/Template.java

@@ -0,0 +1,123 @@
+package com.jozufozu.flywheel.backend.pipeline;
+
+import com.google.common.collect.ImmutableList;
+import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
+import com.jozufozu.flywheel.backend.pipeline.parse.ShaderFunction;
+import com.jozufozu.flywheel.backend.pipeline.parse.ShaderStruct;
+import com.jozufozu.flywheel.backend.pipeline.parse.StructField;
+import com.jozufozu.flywheel.backend.pipeline.parse.Variable;
+
+public class Template {
+
+	public CharSequence footer(ShaderType type, SourceFile file) {
+		switch (type) {
+		case VERTEX:
+			return vertexFooter(file);
+		case FRAGMENT:
+			return fragmentFooter(file);
+		}
+
+		return "";
+	}
+
+	public CharSequence vertexFooter(SourceFile file) {
+		ShaderFunction vertexMain = file.getFunctions()
+				.get("vertex");
+
+		ImmutableList<Variable> parameters = vertexMain.getParameters();
+
+		ShaderStruct interpolant = file.getStructs()
+				.get(vertexMain.returnType());
+
+		ShaderStruct vertex = file.getStructs()
+				.get(parameters.get(0)
+						.typeName()
+						.get());
+
+		ShaderStruct instance = file.getStructs()
+				.get(parameters.get(1)
+						.typeName()
+						.get());
+
+		StringBuilder template = new StringBuilder();
+
+		prefixFields(template, vertex, "attribute", "a_v_");
+		prefixFields(template, instance, "attribute", "a_i_");
+		prefixFields(template, interpolant, "varying", "v2f_");
+
+		template.append("void main() {\n");
+		template.append(vertex.name)
+				.append(" v;\n");
+		assignFields(template, vertex, "v.", "a_v_");
+
+		template.append(instance.name)
+				.append(" i;\n");
+		assignFields(template, instance, "i.", "a_i_");
+
+		template.append(interpolant.name)
+				.append(" o = ")
+				.append(vertexMain.call("v", "i"))
+				.append(";\n");
+
+		assignFields(template, interpolant, "v2f_", "o.");
+
+		template.append('}');
+
+		return template;
+	}
+
+	public CharSequence fragmentFooter(SourceFile file) {
+		ShaderFunction fragmentMain = file.getFunctions()
+				.get("fragment");
+
+		ImmutableList<Variable> parameters = fragmentMain.getParameters();
+
+		ShaderStruct interpolant = file.getStructs()
+				.get(parameters.get(0)
+						.typeName()
+						.get());
+
+		StringBuilder template = new StringBuilder();
+
+		prefixFields(template, interpolant, "varying", "v2f_");
+
+		template.append("void main() {\n");
+		template.append(interpolant.name)
+				.append(" o;\n");
+		assignFields(template, interpolant, "o.", "v2f_");
+
+		template.append(fragmentMain.call("o"))
+				.append(";\n");
+
+		template.append('}');
+
+		return template;
+	}
+
+	public static void prefixFields(StringBuilder builder, ShaderStruct struct, String qualifier, String prefix) {
+		ImmutableList<StructField> fields = struct.getFields();
+
+		for (StructField field : fields) {
+			builder.append(qualifier)
+					.append(' ')
+					.append(field.type)
+					.append(' ')
+					.append(prefix)
+					.append(field.name)
+					.append(";\n");
+		}
+	}
+
+	public static void assignFields(StringBuilder builder, ShaderStruct struct, String prefix1, String prefix2) {
+		ImmutableList<StructField> fields = struct.getFields();
+
+		for (StructField field : fields) {
+			builder.append(prefix1)
+					.append(field.name)
+					.append(" = ")
+					.append(prefix2)
+					.append(field.name)
+					.append(";\n");
+		}
+	}
+}

src/main/java/com/jozufozu/flywheel/backend/pipeline/WorldShaderPipeline.java

@@ -1,45 +1,91 @@
 package com.jozufozu.flywheel.backend.pipeline;
 
+import static org.lwjgl.opengl.GL11.GL_TRUE;
+import static org.lwjgl.opengl.GL20.GL_LINK_STATUS;
+import static org.lwjgl.opengl.GL20.glGetProgramInfoLog;
+import static org.lwjgl.opengl.GL20.glGetProgrami;
+
 import java.util.List;
 
 import javax.annotation.Nullable;
 
+import org.lwjgl.opengl.GL20;
+
+import com.jozufozu.flywheel.backend.Backend;
 import com.jozufozu.flywheel.backend.ShaderSources;
-import com.jozufozu.flywheel.backend.gl.shader.GlProgram;
+import com.jozufozu.flywheel.backend.gl.shader.GlShader;
-import com.jozufozu.flywheel.backend.loading.Program;
+import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
+import com.jozufozu.flywheel.core.shader.ExtensibleGlProgram;
+import com.jozufozu.flywheel.core.shader.GameStateProgram;
 import com.jozufozu.flywheel.core.shader.IMultiProgram;
 import com.jozufozu.flywheel.core.shader.WorldProgram;
 import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
 import com.jozufozu.flywheel.core.shader.spec.ProgramState;
 
+import net.minecraft.util.ResourceLocation;
+
 public class WorldShaderPipeline<P extends WorldProgram> {
 
 	private final ShaderSources sources;
 
-	public WorldShaderPipeline(ShaderSources sources) {
+	private final ExtensibleGlProgram.Factory<P> factory;
+
+	public WorldShaderPipeline(ShaderSources sources, ExtensibleGlProgram.Factory<P> factory) {
 		this.sources = sources;
+		this.factory = factory;
 	}
 
-	@Nullable // TODO: temporary null return
 	public IMultiProgram<P> compile(ProgramSpec spec) {
 
 		SourceFile file = sources.source(spec.vert);
 
-		return compile(file, spec.getStates());
+		return compile(spec.name, file, spec.getStates());
 	}
 
-	@Nullable
+	public IMultiProgram<P> compile(ResourceLocation name, SourceFile file, List<ProgramState> variants) {
-	public IMultiProgram<P> compile(SourceFile file, List<ProgramState> variants) {
+		ShaderBuilder shader = new ShaderBuilder(name, new Template())
-		PipelineProgramBuilder builder = new PipelineProgramBuilder();
+				.setMainSource(file)
+				.setVersion(GLSLVersion.V120);
 
-		builder.includeAll(Includer.recurseIncludes(file));
+		GameStateProgram.Builder<P> builder = GameStateProgram.builder(compile(shader, name, null));
 
-		builder.include(file);
+		for (ProgramState variant : variants) {
+			builder.withVariant(variant.getContext(), compile(shader, name, variant));
+		}
 
-		CharSequence output = builder.build();
+		return builder.build();
+	}
 
-		//Program program = new Program()
+	private P compile(ShaderBuilder shader, ResourceLocation name, @Nullable ProgramState variant) {
 
-		return null;
+		if (variant != null) {
+			shader.setDefines(variant.getDefines());
+		}
+
+		GlShader vertex = shader.compile(name, ShaderType.VERTEX);
+		GlShader fragment = shader.compile(name, ShaderType.FRAGMENT);
+
+		int program = GL20.glCreateProgram();
+
+		GL20.glAttachShader(program, vertex.handle());
+		GL20.glAttachShader(program, fragment.handle());
+
+		String log = glGetProgramInfoLog(program);
+
+		if (!log.isEmpty()) {
+			Backend.log.debug("Program link log for " + name + ": " + log);
+		}
+
+		int result = glGetProgrami(program, GL_LINK_STATUS);
+
+		if (result != GL_TRUE) {
+			throw new RuntimeException("Shader program linking failed, see log for details");
+		}
+
+		if (variant != null) {
+			return factory.create(name, program, variant.getExtensions());
+		} else {
+			return factory.create(name, program, null);
+		}
 	}
 }

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Import.java

@@ -0,0 +1,54 @@
+package com.jozufozu.flywheel.backend.pipeline.parse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import com.jozufozu.flywheel.backend.FileResolution;
+import com.jozufozu.flywheel.backend.ShaderSources;
+import com.jozufozu.flywheel.backend.pipeline.error.ErrorReporter;
+import com.jozufozu.flywheel.backend.pipeline.SourceFile;
+import com.jozufozu.flywheel.backend.pipeline.span.Span;
+
+import net.minecraft.util.ResourceLocation;
+
+public class Import extends AbstractShaderElement {
+
+	public static final List<Import> IMPORTS = new ArrayList<>();
+
+	private final Span file;
+
+	private final FileResolution resolution;
+	private final ResourceLocation fileLoc;
+
+	public Import(ShaderSources parent, Span self, Span file) {
+		super(self);
+		this.file = file;
+
+		fileLoc = toRL(file);
+		resolution = parent.resolveFile(fileLoc);
+		resolution.addSpan(file);
+
+		IMPORTS.add(this);
+	}
+
+	private ResourceLocation toRL(Span file) {
+		try {
+			return new ResourceLocation(file.get());
+		} catch (RuntimeException error) {
+			ErrorReporter.generateSpanError(file, "malformed source name");
+		}
+
+		return new ResourceLocation("");
+	}
+
+	@Nullable
+	public SourceFile getFile() {
+		return resolution.getFile();
+	}
+
+	public ResourceLocation getFileLoc() {
+		return resolution.getFileLoc();
+	}
+}

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Include.java

@@ -1,58 +0,0 @@
-package com.jozufozu.flywheel.backend.pipeline.parse;
-
-import java.util.Optional;
-
-import javax.annotation.Nullable;
-
-import com.jozufozu.flywheel.backend.ShaderSources;
-import com.jozufozu.flywheel.backend.pipeline.error.ErrorReporter;
-import com.jozufozu.flywheel.backend.pipeline.SourceFile;
-import com.jozufozu.flywheel.backend.pipeline.span.Span;
-
-import net.minecraft.util.ResourceLocation;
-
-public class Include extends AbstractShaderElement {
-
-	private final ShaderSources sources;
-	private Span file;
-
-	private ResourceLocation fileLoc;
-	private SourceFile resolution;
-
-
-	public Include(ShaderSources sources, Span self, Span file) {
-		super(self);
-		this.sources = sources;
-		this.file = file;
-
-		try {
-			fileLoc = new ResourceLocation(file.get());
-		} catch (RuntimeException error) {
-			ErrorReporter.generateSpanError(file, "malformed source name");
-		}
-	}
-
-	public boolean isResolved() {
-		return resolution != null;
-	}
-
-	@Nullable
-	public SourceFile getTarget() {
-		return resolution;
-	}
-
-	public ResourceLocation getFile() {
-		return fileLoc;
-	}
-
-	public void resolve() {
-
-		if (fileLoc == null) return;
-
-		try {
-			resolution = sources.source(fileLoc);
-		} catch (RuntimeException error) {
-			ErrorReporter.generateSpanError(file, "could not find source");
-		}
-	}
-}

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderFunction.java

@@ -1,12 +1,10 @@
 package com.jozufozu.flywheel.backend.pipeline.parse;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import com.jozufozu.flywheel.backend.pipeline.error.ErrorReporter;
+import com.google.common.collect.ImmutableList;
 import com.jozufozu.flywheel.backend.pipeline.span.Span;
 
 public class ShaderFunction extends AbstractShaderElement {
@@ -19,7 +17,7 @@ public class ShaderFunction extends AbstractShaderElement {
 	private final Span args;
 	private final Span body;
 
-	private final List<Variable> parameters;
+	private final ImmutableList<Variable> parameters;
 
 	public ShaderFunction(Span self, Span type, Span name, Span args, Span body) {
 		super(self);
@@ -28,34 +26,44 @@ public class ShaderFunction extends AbstractShaderElement {
 		this.args = args;
 		this.body = body;
 
-		this.parameters = new ArrayList<>();
+		this.parameters = parseArguments();
-
-		parseArguments();
 	}
 
 	public String call(String... args) {
-		return name + "(" + String.join(", ", args) + ");";
+		return name + "(" + String.join(", ", args) + ")";
 	}
 
-	protected void parseArguments() {
+	public ImmutableList<Variable> getParameters() {
-		if (args.isErr() || args.isEmpty()) return;
+		return parameters;
+	}
+
+	public String returnType() {
+		return type.get();
+	}
+
+	protected ImmutableList<Variable> parseArguments() {
+		if (args.isErr() || args.isEmpty()) return ImmutableList.of();
 
 		Matcher arguments = argument.matcher(args.get());
 
+		ImmutableList.Builder<Variable> builder = ImmutableList.builder();
+
 		while (arguments.find()) {
 			Span self = Span.fromMatcher(args, arguments);
 			Span type = Span.fromMatcher(args, arguments, 1);
 			Span name = Span.fromMatcher(args, arguments, 2);
 
-			parameters.add(new Variable(self, type, name));
+			builder.add(new Variable(self, type, name));
 		}
+
+		return builder.build();
 	}
 
 	@Override
 	public String toString() {
 
 		String p = parameters.stream()
-				.map(Variable::getType)
+				.map(Variable::typeName)
 				.map(Span::get)
 				.collect(Collectors.joining(","));
 

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderStruct.java

@@ -30,6 +30,10 @@ public class ShaderStruct extends AbstractShaderElement {
 		this.fields2Types = createTypeLookup();
 	}
 
+	public ImmutableList<StructField> getFields() {
+		return fields;
+	}
+
 	private ImmutableMap<String, Span> createTypeLookup() {
 		ImmutableMap.Builder<String, Span> lookup = ImmutableMap.builder();
 		for (StructField field : fields) {
@@ -62,4 +66,9 @@ public class ShaderStruct extends AbstractShaderElement {
 			builder.addAttribute(prefix + field.name, attributeCount);
 		}
 	}
+
+	@Override
+	public String toString() {
+		return "struct " + name;
+	}
 }

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/StructField.java

@@ -27,6 +27,6 @@ public class StructField extends AbstractShaderElement {
 
 	@Override
 	public String toString() {
-		return "TaggedField{" + "name='" + name + '\'' + ", type='" + type + '\'' + '}';
+		return type + " " + name;
 	}
 }

src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Variable.java

@@ -14,11 +14,16 @@ public class Variable extends AbstractShaderElement {
 		this.name = name;
 	}
 
-	public Span getType() {
+	public Span typeName() {
 		return type;
 	}
 
 	public Span getName() {
 		return name;
 	}
+
+	@Override
+	public String toString() {
+		return type + " " + name;
+	}
 }

src/main/java/com/jozufozu/flywheel/core/Contexts.java

@@ -1,5 +1,7 @@
 package com.jozufozu.flywheel.core;
 
+import java.util.List;
+
 import com.jozufozu.flywheel.Flywheel;
 import com.jozufozu.flywheel.backend.Backend;
 import com.jozufozu.flywheel.backend.SpecMetaRegistry;
@@ -7,6 +9,7 @@ import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.core.crumbling.CrumblingProgram;
 import com.jozufozu.flywheel.core.shader.WorldFog;
 import com.jozufozu.flywheel.core.shader.WorldProgram;
+import com.jozufozu.flywheel.core.shader.extension.IProgramExtension;
 import com.jozufozu.flywheel.core.shader.gamestate.FogStateProvider;
 import com.jozufozu.flywheel.core.shader.gamestate.NormalDebugStateProvider;
 import com.jozufozu.flywheel.event.GatherContextEvent;

src/main/java/com/jozufozu/flywheel/core/WorldContext.java

@@ -3,38 +3,26 @@ package com.jozufozu.flywheel.core;
 import java.util.EnumMap;
 import java.util.Map;
 import java.util.function.Supplier;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
-import javax.annotation.Nullable;
-
 import com.jozufozu.flywheel.backend.Backend;
 import com.jozufozu.flywheel.backend.ResourceUtil;
 import com.jozufozu.flywheel.backend.ShaderContext;
 import com.jozufozu.flywheel.backend.ShaderSources;
 import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
 import com.jozufozu.flywheel.backend.loading.InstancedArraysTemplate;
-import com.jozufozu.flywheel.backend.loading.Program;
 import com.jozufozu.flywheel.backend.loading.ProgramTemplate;
-import com.jozufozu.flywheel.backend.loading.Shader;
 import com.jozufozu.flywheel.backend.loading.ShaderLoadingException;
-import com.jozufozu.flywheel.backend.loading.ShaderTransformer;
 import com.jozufozu.flywheel.backend.material.MaterialSpec;
+import com.jozufozu.flywheel.backend.pipeline.IShaderPipeline;
+import com.jozufozu.flywheel.backend.pipeline.LegacyPipeline;
 import com.jozufozu.flywheel.core.shader.ExtensibleGlProgram;
-import com.jozufozu.flywheel.core.shader.GameStateProgram;
 import com.jozufozu.flywheel.core.shader.WorldProgram;
 import com.jozufozu.flywheel.core.shader.spec.ProgramSpec;
-import com.jozufozu.flywheel.core.shader.spec.ProgramState;
-import com.jozufozu.flywheel.util.WorldAttached;
 
 import net.minecraft.util.ResourceLocation;
 
 public class WorldContext<P extends WorldProgram> extends ShaderContext<P> {
-
-	private static final String declaration = "#flwbuiltins";
-	private static final Pattern builtinPattern = Pattern.compile(declaration);
-
 	protected ResourceLocation name;
 	protected Supplier<Stream<ResourceLocation>> specStream;
 	protected TemplateFactory templateFactory;
@@ -44,6 +32,8 @@ public class WorldContext<P extends WorldProgram> extends ShaderContext<P> {
 
 	private final ExtensibleGlProgram.Factory<P> factory;
 
+	public IShaderPipeline<P> pipeline;
+
 	public WorldContext(Backend backend, ExtensibleGlProgram.Factory<P> factory) {
 		super(backend);
 		this.factory = factory;
@@ -79,9 +69,6 @@ public class WorldContext<P extends WorldProgram> extends ShaderContext<P> {
 		return this;
 	}
 
-	protected ShaderTransformer transformer;
-	protected ProgramTemplate template;
-
 	@Override
 	public void load() {
 
@@ -97,53 +84,17 @@ public class WorldContext<P extends WorldProgram> extends ShaderContext<P> {
 			return;
 		}
 
-		template = templateFactory.create(backend.sources);
+		pipeline = new LegacyPipeline<>(backend.sources, templateFactory.create(backend.sources), factory, builtinSources);
-		transformer = new ShaderTransformer().pushStage(this::injectBuiltins)
-				.pushStage(Shader::processIncludes)
-				.pushStage(template)
-				.pushStage(Shader::processIncludes);
 
 		specStream.get()
 				.map(backend::getSpec)
 				.forEach(this::loadSpec);
 	}
 
-	@Override
-	protected Shader getSource(ShaderType type, ResourceLocation name) {
-		Shader source = super.getSource(type, name);
-		transformer.transformSource(source);
-		return source;
-	}
-
-	@Override
-	protected Program link(Program program) {
-		template.attachAttributes(program);
-
-		return super.link(program);
-	}
-
-	/**
-	 * Replace #flwbuiltins with whatever expansion this context provides for the given shader.
-	 */
-	public void injectBuiltins(Shader shader) {
-		Matcher matcher = builtinPattern.matcher(shader.getSource());
-
-		if (matcher.find()) shader.setSource(matcher.replaceFirst(builtinSources.get(shader.type)));
-		else
-			throw new ShaderLoadingException(String.format("%s is missing %s, cannot use in World Context", shader.type.name, declaration));
-	}
-
 	private void loadSpec(ProgramSpec spec) {
 
 		try {
-			GameStateProgram.Builder<P> builder = GameStateProgram.builder(compile(spec, null));
+			programs.put(spec.name, pipeline.compile(spec));
-
-			for (ProgramState state : spec.states) {
-
-				builder.withVariant(state.getContext(), compile(spec, state));
-			}
-
-			programs.put(spec.name, builder.build());
 
 			Backend.log.debug("Loaded program {}", spec.name);
 		} catch (Exception e) {
@@ -152,13 +103,6 @@ public class WorldContext<P extends WorldProgram> extends ShaderContext<P> {
 		}
 	}
 
-	private P compile(ProgramSpec spec, @Nullable ProgramState state) {
-		if (state != null)
-			return factory.create(loadAndLink(spec, state), state.getExtensions());
-		else
-			return factory.create(loadAndLink(spec, null));
-	}
-
 	public interface TemplateFactory {
 		ProgramTemplate create(ShaderSources loader);
 	}

src/main/java/com/jozufozu/flywheel/core/crumbling/CrumblingProgram.java

@@ -4,7 +4,6 @@ import static org.lwjgl.opengl.GL20.glUniform2f;
 
 import java.util.List;
 
-import com.jozufozu.flywheel.backend.loading.Program;
 import com.jozufozu.flywheel.core.atlas.AtlasInfo;
 import com.jozufozu.flywheel.core.shader.WorldProgram;
 import com.jozufozu.flywheel.core.shader.extension.IProgramExtension;
@@ -13,13 +12,14 @@ import net.minecraft.client.renderer.model.ModelBakery;
 import net.minecraft.client.renderer.texture.AtlasTexture;
 import net.minecraft.client.renderer.texture.TextureAtlasSprite;
 import net.minecraft.inventory.container.PlayerContainer;
+import net.minecraft.util.ResourceLocation;
 
 public class CrumblingProgram extends WorldProgram {
 	protected final int uTextureScale;
 	protected int uCrumbling;
 
-	public CrumblingProgram(Program program, List<IProgramExtension> extensions) {
+	public CrumblingProgram(ResourceLocation name, int handle, List<IProgramExtension> extensions) {
-		super(program, extensions);
+		super(name, handle, extensions);
 
 		uTextureScale = getUniformLocation("uTextureScale");
 	}

src/main/java/com/jozufozu/flywheel/core/shader/ExtensibleGlProgram.java

@@ -8,10 +8,11 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.jozufozu.flywheel.backend.gl.shader.GlProgram;
-import com.jozufozu.flywheel.backend.loading.Program;
 import com.jozufozu.flywheel.core.shader.extension.IExtensionInstance;
 import com.jozufozu.flywheel.core.shader.extension.IProgramExtension;
 
+import net.minecraft.util.ResourceLocation;
+
 /**
  * A shader program that be arbitrarily "extended". This class can take in any number of program extensions, and
  * will initialize them and then call their {@link IExtensionInstance#bind() bind} function every subsequent time this
@@ -25,8 +26,8 @@ public class ExtensibleGlProgram extends GlProgram {
 
 	protected final List<IExtensionInstance> extensions;
 
-	public ExtensibleGlProgram(Program program, @Nullable List<IProgramExtension> extensions) {
+	public ExtensibleGlProgram(ResourceLocation name, int handle, @Nullable List<IProgramExtension> extensions) {
-		super(program);
+		super(name, handle);
 
 		if (extensions != null) {
 			List<IExtensionInstance> list = new ArrayList<>();
@@ -71,10 +72,6 @@ public class ExtensibleGlProgram extends GlProgram {
 	public interface Factory<P extends GlProgram> {
 
 		@Nonnull
-		P create(Program program, @Nullable List<IProgramExtension> extensions);
+		P create(ResourceLocation name, int handle, @Nullable List<IProgramExtension> extensions);
-
-		default P create(Program program) {
-			return create(program, null);
-		}
 	}
 }

src/main/java/com/jozufozu/flywheel/core/shader/WorldProgram.java

@@ -6,12 +6,12 @@ import static org.lwjgl.opengl.GL20.glUniform3f;
 
 import java.util.List;
 
-import com.jozufozu.flywheel.backend.loading.Program;
 import com.jozufozu.flywheel.core.shader.extension.IProgramExtension;
 import com.jozufozu.flywheel.util.AnimationTickHolder;
 
 import net.minecraft.client.MainWindow;
 import net.minecraft.client.Minecraft;
+import net.minecraft.util.ResourceLocation;
 import net.minecraft.util.math.vector.Matrix4f;
 
 public class WorldProgram extends ExtensibleGlProgram {
@@ -23,8 +23,8 @@ public class WorldProgram extends ExtensibleGlProgram {
 	protected int uBlockAtlas;
 	protected int uLightMap;
 
-	public WorldProgram(Program program, List<IProgramExtension> extensions) {
+	public WorldProgram(ResourceLocation name, int handle, List<IProgramExtension> extensions) {
-		super(program, extensions);
+		super(name, handle, extensions);
 
 		super.bind();
 		registerSamplers();

src/main/java/com/jozufozu/flywheel/util/StreamUtil.java

@@ -0,0 +1,60 @@
+package com.jozufozu.flywheel.util;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+
+import org.lwjgl.system.MemoryUtil;
+
+import com.mojang.blaze3d.systems.RenderSystem;
+
+public class StreamUtil {
+	public static String readToString(InputStream is) {
+		RenderSystem.assertThread(RenderSystem::isOnRenderThread);
+		ByteBuffer bytebuffer = null;
+
+		try {
+			bytebuffer = readToBuffer(is);
+			int i = bytebuffer.position();
+			((Buffer) bytebuffer).rewind();
+			return MemoryUtil.memASCII(bytebuffer, i);
+		} catch (IOException e) {
+
+		} finally {
+			if (bytebuffer != null) {
+				MemoryUtil.memFree(bytebuffer);
+			}
+
+		}
+
+		return null;
+	}
+
+	public static ByteBuffer readToBuffer(InputStream is) throws IOException {
+		ByteBuffer bytebuffer;
+		if (is instanceof FileInputStream) {
+			FileInputStream fileinputstream = (FileInputStream) is;
+			FileChannel filechannel = fileinputstream.getChannel();
+			bytebuffer = MemoryUtil.memAlloc((int) filechannel.size() + 1);
+
+			while (filechannel.read(bytebuffer) != -1) {
+			}
+		} else {
+			bytebuffer = MemoryUtil.memAlloc(8192);
+			ReadableByteChannel readablebytechannel = Channels.newChannel(is);
+
+			while (readablebytechannel.read(bytebuffer) != -1) {
+				if (bytebuffer.remaining() == 0) {
+					bytebuffer = MemoryUtil.memRealloc(bytebuffer, bytebuffer.capacity() * 2);
+				}
+			}
+		}
+
+		return bytebuffer;
+	}
+}