From 865926e783a6f8b32982504b2c82d583d6e606b2 Mon Sep 17 00:00:00 2001 From: Jozufozu Date: Tue, 6 Jul 2021 12:38:32 -0700 Subject: [PATCH] Common type and character positions - Shader source abstractions now inherit from AbstractShaderElement - Spans keep track of line and column positions --- .../flywheel/backend/loading/TypeHelper.java | 2 +- .../flywheel/backend/pipeline/Includer.java | 9 +- .../flywheel/backend/pipeline/SourceFile.java | 182 +++++++++++------- .../pipeline/parse/AbstractShaderElement.java | 14 ++ .../backend/pipeline/parse/ErrorReporter.java | 14 ++ .../backend/pipeline/parse/Include.java | 49 +++++ .../pipeline/parse/ShaderFunction.java | 12 +- .../backend/pipeline/parse/ShaderStruct.java | 42 ++-- .../backend/pipeline/parse/StructField.java | 24 ++- .../backend/pipeline/parse/Variable.java | 10 +- .../backend/pipeline/parse/package-info.java | 7 + .../backend/pipeline/span/CharPos.java | 28 +++ .../backend/pipeline/span/ErrorSpan.java | 4 + .../flywheel/backend/pipeline/span/Span.java | 29 ++- .../backend/pipeline/span/StringSpan.java | 8 +- 15 files changed, 327 insertions(+), 107 deletions(-) create mode 100644 src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/AbstractShaderElement.java create mode 100644 src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ErrorReporter.java create mode 100644 src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Include.java create mode 100644 src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/package-info.java create mode 100644 src/main/java/com/jozufozu/flywheel/backend/pipeline/span/CharPos.java diff --git a/src/main/java/com/jozufozu/flywheel/backend/loading/TypeHelper.java b/src/main/java/com/jozufozu/flywheel/backend/loading/TypeHelper.java index 2d67ff460..c9738a98d 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/loading/TypeHelper.java +++ b/src/main/java/com/jozufozu/flywheel/backend/loading/TypeHelper.java @@ -26,7 +26,7 @@ public class TypeHelper { return 1; } - public static int getAttributeCount(String type) { + public static int getAttributeCount(CharSequence type) { Matcher mat = matType.matcher(type); if (mat.find()) { return Integer.parseInt(mat.group(1)); diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/Includer.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/Includer.java index 3d9ca5160..022854218 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/Includer.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/Includer.java @@ -8,6 +8,7 @@ import java.util.Stack; import com.google.common.collect.ImmutableList; import com.jozufozu.flywheel.backend.ShaderSources; +import com.jozufozu.flywheel.backend.pipeline.parse.Include; import net.minecraft.util.ResourceLocation; @@ -28,12 +29,12 @@ public class Includer { } private static void process(ShaderSources sources, Set seen, List out, SourceFile source) { - ImmutableList includes = source.getIncludes(); + ImmutableList includes = source.getIncludes(); - for (ResourceLocation include : includes) { + for (Include include : includes) { - if (seen.add(include)) { - SourceFile file = sources.source(include); + if (seen.add(include.getFile())) { + SourceFile file = sources.source(include.getFile()); process(sources, seen, out, file); diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/SourceFile.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/SourceFile.java index 08e815487..8ea1fb28b 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/SourceFile.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/SourceFile.java @@ -1,40 +1,47 @@ package com.jozufozu.flywheel.backend.pipeline; -import java.io.BufferedReader; -import java.io.StringReader; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Stream; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.jozufozu.flywheel.backend.ShaderSources; +import com.jozufozu.flywheel.backend.pipeline.parse.Include; import com.jozufozu.flywheel.backend.pipeline.parse.ShaderFunction; +import com.jozufozu.flywheel.backend.pipeline.span.CharPos; import com.jozufozu.flywheel.backend.pipeline.span.ErrorSpan; import com.jozufozu.flywheel.backend.pipeline.span.Span; import com.jozufozu.flywheel.backend.pipeline.span.StringSpan; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import net.minecraft.util.ResourceLocation; public class SourceFile { // #use "valid_namespace:valid/path_to_file.glsl" private static final Pattern includePattern = Pattern.compile("#use \"(\\w+:[\\w./]+)\""); + private static final Pattern newLine = Pattern.compile("(\\r\\n|\\r|\\n)"); + // https://regexr.com/60n3d public static final Pattern functionDeclaration = Pattern.compile("(\\w+)\\s+(\\w+)\\s*\\(([\\w,\\s]*)\\)\\s*\\{"); public final ResourceLocation name; - private final String source; - private final ShaderSources parent; - // function name -> function object + private final ShaderSources parent; + private final String source; + + private final IntList lineStarts; + + // Function name -> Function object private final ImmutableMap functions; - private final ImmutableList includes; + + // Includes ordered as defined in the source + private final ImmutableList includes; // Sections of the source that must be trimmed for compilation. private final List elisions = new ArrayList<>(); @@ -44,8 +51,10 @@ public class SourceFile { this.name = name; this.source = source; - functions = parseFunctions(); - includes = parseIncludes(); + this.lineStarts = getLinePositions(); + + this.functions = parseFunctions(); + this.includes = parseIncludes(); } public String getSource() { @@ -60,28 +69,81 @@ public class SourceFile { return functions; } - public ImmutableList getIncludes() { + public ImmutableList getIncludes() { return includes; } - private ImmutableList parseIncludes() { - Matcher uses = includePattern.matcher(source); + public CharPos getCharPos(int charPos) { + int lineNo = 0; + for (; lineNo < lineStarts.size(); lineNo++) { + int ls = lineStarts.getInt(lineNo); - List includes = new ArrayList<>(); - - while (uses.find()) { - Span use = Span.fromMatcher(this, uses); - - elisions.add(use); // we have to trim that later - - ResourceLocation loc = new ResourceLocation(uses.group(1)); // TODO: error gracefully - - includes.add(loc); + if (charPos < ls) { + break; + } } - return ImmutableList.copyOf(includes); + int lineStart = lineStarts.getInt(lineNo - 1); + + return new CharPos(charPos, lineNo, charPos - lineStart); } + public String printSource() { + StringBuilder builder = new StringBuilder(); + + builder.append("Source for shader '") + .append(name) + .append("':\n"); + int i = 1; + for (String s : source.split("\n")) { + builder.append(String.format("%1$4s: ", i++)) + .append(s) + .append('\n'); + } + + return builder.toString(); + } + + private CharSequence elided = null; + + public CharSequence getElidedSource() { + if (elided == null) { + StringBuilder out = new StringBuilder(); + + int lastEnd = 0; + + for (Span elision : elisions) { + out.append(source, lastEnd, elision.getStart()); + + lastEnd = elision.getEnd(); + } + + out.append(source, lastEnd, source.length()); + + elided = out.toString(); + } + + return elided; + } + + /** + * Scan the source for line breaks, recording the position of the first character of each line. + */ + private IntList getLinePositions() { + IntList l = new IntArrayList(); + l.add(0); // first line is always at position 0 + + Matcher matcher = newLine.matcher(source); + + while (matcher.find()) { + l.add(matcher.end()); + } + return l; + } + + /** + * Scan the source for function definitions and "parse" them into objects that contain properties of the function. + */ private ImmutableMap parseFunctions() { Matcher matcher = functionDeclaration.matcher(source); @@ -113,59 +175,43 @@ public class SourceFile { return ImmutableMap.copyOf(functions); } - private int findEndOfBlock(int end) { - char[] rest = source.substring(end) - .toCharArray(); + /** + * Scan the source for #use "..." directives. + * Records the contents of the directive into an {@link Include} object, and marks the directive for elision. + */ + private ImmutableList parseIncludes() { + Matcher uses = includePattern.matcher(source); + List includes = new ArrayList<>(); + + while (uses.find()) { + Span use = Span.fromMatcher(this, uses); + Span file = Span.fromMatcher(this, uses, 1); + + includes.add(new Include(parent, use, file)); + + elisions.add(use); // we have to trim that later + } + + return ImmutableList.copyOf(includes); + } + + /** + * Given the position of an opening brace, scans through the source for a paired closing brace. + */ + private int findEndOfBlock(int start) { int blockDepth = 0; - for (int i = 0; i < rest.length; i++) { - char ch = rest[i]; + for (int i = start + 1; i < source.length(); i++) { + char ch = source.charAt(i); if (ch == '{') blockDepth++; - if (ch == '}') blockDepth--; + else if (ch == '}') blockDepth--; if (blockDepth < 0) { - return end + i; + return i; } } return -1; } - - public String printSource() { - StringBuilder builder = new StringBuilder(); - - builder.append("Source for shader '") - .append(name) - .append("':\n"); - int i = 1; - for (String s : source.split("\n")) { - builder.append(String.format("%1$4s: ", i++)) - .append(s) - .append('\n'); - } - - return builder.toString(); - } - - private CharSequence elided = null; - public CharSequence getElidedSource() { - if (elided == null) { - StringBuilder out = new StringBuilder(); - - int lastEnd = 0; - - for (Span elision : elisions) { - out.append(source, lastEnd, elision.getStart()); - - lastEnd = elision.getEnd(); - } - - out.append(source, lastEnd, source.length()); - - elided = out.toString(); - } - - return elided; - } } diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/AbstractShaderElement.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/AbstractShaderElement.java new file mode 100644 index 000000000..c88c0be3a --- /dev/null +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/AbstractShaderElement.java @@ -0,0 +1,14 @@ +package com.jozufozu.flywheel.backend.pipeline.parse; + +import com.jozufozu.flywheel.backend.pipeline.span.Span; + +public abstract class AbstractShaderElement { + + public final Span self; + + public AbstractShaderElement(Span self) { + this.self = self; + } + + public abstract void checkErrors(ErrorReporter e); +} diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ErrorReporter.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ErrorReporter.java new file mode 100644 index 000000000..a41afd9e2 --- /dev/null +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ErrorReporter.java @@ -0,0 +1,14 @@ +package com.jozufozu.flywheel.backend.pipeline.parse; + +import com.jozufozu.flywheel.backend.pipeline.SourceFile; +import com.jozufozu.flywheel.backend.pipeline.span.Span; + +public class ErrorReporter { + + + public String generateSpanError(Span span, String message) { + SourceFile file = span.getSourceFile(); + + return ""; + } +} diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Include.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Include.java new file mode 100644 index 000000000..703435654 --- /dev/null +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Include.java @@ -0,0 +1,49 @@ +package com.jozufozu.flywheel.backend.pipeline.parse; + +import java.util.Optional; + +import com.jozufozu.flywheel.backend.ShaderSources; +import com.jozufozu.flywheel.backend.pipeline.SourceFile; +import com.jozufozu.flywheel.backend.pipeline.span.Span; + +import net.minecraft.util.ResourceLocation; + +public class Include extends AbstractShaderElement { + + private final ShaderSources sources; + private Span file; + + private SourceFile resolution; + + public Include(ShaderSources sources, Span self, Span file) { + super(self); + this.sources = sources; + this.file = file; + } + + public boolean isResolved() { + return resolution != null; + } + + public Optional getTarget() { + return Optional.ofNullable(resolution); + } + + public ResourceLocation getFile() { + return new ResourceLocation(file.get()); + } + + @Override + public void checkErrors(ErrorReporter e) { + + String name = file.get(); + + try { + ResourceLocation loc = new ResourceLocation(name); + resolution = sources.source(loc); + } catch (RuntimeException error) { + + } + + } +} diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderFunction.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderFunction.java index aff3121cd..83503a09e 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderFunction.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderFunction.java @@ -8,7 +8,7 @@ import java.util.stream.Collectors; import com.jozufozu.flywheel.backend.pipeline.span.Span; -public class ShaderFunction { +public class ShaderFunction extends AbstractShaderElement { public static final Pattern argument = Pattern.compile("(\\w+)\\s+(\\w+)"); public static final Pattern assignment = Pattern.compile("(\\w+)\\s*="); @@ -17,16 +17,15 @@ public class ShaderFunction { private final Span name; private final Span args; private final Span body; - private final Span self; private final List parameters; public ShaderFunction(Span self, Span type, Span name, Span args, Span body) { + super(self); this.type = type; this.name = name; this.args = args; this.body = body; - this.self = self; this.parameters = new ArrayList<>(); @@ -59,6 +58,11 @@ public class ShaderFunction { .map(Span::get) .collect(Collectors.joining(",")); - return name + "(" + p + ")"; + return type + " " + name + "(" + p + ")"; + } + + @Override + public void checkErrors(ErrorReporter e) { + } } diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderStruct.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderStruct.java index c556c5cc7..2f3709356 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderStruct.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/ShaderStruct.java @@ -7,36 +7,56 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.jozufozu.flywheel.backend.loading.Program; import com.jozufozu.flywheel.backend.loading.TypeHelper; import com.jozufozu.flywheel.backend.pipeline.span.Span; -public class ShaderStruct { +public class ShaderStruct extends AbstractShaderElement { // https://regexr.com/5t207 public static final Pattern struct = Pattern.compile("struct\\s+([\\w\\d]*)\\s*\\{([\\w\\d \\t#\\[\\](),;\\n]*)}\\s*;"); - public final Span self; public final Span name; public final Span body; - List fields = new ArrayList<>(4); - Map fields2Types = new HashMap<>(); + private final ImmutableList fields; + private final ImmutableMap fields2Types; public ShaderStruct(Span self, Span name, Span body) { - this.self = self; + super(self); this.name = name; this.body = body; - parseFields(); + this.fields = parseFields(); + + ImmutableMap.Builder lookup = ImmutableMap.builder(); + for (StructField field : fields) { + lookup.put(field.name.get(), field.type); + } + + this.fields2Types = lookup.build(); } - private void parseFields() { - Matcher fielder = StructField.fieldPattern.matcher(body.get()); + @Override + public void checkErrors(ErrorReporter e) { - while (fielder.find()) { - fields.add(new StructField(fielder)); - fields2Types.put(fielder.group(2), fielder.group(1)); + } + + private ImmutableList parseFields() { + Matcher matcher = StructField.fieldPattern.matcher(body); + + ImmutableList.Builder fields = ImmutableList.builder(); + + while (matcher.find()) { + Span field = Span.fromMatcher(body, matcher); + Span type = Span.fromMatcher(body, matcher, 1); + Span name = Span.fromMatcher(body, matcher, 2); + + fields.add(new StructField(field, type, name)); } + + return fields.build(); } public void addPrefixedAttributes(Program builder, String prefix) { diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/StructField.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/StructField.java index fd0d0bb2e..07c93720b 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/StructField.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/StructField.java @@ -4,24 +4,25 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import com.jozufozu.flywheel.backend.loading.LayoutTag; +import com.jozufozu.flywheel.backend.pipeline.span.Span; -public class StructField { +public class StructField extends AbstractShaderElement { public static final Pattern fieldPattern = Pattern.compile("(\\S+)\\s*(\\S+);"); - public String name; - public String type; - public LayoutTag layout; + public Span name; + public Span type; - public StructField(Matcher fieldMatcher) { - type = fieldMatcher.group(2); - name = fieldMatcher.group(3); + public StructField(Span self, Span name, Span type) { + super(self); + this.name = name; + this.type = type; } - public String getName() { + public Span getName() { return name; } - public String getType() { + public Span getType() { return type; } @@ -29,4 +30,9 @@ public class StructField { public String toString() { return "TaggedField{" + "name='" + name + '\'' + ", type='" + type + '\'' + '}'; } + + @Override + public void checkErrors(ErrorReporter e) { + + } } diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Variable.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Variable.java index d0bcd343c..6576ddd9f 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Variable.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/Variable.java @@ -2,14 +2,13 @@ package com.jozufozu.flywheel.backend.pipeline.parse; import com.jozufozu.flywheel.backend.pipeline.span.Span; -public class Variable { +public class Variable extends AbstractShaderElement { - private final Span self; private final Span type; private final Span name; public Variable(Span self, Span type, Span name) { - this.self = self; + super(self); this.type = type; this.name = name; } @@ -21,4 +20,9 @@ public class Variable { public Span getName() { return name; } + + @Override + public void checkErrors(ErrorReporter e) { + + } } diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/package-info.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/package-info.java new file mode 100644 index 000000000..12567386d --- /dev/null +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/parse/package-info.java @@ -0,0 +1,7 @@ +@ParametersAreNonnullByDefault +@MethodsReturnNonnullByDefault +package com.jozufozu.flywheel.backend.pipeline.parse; + +import javax.annotation.ParametersAreNonnullByDefault; + +import mcp.MethodsReturnNonnullByDefault; diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/CharPos.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/CharPos.java new file mode 100644 index 000000000..7dfea1301 --- /dev/null +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/CharPos.java @@ -0,0 +1,28 @@ +package com.jozufozu.flywheel.backend.pipeline.span; + +import com.jozufozu.flywheel.backend.pipeline.SourceFile; + +public class CharPos { + + private final int idx; + private final int line; + private final int col; + + public CharPos(int idx, int line, int col) { + this.idx = idx; + this.line = line; + this.col = col; + } + + public int getPos() { + return idx; + } + + public int getLine() { + return line; + } + + public int getCol() { + return col; + } +} diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/ErrorSpan.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/ErrorSpan.java index aaa4ee143..cccb10720 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/ErrorSpan.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/ErrorSpan.java @@ -11,6 +11,10 @@ public class ErrorSpan extends Span { super(in, start, end); } + public ErrorSpan(SourceFile in, CharPos start, CharPos end) { + super(in, start, end); + } + @Override public Span subSpan(int from, int to) { return new ErrorSpan(in, start, end); // the sub-span of an error is an error in the same location diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/Span.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/Span.java index 0b079481e..49285ae69 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/Span.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/Span.java @@ -7,13 +7,17 @@ import com.jozufozu.flywheel.backend.pipeline.SourceFile; /** * A span of code in a {@link SourceFile}. */ -public abstract class Span { +public abstract class Span implements CharSequence { protected final SourceFile in; - protected final int start; - protected final int end; + protected final CharPos start; + protected final CharPos end; public Span(SourceFile in, int start, int end) { + this(in, in.getCharPos(start), in.getCharPos(end)); + } + + public Span(SourceFile in, CharPos start, CharPos end) { this.in = in; this.start = start; this.end = end; @@ -24,11 +28,11 @@ public abstract class Span { } public int getStart() { - return start; + return start.getPos(); } public int getEnd() { - return end; + return end.getPos(); } public boolean isEmpty() { @@ -41,6 +45,21 @@ public abstract class Span { public abstract boolean isErr(); + @Override + public int length() { + return end.getPos() - start.getPos(); + } + + @Override + public char charAt(int index) { + return in.getSource().charAt(start.getPos() + index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return subSpan(start, end); + } + @Override public String toString() { return get(); diff --git a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/StringSpan.java b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/StringSpan.java index 432c7e3f6..4eb14b8b1 100644 --- a/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/StringSpan.java +++ b/src/main/java/com/jozufozu/flywheel/backend/pipeline/span/StringSpan.java @@ -8,15 +8,19 @@ public class StringSpan extends Span { super(in, start, end); } + public StringSpan(SourceFile in, CharPos start, CharPos end) { + super(in, start, end); + } + @Override public Span subSpan(int from, int to) { - return new StringSpan(in, start + from, start + to); + return new StringSpan(in, start.getPos() + from, start.getPos() + to); } @Override public String get() { return in.getSource() - .substring(start, end); + .substring(start.getPos(), end.getPos()); } @Override