-
-
Notifications
You must be signed in to change notification settings - Fork 925
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3519 from jruby/truffle-new-pack-parser
[Truffle] new pack parser
- 9.4.12.0
- 9.4.11.0
- 9.4.10.0
- 9.4.9.0
- 9.4.8.0
- 9.4.7.0
- 9.4.6.0
- 9.4.5.0
- 9.4.4.0
- 9.4.3.0
- 9.4.2.0
- 9.4.1.0
- 9.4.0.0
- 9.3.15.0
- 9.3.14.0
- 9.3.13.0
- 9.3.12.0
- 9.3.11.0
- 9.3.10.0
- 9.3.9.0
- 9.3.8.0
- 9.3.7.0
- 9.3.6.0
- 9.3.5.0
- 9.3.4.0
- 9.3.3.0
- 9.3.2.0
- 9.3.1.0
- 9.3.0.0
- 9.2.21.0
- 9.2.20.1
- 9.2.20.0
- 9.2.19.0
- 9.2.18.0
- 9.2.17.0
- 9.2.16.0
- 9.2.15.0
- 9.2.14.0
- 9.2.13.0
- 9.2.12.0
- 9.2.11.1
- 9.2.11.0
- 9.2.10.0
- 9.2.9.0
- 9.2.8.0
- 9.2.7.0
- 9.2.6.0
- 9.2.5.0
- 9.2.4.1
- 9.2.4.0
- 9.2.3.0
- 9.2.2.0
- 9.2.1.0
- 9.2.0.0
- 9.1.17.0
- 9.1.16.0
- 9.1.15.0
- 9.1.14.0
- 9.1.13.0
- 9.1.12.0
- 9.1.11.0
- 9.1.10.0
- 9.1.9.0
- 9.1.8.0
- 9.1.7.0
- 9.1.6.0
- 9.1.5.0
- 9.1.4.0
- 9.1.3.0
- 9.1.2.0
- 9.1.1.0
- 9.1.0.0
- 9.0.5.0
Showing
15 changed files
with
832 additions
and
791 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
truffle/src/main/antlr4/org/jruby/truffle/format/parser/Pack.g4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This | ||
| * code is released under a tri EPL/GPL/LGPL license. You can use it, | ||
| * redistribute it and/or modify it under the terms of the: | ||
| * | ||
| * Eclipse Public License version 1.0 | ||
| * GNU General Public License version 2 | ||
| * GNU Lesser General Public License version 2.1 | ||
| */ | ||
| grammar Pack; | ||
|
|
||
| sequence : directive* ; | ||
|
|
||
| directive : C count? # character | ||
| | (S NATIVE? LITTLE | 'v') count? # shortLittle | ||
| | (S NATIVE? BIG | 'n') count? # shortBig | ||
| | S NATIVE? count? # shortNative | ||
| | I (LITTLE NATIVE? | NATIVE? LITTLE) count? # intLittle | ||
| | I (BIG NATIVE? | NATIVE? BIG) count? # intBig | ||
| | I NATIVE? count? # intNative | ||
| | (L LITTLE | 'V') count? # longLittle | ||
| | (L BIG | 'N') count? # longBig | ||
| | L count? # longNative | ||
| | (Q | L NATIVE) LITTLE count? # quadLittle | ||
| | (Q | L NATIVE) BIG count? # quadBig | ||
| | (Q | L NATIVE) count? # quadNative | ||
| | 'U' count? # utf8Character | ||
| | 'w' count? # berInteger | ||
| | D count? # doubleNative | ||
| | F count? # floatNative | ||
| | 'E' count? # doubleLittle | ||
| | 'e' count? # floatLittle | ||
| | 'G' count? # doubleBig | ||
| | 'g' count? # floatBig | ||
| | 'A' count? # binaryStringSpacePadded | ||
| | 'a' count? # binaryStringNullPadded | ||
| | 'Z' count? # binaryStringNullStar | ||
| | 'B' count? # bitStringMSBFirst | ||
| | 'b' count? # bitStringMSBLast | ||
| | 'H' count? # hexStringHighFirst | ||
| | 'h' count? # hexStringLowFirst | ||
| | 'u' count? # uuString | ||
| | 'M' INT? # mimeString | ||
| | 'm' count? # base64String | ||
| | ('p' | 'P') # pointer | ||
| | '@' INT? # at | ||
| | 'X' count? # back | ||
| | 'x' count? # nullByte | ||
| | subSequence # subSequenceAlternate | ||
| | ('v' | 'n' | 'V' | 'N' | 'U' | 'w' | D | | ||
| F | 'E' | 'e' | 'g' | 'G' | 'A' | 'a' | | ||
| 'Z' | 'B' | 'b' | 'H' | 'h' | 'u' | 'M' | | ||
| 'm' | 'p' | 'P' | 'X' | 'x') NATIVE #errorDisallowedNative ; | ||
|
|
||
| subSequence : '(' directive+ ')' INT? ; | ||
|
|
||
| count : INT | '*' ; | ||
|
|
||
| C : [cC] ; | ||
| S : [sS] ; | ||
| I : [iI] ; | ||
| L : [lL] ; | ||
| Q : [qQ] ; | ||
| D : [dD] ; | ||
| F : [fF] ; | ||
|
|
||
| LITTLE : '<' ; | ||
| BIG : '>' ; | ||
| NATIVE : [_!] ; | ||
|
|
||
| INT : [0-9]+ ; | ||
|
|
||
| WS : [ \t\n\u000b\f\r\u0000]+ -> skip ; | ||
| COMMENT : '#' .*? (('\r'? '\n') | EOF) -> skip ; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
154 changes: 154 additions & 0 deletions
154
truffle/src/main/java/org/jruby/truffle/format/parser/PackCompiler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| /* | ||
| * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This | ||
| * code is released under a tri EPL/GPL/LGPL license. You can use it, | ||
| * redistribute it and/or modify it under the terms of the: | ||
| * | ||
| * Eclipse Public License version 1.0 | ||
| * GNU General Public License version 2 | ||
| * GNU Lesser General Public License version 2.1 | ||
| */ | ||
| package org.jruby.truffle.format.parser; | ||
|
|
||
| import com.oracle.truffle.api.CallTarget; | ||
| import com.oracle.truffle.api.Truffle; | ||
| import org.antlr.v4.runtime.ANTLRInputStream; | ||
| import org.antlr.v4.runtime.CommonTokenStream; | ||
| import org.jruby.truffle.format.nodes.PackRootNode; | ||
| import org.jruby.truffle.nodes.RubyNode; | ||
| import org.jruby.truffle.runtime.RubyContext; | ||
|
|
||
| import org.jruby.truffle.format.parser.PackLexer; | ||
| import org.jruby.truffle.format.parser.PackParser; | ||
|
|
||
| public class PackCompiler { | ||
|
|
||
| private final RubyContext context; | ||
| private final RubyNode currentNode; | ||
|
|
||
| public PackCompiler(RubyContext context, RubyNode currentNode) { | ||
| this.context = context; | ||
| this.currentNode = currentNode; | ||
| } | ||
|
|
||
| public CallTarget compile(String format) { | ||
| if (format.length() > 32) { | ||
| format = recoverLoop(format); | ||
| } | ||
|
|
||
| final PackErrorListener errorListener = new PackErrorListener(context, currentNode); | ||
|
|
||
| final ANTLRInputStream input = new ANTLRInputStream(format.toString()); | ||
|
|
||
| final PackLexer lexer = new PackLexer(input); | ||
| lexer.removeErrorListeners(); | ||
| lexer.addErrorListener(errorListener); | ||
|
|
||
| final CommonTokenStream tokens = new CommonTokenStream(lexer); | ||
|
|
||
| final PackParser parser = new PackParser(tokens); | ||
|
|
||
| final PackTreeBuilder builder = new PackTreeBuilder(context, currentNode); | ||
| parser.addParseListener(builder); | ||
|
|
||
| parser.removeErrorListeners(); | ||
| parser.addErrorListener(errorListener); | ||
|
|
||
| parser.sequence(); | ||
|
|
||
| return Truffle.getRuntime().createCallTarget( | ||
| new PackRootNode(describe(format), builder.getEncoding(), builder.getNode())); | ||
| } | ||
|
|
||
| /** | ||
| * Format strings can sometimes be dynamically generated with code such as: | ||
| * <p> | ||
| * <code>'x' + ('NX' * size)</code> | ||
| * <p> | ||
| * This is problematic for us as it expands to: | ||
| * <p> | ||
| * <code>xNXNXNXNXNXNXNXNXNXNX...</code> | ||
| * <p> | ||
| * We will struggle to compile that with a large value for size because it | ||
| * will generate a huge number of nodes. Even if we could compile it, it's | ||
| * not great for memory consumption or the instruction cache. Instead, we'd | ||
| * like to recover the loop in there and convert it to: | ||
| * <p> | ||
| * <code>x(NX)1000</code> | ||
| * <p> | ||
| * We could try and do something really sophisticated here, with nested | ||
| * loops and finding the optimal pattern, but for the moment we just look | ||
| * for one simple loop. | ||
| * <p> | ||
| * To do that, for each character we look 1..n characters behind and see if | ||
| * that pattern is repeated. If it is we have the loop. Nothing more | ||
| * complicated than that. | ||
| */ | ||
| private String recoverLoop(String format) { | ||
| int break_point = 0; | ||
|
|
||
| while (break_point < format.length()) { | ||
| if ("0123456789*".indexOf(format.charAt(break_point)) != -1) { | ||
| break_point++; | ||
| continue; | ||
| } | ||
|
|
||
| int repeated_length = 1; | ||
| int max_repeated_length = -1; | ||
|
|
||
| while (repeated_length <= break_point && break_point + repeated_length <= format.length()) { | ||
| if (format.substring(break_point - repeated_length, break_point) | ||
| .equals(format.substring(break_point, break_point + repeated_length))) { | ||
| max_repeated_length = repeated_length; | ||
| } | ||
|
|
||
| repeated_length++; | ||
| } | ||
|
|
||
| if (max_repeated_length == -1) { | ||
| break_point++; | ||
| } else { | ||
| final String repeated = format.substring(break_point, break_point + max_repeated_length); | ||
|
|
||
| int count = 2; | ||
| int rep_point = break_point + max_repeated_length; | ||
|
|
||
| while (rep_point + max_repeated_length <= format.length()) { | ||
| if (!format.substring(rep_point, rep_point + max_repeated_length).equals(repeated)) { | ||
| break; | ||
| } | ||
|
|
||
| count++; | ||
| rep_point += max_repeated_length; | ||
| } | ||
|
|
||
| final StringBuilder builder = new StringBuilder(); | ||
| builder.append(format.substring(0, break_point - max_repeated_length)); | ||
| builder.append('('); | ||
| builder.append(repeated); | ||
| builder.append(')'); | ||
| builder.append(count); | ||
| builder.append(format.substring(rep_point)); | ||
| format = builder.toString(); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| return format; | ||
| } | ||
|
|
||
| /** | ||
| * Provide a simple string describing the format expression that is short | ||
| * enough to be used in Truffle and Graal diagnostics. | ||
| */ | ||
| public static String describe(String format) { | ||
| format = format.replace("\\s+", ""); | ||
|
|
||
| if (format.length() > 10) { | ||
| format = format.substring(0, 10) + "…"; | ||
| } | ||
|
|
||
| return format; | ||
| } | ||
|
|
||
|
|
||
| } |
34 changes: 34 additions & 0 deletions
34
truffle/src/main/java/org/jruby/truffle/format/parser/PackErrorListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This | ||
| * code is released under a tri EPL/GPL/LGPL license. You can use it, | ||
| * redistribute it and/or modify it under the terms of the: | ||
| * | ||
| * Eclipse Public License version 1.0 | ||
| * GNU General Public License version 2 | ||
| * GNU Lesser General Public License version 2.1 | ||
| */ | ||
| package org.jruby.truffle.format.parser; | ||
|
|
||
| import org.antlr.v4.runtime.BaseErrorListener; | ||
| import org.antlr.v4.runtime.RecognitionException; | ||
| import org.antlr.v4.runtime.Recognizer; | ||
| import org.jruby.truffle.nodes.RubyNode; | ||
| import org.jruby.truffle.runtime.RubyContext; | ||
| import org.jruby.truffle.runtime.control.RaiseException; | ||
|
|
||
| public class PackErrorListener extends BaseErrorListener { | ||
|
|
||
| private final RubyContext context; | ||
| private final RubyNode currentNode; | ||
|
|
||
| public PackErrorListener(RubyContext context, RubyNode currentNode) { | ||
| this.context = context; | ||
| this.currentNode = currentNode; | ||
| } | ||
|
|
||
| @Override | ||
| public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { | ||
| throw new RaiseException(context.getCoreLibrary().argumentError(msg, currentNode)); | ||
| } | ||
|
|
||
| } |
576 changes: 0 additions & 576 deletions
576
truffle/src/main/java/org/jruby/truffle/format/parser/PackParser.java
This file was deleted.
Oops, something went wrong.
128 changes: 0 additions & 128 deletions
128
truffle/src/main/java/org/jruby/truffle/format/parser/PackTokenizer.java
This file was deleted.
Oops, something went wrong.
489 changes: 489 additions & 0 deletions
489
truffle/src/main/java/org/jruby/truffle/format/parser/PackTreeBuilder.java
Large diffs are not rendered by default.
Oops, something went wrong.
15 changes: 0 additions & 15 deletions
15
truffle/src/main/java/org/jruby/truffle/format/runtime/Endianness.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters