Merge pull request #3551 from jruby/truffle-antlr-printf

[Truffle] Antlr printf
jruby · Dec 20, 2015 · 9016ea4 · 9016ea4
2 parents 7c16fea + 60374d7
commit 9016ea4
Showing 13 changed files with 406 additions and 434 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,7 @@
 *.versionsBackup
 *.zip
 *~
+*.tokens
 
 .DS_Store
 .debug.properties

diff --git a/spec/truffle/tags/core/string/modulo_tags.txt b/spec/truffle/tags/core/string/modulo_tags.txt
@@ -79,3 +79,4 @@ fails:String#% behaves as if calling Kernel#Float for %g arguments, when the pas
 fails:String#% behaves as if calling Kernel#Float for %g arguments, when the passed argument is hexadecimal string
 fails:String#% behaves as if calling Kernel#Float for %G arguments, when the passed argument does not respond to #to_ary
 fails:String#% behaves as if calling Kernel#Float for %G arguments, when the passed argument is hexadecimal string
+fails:String#% when format string contains %<> formats should raise ArgumentError if no hash given
diff --git a/tool/truffle-findbugs-exclude.xml b/tool/truffle-findbugs-exclude.xml
@@ -25,6 +25,14 @@
     <Class name="org.jruby.truffle.format.parser.PackParser" />
   </Match>
 
+  <Match>
+    <Class name="org.jruby.truffle.format.parser.PrintfLexer" />
+  </Match>
+
+  <Match>
+    <Class name="org.jruby.truffle.format.parser.PrintfParser" />
+  </Match>
+
   <!-- Sometimes we really do want to exit - but we should rethink for multi-tennant -->
 
   <Match>

diff --git a/truffle/src/main/antlr4/org/jruby/truffle/format/parser/PrintfLexer.g4 b/truffle/src/main/antlr4/org/jruby/truffle/format/parser/PrintfLexer.g4
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This
+ * code is released under a tri EPL/GPL/LGPL license. You can use it,
+ * redistribute it and/or modify it under the terms of the:
+ *
+ * Eclipse Public License version 1.0
+ * GNU General Public License version 2
+ * GNU Lesser General Public License version 2.1
+ */
+lexer grammar PrintfLexer;
+
+FORMAT  : '%' -> mode(FORMAT_MODE);
+LITERAL : (~'%')* ;
+
+mode FORMAT_MODE;
+
+ANGLE_KEY   : '<' .*? '>' ;
+ZERO        : '0' ;
+NUMBER      : [1-9] [0-9]* ;
+SPACE       : ' ' ;
+PLUS        : '+' ;
+MINUS       : '-' ;
+STAR        : '*' ;
+DOLLAR      : '$' ;
+DOT         : '.' ;
+CURLY_KEY   : '{' .*? '}' -> mode(DEFAULT_MODE) ;
+TYPE        : [bBdiouxXeEfgGaAcps] -> mode(DEFAULT_MODE) ;
+ESCAPED     : '%' -> mode(DEFAULT_MODE) ;
diff --git a/truffle/src/main/antlr4/org/jruby/truffle/format/parser/PrintfParser.g4 b/truffle/src/main/antlr4/org/jruby/truffle/format/parser/PrintfParser.g4
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This
+ * code is released under a tri EPL/GPL/LGPL license. You can use it,
+ * redistribute it and/or modify it under the terms of the:
+ *
+ * Eclipse Public License version 1.0
+ * GNU General Public License version 2
+ * GNU Lesser General Public License version 2.1
+ */
+parser grammar PrintfParser;
+
+options { tokenVocab=PrintfLexer; }
+
+sequence : (FORMAT directive | literal)* ;
+
+directive : CURLY_KEY                  # string
+          | ESCAPED                    # escaped
+          | ANGLE_KEY?
+            flag*
+            width=NUMBER?
+            (DOT precision=NUMBER)?
+            TYPE                       # format ;
+
+flag : SPACE
+     | ZERO
+     | PLUS
+     | MINUS
+     | STAR
+     | NUMBER DOLLAR ;
+
+literal : LITERAL ;
diff --git a/truffle/src/main/java/org/jruby/truffle/format/nodes/format/FormatFloatNode.java b/truffle/src/main/java/org/jruby/truffle/format/nodes/format/FormatFloatNode.java
@@ -14,7 +14,7 @@
 import com.oracle.truffle.api.dsl.NodeChildren;
 import com.oracle.truffle.api.dsl.Specialization;
 import org.jruby.truffle.format.nodes.PackNode;
-import org.jruby.truffle.format.parser.FormatDirective;
+import org.jruby.truffle.format.parser.PrintfTreeBuilder;
 import org.jruby.truffle.runtime.RubyContext;
 import org.jruby.truffle.runtime.core.StringOperations;
 import org.jruby.util.ByteList;
@@ -68,12 +68,12 @@ public ByteList format(double value) {
 
             if (Double.isInfinite(value)) {
 
-                if (spacePadding != FormatDirective.DEFAULT) {
+                if (spacePadding != PrintfTreeBuilder.DEFAULT) {
                     builder.append(" ");
                     builder.append(spacePadding + 5);
                 }
 
-                if (zeroPadding != FormatDirective.DEFAULT && zeroPadding != 0) {
+                if (zeroPadding != PrintfTreeBuilder.DEFAULT && zeroPadding != 0) {
                     builder.append("0");
                     builder.append(zeroPadding + 5);
                 }
@@ -86,20 +86,20 @@ public ByteList format(double value) {
 
             } else {
 
-                if (spacePadding != FormatDirective.DEFAULT) {
+                if (spacePadding != PrintfTreeBuilder.DEFAULT) {
                     builder.append(" ");
                     builder.append(spacePadding);
 
-                    if (zeroPadding != FormatDirective.DEFAULT) {
+                    if (zeroPadding != PrintfTreeBuilder.DEFAULT) {
                         builder.append(".");
                         builder.append(zeroPadding);
                     }
-                } else if (zeroPadding != FormatDirective.DEFAULT && zeroPadding != 0) {
+                } else if (zeroPadding != PrintfTreeBuilder.DEFAULT && zeroPadding != 0) {
                     builder.append("0");
                     builder.append(zeroPadding);
                 }
 
-                if (precision != FormatDirective.DEFAULT) {
+                if (precision != PrintfTreeBuilder.DEFAULT) {
                     builder.append(".");
                     builder.append(precision);
                 }

diff --git a/truffle/src/main/java/org/jruby/truffle/format/nodes/format/FormatIntegerNode.java b/truffle/src/main/java/org/jruby/truffle/format/nodes/format/FormatIntegerNode.java
@@ -16,7 +16,7 @@
 import com.oracle.truffle.api.object.DynamicObject;
 
 import org.jruby.truffle.format.nodes.PackNode;
-import org.jruby.truffle.format.parser.FormatDirective;
+import org.jruby.truffle.format.parser.PrintfTreeBuilder;
 import org.jruby.truffle.runtime.RubyContext;
 import org.jruby.truffle.runtime.layouts.Layouts;
 import org.jruby.util.ByteList;
@@ -98,15 +98,15 @@ protected ByteList doFormat(Object value, int spacePadding, int zeroPadding) {
 
         builder.append("%");
 
-        if (spacePadding != FormatDirective.DEFAULT) {
+        if (spacePadding != PrintfTreeBuilder.DEFAULT) {
             builder.append(" ");
             builder.append(spacePadding);
 
-            if (zeroPadding != FormatDirective.DEFAULT) {
+            if (zeroPadding != PrintfTreeBuilder.DEFAULT) {
                 builder.append(".");
                 builder.append(zeroPadding);
             }
-        } else if (zeroPadding != FormatDirective.DEFAULT) {
+        } else if (zeroPadding != PrintfTreeBuilder.DEFAULT) {
             builder.append("0");
             builder.append(zeroPadding);
         }

diff --git a/truffle/src/main/java/org/jruby/truffle/format/parser/FormatDirective.java b/truffle/src/main/java/org/jruby/truffle/format/parser/FormatDirective.java
diff --git a/truffle/src/main/java/org/jruby/truffle/format/parser/FormatParser.java b/truffle/src/main/java/org/jruby/truffle/format/parser/FormatParser.java
diff --git a/truffle/src/main/java/org/jruby/truffle/format/parser/FormatTokenizer.java b/truffle/src/main/java/org/jruby/truffle/format/parser/FormatTokenizer.java
diff --git a/truffle/src/main/java/org/jruby/truffle/format/parser/PrintfCompiler.java b/truffle/src/main/java/org/jruby/truffle/format/parser/PrintfCompiler.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This
+ * code is released under a tri EPL/GPL/LGPL license. You can use it,
+ * redistribute it and/or modify it under the terms of the:
+ *
+ * Eclipse Public License version 1.0
+ * GNU General Public License version 2
+ * GNU Lesser General Public License version 2.1
+ */
+package org.jruby.truffle.format.parser;
+
+import com.oracle.truffle.api.CallTarget;
+import com.oracle.truffle.api.Truffle;
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.jruby.truffle.format.nodes.PackRootNode;
+import org.jruby.truffle.format.runtime.PackEncoding;
+import org.jruby.truffle.nodes.RubyNode;
+import org.jruby.truffle.runtime.RubyContext;
+
+import org.jruby.truffle.format.parser.PrintfLexer;
+import org.jruby.truffle.format.parser.PrintfParser;
+import org.jruby.util.ByteList;
+
+public class PrintfCompiler {
+
+    private final RubyContext context;
+    private final RubyNode currentNode;
+
+    public PrintfCompiler(RubyContext context, RubyNode currentNode) {
+        this.context = context;
+        this.currentNode = currentNode;
+    }
+
+    public CallTarget compile(ByteList format) {
+        final PackErrorListener errorListener = new PackErrorListener(context, currentNode);
+
+        final ANTLRInputStream input = new ANTLRInputStream(bytesToChars(format.bytes()), format.realSize());
+
+        final PrintfLexer lexer = new PrintfLexer(input);
+        lexer.removeErrorListeners();
+        lexer.addErrorListener(errorListener);
+
+        final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+        final PrintfParser parser = new PrintfParser(tokens);
+
+        final PrintfTreeBuilder builder = new PrintfTreeBuilder(context, format);
+        parser.addParseListener(builder);
+
+        parser.removeErrorListeners();
+        parser.addErrorListener(errorListener);
+
+        parser.sequence();
+
+        return Truffle.getRuntime().createCallTarget(
+                new PackRootNode(PackCompiler.describe(format.toString()), PackEncoding.DEFAULT, builder.getNode()));
+    }
+
+    public static char[] bytesToChars(byte[] bytes) {
+        final char[] chars = new char[bytes.length];
+
+        for (int n = 0; n < bytes.length; n++) {
+            chars[n] = (char) bytes[n];
+        }
+
+        return chars;
+    }
+
+}
diff --git a/truffle/src/main/java/org/jruby/truffle/format/parser/PrintfTreeBuilder.java b/truffle/src/main/java/org/jruby/truffle/format/parser/PrintfTreeBuilder.java
@@ -0,0 +1,252 @@
+/*
+ * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. This
+ * code is released under a tri EPL/GPL/LGPL license. You can use it,
+ * redistribute it and/or modify it under the terms of the:
+ *
+ * Eclipse Public License version 1.0
+ * GNU General Public License version 2
+ * GNU Lesser General Public License version 2.1
+ */
+package org.jruby.truffle.format.parser;
+
+import com.oracle.truffle.api.object.DynamicObject;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.misc.Interval;
+import org.jruby.truffle.format.nodes.PackNode;
+import org.jruby.truffle.format.nodes.SourceNode;
+import org.jruby.truffle.format.nodes.control.SequenceNode;
+import org.jruby.truffle.format.nodes.format.FormatFloatNodeGen;
+import org.jruby.truffle.format.nodes.format.FormatIntegerNodeGen;
+import org.jruby.truffle.format.nodes.read.*;
+import org.jruby.truffle.format.nodes.type.ToDoubleWithCoercionNodeGen;
+import org.jruby.truffle.format.nodes.type.ToIntegerNodeGen;
+import org.jruby.truffle.format.nodes.type.ToStringNodeGen;
+import org.jruby.truffle.format.nodes.write.WriteByteNode;
+import org.jruby.truffle.format.nodes.write.WriteBytesNodeGen;
+import org.jruby.truffle.format.nodes.write.WritePaddedBytesNodeGen;
+import org.jruby.truffle.runtime.RubyContext;
+import org.jruby.truffle.runtime.layouts.Layouts;
+import org.jruby.util.ByteList;
+import org.jruby.util.StringSupport;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PrintfTreeBuilder extends org.jruby.truffle.format.parser.PrintfParserBaseListener {
+
+    public static final int PADDING_FROM_ARGUMENT = -2;
+
+    public static final int DEFAULT = -1;
+
+    private final RubyContext context;
+    private final ByteList source;
+
+    private final List<PackNode> sequence = new ArrayList<>();
+
+    public PrintfTreeBuilder(RubyContext context, ByteList source) {
+        this.context = context;
+        this.source = source;
+    }
+
+    @Override
+    public void exitEscaped(org.jruby.truffle.format.parser.PrintfParser.EscapedContext ctx) {
+        sequence.add(new WriteByteNode(context, (byte) '%'));
+    }
+
+    @Override
+    public void exitString(org.jruby.truffle.format.parser.PrintfParser.StringContext ctx) {
+        final ByteList keyBytes = tokenAsBytes(ctx.CURLY_KEY().getSymbol(), 1);
+        final DynamicObject key = context.getSymbol(keyBytes);
+
+        sequence.add(
+                WriteBytesNodeGen.create(context,
+                    ToStringNodeGen.create(context, true, "to_s", false, new ByteList(),
+                        ReadHashValueNodeGen.create(context, key, new SourceNode()))));
+    }
+
+    @Override
+    public void exitFormat(org.jruby.truffle.format.parser.PrintfParser.FormatContext ctx) {
+        final int width;
+
+        if (ctx.width != null) {
+            width = Integer.parseInt(ctx.width.getText());
+        } else {
+            width = DEFAULT;
+        }
+
+        boolean leftJustified = false;
+        int spacePadding = DEFAULT;
+        int zeroPadding = DEFAULT;
+
+
+        for (int n = 0; n < ctx.flag().size(); n++) {
+            final org.jruby.truffle.format.parser.PrintfParser.FlagContext flag = ctx.flag(n);
+
+            if (flag.MINUS() != null) {
+                leftJustified = true;
+            } else if (flag.SPACE() != null) {
+                if (n + 1 < ctx.flag().size() && ctx.flag(n + 1).STAR() != null) {
+                    spacePadding = PADDING_FROM_ARGUMENT;
+                } else {
+                    spacePadding = width;
+                }
+            } else if (flag.ZERO() != null) {
+                if (n + 1 < ctx.flag().size() && ctx.flag(n + 1).STAR() != null) {
+                    zeroPadding = PADDING_FROM_ARGUMENT;
+                } else {
+                    zeroPadding = width;
+                }
+            } else if (flag.STAR() != null) {
+                // Handled in space and zero, above
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        }
+
+        if (spacePadding == DEFAULT && zeroPadding == DEFAULT) {
+            spacePadding = width;
+        }
+
+        final char type = ctx.TYPE().getSymbol().getText().charAt(0);
+
+        final PackNode valueNode;
+
+        if (ctx.ANGLE_KEY() == null) {
+            valueNode = ReadValueNodeGen.create(context, new SourceNode());
+        } else {
+            final ByteList keyBytes = tokenAsBytes(ctx.ANGLE_KEY().getSymbol(), 1);
+            final DynamicObject key = context.getSymbol(keyBytes);
+            valueNode = ReadHashValueNodeGen.create(context, key, new SourceNode());
+        }
+
+        final int precision;
+
+        if (ctx.precision != null) {
+            precision = Integer.parseInt(ctx.precision.getText());
+        } else {
+            precision = DEFAULT;
+        }
+
+        final PackNode node;
+
+        switch (type) {
+            case 's':
+                if (ctx.ANGLE_KEY() == null) {
+                    if (spacePadding == DEFAULT) {
+                        node = WriteBytesNodeGen.create(context, ReadStringNodeGen.create(
+                                context, true, "to_s", false, new ByteList(), new SourceNode()));
+                    } else {
+                        node = WritePaddedBytesNodeGen.create(context, spacePadding, leftJustified,
+                                ReadStringNodeGen.create(context, true, "to_s", false, new ByteList(), new SourceNode()));
+                    }
+                } else {
+                    if (spacePadding == DEFAULT) {
+                        node = WriteBytesNodeGen.create(context, ToStringNodeGen.create(
+                                context, true, "to_s", false, new ByteList(), valueNode));
+                    } else {
+                        node = WritePaddedBytesNodeGen.create(context, spacePadding, leftJustified,
+                                ToStringNodeGen.create(context, true, "to_s", false, new ByteList(), valueNode));
+                    }
+                }
+                break;
+            case 'd':
+            case 'i':
+            case 'o':
+            case 'u':
+            case 'x':
+            case 'X':
+                final PackNode spacePaddingNode;
+                if (spacePadding == PADDING_FROM_ARGUMENT) {
+                    spacePaddingNode = ReadIntegerNodeGen.create(context, new SourceNode());
+                } else {
+                    spacePaddingNode = new LiteralIntegerNode(context, spacePadding);
+                }
+
+                final PackNode zeroPaddingNode;
+
+                        /*
+                         * Precision and zero padding both set zero padding -
+                         * but precision has priority and explicit zero padding
+                         * is actually ignored if it's set.
+                         */
+
+                if (zeroPadding == PADDING_FROM_ARGUMENT) {
+                    zeroPaddingNode = ReadIntegerNodeGen.create(context, new SourceNode());
+                } else if (ctx.precision != null) {
+                    zeroPaddingNode = new LiteralIntegerNode(context, Integer.parseInt(ctx.precision.getText()));
+                } else {
+                    zeroPaddingNode = new LiteralIntegerNode(context, zeroPadding);
+                }
+
+                final char format;
+
+                switch (type) {
+                    case 'd':
+                    case 'i':
+                    case 'u':
+                        format = 'd';
+                        break;
+                    case 'o':
+                        format = 'o';
+                        break;
+                    case 'x':
+                    case 'X':
+                        format = type;
+                        break;
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+
+                node = WriteBytesNodeGen.create(context,
+                        FormatIntegerNodeGen.create(context, format,
+                                spacePaddingNode,
+                                zeroPaddingNode,
+                                ToIntegerNodeGen.create(context, valueNode)));
+                break;
+            case 'f':
+            case 'g':
+            case 'G':
+            case 'e':
+            case 'E':
+                node = WriteBytesNodeGen.create(context,
+                        FormatFloatNodeGen.create(context, spacePadding,
+                                zeroPadding, precision,
+                                type,
+                                ToDoubleWithCoercionNodeGen.create(context,
+                                        valueNode)));
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+
+        sequence.add(node);
+    }
+
+    @Override
+    public void exitLiteral(org.jruby.truffle.format.parser.PrintfParser.LiteralContext ctx) {
+        final ByteList text = tokenAsBytes(ctx.LITERAL().getSymbol());
+
+        final PackNode node;
+
+        if (text.length() == 1) {
+            node = new WriteByteNode(context, (byte) text.get(0));
+        } else {
+            node = WriteBytesNodeGen.create(context, new LiteralBytesNode(context, text));
+        }
+
+        sequence.add(node);
+    }
+
+    public PackNode getNode() {
+        return new SequenceNode(context, sequence.toArray(new PackNode[sequence.size()]));
+    }
+
+    private ByteList tokenAsBytes(Token token) {
+        return tokenAsBytes(token, 0);
+    }
+
+    private ByteList tokenAsBytes(Token token, int trim) {
+        return new ByteList(source, token.getStartIndex() + trim, token.getStopIndex() - token.getStartIndex() + 1 - 2 * trim);
+    }
+
+}
diff --git a/truffle/src/main/java/org/jruby/truffle/nodes/core/KernelNodes.java b/truffle/src/main/java/org/jruby/truffle/nodes/core/KernelNodes.java
@@ -33,6 +33,7 @@
 import org.jcodings.specific.UTF8Encoding;
 import org.jruby.exceptions.MainExitException;
 import org.jruby.runtime.Visibility;
+import org.jruby.truffle.format.parser.PrintfCompiler;
 import org.jruby.truffle.nodes.RubyGuards;
 import org.jruby.truffle.nodes.RubyNode;
 import org.jruby.truffle.nodes.RubyRootNode;
@@ -64,7 +65,6 @@
 import org.jruby.truffle.nodes.objectstorage.WriteHeadObjectFieldNodeGen;
 import org.jruby.truffle.nodes.rubinius.ObjectPrimitiveNodes;
 import org.jruby.truffle.nodes.rubinius.ObjectPrimitiveNodesFactory;
-import org.jruby.truffle.format.parser.FormatParser;
 import org.jruby.truffle.format.runtime.PackResult;
 import org.jruby.truffle.format.runtime.exceptions.*;
 import org.jruby.truffle.runtime.*;
@@ -1947,11 +1947,11 @@ public Long block() throws InterruptedException {
 
     @CoreMethod(names = { "format", "sprintf" }, isModuleFunction = true, rest = true, required = 1, taintFromParameter = 0)
     @ImportStatic(StringCachingGuards.class)
-    public abstract static class FormatNode extends CoreMethodArrayArgumentsNode {
+    public abstract static class SprintfNode extends CoreMethodArrayArgumentsNode {
 
         @Child private TaintNode taintNode;
 
-        public FormatNode(RubyContext context, SourceSection sourceSection) {
+        public SprintfNode(RubyContext context, SourceSection sourceSection) {
             super(context, sourceSection);
         }
 
@@ -2048,7 +2048,7 @@ protected CallTarget compileFormat(DynamicObject format) {
             assert RubyGuards.isRubyString(format);
 
             try {
-                return new FormatParser(getContext()).parse(StringOperations.getByteList(format));
+                return new PrintfCompiler(getContext(), this).compile(StringOperations.getByteList(format));
             } catch (FormatException e) {
                 CompilerDirectives.transferToInterpreter();
                 throw new RaiseException(getContext().getCoreLibrary().argumentError(e.getMessage(), this));