Uffda. Change keyword names to match MRI. Fixes one MRI test but real…

…ly it just makes our syntax error messages say 'keyword_foo' rather than the old school 'kFOO'. A second change is to remove Tokens interface altogether and just use the int constants directly from the parser.
jruby · Jun 30, 2017 · c02838c · c02838c
1 parent 9d4c90b
commit c02838c
Showing 14 changed files with 1,414 additions and 1,816 deletions.
diff --git a/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java b/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java
@@ -31,6 +31,7 @@
 
 import org.jcodings.Encoding;
 import org.jruby.lexer.LexerSource;
+import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 
 import static org.jruby.lexer.LexingCommon.*;
@@ -79,7 +80,7 @@ protected int error(RipperLexer lexer, int len, ByteList str, ByteList eos) {
         lexer.compile_error("can't find string \"" + eos.toString() + "\" anywhere before EOF");
 
         if (lexer.delayed == null) {
-            lexer.dispatchScanEvent(Tokens.tSTRING_CONTENT);
+            lexer.dispatchScanEvent(RubyParser.tSTRING_CONTENT);
         } else {
             if (str != null) {
                 lexer.delayed.append(str);
@@ -89,7 +90,7 @@ protected int error(RipperLexer lexer, int len, ByteList str, ByteList eos) {
                     lexer.delayed.append(lexer.lexb.makeShared(lexer.tokp, len));
                 }
             }
-            lexer.dispatchDelayedToken(Tokens.tSTRING_CONTENT);
+            lexer.dispatchDelayedToken(RubyParser.tSTRING_CONTENT);
         }
         lexer.lex_goto_eol();
 
@@ -117,7 +118,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
         if (lexer.was_bol() && lexer.whole_match_p(nd_lit, indent)) {
             lexer.dispatchHeredocEnd();
             lexer.heredoc_restore(this);
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
         }
 
         if ((flags & STR_FUNC_EXPAND) == 0) {
@@ -156,7 +157,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
 
                 if (lexer.getHeredocIndent() > 0) {
                     lexer.setValue(lexer.createStr(str, 0));
-                    return Tokens.tSTRING_CONTENT;
+                    return RubyParser.tSTRING_CONTENT;
                 }
                 // MRI null checks str in this case but it is unconditionally non-null?
                 if (lexer.nextc() == -1) return error(lexer, len, null, eos);
@@ -169,10 +170,10 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
                 case '$':
                 case '@':
                     lexer.pushback(c);
-                    return Tokens.tSTRING_DVAR;
+                    return RubyParser.tSTRING_DVAR;
                 case '{':
                     lexer.commandStart = true;
-                    return Tokens.tSTRING_DBEG;
+                    return RubyParser.tSTRING_DBEG;
                 }
                 tok.append('#');
             }
@@ -191,7 +192,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
                 if (c != '\n') {
                     lexer.setValue(lexer.createStr(tok, 0));
                     lexer.flush_string_content(enc[0]);
-                    return Tokens.tSTRING_CONTENT;
+                    return RubyParser.tSTRING_CONTENT;
                 }
                 tok.append(lexer.nextc());
 
@@ -204,6 +205,6 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
         lexer.heredoc_restore(this);
         lexer.setStrTerm(new StringTerm(-1, '\0', '\0'));
         lexer.setValue(lexer.createStr(str, 0));
-        return Tokens.tSTRING_CONTENT;
+        return RubyParser.tSTRING_CONTENT;
     }
 }
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperLexer.java b/core/src/main/java/org/jruby/ext/ripper/RipperLexer.java
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.java b/core/src/main/java/org/jruby/ext/ripper/RipperParser.java
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.y b/core/src/main/java/org/jruby/ext/ripper/RipperParser.y
diff --git a/core/src/main/java/org/jruby/ext/ripper/StringTerm.java b/core/src/main/java/org/jruby/ext/ripper/StringTerm.java
@@ -30,6 +30,7 @@
 import java.io.IOException;
 import org.jcodings.Encoding;
 import org.jruby.lexer.LexerSource;
+import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 
 import static org.jruby.lexer.LexingCommon.*;
@@ -73,11 +74,11 @@ private int endFound(RipperLexer lexer, LexerSource src, ByteList buffer) throws
                 String options = parseRegexpFlags(lexer, src);
                 buffer.append(options.getBytes());
 
-                return Tokens.tREGEXP_END;
+                return RubyParser.tREGEXP_END;
             }
 
             buffer.append(end);
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
     }
 
     @Override
@@ -89,7 +90,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
         // Heredoc already parsed this and saved string...Do not parse..just return
         if (flags == -1) {
             lexer.ignoreNextScanEvent = true;
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
         }
 
         ByteList buffer = createByteList(lexer);        
@@ -118,9 +119,9 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
             case '$':
             case '@':
                 lexer.pushback(c);
-                return Tokens.tSTRING_DVAR;
+                return RubyParser.tSTRING_DVAR;
             case '{':
-                return Tokens.tSTRING_DBEG;
+                return RubyParser.tSTRING_DBEG;
             }
             buffer.append((byte) '#');
         }
@@ -132,16 +133,16 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
         if (parseStringIntoBuffer(lexer, src, buffer, enc) == EOF) {
             if ((flags & STR_FUNC_REGEXP) != 0) {
                 if (lexer.eofp) lexer.compile_error("unterminated regexp meets end of file");
-                return Tokens.tREGEXP_END;
+                return RubyParser.tREGEXP_END;
             } else {
                 if (lexer.eofp) lexer.compile_error("unterminated string meets end of file");
-                return Tokens.tSTRING_END;
+                return RubyParser.tSTRING_END;
             }
         }
 
         lexer.setValue(lexer.createStr(buffer, flags));
         lexer.flush_string_content(enc[0]);
-        return Tokens.tSTRING_CONTENT;
+        return RubyParser.tSTRING_CONTENT;
     }
 
     private String parseRegexpFlags(RipperLexer lexer, LexerSource src) throws IOException {

diff --git a/core/src/main/java/org/jruby/ext/ripper/Tokens.java b/core/src/main/java/org/jruby/ext/ripper/Tokens.java
diff --git a/core/src/main/java/org/jruby/lexer/yacc/HeredocTerm.java b/core/src/main/java/org/jruby/lexer/yacc/HeredocTerm.java
@@ -30,7 +30,7 @@
 package org.jruby.lexer.yacc;
 
 import org.jcodings.Encoding;
-import org.jruby.parser.Tokens;
+import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 
 import static org.jruby.lexer.LexingCommon.*;
@@ -100,7 +100,7 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
         // Found end marker for this heredoc
         if (lexer.was_bol() && lexer.whole_match_p(nd_lit, indent)) {
             lexer.heredoc_restore(this);
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
         }
 
         if ((flags & STR_FUNC_EXPAND) == 0) {
@@ -139,7 +139,7 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
 
                 if (lexer.getHeredocIndent() > 0) {
                     lexer.setValue(lexer.createStr(str, 0));
-                    return Tokens.tSTRING_CONTENT;
+                    return RubyParser.tSTRING_CONTENT;
                 }
                 // MRI null checks str in this case but it is unconditionally non-null?
                 if (lexer.nextc() == -1) return error(lexer, len, null, eos);
@@ -152,10 +152,10 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
                     case '$':
                     case '@':
                         lexer.pushback(c);
-                        return Tokens.tSTRING_DVAR;
+                        return RubyParser.tSTRING_DVAR;
                     case '{':
                         lexer.commandStart = true;
-                        return Tokens.tSTRING_DBEG;
+                        return RubyParser.tSTRING_DBEG;
                 }
                 tok.append('#');
             }
@@ -173,14 +173,14 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
                 }
                 if (c != '\n') {
                     lexer.setValue(lexer.createStr(tok, 0));
-                    return Tokens.tSTRING_CONTENT;
+                    return RubyParser.tSTRING_CONTENT;
                 }
                 tok.append(lexer.nextc());
 
                 if (lexer.getHeredocIndent() > 0) {
                     lexer.lex_goto_eol();
                     lexer.setValue(lexer.createStr(tok, 0));
-                    return Tokens.tSTRING_CONTENT;
+                    return RubyParser.tSTRING_CONTENT;
                 }
 
                 if ((c = lexer.nextc()) == EOF) return error(lexer, len, str, eos);
@@ -191,6 +191,6 @@ public int parseString(RubyLexer lexer) throws java.io.IOException {
         lexer.heredoc_restore(this);
         lexer.setStrTerm(new StringTerm(-1, '\0', '\0', lexer.getRubySourceline()));
         lexer.setValue(lexer.createStr(str, 0));
-        return Tokens.tSTRING_CONTENT;
+        return RubyParser.tSTRING_CONTENT;
     }
 }
diff --git a/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java b/core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
diff --git a/core/src/main/java/org/jruby/lexer/yacc/StringTerm.java b/core/src/main/java/org/jruby/lexer/yacc/StringTerm.java
@@ -31,7 +31,7 @@
 import org.jcodings.Encoding;
 import org.jruby.ast.RegexpNode;
 import org.jruby.lexer.yacc.SyntaxException.PID;
-import org.jruby.parser.Tokens;
+import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 import org.jruby.util.KCode;
 import org.jruby.util.RegexpOptions;
@@ -84,11 +84,11 @@ private int endFound(RubyLexer lexer) throws IOException {
                 ByteList regexpBytelist = ByteList.create("");
 
                 lexer.setValue(new RegexpNode(lexer.getPosition(), regexpBytelist, options));
-                return Tokens.tREGEXP_END;
+                return RubyParser.tREGEXP_END;
             }
 
             lexer.setValue("" + end);
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
     }
 
     // Return of 0 means failed to find anything.  Non-zero means return that from lexer.
@@ -114,7 +114,7 @@ private int parsePeekVariableName(RubyLexer lexer) throws IOException {
                     lexer.setValue("#" + (char) c2);
 
                     lexer.pushback(c2); lexer.pushback(c);
-                    return Tokens.tSTRING_DVAR;
+                    return RubyParser.tSTRING_DVAR;
                 }
 
                 significant = c2;                                  // $FOO potentially
@@ -145,7 +145,7 @@ private int parsePeekVariableName(RubyLexer lexer) throws IOException {
                 //lexer.setBraceNest(lexer.getBraceNest() + 1);
                 lexer.setValue("#" + (char) c);
                 lexer.commandStart = true;
-                return Tokens.tSTRING_DBEG;
+                return RubyParser.tSTRING_DBEG;
             default:
                 // We did not find significant char after # so push it back to
                 // be processed as an ordinary string.
@@ -156,7 +156,7 @@ private int parsePeekVariableName(RubyLexer lexer) throws IOException {
         if (significant != -1 && Character.isAlphabetic(significant) || significant == '_') {
             lexer.pushback(c);
             lexer.setValue("#" + significant);
-            return Tokens.tSTRING_DVAR;
+            return RubyParser.tSTRING_DVAR;
         }
 
         return 0;
@@ -170,7 +170,7 @@ public int parseString(RubyLexer lexer) throws IOException {
         // Heredoc already parsed this and saved string...Do not parse..just return
         if (flags == -1) {
             lexer.setValue("" + end);
-            return Tokens.tSTRING_END;
+            return RubyParser.tSTRING_END;
         }
 
         c = lexer.nextc();
@@ -205,7 +205,7 @@ public int parseString(RubyLexer lexer) throws IOException {
         }
 
         lexer.setValue(lexer.createStr(buffer, flags));
-        return Tokens.tSTRING_CONTENT;
+        return RubyParser.tSTRING_CONTENT;
     }
 
     private RegexpOptions parseRegexpFlags(RubyLexer lexer) throws IOException {

diff --git a/core/src/main/java/org/jruby/parser/RubyParser.java b/core/src/main/java/org/jruby/parser/RubyParser.java
diff --git a/core/src/main/java/org/jruby/parser/RubyParser.y b/core/src/main/java/org/jruby/parser/RubyParser.y
diff --git a/core/src/main/java/org/jruby/parser/Tokens.java b/core/src/main/java/org/jruby/parser/Tokens.java
diff --git a/core/src/main/java/org/jruby/parser/YyTables.java b/core/src/main/java/org/jruby/parser/YyTables.java
@@ -1,3 +1,31 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+ * Version: EPL 1.0/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Eclipse Public
+ * License Version 1.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ *
+ * Copyright (C) 2013-2017 The JRuby Team (jruby@jruby.org)
+ * 
+ * Alternatively, the contents of this file may be used under the terms of
+ * either of the GNU General Public License Version 2 or later (the "GPL"),
+ * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the EPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the EPL, the GPL or the LGPL.
+ ***** END LICENSE BLOCK *****/
 package org.jruby.parser;
 
 public class YyTables {

diff --git a/test/mri/excludes/TestSyntax.rb b/test/mri/excludes/TestSyntax.rb
@@ -12,6 +12,5 @@
 exclude :test_duplicated_when, "needs investigation"
 exclude :test_integer_suffix, "needs investigation"
 exclude :test_invalid_next, "needs investigation"
-exclude :test_parenthesised_statement_argument, "needs investigation"
 exclude :test_warn_unreachable, "needs investigation"
 exclude :test_warning_literal_in_condition, "needs investigation"