All hail the world of getting better expected token strings in syntax…

… errors
jruby · Apr 25, 2018 · f6eda64 · f6eda64
1 parent 9b1e93b
commit f6eda64
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 24 deletions.
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.java b/core/src/main/java/org/jruby/ext/ripper/RipperParser.java
@@ -49,7 +49,7 @@ public class RipperParser extends RipperParserBase {
     public RipperParser(ThreadContext context, IRubyObject ripper, LexerSource source) {
         super(context, ripper, source);
     }
-					// line 52 "-"
+					// line 53 "-"
   // %token constants
   public static final int keyword_class = 257;
   public static final int keyword_module = 258;
@@ -759,20 +759,20 @@ public RipperParser(ThreadContext context, IRubyObject ripper, LexerSource sourc
     "modifier_rescue","keyword_alias","keyword_defined","keyword_BEGIN",
     "keyword_END","keyword__LINE__","keyword__FILE__",
     "keyword__ENCODING__","keyword_do_lambda","tIDENTIFIER","tFID",
-    "tGVAR","tIVAR","tCONSTANT","tCVAR","tLABEL","tCHAR","tUPLUS",
-    "tUMINUS","tUMINUS_NUM","tPOW","tCMP","tEQ","tEQQ","tNEQ","tGEQ",
-    "tLEQ","tANDOP","tOROP","tMATCH","tNMATCH","tDOT","tDOT2","tDOT3",
-    "tAREF","tASET","tLSHFT","tRSHFT","tANDDOT","tCOLON2","tCOLON3",
-    "tOP_ASGN","tASSOC","tLPAREN","tLPAREN2","tRPAREN","tLPAREN_ARG",
-    "tLBRACK","tRBRACK","tLBRACE","tLBRACE_ARG","tSTAR","tSTAR2","tAMPER",
-    "tAMPER2","tTILDE","tPERCENT","tDIVIDE","tPLUS","tMINUS","tLT","tGT",
-    "tPIPE","tBANG","tCARET","tLCURLY","tRCURLY","tBACK_REF2","tSYMBEG",
+    "tGVAR","tIVAR","tCONSTANT","tCVAR","tLABEL","tCHAR","unary+",
+"unary-","tUMINUS_NUM","'**'","'<=>'","'=='","'==='","'!='","'>='",
+"'<='","'&&'","'||'","'=~'","'!~'","'.'","'..'","'...'",
+"'[]'","'[]='","'<<'","'>>'","'&.'","'::'","':: at EXPR_BEG'",
+    "tOP_ASGN","'=>'","'('","'( arg'","')'","'['",
+"'{'","'{ arg'","'['","'[ args'","'*'","'*'","'&'",
+"'&'","'~'","'%'","'/'","'+'","'-'","'<'","'>'",
+"'|'","'!'","'^'","'{'","'}'","'`'","':'",
     "tSTRING_BEG","tXSTRING_BEG","tREGEXP_BEG","tWORDS_BEG","tQWORDS_BEG",
-    "tSTRING_DBEG","tSTRING_DVAR","tSTRING_END","tLAMBDA","tLAMBEG",
+    "tSTRING_DBEG","tSTRING_DVAR","tSTRING_END","'->'","tLAMBEG",
     "tNTH_REF","tBACK_REF","tSTRING_CONTENT","tINTEGER","tIMAGINARY",
     "tFLOAT","tRATIONAL","tREGEXP_END","tIGNORED_NL","tCOMMENT",
     "tEMBDOC_BEG","tEMBDOC","tEMBDOC_END","tSP","tHEREDOC_BEG",
-    "tHEREDOC_END","tSYMBOLS_BEG","tQSYMBOLS_BEG","tDSTAR","tSTRING_DEND",
+    "tHEREDOC_END","tSYMBOLS_BEG","tQSYMBOLS_BEG","'**'","tSTRING_DEND",
     "tLABEL_END","tLOWEST","k__END__",
     };
 
@@ -4846,6 +4846,6 @@ public Object yyparse (RipperLexer yyLex) throws java.io.IOException {
   }
 };
 }
-					// line 2186 "RipperParser.y"
+					// line 2187 "RipperParser.y"
 }
-					// line 9736 "-"
+					// line 9737 "-"
diff --git a/core/src/main/java/org/jruby/ext/ripper/YyTables.java b/core/src/main/java/org/jruby/ext/ripper/YyTables.java
@@ -26,7 +26,6 @@
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the EPL, the GPL or the LGPL.
  ***** END LICENSE BLOCK *****/
-
 package org.jruby.ext.ripper;
 
 public class YyTables {

diff --git a/core/src/main/java/org/jruby/parser/RubyParser.java b/core/src/main/java/org/jruby/parser/RubyParser.java
@@ -863,19 +863,19 @@ public void setWarnings(IRubyWarnings warnings) {
     "modifier_rescue","keyword_alias","keyword_defined","keyword_BEGIN",
     "keyword_END","keyword__LINE__","keyword__FILE__",
     "keyword__ENCODING__","keyword_do_lambda","tIDENTIFIER","tFID",
-    "tGVAR","tIVAR","tCONSTANT","tCVAR","tLABEL","tCHAR","tUPLUS",
-    "tUMINUS","tUMINUS_NUM","tPOW","tCMP","tEQ","tEQQ","tNEQ","tGEQ",
-    "tLEQ","tANDOP","tOROP","tMATCH","tNMATCH","tDOT","tDOT2","tDOT3",
-    "tAREF","tASET","tLSHFT","tRSHFT","tANDDOT","tCOLON2","tCOLON3",
-    "tOP_ASGN","tASSOC","tLPAREN","tLPAREN2","tRPAREN","tLPAREN_ARG",
-    "tLBRACK","tRBRACK","tLBRACE","tLBRACE_ARG","tSTAR","tSTAR2","tAMPER",
-    "tAMPER2","tTILDE","tPERCENT","tDIVIDE","tPLUS","tMINUS","tLT","tGT",
-    "tPIPE","tBANG","tCARET","tLCURLY","tRCURLY","tBACK_REF2","tSYMBEG",
+    "tGVAR","tIVAR","tCONSTANT","tCVAR","tLABEL","tCHAR","unary+",
+"unary-","tUMINUS_NUM","'**'","'<=>'","'=='","'==='","'!='","'>='",
+"'<='","'&&'","'||'","'=~'","'!~'","'.'","'..'","'...'",
+"'[]'","'[]='","'<<'","'>>'","'&.'","'::'","':: at EXPR_BEG'",
+    "tOP_ASGN","'=>'","'('","'( arg'","')'","'['",
+"'{'","'{ arg'","'['","'[ args'","'*'","'*'","'&'",
+"'&'","'~'","'%'","'/'","'+'","'-'","'<'","'>'",
+"'|'","'!'","'^'","'{'","'}'","'`'","':'",
     "tSTRING_BEG","tXSTRING_BEG","tREGEXP_BEG","tWORDS_BEG","tQWORDS_BEG",
-    "tSTRING_DBEG","tSTRING_DVAR","tSTRING_END","tLAMBDA","tLAMBEG",
+    "tSTRING_DBEG","tSTRING_DVAR","tSTRING_END","'->'","tLAMBEG",
     "tNTH_REF","tBACK_REF","tSTRING_CONTENT","tINTEGER","tIMAGINARY",
     "tFLOAT","tRATIONAL","tREGEXP_END","tSYMBOLS_BEG","tQSYMBOLS_BEG",
-    "tDSTAR","tSTRING_DEND","tLABEL_END","tLOWEST",
+"'**'","tSTRING_DEND","tLABEL_END","tLOWEST",
     };
 
   /** printable rules for debugging.

diff --git a/tool/patch_parser.rb b/tool/patch_parser.rb
@@ -1,3 +1,58 @@
+REMAPS = {
+  '"tUPLUS"' => '"unary+"',
+  '"tUMINUS"'=>  '"unary-"',
+  '"tPOW"' => '"\'**\'"',
+  '"tCMP"' => '"\'<=>\'"',
+  '"tEQ"' => '"\'==\'"',
+  '"tEQQ"' => '"\'===\'"',
+  '"tNEQ"' => '"\'!=\'"',
+  '"tGEQ"' => '"\'>=\'"',
+  '"tLEQ"' => '"\'<=\'"',
+  '"tANDOP"' => '"\'&&\'"',
+  '"tOROP"' => '"\'||\'"',
+  '"tMATCH"' => '"\'=~\'"',
+  '"tNMATCH"'=> '"\'!~\'"',
+  '"tDOT"' => '"\'.\'"',
+  '"tDOT2"' => '"\'..\'"',
+  '"tDOT3"' => '"\'...\'"',
+  '"tAREF"' => '"\'[]\'"',
+  '"tASET"' => '"\'[]=\'"',
+  '"tLSHFT"' => '"\'<<\'"',
+  '"tRSHFT"' => '"\'>>\'"',
+  '"tANDDOT"' => '"\'&.\'"',
+  '"tCOLON2"' => '"\'::\'"',
+  '"tCOLON3"' => '"\':: at EXPR_BEG\'"',
+  '"tASSOC"' => '"\'=>\'"',
+  '"tLPAREN"' => '"\'(\'"',
+  '"tLPAREN2"'=> '"\'( arg\'"',
+  '"tRPAREN"' => '"\')\'"',
+  '"tLPAREN_ARG"' => '"\'[\'"',
+  '"tLBRACK"' => '"\'{\'"',
+  '"tRBRACK"' => '"\'{ arg\'"',
+  '"tLBRACE"' => '"\'[\'"',
+  '"tLBRACE_ARG"' => '"\'[ args\'"', 
+  '"tSTAR"' => '"\'*\'"',
+  '"tSTAR2"' => '"\'*\'"',
+  '"tAMPER"' => '"\'&\'"',
+  '"tAMPER2"' => '"\'&\'"',
+  '"tTILDE"' => '"\'~\'"',
+  '"tPERCENT"' => '"\'%\'"',
+  '"tDIVIDE"' => '"\'/\'"',
+  '"tPLUS"' => '"\'+\'"',
+  '"tMINUS"' => '"\'-\'"',
+  '"tLT"' => '"\'<\'"',
+  '"tGT"' => '"\'>\'"',
+  '"tPIPE"' => '"\'|\'"',
+  '"tBANG"' => '"\'!\'"',
+  '"tCARET"' => '"\'^\'"',
+  '"tLCURLY"' => '"\'{\'"',
+  '"tRCURLY"' => '"\'}\'"',
+  '"tBACK_REF2"' => '"\'`\'"',
+  '"tSYMBEG"' => '"\':\'"',
+  '"tLAMBDA"'  => '"\'->\'"',
+  '"tDSTAR"' => '"\'**\'"',
+}
+
 def get_numbers_until_end_block(table)
   while line = gets
     break if /\};/ =~ line
@@ -10,6 +65,20 @@ def get_numbers_until_end_block(table)
   table
 end
 
+def print_names_until_end_block
+  while line = gets
+    if /\};/ =~ line
+      print line
+      break
+    end
+    next if /^\/\// =~ line
+    new_line = line.split(/,/).map do |element|
+      REMAPS[element.strip] || element
+    end.join(',')
+    puts new_line
+  end
+end
+
 # We use this script to generate our normal parser and the parser for 
 # the ripper extension.
 if ARGV[0] =~ /Ripper/
@@ -39,6 +108,13 @@ def get_numbers_until_end_block(table)
 
 puts "    protected static final short[] yyCheck = #{yytable_prefix}YyTables.yyCheck();"
 
+while gets
+  print $_
+  break if /protected static final String\[\] yyNames = \{/ =~ $_
+end
+
+print_names_until_end_block
+
 while gets
   print $_
 end