lol. Accidentally using main line parser token values for ripper toke…

…n values. In theory they should be ordered the same but apparently are a little off towards the end (no doubt to ripper specific tokens mainline does not have).
jruby · Mar 25, 2018 · 4932393 · 4932393
1 parent 957d0f2
commit 4932393
Showing 2 changed files with 15 additions and 17 deletions.
diff --git a/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java b/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java
@@ -31,7 +31,6 @@
 
 import org.jcodings.Encoding;
 import org.jruby.lexer.LexerSource;
-import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 
 import static org.jruby.lexer.LexingCommon.*;
@@ -80,7 +79,7 @@ protected int error(RipperLexer lexer, int len, ByteList str, ByteList eos) {
         lexer.compile_error("can't find string \"" + eos.toString() + "\" anywhere before EOF");
 
         if (lexer.delayed == null) {
-            lexer.dispatchScanEvent(RubyParser.tSTRING_CONTENT);
+            lexer.dispatchScanEvent(RipperParser.tSTRING_CONTENT);
         } else {
             if (str != null) {
                 lexer.delayed.append(str);
@@ -90,7 +89,7 @@ protected int error(RipperLexer lexer, int len, ByteList str, ByteList eos) {
                     lexer.delayed.append(lexer.lexb.makeShared(lexer.tokp, len));
                 }
             }
-            lexer.dispatchDelayedToken(RubyParser.tSTRING_CONTENT);
+            lexer.dispatchDelayedToken(RipperParser.tSTRING_CONTENT);
         }
         lexer.lex_goto_eol();
 
@@ -101,7 +100,7 @@ protected int restore(RipperLexer lexer) {
         lexer.heredoc_restore(this);
         lexer.setStrTerm(new StringTerm(flags | STR_FUNC_TERM, 0, 0)); // Weird way of ending
 
-        return RubyParser.tSTRING_CONTENT;
+        return RipperParser.tSTRING_CONTENT;
     }
 
     @Override
@@ -120,7 +119,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
             lexer.heredoc_restore(this);
             lexer.setStrTerm(null);
             lexer.setState(EXPR_END);
-            return RubyParser.tSTRING_END;
+            return RipperParser.tSTRING_END;
         }
 
         if ((flags & STR_FUNC_EXPAND) == 0) {
@@ -159,7 +158,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
 
                 if (lexer.getHeredocIndent() > 0) {
                     lexer.setValue(lexer.createStr(str, 0));
-                    return RubyParser.tSTRING_CONTENT;
+                    return RipperParser.tSTRING_CONTENT;
                 }
                 // MRI null checks str in this case but it is unconditionally non-null?
                 if (lexer.nextc() == -1) return error(lexer, len, null, eos);
@@ -192,15 +191,15 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
                 if (c != '\n') {
                     lexer.setValue(lexer.createStr(tok, 0));
                     lexer.flush_string_content(enc[0]);
-                    return RubyParser.tSTRING_CONTENT;
+                    return RipperParser.tSTRING_CONTENT;
                 }
                 tok.append(lexer.nextc());
 
                 if (lexer.getHeredocIndent() > 0) {
                     lexer.lex_goto_eol();
                     lexer.setValue(lexer.createStr(tok, 0));
                     lexer.flush_string_content(enc[0]);
-                    return RubyParser.tSTRING_CONTENT;
+                    return RipperParser.tSTRING_CONTENT;
                 }
 
                 if ((c = lexer.nextc()) == EOF) return error(lexer, len, str, eos);
@@ -211,6 +210,6 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
         lexer.pushback(c);
         lexer.setValue(lexer.createStr(str, 0));
         lexer.flush_string_content(lexer.getEncoding());
-        return RubyParser.tSTRING_CONTENT;
+        return RipperParser.tSTRING_CONTENT;
     }
 }
diff --git a/core/src/main/java/org/jruby/ext/ripper/StringTerm.java b/core/src/main/java/org/jruby/ext/ripper/StringTerm.java
@@ -33,7 +33,6 @@
 import org.jcodings.Encoding;
 import org.jruby.Ruby;
 import org.jruby.lexer.LexerSource;
-import org.jruby.parser.RubyParser;
 import org.jruby.util.ByteList;
 import org.jruby.util.RegexpOptions;
 
@@ -84,19 +83,19 @@ private int endFound(RipperLexer lexer) throws IOException {
 
         if ((flags & STR_FUNC_REGEXP) != 0) {
             validateRegexp(lexer);
-            lexer.dispatchScanEvent(RubyParser.tREGEXP_END);
+            lexer.dispatchScanEvent(RipperParser.tREGEXP_END);
             lexer.setState(EXPR_END | EXPR_ENDARG);
-            return RubyParser.tREGEXP_END;
+            return RipperParser.tREGEXP_END;
         }
 
         if ((flags & STR_FUNC_LABEL) != 0 && lexer.isLabelSuffix()) {
             lexer.nextc();
             lexer.setState(EXPR_BEG | EXPR_LABEL);
-            return RubyParser.tLABEL_END;
+            return RipperParser.tLABEL_END;
         }
 
         lexer.setState(EXPR_END | EXPR_ENDARG);
-        return RubyParser.tSTRING_END;
+        return RipperParser.tSTRING_END;
     }
 
     private void validateRegexp(RipperLexer lexer) throws IOException {
@@ -121,7 +120,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
             if ((flags & STR_FUNC_QWORDS) != 0) lexer.nextc(); // delayed terminator char
             lexer.ignoreNextScanEvent = true;
             lexer.setStrTerm(null);
-            return ((flags & STR_FUNC_REGEXP) != 0) ? RubyParser.tREGEXP_END : RubyParser.tSTRING_END;
+            return ((flags & STR_FUNC_REGEXP) != 0) ? RipperParser.tREGEXP_END : RipperParser.tSTRING_END;
         }
 
         ByteList buffer = createByteList(lexer);        
@@ -150,7 +149,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
         }        
 
         if ((flags & STR_FUNC_EXPAND) != 0 && c == '#') {
-            int token = lexer.peekVariableName(RubyParser.tSTRING_DVAR, RubyParser.tSTRING_DBEG);
+            int token = lexer.peekVariableName(RipperParser.tSTRING_DVAR, RipperParser.tSTRING_DBEG);
 
             if (token != 0) {
                 if ((flags & STR_FUNC_REGEXP) != 0) {
@@ -180,7 +179,7 @@ public int parseString(RipperLexer lexer, LexerSource src) throws IOException {
             regexpFragments.add(buffer);
         }
         lexer.flush_string_content(enc[0]);
-        return RubyParser.tSTRING_CONTENT;
+        return RipperParser.tSTRING_CONTENT;
     }
 
     private void mixedEscape(RipperLexer lexer, Encoding foundEncoding, Encoding parserEncoding) {