Update ripper parser to line up with mainline parser. Fix lineno to p…

…ass all but 8 specs on Yard.
jruby · Jun 8, 2015 · 86dfb5d · 86dfb5d
1 parent 828fad3
commit 86dfb5d
Showing 7 changed files with 5,234 additions and 4,971 deletions.
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperLexer.java b/core/src/main/java/org/jruby/ext/ripper/RipperLexer.java
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.java b/core/src/main/java/org/jruby/ext/ripper/RipperParser.java
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.y b/core/src/main/java/org/jruby/ext/ripper/RipperParser.y
@@ -887,8 +887,12 @@ arg             : lhs '=' arg {
                 | kDEFINED opt_nl arg {
                     $$ = p.dispatch("on_defined", $3);
                 }
-                | arg '?' arg opt_nl ':' arg {
-                    $$ = p.dispatch("on_ifop", $1, $3, $6);
+                | arg '?' {
+                    p.getConditionState().begin();
+                } arg opt_nl ':' {
+                    p.getConditionState().end();
+                } arg {
+                    $$ = p.dispatch("on_ifop", $1, $4, $8);
                 }
                 | primary {
                     $$ = $1;
@@ -1024,19 +1028,27 @@ primary         : literal
                 | tFID {
                     $$ = p.dispatch("on_method_add_arg", p.dispatch("on_fcall", $1), p.dispatch("on_args_new"));
                 }
-                | kBEGIN bodystmt kEND {
-                    $$ = p.dispatch("on_begin", $2);
+                | kBEGIN {
+                    $$ = p.getCmdArgumentState().getStack();
+                    p.getCmdArgumentState().reset();
+                } bodystmt kEND {
+                    p.getCmdArgumentState().reset($<Long>2.longValue());
+                    $$ = p.dispatch("on_begin", $3);
                 }
                 | tLPAREN_ARG {
                     p.setState(LexState.EXPR_ENDARG);
                 } rparen {
                     $$ = p.dispatch("on_paren", null);
                 }
-                | tLPAREN_ARG expr {
+                | tLPAREN_ARG {
+                    $$ = p.getCmdArgumentState().getStack();
+                    p.getCmdArgumentState().reset();
+                } expr {
                     p.setState(LexState.EXPR_ENDARG); 
                 } rparen {
+                    p.getCmdArgumentState().reset($<Long>2.longValue());
                     p.warning("(...) interpreted as grouped expression");
-                    $$ = p.dispatch("on_paren", $2);
+                    $$ = p.dispatch("on_paren", $3);
                 }
                 | tLPAREN compstmt tRPAREN {
                     $$ = p.dispatch("on_paren", $2);
@@ -1611,15 +1623,24 @@ string_content  : tSTRING_CONTENT
                 }
                 | tSTRING_DBEG {
                    $$ = p.getStrTerm();
-                   p.getConditionState().stop();
-                   p.getCmdArgumentState().stop();
                    p.setStrTerm(null);
+                   p.getConditionState().stop();
+                } {
+                   $$ = p.getCmdArgumentState().getStack();
+                   p.getCmdArgumentState().reset();
+                } {
+                   $$ = p.getState();
                    p.setState(LexState.EXPR_BEG);
-                } compstmt tRCURLY {
+                } {
+                   $$ = p.getBraceNest();
+                   p.setBraceNest(0);
+                } compstmt tSTRING_DEND {
                    p.getConditionState().restart();
-                   p.getCmdArgumentState().restart();
                    p.setStrTerm($<StrTerm>2);
-                   $$ = p.dispatch("on_string_embexpr", $3);
+                   p.getCmdArgumentState().reset($<Long>3.longValue());
+                   p.setState($<LexState>4);
+                   p.setBraceNest($<Integer>5);
+                   $$ = p.dispatch("on_string_embexpr", $6);
                 }
 
 string_dvar     : tGVAR {

diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParserBase.java b/core/src/main/java/org/jruby/ext/ripper/RipperParserBase.java
@@ -351,6 +351,18 @@ public int getInSingle() {
         return inSingleton;
     }
 
+    public int getBraceNest() {
+        return lexer.getBraceNest();
+    }
+
+    public LexState getState() {
+        return lexer.getState();
+    }
+
+    public void setBraceNest(int braceNest) {
+        lexer.setBraceNest(braceNest);
+    }
+
     public void setState(LexState lexState) {
         lexer.setState(lexState);
     }

diff --git a/core/src/main/java/org/jruby/ext/ripper/YyTables.java b/core/src/main/java/org/jruby/ext/ripper/YyTables.java
diff --git a/test/mri/excludes/TestRipper/Filter.rb b/test/mri/excludes/TestRipper/Filter.rb
@@ -1,6 +1,3 @@
-exclude :test_filter_column, "needs investigation"
-exclude :test_filter_filename, "needs investigation"
-exclude :test_filter_filename_unset, "needs investigation"
 exclude :test_filter_lineno, "needs investigation"
 exclude :test_filter_lineno_set, "needs investigation"
-exclude :test_filter_token, "needs investigation"
+
diff --git a/test/mri/excludes/TestRipper/ScannerEvents.rb b/test/mri/excludes/TestRipper/ScannerEvents.rb
@@ -1,4 +1,3 @@
-exclude :test_embexpr_end, "needs investigation"
-exclude :test_event_coverage, "embexpr_end due to parser/lexer bug"
-exclude :test_tokenize, "needs investigation"
+exclude :test_lex, "off-by-one one-liner no newline"
+exclude :test_location, "off-by-one one-liner no newline"