Update for ripper to use closer to MRI's ripper .rb files.

Update ripper to keep track of dedents. Unmark passing ripper tests
jruby · Feb 25, 2016 · 5c712c5 · mkristian · May 25, 2016 · enebo
1 parent 080709d
commit 5c712c5
Showing 15 changed files with 4,839 additions and 4,314 deletions.
diff --git a/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java b/core/src/main/java/org/jruby/ext/ripper/HeredocTerm.java
@@ -139,6 +139,12 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
                             break;
                     }
                 }
+
+                if (lexer.getHeredocIndent() > 0) {
+                    for (long i = 0; p + i < pend && lexer.update_heredoc_indent(lexer.p(p)); i++) {}
+                    lexer.setHeredocLineIndent(0);
+                }
+
                 if (str != null) {
                     str.append(lbuf.makeShared(p, pend - p));
                 } else {
@@ -147,6 +153,11 @@ public int parseString(RipperLexer lexer, LexerSource src) throws java.io.IOExce
 
                 if (pend < lexer.lex_pend) str.append('\n');
                 lexer.lex_goto_eol();
+
+                if (lexer.getHeredocIndent() > 0) {
+                    lexer.setValue(str);
+                    return Tokens.tSTRING_CONTENT;
+                }
                 // MRI null checks str in this case but it is unconditionally non-null?
                 if (lexer.nextc() == -1) return error(lexer, len, null, eos);
             } while(!lexer.whole_match_p(eos, indent));

diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.java b/core/src/main/java/org/jruby/ext/ripper/RipperParser.java
diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParser.y b/core/src/main/java/org/jruby/ext/ripper/RipperParser.y
@@ -1520,10 +1520,14 @@ string          : tCHAR
                 }
 
 string1         : tSTRING_BEG string_contents tSTRING_END {
+                    p.heredoc_dedent($2);
+                    p.setHeredocIndent(0);
                     $$ = p.dispatch("on_string_literal", $2);
                 }
 
 xstring         : tXSTRING_BEG xstring_contents tSTRING_END {
+                    p.heredoc_dedent($2);
+                    p.setHeredocIndent(0);
                     $$ = p.dispatch("on_xstring_literal", $2);
                 }
 
@@ -1637,13 +1641,17 @@ string_content  : tSTRING_CONTENT
                 } {
                    $$ = p.getBraceNest();
                    p.setBraceNest(0);
+                } {
+                   $$ = p.getHeredocIndent();
+                   p.setHeredocIndent(0);
                 } compstmt tSTRING_DEND {
                    p.getConditionState().restart();
                    p.setStrTerm($<StrTerm>2);
                    p.getCmdArgumentState().reset($<Long>3.longValue());
                    p.setState($<Integer>4);
                    p.setBraceNest($<Integer>5);
-                   $$ = p.dispatch("on_string_embexpr", $6);
+                   p.setHeredocIndent($<Integer>6);
+                   $$ = p.dispatch("on_string_embexpr", $7);
                 }
 
 string_dvar     : tGVAR {

diff --git a/core/src/main/java/org/jruby/ext/ripper/RipperParserBase.java b/core/src/main/java/org/jruby/ext/ripper/RipperParserBase.java
@@ -268,6 +268,20 @@ public void pushBlockScope() {
     public void pushLocalScope() {
         currentScope = getRuntime().getStaticScopeFactory().newLocalScope(currentScope);
     }
+
+    public int getHeredocIndent() {
+        return lexer.getHeredocIndent();
+    }
+
+    public void setHeredocIndent(int indent) {
+        lexer.setHeredocIndent(indent);
+    }
+
+    public void heredoc_dedent(IRubyObject array) {
+        if (lexer.getHeredocIndent() <= 0) return;
+
+        dispatch("on_heredoc_dedent", array, getRuntime().newFixnum(lexer.getHeredocIndent()));
+    }
 
     public void setCommandStart(boolean value) {
         lexer.commandStart = value;

diff --git a/core/src/main/java/org/jruby/ext/ripper/RubyRipper.java b/core/src/main/java/org/jruby/ext/ripper/RubyRipper.java
@@ -173,6 +173,7 @@ private static IRubyObject createParserEventTable(Ruby runtime, RubyClass ripper
         hash.fastASet(runtime.newSymbol("field"), runtime.newFixnum(3));
         hash.fastASet(runtime.newSymbol("for"), runtime.newFixnum(3));
         hash.fastASet(runtime.newSymbol("hash"), runtime.newFixnum(1));
+        hash.fastASet(runtime.newSymbol("heredoc_dedent"), runtime.newFixnum(1));
         hash.fastASet(runtime.newSymbol("if"), runtime.newFixnum(3));
         hash.fastASet(runtime.newSymbol("if_mod"), runtime.newFixnum(2));
         hash.fastASet(runtime.newSymbol("ifop"), runtime.newFixnum(3));

diff --git a/core/src/main/java/org/jruby/ext/ripper/StringTerm.java b/core/src/main/java/org/jruby/ext/ripper/StringTerm.java
@@ -184,6 +184,10 @@ public int parseStringIntoBuffer(RipperLexer lexer, LexerSource src, ByteList bu
         int c;
 
         while ((c = lexer.nextc()) != EOF) {
+            if (lexer.getHeredocIndent() > 0) {
+                lexer.update_heredoc_indent(c);
+            }
+
             if (begin != '\0' && c == begin) {
                 nest++;
             } else if (c == end) {

diff --git a/core/src/main/java/org/jruby/ext/ripper/YyTables.java b/core/src/main/java/org/jruby/ext/ripper/YyTables.java
diff --git a/lib/ruby/stdlib/irb/#context.rb# b/lib/ruby/stdlib/irb/#context.rb#
diff --git a/lib/ruby/stdlib/joda-time/joda-time/2.3/joda-time-2.3.jar b/lib/ruby/stdlib/joda-time/joda-time/2.3/joda-time-2.3.jar
diff --git a/lib/ruby/stdlib/joda-time/joda-time/maven-metadata-local.xml b/lib/ruby/stdlib/joda-time/joda-time/maven-metadata-local.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>joda-time</groupId>
+  <artifactId>joda-time</artifactId>
+  <versioning>
+    <release>2.3</release>
+    <versions>
+      <version>2.3</version>
+    </versions>
+    <lastUpdated>20150120203521</lastUpdated>
+  </versioning>
+</metadata>
diff --git a/lib/ruby/stdlib/ripper/core.rb b/lib/ruby/stdlib/ripper/core.rb
@@ -1,3 +1,4 @@
+# frozen_string_literal: false
 #
 # $Id$
 #

diff --git a/lib/ruby/stdlib/ripper/filter.rb b/lib/ruby/stdlib/ripper/filter.rb
@@ -1,3 +1,4 @@
+# frozen_string_literal: false
 #
 # $Id$
 #

diff --git a/lib/ruby/stdlib/ripper/lexer.rb b/lib/ruby/stdlib/ripper/lexer.rb
@@ -1,3 +1,4 @@
+# frozen_string_literal: false
 #
 # $Id$
 #
@@ -44,26 +45,54 @@ def Ripper.lex(src, filename = '-', lineno = 1)
   end
 
   class Lexer < ::Ripper   #:nodoc: internal use only
+    Elem = Struct.new(:pos, :event, :tok)
+
     def tokenize
-      lex().map {|pos, event, tok| tok }
+      parse().sort_by(&:pos).map(&:tok)
     end
 
     def lex
-      parse().sort_by {|pos, event, tok| pos }
+      parse().sort_by(&:pos).map(&:to_a)
     end
 
     def parse
       @buf = []
+      @stack = []
       super
+      @buf.flatten!
       @buf
     end
 
     private
 
-    SCANNER_EVENTS.each do |event|
+    def on_heredoc_dedent(v, w)
+      @buf.last.each do |e|
+        if e.event == :on_tstring_content
+          if (n = dedent_string(e.tok, w)) > 0
+            e.pos[1] += n
+          end
+        end
+      end
+      v
+    end
+
+    def on_heredoc_beg(tok)
+      @stack.push @buf
+      buf = []
+      @buf << buf
+      @buf = buf
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+    end
+
+    def on_heredoc_end(tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf = @stack.pop
+    end
+
+    (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
       module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
-        def on_#{event}(tok)
-          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        def #{event}(tok)
+          @buf.push Elem.new([lineno(), column()], :#{event}, tok)
         end
       End
     end

diff --git a/lib/ruby/stdlib/ripper/sexp.rb b/lib/ruby/stdlib/ripper/sexp.rb
@@ -1,3 +1,4 @@
+# frozen_string_literal: false
 #
 # $Id$
 #
@@ -59,45 +60,38 @@ def Ripper.sexp_raw(src, filename = '-', lineno = 1)
     sexp unless builder.error?
   end
 
-  class SexpBuilderPP < ::Ripper   #:nodoc:
+  class SexpBuilder < ::Ripper   #:nodoc:
     private
 
-    PARSER_EVENT_TABLE.each do |event, arity|
-      if /_new\z/ =~ event.to_s and arity == 0
-        module_eval(<<-End, __FILE__, __LINE__ + 1)
-          def on_#{event}
-            []
-          end
-        End
-      elsif /_add\z/ =~ event.to_s
-        module_eval(<<-End, __FILE__, __LINE__ + 1)
-          def on_#{event}(list, item)
-            list.push item
-            list
-          end
-        End
-      else
-        module_eval(<<-End, __FILE__, __LINE__ + 1)
-          def on_#{event}(*args)
-            [:#{event}, *args]
-          end
-        End
+    def dedent_element(e, width)
+      if (n = dedent_string(e[1], width)) > 0
+        e[2][1] += n
       end
+      e
     end
 
-    SCANNER_EVENTS.each do |event|
-      module_eval(<<-End, __FILE__, __LINE__ + 1)
-        def on_#{event}(tok)
-          [:@#{event}, tok, [lineno(), column()]]
+    def on_heredoc_dedent(val, width)
+      sub = proc do |cont|
+        cont.map! do |e|
+          if Array === e
+            case e[0]
+            when :@tstring_content
+              e = dedent_element(e, width)
+            when /_add\z/
+              e[1] = sub[e[1]]
+            end
+          elsif String === e
+            dedent_string(e, width)
+          end
+          e
         end
-      End
+      end
+      sub[val]
+      val
     end
-  end
-
-  class SexpBuilder < ::Ripper   #:nodoc:
-    private
 
-    PARSER_EVENTS.each do |event|
+    events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym}
+    (PARSER_EVENTS - events).each do |event|
       module_eval(<<-End, __FILE__, __LINE__ + 1)
         def on_#{event}(*args)
           args.unshift :#{event}
@@ -115,4 +109,39 @@ def on_#{event}(tok)
     end
   end
 
+  class SexpBuilderPP < SexpBuilder #:nodoc:
+    private
+
+    def on_heredoc_dedent(val, width)
+      val.map! do |e|
+        next e if Symbol === e and /_content\z/ =~ e
+        if Array === e and e[0] == :@tstring_content
+          e = dedent_element(e, width)
+        elsif String === e
+          dedent_string(e, width)
+        end
+        e
+      end
+      val
+    end
+
+    PARSER_EVENT_TABLE.each do |event, arity|
+      if /_new\z/ =~ event.to_s and arity == 0
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}
+            []
+          end
+        End
+      elsif /_add\z/ =~ event.to_s
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(list, item)
+            list.push item
+            list
+          end
+        End
+      end
+    end
+  end
 end
+
+
diff --git a/test/mri/excludes/TestRipper/ParserEvents.rb b/test/mri/excludes/TestRipper/ParserEvents.rb
@@ -1,8 +1,5 @@
 exclude :test_block_variables, "missing rlimit used in this test; related to #2776"
-exclude :test_dyna_symbol, "needs investigation"
 exclude :test_event_coverage, "needs investigation"
-exclude :test_heredoc_dedent, "needs investigation"
 exclude :test_local_variables, "needs investigation"
 exclude :test_magic_comment, "needs investigation"
 exclude :test_operator_ambiguous, "needs investigation"
-exclude :test_var_ref, "needs investigation"