Fix lots of heredoc parsing bugs (needs cleanup)

opal · Oct 31, 2013 · b68dc4e · b68dc4e
1 parent 0a9aad2
commit b68dc4e
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 12 deletions.
diff --git a/lib/opal/parser/lexer.rb b/lib/opal/parser/lexer.rb
@@ -99,6 +99,23 @@ def next_string_token
       # if not end of string, so we must be parsing contents
       str_buffer = []
 
+      if str_parse[:type] == :heredoc
+        eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
+
+        if scanner.check(eos_regx)
+          scanner.scan(/[ \t]*#{Regexp.escape(str_parse[:end])}/)
+          self.strterm = nil
+
+          if str_parse[:scanner]
+            @scanner_stack << str_parse[:scanner]
+            @scanner = str_parse[:scanner]
+          end
+
+          @lex_state = :expr_end
+          return :STRING_END, scanner.matched
+        end
+      end
+
       # see if we can read end of string/xstring/regecp markers
       # if scanner.scan /#{str_parse[:end]}/
       if scanner.scan Regexp.new(Regexp.escape(str_parse[:end]))
@@ -174,12 +191,71 @@ def next_string_token
         str_buffer << '#'
       end
 
-      add_string_content str_buffer, str_parse
+      if str_parse[:type] == :heredoc
+        add_heredoc_content str_buffer, str_parse
+      else
+        add_string_content str_buffer, str_parse
+      end
+
       complete_str = str_buffer.join ''
       @line += complete_str.count("\n")
       return :STRING_CONTENT, complete_str
     end
 
+    def add_heredoc_content(str_buffer, str_parse)
+      scanner = @scanner
+
+      eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
+      expand = true
+
+      until scanner.eos?
+        c = nil
+        handled = true
+
+        if scanner.scan(/\n/)
+          c = scanner.matched
+        elsif scanner.check(eos_regx) && scanner.bol?
+          break # eos!
+        elsif expand && scanner.check(/#(?=[\$\@\{])/)
+          break
+        elsif scanner.scan(/\\/)
+          if str_parse[:regexp]
+            if scanner.scan(/(.)/)
+              c = "\\" + scanner.matched
+            end
+          else
+            c = if scanner.scan(/n/)
+              "\n"
+            elsif scanner.scan(/r/)
+              "\r"
+            elsif scanner.scan(/\n/)
+              "\n"
+            elsif scanner.scan(/t/)
+              "\t"
+            else
+              # escaped char doesnt need escaping, so just return it
+              scanner.scan(/./)
+              scanner.matched
+            end
+          end
+        else
+          handled = false
+        end
+
+        unless handled
+          reg = Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\\n]+|.")
+
+          scanner.scan reg
+          c = scanner.matched
+        end
+
+        c ||= scanner.matched
+        str_buffer << c
+      end
+
+      raise "reached EOF while in string" if scanner.eos?
+    end
+
     def add_string_content(str_buffer, str_parse)
       scanner = @scanner
       # regexp for end of string/regexp

diff --git a/spec/opal/parser/heredoc_spec.rb b/spec/opal/parser/heredoc_spec.rb
@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe "Heredocs" do
+
+  it "parses as a s(:str)" do
+    opal_parse("a = <<-FOO\nbar\nFOO")[2].should == [:str, "bar\n"]
+  end
+
+  it "allows start marker to be wrapped in quotes" do
+    opal_parse("a = <<-'FOO'\nbar\nFOO")[2].should == [:str, "bar\n"]
+    opal_parse("a = <<-\"FOO\"\nbar\nFOO")[2].should == [:str, "bar\n"]
+  end
+
+  it "does not parse EOS unless beginning of line" do
+    opal_parse("<<-FOO\ncontentFOO\nFOO").should == [:str, "contentFOO\n"]
+  end
+
+  it "does not parse EOS unless end of line" do
+    opal_parse("<<-FOO\nsome FOO content\nFOO").should == [:str, "some FOO content\n"]
+  end
+
+  it "parses postfix code as if it appeared after heredoc" do
+    opal_parse("<<-FOO.class\ncode\nFOO").should == [:call, [:str, "code\n"], :class, [:arglist]]
+    opal_parse("bar(<<-FOO, 1, 2, 3)\ncode\nFOO").should == [:call, nil, :bar,
+                                                              [:arglist, [:str, "code\n"],
+                                                                         [:int, 1],
+                                                                         [:int, 2],
+                                                                         [:int, 3]]]
+  end
+end
diff --git a/spec/opal/parser/str_spec.rb b/spec/opal/parser/str_spec.rb
@@ -104,15 +104,4 @@
       opal_parse("foo ?a").should == [:call, nil, :foo, [:arglist, [:str, "a"]]]
     end
   end
-
-  describe "parsing heredocs" do
-    it "parses as a s(:str)" do
-      opal_parse("a = <<-FOO\nbar\nFOO")[2].should == [:str, "bar\n"]
-    end
-
-    it "allows start marker to be wrapped in quotes" do
-      opal_parse("a = <<-'FOO'\nbar\nFOO")[2].should == [:str, "bar\n"]
-      opal_parse("a = <<-\"FOO\"\nbar\nFOO")[2].should == [:str, "bar\n"]
-    end
-  end
 end