Support a stack of StringScanner in parser

opal · Oct 25, 2013 · 36c74a6 · 36c74a6
1 parent 04f480d
commit 36c74a6
Showing 1 changed file with 12 additions and 1 deletion.
diff --git a/lib/opal/lexer.rb b/lib/opal/lexer.rb
@@ -15,7 +15,9 @@ def initialize(source, file)
       @file       = file
 
       @string_parse_stack = []
+
       @scanner = StringScanner.new(source)
+      @scanner_stack = [@scanner]
     end
 
     def cond_push(n)
@@ -1235,7 +1237,16 @@ def next_token
           return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
 
         end
-        return [false, false] if scanner.eos?
+
+        if scanner.eos?
+          if @scanner_stack.size == 1 # our main scanner, we cant pop this
+            return [false, false]
+          else # we were probably parsing a heredoc, so pop that parser and continue
+            @scanner_stack.pop
+            @scanner = @scanner_stack.last
+            return next_token
+          end
+        end
 
         raise "Unexpected content in parsing stream `#{scanner.peek 5}` :#{@file}:#{@line}"
       end