Correct implementation of heredoc (#5578)

asterite · RX14 · commit 8eb85541c161 · 2018-01-17T18:58:57.000Z
Now you can specify multiple heredocs in a single line, just like in Ruby.
diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr
@@ -881,6 +881,9 @@ describe Crystal::Formatter do
   assert_format "<<-HTML\n  hello \n  world   \n  HTML"
   assert_format "  <<-HTML\n    hello \n    world   \n    HTML", "<<-HTML\n  hello \n  world   \n  HTML"
 
+  assert_format "x, y = <<-FOO, <<-BAR\n  hello\n  FOO\n  world\n  BAR"
+  assert_format "x, y, z = <<-FOO, <<-BAR, <<-BAZ\n  hello\n  FOO\n  world\n  BAR\n  qux\nBAZ"
+
   assert_format "#!shebang\n1 + 2"
 
   assert_format "   {{\n1 + 2 }}", "{{\n  1 + 2\n}}"
@@ -1029,9 +1032,6 @@ describe Crystal::Formatter do
 
   assert_format "lib Foo\n  {% if 1 %}\n    2\n  {% end %}\nend\n\nmacro bar\n  1\nend"
 
-  assert_format %(puts(<<-FOO\n1\nFOO, 2))
-  assert_format %(puts <<-FOO\n1\nFOO, 2)
-
   assert_format "x : Int32 |\nString", "x : Int32 |\n    String"
 
   assert_format %(foo("bar" \\\n"baz")), %(foo("bar" \\\n    "baz"))
diff --git a/spec/compiler/lexer/lexer_string_spec.cr b/spec/compiler/lexer/lexer_string_spec.cr
@@ -161,6 +161,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("Hello, mom! I am HERE.")
     tester.next_string_token_should_be("\nHER dress is beautiful.")
     tester.next_string_token_should_be("\nHE is OK.")
@@ -173,6 +174,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("foo")
     tester.next_string_token_should_be("\n")
     tester.string_should_end_correctly
@@ -183,6 +185,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("foo")
     tester.next_string_token_should_be("\r\n")
     tester.string_should_end_correctly
@@ -193,6 +196,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("foo")
     tester.string_should_end_correctly
   end
@@ -203,6 +207,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("Hello, mom! I am HERE.")
     tester.token_should_be_at(line: 2)
     tester.next_string_token_should_be("\nHER dress is beautiful.")
@@ -221,6 +226,7 @@ describe "Lexer string" do
     tester = LexerObjects::Strings.new(lexer)
 
     tester.string_should_start_correctly
+    tester.next_token_should_be(:NEWLINE)
     tester.next_string_token_should_be("abc")
     tester.string_should_have_an_interpolation_of("foo")
     tester.string_should_end_correctly
@@ -239,46 +245,14 @@ describe "Lexer string" do
     end
   end
 
-  it "raises on invalid heredoc identifier (<<-HERE A)" do
-    lexer = Lexer.new("<<-HERE A\ntest\nHERE\n")
-
-    expect_raises Crystal::SyntaxException, /invalid character '.+' for heredoc identifier/ do
-      lexer.next_token
-    end
-  end
-
-  it "raises on invalid heredoc identifier (<<-HERE\\n)" do
-    lexer = Lexer.new("<<-HERE\\ntest\nHERE\n")
-
-    expect_raises Crystal::SyntaxException, /invalid character '.+' for heredoc identifier/ do
-      lexer.next_token
-    end
-  end
-
-  it "raises when identifier doesn't start with a leter" do
-    lexer = Lexer.new("<<-123\\ntest\n123\n")
+  it "raises when identifier doesn't start with a leter or number" do
+    lexer = Lexer.new("<<-!!!\\ntest\n!!!\n")
 
     expect_raises Crystal::SyntaxException, /heredoc identifier starts with invalid character/ do
       lexer.next_token
     end
   end
 
-  it "raises when identifier contains a character not for identifier" do
-    lexer = Lexer.new("<<-aaa.bbb?\\ntest\naaa.bbb?\n")
-
-    expect_raises Crystal::SyntaxException, /invalid character '.+' for heredoc identifier/ do
-      lexer.next_token
-    end
-  end
-
-  it "raises when identifier contains spaces" do
-    lexer = Lexer.new("<<-aaa  bbb\\ntest\naaabbb\n")
-
-    expect_raises Crystal::SyntaxException, /invalid character '.+' for heredoc identifier/ do
-      lexer.next_token
-    end
-  end
-
   it "raises on unexpected EOF while lexing heredoc" do
     lexer = Lexer.new("<<-aaa")
 
diff --git a/spec/compiler/normalize/string_interpolation_spec.cr b/spec/compiler/normalize/string_interpolation_spec.cr
@@ -10,4 +10,8 @@ describe "Normalize: string interpolation" do
     assert_expand "\"foo\#{bar}#{s}\"",
       "((((::String::Builder.new(218)) << \"foo\") << bar) << \"#{s}\").to_s"
   end
+
+  it "normalizes heredoc" do
+    assert_normalize "<<-FOO\nhello\nFOO", %("hello")
+  end
 end
diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr
@@ -1180,16 +1180,17 @@ describe "Parser" do
   it_parses %("hello " \\\n "world"), StringLiteral.new("hello world")
   it_parses %("hello "\\\n"world"), StringLiteral.new("hello world")
   it_parses %("hello \#{1}" \\\n "\#{2} world"), StringInterpolation.new(["hello ".string, 1.int32, 2.int32, " world".string] of ASTNode)
-  it_parses "<<-HERE\nHello, mom! I am HERE.\nHER dress is beautiful.\nHE is OK.\n  HERESY\nHERE", "Hello, mom! I am HERE.\nHER dress is beautiful.\nHE is OK.\n  HERESY".string
-  it_parses "<<-HERE\n   One\n  Zero\n  HERE", " One\nZero".string
-  it_parses "<<-HERE\n   One \\n Two\n  Zero\n  HERE", " One \n Two\nZero".string
-  it_parses "<<-HERE\n   One\n\n  Zero\n  HERE", " One\n\nZero".string
-  it_parses "<<-HERE\n   One\n \n  Zero\n  HERE", " One\n\nZero".string
+  it_parses "<<-HERE\nHello, mom! I am HERE.\nHER dress is beautiful.\nHE is OK.\n  HERESY\nHERE",
+    "Hello, mom! I am HERE.\nHER dress is beautiful.\nHE is OK.\n  HERESY".string_interpolation
+  it_parses "<<-HERE\n   One\n  Zero\n  HERE", " One\nZero".string_interpolation
+  it_parses "<<-HERE\n   One \\n Two\n  Zero\n  HERE", " One \n Two\nZero".string_interpolation
+  it_parses "<<-HERE\n   One\n\n  Zero\n  HERE", " One\n\nZero".string_interpolation
+  it_parses "<<-HERE\n   One\n \n  Zero\n  HERE", " One\n\nZero".string_interpolation
   it_parses "<<-HERE\n   \#{1}One\n  \#{2}Zero\n  HERE", StringInterpolation.new([" ".string, 1.int32, "One\n".string, 2.int32, "Zero".string] of ASTNode)
   it_parses "<<-HERE\n  foo\#{1}bar\n   baz\n  HERE", StringInterpolation.new(["foo".string, 1.int32, "bar\n baz".string] of ASTNode)
-  it_parses "<<-HERE\r\n   One\r\n  Zero\r\n  HERE", " One\r\nZero".string
-  it_parses "<<-HERE\r\n   One\r\n  Zero\r\n  HERE\r\n", " One\r\nZero".string
-  it_parses "<<-SOME\n  Sa\n  Se\n  SOME", "Sa\nSe".string
+  it_parses "<<-HERE\r\n   One\r\n  Zero\r\n  HERE", " One\r\nZero".string_interpolation
+  it_parses "<<-HERE\r\n   One\r\n  Zero\r\n  HERE\r\n", " One\r\nZero".string_interpolation
+  it_parses "<<-SOME\n  Sa\n  Se\n  SOME", "Sa\nSe".string_interpolation
   it_parses "<<-HERE\n  \#{1} \#{2}\n  HERE", StringInterpolation.new([1.int32, " ".string, 2.int32] of ASTNode)
   it_parses "<<-HERE\n  \#{1} \\n \#{2}\n  HERE", StringInterpolation.new([1.int32, " \n ".string, 2.int32] of ASTNode)
   assert_syntax_error "<<-HERE\n   One\nwrong\n  Zero\n  HERE", "heredoc line must have an indent greater or equal than 2", 3, 1
@@ -1200,16 +1201,24 @@ describe "Parser" do
   assert_syntax_error "<<-HERE\n One\n  \#{1}\n  HERE", "heredoc line must have an indent greater or equal than 2", 2, 1
   assert_syntax_error %("foo" "bar")
 
-  it_parses "<<-'HERE'\n  hello \\n world\n  \#{1}\n  HERE", StringLiteral.new("hello \\n world\n\#{1}")
+  it_parses "<<-'HERE'\n  hello \\n world\n  \#{1}\n  HERE", "hello \\n world\n\#{1}".string_interpolation
   assert_syntax_error "<<-'HERE\n", "expecting closing single quote"
 
-  it_parses "<<-FOO\n1\nFOO.bar", Call.new("1".string, "bar")
-  it_parses "<<-FOO\n1\nFOO + 2", Call.new("1".string, "+", 2.int32)
+  it_parses "<<-'HERE COMES HEREDOC'\n  hello \\n world\n  \#{1}\n  HERE COMES HEREDOC", "hello \\n world\n\#{1}".string_interpolation
 
-  it_parses "<<-FOO\n\t1\n\tFOO", StringLiteral.new("1")
-  it_parses "<<-FOO\n \t1\n \tFOO", StringLiteral.new("1")
-  it_parses "<<-FOO\n \t 1\n \t FOO", StringLiteral.new("1")
-  it_parses "<<-FOO\n\t 1\n\t FOO", StringLiteral.new("1")
+  assert_syntax_error "<<-FOO\n1\nFOO.bar", "Unterminated heredoc: can't find \"FOO\" anywhere before the end of file"
+  assert_syntax_error "<<-FOO\n1\nFOO + 2", "Unterminated heredoc: can't find \"FOO\" anywhere before the end of file"
+
+  it_parses "<<-FOO\n\t1\n\tFOO", "1".string_interpolation
+  it_parses "<<-FOO\n \t1\n \tFOO", "1".string_interpolation
+  it_parses "<<-FOO\n \t 1\n \t FOO", "1".string_interpolation
+  it_parses "<<-FOO\n\t 1\n\t FOO", "1".string_interpolation
+
+  it_parses "x, y = <<-FOO, <<-BAR\nhello\nFOO\nworld\nBAR",
+    MultiAssign.new(["x".var, "y".var] of ASTNode, ["hello".string_interpolation, "world".string_interpolation] of ASTNode)
+
+  it_parses "x, y, z = <<-FOO, <<-BAR, <<-BAZ\nhello\nFOO\nworld\nBAR\n!\nBAZ",
+    MultiAssign.new(["x".var, "y".var, "z".var] of ASTNode, ["hello".string_interpolation, "world".string_interpolation, "!".string_interpolation] of ASTNode)
 
   it_parses "enum Foo; A\nB, C\nD = 1; end", EnumDef.new("Foo".path, [Arg.new("A"), Arg.new("B"), Arg.new("C"), Arg.new("D", 1.int32)] of ASTNode)
   it_parses "enum Foo; A = 1, B; end", EnumDef.new("Foo".path, [Arg.new("A", 1.int32), Arg.new("B")] of ASTNode)
diff --git a/spec/support/syntax.cr b/spec/support/syntax.cr
@@ -90,6 +90,10 @@ class String
     StringLiteral.new self
   end
 
+  def string_interpolation
+    StringInterpolation.new([self.string] of ASTNode)
+  end
+
   def float32
     NumberLiteral.new self, :f32
   end
diff --git a/src/compiler/crystal/semantic/normalizer.cr b/src/compiler/crystal/semantic/normalizer.cr
@@ -375,5 +375,16 @@ module Crystal
         Assign.new(target.clone, call).at(node)
       end
     end
+
+    def transform(node : StringInterpolation)
+      # If the interpolation has just one string literal inside it,
+      # return that instead of an interpolation
+      if node.expressions.size == 1
+        first = node.expressions.first
+        return first if first.is_a?(StringLiteral)
+      end
+
+      super
+    end
   end
 end
diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr
@@ -18,6 +18,13 @@ module Crystal
     @token_end_location : Location?
     @string_pool : StringPool
 
+    # This is an interface for storing data associated to a heredoc
+    module HeredocItem
+    end
+
+    # Heredocs pushed when found. Should be processed when encountering a newline
+    getter heredocs = [] of {Token::DelimiterState, HeredocItem}
+
     def initialize(string, string_pool : StringPool? = nil)
       @reader = Char::Reader.new(string)
       @token = Token.new
@@ -158,30 +165,36 @@ module Crystal
             found_closing_single_quote = false
 
             char = next_char
+            start_here = current_pos
+
             if char == '\''
               has_single_quote = true
               char = next_char
+              start_here = current_pos
             end
 
-            unless ident_start?(char)
+            unless ident_part?(char)
               raise "heredoc identifier starts with invalid character"
             end
 
-            here << char
+            end_here = 0
+
             while true
               char = next_char
               case
               when char == '\r'
                 if peek_next_char == '\n'
-                  next
+                  end_here = current_pos
+                  next_char
+                  break
                 else
                   raise "expecting '\\n' after '\\r'"
                 end
               when char == '\n'
-                incr_line_number 0
+                end_here = current_pos
                 break
               when ident_part?(char)
-                here << char
+                # ok
               when char == '\0'
                 raise "Unexpected EOF on heredoc identifier"
               else
@@ -191,8 +204,11 @@ module Crystal
                   if peek != '\r' && peek != '\n'
                     raise "expecting '\\n' or '\\r' after closing single quote"
                   end
+                elsif has_single_quote
+                  # wait until another quote
                 else
-                  raise "invalid character #{char.inspect} for heredoc identifier"
+                  end_here = current_pos
+                  break
                 end
               end
             end
@@ -201,8 +217,11 @@ module Crystal
               raise "expecting closing single quote"
             end
 
-            here = here.to_s
-            delimited_pair :heredoc, here, here, start, allow_escapes: !has_single_quote
+            end_here -= 1 if has_single_quote
+
+            here = string_range(start_here, end_here)
+
+            delimited_pair :heredoc, here, here, start, allow_escapes: !has_single_quote, advance: false
           else
             @token.type = :"<<"
           end
@@ -1176,6 +1195,10 @@ module Crystal
     end
 
     def consume_newlines
+      # If there are heredocs we don't freely consume newlines because
+      # these will be part of the heredoc string
+      return unless @heredocs.empty?
+
       if @count_whitespace
         return
       end
@@ -1721,6 +1744,7 @@ module Crystal
 
     def next_string_token(delimiter_state)
       @token.line_number = @line_number
+      @token.delimiter_state = delimiter_state
 
       start = current_pos
       string_end = delimiter_state.end
@@ -1737,13 +1761,13 @@ module Crystal
         else
           @token.type = :STRING
           @token.value = string_end.to_s
-          @token.delimiter_state = @token.delimiter_state.with_open_count_delta(-1)
+          @token.delimiter_state = delimiter_state.with_open_count_delta(-1)
         end
       when string_nest
         next_char
         @token.type = :STRING
         @token.value = string_nest.to_s
-        @token.delimiter_state = @token.delimiter_state.with_open_count_delta(+1)
+        @token.delimiter_state = delimiter_state.with_open_count_delta(+1)
       when '\\'
         if delimiter_state.allow_escapes
           if delimiter_state.kind == :regex
@@ -1877,10 +1901,9 @@ module Crystal
 
             if reached_end &&
                (current_char == '\n' || current_char == '\0' ||
-               (current_char == '\r' && peek_next_char == '\n' && next_char) ||
-               !ident_part?(current_char))
+               (current_char == '\r' && peek_next_char == '\n' && next_char))
               @token.type = :DELIMITER_END
-              @token.delimiter_state = @token.delimiter_state.with_heredoc_indent(indent)
+              @token.delimiter_state = delimiter_state.with_heredoc_indent(indent)
             else
               @reader.pos = old_pos
               @column_number = old_column
@@ -1923,8 +1946,9 @@ module Crystal
       msg = case delimiter_state.kind
             when :command then "Unterminated command literal"
             when :regex   then "Unterminated regular expression"
-            when :heredoc then "Unterminated heredoc"
-            when :string  then "Unterminated string literal"
+            when :heredoc
+              "Unterminated heredoc: can't find \"#{delimiter_state.end}\" anywhere before the end of file"
+            when :string then "Unterminated string literal"
             else
               ::raise "unreachable"
             end
@@ -2409,8 +2433,8 @@ module Crystal
       @token.value = value
     end
 
-    def delimited_pair(kind, string_nest, string_end, start, allow_escapes = true)
-      next_char
+    def delimited_pair(kind, string_nest, string_end, start, allow_escapes = true, advance = true)
+      next_char if advance
       @token.type = :DELIMITER_START
       @token.delimiter_state = Token::DelimiterState.new(kind, string_nest, string_end, allow_escapes)
       set_token_raw_from_start(start)
diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr
diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr

-Original file line number
+Diff line change
       @wants_doc = false
       @doc_enabled = false
       @no_type_declaration = 0
 +      @consuming_heredocs = false
       # This flags tells the parser where it has to consider a "do"
       # as belonging to the current parsed call. For example when writing
       check :DELIMITER_START
 +      if delimiter_state.kind == :heredoc
 +        node = StringInterpolation.new([] of ASTNode).at(location)
 +        @heredocs << {delimiter_state, node}
 +        next_token
 +        return node
 +      end
++
       next_string_token(delimiter_state)
       delimiter_state = @token.delimiter_state
       end
       if has_interpolation
 -        if needs_heredoc_indent_removed?(delimiter_state)
 -          pieces = remove_heredoc_indent(pieces, delimiter_state.heredoc_indent)
 -        else
 -          pieces = pieces.map do |piece|
 -            value = piece.value
 -            value.is_a?(String) ? StringLiteral.new(value) : value
 -          end
 -        end
 +        pieces = combine_interpolation_pieces(pieces, delimiter_state)
         result = StringInterpolation.new(pieces).at(location)
       else
 -        if needs_heredoc_indent_removed?(delimiter_state)
 -          pieces = remove_heredoc_indent(pieces, delimiter_state.heredoc_indent)
 -          string = pieces.join { |piece| piece.as(StringLiteral).value }
 -        else
 -          string = pieces.map(&.value).join
 -        end
 +        string = combine_pieces(pieces, delimiter_state)
         result = StringLiteral.new string
       end
       result
     end
 +    private def combine_interpolation_pieces(pieces, delimiter_state)
 +      if needs_heredoc_indent_removed?(delimiter_state)
 +        remove_heredoc_indent(pieces, delimiter_state.heredoc_indent)
 +      else
 +        pieces.map do |piece|
 +          value = piece.value
 +          value.is_a?(String) ? StringLiteral.new(value) : value
 +        end
 +      end
 +    end
++
 +    private def combine_pieces(pieces, delimiter_state)
 +      if needs_heredoc_indent_removed?(delimiter_state)
 +        pieces = remove_heredoc_indent(pieces, delimiter_state.heredoc_indent)
 +        pieces.join { |piece| piece.as(StringLiteral).value }
 +      else
 +        pieces.map(&.value).join
 +      end
 +    end
++
     def consume_delimiter(pieces, delimiter_state, has_interpolation)
       options = Regex::Options::None
       token_end_location = nil
       options
     end
 +    def consume_heredocs
 +      @consuming_heredocs = true
 +      @heredocs.reverse!
 +      while heredoc = @heredocs.pop?
 +        consume_heredoc(heredoc[0], heredoc[1].as(StringInterpolation))
 +      end
 +      @consuming_heredocs = false
 +    end
++
 +    def consume_heredoc(delimiter_state, node)
 +      next_string_token(delimiter_state)
 +      delimiter_state = @token.delimiter_state
++
 +      pieces = [] of Piece
 +      has_interpolation = false
++
 +      delimiter_state, has_interpolation, options, token_end_location = consume_delimiter pieces, delimiter_state, has_interpolation
++
 +      if has_interpolation
 +        pieces = combine_interpolation_pieces(pieces, delimiter_state)
 +        node.expressions.concat(pieces)
 +      else
 +        string = combine_pieces(pieces, delimiter_state)
 +        node.expressions.push(StringLiteral.new(string).at(node.location).at_end(token_end_location))
 +      end
++
 +      node.end_location = token_end_location
 +    end
++
     def needs_heredoc_indent_removed?(delimiter_state)
       delimiter_state.kind == :heredoc && delimiter_state.heredoc_indent > 0
     end
       pieces.each_with_index do |piece, i|
         value = piece.value
         line_number = piece.line_number
++
         this_piece_is_in_new_line = line_number != previous_line_number
         next_piece_is_in_new_line = i == pieces.size - 1 || pieces[i + 1].line_number != line_number
         if value.is_a?(String)
       @visibility = old_visibility
       value
     end
++
 +    def next_token
 +      token = super
++
 +      if token.type == :NEWLINE && !@consuming_heredocs && !@heredocs.empty?
 +        consume_heredocs
 +      end
++
 +      token
 +    end
 +  end
++
 +  class StringInterpolation
 +    include Lexer::HeredocItem
   end
 end
-Original file line number
+Diff line change
       end_line : Int32,
       difference : Int32
 +    record HeredocInfo,
 +      node : StringInterpolation,
 +      token : Token,
 +      line : Int32,
 +      column : Int32,
 +      indent : Int32,
 +      string_continuation : Int32 do
 +      include Lexer::HeredocItem
 +    end
++
     @lexer : Lexer
     @comment_columns : Array(Int32?)
     @indent : Int32
       @indent = 0
       @line = 0
       @column = 0
 -      @token = @lexer.token
 -      @token = next_token
 +      @token = @lexer.next_token
       @output = IO::Memory.new(source.bytesize)
       @line_output = IO::Memory.new
     end
     def visit(node : StringInterpolation)
 +      if @token.delimiter_state.kind == :heredoc
 +        # For heredoc, only write the start: on a newline will print it
 +        @lexer.heredocs << {@token.delimiter_state, HeredocInfo.new(node, @token.dup, @line, @column, @indent, @string_continuation)}
 +        write @token.raw
 +        next_token
 +        return false
 +      end
++
       check :DELIMITER_START
 -      column = @column
 -      old_indent = @indent
 -      old_string_continuation = @string_continuation
 -      is_regex = @token.delimiter_state.kind == :regex
 -      indent_difference = @token.column_number - (@column + 1)
 +      visit_string_interpolation(node, @token, @line, @column, @indent, @string_continuation)
 +    end
 -      write @token.raw
 +    def visit_string_interpolation(node, token, line, column, old_indent, old_string_continuation, wrote_token = false)
 +      @token = token
++
 +      is_regex = token.delimiter_state.kind == :regex
 +      indent_difference = token.column_number - (column + 1)
++
 +      write token.raw unless wrote_token
       next_string_token
 -      delimiter_state = @token.delimiter_state
 -      is_heredoc = @token.delimiter_state.kind == :heredoc
 +      delimiter_state = token.delimiter_state
 +      is_heredoc = token.delimiter_state.kind == :heredoc
       @last_is_heredoc = is_heredoc
       heredoc_line = @line
       end
     end
 +    private def consume_heredocs
 +      @consuming_heredocs = true
 +      @lexer.heredocs.reverse!
 +      while heredoc = @lexer.heredocs.pop?
 +        consume_heredoc(heredoc[0], heredoc[1].as(HeredocInfo))
 +        write_line unless @lexer.heredocs.empty?
 +      end
 +      @consuming_heredocs = false
 +    end
++
 +    private def consume_heredoc(delimiter_state, info)
 +      visit_string_interpolation(
 +        info.node,
 +        info.token,
 +        info.line,
 +        info.column,
 +        info.indent, info.string_continuation,
 +        wrote_token: true)
 +    end
++
     def visit(node : RegexLiteral)
       accept node.value
       io << @output
     end
 -    def maybe_reset_passed_backslash_newline
 -    end
+-
     def next_token
       current_line_number = @lexer.line_number
       @token = @lexer.next_token
       if @token.type == :DELIMITER_START
         increment_lines(@lexer.line_number - current_line_number)
 +      elsif @token.type == :NEWLINE
 +        if !@lexer.heredocs.empty? && !@consuming_heredocs
 +          write_line
 +          consume_heredocs
 +        end
       end
       @token
     end