Parse __END__ constructs in the lexer

opal · Dec 7, 2013 · 17690dc · 17690dc
1 parent 1aef0d4
commit 17690dc
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -51,6 +51,9 @@
 *   Fix bug where sometimes the wrong regexp flags would be generated in the
     output javascript.
 
+*   Support parsing `__END__` constructs in ruby code, inside the lexer. The
+    content is gathered up by use of the parser.
+
 *   Support single character strings (using ? prefix) with escaped characters.
 
 ## 0.5.5 2013-11-25

diff --git a/lib/opal/parser/lexer.rb b/lib/opal/parser/lexer.rb
@@ -23,6 +23,7 @@ class Lexer
 
     attr_reader :line
     attr_reader :scope
+    attr_reader :eof_content
 
     attr_accessor :lex_state
     attr_accessor :strterm
@@ -1143,6 +1144,17 @@ def yylex
           cmdarg_push 0
           return result
 
+        elsif scanner.bol? and skip(/\__END__(\n|$)/)
+          while true
+            if scanner.eos?
+              @eof_content = self.yylval
+              return false
+            end
+
+            scan(/(.*)/)
+            scan(/\n/)
+          end
+
         elsif check(/[0-9]/)
           return process_numeric
 

diff --git a/spec/cli/lexer_spec.rb b/spec/cli/lexer_spec.rb
@@ -76,4 +76,28 @@
       end
     end
   end
+
+  describe "__END__ content in a source file" do
+    it "ignores token and following content if at start of line and followed by newline" do
+      expect_parsed("42\n__END__").to eq([:int, 42])
+      expect_parsed("42\n__END__\nFred").to eq([:int, 42])
+      expect_parsed(" __END__").to eq([:call, nil, :__END__, [:arglist]])
+      expect_parsed("__END__ ").to eq([:call, nil, :__END__, [:arglist]])
+      expect_parsed("__END__ 42").to eq([:call, nil, :__END__, [:arglist, [:int, 42]]])
+    end
+
+    it "adds any __END__ content to lexer.eof_content" do
+      parser = Opal::Parser.new
+      parser.parse("42")
+      expect(parser.lexer.eof_content).to eq(nil)
+
+      parser = Opal::Parser.new
+      parser.parse("42\n__END__\nFord\nPerfect")
+      expect(parser.lexer.eof_content).to eq("Ford\nPerfect")
+
+      parser = Opal::Parser.new
+      parser.parse("42\n__END__\n")
+      expect(parser.lexer.eof_content).to eq("")
+    end
+  end
 end
diff --git a/spec/cli/spec_helper.rb b/spec/cli/spec_helper.rb
@@ -5,6 +5,10 @@ def parsed(source, file='(string)')
     Opal::Parser.new.parse(source, file)
   end
 
+  def expect_parsed(source)
+    expect(parsed(source))
+  end
+
   def expect_parsed_string(source)
     expect(parsed(source)[1])
   end