Lexer now returns column info for every token

opal · Nov 23, 2013 · 6e108ec · 6e108ec
1 parent 7a36c3f
commit 6e108ec
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 5 deletions.
diff --git a/lib/opal/fragment.rb b/lib/opal/fragment.rb
@@ -16,7 +16,7 @@ def initialize(code, sexp = nil)
     # In debug mode we may wish to include the original line as a comment
     def to_code
       if @sexp
-        "/*:#{@sexp.line}*/#{@code}"
+        "/*:#{@sexp.line}:#{@sexp.column}*/#{@code}"
       else
         @code
       end

diff --git a/lib/opal/parser/lexer.rb b/lib/opal/parser/lexer.rb
@@ -16,6 +16,7 @@ def initialize(source, file)
       @cmdarg     = 0
       @line       = 1
       @column     = 0
+      @tok_column = 0
       @file       = file
 
       @scanner = StringScanner.new(source)
@@ -84,14 +85,20 @@ def set_arg_state
 
     def scan(regexp)
       if result = @scanner.scan(regexp)
+        @column += result.length
         @yylval += @scanner.matched
       end
 
       result
     end
 
     def skip(regexp)
-      @scanner.scan regexp
+      if result = @scanner.scan(regexp)
+        @column += result.length
+        @tok_column = @column
+      end
+
+      result
     end
 
     def check(regexp)
@@ -106,10 +113,15 @@ def matched
       @scanner.matched
     end
 
+    def line=(line)
+      @column = @tok_column = 0
+      @line = line
+    end
+
     def next_token
       token     = self.yylex
       value     = self.yylval
-      location  = [@line, @column]
+      location  = [@line, @tok_column]
 
       [token, [value, location]]
     end
@@ -547,10 +559,17 @@ def yylex
 
         elsif skip(/(\n|#)/)
           c = scanner.matched
-          if c == '#' then skip(/(.*)/) else @line += 1; end
+          if c == '#'
+            skip(/(.*)/)
+          else
+            self.line += 1
+          end
 
           skip(/(\n+)/)
-          @line += scanner.matched.length if scanner.matched
+
+          if scanner.matched
+            self.line += scanner.matched.length
+          end
 
           next if [:expr_beg, :expr_dot].include? @lex_state