Skip to content

Commit

Permalink
Showing 5 changed files with 2,292 additions and 2,533 deletions.
1 change: 1 addition & 0 deletions lib/opal/parser.rb
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@ def pop_scope
end

def on_error(t, val, vstack)
puts lexer.instance_variable_get(:@scanner).peek(50)
raise "parse error on value #{val.inspect} (#{token_to_str(t) || '?'}) :#{@file}:#{lexer.line}"
end

4,320 changes: 2,163 additions & 2,157 deletions lib/opal/parser/grammar.rb

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion lib/opal/parser/grammar.y
Original file line number Diff line number Diff line change
@@ -435,6 +435,7 @@ reswords:
| OR | REDO | RESCUE | RETRY | RETURN | SELF
| SUPER | THEN | TRUE | UNDEF | WHEN | YIELD
| IF_MOD | UNLESS_MOD | WHILE_MOD | UNTIL_MOD | RESCUE_MOD
| IF | WHILE | UNTIL

arg:
lhs '=' arg
@@ -945,7 +946,7 @@ primary:
}
| DEF singleton dot_or_colon
{
# ..
lexer.lex_state = :expr_fname
}
fname
{
30 changes: 27 additions & 3 deletions lib/opal/parser/keywords.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
module Opal
module Keywords

class KeywordTable
attr_accessor :name, :id, :state

def initialize(name, id, state)
@name = name
@id = id
@state = state
end
end

KEYWORDS = [
["__LINE__", [:LINE, :LINE], :expr_end],
["__FILE__", [:FILE, :FILE], :expr_end],
@@ -13,7 +24,7 @@ module Keywords
["defined?", [:DEFINED, :DEFINED], :expr_arg],
["do", [:DO, :DO], :expr_beg],
["else", [:ELSE, :ELSE], :expr_beg],
["elsif", [:ELSE, :ELSE], :expr_beg],
["elsif", [:ELSIF, :ELSIF], :expr_beg],
["end", [:END, :END], :expr_end],
["ensure", [:ENSURE, :ENSURE], :expr_beg],
["false", [:FALSE, :FALSE], :expr_end],
@@ -26,15 +37,28 @@ module Keywords
["redo", [:REDO, :REDO], :expr_end],
["rescue", [:RESCUE, :RESCUE_MOD], :expr_mid],
["return", [:RETURN, :RETURN], :expr_mid],
["self", [:SELF, :SELF], :expr_end],
["super", [:SUPER, :SUPER], :expr_arg],
["then", [:THEN, :THEN], :expr_beg],
["true", [:TRUE, :TRUE], :expr_end],
["undef", [:UNDEF, :UNDEF], :expr_fname],
["unless", [:UNLESS, :UNLESS_MOD], :expr_beg],
["until", [:UNTIL, :UNTIL_MOD], :expr_beg],
["when", [:CASE, :CASE], :expr_beg],
["when", [:WHEN, :WHEN], :expr_beg],
["while", [:WHILE, :WHILE_MOD], :expr_beg],
["yield", [:YIELD, :YIELD], :expr_arg]
]
].map { |decl| KeywordTable.new(*decl) }

def self.map
unless @map
@map = {}
KEYWORDS.each { |k| @map[k.name] = k }
end
@map
end

def self.keyword(kw)
map[kw]
end
end
end
471 changes: 99 additions & 372 deletions lib/opal/parser/lexer.rb
Original file line number Diff line number Diff line change
@@ -77,6 +77,13 @@ def space?
@scanner.check(/\s/)
end

def next_token
# if we are trying to parse a string, then delegate to that
return next_string_token if @string_parse

self.yylex
end

def next_string_token
# str_parse, scanner = current_string_parse, @scanner
str_parse = @string_parse
@@ -254,10 +261,98 @@ def add_string_content(str_buffer, str_parse)
raise "reached EOF while in string" if scanner.eos?
end

def next_token
# if we are trying to parse a string, then delegate to that
return next_string_token if @string_parse
def process_identifier(matched, cmd_start)
scanner = @scanner
matched = scanner.matched

if scanner.peek(2) != '::' && scanner.scan(/:/)
@lex_state = :expr_beg
return :LABEL, "#{matched}"
end

if matched == 'defined?'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_arg
return :DEFINED, 'defined?'
end

if matched.end_with? '?', '!'
result = :IDENTIFIER
else
if @lex_state == :expr_fname
if scanner.scan(/\=/)
result = :IDENTIFIER
matched += scanner.matched
end

elsif matched =~ /^[A-Z]/
result = :CONSTANT
else
result = :IDENTIFIER
end
end

if @lex_state != :expr_dot and kw = Keywords.keyword(matched)
old_state = @lex_state
@lex_state = kw.state

if old_state == :expr_fname
return [kw.id[0], kw.name]
end

if @lex_state == :expr_beg
cmd_start = true
end

if matched == "do"
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

if @start_of_lambda
@start_of_lambda = false
@lex_state = :expr_beg
return [:DO_LAMBDA, scanner.matched]
elsif cond?
@lex_state = :expr_beg
return :DO_COND, matched
elsif cmdarg? && @lex_state != :expr_cmdarg
@lex_state = :expr_beg
return :DO_BLOCK, matched
elsif @lex_state == :expr_endarg
return :DO_BLOCK, matched
else
@lex_state = :expr_beg
return :DO, matched
end
else
if old_state == :expr_beg or old_state == :expr_value
return [kw.id[0], matched]
else
if kw.id[0] != kw.id[1]
@lex_state = :expr_beg
end

return [kw.id[1], matched]
end
end
end

if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
@lex_state = cmd_start ? :expr_cmdarg : :expr_arg
else
@lex_state = :expr_end
end

return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
end

def yylex
# scanner, @space_seen, cmd_start, c = @scanner, false, false, ''
scanner = @scanner
@space_seen = false
@@ -868,375 +963,7 @@ def next_token
end

elsif scanner.scan(/(\w)+[\?\!]?/)
matched = scanner.matched

if scanner.peek(2) != '::' && scanner.scan(/:/)
@lex_state = :expr_beg
return :LABEL, "#{matched}"
end

if matched == 'self'
if @lex_state == :expr_dot
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end unless @lex_state == :expr_fname
return :SELF, matched
end

if @lex_state == :expr_fname
if scanner.scan(/\=/)
@lex_state = :expr_end
return :IDENTIFIER, matched + scanner.matched
end
end

case matched
when 'class'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_class
return :CLASS, matched

when 'module'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :MODULE, matched

when 'defined?'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_arg
return :DEFINED, 'defined?'

when 'def'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_fname
@scope_line = @line
return :DEF, matched

when 'undef'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_fname
return :UNDEF, matched

when 'end'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :END, matched

when 'do'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

if @start_of_lambda
@start_of_lambda = false
@lex_state = :expr_beg
return [:DO_LAMBDA, scanner.matched]
elsif cond?
@lex_state = :expr_beg
return :DO_COND, matched
elsif cmdarg? && @lex_state != :expr_cmdarg
@lex_state = :expr_beg
return :DO_BLOCK, matched
elsif @lex_state == :expr_endarg
return :DO_BLOCK, matched
else
@lex_state = :expr_beg
return :DO, matched
end

when 'if'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :IF, matched if @lex_state == :expr_beg
@lex_state = :expr_beg
return :IF_MOD, matched

when 'unless'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :UNLESS, matched if @lex_state == :expr_beg
@lex_state = :expr_beg
return :UNLESS_MOD, matched

when 'else'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :ELSE, matched

when 'elsif'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :ELSIF, matched

when 'true'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :TRUE, matched

when 'false'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :FALSE, matched

when 'nil'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :NIL, matched

when '__LINE__'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :LINE, @line.to_s

when '__FILE__'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :FILE, matched

when 'begin'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :BEGIN, matched

when 'rescue'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

if @lex_state == :expr_beg
@lex_state = :expr_mid
return :RESCUE, matched
end

@lex_state = :expr_beg
return :RESCUE_MOD, matched

when 'ensure'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :ENSURE, matched

when 'case'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :CASE, matched

when 'when'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :WHEN, matched

when 'or'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :OR, matched

when 'and'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :AND, matched

when 'not'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :NOT, matched

when 'return'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_mid
return :RETURN, matched

when 'next'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_mid
return :NEXT, matched

when 'redo'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_end
return :REDO, matched

when 'break'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_mid
return :BREAK, matched

when 'super'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_arg
return :SUPER, matched

when 'then'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_beg
return :THEN, matched

when 'while'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :WHILE, matched if @lex_state == :expr_beg
@lex_state = :expr_beg
return :WHILE_MOD, matched

when 'until'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

return :UNTIL, matched if @lex_state == :expr_beg
@lex_state = :expr_beg
return :UNTIL_MOD, matched

when 'yield'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_arg
return :YIELD, matched

when 'alias'
if after_operator?
@lex_state = :expr_end
return :IDENTIFIER, matched
end

@lex_state = :expr_fname
return :ALIAS, matched
end

if @lex_state == :expr_fname
if scanner.scan(/\=/)
@lex_state = :expr_end
return :IDENTIFIER, matched + scanner.matched
end
end

if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
@lex_state = cmd_start ? :expr_cmdarg : :expr_arg
else
@lex_state = :expr_end
end

return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]

return process_identifier scanner.matched, cmd_start
end

if scanner.eos?

0 comments on commit 1c61665

Please sign in to comment.