Skip to content

Commit 6b7866c

Browse files
adambeynonadambeynon
committedJul 31, 2013
Rewrite some lexer logic to match parse.y
·
v1.8.3.rc1lerna-0.0.1-alpha.2
1 parent 03b6ab5 commit 6b7866c

File tree

1 file changed

+107
-78
lines changed

1 file changed

+107
-78
lines changed
 

Diff for: ‎lib/opal/lexer.rb

+107-78
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,22 @@ def cmdarg?
8686
(@cmdarg & 1) != 0
8787
end
8888

89+
def arg?
90+
[:expr_arg, :expr_cmdarg].include? @lex_state
91+
end
92+
93+
def end?
94+
[:expr_end, :expr_endarg, :expr_endfn].include? @lex_state
95+
end
96+
97+
def beg?
98+
[:expr_beg, :expr_value, :expr_mid, :expr_class].include? @lex_state
99+
end
100+
101+
def after_operator?
102+
[:expr_fname, :expr_dot].include? @lex_state
103+
end
104+
89105
def next_string_token
90106
# str_parse, scanner = current_string_parse, @scanner
91107
str_parse = @string_parse
@@ -269,15 +285,15 @@ def next_token
269285
# if we are trying to parse a string, then delegate to that
270286
return next_string_token if @string_parse
271287

272-
# scanner, space_seen, cmd_start, c = @scanner, false, false, ''
288+
# scanner, @space_seen, cmd_start, c = @scanner, false, false, ''
273289
scanner = @scanner
274-
space_seen = false
290+
@space_seen = false
275291
cmd_start = false
276292
c = ''
277293

278294
while true
279295
if scanner.scan(/\ |\t|\r/)
280-
space_seen = true
296+
@space_seen = true
281297
next
282298

283299
elsif scanner.scan(/(\n|#)/)
@@ -290,7 +306,7 @@ def next_token
290306
next if [:expr_beg, :expr_dot].include? @lex_state
291307

292308
if scanner.scan(/([\ \t\r\f\v]*)\./)
293-
space_seen = true unless scanner[1].empty?
309+
@space_seen = true unless scanner[1].empty?
294310
scanner.pos = scanner.pos - 1
295311

296312
next unless scanner.check(/\.\./)
@@ -304,6 +320,83 @@ def next_token
304320
@lex_state = :expr_beg
305321
return ';', ';'
306322

323+
elsif scanner.scan(/\*/)
324+
if scanner.scan(/\*/)
325+
if scanner.scan(/\=/)
326+
@lex_state = :expr_beg
327+
return :OP_ASGN, '**'
328+
end
329+
330+
if @lex_state == :expr_fname or @lex_state == :expr_dot
331+
@lex_state = :expr_arg
332+
else
333+
@lex_state = :expr_beg
334+
end
335+
336+
return '**', '**'
337+
338+
else
339+
if scanner.scan(/\=/)
340+
@lex_state = :expr_beg
341+
return :OP_ASGN, '*'
342+
end
343+
end
344+
345+
if scanner.scan(/\*\=/)
346+
@lex_state = :expr_beg
347+
return :OP_ASGN, '**'
348+
end
349+
350+
if scanner.scan(/\*/)
351+
if after_operator?
352+
@lex_state = :expr_arg
353+
else
354+
@lex_state = :expr_beg
355+
end
356+
357+
return '**', '**'
358+
end
359+
360+
if scanner.scan(/\=/)
361+
@lex_state = :expr_beg
362+
return :OP_ASGN, '*'
363+
else
364+
result = '*'
365+
if @lex_state == :expr_fname
366+
@lex_state = :expr_end
367+
return '*', result
368+
elsif @space_seen && scanner.check(/\S/)
369+
@lex_state = :expr_beg
370+
return :SPLAT, result
371+
elsif [:expr_beg, :expr_mid].include? @lex_state
372+
@lex_state = :expr_beg
373+
return :SPLAT, result
374+
else
375+
@lex_state = :expr_beg
376+
return '*', result
377+
end
378+
end
379+
380+
elsif scanner.scan(/\!/)
381+
c = scanner.scan(/./)
382+
if after_operator?
383+
@lex_state = :expr_arg
384+
if c == "@"
385+
return '!', '!'
386+
end
387+
else
388+
@lex_state = :expr_beg
389+
end
390+
391+
if c == '='
392+
return '!=', '!='
393+
elsif c == '~'
394+
return '!~', '!~'
395+
end
396+
397+
scanner.pos = scanner.pos - 1
398+
return '!', '!'
399+
307400
elsif scanner.scan(/\"/)
308401
@string_parse = { :beg => '"', :end => '"', :interpolate => true }
309402
return :STRING_BEG, scanner.matched
@@ -359,7 +452,7 @@ def next_token
359452
elsif @lex_state == :expr_fname
360453
@lex_state = :expr_end
361454
elsif @lex_state == :expr_cmdarg || @lex_state == :expr_arg
362-
if !scanner.check(/\s/) && space_seen
455+
if !scanner.check(/\s/) && @space_seen
363456
@string_parse = { :beg => '/', :end => '/', :interpolate => true, :regexp => true }
364457
return :REGEXP_BEG, scanner.matched
365458
end
@@ -374,7 +467,7 @@ def next_token
374467
@lex_state = :expr_beg
375468
return :OP_ASGN, '%'
376469
elsif scanner.check(/[^\s]/)
377-
if @lex_state == :expr_beg or (@lex_state == :expr_arg && space_seen)
470+
if @lex_state == :expr_beg or (@lex_state == :expr_arg && @space_seen)
378471
interpolate = true
379472
start_word = scanner.scan(/./)
380473
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
@@ -387,7 +480,7 @@ def next_token
387480

388481
elsif scanner.scan(/\\/)
389482
if scanner.scan(/\r?\n/)
390-
space_seen = true
483+
@space_seen = true
391484
next
392485
end
393486

@@ -397,7 +490,7 @@ def next_token
397490
result = scanner.matched
398491
if [:expr_beg, :expr_mid].include? @lex_state
399492
result = :PAREN_BEG
400-
elsif space_seen && [:expr_arg, :expr_cmdarg].include?(@lex_state)
493+
elsif @space_seen && [:expr_arg, :expr_cmdarg].include?(@lex_state)
401494
result = :tLPAREN_ARG
402495
else
403496
result = '('
@@ -427,7 +520,7 @@ def next_token
427520
else
428521
raise "Unexpected '[' token"
429522
end
430-
elsif [:expr_beg, :expr_mid].include?(@lex_state) || space_seen
523+
elsif [:expr_beg, :expr_mid].include?(@lex_state) || @space_seen
431524
@lex_state = :expr_beg
432525
cond_push 0
433526
cmdarg_push 0
@@ -464,43 +557,11 @@ def next_token
464557
@lex_state = :expr_dot unless @lex_state == :expr_fname
465558
return '.', scanner.matched
466559

467-
elsif scanner.scan(/\*\*\=/)
468-
@lex_state = :expr_beg
469-
return :OP_ASGN, '**'
470-
471-
elsif scanner.scan(/\*\*/)
472-
if @lex_state == :expr_fname or @lex_state == :expr_dot
473-
@lex_state = :expr_arg
474-
else
475-
@lex_state = :expr_beg
476-
end
477-
return '**', '**'
478-
479-
elsif scanner.scan(/\*\=/)
480-
@lex_state = :expr_beg
481-
return :OP_ASGN, '*'
482-
483-
elsif scanner.scan(/\*/)
484-
result = scanner.matched
485-
if @lex_state == :expr_fname
486-
@lex_state = :expr_end
487-
return '*', result
488-
elsif space_seen && scanner.check(/\S/)
489-
@lex_state = :expr_beg
490-
return :SPLAT, result
491-
elsif [:expr_beg, :expr_mid].include? @lex_state
492-
@lex_state = :expr_beg
493-
return :SPLAT, result
494-
else
495-
@lex_state = :expr_beg
496-
return '*', result
497-
end
498-
499560
elsif scanner.scan(/\:\:/)
500561
if [:expr_beg, :expr_mid, :expr_class].include? @lex_state
501562
@lex_state = :expr_beg
502563
return '::@', scanner.matched
503-
elsif space_seen && @lex_state == :expr_arg
564+
elsif @space_seen && @lex_state == :expr_arg
504565
@lex_state = :expr_beg
505566
return '::@', scanner.matched
506567
end
@@ -571,7 +632,7 @@ def next_token
571632
@lex_state = :expr_beg
572633
return :OP_ASGN, '&'
573634
elsif scanner.scan(/\&/)
574-
if space_seen && !scanner.check(/\s/) && (@lex_state == :expr_cmdarg || @lex_state == :expr_arg)
635+
if @space_seen && !scanner.check(/\s/) && (@lex_state == :expr_cmdarg || @lex_state == :expr_arg)
575636
return '&@', '&'
576637
elsif [:expr_beg, :expr_mid].include? @lex_state
577638
return '&@', '&'
@@ -594,7 +655,7 @@ def next_token
594655
if @lex_state == :expr_fname
595656
@lex_state = :expr_end
596657
return '<<', '<<'
597-
elsif ![:expr_end, :expr_dot, :expr_endarg, :expr_class].include?(@lex_state) && space_seen
658+
elsif ![:expr_end, :expr_dot, :expr_endarg, :expr_class].include?(@lex_state) && @space_seen
598659
if scanner.scan(/(-?)['"]?(\w+)['"]?/)
599660
heredoc = scanner[2]
600661
# for now just scrap rest of line + skip down one line for
@@ -682,7 +743,7 @@ def next_token
682743
end
683744

684745
if @lex_state == :expr_cmdarg || @lex_state == :expr_arg
685-
if !scanner.check(/\s/) && space_seen
746+
if !scanner.check(/\s/) && @space_seen
686747
@lex_state = :expr_mid
687748
return [sign, sign]
688749
end
@@ -701,19 +762,7 @@ def next_token
701762
@lex_state = :expr_end
702763
return :STRING, scanner.scan(/./)
703764
end
704-
#if scanner.scan(/\\/)
705-
#c = if scanner.scan(/n/)
706-
#"\n"
707-
#else
708-
#scanner.scan(/./)
709-
#scanner.matched
710-
#end
711-
#else
712-
#c = scanner.scan(/./)
713-
#end
714-
715-
#@lex_state = :expr_end
716-
#return :STRING, c
765+
717766
@lex_state = :expr_beg
718767
return '?', scanner.matched
719768

@@ -749,26 +798,6 @@ def next_token
749798
@lex_state = :expr_beg
750799
return '=', '='
751800

752-
elsif scanner.scan(/\!\=/)
753-
if @lex_state == :expr_fname
754-
@lex_state == :expr_end
755-
return '!=', '!='
756-
end
757-
@lex_state = :expr_beg
758-
return '!=', '!='
759-
760-
elsif scanner.scan(/\!\~/)
761-
@lex_state = :expr_beg
762-
return '!~', '!~'
763-
764-
elsif scanner.scan(/\!/)
765-
if @lex_state == :expr_fname
766-
@lex_state = :expr_end
767-
return '!', '!'
768-
end
769-
@lex_state = :expr_beg
770-
return '!', '!'
771-
772801
elsif scanner.scan(/\~/)
773802
if @lex_state == :expr_fname
774803
@lex_state = :expr_end

0 commit comments

Comments
 (0)
Please sign in to comment.