@@ -8,6 +8,8 @@ class Lexer
88
99 attr_accessor :lex_state , :strterm , :scanner
1010
11+ attr_accessor :yylval
12+
1113 def initialize ( source , file )
1214 @lex_state = :expr_beg
1315 @cond = 0
@@ -81,6 +83,14 @@ def set_arg_state
8183 end
8284
8385 def scan ( regexp )
86+ if result = @scanner . scan ( regexp )
87+ @yylval += @scanner . matched
88+ end
89+
90+ result
91+ end
92+
93+ def skip_scan ( regexp )
8494 @scanner . scan regexp
8595 end
8696
@@ -97,9 +107,11 @@ def matched
97107 end
98108
99109 def next_token
100- old = self . yylex
101- loc = [ @line , @column ]
102- [ old [ 0 ] , [ old [ 1 ] , loc ] ]
110+ token = self . yylex
111+ value = self . yylval
112+ location = [ @line , @column ]
113+
114+ [ token , [ value , location ] ]
103115 end
104116
105117 def strterm_expand? ( strterm )
@@ -122,15 +134,20 @@ def process_numeric
122134 scanner = @scanner
123135
124136 if scan ( /0b?(0|1|_)+/ )
125- return [ :tINTEGER , scanner . matched . to_i ( 2 ) ]
137+ self . yylval = scanner . matched . to_i ( 2 )
138+ return :tINTEGER
126139 elsif scan ( /0o?([0-7]|_)+/ )
127- return [ :tINTEGER , scanner . matched . to_i ( 8 ) ]
140+ self . yylval = scanner . matched . to_i ( 8 )
141+ return :tINTEGER
128142 elsif scan ( /[\d _]+\. [\d _]+\b |[\d _]+(\. [\d _]+)?[eE][-+]?[\d _]+\b / )
129- return [ :tFLOAT , scanner . matched . gsub ( /_/ , '' ) . to_f ]
143+ self . yylval = scanner . matched . gsub ( /_/ , '' ) . to_f
144+ return :tFLOAT
130145 elsif scan ( /[\d _]+\b / )
131- return [ :tINTEGER , scanner . matched . gsub ( /_/ , '' ) . to_i ]
146+ self . yylval = scanner . matched . gsub ( /_/ , '' ) . to_i
147+ return :tINTEGER
132148 elsif scan ( /0(x|X)(\d |[a-f]|[A-F]|_)+/ )
133- return [ :tINTEGER , scanner . matched . to_i ( 16 ) ]
149+ self . yylval = scanner . matched . to_i ( 16 )
150+ return :tINTEGER
134151 else
135152 raise "Lexing error on numeric type: `#{ scanner . peek 5 }
136153 end
@@ -163,7 +180,7 @@ def next_string_token
163180 end
164181
165182 @lex_state = :expr_end
166- return :tSTRING_END , scanner . matched
183+ return :tSTRING_END
167184 end
168185 end
169186
@@ -173,7 +190,8 @@ def next_string_token
173190 if words && !str_parse [ :done_last_space ] #&& space
174191 str_parse [ :done_last_space ] = true
175192 pushback ( 1 )
176- return :tSPACE , ' '
193+ self . yylval = ' '
194+ return :tSPACE
177195 end
178196 self . strterm = nil
179197
@@ -182,10 +200,10 @@ def next_string_token
182200 @lex_state = :expr_end
183201
184202 if str_parse [ :type ] == :regexp
185- result = scan ( /\w +/ )
186- return :tREGEXP_END , result
203+ self . yylval = scan ( /\w +/ )
204+ return :tREGEXP_END
187205 end
188- return :tSTRING_END , scanner . matched
206+ return :tSTRING_END
189207 else
190208 str_buffer << scanner . matched
191209 str_parse [ :nesting ] -= 1
@@ -194,16 +212,16 @@ def next_string_token
194212
195213 elsif [ '"' , "'" ] . include? str_parse [ :beg ]
196214 @lex_state = :expr_end
197- return :tSTRING_END , scanner . matched
215+ return :tSTRING_END
198216
199217 elsif str_parse [ :beg ] == '`'
200218 @lex_state = :expr_end
201- return :tSTRING_END , scanner . matched
219+ return :tSTRING_END
202220
203221 elsif str_parse [ :beg ] == '/' || str_parse [ :type ] == :regexp
204- result = scan ( /\w +/ )
205222 @lex_state = :expr_end
206- return :tREGEXP_END , result
223+ self . yylval = scan ( /\w +/ )
224+ return :tREGEXP_END
207225
208226 else
209227 if str_parse [ :scanner ]
@@ -212,27 +230,30 @@ def next_string_token
212230 end
213231
214232 @lex_state = :expr_end
215- return :tSTRING_END , scanner . matched
233+ return :tSTRING_END
216234 end
217235 end
218236
219- return :tSPACE , ' ' if space
237+ if space
238+ self . yylval = ' '
239+ return :tSPACE
240+ end
220241
221242 if str_parse [ :balance ] and scan Regexp . new ( Regexp . escape ( str_parse [ :beg ] ) )
222243 str_buffer << scanner . matched
223244 str_parse [ :nesting ] += 1
224245 elsif check ( /#[@$]/ )
225246 scan ( /#/ )
226247 if expand
227- return :tSTRING_DVAR , scanner . matched
248+ return :tSTRING_DVAR
228249 else
229250 str_buffer << scanner . matched
230251 end
231252
232253 elsif scan ( /#\{ / )
233254 if expand
234255 # we are into ruby code, so stop parsing content (for now)
235- return :tSTRING_DBEG , scanner . matched
256+ return :tSTRING_DBEG
236257 else
237258 str_buffer << scanner . matched
238259 end
@@ -250,7 +271,9 @@ def next_string_token
250271
251272 complete_str = str_buffer . join ''
252273 @line += complete_str . count ( "\n " )
253- return :tSTRING_CONTENT , complete_str
274+
275+ self . yylval = complete_str
276+ return :tSTRING_CONTENT
254277 end
255278
256279 def add_heredoc_content ( str_buffer , str_parse )
@@ -402,7 +425,8 @@ def heredoc_identifier
402425 end_of_line = @scanner . scan ( /.*\n / )
403426 self . strterm [ :scanner ] = StringScanner . new ( end_of_line ) if end_of_line != "\n "
404427
405- return :tSTRING_BEG , heredoc
428+ self . yylval = heredoc
429+ return :tSTRING_BEG
406430 end
407431 end
408432
@@ -412,17 +436,18 @@ def process_identifier(matched, cmd_start)
412436
413437 if scanner . peek ( 2 ) != '::' && scan ( /:/ )
414438 @lex_state = :expr_beg
415- return :tLABEL , "#{ matched }
439+ self . yylval = matched
440+ return :tLABEL
416441 end
417442
418443 if matched == 'defined?'
419444 if after_operator?
420445 @lex_state = :expr_end
421- return :tIDENTIFIER , matched
446+ return :tIDENTIFIER
422447 end
423448
424449 @lex_state = :expr_arg
425- return :kDEFINED , 'defined?'
450+ return :kDEFINED
426451 end
427452
428453 if matched . end_with? '?' , '!'
@@ -446,7 +471,8 @@ def process_identifier(matched, cmd_start)
446471 @lex_state = kw . state
447472
448473 if old_state == :expr_fname
449- return [ kw . id [ 0 ] , kw . name ]
474+ self . yylval = kw . name
475+ return kw . id [ 0 ]
450476 end
451477
452478 if @lex_state == :expr_beg
@@ -456,34 +482,36 @@ def process_identifier(matched, cmd_start)
456482 if matched == "do"
457483 if after_operator?
458484 @lex_state = :expr_end
459- return :tIDENTIFIER , matched
485+ return :tIDENTIFIER
460486 end
461487
462488 if @start_of_lambda
463489 @start_of_lambda = false
464490 @lex_state = :expr_beg
465- return [ :kDO_LAMBDA , scanner . matched ]
491+ return :kDO_LAMBDA
466492 elsif cond?
467493 @lex_state = :expr_beg
468- return :kDO_COND , matched
494+ return :kDO_COND
469495 elsif cmdarg? && @lex_state != :expr_cmdarg
470496 @lex_state = :expr_beg
471- return :kDO_BLOCK , matched
497+ return :kDO_BLOCK
472498 elsif @lex_state == :expr_endarg
473- return :kDO_BLOCK , matched
499+ return :kDO_BLOCK
474500 else
475501 @lex_state = :expr_beg
476- return :kDO , matched
502+ return :kDO
477503 end
478504 else
479505 if old_state == :expr_beg or old_state == :expr_value
480- return [ kw . id [ 0 ] , matched ]
506+ self . yylval = matched
507+ return kw . id [ 0 ]
481508 else
482509 if kw . id [ 0 ] != kw . id [ 1 ]
483510 @lex_state = :expr_beg
484511 end
485512
486- return [ kw . id [ 1 ] , matched ]
513+ self . yylval = matched
514+ return kw . id [ 1 ]
487515 end
488516 end
489517 end
@@ -494,10 +522,11 @@ def process_identifier(matched, cmd_start)
494522 @lex_state = :expr_end
495523 end
496524
497- return [ matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER , matched ]
525+ return matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER
498526 end
499527
500528 def yylex
529+ @yylval = ''
501530 @space_seen = false
502531 cmd_start = false
503532 c = ''
@@ -507,20 +536,20 @@ def yylex
507536 end
508537
509538 while true
510- if scan ( /\ |\t |\r / )
539+ if skip_scan ( /\ |\t |\r / )
511540 @space_seen = true
512541 next
513542
514- elsif scan ( /(\n |#)/ )
543+ elsif skip_scan ( /(\n |#)/ )
515544 c = scanner . matched
516- if c == '#' then scan ( /(.*)/ ) else @line += 1 ; end
545+ if c == '#' then skip_scan ( /(.*)/ ) else @line += 1 ; end
517546
518- scan ( /(\n +)/ )
547+ skip_scan ( /(\n +)/ )
519548 @line += scanner . matched . length if scanner . matched
520549
521550 next if [ :expr_beg , :expr_dot ] . include? @lex_state
522551
523- if scan ( /([\ \t \r \f \v ]*)\. / )
552+ if skip_scan ( /([\ \t \r \f \v ]*)\. / )
524553 @space_seen = true unless scanner [ 1 ] . empty?
525554 scanner . pos = scanner . pos - 1
526555
@@ -529,37 +558,40 @@ def yylex
529558
530559 cmd_start = true
531560 @lex_state = :expr_beg
532- return :tNL , '\\n'
561+ self . yylval = '\\n'
562+ return :tNL
533563
534564 elsif scan ( /\; / )
535565 @lex_state = :expr_beg
536- return :tSEMI , ';'
566+ return :tSEMI
537567
538568 elsif check ( /\* / )
539569 if scan ( /\* \* \= / )
540570 @lex_state = :expr_beg
541- return :tOP_ASGN , '**'
571+ self . yylval = '**'
572+ return :tOP_ASGN
542573 elsif scan ( /\* \* / )
543574 self . set_arg_state
544- return :tPOW , '**'
575+ return :tPOW
545576 elsif scan ( /\* \= / )
546577 @lex_state = :expr_beg
547- return :tOP_ASGN , '*'
578+ self . yylval = '*'
579+ return :tOP_ASGN
548580 else
549- result = scan ( /\* / )
581+ scan ( /\* / )
550582
551583 if after_operator?
552584 @lex_state = :expr_arg
553- return :tSTAR2 , result
585+ return :tSTAR2
554586 elsif @space_seen && check ( /\S / )
555587 @lex_state = :expr_beg
556- return :tSTAR , result
588+ return :tSTAR
557589 elsif [ :expr_beg , :expr_mid ] . include? @lex_state
558590 @lex_state = :expr_beg
559- return :tSTAR , result
591+ return :tSTAR
560592 else
561593 @lex_state = :expr_beg
562- return :tSTAR2 , result
594+ return :tSTAR2
563595 end
564596 end
565597
@@ -575,13 +607,13 @@ def yylex
575607 end
576608
577609 if c == '='
578- return :tNEQ , '!='
610+ return :tNEQ
579611 elsif c == '~'
580- return :tNMATCH , '!~'
612+ return :tNMATCH
581613 end
582614
583- scanner . pos = scanner . pos - 1
584- return :tBANG , '!'
615+ pushback ( 1 )
616+ return :tBANG
585617
586618 elsif scan ( /\= / )
587619 if @lex_state == :expr_beg and !@space_seen
@@ -613,45 +645,47 @@ def yylex
613645
614646 if scan ( /\= / )
615647 if scan ( /\= / )
616- return :tEQQ , '==='
648+ return :tEQQ
617649 end
618650
619- return :tEQ , '=='
651+ return :tEQ
620652 end
621653
622654 if scan ( /\~ / )
623- return :tMATCH , '=~'
655+ return :tMATCH
624656 elsif scan ( /\> / )
625- return :tASSOC , '=>'
657+ return :tASSOC
626658 end
627659
628- return :tEQL , '='
660+ return :tEQL
629661
630662 elsif scan ( /\" / )
631663 self . strterm = new_strterm ( :dquote , '"' , '"' )
632- return :tSTRING_BEG , scanner . matched
664+ return :tSTRING_BEG
633665
634666 elsif scan ( /\' / )
635667 self . strterm = new_strterm ( :squote , "'" , "'" )
636- return :tSTRING_BEG , scanner . matched
668+ return :tSTRING_BEG
637669
638670 elsif scan ( /\` / )
639671 self . strterm = new_strterm ( :xquote , '`' , '`' )
640- return :tXSTRING_BEG , scanner . matched
672+ return :tXSTRING_BEG
641673
642674 elsif scan ( /\& / )
643675 if scan ( /\& / )
644676 @lex_state = :expr_beg
645677
646678 if scan ( /\= / )
647- return :tOP_ASGN , '&&'
679+ self . yylval = '&&'
680+ return :tOP_ASGN
648681 end
649682
650- return :tANDOP , '&&'
683+ return :tANDOP
651684
652685 elsif scan ( /\= / )
653686 @lex_state = :expr_beg
654- return :tOP_ASGN , '&'
687+ self . yylval = '&'
688+ return :tOP_ASGN
655689 end
656690
657691 if spcarg?
@@ -665,23 +699,25 @@ def yylex
665699 end
666700
667701 self . set_arg_state
668- return result , '&'
702+ return result
669703
670704 elsif scan ( /\| / )
671705 if scan ( /\| / )
672706 @lex_state = :expr_beg
673707 if scan ( /\= / )
674- return :tOP_ASGN , '||'
708+ self . yylval = '||'
709+ return :tOP_ASGN
675710 end
676711
677- return :tOROP , '||'
712+ return :tOROP
678713
679714 elsif scan ( /\= / )
680- return :tOP_ASGN , '|'
715+ self . yylval = '|'
716+ return :tOP_ASGN
681717 end
682718
683719 self . set_arg_state
684- return :tPIPE , '|'
720+ return :tPIPE
685721
686722 elsif scan ( /\% [QqWwixr]/ )
687723 str_type = scanner . matched [ 1 , 1 ]
@@ -697,62 +733,64 @@ def yylex
697733 case str_type
698734 when 'Q'
699735 self . strterm = new_strterm2 ( :dquote , paren , term )
700- return :tSTRING_BEG , scanner . matched
736+ return :tSTRING_BEG
701737 when 'q'
702738 self . strterm = new_strterm2 ( :squote , paren , term )
703- return :tSTRING_BEG , scanner . matched
739+ return :tSTRING_BEG
704740 when 'W'
705741 self . strterm = new_strterm ( :dword , 'W' , term )
706- scan ( /\s */ )
707- return :tWORDS_BEG , scanner . matched
742+ skip_scan ( /\s */ )
743+ return :tWORDS_BEG
708744 when 'w' , 'i'
709745 self . strterm = new_strterm ( :sword , 'w' , term )
710- scan ( /\s */ )
711- return :tAWORDS_BEG , scanner . matched
746+ skip_scan ( /\s */ )
747+ return :tAWORDS_BEG
712748 when 'x'
713749 self . strterm = new_strterm2 ( :xquote , paren , term )
714- return :tXSTRING_BEG , scanner . matched
750+ return :tXSTRING_BEG
715751 when 'r'
716752 self . strterm = new_strterm2 ( :regexp , paren , term )
717- return :tREGEXP_BEG , scanner . matched
753+ return :tREGEXP_BEG
718754 end
719755
720756 elsif scan ( /\/ / )
721757 if beg?
722758 self . strterm = new_strterm ( :regexp , '/' , '/' )
723- return :tREGEXP_BEG , scanner . matched
759+ return :tREGEXP_BEG
724760 elsif scan ( /\= / )
725761 @lex_state = :expr_beg
726- return :tOP_ASGN , '/'
762+ self . yylval = '/'
763+ return :tOP_ASGN
727764 elsif after_operator?
728765 @lex_state = :expr_arg
729766 elsif arg?
730767 if !check ( /\s / ) && @space_seen
731768 self . strterm = new_strterm ( :regexp , '/' , '/' )
732- return :tREGEXP_BEG , scanner . matched
769+ return :tREGEXP_BEG
733770 end
734771 else
735772 @lex_state = :expr_beg
736773 end
737774
738- return :tDIVIDE , '/'
775+ return :tDIVIDE
739776
740777 elsif scan ( /\% / )
741778 if scan ( /\= / )
742779 @lex_state = :expr_beg
743- return :tOP_ASGN , '%'
780+ self . yylval = '%'
781+ return :tOP_ASGN
744782 elsif check ( /[^\s ]/ )
745783 if @lex_state == :expr_beg or ( @lex_state == :expr_arg && @space_seen )
746784 start_word = scan ( /./ )
747785 end_word = { '(' => ')' , '[' => ']' , '{' => '}' } [ start_word ] || start_word
748786 self . strterm = new_strterm2 ( :dquote , start_word , end_word )
749- return :tSTRING_BEG , scanner . matched
787+ return :tSTRING_BEG
750788 end
751789 end
752790
753791 self . set_arg_state
754792
755- return :tPERCENT , '%'
793+ return :tPERCENT
756794
757795 elsif scan ( /\\ / )
758796 if scan ( /\r ?\n / )
@@ -776,84 +814,84 @@ def yylex
776814 cond_push 0
777815 cmdarg_push 0
778816
779- return result , scanner . matched
817+ return result
780818
781819 elsif scan ( /\) / )
782820 cond_lexpop
783821 cmdarg_lexpop
784822 @lex_state = :expr_end
785- return :tRPAREN , scanner . matched
823+ return :tRPAREN
786824
787825 elsif scan ( /\[ / )
788826 result = scanner . matched
789827
790828 if after_operator?
791829 @lex_state = :expr_arg
792830 if scan ( /\] =/ )
793- return :tASET , '[]='
831+ return :tASET
794832 elsif scan ( /\] / )
795- return :tAREF , '[]'
833+ return :tAREF
796834 else
797835 raise "Unexpected '[' token"
798836 end
799837 elsif beg? || @space_seen
800838 @lex_state = :expr_beg
801839 cond_push 0
802840 cmdarg_push 0
803- return :tLBRACK , scanner . matched
841+ return :tLBRACK
804842 else
805843 @lex_state = :expr_beg
806844 cond_push 0
807845 cmdarg_push 0
808- return :tLBRACK2 , scanner . matched
846+ return :tLBRACK2
809847 end
810848
811849 elsif scan ( /\] / )
812850 cond_lexpop
813851 cmdarg_lexpop
814852 @lex_state = :expr_end
815- return :tRBRACK , scanner . matched
853+ return :tRBRACK
816854
817855 elsif scan ( /\} / )
818856 cond_lexpop
819857 cmdarg_lexpop
820858 @lex_state = :expr_end
821859
822- return :tRCURLY , scanner . matched
860+ return :tRCURLY
823861
824862 elsif scan ( /\. \. \. / )
825863 @lex_state = :expr_beg
826- return :tDOT3 , scanner . matched
864+ return :tDOT3
827865
828866 elsif scan ( /\. \. / )
829867 @lex_state = :expr_beg
830- return :tDOT2 , scanner . matched
868+ return :tDOT2
831869
832870 elsif scan ( /\. / )
833871 @lex_state = :expr_dot unless @lex_state == :expr_fname
834- return :tDOT , scanner . matched
872+ return :tDOT
835873
836874 elsif scan ( /\: \: / )
837875 if beg?
838876 @lex_state = :expr_beg
839- return :tCOLON3 , scanner . matched
877+ return :tCOLON3
840878 elsif spcarg?
841879 @lex_state = :expr_beg
842- return :tCOLON3 , scanner . matched
880+ return :tCOLON3
843881 end
844882
845883 @lex_state = :expr_dot
846- return :tCOLON2 , scanner . matched
884+ return :tCOLON2
847885
848886 elsif scan ( /\: / )
849887 if end? || check ( /\s / )
850888 unless check ( /\w / )
851889 @lex_state = :expr_beg
852- return :tCOLON , ':'
890+ return :tCOLON
853891 end
854892
855893 @lex_state = :expr_fname
856- return :tSYMBEG , ':'
894+ return :tSYMBEG
857895 end
858896
859897 if scan ( /\' / )
@@ -863,33 +901,37 @@ def yylex
863901 end
864902
865903 @lex_state = :expr_fname
866- return :tSYMBEG , ':'
904+ return :tSYMBEG
867905
868906 elsif scan ( /\^ \= / )
869907 @lex_state = :expr_beg
870- return :tOP_ASGN , '^'
908+ self . yylval = '^'
909+ return :tOP_ASGN
910+
871911 elsif scan ( /\^ / )
872912 self . set_arg_state
873- return :tCARET , scanner . matched
913+ return :tCARET
874914
875915 elsif check ( /\< / )
876916 if scan ( /\< \< \= / )
877917 @lex_state = :expr_beg
878- return :tOP_ASGN , '<<'
918+ self . yylval = '<<'
919+ return :tOP_ASGN
920+
879921 elsif scan ( /\< \< / )
880922 if after_operator?
881923 @lex_state = :expr_arg
882- return :tLSHFT , '<<'
924+ return :tLSHFT
883925 elsif !after_operator? && !end? && ( !arg? || @space_seen )
884926 if token = heredoc_identifier
885927 return token
886928 end
887929
888930 @lex_state = :expr_beg
889- return :tLSHFT , '<<'
931+ return :tLSHFT
890932 end
891933 @lex_state = :expr_beg
892- return :tLSHFT , '<<'
934+ return :tLSHFT
893935 elsif scan ( /\< \= \> / )
894936 if after_operator?
895937 @lex_state = :expr_arg
@@ -901,37 +943,39 @@ def yylex
901943 @lex_state = :expr_beg
902944 end
903945
904- return :tCMP , '<=>'
946+ return :tCMP
905947 elsif scan ( /\< \= / )
906948 self . set_arg_state
907- return :tLEQ , '<='
949+ return :tLEQ
908950
909951 elsif scan ( /\< / )
910952 self . set_arg_state
911- return :tLT , '<'
953+ return :tLT
912954 end
913955
914956 elsif check ( /\> / )
915957 if scan ( /\> \> \= / )
916- return :tOP_ASGN , '>>'
958+ self . yylval = '>>'
959+ return :tOP_ASGN
960+
917961 elsif scan ( /\> \> / )
918962 self . set_arg_state
919- return :tRSHFT , '>>'
963+ return :tRSHFT
920964
921965 elsif scan ( /\> \= / )
922966 self . set_arg_state
923- return :tGEQ , scanner . matched
967+ return :tGEQ
924968
925969 elsif scan ( /\> / )
926970 self . set_arg_state
927- return :tGT , '>'
971+ return :tGT
928972 end
929973
930974 elsif scan ( /->/ )
931975 # FIXME: # should be :expr_arg, but '(' breaks it...
932976 @lex_state = :expr_end
933977 @start_of_lambda = true
934- return [ :tLAMBDA , scanner . matched ]
978+ return :tLAMBDA
935979
936980 elsif scan ( /[+-]/ )
937981 matched = scanner . matched
@@ -943,84 +987,95 @@ def yylex
943987
944988 if beg?
945989 @lex_state = :expr_mid
946- return [ utype , matched ]
990+ self . yylval = matched
991+ return utype
947992 elsif after_operator?
948993 @lex_state = :expr_arg
949- return [ :tIDENTIFIER , matched + '@' ] if scan ( /@/ )
950- return [ sign , matched ]
994+ if scan ( /@/ )
995+ self . yylval = matched + '@'
996+ return :tIDENTIFIER
997+ end
998+
999+ self . yylval = matched
1000+ return sign
9511001 end
9521002
9531003 if scan ( /\= / )
9541004 @lex_state = :expr_beg
955- return [ :tOP_ASGN , matched ]
1005+ self . yylval = matched
1006+ return :tOP_ASGN
9561007 end
9571008
9581009 if spcarg?
9591010 @lex_state = :expr_mid
960- return [ utype , matched ]
1011+ self . yylval = matched
1012+ return utype
9611013 end
9621014
9631015 @lex_state = :expr_beg
964- return [ sign , matched ]
1016+ self . yylval = matched
1017+ return sign
9651018
9661019 elsif scan ( /\? / )
9671020 if end?
9681021 @lex_state = :expr_beg
969- return :tEH , scanner . matched
1022+ return :tEH
9701023 end
9711024
9721025 unless check ( /\ |\t |\r |\s / )
9731026 @lex_state = :expr_end
974- return :tSTRING , scan ( /./ )
1027+ self . yylval = scan ( /./ )
1028+ return :tSTRING
9751029 end
9761030
9771031 @lex_state = :expr_beg
978- return :tEH , scanner . matched
1032+ return :tEH
9791033
9801034 elsif scan ( /\~ / )
9811035 self . set_arg_state
982- return :tTILDE , '~'
1036+ return :tTILDE
9831037
9841038 elsif check ( /\$ / )
9851039 if scan ( /\$ ([1-9]\d *)/ )
9861040 @lex_state = :expr_end
987- return :tNTH_REF , scanner . matched . sub ( '$' , '' )
1041+ self . yylval = scanner . matched . sub ( '$' , '' )
1042+ return :tNTH_REF
9881043
9891044 elsif scan ( /(\$ _)(\w +)/ )
9901045 @lex_state = :expr_end
991- return :tGVAR , scanner . matched
1046+ return :tGVAR
9921047
9931048 elsif scan ( /\$ [\+ \' \` \& !@\" ~*$?\/ \\ :;=.,<>_]/ )
9941049 @lex_state = :expr_end
995- return :tGVAR , scanner . matched
1050+ return :tGVAR
9961051 elsif scan ( /\$ \w +/ )
9971052 @lex_state = :expr_end
998- return :tGVAR , scanner . matched
1053+ return :tGVAR
9991054 else
10001055 raise "Bad gvar name: #{ scanner . peek ( 5 ) . inspect }
10011056 end
10021057
10031058 elsif scan ( /\$ \w +/ )
10041059 @lex_state = :expr_end
1005- return :tGVAR , scanner . matched
1060+ return :tGVAR
10061061
10071062 elsif scan ( /\@ \@ \w */ )
10081063 @lex_state = :expr_end
1009- return :tCVAR , scanner . matched
1064+ return :tCVAR
10101065
10111066 elsif scan ( /\@ \w */ )
10121067 @lex_state = :expr_end
1013- return :tIVAR , scanner . matched
1068+ return :tIVAR
10141069
10151070 elsif scan ( /\, / )
10161071 @lex_state = :expr_beg
1017- return :tCOMMA , scanner . matched
1072+ return :tCOMMA
10181073
10191074 elsif scan ( /\{ / )
10201075 if @start_of_lambda
10211076 @start_of_lambda = false
10221077 @lex_state = :expr_beg
1023- return [ :tLAMBEG , scanner . matched ]
1078+ return :tLAMBEG
10241079
10251080 elsif arg? or @lex_state == :expr_end
10261081 result = :tLCURLY
@@ -1033,7 +1088,7 @@ def yylex
10331088 @lex_state = :expr_beg
10341089 cond_push 0
10351090 cmdarg_push 0
1036- return result , scanner . matched
1091+ return result
10371092
10381093 elsif check ( /[0-9]/ )
10391094 return process_numeric
@@ -1044,7 +1099,8 @@ def yylex
10441099
10451100 if scanner . eos?
10461101 if @scanner_stack . size == 1 # our main scanner, we cant pop this
1047- return [ false , false ]
1102+ self . yylval = false
1103+ return false
10481104 else # we were probably parsing a heredoc, so pop that parser and continue
10491105 @scanner_stack . pop
10501106 @scanner = @scanner_stack . last
0 commit comments