Skip to content

Commit b9d776e

Browse files
adambeynonadambeynon
committedNov 12, 2013
Use 't' prefix for parser tokens more consistently
·
v1.8.3.rc1lerna-0.0.1-alpha.2
1 parent e9b8a97 commit b9d776e

File tree

3 files changed

+244
-244
lines changed

3 files changed

+244
-244
lines changed
 

‎lib/opal/parser/grammar.rb

+66-66
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/opal/parser/grammar.y

+123-123
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,16 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
55
kREDO kRETRY kIN kDO kDO_COND kDO_BLOCK kRETURN kYIELD kSUPER
66
kSELF kNIL kTRUE kFALSE kAND kOR kNOT kIF_MOD kUNLESS_MOD kWHILE_MOD
77
kUNTIL_MOD kRESCUE_MOD kALIAS kDEFINED klBEGIN klEND k__LINE__
8-
k__FILE__ IDENTIFIER FID GVAR IVAR CONSTANT CVAR NTH_REF
9-
BACK_REF STRING_CONTENT INTEGER FLOAT REGEXP_END '+@'
8+
k__FILE__ tIDENTIFIER tFID tGVAR tIVAR tCONSTANT tCVAR tNTH_REF
9+
tBACK_REF tSTRING_CONTENT tINTEGER tFLOAT tREGEXP_END '+@'
1010
'-@' '-@NUM' tPOW tCMP tEQ tEQQ tNEQ tGEQ tLEQ tANDOP
1111
tOROP tMATCH tNMATCH '.' tDOT2 tDOT3 '[]' '[]=' tLSHFT tRSHFT
12-
'::' '::@' tOP_ASGN '=>' PAREN_BEG '(' ')' tLPAREN_ARG
13-
ARRAY_BEG ']' tLBRACE tLBRACE_ARG SPLAT tSTAR2 '&@' tAMPER2
12+
'::' '::@' tOP_ASGN tASSOC tLPAREN '(' ')' tLPAREN_ARG
13+
ARRAY_BEG ']' tLBRACE tLBRACE_ARG tSTAR tSTAR2 '&@' tAMPER2
1414
tTILDE tPERCENT tDIVIDE '+' '-' tLT tGT tPIPE tBANG tCARET
15-
LCURLY '}' BACK_REF2 SYMBOL_BEG STRING_BEG XSTRING_BEG REGEXP_BEG
16-
WORDS_BEG AWORDS_BEG STRING_DBEG STRING_DVAR STRING_END STRING
17-
SYMBOL '\\n' tEH tCOLON ',' SPACE ';' LABEL LAMBDA LAMBEG kDO_LAMBDA
15+
tLCURLY '}' tBACK_REF2 tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG
16+
tWORDS_BEG tAWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END tSTRING
17+
tSYMBOL '\\n' tEH tCOLON ',' tSPACE ';' tLABEL tLAMBDA tLAMBEG kDO_LAMBDA
1818

1919
prechigh
2020
right tBANG tTILDE '+@'
@@ -38,7 +38,7 @@ prechigh
3838
left kOR kAND
3939
nonassoc kIF_MOD kUNLESS_MOD kWHILE_MOD kUNTIL_MOD
4040
nonassoc tLBRACE_ARG
41-
nonassoc LOWEST
41+
nonassoc tLOWEST
4242
preclow
4343

4444
rule
@@ -86,12 +86,12 @@ rule
8686
{
8787
result = s(:alias, val[1], val[3])
8888
}
89-
| kALIAS GVAR GVAR
89+
| kALIAS tGVAR tGVAR
9090
{
9191
result = s(:valias, val[1].intern, val[2].intern)
9292
}
93-
| kALIAS GVAR BACK_REF
94-
| kALIAS GVAR NTH_REF
93+
| kALIAS tGVAR tBACK_REF
94+
| kALIAS tGVAR tNTH_REF
9595
{
9696
result = s(:valias, val[1].intern, val[2].intern)
9797
}
@@ -119,8 +119,8 @@ rule
119119
{
120120
result = s(:rescue_mod, val[0], val[2])
121121
}
122-
| klBEGIN LCURLY compstmt '}'
123-
| klEND LCURLY compstmt '}'
122+
| klBEGIN tLCURLY compstmt '}'
123+
| klEND tLCURLY compstmt '}'
124124
| lhs tEQL command_call
125125
{
126126
result = new_assign val[0], val[2]
@@ -134,12 +134,12 @@ rule
134134
result = new_op_asgn val[1].intern, val[0], val[2]
135135
}
136136
| primary_value '[@' aref_args ']' tOP_ASGN command_call
137-
| primary_value '.' IDENTIFIER tOP_ASGN command_call
137+
| primary_value '.' tIDENTIFIER tOP_ASGN command_call
138138
{
139139
result = s(:op_asgn2, val[0], "#{val[2]}=".intern, val[3].intern, val[4])
140140
}
141-
| primary_value '.' CONSTANT tOP_ASGN command_call
142-
| primary_value '::' IDENTIFIER tOP_ASGN command_call
141+
| primary_value '.' tCONSTANT tOP_ASGN command_call
142+
| primary_value '::' tIDENTIFIER tOP_ASGN command_call
143143
| backref tOP_ASGN command_call
144144
| lhs tEQL mrhs
145145
{
@@ -206,17 +206,17 @@ rule
206206

207207
cmd_brace_block: tLBRACE_ARG opt_block_var compstmt '}'
208208

209-
command: operation command_args =LOWEST
209+
command: operation command_args =tLOWEST
210210
{
211211
result = new_call nil, val[0].intern, val[1]
212212
}
213213
| operation command_args cmd_brace_block
214-
| primary_value '.' operation2 command_args =LOWEST
214+
| primary_value '.' operation2 command_args =tLOWEST
215215
{
216216
result = new_call val[0], val[2].intern, val[3]
217217
}
218218
| primary_value '.' operation2 command_args cmd_brace_block
219-
| primary_value '::' operation2 command_args =LOWEST
219+
| primary_value '::' operation2 command_args =tLOWEST
220220
{
221221
result = new_call val[0], val[2].intern, val[3]
222222
}
@@ -234,7 +234,7 @@ rule
234234
{
235235
result = val[0]
236236
}
237-
| PAREN_BEG mlhs_entry ')'
237+
| tLPAREN mlhs_entry ')'
238238
{
239239
result = val[1]
240240
}
@@ -243,7 +243,7 @@ rule
243243
{
244244
result = val[0]
245245
}
246-
| PAREN_BEG mlhs_entry ')'
246+
| tLPAREN mlhs_entry ')'
247247
{
248248
result = val[1]
249249
}
@@ -256,31 +256,31 @@ rule
256256
{
257257
result = val[0] << val[1]
258258
}
259-
| mlhs_head SPLAT mlhs_node
259+
| mlhs_head tSTAR mlhs_node
260260
{
261261
result = val[0] << s(:splat, val[2])
262262
}
263-
| mlhs_head SPLAT mlhs_node ',' mlhs_post
264-
| mlhs_head SPLAT
263+
| mlhs_head tSTAR mlhs_node ',' mlhs_post
264+
| mlhs_head tSTAR
265265
{
266266
result = val[0] << s(:splat)
267267
}
268-
| mlhs_head SPLAT ',' mlhs_post
269-
| SPLAT mlhs_node
268+
| mlhs_head tSTAR ',' mlhs_post
269+
| tSTAR mlhs_node
270270
{
271271
result = s(:array, s(:splat, val[1]))
272272
}
273-
| SPLAT
273+
| tSTAR
274274
{
275275
result = s(:array, s(:splat))
276276
}
277-
| SPLAT ',' mlhs_post
277+
| tSTAR ',' mlhs_post
278278

279279
mlhs_item: mlhs_node
280280
{
281281
result = val[0]
282282
}
283-
| PAREN_BEG mlhs_entry ')'
283+
| tLPAREN mlhs_entry ')'
284284
{
285285
result = val[1]
286286
}
@@ -307,14 +307,14 @@ rule
307307
args.type = :arglist if args.type == :array
308308
result = s(:attrasgn, val[0], :[]=, args)
309309
}
310-
| primary_value '.' IDENTIFIER
310+
| primary_value '.' tIDENTIFIER
311311
{
312312
result = new_call val[0], val[2].intern, s(:arglist)
313313
}
314-
| primary_value '::' IDENTIFIER
315-
| primary_value '.' CONSTANT
316-
| primary_value '::' CONSTANT
317-
| '::@' CONSTANT
314+
| primary_value '::' tIDENTIFIER
315+
| primary_value '.' tCONSTANT
316+
| primary_value '::' tCONSTANT
317+
| '::@' tCONSTANT
318318
| backref
319319

320320
lhs: variable
@@ -327,29 +327,29 @@ rule
327327
args.type = :arglist if args.type == :array
328328
result = s(:attrasgn, val[0], :[]=, args)
329329
}
330-
| primary_value '.' IDENTIFIER
330+
| primary_value '.' tIDENTIFIER
331331
{
332332
result = s(:attrasgn, val[0], "#{val[2]}=".intern, s(:arglist))
333333
}
334-
| primary_value '::' IDENTIFIER
334+
| primary_value '::' tIDENTIFIER
335335
{
336336
result = s(:attrasgn, val[0], "#{val[2]}=".intern, s(:arglist))
337337
}
338-
| primary_value '.' CONSTANT
338+
| primary_value '.' tCONSTANT
339339
{
340340
result = s(:attrasgn, val[0], "#{val[2]}=".intern, s(:arglist))
341341
}
342-
| primary_value '::' CONSTANT
342+
| primary_value '::' tCONSTANT
343343
{
344344
result = s(:colon2, val[0], val[2].intern)
345345
}
346-
| '::@' CONSTANT
346+
| '::@' tCONSTANT
347347
{
348348
result = s(:colon3, val[1].intern)
349349
}
350350
| backref
351351

352-
cname: CONSTANT
352+
cname: tCONSTANT
353353

354354
cpath: '::@' cname
355355
{
@@ -364,9 +364,9 @@ rule
364364
result = s(:colon2, val[0], val[2].intern)
365365
}
366366

367-
fname: IDENTIFIER
368-
| CONSTANT
369-
| FID
367+
fname: tIDENTIFIER
368+
| tCONSTANT
369+
| tFID
370370
| op
371371
{
372372
lexer.lex_state = :expr_end
@@ -395,9 +395,9 @@ rule
395395

396396
op: tPIPE | tCARET | tAMPER2 | tCMP | tEQ | tEQQ
397397
| tMATCH | tGT | tGEQ | tLT | tLEQ | tLSHFT
398-
| tRSHFT | '+' | '-' | tSTAR2 | SPLAT | tDIVIDE
398+
| tRSHFT | '+' | '-' | tSTAR2 | tSTAR | tDIVIDE
399399
| tPERCENT | tPOW | tTILDE | '+@' | '-@' | '[]'
400-
| '[]=' | BACK_REF2 | tBANG | tNEQ
400+
| '[]=' | tBACK_REF2 | tBANG | tNEQ
401401

402402
reswords: k__LINE__ | k__FILE__ | klBEGIN | klEND | kALIAS | kAND
403403
| kBEGIN | kBREAK | kCASE | kCLASS | kDEF | kDEFINED
@@ -427,14 +427,14 @@ rule
427427
result = s(:op_asgn1, val[0], val[2], val[4].intern, val[5])
428428
result.line = val[0].line
429429
}
430-
| primary_value '.' IDENTIFIER tOP_ASGN arg
430+
| primary_value '.' tIDENTIFIER tOP_ASGN arg
431431
{
432432
result = s(:op_asgn2, val[0], "#{val[2]}=".intern, val[3].intern, val[4])
433433
}
434-
| primary_value '.' CONSTANT tOP_ASGN arg
435-
| primary_value '::' IDENTIFIER tOP_ASGN arg
436-
| primary_value '::' CONSTANT tOP_ASGN arg
437-
| '::@' CONSTANT tOP_ASGN arg
434+
| primary_value '.' tCONSTANT tOP_ASGN arg
435+
| primary_value '::' tIDENTIFIER tOP_ASGN arg
436+
| primary_value '::' tCONSTANT tOP_ASGN arg
437+
| '::@' tCONSTANT tOP_ASGN arg
438438
| backref tOP_ASGN arg
439439
| arg tDOT2 arg
440440
{
@@ -470,8 +470,8 @@ rule
470470
{
471471
result = new_call val[0], :"**", s(:arglist, val[2])
472472
}
473-
| '-@NUM' INTEGER tPOW arg
474-
| '-@NUM' FLOAT tPOW arg
473+
| '-@NUM' tINTEGER tPOW arg
474+
| '-@NUM' tFLOAT tPOW arg
475475
| '+@' arg
476476
{
477477
result = new_call val[1], :"+@", s(:arglist)
@@ -680,15 +680,15 @@ rule
680680
{
681681
result = s(:array, val[0])
682682
}
683-
| SPLAT arg_value
683+
| tSTAR arg_value
684684
{
685685
result = s(:array, s(:splat, val[1]))
686686
}
687687
| args ',' arg_value
688688
{
689689
result = val[0] << val[2]
690690
}
691-
| args ',' SPLAT arg_value
691+
| args ',' tSTAR arg_value
692692
{
693693
result = val[0] << s(:splat, val[3])
694694
}
@@ -698,8 +698,8 @@ rule
698698
val[0] << val[2]
699699
result = val[0]
700700
}
701-
| args ',' SPLAT arg_value
702-
| SPLAT arg_value
701+
| args ',' tSTAR arg_value
702+
| tSTAR arg_value
703703
{
704704
result = s(:splat, val[1])
705705
}
@@ -712,7 +712,7 @@ rule
712712
| awords
713713
| var_ref
714714
| backref
715-
| FID
715+
| tFID
716716
| kBEGIN
717717
{
718718
result = lexer.line
@@ -726,15 +726,15 @@ rule
726726
{
727727
result = val[1]
728728
}
729-
| PAREN_BEG compstmt ')'
729+
| tLPAREN compstmt ')'
730730
{
731731
result = s(:paren, val[1] || s(:nil))
732732
}
733-
| primary_value '::' CONSTANT
733+
| primary_value '::' tCONSTANT
734734
{
735735
result = s(:colon2, val[0], val[2].intern)
736736
}
737-
| '::@' CONSTANT
737+
| '::@' tCONSTANT
738738
{
739739
result = s(:colon3, val[1])
740740
}
@@ -790,7 +790,7 @@ rule
790790
val[0] << val[1]
791791
result = val[0]
792792
}
793-
| LAMBDA lambda
793+
| tLAMBDA lambda
794794
{
795795
result = val[1]
796796
}
@@ -961,7 +961,7 @@ rule
961961
| block_param
962962
| none
963963

964-
lambda_body: LAMBEG compstmt '}'
964+
lambda_body: tLAMBEG compstmt '}'
965965
{
966966
result = val[1]
967967
}
@@ -1000,7 +1000,7 @@ rule
10001000
result = val[0]
10011001
}
10021002

1003-
f_block_opt: IDENTIFIER tEQL primary_value
1003+
f_block_opt: tIDENTIFIER tEQL primary_value
10041004
{
10051005
result = new_assign new_assignable(s(:identifier, val[0].intern)), val[2]
10061006
}
@@ -1119,7 +1119,7 @@ opt_block_args_tail: ',' block_args_tail
11191119
result = s(:super, nil)
11201120
}
11211121

1122-
brace_block: LCURLY
1122+
brace_block: tLCURLY
11231123
{
11241124
push_scope :block
11251125
result = lexer.line
@@ -1179,7 +1179,7 @@ opt_block_args_tail: ',' block_args_tail
11791179
| mrhs
11801180
| none
11811181

1182-
exc_var: '=>' lhs
1182+
exc_var: tASSOC lhs
11831183
{
11841184
result = val[1]
11851185
}
@@ -1206,30 +1206,30 @@ opt_block_args_tail: ',' block_args_tail
12061206
string: string1
12071207
| string string1
12081208

1209-
string1: STRING_BEG string_contents STRING_END
1209+
string1: tSTRING_BEG string_contents tSTRING_END
12101210
{
12111211
result = val[1]
12121212
}
1213-
| STRING
1213+
| tSTRING
12141214
{
12151215
result = s(:str, val[0])
12161216
}
12171217

1218-
xstring: XSTRING_BEG xstring_contents STRING_END
1218+
xstring: tXSTRING_BEG xstring_contents tSTRING_END
12191219
{
12201220
result = new_xstr val[1]
12211221
}
12221222

1223-
regexp: REGEXP_BEG xstring_contents REGEXP_END
1223+
regexp: tREGEXP_BEG xstring_contents tREGEXP_END
12241224
{
12251225
result = new_regexp val[1], val[2]
12261226
}
12271227

1228-
words: WORDS_BEG SPACE STRING_END
1228+
words: tWORDS_BEG tSPACE tSTRING_END
12291229
{
12301230
result = s(:array)
12311231
}
1232-
| WORDS_BEG word_list STRING_END
1232+
| tWORDS_BEG word_list tSTRING_END
12331233
{
12341234
result = val[1]
12351235
}
@@ -1238,7 +1238,7 @@ opt_block_args_tail: ',' block_args_tail
12381238
{
12391239
result = s(:array)
12401240
}
1241-
| word_list word SPACE
1241+
| word_list word tSPACE
12421242
{
12431243
part = val[1]
12441244
part = s(:dstr, "", val[1]) if part.type == :evstr
@@ -1254,11 +1254,11 @@ opt_block_args_tail: ',' block_args_tail
12541254
result = val[0].concat([val[1]])
12551255
}
12561256

1257-
awords: AWORDS_BEG SPACE STRING_END
1257+
awords: tAWORDS_BEG tSPACE tSTRING_END
12581258
{
12591259
result = s(:array)
12601260
}
1261-
| AWORDS_BEG qword_list STRING_END
1261+
| tAWORDS_BEG qword_list tSTRING_END
12621262
{
12631263
result = val[1]
12641264
}
@@ -1267,7 +1267,7 @@ opt_block_args_tail: ',' block_args_tail
12671267
{
12681268
result = s(:array)
12691269
}
1270-
| qword_list STRING_CONTENT SPACE
1270+
| qword_list tSTRING_CONTENT tSPACE
12711271
{
12721272
result = val[0] << s(:str, val[1])
12731273
}
@@ -1290,11 +1290,11 @@ xstring_contents: none
12901290
result = str_append val[0], val[1]
12911291
}
12921292

1293-
string_content: STRING_CONTENT
1293+
string_content: tSTRING_CONTENT
12941294
{
12951295
result = s(:str, val[0])
12961296
}
1297-
| STRING_DVAR
1297+
| tSTRING_DVAR
12981298
{
12991299
result = lexer.strterm
13001300
lexer.strterm = nil
@@ -1304,7 +1304,7 @@ xstring_contents: none
13041304
lexer.strterm = val[1]
13051305
result = s(:evstr, val[2])
13061306
}
1307-
| STRING_DBEG
1307+
| tSTRING_DBEG
13081308
{
13091309
lexer.cond_push 0
13101310
lexer.cmdarg_push 0
@@ -1320,69 +1320,69 @@ xstring_contents: none
13201320
result = s(:evstr, val[2])
13211321
}
13221322

1323-
string_dvar: GVAR
1323+
string_dvar: tGVAR
13241324
{
13251325
result = s(:gvar, val[0].intern)
13261326
}
1327-
| IVAR
1327+
| tIVAR
13281328
{
13291329
result = s(:ivar, val[0].intern)
13301330
}
1331-
| CVAR
1331+
| tCVAR
13321332
{
13331333
result = s(:cvar, val[0].intern)
13341334
}
13351335
| backref
13361336

13371337

1338-
symbol: SYMBOL_BEG sym
1338+
symbol: tSYMBEG sym
13391339
{
13401340
result = s(:sym, val[1].intern)
13411341
lexer.lex_state = :expr_end
13421342
}
1343-
| SYMBOL
1343+
| tSYMBOL
13441344
{
13451345
result = s(:sym, val[0].intern)
13461346
}
13471347

13481348
sym: fname
1349-
| IVAR
1350-
| GVAR
1351-
| CVAR
1349+
| tIVAR
1350+
| tGVAR
1351+
| tCVAR
13521352

1353-
dsym: SYMBOL_BEG xstring_contents STRING_END
1353+
dsym: tSYMBEG xstring_contents tSTRING_END
13541354
{
13551355
result = new_dsym val[1]
13561356
}
13571357

1358-
numeric: INTEGER
1358+
numeric: tINTEGER
13591359
{
13601360
result = s(:int, val[0])
13611361
}
1362-
| FLOAT
1362+
| tFLOAT
13631363
{
13641364
result = s(:float, val[0])
13651365
}
1366-
| '-@NUM' INTEGER =LOWEST
1367-
| '-@NUM' FLOAT =LOWEST
1366+
| '-@NUM' tINTEGER =tLOWEST
1367+
| '-@NUM' tFLOAT =tLOWEST
13681368

1369-
variable: IDENTIFIER
1369+
variable: tIDENTIFIER
13701370
{
13711371
result = s(:identifier, val[0].intern)
13721372
}
1373-
| IVAR
1373+
| tIVAR
13741374
{
13751375
result = s(:ivar, val[0].intern)
13761376
}
1377-
| GVAR
1377+
| tGVAR
13781378
{
13791379
result = s(:gvar, val[0].intern)
13801380
}
1381-
| CONSTANT
1381+
| tCONSTANT
13821382
{
13831383
result = s(:const, val[0].intern)
13841384
}
1385-
| CVAR
1385+
| tCVAR
13861386
{
13871387
result = s(:cvar, val[0].intern)
13881388
}
@@ -1421,11 +1421,11 @@ xstring_contents: none
14211421
result = new_assignable val[0]
14221422
}
14231423

1424-
backref: NTH_REF
1424+
backref: tNTH_REF
14251425
{
14261426
result = s(:nth_ref, val[0])
14271427
}
1428-
| BACK_REF
1428+
| tBACK_REF
14291429

14301430
superclass: term
14311431
{
@@ -1487,23 +1487,23 @@ xstring_contents: none
14871487
result = s(:args)
14881488
}
14891489

1490-
f_norm_arg: CONSTANT
1490+
f_norm_arg: tCONSTANT
14911491
{
14921492
raise 'formal argument cannot be a constant'
14931493
}
1494-
| IVAR
1494+
| tIVAR
14951495
{
14961496
raise 'formal argument cannot be an instance variable'
14971497
}
1498-
| CVAR
1498+
| tCVAR
14991499
{
15001500
raise 'formal argument cannot be a class variable'
15011501
}
1502-
| GVAR
1502+
| tGVAR
15031503
{
15041504
raise 'formal argument cannot be a global variable'
15051505
}
1506-
| IDENTIFIER
1506+
| tIDENTIFIER
15071507
{
15081508
result = val[0].intern
15091509
scope.add_local result
@@ -1513,7 +1513,7 @@ xstring_contents: none
15131513
{
15141514
result = val[0]
15151515
}
1516-
| PAREN_BEG f_margs ')'
1516+
| tLPAREN f_margs ')'
15171517
{
15181518
result = val[1]
15191519
}
@@ -1522,7 +1522,7 @@ xstring_contents: none
15221522
{
15231523
result = s(:lasgn, val[0])
15241524
}
1525-
| PAREN_BEG f_margs ')'
1525+
| tLPAREN f_margs ')'
15261526

15271527
f_marg_list: f_marg
15281528
{
@@ -1535,10 +1535,10 @@ xstring_contents: none
15351535
}
15361536

15371537
f_margs: f_marg_list
1538-
| f_marg_list ',' SPLAT f_norm_arg
1539-
| f_marg_list ',' SPLAT
1540-
| SPLAT f_norm_arg
1541-
| SPLAT
1538+
| f_marg_list ',' tSTAR f_norm_arg
1539+
| f_marg_list ',' tSTAR
1540+
| tSTAR f_norm_arg
1541+
| tSTAR
15421542

15431543
f_arg: f_arg_item
15441544
{
@@ -1550,7 +1550,7 @@ xstring_contents: none
15501550
result = val[0]
15511551
}
15521552

1553-
f_opt: IDENTIFIER tEQL arg_value
1553+
f_opt: tIDENTIFIER tEQL arg_value
15541554
{
15551555
result = new_assign new_assignable(s(:identifier, val[0].intern)), val[2]
15561556
}
@@ -1566,9 +1566,9 @@ xstring_contents: none
15661566
}
15671567

15681568
restarg_mark: tSTAR2
1569-
| SPLAT
1569+
| tSTAR
15701570

1571-
f_rest_arg: restarg_mark IDENTIFIER
1571+
f_rest_arg: restarg_mark tIDENTIFIER
15721572
{
15731573
result = "*#{val[1]}".intern
15741574
}
@@ -1580,7 +1580,7 @@ xstring_contents: none
15801580
blkarg_mark: tAMPER2
15811581
| '&@'
15821582

1583-
f_block_arg: blkarg_mark IDENTIFIER
1583+
f_block_arg: blkarg_mark tIDENTIFIER
15841584
{
15851585
result = "&#{val[1]}".intern
15861586
}
@@ -1625,26 +1625,26 @@ xstring_contents: none
16251625
result = val[0].push *val[2]
16261626
}
16271627

1628-
assoc: arg_value '=>' arg_value
1628+
assoc: arg_value tASSOC arg_value
16291629
{
16301630
result = [val[0], val[2]]
16311631
}
1632-
| LABEL arg_value
1632+
| tLABEL arg_value
16331633
{
16341634
result = [s(:sym, val[0].intern), val[1]]
16351635
}
16361636

1637-
operation: IDENTIFIER
1638-
| CONSTANT
1639-
| FID
1637+
operation: tIDENTIFIER
1638+
| tCONSTANT
1639+
| tFID
16401640

1641-
operation2: IDENTIFIER
1642-
| CONSTANT
1643-
| FID
1641+
operation2: tIDENTIFIER
1642+
| tCONSTANT
1643+
| tFID
16441644
| op
16451645

1646-
operation3: IDENTIFIER
1647-
| FID
1646+
operation3: tIDENTIFIER
1647+
| tFID
16481648
| op
16491649

16501650
dot_or_colon: '.'

‎lib/opal/parser/lexer.rb

+55-55
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ def next_string_token
112112
end
113113

114114
@lex_state = :expr_end
115-
return :STRING_END, scanner.matched
115+
return :tSTRING_END, scanner.matched
116116
end
117117
end
118118

@@ -122,7 +122,7 @@ def next_string_token
122122
if words && !str_parse[:done_last_space]#&& space
123123
str_parse[:done_last_space] = true
124124
scanner.pos -= 1
125-
return :SPACE, ' '
125+
return :tSPACE, ' '
126126
end
127127
self.strterm = nil
128128

@@ -132,9 +132,9 @@ def next_string_token
132132

133133
if str_parse[:regexp]
134134
result = scanner.scan(/\w+/)
135-
return :REGEXP_END, result
135+
return :tREGEXP_END, result
136136
end
137-
return :STRING_END, scanner.matched
137+
return :tSTRING_END, scanner.matched
138138
else
139139
str_buffer << scanner.matched
140140
str_parse[:nesting] -= 1
@@ -143,16 +143,16 @@ def next_string_token
143143

144144
elsif ['"', "'"].include? str_parse[:beg]
145145
@lex_state = :expr_end
146-
return :STRING_END, scanner.matched
146+
return :tSTRING_END, scanner.matched
147147

148148
elsif str_parse[:beg] == '`'
149149
@lex_state = :expr_end
150-
return :STRING_END, scanner.matched
150+
return :tSTRING_END, scanner.matched
151151

152152
elsif str_parse[:beg] == '/' || str_parse[:regexp]
153153
result = scanner.scan(/\w+/)
154154
@lex_state = :expr_end
155-
return :REGEXP_END, result
155+
return :tREGEXP_END, result
156156

157157
else
158158
if str_parse[:scanner]
@@ -161,27 +161,27 @@ def next_string_token
161161
end
162162

163163
@lex_state = :expr_end
164-
return :STRING_END, scanner.matched
164+
return :tSTRING_END, scanner.matched
165165
end
166166
end
167167

168-
return :SPACE, ' ' if space
168+
return :tSPACE, ' ' if space
169169

170170
if str_parse[:balance] and scanner.scan Regexp.new(Regexp.escape(str_parse[:beg]))
171171
str_buffer << scanner.matched
172172
str_parse[:nesting] += 1
173173
elsif scanner.check(/#[@$]/)
174174
scanner.scan(/#/)
175175
if expand
176-
return :STRING_DVAR, scanner.matched
176+
return :tSTRING_DVAR, scanner.matched
177177
else
178178
str_buffer << scanner.matched
179179
end
180180

181181
elsif scanner.scan(/#\{/)
182182
if expand
183183
# we are into ruby code, so stop parsing content (for now)
184-
return :STRING_DBEG, scanner.matched
184+
return :tSTRING_DBEG, scanner.matched
185185
else
186186
str_buffer << scanner.matched
187187
end
@@ -199,7 +199,7 @@ def next_string_token
199199

200200
complete_str = str_buffer.join ''
201201
@line += complete_str.count("\n")
202-
return :STRING_CONTENT, complete_str
202+
return :tSTRING_CONTENT, complete_str
203203
end
204204

205205
def add_heredoc_content(str_buffer, str_parse)
@@ -351,7 +351,7 @@ def heredoc_identifier
351351
end_of_line = @scanner.scan(/.*\n/)
352352
self.strterm[:scanner] = StringScanner.new(end_of_line) if end_of_line != "\n"
353353

354-
return :STRING_BEG, heredoc
354+
return :tSTRING_BEG, heredoc
355355
end
356356
end
357357

@@ -361,32 +361,32 @@ def process_identifier(matched, cmd_start)
361361

362362
if scanner.peek(2) != '::' && scanner.scan(/:/)
363363
@lex_state = :expr_beg
364-
return :LABEL, "#{matched}"
364+
return :tLABEL, "#{matched}"
365365
end
366366

367367
if matched == 'defined?'
368368
if after_operator?
369369
@lex_state = :expr_end
370-
return :IDENTIFIER, matched
370+
return :tIDENTIFIER, matched
371371
end
372372

373373
@lex_state = :expr_arg
374374
return :kDEFINED, 'defined?'
375375
end
376376

377377
if matched.end_with? '?', '!'
378-
result = :IDENTIFIER
378+
result = :tIDENTIFIER
379379
else
380380
if @lex_state == :expr_fname
381381
if scanner.scan(/\=/)
382-
result = :IDENTIFIER
382+
result = :tIDENTIFIER
383383
matched += scanner.matched
384384
end
385385

386386
elsif matched =~ /^[A-Z]/
387-
result = :CONSTANT
387+
result = :tCONSTANT
388388
else
389-
result = :IDENTIFIER
389+
result = :tIDENTIFIER
390390
end
391391
end
392392

@@ -405,7 +405,7 @@ def process_identifier(matched, cmd_start)
405405
if matched == "do"
406406
if after_operator?
407407
@lex_state = :expr_end
408-
return :IDENTIFIER, matched
408+
return :tIDENTIFIER, matched
409409
end
410410

411411
if @start_of_lambda
@@ -443,7 +443,7 @@ def process_identifier(matched, cmd_start)
443443
@lex_state = :expr_end
444444
end
445445

446-
return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
446+
return [matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER, matched]
447447
end
448448

449449
def yylex
@@ -532,10 +532,10 @@ def yylex
532532
return :tSTAR2, result
533533
elsif @space_seen && scanner.check(/\S/)
534534
@lex_state = :expr_beg
535-
return :SPLAT, result
535+
return :tSTAR, result
536536
elsif [:expr_beg, :expr_mid].include? @lex_state
537537
@lex_state = :expr_beg
538-
return :SPLAT, result
538+
return :tSTAR, result
539539
else
540540
@lex_state = :expr_beg
541541
return :tSTAR2, result
@@ -605,22 +605,22 @@ def yylex
605605
if scanner.scan(/\~/)
606606
return :tMATCH, '=~'
607607
elsif scanner.scan(/\>/)
608-
return '=>', '=>'
608+
return :tASSOC, '=>'
609609
end
610610

611611
return :tEQL, '='
612612

613613
elsif scanner.scan(/\"/)
614614
self.strterm = { :type => :dquote, :beg => '"', :end => '"' }
615-
return :STRING_BEG, scanner.matched
615+
return :tSTRING_BEG, scanner.matched
616616

617617
elsif scanner.scan(/\'/)
618618
self.strterm = { :type => :squote, :beg => "'", :end => "'" }
619-
return :STRING_BEG, scanner.matched
619+
return :tSTRING_BEG, scanner.matched
620620

621621
elsif scanner.scan(/\`/)
622622
self.strterm = { :type => :xquote, :beg => "`", :end => "`" }
623-
return :XSTRING_BEG, scanner.matched
623+
return :tXSTRING_BEG, scanner.matched
624624

625625
elsif scanner.scan(/\&/)
626626
if scanner.scan(/\&/)
@@ -671,38 +671,38 @@ def yylex
671671
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
672672
self.strterm = { :type => :dword, :beg => 'W', :end => end_word }
673673
scanner.scan(/\s*/)
674-
return :WORDS_BEG, scanner.matched
674+
return :tWORDS_BEG, scanner.matched
675675

676676
elsif scanner.scan(/\%w/) or scanner.scan(/\%i/)
677677
start_word = scanner.scan(/./)
678678
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
679679
self.strterm = { :type => :sword, :beg => 'w', :end => end_word }
680680
scanner.scan(/\s*/)
681-
return :AWORDS_BEG, scanner.matched
681+
return :tAWORDS_BEG, scanner.matched
682682

683683
elsif scanner.scan(/\%[Qq]/)
684684
type = scanner.matched.end_with?('Q') ? :dquote : :squote
685685
start_word = scanner.scan(/./)
686686
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
687687
self.strterm = { :type => type, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
688-
return :STRING_BEG, scanner.matched
688+
return :tSTRING_BEG, scanner.matched
689689

690690
elsif scanner.scan(/\%x/)
691691
start_word = scanner.scan(/./)
692692
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
693693
self.strterm = { :type => :xquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
694-
return :XSTRING_BEG, scanner.matched
694+
return :tXSTRING_BEG, scanner.matched
695695

696696
elsif scanner.scan(/\%r/)
697697
start_word = scanner.scan(/./)
698698
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
699699
self.strterm = { :type => :regexp, :beg => start_word, :end => end_word, :regexp => true, :balance => true, :nesting => 0 }
700-
return :REGEXP_BEG, scanner.matched
700+
return :tREGEXP_BEG, scanner.matched
701701

702702
elsif scanner.scan(/\//)
703703
if [:expr_beg, :expr_mid].include? @lex_state
704704
self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
705-
return :REGEXP_BEG, scanner.matched
705+
return :tREGEXP_BEG, scanner.matched
706706
elsif scanner.scan(/\=/)
707707
@lex_state = :expr_beg
708708
return :tOP_ASGN, '/'
@@ -711,7 +711,7 @@ def yylex
711711
elsif @lex_state == :expr_cmdarg || @lex_state == :expr_arg
712712
if !scanner.check(/\s/) && @space_seen
713713
self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
714-
return :REGEXP_BEG, scanner.matched
714+
return :tREGEXP_BEG, scanner.matched
715715
end
716716
else
717717
@lex_state = :expr_beg
@@ -728,7 +728,7 @@ def yylex
728728
start_word = scanner.scan(/./)
729729
end_word = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
730730
self.strterm = { :type => :dquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
731-
return :STRING_BEG, scanner.matched
731+
return :tSTRING_BEG, scanner.matched
732732
end
733733
end
734734

@@ -747,7 +747,7 @@ def yylex
747747
elsif scanner.scan(/\(/)
748748
result = scanner.matched
749749
if [:expr_beg, :expr_mid].include? @lex_state
750-
result = :PAREN_BEG
750+
result = :tLPAREN
751751
elsif @space_seen && [:expr_arg, :expr_cmdarg].include?(@lex_state)
752752
result = :tLPAREN_ARG
753753
else
@@ -835,7 +835,7 @@ def yylex
835835
end
836836

837837
@lex_state = :expr_fname
838-
return :SYMBOL_BEG, ':'
838+
return :tSYMBEG, ':'
839839
end
840840

841841
if scanner.scan(/\'/)
@@ -845,7 +845,7 @@ def yylex
845845
end
846846

847847
@lex_state = :expr_fname
848-
return :SYMBOL_BEG, ':'
848+
return :tSYMBEG, ':'
849849

850850
elsif scanner.scan(/\^\=/)
851851
@lex_state = :expr_beg
@@ -935,7 +935,7 @@ def yylex
935935
# FIXME: # should be :expr_arg, but '(' breaks it...
936936
@lex_state = :expr_end
937937
@start_of_lambda = true
938-
return [:LAMBDA, scanner.matched]
938+
return [:tLAMBDA, scanner.matched]
939939

940940
elsif scanner.scan(/[+-]/)
941941
result = scanner.matched
@@ -946,7 +946,7 @@ def yylex
946946
return [sign, sign]
947947
elsif @lex_state == :expr_fname or @lex_state == :expr_dot
948948
@lex_state = :expr_arg
949-
return [:IDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
949+
return [:tIDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
950950
return [result, result]
951951
end
952952

@@ -973,7 +973,7 @@ def yylex
973973

974974
unless scanner.check(/\ |\t|\r|\s/)
975975
@lex_state = :expr_end
976-
return :STRING, scanner.scan(/./)
976+
return :tSTRING, scanner.scan(/./)
977977
end
978978

979979
@lex_state = :expr_beg
@@ -990,33 +990,33 @@ def yylex
990990
elsif scanner.check(/\$/)
991991
if scanner.scan(/\$([1-9]\d*)/)
992992
@lex_state = :expr_end
993-
return :NTH_REF, scanner.matched.sub('$', '')
993+
return :tNTH_REF, scanner.matched.sub('$', '')
994994

995995
elsif scanner.scan(/(\$_)(\w+)/)
996996
@lex_state = :expr_end
997-
return :GVAR, scanner.matched
997+
return :tGVAR, scanner.matched
998998

999999
elsif scanner.scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
10001000
@lex_state = :expr_end
1001-
return :GVAR, scanner.matched
1001+
return :tGVAR, scanner.matched
10021002
elsif scanner.scan(/\$\w+/)
10031003
@lex_state = :expr_end
1004-
return :GVAR, scanner.matched
1004+
return :tGVAR, scanner.matched
10051005
else
10061006
raise "Bad gvar name: #{scanner.peek(5).inspect}"
10071007
end
10081008

10091009
elsif scanner.scan(/\$\w+/)
10101010
@lex_state = :expr_end
1011-
return :GVAR, scanner.matched
1011+
return :tGVAR, scanner.matched
10121012

10131013
elsif scanner.scan(/\@\@\w*/)
10141014
@lex_state = :expr_end
1015-
return :CVAR, scanner.matched
1015+
return :tCVAR, scanner.matched
10161016

10171017
elsif scanner.scan(/\@\w*/)
10181018
@lex_state = :expr_end
1019-
return :IVAR, scanner.matched
1019+
return :tIVAR, scanner.matched
10201020

10211021
elsif scanner.scan(/\,/)
10221022
@lex_state = :expr_beg
@@ -1026,10 +1026,10 @@ def yylex
10261026
if @start_of_lambda
10271027
@start_of_lambda = false
10281028
@lex_state = :expr_beg
1029-
return [:LAMBEG, scanner.matched]
1029+
return [:tLAMBEG, scanner.matched]
10301030

10311031
elsif [:expr_end, :expr_arg, :expr_cmdarg].include? @lex_state
1032-
result = :LCURLY
1032+
result = :tLCURLY
10331033
elsif @lex_state == :expr_endarg
10341034
result = :LBRACE_ARG
10351035
else
@@ -1044,15 +1044,15 @@ def yylex
10441044
elsif scanner.check(/[0-9]/)
10451045
@lex_state = :expr_end
10461046
if scanner.scan(/0b?(0|1|_)+/)
1047-
return [:INTEGER, scanner.matched.to_i(2)]
1047+
return [:tINTEGER, scanner.matched.to_i(2)]
10481048
elsif scanner.scan(/0o?([0-7]|_)+/)
1049-
return [:INTEGER, scanner.matched.to_i(8)]
1049+
return [:tINTEGER, scanner.matched.to_i(8)]
10501050
elsif scanner.scan(/[\d_]+\.[\d_]+\b|[\d_]+(\.[\d_]+)?[eE][-+]?[\d_]+\b/)
1051-
return [:FLOAT, scanner.matched.gsub(/_/, '').to_f]
1051+
return [:tFLOAT, scanner.matched.gsub(/_/, '').to_f]
10521052
elsif scanner.scan(/[\d_]+\b/)
1053-
return [:INTEGER, scanner.matched.gsub(/_/, '').to_i]
1053+
return [:tINTEGER, scanner.matched.gsub(/_/, '').to_i]
10541054
elsif scanner.scan(/0(x|X)(\d|[a-f]|[A-F]|_)+/)
1055-
return [:INTEGER, scanner.matched.to_i(16)]
1055+
return [:tINTEGER, scanner.matched.to_i(16)]
10561056
else
10571057
raise "Lexing error on numeric type: `#{scanner.peek 5}`"
10581058
end

0 commit comments

Comments
 (0)
Please sign in to comment.