Skip to content

Commit bc28d95

Browse files
author
whitequark
committedMay 7, 2015
Ensure AST is consistent with Python's builtin ast.
·
1.41.0
1 parent 25e4d3f commit bc28d95

File tree

2 files changed

+88
-38
lines changed

2 files changed

+88
-38
lines changed
 

‎pyparser/parser.py

+20-11
Original file line numberDiff line numberDiff line change
@@ -591,14 +591,14 @@ def expr_stmt(self, lhs, rhs):
591591
@action(List(Rule('test'), ',', trailing=True))
592592
def print_stmt_1(self, values):
593593
loc = values.trailing_comma.loc if values.trailing_comma else values[-1].loc
594-
nl = True if values.trailing_comma else False
594+
nl = False if values.trailing_comma else True
595595
return ast.Print(dest=None, values=values, nl=nl,
596596
dest_loc=None, loc=loc)
597597

598598
@action(Seq(Loc('>>'), Rule('test'), Tok(','), List(Rule('test'), ',', trailing=True)))
599599
def print_stmt_2(self, dest_loc, dest, comma_tok, values):
600600
loc = values.trailing_comma.loc if values.trailing_comma else values[-1].loc
601-
nl = True if values.trailing_comma else False
601+
nl = False if values.trailing_comma else True
602602
return ast.Print(dest=dest, values=values, nl=nl,
603603
dest_loc=dest_loc, loc=loc)
604604

@@ -613,11 +613,13 @@ def print_stmt(self, print_loc, stmt):
613613
stmt.loc = print_loc.join(stmt.loc)
614614
return stmt
615615

616-
@action(Seq(Loc('del'), Rule('exprlist')))
616+
@action(Seq(Loc('del'), List(Rule('expr'), ',', trailing=True)))
617617
def del_stmt(self, stmt_loc, exprs):
618+
# Python uses exprlist here, but does *not* obey the usual
619+
# tuple-wrapping semantics, so we embed the rule directly.
618620
"""del_stmt: 'del' exprlist"""
619-
return ast.Delete(targets=self._assignable(exprs),
620-
loc=stmt_loc.join(exprs.loc), keyword_loc=stmt_loc)
621+
return ast.Delete(targets=list(map(self._assignable, exprs)),
622+
loc=stmt_loc.join(exprs[-1].loc), keyword_loc=stmt_loc)
621623

622624
@action(Loc('pass'))
623625
def pass_stmt(self, stmt_loc):
@@ -772,14 +774,14 @@ def global_stmt(self, global_loc, names):
772774
Opt(SeqN(1, Loc(','), Rule('test')))))))
773775
def exec_stmt(self, exec_loc, body, in_opt):
774776
"""exec_stmt: 'exec' expr ['in' test [',' test]]"""
775-
in_loc, locals, globals = None, None, None
777+
in_loc, globals, locals = None, None, None
776778
loc = exec_loc.join(body.loc)
777779
if in_opt:
778-
in_loc, locals, globals = in_opt
779-
if globals:
780-
loc = loc.join(globals.loc)
781-
else:
780+
in_loc, globals, locals = in_opt
781+
if locals:
782782
loc = loc.join(locals.loc)
783+
else:
784+
loc = loc.join(globals.loc)
783785
return ast.Exec(body=body, locals=locals, globals=globals,
784786
loc=loc, keyword_loc=exec_loc, in_loc=in_loc)
785787

@@ -863,7 +865,7 @@ def try_stmt_1(self, clauses, else_opt, finally_opt):
863865
handler.loc = handler.loc.join(handler.body[-1].loc)
864866
handlers.append(handler)
865867

866-
else_loc = else_colon_loc = orelse = None
868+
else_loc, else_colon_loc, orelse = None, None, []
867869
loc = handlers[-1].loc
868870
if else_opt:
869871
else_loc, else_colon_loc, orelse = else_opt
@@ -1169,6 +1171,13 @@ def subscriptlist(self, subscripts):
11691171
"""subscriptlist: subscript (',' subscript)* [',']"""
11701172
if len(subscripts) == 1:
11711173
return ast.Subscript(slice=subscripts[0], ctx=None, loc=None)
1174+
elif all([isinstance(x, ast.Index) for x in subscripts]):
1175+
elts = [x.value for x in subscripts]
1176+
loc = subscripts[0].loc.join(subscripts[-1].loc)
1177+
index = ast.Index(value=ast.Tuple(elts=elts, ctx=None,
1178+
begin_loc=None, end_loc=None, loc=loc),
1179+
loc=loc)
1180+
return ast.Subscript(slice=index, ctx=None, loc=None)
11721181
else:
11731182
extslice = ast.ExtSlice(dims=subscripts,
11741183
loc=subscripts[0].loc.join(subscripts[-1].loc))

‎pyparser/test/test_parser.py

+68-27
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from __future__ import absolute_import, division, print_function, unicode_literals
44
from .. import source, lexer, diagnostic, ast, coverage
55
from ..coverage import parser
6-
import unittest, sys, re
6+
import unittest, sys, re, ast as pyast
77

88
if sys.version_info >= (3,):
99
def unicode(x): return x
@@ -51,6 +51,21 @@ def flatten_ast(self, node):
5151
flat_node[unicode(field)] = value
5252
return flat_node
5353

54+
def flatten_python_ast(self, node):
55+
flat_node = { 'ty': unicode(type(node).__name__) }
56+
for field in node._fields:
57+
if field == 'ctx':
58+
flat_node['ctx'] = None
59+
continue
60+
61+
value = getattr(node, field)
62+
if isinstance(value, ast.AST):
63+
value = self.flatten_python_ast(value)
64+
if isinstance(value, list) and len(value) > 0 and isinstance(value[0], ast.AST):
65+
value = list(map(self.flatten_python_ast, value))
66+
flat_node[unicode(field)] = value
67+
return flat_node
68+
5469
_loc_re = re.compile(r"\s*([~^]*)<?\s+([a-z_0-9.]+)")
5570
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")
5671

@@ -92,26 +107,35 @@ def match_loc(self, ast, matcher, root=lambda x: x):
92107
matcher_pos = matcher_match.end(0)
93108

94109
def _assertParsesGen(self, expected_flat_ast, code,
95-
loc_matcher="", ast_slicer=lambda x: (0, x)):
110+
loc_matcher="", ast_slicer=lambda x: (0, x),
111+
validate_if=lambda: True):
96112
ast = self.parser_for(code + "\n").file_input()
97113
flat_ast = self.flatten_ast(ast)
114+
python_ast = pyast.parse(code.replace("·", "\n") + "\n")
115+
flat_python_ast = self.flatten_python_ast(python_ast)
98116
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
99117
flat_ast)
118+
if validate_if():
119+
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
120+
flat_python_ast)
100121
self.match_loc(ast, loc_matcher, ast_slicer)
101122

102-
def assertParsesSuite(self, expected_flat_ast, code, loc_matcher=""):
123+
def assertParsesSuite(self, expected_flat_ast, code, loc_matcher="", **kwargs):
103124
self._assertParsesGen(expected_flat_ast, code,
104-
loc_matcher, lambda x: (0, x.body))
125+
loc_matcher, lambda x: (0, x.body),
126+
**kwargs)
105127

106-
def assertParsesExpr(self, expected_flat_ast, code, loc_matcher=""):
128+
def assertParsesExpr(self, expected_flat_ast, code, loc_matcher="", **kwargs):
107129
self._assertParsesGen([{'ty': 'Expr', 'value': expected_flat_ast}], code,
108-
loc_matcher, lambda x: (0, x.body[0].value))
130+
loc_matcher, lambda x: (0, x.body[0].value),
131+
**kwargs)
109132

110-
def assertParsesArgs(self, expected_flat_ast, code, loc_matcher=""):
133+
def assertParsesArgs(self, expected_flat_ast, code, loc_matcher="", **kwargs):
111134
self._assertParsesGen([{'ty': 'Expr', 'value': {'ty': 'Lambda', 'body': self.ast_1,
112135
'args': expected_flat_ast}}],
113136
"lambda %s: 1" % code,
114-
loc_matcher, lambda x: (7, x.body[0].value.args))
137+
loc_matcher, lambda x: (7, x.body[0].value.args),
138+
**kwargs)
115139

116140
def assertParsesToplevel(self, expected_flat_ast, code,
117141
mode="file_input", interactive=False):
@@ -200,8 +224,8 @@ def test_unary(self):
200224
"~ op.loc")
201225

202226
self.assertParsesExpr(
203-
{'ty': 'UnaryOp', 'op': {'ty': 'USub'}, 'operand': self.ast_1},
204-
"-1",
227+
{'ty': 'UnaryOp', 'op': {'ty': 'USub'}, 'operand': self.ast_x},
228+
"-x",
205229
"~~ loc"
206230
"~ op.loc")
207231

@@ -669,10 +693,9 @@ def test_subscript(self):
669693

670694
self.assertParsesExpr(
671695
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
672-
'slice': {'ty': 'ExtSlice', 'dims': [
673-
{'ty': 'Index', 'value': self.ast_1},
674-
{'ty': 'Index', 'value': self.ast_2},
675-
]}},
696+
'slice': {'ty': 'Index', 'value': {'ty': 'Tuple', 'ctx': None, 'elts': [
697+
self.ast_1, self.ast_2
698+
]}}},
676699
"x[1, 2]",
677700
" ~~~~ slice.loc"
678701
"~~~~~~~ loc")
@@ -701,14 +724,26 @@ def test_subscript(self):
701724
" ~~~ slice.loc"
702725
"~~~~~~ loc")
703726

727+
self.assertParsesExpr(
728+
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
729+
'slice': {'ty': 'ExtSlice', 'dims': [
730+
{'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None},
731+
{'ty': 'Index', 'value': self.ast_2},
732+
]}},
733+
"x[1:2, 2]",
734+
" ~~~~~~ slice.loc"
735+
"~~~~~~~~~ loc")
736+
704737
self.assertParsesExpr(
705738
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
706739
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
707740
"x[1:2:]",
708741
" ^ slice.bound_colon_loc"
709742
" ^ slice.step_colon_loc"
710743
" ~~~~ slice.loc"
711-
"~~~~~~~ loc")
744+
"~~~~~~~ loc",
745+
# A Python bug places ast.Name(id='None') instead of None in step on <3.0
746+
validate_if=lambda: sys.version_info[0] > 2)
712747

713748
self.assertParsesExpr(
714749
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
@@ -839,31 +874,37 @@ def test_augassign(self):
839874

840875
def test_print(self):
841876
self.assertParsesSuite(
842-
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': False}],
877+
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': True}],
843878
"print 1",
844879
"~~~~~ 0.keyword_loc"
845880
"~~~~~~~ 0.loc")
846881

847882
self.assertParsesSuite(
848-
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': True}],
883+
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': False}],
849884
"print 1,",
850885
"~~~~~ 0.keyword_loc"
851886
"~~~~~~~~ 0.loc")
852887

853888
self.assertParsesSuite(
854-
[{'ty': 'Print', 'dest': self.ast_2, 'values': [self.ast_1], 'nl': False}],
889+
[{'ty': 'Print', 'dest': self.ast_2, 'values': [self.ast_1], 'nl': True}],
855890
"print >>2, 1",
856891
"~~~~~ 0.keyword_loc"
857892
" ~~ 0.dest_loc"
858893
"~~~~~~~~~~~~ 0.loc")
859894

860895
def test_del(self):
861896
self.assertParsesSuite(
862-
[{'ty': 'Delete', 'targets': self.ast_x}],
897+
[{'ty': 'Delete', 'targets': [self.ast_x]}],
863898
"del x",
864899
"~~~ 0.keyword_loc"
865900
"~~~~~ 0.loc")
866901

902+
self.assertParsesSuite(
903+
[{'ty': 'Delete', 'targets': [self.ast_x, self.ast_y]}],
904+
"del x, y",
905+
"~~~ 0.keyword_loc"
906+
"~~~~~~~~ 0.loc")
907+
867908
def test_pass(self):
868909
self.assertParsesSuite(
869910
[{'ty': 'Pass'}],
@@ -1039,20 +1080,20 @@ def test_global(self):
10391080

10401081
def test_exec(self):
10411082
self.assertParsesSuite(
1042-
[{'ty': 'Exec', 'body': self.ast_1, 'locals': None, 'globals': None}],
1083+
[{'ty': 'Exec', 'body': self.ast_1, 'globals': None, 'locals': None}],
10431084
"exec 1",
10441085
"~~~~ 0.keyword_loc"
10451086
"~~~~~~ 0.loc")
10461087

10471088
self.assertParsesSuite(
1048-
[{'ty': 'Exec', 'body': self.ast_1, 'locals': self.ast_2, 'globals': None}],
1089+
[{'ty': 'Exec', 'body': self.ast_1, 'globals': self.ast_2, 'locals': None}],
10491090
"exec 1 in 2",
10501091
"~~~~ 0.keyword_loc"
10511092
" ~~ 0.in_loc"
10521093
"~~~~~~~~~~~ 0.loc")
10531094

10541095
self.assertParsesSuite(
1055-
[{'ty': 'Exec', 'body': self.ast_1, 'locals': self.ast_2, 'globals': self.ast_3}],
1096+
[{'ty': 'Exec', 'body': self.ast_1, 'globals': self.ast_2, 'locals': self.ast_3}],
10561097
"exec 1 in 2, 3",
10571098
"~~~~ 0.keyword_loc"
10581099
" ~~ 0.in_loc"
@@ -1146,7 +1187,7 @@ def test_for(self):
11461187

11471188
def test_try(self):
11481189
self.assertParsesSuite(
1149-
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
1190+
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
11501191
'handlers': [
11511192
{'ty': 'ExceptHandler', 'type': None, 'name': None,
11521193
'body': [self.ast_expr_2]}
@@ -1160,15 +1201,15 @@ def test_try(self):
11601201
"~~~~~~~~~~~~~~~~~~~~ 0.loc")
11611202

11621203
self.assertParsesSuite(
1163-
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
1204+
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
11641205
'handlers': [
11651206
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': None,
11661207
'body': [self.ast_expr_2]}
11671208
]}],
11681209
"try:· 1·except y:· 2")
11691210

11701211
self.assertParsesSuite(
1171-
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
1212+
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
11721213
'handlers': [
11731214
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': self.ast_t,
11741215
'body': [self.ast_expr_2]}
@@ -1177,7 +1218,7 @@ def test_try(self):
11771218
" ~~ 0.handlers.0.as_loc")
11781219

11791220
self.assertParsesSuite(
1180-
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
1221+
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
11811222
'handlers': [
11821223
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': self.ast_t,
11831224
'body': [self.ast_expr_2]}
@@ -1208,7 +1249,7 @@ def test_finally(self):
12081249

12091250
self.assertParsesSuite(
12101251
[{'ty': 'TryFinally', 'finalbody': [self.ast_expr_3], 'body': [
1211-
{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None, 'handlers': [
1252+
{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [], 'handlers': [
12121253
{'ty': 'ExceptHandler', 'type': None, 'name': None,
12131254
'body': [self.ast_expr_2]}
12141255
]}

0 commit comments

Comments
 (0)
Please sign in to comment.