Skip to content

Commit

Permalink
Ensure AST is consistent with Python's builtin ast.
Browse files Browse the repository at this point in the history
whitequark committed May 7, 2015
1 parent 25e4d3f commit bc28d95
Showing 2 changed files with 88 additions and 38 deletions.
31 changes: 20 additions & 11 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -591,14 +591,14 @@ def expr_stmt(self, lhs, rhs):
@action(List(Rule('test'), ',', trailing=True))
def print_stmt_1(self, values):
loc = values.trailing_comma.loc if values.trailing_comma else values[-1].loc
nl = True if values.trailing_comma else False
nl = False if values.trailing_comma else True
return ast.Print(dest=None, values=values, nl=nl,
dest_loc=None, loc=loc)

@action(Seq(Loc('>>'), Rule('test'), Tok(','), List(Rule('test'), ',', trailing=True)))
def print_stmt_2(self, dest_loc, dest, comma_tok, values):
loc = values.trailing_comma.loc if values.trailing_comma else values[-1].loc
nl = True if values.trailing_comma else False
nl = False if values.trailing_comma else True
return ast.Print(dest=dest, values=values, nl=nl,
dest_loc=dest_loc, loc=loc)

@@ -613,11 +613,13 @@ def print_stmt(self, print_loc, stmt):
stmt.loc = print_loc.join(stmt.loc)
return stmt

@action(Seq(Loc('del'), Rule('exprlist')))
@action(Seq(Loc('del'), List(Rule('expr'), ',', trailing=True)))
def del_stmt(self, stmt_loc, exprs):
# Python uses exprlist here, but does *not* obey the usual
# tuple-wrapping semantics, so we embed the rule directly.
"""del_stmt: 'del' exprlist"""
return ast.Delete(targets=self._assignable(exprs),
loc=stmt_loc.join(exprs.loc), keyword_loc=stmt_loc)
return ast.Delete(targets=list(map(self._assignable, exprs)),
loc=stmt_loc.join(exprs[-1].loc), keyword_loc=stmt_loc)

@action(Loc('pass'))
def pass_stmt(self, stmt_loc):
@@ -772,14 +774,14 @@ def global_stmt(self, global_loc, names):
Opt(SeqN(1, Loc(','), Rule('test')))))))
def exec_stmt(self, exec_loc, body, in_opt):
"""exec_stmt: 'exec' expr ['in' test [',' test]]"""
in_loc, locals, globals = None, None, None
in_loc, globals, locals = None, None, None
loc = exec_loc.join(body.loc)
if in_opt:
in_loc, locals, globals = in_opt
if globals:
loc = loc.join(globals.loc)
else:
in_loc, globals, locals = in_opt
if locals:
loc = loc.join(locals.loc)
else:
loc = loc.join(globals.loc)
return ast.Exec(body=body, locals=locals, globals=globals,
loc=loc, keyword_loc=exec_loc, in_loc=in_loc)

@@ -863,7 +865,7 @@ def try_stmt_1(self, clauses, else_opt, finally_opt):
handler.loc = handler.loc.join(handler.body[-1].loc)
handlers.append(handler)

else_loc = else_colon_loc = orelse = None
else_loc, else_colon_loc, orelse = None, None, []
loc = handlers[-1].loc
if else_opt:
else_loc, else_colon_loc, orelse = else_opt
@@ -1169,6 +1171,13 @@ def subscriptlist(self, subscripts):
"""subscriptlist: subscript (',' subscript)* [',']"""
if len(subscripts) == 1:
return ast.Subscript(slice=subscripts[0], ctx=None, loc=None)
elif all([isinstance(x, ast.Index) for x in subscripts]):
elts = [x.value for x in subscripts]
loc = subscripts[0].loc.join(subscripts[-1].loc)
index = ast.Index(value=ast.Tuple(elts=elts, ctx=None,
begin_loc=None, end_loc=None, loc=loc),
loc=loc)
return ast.Subscript(slice=index, ctx=None, loc=None)
else:
extslice = ast.ExtSlice(dims=subscripts,
loc=subscripts[0].loc.join(subscripts[-1].loc))
95 changes: 68 additions & 27 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from .. import source, lexer, diagnostic, ast, coverage
from ..coverage import parser
import unittest, sys, re
import unittest, sys, re, ast as pyast

if sys.version_info >= (3,):
def unicode(x): return x
@@ -51,6 +51,21 @@ def flatten_ast(self, node):
flat_node[unicode(field)] = value
return flat_node

def flatten_python_ast(self, node):
flat_node = { 'ty': unicode(type(node).__name__) }
for field in node._fields:
if field == 'ctx':
flat_node['ctx'] = None
continue

value = getattr(node, field)
if isinstance(value, ast.AST):
value = self.flatten_python_ast(value)
if isinstance(value, list) and len(value) > 0 and isinstance(value[0], ast.AST):
value = list(map(self.flatten_python_ast, value))
flat_node[unicode(field)] = value
return flat_node

_loc_re = re.compile(r"\s*([~^]*)<?\s+([a-z_0-9.]+)")
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")

@@ -92,26 +107,35 @@ def match_loc(self, ast, matcher, root=lambda x: x):
matcher_pos = matcher_match.end(0)

def _assertParsesGen(self, expected_flat_ast, code,
loc_matcher="", ast_slicer=lambda x: (0, x)):
loc_matcher="", ast_slicer=lambda x: (0, x),
validate_if=lambda: True):
ast = self.parser_for(code + "\n").file_input()
flat_ast = self.flatten_ast(ast)
python_ast = pyast.parse(code.replace("·", "\n") + "\n")
flat_python_ast = self.flatten_python_ast(python_ast)
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
flat_ast)
if validate_if():
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
flat_python_ast)
self.match_loc(ast, loc_matcher, ast_slicer)

def assertParsesSuite(self, expected_flat_ast, code, loc_matcher=""):
def assertParsesSuite(self, expected_flat_ast, code, loc_matcher="", **kwargs):
self._assertParsesGen(expected_flat_ast, code,
loc_matcher, lambda x: (0, x.body))
loc_matcher, lambda x: (0, x.body),
**kwargs)

def assertParsesExpr(self, expected_flat_ast, code, loc_matcher=""):
def assertParsesExpr(self, expected_flat_ast, code, loc_matcher="", **kwargs):
self._assertParsesGen([{'ty': 'Expr', 'value': expected_flat_ast}], code,
loc_matcher, lambda x: (0, x.body[0].value))
loc_matcher, lambda x: (0, x.body[0].value),
**kwargs)

def assertParsesArgs(self, expected_flat_ast, code, loc_matcher=""):
def assertParsesArgs(self, expected_flat_ast, code, loc_matcher="", **kwargs):
self._assertParsesGen([{'ty': 'Expr', 'value': {'ty': 'Lambda', 'body': self.ast_1,
'args': expected_flat_ast}}],
"lambda %s: 1" % code,
loc_matcher, lambda x: (7, x.body[0].value.args))
loc_matcher, lambda x: (7, x.body[0].value.args),
**kwargs)

def assertParsesToplevel(self, expected_flat_ast, code,
mode="file_input", interactive=False):
@@ -200,8 +224,8 @@ def test_unary(self):
"~ op.loc")

self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'USub'}, 'operand': self.ast_1},
"-1",
{'ty': 'UnaryOp', 'op': {'ty': 'USub'}, 'operand': self.ast_x},
"-x",
"~~ loc"
"~ op.loc")

@@ -669,10 +693,9 @@ def test_subscript(self):

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'ExtSlice', 'dims': [
{'ty': 'Index', 'value': self.ast_1},
{'ty': 'Index', 'value': self.ast_2},
]}},
'slice': {'ty': 'Index', 'value': {'ty': 'Tuple', 'ctx': None, 'elts': [
self.ast_1, self.ast_2
]}}},
"x[1, 2]",
" ~~~~ slice.loc"
"~~~~~~~ loc")
@@ -701,14 +724,26 @@ def test_subscript(self):
" ~~~ slice.loc"
"~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'ExtSlice', 'dims': [
{'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None},
{'ty': 'Index', 'value': self.ast_2},
]}},
"x[1:2, 2]",
" ~~~~~~ slice.loc"
"~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
"x[1:2:]",
" ^ slice.bound_colon_loc"
" ^ slice.step_colon_loc"
" ~~~~ slice.loc"
"~~~~~~~ loc")
"~~~~~~~ loc",
# A Python bug places ast.Name(id='None') instead of None in step on <3.0
validate_if=lambda: sys.version_info[0] > 2)

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
@@ -839,31 +874,37 @@ def test_augassign(self):

def test_print(self):
self.assertParsesSuite(
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': False}],
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': True}],
"print 1",
"~~~~~ 0.keyword_loc"
"~~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': True}],
[{'ty': 'Print', 'dest': None, 'values': [self.ast_1], 'nl': False}],
"print 1,",
"~~~~~ 0.keyword_loc"
"~~~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Print', 'dest': self.ast_2, 'values': [self.ast_1], 'nl': False}],
[{'ty': 'Print', 'dest': self.ast_2, 'values': [self.ast_1], 'nl': True}],
"print >>2, 1",
"~~~~~ 0.keyword_loc"
" ~~ 0.dest_loc"
"~~~~~~~~~~~~ 0.loc")

def test_del(self):
self.assertParsesSuite(
[{'ty': 'Delete', 'targets': self.ast_x}],
[{'ty': 'Delete', 'targets': [self.ast_x]}],
"del x",
"~~~ 0.keyword_loc"
"~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Delete', 'targets': [self.ast_x, self.ast_y]}],
"del x, y",
"~~~ 0.keyword_loc"
"~~~~~~~~ 0.loc")

def test_pass(self):
self.assertParsesSuite(
[{'ty': 'Pass'}],
@@ -1039,20 +1080,20 @@ def test_global(self):

def test_exec(self):
self.assertParsesSuite(
[{'ty': 'Exec', 'body': self.ast_1, 'locals': None, 'globals': None}],
[{'ty': 'Exec', 'body': self.ast_1, 'globals': None, 'locals': None}],
"exec 1",
"~~~~ 0.keyword_loc"
"~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Exec', 'body': self.ast_1, 'locals': self.ast_2, 'globals': None}],
[{'ty': 'Exec', 'body': self.ast_1, 'globals': self.ast_2, 'locals': None}],
"exec 1 in 2",
"~~~~ 0.keyword_loc"
" ~~ 0.in_loc"
"~~~~~~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Exec', 'body': self.ast_1, 'locals': self.ast_2, 'globals': self.ast_3}],
[{'ty': 'Exec', 'body': self.ast_1, 'globals': self.ast_2, 'locals': self.ast_3}],
"exec 1 in 2, 3",
"~~~~ 0.keyword_loc"
" ~~ 0.in_loc"
@@ -1146,7 +1187,7 @@ def test_for(self):

def test_try(self):
self.assertParsesSuite(
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
'handlers': [
{'ty': 'ExceptHandler', 'type': None, 'name': None,
'body': [self.ast_expr_2]}
@@ -1160,15 +1201,15 @@ def test_try(self):
"~~~~~~~~~~~~~~~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
'handlers': [
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': None,
'body': [self.ast_expr_2]}
]}],
"try:· 1·except y:· 2")

self.assertParsesSuite(
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
'handlers': [
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': self.ast_t,
'body': [self.ast_expr_2]}
@@ -1177,7 +1218,7 @@ def test_try(self):
" ~~ 0.handlers.0.as_loc")

self.assertParsesSuite(
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [],
'handlers': [
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': self.ast_t,
'body': [self.ast_expr_2]}
@@ -1208,7 +1249,7 @@ def test_finally(self):

self.assertParsesSuite(
[{'ty': 'TryFinally', 'finalbody': [self.ast_expr_3], 'body': [
{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None, 'handlers': [
{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': [], 'handlers': [
{'ty': 'ExceptHandler', 'type': None, 'name': None,
'body': [self.ast_expr_2]}
]}

0 comments on commit bc28d95

Please sign in to comment.