Skip to content

Commit

Permalink
45% coverage.
Browse files Browse the repository at this point in the history
All expressions except lambdas and a few forms of calls and
subscripts are handled. The omissions are due to LL conflicts.
whitequark committed May 4, 2015
1 parent da78a09 commit 656fb73
Showing 4 changed files with 332 additions and 33 deletions.
6 changes: 4 additions & 2 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -160,6 +160,7 @@ class Attribute(expr, ast.Attribute):
:ivar value: (node) left-hand side
:ivar attr: (string) attribute name
"""
_locs = expr._locs + ('dot_loc', 'attr_loc')
class BinOp(expr, ast.BinOp):
"""
A binary operation, e.g. ``x + y``.
@@ -316,10 +317,11 @@ class UnaryOp(expr, ast.UnaryOp):
"""
class Yield(expr, ast.Yield):
"""
A yield expression, e.g. ``yield x``.
A yield expression, e.g. ``(yield x)``.
:ivar value: (node) yielded value
"""
_locs = expr._locs + ('keyword_loc',)

# expr_context
# AugLoad
@@ -337,7 +339,7 @@ class keyword(commonloc, ast.keyword):
:ivar value: (node) value
:ivar equals_loc: location of ``=``
"""
_locs = commonloc._locs + ('equals_loc',)
_locs = commonloc._locs + ('arg_loc', 'equals_loc')

class mod(commonloc):
"""Base class for modules (groups of statements)."""
3 changes: 3 additions & 0 deletions pyparser/coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -29,6 +29,9 @@ def instrument():
if not in_grammar:
continue

if token.kind == '.': # skip ast.List
lex.next()

if token.kind == 'ident' and \
token.value in ('action', 'Eps', 'Tok', 'Loc', 'Rule', 'Expect',
'Seq', 'SeqN', 'Alt', 'Opt', 'Star', 'Plus', 'List',
67 changes: 42 additions & 25 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -199,6 +199,9 @@ def rule(parser):
results.append(result)
return rule

class commalist(list):
__slots__ = ('trailing_comma',)

def List(inner_rule, separator_tok, trailing, leading=True, loc=None):
if not trailing:
@action(Seq(inner_rule, Star(SeqN(1, Tok(separator_tok), inner_rule))), loc=loc)
@@ -213,7 +216,7 @@ def outer_rule(parser, first, rest):
separator_rule = Tok(separator_tok)
@llrule(loc, inner_rule.expected)
def rule(parser):
results = []
results = commalist()

if leading:
result = inner_rule(parser)
@@ -225,12 +228,12 @@ def rule(parser):
while True:
result = separator_rule(parser)
if result is unmatched:
#results.trailing = True
results.trailing_comma = False
return results

result = inner_rule(parser)
if result is unmatched:
#results.trailing = False
results.trailing_comma = True
return results
else:
results.append(result)
@@ -271,9 +274,15 @@ def BeginEnd(begin_tok, inner_rule, end_tok, empty=None, loc=None):
def rule(parser, begin_loc, node, end_loc):
if node is None:
node = empty()
node.ctx = None
node.begin_loc, node.end_loc, node.loc = \
begin_loc, end_loc, begin_loc.join(end_loc)

# Collection nodes don't have loc yet. If a node has loc at this
# point, it means it's an expression passed in parentheses.
if node.loc is None and type(node) in [
ast.List, ast.Dict, ast.Tuple, ast.Repr,
ast.ListComp, ast.GeneratorExp,
ast.Call, ast.Subscript]:
node.begin_loc, node.end_loc, node.loc = \
begin_loc, end_loc, begin_loc.join(end_loc)
return node
return rule

@@ -838,6 +847,7 @@ def power(self, atom, trailers, factor_opt):
trailer.value = atom
elif isinstance(trailer, ast.Call):
trailer.func = atom
trailer.loc = atom.loc.join(trailer.loc)
atom = trailer
if factor_opt:
op_loc, factor = factor_opt
@@ -862,14 +872,15 @@ def atom_3(self, begin_tok, data_tok, end_tok):

@action(Rule('testlist1'))
def atom_4(self, expr):
return ast.Repr(value=expr, loc=expr.loc)
return ast.Repr(value=expr, loc=None)

atom = Alt(BeginEnd('(', Opt(Alt(Rule('yield_expr'), Rule('testlist_gexp'))), ')',
empty=lambda: ast.Tuple(elts=[])),
empty=lambda: ast.Tuple(elts=[], ctx=None, loc=None)),
BeginEnd('[', Opt(Rule('listmaker')), ']',
empty=lambda: ast.List(elts=[])),
empty=lambda: ast.List(elts=[], ctx=None, loc=None)),
BeginEnd('{', Opt(Rule('dictmaker')), '}',
empty=lambda: ast.Dict(keys=[], values=[], colon_locs=[])),
empty=lambda: ast.Dict(keys=[], values=[], colon_locs=[],
ctx=None, loc=None)),
BeginEnd('`', atom_4, '`'),
atom_1, atom_2, atom_3)
"""atom: ('(' [yield_expr|testlist_gexp] ')' |
@@ -890,11 +901,11 @@ def list_gen_action(self, lhs, rhs):

@action(Rule('list_for'))
def listmaker_1(self, compose):
return ast.ListComp(generators=compose([]))
return ast.ListComp(generators=compose([]), loc=None)

@action(List(Rule('test'), ',', trailing=True, leading=False))
def listmaker_2(self, elts):
return ast.List(elts=elts, ctx=None)
return ast.List(elts=elts, ctx=None, loc=None)

listmaker = action(
Seq(Rule('test'),
@@ -904,14 +915,14 @@ def listmaker_2(self, elts):

@action(Rule('gen_for'))
def testlist_gexp_1(self, compose):
return ast.GeneratorExp(generators=compose([]))
return ast.GeneratorExp(generators=compose([]), loc=None)

@action(List(Rule('test'), ',', trailing=True))
@action(List(Rule('test'), ',', trailing=True, leading=False))
def testlist_gexp_2(self, elts):
if elts == [] and not elts.trailing:
if elts == [] and not elts.trailing_comma:
return None
else:
return ast.Tuple(elts=elts, ctx=None)
return ast.Tuple(elts=elts, ctx=None, loc=None)

testlist_gexp = action(
Seq(Rule('test'), Alt(testlist_gexp_1, testlist_gexp_2))) \
@@ -929,7 +940,8 @@ def lambdef(self, lambda_loc, args_opt, colon_loc, body):
@action(Seq(Loc('.'), Tok('ident')))
def trailer_1(self, dot_loc, ident_tok):
return ast.Attribute(attr=ident_tok.value, ctx=None,
loc=dot_loc.join(ident_tok.loc), dot_loc=dot_loc)
loc=dot_loc.join(ident_tok.loc),
attr_loc=ident_tok.loc, dot_loc=dot_loc)

trailer = Alt(BeginEnd('(', Rule('arglist'), ')'),
BeginEnd('[', Rule('subscriptlist'), ']'),
@@ -940,11 +952,11 @@ def trailer_1(self, dot_loc, ident_tok):
def subscriptlist(self, subscripts):
"""subscriptlist: subscript (',' subscript)* [',']"""
if len(subscripts) == 1:
return ast.Subscript(slice=subscripts[0], ctx=None)
return ast.Subscript(slice=subscripts[0], ctx=None, loc=None)
else:
extslice = ast.ExtSlice(dims=subscripts,
loc=subscripts[0].loc.join(subscripts[-1].loc))
return ast.Subscript(slice=extslice, ctx=None)
return ast.Subscript(slice=extslice, ctx=None, loc=None)

@action(Seq(Loc('.'), Loc('.'), Loc('.')))
def subscript_1(self, dot_1_loc, dot_2_loc, dot_3_loc):
@@ -970,6 +982,7 @@ def subscript_3(self, lower_opt, colon_loc, upper_opt, step_opt):
loc=loc, bound_colon_loc=colon_loc, step_colon_loc=step_colon_loc)

subscript_3_inner = Expect(Seq(Opt(Rule('test')), Opt(Rule('sliceop'))))

def subscript(self):
"""subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]"""
# This requires manual disambiguation of `test | [test] ...`
@@ -987,6 +1000,8 @@ def subscript(self):

result_2 = self.subscript_3_inner()
return self.subscript_3(result, result_1.loc, *result_2)
subscript.expected = \
lambda parser: parser.subscript_1.expected(parser) + parser.test.expected(parser)

sliceop = Seq(Loc(':'), Opt(Rule('test')))
"""sliceop: ':' [test]"""
@@ -1006,7 +1021,8 @@ def dictmaker(self, elts):
"""dictmaker: test ':' test (',' test ':' test)* [',']"""
return ast.Dict(keys=list(map(lambda x: x[0], elts)),
values=list(map(lambda x: x[2], elts)),
colon_locs=list(map(lambda x: x[1], elts)))
colon_locs=list(map(lambda x: x[1], elts)),
loc=None)

@action(Seq(Loc('class'), Tok('ident'),
Opt(BeginEnd('(', Rule('testlist'), ')')),
@@ -1035,12 +1051,12 @@ def arglist_1(self, star_loc, stararg, postargs, kwarg_opt):
raise diagnostic.DiagnosticException(error)

return ast.Call(starargs=stararg, kwargs=kwarg, keywords=postargs,
star_loc=star_loc, dstar_loc=dstar_loc)
star_loc=star_loc, dstar_loc=dstar_loc, loc=None)

@action(Seq(Loc('**'), Rule('test')))
def arglist_2(self, dstar_loc, kwarg):
return ast.Call(starargs=None, kwargs=kwarg, keywords=[],
star_loc=None, dstar_loc=dstar_loc)
star_loc=None, dstar_loc=dstar_loc, loc=None)

arglist_3 = Alt(arglist_1, arglist_2)

@@ -1066,8 +1082,8 @@ def arglist(self):

call = self.arglist_3()
if call is unmatched:
call = ast.Call(args=args, keywords=keywords, starargs=[], kwargs=[],
star_loc=None, dstar_loc=None)
call = ast.Call(args=args, keywords=keywords, starargs=None, kwargs=None,
star_loc=None, dstar_loc=None, loc=None)
else:
call.args = args
call.keywords = keywords + call.keywords
@@ -1106,7 +1122,8 @@ def argument(self):
"error", "keyword must be an identifier", {}, lhs.loc)
raise diagnostic.DiagnosticException(error)
return ast.keyword(arg=lhs.id, value=rhs,
loc=lhs.loc.join(rhs.loc), equals_loc=equals_loc)
loc=lhs.loc.join(rhs.loc),
arg_loc=lhs.loc, equals_loc=equals_loc)
else:
return lhs
argument.expected = test.expected
289 changes: 283 additions & 6 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,8 @@ def tearDownModule():

class ParserTestCase(unittest.TestCase):

maxDiff = None

def parser_for(self, code, version=(2, 6)):
code = code.replace("·", "\n")

@@ -36,7 +38,8 @@ def flatten_ast(self, node):
self.assertTrue(attr in node._locs,
"%s not in %s._locs" % (attr, repr(node)))
for loc in node._locs:
self.assertTrue(loc in node.__dict__)
self.assertTrue(loc in node.__dict__,
"%s not in %s._locs" % (loc, repr(node)))

flat_node = { 'ty': unicode(type(node).__name__) }
for field in node._fields:
@@ -122,6 +125,17 @@ def assertDiagnosesUnexpected(self, code, err_token, loc):
self.assertDiagnoses(code,
("error", "unexpected {actual}: expected {expected}", {'actual': err_token}, loc))

# Fixtures

ast_1 = {'ty': 'Num', 'n': 1}
ast_2 = {'ty': 'Num', 'n': 2}
ast_3 = {'ty': 'Num', 'n': 3}

ast_x = {'ty': 'Name', 'id': 'x', 'ctx': None}
ast_y = {'ty': 'Name', 'id': 'y', 'ctx': None}
ast_z = {'ty': 'Name', 'id': 'z', 'ctx': None}
ast_t = {'ty': 'Name', 'id': 't', 'ctx': None}

#
# LITERALS
#
@@ -162,8 +176,6 @@ def test_ident(self):
# OPERATORS
#

ast_1 = {'ty': 'Num', 'n': 1}

def test_unary(self):
self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'UAdd'}, 'operand': self.ast_1},
@@ -373,11 +385,276 @@ def test_boolop_multi(self):
" ~~ op_locs.1")

#
# STATEMENTS
# COMPOUND LITERALS
#

ast_x = {'ty': 'Name', 'id': 'x', 'ctx': None}
ast_y = {'ty': 'Name', 'id': 'y', 'ctx': None}
def test_tuple(self):
self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [], 'ctx': None},
"()",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [self.ast_1], 'ctx': None},
"(1,)",
"~~~~ loc")

self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [self.ast_1, self.ast_1], 'ctx': None},
"(1,1)",
"~~~~~ loc")

self.assertParsesExpr(
self.ast_1,
"(1)",
" ~ loc")

def test_list(self):
self.assertParsesExpr(
{'ty': 'List', 'elts': [], 'ctx': None},
"[]",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'List', 'elts': [self.ast_1], 'ctx': None},
"[1]",
"~~~ loc")

self.assertParsesExpr(
{'ty': 'List', 'elts': [self.ast_1, self.ast_1], 'ctx': None},
"[1,1]",
"~~~~~ loc")

def test_dict(self):
self.assertParsesExpr(
{'ty': 'Dict', 'keys': [], 'values': []},
"{}",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'Dict', 'keys': [self.ast_x], 'values': [self.ast_1]},
"{x: 1}",
"^ begin_loc"
" ^ end_loc"
" ^ colon_locs.0"
"~~~~~~ loc")

def test_repr(self):
self.assertParsesExpr(
{'ty': 'Repr', 'value': self.ast_1},
"`1`",
"^ begin_loc"
" ^ end_loc"
"~~~ loc")

#
# GENERATOR AND CONDITIONAL EXPRESSIONS
#

def test_list_comp(self):
self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y, 'ifs': []}
]},
"[x for y in z]",
"^ begin_loc"
" ~~~ generators.0.for_loc"
" ~~ generators.0.in_loc"
" ~~~~~~~~~~ generators.0.loc"
" ^ end_loc"
"~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_t]}
]},
"[x for y in z if t]",
" ~~ generators.0.if_locs.0"
" ~~~~~~~~~~~~~~~ generators.0.loc"
"~~~~~~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_x]},
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_t, 'ifs': []}
]},
"[x for y in z if x for t in z]",
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ loc")

def test_gen_comp(self):
self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y, 'ifs': []}
]},
"(x for y in z)",
"^ begin_loc"
" ~~~ generators.0.for_loc"
" ~~ generators.0.in_loc"
" ~~~~~~~~~~ generators.0.loc"
" ^ end_loc"
"~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_t]}
]},
"(x for y in z if t)",
" ~~ generators.0.if_locs.0"
" ~~~~~~~~~~~~~~~ generators.0.loc"
"~~~~~~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_x]},
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_t, 'ifs': []}
]},
"(x for y in z if x for t in z)",
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ loc")

def test_yield_expr(self):
self.assertParsesExpr(
{'ty': 'Yield', 'value': self.ast_1},
"(yield 1)",
" ~~~~~ keyword_loc"
" ~~~~~~~ loc")

def test_if_expr(self):
self.assertParsesExpr(
{'ty': 'IfExp', 'body': self.ast_x, 'test': self.ast_y, 'orelse': self.ast_z},
"x if y else z",
" ~~ if_loc"
" ~~~~ else_loc"
"~~~~~~~~~~~~~ loc")

#
# CALLS, ATTRIBUTES AND SUBSCRIPTS
#

def test_call(self):
self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': None,
'args': [], 'keywords': []},
"x()",
" ^ begin_loc"
" ^ end_loc"
"~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': None,
'args': [self.ast_y], 'keywords': [
{ 'ty': 'keyword', 'arg': 'z', 'value': self.ast_z}
]},
"x(y, z=z)",
" ^ keywords.0.arg_loc"
" ^ keywords.0.equals_loc"
" ~~~ keywords.0.loc"
"~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': None,
'args': [], 'keywords': []},
"x(*y)",
" ^ star_loc"
"~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': self.ast_z,
# 'args': [], 'keywords': []},
# "x(*y, **z)",
# " ^ star_loc"
# " ^^ dstar_loc"
# "~~~~~~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': self.ast_z,
# 'args': [self.ast_t], 'keywords': []},
# "x(*y, t, **z)",
# " ^ star_loc"
# " ^^ dstar_loc"
# "~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': self.ast_z,
'args': [], 'keywords': []},
"x(**z)",
" ^^ dstar_loc"
"~~~~~~ loc")

def test_subscript(self):
self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Index', 'value': self.ast_1}},
"x[1]",
" ^ begin_loc"
" ^ slice.loc"
" ^ end_loc"
"~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'ExtSlice', 'dims': [
{'ty': 'Index', 'value': self.ast_1},
{'ty': 'Index', 'value': self.ast_2},
]}},
"x[1, 2]",
" ~~~~ slice.loc"
"~~~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
# 'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': None, 'step': None}},
# "x[:1]",
# " ^ slice.bound_colon_loc"
# " ~~ slice.loc"
# "~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
"x[1:2]",
" ^ slice.bound_colon_loc"
" ~~~ slice.loc"
"~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
"x[1:2:]",
" ^ slice.bound_colon_loc"
" ^ slice.step_colon_loc"
" ~~~~ slice.loc"
"~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': self.ast_3}},
"x[1:2:3]",
" ^ slice.bound_colon_loc"
" ^ slice.step_colon_loc"
" ~~~~~ slice.loc"
"~~~~~~~~ loc")

def test_attribute(self):
self.assertParsesExpr(
{'ty': 'Attribute', 'value': self.ast_x, 'attr': 'zz', 'ctx': None},
"x.zz",
" ^ dot_loc"
" ~~ attr_loc"
"~~~~ loc")

#
# STATEMENTS
#

def test_assign(self):
self.assertParsesSuite(

0 comments on commit 656fb73

Please sign in to comment.