Skip to content

Commit

Permalink
45% coverage.
Browse files Browse the repository at this point in the history
All expressions except lambdas and a few forms of calls and
subscripts are handled. The omissions are due to LL conflicts.
whitequark committed May 4, 2015

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent da78a09 commit 656fb73
Showing 4 changed files with 332 additions and 33 deletions.
6 changes: 4 additions & 2 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -160,6 +160,7 @@ class Attribute(expr, ast.Attribute):
:ivar value: (node) left-hand side
:ivar attr: (string) attribute name
"""
_locs = expr._locs + ('dot_loc', 'attr_loc')
class BinOp(expr, ast.BinOp):
"""
A binary operation, e.g. ``x + y``.
@@ -316,10 +317,11 @@ class UnaryOp(expr, ast.UnaryOp):
"""
class Yield(expr, ast.Yield):
"""
A yield expression, e.g. ``yield x``.
A yield expression, e.g. ``(yield x)``.
:ivar value: (node) yielded value
"""
_locs = expr._locs + ('keyword_loc',)

# expr_context
# AugLoad
@@ -337,7 +339,7 @@ class keyword(commonloc, ast.keyword):
:ivar value: (node) value
:ivar equals_loc: location of ``=``
"""
_locs = commonloc._locs + ('equals_loc',)
_locs = commonloc._locs + ('arg_loc', 'equals_loc')

class mod(commonloc):
"""Base class for modules (groups of statements)."""
3 changes: 3 additions & 0 deletions pyparser/coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -29,6 +29,9 @@ def instrument():
if not in_grammar:
continue

if token.kind == '.': # skip ast.List
lex.next()

if token.kind == 'ident' and \
token.value in ('action', 'Eps', 'Tok', 'Loc', 'Rule', 'Expect',
'Seq', 'SeqN', 'Alt', 'Opt', 'Star', 'Plus', 'List',
67 changes: 42 additions & 25 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -199,6 +199,9 @@ def rule(parser):
results.append(result)
return rule

class commalist(list):
__slots__ = ('trailing_comma',)

def List(inner_rule, separator_tok, trailing, leading=True, loc=None):
if not trailing:
@action(Seq(inner_rule, Star(SeqN(1, Tok(separator_tok), inner_rule))), loc=loc)
@@ -213,7 +216,7 @@ def outer_rule(parser, first, rest):
separator_rule = Tok(separator_tok)
@llrule(loc, inner_rule.expected)
def rule(parser):
results = []
results = commalist()

if leading:
result = inner_rule(parser)
@@ -225,12 +228,12 @@ def rule(parser):
while True:
result = separator_rule(parser)
if result is unmatched:
#results.trailing = True
results.trailing_comma = False
return results

result = inner_rule(parser)
if result is unmatched:
#results.trailing = False
results.trailing_comma = True
return results
else:
results.append(result)
@@ -271,9 +274,15 @@ def BeginEnd(begin_tok, inner_rule, end_tok, empty=None, loc=None):
def rule(parser, begin_loc, node, end_loc):
if node is None:
node = empty()
node.ctx = None
node.begin_loc, node.end_loc, node.loc = \
begin_loc, end_loc, begin_loc.join(end_loc)

# Collection nodes don't have loc yet. If a node has loc at this
# point, it means it's an expression passed in parentheses.
if node.loc is None and type(node) in [
ast.List, ast.Dict, ast.Tuple, ast.Repr,
ast.ListComp, ast.GeneratorExp,
ast.Call, ast.Subscript]:
node.begin_loc, node.end_loc, node.loc = \
begin_loc, end_loc, begin_loc.join(end_loc)
return node
return rule

@@ -838,6 +847,7 @@ def power(self, atom, trailers, factor_opt):
trailer.value = atom
elif isinstance(trailer, ast.Call):
trailer.func = atom
trailer.loc = atom.loc.join(trailer.loc)
atom = trailer
if factor_opt:
op_loc, factor = factor_opt
@@ -862,14 +872,15 @@ def atom_3(self, begin_tok, data_tok, end_tok):

@action(Rule('testlist1'))
def atom_4(self, expr):
return ast.Repr(value=expr, loc=expr.loc)
return ast.Repr(value=expr, loc=None)

atom = Alt(BeginEnd('(', Opt(Alt(Rule('yield_expr'), Rule('testlist_gexp'))), ')',
empty=lambda: ast.Tuple(elts=[])),
empty=lambda: ast.Tuple(elts=[], ctx=None, loc=None)),
BeginEnd('[', Opt(Rule('listmaker')), ']',
empty=lambda: ast.List(elts=[])),
empty=lambda: ast.List(elts=[], ctx=None, loc=None)),
BeginEnd('{', Opt(Rule('dictmaker')), '}',
empty=lambda: ast.Dict(keys=[], values=[], colon_locs=[])),
empty=lambda: ast.Dict(keys=[], values=[], colon_locs=[],
ctx=None, loc=None)),
BeginEnd('`', atom_4, '`'),
atom_1, atom_2, atom_3)
"""atom: ('(' [yield_expr|testlist_gexp] ')' |
@@ -890,11 +901,11 @@ def list_gen_action(self, lhs, rhs):

@action(Rule('list_for'))
def listmaker_1(self, compose):
return ast.ListComp(generators=compose([]))
return ast.ListComp(generators=compose([]), loc=None)

@action(List(Rule('test'), ',', trailing=True, leading=False))
def listmaker_2(self, elts):
return ast.List(elts=elts, ctx=None)
return ast.List(elts=elts, ctx=None, loc=None)

listmaker = action(
Seq(Rule('test'),
@@ -904,14 +915,14 @@ def listmaker_2(self, elts):

@action(Rule('gen_for'))
def testlist_gexp_1(self, compose):
return ast.GeneratorExp(generators=compose([]))
return ast.GeneratorExp(generators=compose([]), loc=None)

@action(List(Rule('test'), ',', trailing=True))
@action(List(Rule('test'), ',', trailing=True, leading=False))
def testlist_gexp_2(self, elts):
if elts == [] and not elts.trailing:
if elts == [] and not elts.trailing_comma:
return None
else:
return ast.Tuple(elts=elts, ctx=None)
return ast.Tuple(elts=elts, ctx=None, loc=None)

testlist_gexp = action(
Seq(Rule('test'), Alt(testlist_gexp_1, testlist_gexp_2))) \
@@ -929,7 +940,8 @@ def lambdef(self, lambda_loc, args_opt, colon_loc, body):
@action(Seq(Loc('.'), Tok('ident')))
def trailer_1(self, dot_loc, ident_tok):
return ast.Attribute(attr=ident_tok.value, ctx=None,
loc=dot_loc.join(ident_tok.loc), dot_loc=dot_loc)
loc=dot_loc.join(ident_tok.loc),
attr_loc=ident_tok.loc, dot_loc=dot_loc)

trailer = Alt(BeginEnd('(', Rule('arglist'), ')'),
BeginEnd('[', Rule('subscriptlist'), ']'),
@@ -940,11 +952,11 @@ def trailer_1(self, dot_loc, ident_tok):
def subscriptlist(self, subscripts):
"""subscriptlist: subscript (',' subscript)* [',']"""
if len(subscripts) == 1:
return ast.Subscript(slice=subscripts[0], ctx=None)
return ast.Subscript(slice=subscripts[0], ctx=None, loc=None)
else:
extslice = ast.ExtSlice(dims=subscripts,
loc=subscripts[0].loc.join(subscripts[-1].loc))
return ast.Subscript(slice=extslice, ctx=None)
return ast.Subscript(slice=extslice, ctx=None, loc=None)

@action(Seq(Loc('.'), Loc('.'), Loc('.')))
def subscript_1(self, dot_1_loc, dot_2_loc, dot_3_loc):
@@ -970,6 +982,7 @@ def subscript_3(self, lower_opt, colon_loc, upper_opt, step_opt):
loc=loc, bound_colon_loc=colon_loc, step_colon_loc=step_colon_loc)

subscript_3_inner = Expect(Seq(Opt(Rule('test')), Opt(Rule('sliceop'))))

def subscript(self):
"""subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]"""
# This requires manual disambiguation of `test | [test] ...`
@@ -987,6 +1000,8 @@ def subscript(self):

result_2 = self.subscript_3_inner()
return self.subscript_3(result, result_1.loc, *result_2)
subscript.expected = \
lambda parser: parser.subscript_1.expected(parser) + parser.test.expected(parser)

sliceop = Seq(Loc(':'), Opt(Rule('test')))
"""sliceop: ':' [test]"""
@@ -1006,7 +1021,8 @@ def dictmaker(self, elts):
"""dictmaker: test ':' test (',' test ':' test)* [',']"""
return ast.Dict(keys=list(map(lambda x: x[0], elts)),
values=list(map(lambda x: x[2], elts)),
colon_locs=list(map(lambda x: x[1], elts)))
colon_locs=list(map(lambda x: x[1], elts)),
loc=None)

@action(Seq(Loc('class'), Tok('ident'),
Opt(BeginEnd('(', Rule('testlist'), ')')),
@@ -1035,12 +1051,12 @@ def arglist_1(self, star_loc, stararg, postargs, kwarg_opt):
raise diagnostic.DiagnosticException(error)

return ast.Call(starargs=stararg, kwargs=kwarg, keywords=postargs,
star_loc=star_loc, dstar_loc=dstar_loc)
star_loc=star_loc, dstar_loc=dstar_loc, loc=None)

@action(Seq(Loc('**'), Rule('test')))
def arglist_2(self, dstar_loc, kwarg):
return ast.Call(starargs=None, kwargs=kwarg, keywords=[],
star_loc=None, dstar_loc=dstar_loc)
star_loc=None, dstar_loc=dstar_loc, loc=None)

arglist_3 = Alt(arglist_1, arglist_2)

@@ -1066,8 +1082,8 @@ def arglist(self):

call = self.arglist_3()
if call is unmatched:
call = ast.Call(args=args, keywords=keywords, starargs=[], kwargs=[],
star_loc=None, dstar_loc=None)
call = ast.Call(args=args, keywords=keywords, starargs=None, kwargs=None,
star_loc=None, dstar_loc=None, loc=None)
else:
call.args = args
call.keywords = keywords + call.keywords
@@ -1106,7 +1122,8 @@ def argument(self):
"error", "keyword must be an identifier", {}, lhs.loc)
raise diagnostic.DiagnosticException(error)
return ast.keyword(arg=lhs.id, value=rhs,
loc=lhs.loc.join(rhs.loc), equals_loc=equals_loc)
loc=lhs.loc.join(rhs.loc),
arg_loc=lhs.loc, equals_loc=equals_loc)
else:
return lhs
argument.expected = test.expected
289 changes: 283 additions & 6 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,8 @@ def tearDownModule():

class ParserTestCase(unittest.TestCase):

maxDiff = None

def parser_for(self, code, version=(2, 6)):
code = code.replace("·", "\n")

@@ -36,7 +38,8 @@ def flatten_ast(self, node):
self.assertTrue(attr in node._locs,
"%s not in %s._locs" % (attr, repr(node)))
for loc in node._locs:
self.assertTrue(loc in node.__dict__)
self.assertTrue(loc in node.__dict__,
"%s not in %s._locs" % (loc, repr(node)))

flat_node = { 'ty': unicode(type(node).__name__) }
for field in node._fields:
@@ -122,6 +125,17 @@ def assertDiagnosesUnexpected(self, code, err_token, loc):
self.assertDiagnoses(code,
("error", "unexpected {actual}: expected {expected}", {'actual': err_token}, loc))

# Fixtures

ast_1 = {'ty': 'Num', 'n': 1}
ast_2 = {'ty': 'Num', 'n': 2}
ast_3 = {'ty': 'Num', 'n': 3}

ast_x = {'ty': 'Name', 'id': 'x', 'ctx': None}
ast_y = {'ty': 'Name', 'id': 'y', 'ctx': None}
ast_z = {'ty': 'Name', 'id': 'z', 'ctx': None}
ast_t = {'ty': 'Name', 'id': 't', 'ctx': None}

#
# LITERALS
#
@@ -162,8 +176,6 @@ def test_ident(self):
# OPERATORS
#

ast_1 = {'ty': 'Num', 'n': 1}

def test_unary(self):
self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'UAdd'}, 'operand': self.ast_1},
@@ -373,11 +385,276 @@ def test_boolop_multi(self):
" ~~ op_locs.1")

#
# STATEMENTS
# COMPOUND LITERALS
#

ast_x = {'ty': 'Name', 'id': 'x', 'ctx': None}
ast_y = {'ty': 'Name', 'id': 'y', 'ctx': None}
def test_tuple(self):
self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [], 'ctx': None},
"()",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [self.ast_1], 'ctx': None},
"(1,)",
"~~~~ loc")

self.assertParsesExpr(
{'ty': 'Tuple', 'elts': [self.ast_1, self.ast_1], 'ctx': None},
"(1,1)",
"~~~~~ loc")

self.assertParsesExpr(
self.ast_1,
"(1)",
" ~ loc")

def test_list(self):
self.assertParsesExpr(
{'ty': 'List', 'elts': [], 'ctx': None},
"[]",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'List', 'elts': [self.ast_1], 'ctx': None},
"[1]",
"~~~ loc")

self.assertParsesExpr(
{'ty': 'List', 'elts': [self.ast_1, self.ast_1], 'ctx': None},
"[1,1]",
"~~~~~ loc")

def test_dict(self):
self.assertParsesExpr(
{'ty': 'Dict', 'keys': [], 'values': []},
"{}",
"^ begin_loc"
" ^ end_loc"
"~~ loc")

self.assertParsesExpr(
{'ty': 'Dict', 'keys': [self.ast_x], 'values': [self.ast_1]},
"{x: 1}",
"^ begin_loc"
" ^ end_loc"
" ^ colon_locs.0"
"~~~~~~ loc")

def test_repr(self):
self.assertParsesExpr(
{'ty': 'Repr', 'value': self.ast_1},
"`1`",
"^ begin_loc"
" ^ end_loc"
"~~~ loc")

#
# GENERATOR AND CONDITIONAL EXPRESSIONS
#

def test_list_comp(self):
self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y, 'ifs': []}
]},
"[x for y in z]",
"^ begin_loc"
" ~~~ generators.0.for_loc"
" ~~ generators.0.in_loc"
" ~~~~~~~~~~ generators.0.loc"
" ^ end_loc"
"~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_t]}
]},
"[x for y in z if t]",
" ~~ generators.0.if_locs.0"
" ~~~~~~~~~~~~~~~ generators.0.loc"
"~~~~~~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_x]},
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_t, 'ifs': []}
]},
"[x for y in z if x for t in z]",
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ loc")

def test_gen_comp(self):
self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y, 'ifs': []}
]},
"(x for y in z)",
"^ begin_loc"
" ~~~ generators.0.for_loc"
" ~~ generators.0.in_loc"
" ~~~~~~~~~~ generators.0.loc"
" ^ end_loc"
"~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_t]}
]},
"(x for y in z if t)",
" ~~ generators.0.if_locs.0"
" ~~~~~~~~~~~~~~~ generators.0.loc"
"~~~~~~~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'GeneratorExp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [self.ast_x]},
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_t, 'ifs': []}
]},
"(x for y in z if x for t in z)",
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ loc")

def test_yield_expr(self):
self.assertParsesExpr(
{'ty': 'Yield', 'value': self.ast_1},
"(yield 1)",
" ~~~~~ keyword_loc"
" ~~~~~~~ loc")

def test_if_expr(self):
self.assertParsesExpr(
{'ty': 'IfExp', 'body': self.ast_x, 'test': self.ast_y, 'orelse': self.ast_z},
"x if y else z",
" ~~ if_loc"
" ~~~~ else_loc"
"~~~~~~~~~~~~~ loc")

#
# CALLS, ATTRIBUTES AND SUBSCRIPTS
#

def test_call(self):
self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': None,
'args': [], 'keywords': []},
"x()",
" ^ begin_loc"
" ^ end_loc"
"~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': None,
'args': [self.ast_y], 'keywords': [
{ 'ty': 'keyword', 'arg': 'z', 'value': self.ast_z}
]},
"x(y, z=z)",
" ^ keywords.0.arg_loc"
" ^ keywords.0.equals_loc"
" ~~~ keywords.0.loc"
"~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': None,
'args': [], 'keywords': []},
"x(*y)",
" ^ star_loc"
"~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': self.ast_z,
# 'args': [], 'keywords': []},
# "x(*y, **z)",
# " ^ star_loc"
# " ^^ dstar_loc"
# "~~~~~~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Call', 'func': self.ast_x, 'starargs': self.ast_y, 'kwargs': self.ast_z,
# 'args': [self.ast_t], 'keywords': []},
# "x(*y, t, **z)",
# " ^ star_loc"
# " ^^ dstar_loc"
# "~~~~~~~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Call', 'func': self.ast_x, 'starargs': None, 'kwargs': self.ast_z,
'args': [], 'keywords': []},
"x(**z)",
" ^^ dstar_loc"
"~~~~~~ loc")

def test_subscript(self):
self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Index', 'value': self.ast_1}},
"x[1]",
" ^ begin_loc"
" ^ slice.loc"
" ^ end_loc"
"~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'ExtSlice', 'dims': [
{'ty': 'Index', 'value': self.ast_1},
{'ty': 'Index', 'value': self.ast_2},
]}},
"x[1, 2]",
" ~~~~ slice.loc"
"~~~~~~~ loc")

# self.assertParsesExpr(
# {'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
# 'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': None, 'step': None}},
# "x[:1]",
# " ^ slice.bound_colon_loc"
# " ~~ slice.loc"
# "~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
"x[1:2]",
" ^ slice.bound_colon_loc"
" ~~~ slice.loc"
"~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': None}},
"x[1:2:]",
" ^ slice.bound_colon_loc"
" ^ slice.step_colon_loc"
" ~~~~ slice.loc"
"~~~~~~~ loc")

self.assertParsesExpr(
{'ty': 'Subscript', 'value': self.ast_x, 'ctx': None,
'slice': {'ty': 'Slice', 'lower': self.ast_1, 'upper': self.ast_2, 'step': self.ast_3}},
"x[1:2:3]",
" ^ slice.bound_colon_loc"
" ^ slice.step_colon_loc"
" ~~~~~ slice.loc"
"~~~~~~~~ loc")

def test_attribute(self):
self.assertParsesExpr(
{'ty': 'Attribute', 'value': self.ast_x, 'attr': 'zz', 'ctx': None},
"x.zz",
" ^ dot_loc"
" ~~ attr_loc"
"~~~~ loc")

#
# STATEMENTS
#

def test_assign(self):
self.assertParsesSuite(

0 comments on commit 656fb73

Please sign in to comment.