Skip to content

Commit

Permalink
25% grammar coverage.
Browse files Browse the repository at this point in the history
whitequark committed Apr 29, 2015
1 parent 45914f5 commit 8861f5d
Showing 4 changed files with 363 additions and 22 deletions.
16 changes: 13 additions & 3 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -86,8 +86,15 @@ class arguments(commonloc):
:ivar default_equals_locs: locations of ``=``
"""

class boolop(commonloc):
"""Base class for binary boolean operators."""
class boolop:
"""
Base class for binary boolean operators.
This class is unlike others in that it does not have the ``loc`` field.
It serves only as an indicator of operation and corresponds to no source
itself; locations are recorded in :class:`BoolOp`.
"""
_locs = ()
class And(boolop, ast.And):
"""The ``and`` operator."""
class Or(boolop, ast.Or):
@@ -168,7 +175,9 @@ class BoolOp(expr, ast.BoolOp):
:ivar left: (node) left-hand side
:ivar op: (:class:`boolop`) operator
:ivar right: (node) right-hand side
:ivar op_locs: locations of operators
"""
_locs = expr._locs + ('op_locs',)
class Call(beginendloc, expr, ast.Call):
"""
A function call, e.g. ``f(x, y=1, *z, **t)``.
@@ -414,8 +423,9 @@ class Assign(stmt, ast.Assign):
:ivar targets: (list of assignable node) left-hand sides
:ivar value: (node) right-hand side
:ivar ops_loc: location of equality signs corresponding to ``targets``
:ivar op_locs: location of equality signs corresponding to ``targets``
"""
_locs = stmt._locs + ('op_locs',)
class AugAssign(stmt, ast.AugAssign):
"""
The operator-assignment statement, e.g. ``+=``.
4 changes: 2 additions & 2 deletions pyparser/coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -79,7 +79,7 @@ def report(parser, name='parser'):
total_pts += pts
total_covered += covered

print("GRAMMAR COVERAGE: %.2f%%" % (total_covered / total_pts))
print("GRAMMAR COVERAGE: %.2f%%" % (total_covered / total_pts * 100))

content = rewriter.rewrite().source
content = '\n'.join(map(
@@ -117,7 +117,7 @@ def report(parser, name='parser'):
<pre>{content}</pre>
</body>
</html>
""".format(percentage=total_covered / total_pts,
""".format(percentage=total_covered / total_pts * 100,
pts=total_pts, covered=total_covered,
file=os.path.basename(_buf.name),
content=content))
40 changes: 28 additions & 12 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -414,7 +414,7 @@ def expr_stmt_1(self, augassign, rhs_expr):
def expr_stmt_2(self, seq):
if len(seq) > 0:
return ast.Assign(targets=map(lambda x: x[1], seq[:-1]), value=seq[-1][1],
ops_loc=map(lambda x: x[0], seq))
op_locs=map(lambda x: x[0], seq))
else:
return None

@@ -423,18 +423,18 @@ def expr_stmt(self, lhs, rhs):
"""expr_stmt: testlist (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist))*)"""
if isinstance(rhs, ast.AugAssign):
if len(lhs.elts) == 1:
rhs.target = self._assignable(lhs.elts[0])
rhs.loc = rhs.target.loc.join(rhs.value.loc)
return rhs
else:
if isinstance(lhs, ast.Tuple):
error = diagnostic.Diagnostic(
"error", "illegal expression for augmented assignment", {}, rhs.loc)
raise diagnostic.DiagnosticException(error)
else:
rhs.target = self._assignable(lhs)
rhs.loc = rhs.target.loc.join(rhs.value.loc)
return rhs
elif rhs is not None:
rhs.targets = map(self._assignable, [lhs] + rhs.targets)
rhs.loc = lhs.loc.join(rhs.value.loc)
return ast.Expr(value=rhs, loc=rhs.loc)
return rhs
else:
return ast.Expr(value=lhs, loc=lhs.loc)

@@ -754,11 +754,27 @@ def test_1(self, lhs, rhs_opt):
test = Alt(test_1, Rule('lambdef'))
"""test: or_test ['if' or_test 'else' test] | lambdef"""

or_test = BinOper('and_test', Oper(ast.Or, 'or'), node=ast.BoolOp)
"""or_test: and_test ('or' and_test)*"""
@action(Seq(Rule('and_test'), Star(Seq(Loc('or'), Rule('and_test')))))
def or_test(self, lhs, rhs):
"""or_test: and_test ('or' and_test)*"""
if len(rhs) > 0:
return ast.BoolOp(op=ast.Or(),
values=[lhs] + map(lambda x: x[1], rhs),
loc=lhs.loc.join(rhs[-1][1].loc),
op_locs=map(lambda x: x[0], rhs))
else:
return lhs

and_test = BinOper('not_test', Oper(ast.And, 'and'), node=ast.BoolOp)
"""and_test: not_test ('and' not_test)*"""
@action(Seq(Rule('not_test'), Star(Seq(Loc('and'), Rule('not_test')))))
def and_test(self, lhs, rhs):
"""and_test: not_test ('and' not_test)*"""
if len(rhs) > 0:
return ast.BoolOp(op=ast.And(),
values=[lhs] + map(lambda x: x[1], rhs),
loc=lhs.loc.join(rhs[-1][1].loc),
op_locs=map(lambda x: x[0], rhs))
else:
return lhs

@action(Seq(Oper(ast.Not, 'not'), Rule('not_test')))
def not_test_1(self, op, operand):
@@ -779,7 +795,7 @@ def comparison(self, lhs, rhs):
return lhs

comp_op = Alt(Oper(ast.Lt, '<'), Oper(ast.Gt, '>'), Oper(ast.Eq, '=='),
Oper(ast.GtE, '>='), Oper(ast.GtE, '<='), Oper(ast.NotEq, '<>'),
Oper(ast.GtE, '>='), Oper(ast.LtE, '<='), Oper(ast.NotEq, '<>'),
Oper(ast.NotEq, '!='),
Oper(ast.In, 'in'), Oper(ast.NotIn, 'not', 'in'),
Oper(ast.Is, 'is'), Oper(ast.IsNot, 'is', 'not'))
325 changes: 320 additions & 5 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -30,7 +30,8 @@ def flatten_ast(self, node):
# Validate locs
for attr in node.__dict__:
if attr.endswith('_loc') or attr.endswith('_locs'):
self.assertTrue(attr in node._locs)
self.assertTrue(attr in node._locs,
"%s not in %s._locs" % (attr, repr(node)))
for loc in node._locs:
self.assertTrue(loc in node.__dict__)

@@ -45,7 +46,7 @@ def flatten_ast(self, node):
return flat_node

_loc_re = re.compile(r"\s*([~^]+)\s+([a-z_0-9.]+)")
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(.)?")
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")

def match_loc(self, ast, matcher, root=lambda x: x):
ast = root(ast)
@@ -68,9 +69,9 @@ def match_loc(self, ast, matcher, root=lambda x: x):
if path_match is None:
raise Exception("invalid location matcher path %s" % path)

path_field = path_match.group(1)
path_index = path_match.group(2)
path_last = not path_match.group(3)
path_field = path_match.group(2)
path_index = path_match.group(3)
path_last = not path_match.group(4)

if path_field is not None:
obj = getattr(obj, path_field)
@@ -148,3 +149,317 @@ def test_string(self):
"^ begin_loc"
" ^ end_loc")

def test_ident(self):
self.assertParsesExpr(
{'ty': 'Name', 'id': 'foo', 'ctx': None},
"foo",
"~~~ loc")

#
# OPERATORS
#

ast_1 = {'ty': 'Num', 'n': 1}

def test_unary(self):
self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'UAdd'}, 'operand': self.ast_1},
"+1",
"~~ loc"
"~ op.loc")

self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'USub'}, 'operand': self.ast_1},
"-1",
"~~ loc"
"~ op.loc")

self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'Invert'}, 'operand': self.ast_1},
"~1",
"~~ loc"
"~ op.loc")

def test_binary(self):
self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Pow'}, 'left': self.ast_1, 'right': self.ast_1},
"1 ** 1",
"~~~~~~ loc"
" ~~ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Mult'}, 'left': self.ast_1, 'right': self.ast_1},
"1 * 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Div'}, 'left': self.ast_1, 'right': self.ast_1},
"1 / 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Mod'}, 'left': self.ast_1, 'right': self.ast_1},
"1 % 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'FloorDiv'}, 'left': self.ast_1, 'right': self.ast_1},
"1 // 1",
"~~~~~~ loc"
" ~~ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Add'}, 'left': self.ast_1, 'right': self.ast_1},
"1 + 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Sub'}, 'left': self.ast_1, 'right': self.ast_1},
"1 - 1",
"~~~~~ loc"
" ^ op.loc")

def test_bitwise(self):
self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'LShift'}, 'left': self.ast_1, 'right': self.ast_1},
"1 << 1",
"~~~~~~ loc"
" ~~ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'RShift'}, 'left': self.ast_1, 'right': self.ast_1},
"1 >> 1",
"~~~~~~ loc"
" ~~ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'BitAnd'}, 'left': self.ast_1, 'right': self.ast_1},
"1 & 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'BitOr'}, 'left': self.ast_1, 'right': self.ast_1},
"1 | 1",
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'BitXor'}, 'left': self.ast_1, 'right': self.ast_1},
"1 ^ 1",
"~~~~~ loc"
" ^ op.loc")

def test_compare(self):
self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'Lt'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 < 1",
"~~~~~ loc"
" ^ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'LtE'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 <= 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'Gt'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 > 1",
"~~~~~ loc"
" ^ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'GtE'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 >= 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'Eq'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 == 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'NotEq'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 != 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'NotEq'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 <> 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'In'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 in 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'NotIn'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 not in 1",
"~~~~~~~~~~ loc"
" ~~~~~~ ops.0.loc")

self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'Is'}],
'left': self.ast_1, 'comparators': [self.ast_1]},
"1 is 1",
"~~~~~~ loc"
" ~~ ops.0.loc")

# self.assertParsesExpr(
# {'ty': 'Compare', 'ops': [{'ty': 'NotIs'}],
# 'left': self.ast_1, 'comparators': [self.ast_1]},
# "1 is not 1",
# "~~~~~~~~~~ loc"
# " ~~~~~~ ops.0.loc")

def test_compare_multi(self):
self.assertParsesExpr(
{'ty': 'Compare', 'ops': [{'ty': 'Lt'}, {'ty': 'LtE'}],
'left': self.ast_1,
'comparators': [{'ty': 'Num', 'n': 2}, {'ty': 'Num', 'n': 3}]},
"1 < 2 <= 3",
"~~~~~~~~~~ loc"
" ^ ops.0.loc"
" ~~ ops.1.loc")

def test_boolop(self):
self.assertParsesExpr(
{'ty': 'BoolOp', 'op': {'ty': 'And'}, 'values': [self.ast_1, self.ast_1]},
"1 and 1",
"~~~~~~~ loc"
" ~~~ op_locs.0")

self.assertParsesExpr(
{'ty': 'BoolOp', 'op': {'ty': 'Or'}, 'values': [self.ast_1, self.ast_1]},
"1 or 1",
"~~~~~~ loc"
" ~~ op_locs.0")

self.assertParsesExpr(
{'ty': 'UnaryOp', 'op': {'ty': 'Not'}, 'operand': self.ast_1},
"not 1",
"~~~~~ loc"
"~~~ op.loc")

def test_boolop_multi(self):
self.assertParsesExpr(
{'ty': 'BoolOp', 'op': {'ty': 'Or'}, 'values': [self.ast_1, self.ast_1, self.ast_1]},
"1 or 1 or 1",
"~~~~~~~~~~~ loc"
" ~~ op_locs.0"
" ~~ op_locs.1")

#
# STATEMENTS
#

ast_x = {'ty': 'Name', 'id': 'x', 'ctx': None}
ast_y = {'ty': 'Name', 'id': 'y', 'ctx': None}

def test_assign(self):
self.assertParsesSuite(
[{'ty': 'Assign', 'targets': [self.ast_x], 'value': self.ast_1}],
"x = 1",
"~~~~~ 0.loc"
" ^ 0.op_locs.0")

self.assertParsesSuite(
[{'ty': 'Assign', 'targets': [self.ast_x, self.ast_y], 'value': self.ast_1}],
"x = y = 1",
"~~~~~~~~~ 0.loc"
" ^ 0.op_locs.0"
" ^ 0.op_locs.1")

def test_augassign(self):
self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Add'}, 'target': self.ast_x, 'value': self.ast_1}],
"x += 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Sub'}, 'target': self.ast_x, 'value': self.ast_1}],
"x -= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Mult'}, 'target': self.ast_x, 'value': self.ast_1}],
"x *= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Div'}, 'target': self.ast_x, 'value': self.ast_1}],
"x /= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Mod'}, 'target': self.ast_x, 'value': self.ast_1}],
"x %= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Pow'}, 'target': self.ast_x, 'value': self.ast_1}],
"x **= 1",
"~~~~~~~ 0.loc"
" ~~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'FloorDiv'}, 'target': self.ast_x, 'value': self.ast_1}],
"x //= 1",
"~~~~~~~ 0.loc"
" ~~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'RShift'}, 'target': self.ast_x, 'value': self.ast_1}],
"x >>= 1",
"~~~~~~~ 0.loc"
" ~~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'LShift'}, 'target': self.ast_x, 'value': self.ast_1}],
"x <<= 1",
"~~~~~~~ 0.loc"
" ~~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'BitAnd'}, 'target': self.ast_x, 'value': self.ast_1}],
"x &= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'BitOr'}, 'target': self.ast_x, 'value': self.ast_1}],
"x |= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'BitXor'}, 'target': self.ast_x, 'value': self.ast_1}],
"x ^= 1",
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

0 comments on commit 8861f5d

Please sign in to comment.