Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: a9fd0a06a2b2
Choose a base ref
...
head repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 45914f5a7fd0
Choose a head ref
  • 4 commits
  • 5 files changed
  • 1 contributor

Commits on Apr 24, 2015

  1. Basic human-friendly AST assertions in parser tests.

    whitequark committed Apr 24, 2015
    Copy the full SHA
    edf6bae View commit details
  2. Display detailed coverage information on Alt-rules.

    whitequark committed Apr 24, 2015
    Copy the full SHA
    83287d6 View commit details
  3. Clear up a few comments in lexer.

    whitequark committed Apr 24, 2015
    Copy the full SHA
    1da96c6 View commit details
  4. Add a DSL for AST location assertions.

    whitequark committed Apr 24, 2015
    Copy the full SHA
    45914f5 View commit details
Showing with 128 additions and 20 deletions.
  1. +1 −0 pyparser/ast.py
  2. +12 −6 pyparser/coverage/__init__.py
  3. +3 −3 pyparser/lexer.py
  4. +5 −3 pyparser/parser.py
  5. +107 −8 pyparser/test/test_parser.py
1 change: 1 addition & 0 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@

from __future__ import absolute_import, division, print_function, unicode_literals
from .shim import ast
from .shim.ast import AST

class commonloc(object):
"""
18 changes: 12 additions & 6 deletions pyparser/coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -63,20 +63,23 @@ def report(parser, name='parser'):
pts = len(rule.covered)
covered = len(filter(lambda x: x, rule.covered))
if covered == 0:
klass = 'uncovered'
klass, hint = 'uncovered', None
elif covered < pts:
klass = 'partial'
klass, hint = 'partial', ', '.join(map(lambda x: "yes" if x else "no", rule.covered))
else:
klass = 'covered'
klass, hint = 'covered', None

loc = source.Range(_buf, *rule.loc)
rewriter.insert_before(loc, r"<span class='%s'>" % klass)
if hint:
rewriter.insert_before(loc, r"<span class='%s' title='%s'>" % (klass, hint))
else:
rewriter.insert_before(loc, r"<span class='%s'>" % klass)
rewriter.insert_after(loc, r"</span>")

total_pts += pts
total_covered += covered

print("GRAMMAR COVERAGE: %.2f" % (total_covered / total_pts))
print("GRAMMAR COVERAGE: %.2f%%" % (total_covered / total_pts))

content = rewriter.rewrite().source
content = '\n'.join(map(
@@ -92,7 +95,10 @@ def report(parser, name='parser'):
<title>{percentage:.2f}%: {file} coverage report</title>
<style type="text/css">
.uncovered {{ background-color: #FFCAAD; }}
.partial {{ background-color: #FFFFB4; }}
.partial {{
background-color: #FFFFB4;
border-bottom: 1px dotted black;
}}
.covered {{ background-color: #9CE4B7; }}
pre {{ counter-reset: line; }}
.line::before {{
6 changes: 3 additions & 3 deletions pyparser/lexer.py
Original file line number Diff line number Diff line change
@@ -219,6 +219,8 @@ def peek(self, eof_token=False):

return self.queue[-1]

# We need separate next and _refill because lexing can sometimes
# generate several tokens, e.g. INDENT
def _refill(self, eof_token):
if self.offset == len(self.source_buffer.source):
if eof_token:
@@ -227,8 +229,6 @@ def _refill(self, eof_token):
else:
raise StopIteration

# We need separate next and _refill because lexing can sometimes
# generate several tokens, e.g. INDENT
match = self._lex_token_re.match(self.source_buffer.source, self.offset)
if match is None:
diag = diagnostic.Diagnostic(
@@ -239,7 +239,7 @@ def _refill(self, eof_token):

# Should we emit indent/dedent?
if self.new_line and \
match.group(3) is None: # not a newline
match.group(3) is None: # not a blank line
whitespace = match.string[match.start(0):match.start(1)]
level = len(whitespace.expandtabs())
range = source.Range(self.source_buffer, match.start(1), match.start(1))
8 changes: 5 additions & 3 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -321,10 +321,11 @@ def single_input(self, body):
return ast.Interactive(body=body, loc=loc)

@action(SeqN(0, Star(Alt(Newline(), Rule('stmt'))), Tok('eof')))
def file_input(parser, stmts):
def file_input(parser, body):
"""file_input: (NEWLINE | stmt)* ENDMARKER"""
body = reduce(list.__add__, body, [])
loc = None if body == [] else body[0].loc
return ast.Module(body=reduce(list.__add__, stmts, []), loc=loc)
return ast.Module(body=body, loc=loc)

@action(SeqN(0, Rule('testlist'), Star(Tok('newline')), Tok('eof')))
def eval_input(self, expr):
@@ -839,7 +840,8 @@ def atom_2(self, tok):
@action(Seq(Tok('strbegin'), Tok('strdata'), Tok('strend')))
def atom_3(self, begin_tok, data_tok, end_tok):
return ast.Str(s=data_tok.value,
begin_loc=begin_tok.loc, end_loc=end_tok.loc)
begin_loc=begin_tok.loc, end_loc=end_tok.loc,
loc=begin_tok.loc.join(end_tok.loc))

@action(Rule('testlist1'))
def atom_4(self, expr):
115 changes: 107 additions & 8 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
# coding:utf-8

from __future__ import absolute_import, division, print_function, unicode_literals
from .. import source, lexer, diagnostic, coverage
from .. import source, lexer, diagnostic, ast, coverage
from ..coverage import parser
import unittest
import unittest, re

def tearDownModule():
coverage.report(parser)

class ParserTestCase(unittest.TestCase):

def parser_for(self, code):
def parser_for(self, code, version=(2, 6)):
code = code.replace("·", "\n")

self.source_buffer = source.Buffer(code)
self.lexer = lexer.Lexer(self.source_buffer, (2,7))
self.lexer = lexer.Lexer(self.source_buffer, version)
self.parser = parser.Parser(self.lexer)

old_next = self.lexer.next
@@ -24,8 +26,78 @@ def lexer_next(**args):

return self.parser

def assertParses(self, ast, code):
self.assertEqual(ast, self.parser_for(code).file_input())
def flatten_ast(self, node):
# Validate locs
for attr in node.__dict__:
if attr.endswith('_loc') or attr.endswith('_locs'):
self.assertTrue(attr in node._locs)
for loc in node._locs:
self.assertTrue(loc in node.__dict__)

flat_node = { 'ty': unicode(type(node).__name__) }
for field in node._fields:
value = getattr(node, field)
if isinstance(value, ast.AST):
value = self.flatten_ast(value)
if isinstance(value, list) and len(value) > 0 and isinstance(value[0], ast.AST):
value = map(self.flatten_ast, value)
flat_node[unicode(field)] = value
return flat_node

_loc_re = re.compile(r"\s*([~^]+)\s+([a-z_0-9.]+)")
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(.)?")

def match_loc(self, ast, matcher, root=lambda x: x):
ast = root(ast)

matcher_pos = 0
while matcher_pos < len(matcher):
matcher_match = self._loc_re.match(matcher, matcher_pos)
if matcher_match is None:
raise Exception("invalid location matcher %s" % matcher[matcher_pos:])

range = source.Range(self.source_buffer,
matcher_match.start(1) - matcher_pos,
matcher_match.end(1) - matcher_pos)
path = matcher_match.group(2)

path_pos = 0
obj = ast
while path_pos < len(path):
path_match = self._path_re.match(path, path_pos)
if path_match is None:
raise Exception("invalid location matcher path %s" % path)

path_field = path_match.group(1)
path_index = path_match.group(2)
path_last = not path_match.group(3)

if path_field is not None:
obj = getattr(obj, path_field)
elif path_index is not None:
obj = obj[int(path_index)]

if path_last:
self.assertEqual(obj, range)

path_pos = path_match.end(0)

matcher_pos = matcher_match.end(0)

def assertParsesGen(self, expected_flat_ast, code):
ast = self.parser_for(code + "\n").file_input()
flat_ast = self.flatten_ast(ast)
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
flat_ast)
return ast

def assertParsesSuite(self, expected_flat_ast, code, loc_matcher=""):
ast = self.assertParsesGen(expected_flat_ast, code)
self.match_loc(ast, loc_matcher, lambda x: x.body)

def assertParsesExpr(self, expected_flat_ast, code, loc_matcher=""):
ast = self.assertParsesGen([{'ty': 'Expr', 'value': expected_flat_ast}], code)
self.match_loc(ast, loc_matcher, lambda x: x.body[0].value)

def assertDiagnoses(self, code, diag):
try:
@@ -46,6 +118,33 @@ def assertDiagnosesUnexpected(self, code, err_token, loc):
self.assertDiagnoses(code,
("error", "unexpected {actual}: expected {expected}", {'actual': err_token}, loc))

def test_pass(self):
self.assertParses(None, "pass\n")
#
# LITERALS
#

def test_int(self):
self.assertParsesExpr(
{'ty': 'Num', 'n': 1},
"1",
"^ loc")

def test_float(self):
self.assertParsesExpr(
{'ty': 'Num', 'n': 1.0},
"1.0",
"~~~ loc")

def test_complex(self):
self.assertParsesExpr(
{'ty': 'Num', 'n': 1j},
"1j",
"~~ loc")

def test_string(self):
self.assertParsesExpr(
{'ty': 'Str', 's': 'foo'},
"'foo'",
"~~~~~ loc"
"^ begin_loc"
" ^ end_loc")