Skip to content

Commit

Permalink
100% grammar coverage.
Browse files Browse the repository at this point in the history
whitequark committed May 7, 2015
1 parent f52cd79 commit 25e4d3f
Showing 3 changed files with 175 additions and 46 deletions.
4 changes: 2 additions & 2 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -89,10 +89,10 @@ class arguments(beginendloc, ast.arguments):
:ivar vararg_loc: location of splat formal argument, if any
:ivar dstar_loc: location of ``**``, if any
:ivar kwarg_loc: location of keyword splat formal argument, if any
:ivar default_equals_locs: locations of ``=``
:ivar equals_locs: locations of ``=``
"""
_locs = beginendloc._locs + ('star_loc', 'vararg_loc', 'dstar_loc',
'vararg_loc', 'kwarg_loc', 'default_equals_locs')
'vararg_loc', 'kwarg_loc', 'equals_locs')

class boolop:
"""
80 changes: 61 additions & 19 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -392,7 +392,7 @@ def _assignable(self, node):
def _empty_arguments(self):
return ast.arguments(args=[], defaults=[], vararg=None, kwarg=None,
star_loc=None, vararg_loc=None, dstar_loc=None, kwarg_loc=None,
default_equals_locs=[], begin_loc=None, end_loc=None, loc=None)
equals_locs=[], begin_loc=None, end_loc=None, loc=None)

def _empty_arglist(self):
return ast.Call(args=[], keywords=[], starargs=None, kwargs=None,
@@ -464,36 +464,78 @@ def parameters_1(self, args):
parameters = BeginEnd('(', parameters_1, ')')
"""parameters: '(' [varargslist] ')'"""

varargslist_1 = Seq(Rule('fpdef'), Opt(Seq(Loc('='), Rule('test'))))

@action(Seq(Loc('**'), Tok('ident')))
def varargslist_1(self, dstar_loc, kwarg_tok):
return ast.arguments(vararg=None, kwarg=kwarg_tok.value,
def varargslist_2(self, dstar_loc, kwarg_tok):
return ast.arguments(args=[], defaults=[], vararg=None, kwarg=kwarg_tok.value,
star_loc=None, vararg_loc=None,
dstar_loc=dstar_loc, kwarg_loc=kwarg_tok.loc)
dstar_loc=dstar_loc, kwarg_loc=kwarg_tok.loc,
begin_loc=None, end_loc=None, equals_locs=[], loc=kwarg_tok.loc)

@action(Seq(Loc('*'), Tok('ident'),
Opt(Seq(Loc('**'), Tok('ident')))))
def varargslist_2(self, star_loc, vararg_tok, kwarg_opt):
dstar_loc, kwarg, kwarg_loc = None
Opt(Seq(Tok(','), Loc('**'), Tok('ident')))))
def varargslist_3(self, star_loc, vararg_tok, kwarg_opt):
dstar_loc = kwarg = kwarg_loc = None
loc = vararg_tok.loc
if kwarg_opt:
dstar_loc, kwarg, kwarg_loc = kwarg_opt
return ast.arguments(vararg=vararg_tok.value, kwarg=kwarg,
_, dstar_loc, kwarg_tok = kwarg_opt
kwarg, kwarg_loc = kwarg_tok.value, kwarg_tok.loc
loc = kwarg_tok.loc
return ast.arguments(args=[], defaults=[], vararg=vararg_tok.value, kwarg=kwarg,
star_loc=star_loc, vararg_loc=vararg_tok.loc,
dstar_loc=dstar_loc, kwarg_loc=kwarg_loc)
dstar_loc=dstar_loc, kwarg_loc=kwarg_loc,
begin_loc=None, end_loc=None, equals_locs=[], loc=loc)

@action(Eps(value=()))
def varargslist_4(self):
return self._empty_arguments()

@action(Alt(Seq(Star(SeqN(0, varargslist_1, Tok(','))),
Alt(varargslist_2, varargslist_3)),
Seq(List(varargslist_1, ',', trailing=True),
varargslist_4)))
def varargslist(self, fparams, args):
"""varargslist: ((fpdef ['=' test] ',')*
('*' NAME [',' '**' NAME] | '**' NAME) |
fpdef ['=' test] (',' fpdef ['=' test])* [','])"""
for fparam, default_opt in fparams:
args.args.append(fparam)
if default_opt:
equals_loc, default = default_opt
args.equals_locs.append(equals_loc)
args.defaults.append(default)
elif len(args.defaults) > 0:
error = diagnostic.Diagnostic(
"error", "non-default argument follows default argument", {}, fparam.loc)
raise diagnostic.DiagnosticException(error)

def fparam_loc(fparam, default_opt):
if default_opt:
equals_loc, default = default_opt
return fparam.loc.join(default.loc)
else:
return fparam.loc

if args.loc is None:
args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1]))
else:
args.loc = args.loc.join(fparam_loc(*fparams[0]))

varargslist = Tok('no')
"""varargslist: ((fpdef ['=' test] ',')*
('*' NAME [',' '**' NAME] | '**' NAME) |
fpdef ['=' test] (',' fpdef ['=' test])* [','])"""
return args

@action(Tok('ident'))
def fpdef_1(ident_tok):
return ast.Name(id=ident_tok.value, loc=ident_tok.loc)
def fpdef_1(self, ident_tok):
return ast.Name(id=ident_tok.value, loc=ident_tok.loc, ctx=None)

fpdef = Alt(fpdef_1, BeginEnd('(', Rule('fplist'), ')'))
fpdef = Alt(fpdef_1, BeginEnd('(', Rule('fplist'), ')',
empty=lambda: ast.Tuple(elts=[], ctx=None, loc=None)))
"""fpdef: NAME | '(' fplist ')'"""

fplist = List(Rule('fpdef'), ',', trailing=True)
"""fplist: fpdef (',' fpdef)* [',']"""
@action(List(Rule('fpdef'), ',', trailing=True))
def fplist(self, elts):
"""fplist: fpdef (',' fpdef)* [',']"""
return ast.Tuple(elts=elts, ctx=None, loc=None)

stmt = Alt(Rule('simple_stmt'), Rule('compound_stmt'))
"""stmt: simple_stmt | compound_stmt"""
137 changes: 112 additions & 25 deletions pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -55,7 +55,7 @@ def flatten_ast(self, node):
_path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")

def match_loc(self, ast, matcher, root=lambda x: x):
ast = root(ast)
offset, ast = root(ast)

matcher_pos = 0
while matcher_pos < len(matcher):
@@ -64,8 +64,8 @@ def match_loc(self, ast, matcher, root=lambda x: x):
raise Exception("invalid location matcher %s" % matcher[matcher_pos:])

range = source.Range(self.source_buffer,
matcher_match.start(1) - matcher_pos,
matcher_match.end(1) - matcher_pos)
matcher_match.start(1) - matcher_pos + offset,
matcher_match.end(1) - matcher_pos + offset)
path = matcher_match.group(2)

path_pos = 0
@@ -91,25 +91,32 @@ def match_loc(self, ast, matcher, root=lambda x: x):

matcher_pos = matcher_match.end(0)

def assertParsesGen(self, expected_flat_ast, code):
def _assertParsesGen(self, expected_flat_ast, code,
loc_matcher="", ast_slicer=lambda x: (0, x)):
ast = self.parser_for(code + "\n").file_input()
flat_ast = self.flatten_ast(ast)
self.assertEqual({'ty': 'Module', 'body': expected_flat_ast},
flat_ast)
return ast
self.match_loc(ast, loc_matcher, ast_slicer)

def assertParsesSuite(self, expected_flat_ast, code, loc_matcher=""):
ast = self.assertParsesGen(expected_flat_ast, code)
self.match_loc(ast, loc_matcher, lambda x: x.body)
self._assertParsesGen(expected_flat_ast, code,
loc_matcher, lambda x: (0, x.body))

def assertParsesExpr(self, expected_flat_ast, code, loc_matcher=""):
ast = self.assertParsesGen([{'ty': 'Expr', 'value': expected_flat_ast}], code)
self.match_loc(ast, loc_matcher, lambda x: x.body[0].value)
self._assertParsesGen([{'ty': 'Expr', 'value': expected_flat_ast}], code,
loc_matcher, lambda x: (0, x.body[0].value))

def assertParsesToplevel(self, expected_flat_ast, code, loc_matcher, mode, interactive):
def assertParsesArgs(self, expected_flat_ast, code, loc_matcher=""):
self._assertParsesGen([{'ty': 'Expr', 'value': {'ty': 'Lambda', 'body': self.ast_1,
'args': expected_flat_ast}}],
"lambda %s: 1" % code,
loc_matcher, lambda x: (7, x.body[0].value.args))

def assertParsesToplevel(self, expected_flat_ast, code,
mode="file_input", interactive=False):
ast = getattr(self.parser_for(code, interactive=interactive), mode)()
self.assertEqual(expected_flat_ast, self.flatten_ast(ast))
self.match_loc(ast, loc_matcher, lambda x: x.body)

def assertDiagnoses(self, code, diag):
try:
@@ -648,8 +655,7 @@ def test_call(self):
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y, 'ifs': []}
]}
]},
"x(y for y in z)",
"")
"x(y for y in z)")

def test_subscript(self):
self.assertParsesExpr(
@@ -1159,8 +1165,7 @@ def test_try(self):
{'ty': 'ExceptHandler', 'type': self.ast_y, 'name': None,
'body': [self.ast_expr_2]}
]}],
"try:· 1·except y:· 2",
"")
"try:· 1·except y:· 2")

self.assertParsesSuite(
[{'ty': 'TryExcept', 'body': [self.ast_expr_1], 'orelse': None,
@@ -1307,23 +1312,108 @@ def test_decorated(self):
" ^ 0.decorator_list.0.loc"
"~~~~~~~~~~~~~~~~~~ 0.loc")

#
# FUNCTION AND LAMBDA ARGUMENTS
#

def test_args(self):
self.assertParsesArgs(
{'ty': 'arguments', 'args': [], 'defaults': [],
'vararg': None, 'kwarg': None},
"")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'vararg': None, 'kwarg': None},
"x",
"~ args.0.loc"
"~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x], 'defaults': [self.ast_1],
'vararg': None, 'kwarg': None},
"x=1",
"~ args.0.loc"
" ~ equals_locs.0"
" ~ defaults.0.loc"
"~~~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x, self.ast_y], 'defaults': [],
'vararg': None, 'kwarg': None},
"x, y",
"~~~~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'vararg': 'y', 'kwarg': None},
"x, *y",
" ^ star_loc"
" ~ vararg_loc"
"~~~~~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'vararg': None, 'kwarg': 'y'},
"x, **y",
" ^^ dstar_loc"
" ~ kwarg_loc"
"~~~~~~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'vararg': 'y', 'kwarg': 'z'},
"x, *y, **z",
" ^ star_loc"
" ~ vararg_loc"
" ^^ dstar_loc"
" ~ kwarg_loc"
"~~~~~~~~~~ loc")

self.assertParsesArgs(
{'ty': 'arguments', 'defaults': [], 'vararg': None, 'kwarg': None,
'args': [{'ty': 'Tuple', 'ctx': None, 'elts': [self.ast_x, self.ast_y]}]},
"(x,y)",
"^ args.0.begin_loc"
" ^ args.0.end_loc"
"~~~~~ args.0.loc"
"~~~~~ loc")

def test_args_def(self):
self.assertParsesSuite(
[{'ty': 'FunctionDef', 'name': 'foo',
'args': {'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'kwarg': None, 'vararg': None},
'body': [{'ty': 'Pass'}], 'decorator_list': []}],
"def foo(x):· pass")

def test_args_oldlambda(self):
self.assertParsesExpr(
{'ty': 'ListComp', 'elt': self.ast_x, 'generators': [
{'ty': 'comprehension', 'iter': self.ast_z, 'target': self.ast_y,
'ifs': [{'ty': 'Lambda',
'args': {'ty': 'arguments', 'args': [self.ast_x], 'defaults': [],
'kwarg': None, 'vararg': None},
'body': self.ast_t}
]}
]},
"[x for y in z if lambda x: t]")

#
# PARSING MODES
#

def test_single_input(self):
self.assertParsesToplevel(
{'ty': 'Interactive', 'body': []},
"\n",
"",
"·",
mode='single_input', interactive=True)

self.assertParsesToplevel(
{'ty': 'Interactive', 'body': [
{'ty': 'Expr', 'value': self.ast_1}
]},
"1\n",
"",
"1·",
mode='single_input', interactive=True)

self.assertParsesToplevel(
@@ -1332,20 +1422,17 @@ def test_single_input(self):
{'ty': 'Expr', 'value': self.ast_1}
], 'orelse': []}
]},
"if x: 1\n\n",
"",
"if x: 1··",
mode='single_input', interactive=True)

def test_file_input(self):
self.assertParsesToplevel(
{'ty': 'Module', 'body': []},
"\n",
"",
"·",
mode='file_input', interactive=True)

def test_eval_input(self):
self.assertParsesToplevel(
{'ty': 'Expression', 'body': [self.ast_1]},
"1\n",
"",
"1·",
mode='eval_input', interactive=True)

0 comments on commit 25e4d3f

Please sign in to comment.