Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 1a49d4d2a9f9
Choose a base ref
...
head repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 35749e24fd53
Choose a head ref
  • 2 commits
  • 5 files changed
  • 1 contributor

Commits on May 10, 2015

  1. Add Python 3.3-3.4 support.

    whitequark committed May 10, 2015
    Copy the full SHA
    83bd532 View commit details
  2. Add Python 3.5 support.

    whitequark committed May 10, 2015
    Copy the full SHA
    35749e2 View commit details
Showing with 97 additions and 16 deletions.
  1. +4 −2 doc/index.rst
  2. +13 −1 pyparser/ast.py
  3. +4 −0 pyparser/lexer.py
  4. +45 −12 pyparser/parser.py
  5. +31 −1 pyparser/test/test_parser.py
6 changes: 4 additions & 2 deletions doc/index.rst
Original file line number Diff line number Diff line change
@@ -69,10 +69,12 @@ stream of tokens.
comprehension,
excepthandler, ExceptHandler,
expr, BinOp, BoolOp, Call, Compare, Dict, DictComp, Ellipsis, GeneratorExp, IfExp, Lambda,
List, ListComp, Name, Num, Repr, Set, SetComp, Str, Subscript, Tuple, UnaryOp, Yield,
List, ListComp, Name, Num, Repr, Set, SetComp, Str, Subscript, Tuple, UnaryOp,
Yield, YieldFrom
keyword,
mod, Expression, Interactive, Module,
operator, Add, BitAnd, BitOr, BitXor, Div, FloorDiv, LShift, Mod, Mult, Pow, RShift, Sub,
operator, Add, BitAnd, BitOr, BitXor, Div, FloorDiv, LShift, MatMult, Mod, Mult,
Pow, RShift, Sub,
slice, ExtSlice, Index, Slice,
stmt, Assert, Assign, AugAssign, Break, ClassDef, Continue, Delete, Exec, Expr, For,
FunctionDef, Global, If, Import, ImportFrom, Nonlocal, Pass, Print, Raise, Return,
14 changes: 13 additions & 1 deletion pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -411,13 +411,23 @@ class UnaryOp(expr):
_fields = ('op', 'operand')
class Yield(expr):
"""
A yield expression, e.g. ``(yield x)``.
A yield expression, e.g. ``yield x``.
:ivar value: (:class:`AST`) yielded value
:ivar yield_loc: location of ``yield``
"""
_fields = ('value',)
_locs = expr._locs + ('yield_loc',)
class YieldFrom(expr):
"""
A yield from expression, e.g. ``yield from x``.
:ivar value: (:class:`AST`) yielded value
:ivar yield_loc: location of ``yield``
:ivar from_loc: location of ``from``
"""
_fields = ('value',)
_locs = expr._locs + ('yield_loc', 'from_loc')

# expr_context
# AugLoad
@@ -464,6 +474,8 @@ class FloorDiv(operator):
"""The ``\\\\`` operator."""
class LShift(operator):
"""The ``<<`` operator."""
class MatMult(operator):
"""The ``@`` operator."""
class Mod(operator):
"""The ``%`` operator."""
class Mult(operator):
4 changes: 4 additions & 0 deletions pyparser/lexer.py
Original file line number Diff line number Diff line change
@@ -60,6 +60,9 @@ class Lexer:
_reserved_3_1 = _reserved_3_0 \
| set(["<>"])

_reserved_3_5 = _reserved_3_1 \
| set(["@", "@="])

_reserved = {
(2, 6): _reserved_2_6,
(2, 7): _reserved_2_6,
@@ -68,6 +71,7 @@ class Lexer:
(3, 2): _reserved_3_1,
(3, 3): _reserved_3_1,
(3, 4): _reserved_3_1,
(3, 5): _reserved_3_5,
}
"""
A map from a tuple (*major*, *minor*) corresponding to Python version to
57 changes: 45 additions & 12 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -416,8 +416,9 @@ def _init_version(self, version):
self.exprlist_1 = self.exprlist_1__26
self.testlist_comp_1 = self.testlist_comp_1__26
self.expr_stmt_1 = self.expr_stmt_1__26
self.yield_expr = self.yield_expr__26
return
elif version in ((3, 0), (3, 1), (3, 2)):
elif version in ((3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)):
if version == (3, 0):
self.with_stmt = self.with_stmt__26 # lol
else:
@@ -446,6 +447,10 @@ def _init_version(self, version):
self.exprlist_1 = self.exprlist_1__32
self.testlist_comp_1 = self.testlist_comp_1__32
self.expr_stmt_1 = self.expr_stmt_1__32
if version < (3, 3):
self.yield_expr = self.yield_expr__26
else:
self.yield_expr = self.yield_expr__33
return

raise NotImplementedError("pyparser.parser.Parser cannot parse Python %s" %
@@ -862,10 +867,11 @@ def expr_stmt(self, lhs, rhs):
(_wrap_tuple)
"""(3.2-) testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']"""

augassign = Alt(Oper(ast.Add, '+='), Oper(ast.Sub, '-='), Oper(ast.Mult, '*='),
Oper(ast.Div, '/='), Oper(ast.Mod, '%='), Oper(ast.BitAnd, '&='),
Oper(ast.BitOr, '|='), Oper(ast.BitXor, '^='), Oper(ast.LShift, '<<='),
Oper(ast.RShift, '>>='), Oper(ast.Pow, '**='), Oper(ast.FloorDiv, '//='))
augassign = Alt(Oper(ast.Add, '+='), Oper(ast.Sub, '-='), Oper(ast.MatMult, '@='),
Oper(ast.Mult, '*='), Oper(ast.Div, '/='), Oper(ast.Mod, '%='),
Oper(ast.BitAnd, '&='), Oper(ast.BitOr, '|='), Oper(ast.BitXor, '^='),
Oper(ast.LShift, '<<='), Oper(ast.RShift, '>>='),
Oper(ast.Pow, '**='), Oper(ast.FloorDiv, '//='))
"""augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')"""

@@ -1439,8 +1445,9 @@ def star_expr__32(self, star_loc, expr):
arith_expr = BinOper('term', Alt(Oper(ast.Add, '+'), Oper(ast.Sub, '-')))
"""arith_expr: term (('+'|'-') term)*"""

term = BinOper('factor', Alt(Oper(ast.Mult, '*'), Oper(ast.Div, '/'),
Oper(ast.Mod, '%'), Oper(ast.FloorDiv, '//')))
term = BinOper('factor', Alt(Oper(ast.Mult, '*'), Oper(ast.MatMult, '@'),
Oper(ast.Div, '/'), Oper(ast.Mod, '%'),
Oper(ast.FloorDiv, '//')))
"""term: factor (('*'|'/'|'%'|'//') factor)*"""

@action(Seq(Alt(Oper(ast.UAdd, '+'), Oper(ast.USub, '-'), Oper(ast.Invert, '~')),
@@ -1932,8 +1939,34 @@ def compose(comprehensions):
testlist1 = action(List(Rule('test'), ',', trailing=False))(_wrap_tuple)
"""testlist1: test (',' test)*"""

@action(Seq(Loc('yield'), Rule('testlist')))
def yield_expr(self, stmt_loc, exprs):
"""yield_expr: 'yield' [testlist]"""
return ast.Yield(value=exprs,
yield_loc=stmt_loc, loc=stmt_loc.join(exprs.loc))
@action(Seq(Loc('yield'), Opt(Rule('testlist'))))
def yield_expr__26(self, yield_loc, exprs):
"""(2.6, 2.7, 3.0, 3.1, 3.2) yield_expr: 'yield' [testlist]"""
if exprs is not None:
return ast.Yield(value=exprs,
yield_loc=yield_loc, loc=yield_loc.join(exprs.loc))
else:
return ast.Yield(value=None,
yield_loc=yield_loc, loc=yield_loc)

@action(Seq(Loc('yield'), Opt(Rule('yield_arg'))))
def yield_expr__33(self, yield_loc, arg):
"""(3.3-) yield_expr: 'yield' [yield_arg]"""
if isinstance(arg, ast.YieldFrom):
arg.yield_loc = yield_loc
arg.loc = arg.loc.join(arg.yield_loc)
return arg
elif arg is not None:
return ast.Yield(value=arg,
yield_loc=yield_loc, loc=yield_loc.join(arg.loc))
else:
return ast.Yield(value=None,
yield_loc=yield_loc, loc=yield_loc)

@action(Seq(Loc('from'), Rule('test')))
def yield_arg_1(self, from_loc, value):
return ast.YieldFrom(value=value,
from_loc=from_loc, loc=from_loc.join(value.loc))

yield_arg = Alt(yield_arg_1, Rule('testlist'))
"""(3.3-) yield_arg: 'from' test | testlist"""
32 changes: 31 additions & 1 deletion pyparser/test/test_parser.py
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ class ParserTestCase(unittest.TestCase):

maxDiff = None

versions = [(2, 6), (2, 7), (3, 0), (3, 1), (3, 2)]
versions = [(2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)]

def parser_for(self, code, version, interactive=False):
code = code.replace("·", "\n")
@@ -299,6 +299,13 @@ def test_binary(self):
"~~~~~ loc"
" ^ op.loc")

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'MatMult'}, 'left': self.ast_x, 'right': self.ast_x},
"x @ x",
"~~~~~ loc"
" ^ op.loc",
only_if=lambda ver: ver >= (3, 5))

self.assertParsesExpr(
{'ty': 'BinOp', 'op': {'ty': 'Div'}, 'left': self.ast_1, 'right': self.ast_1},
"1 / 1",
@@ -978,6 +985,13 @@ def test_augassign(self):
"~~~~~~ 0.loc"
" ~~ 0.op.loc")

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'MatMult'}, 'target': self.ast_x, 'value': self.ast_y}],
"x @= y",
"~~~~~~ 0.loc"
" ~~ 0.op.loc",
only_if=lambda ver: ver >= (3, 5))

self.assertParsesSuite(
[{'ty': 'AugAssign', 'op': {'ty': 'Div'}, 'target': self.ast_x, 'value': self.ast_1}],
"x /= 1",
@@ -1117,13 +1131,29 @@ def test_return(self):
"~~~~~~~~ 0.loc")

def test_yield(self):
self.assertParsesSuite(
[{'ty': 'Expr', 'value': {'ty': 'Yield', 'value': None}}],
"yield",
"~~~~~ 0.value.yield_loc"
"~~~~~ 0.value.loc"
"~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Expr', 'value': {'ty': 'Yield', 'value': self.ast_x}}],
"yield x",
"~~~~~ 0.value.yield_loc"
"~~~~~~~ 0.value.loc"
"~~~~~~~ 0.loc")

self.assertParsesSuite(
[{'ty': 'Expr', 'value': {'ty': 'YieldFrom', 'value': self.ast_x}}],
"yield from x",
"~~~~~ 0.value.yield_loc"
" ~~~~ 0.value.from_loc"
"~~~~~~~~~~~~ 0.value.loc"
"~~~~~~~~~~~~ 0.loc",
only_if=lambda ver: ver >= (3, 3))

def test_raise(self):
self.assertParsesSuite(
[{'ty': 'Raise', 'exc': None, 'inst': None,