Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 11ec15762d83
Choose a base ref
...
head repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: cb25dc3f7d6b
Choose a head ref
  • 2 commits
  • 7 files changed
  • 1 contributor

Commits on May 10, 2015

  1. Add algorithm module.

    whitequark committed May 10, 2015
    Copy the full SHA
    71bc9c1 View commit details
  2. Update README.

    whitequark committed May 10, 2015
    Copy the full SHA
    cb25dc3 View commit details
Showing with 121 additions and 8 deletions.
  1. +1 −0 README.md
  2. 0 README.rst
  3. +12 −3 doc/index.rst
  4. +6 −5 pyparser/__init__.py
  5. +72 −0 pyparser/algorithm.py
  6. +1 −0 pyparser/diagnostic.py
  7. +29 −0 pyparser/test/test_algorithm.py
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
See http://m-labs.hk/pyparser/.
Empty file removed README.rst
Empty file.
15 changes: 12 additions & 3 deletions doc/index.rst
Original file line number Diff line number Diff line change
@@ -15,14 +15,16 @@ code that consumes the ASTs. The :class:`pyparser.source.Range` class,
instances of which are contained in the AST, allows to extract
location information in various convenient formats.

TODO: algorithms
The :mod:`pyparser.algorithm` module contains some useful algorithms
to manipulate ASTs.

If a consumer of ASTs wishes to modify the original source without
losing formatting, it can use :class:`pyparser.source.Rewriter`
to insert code fragments around or instead of a known
:class:`pyparser.source.Range`. If the AST is not expected to
change after the modification, it is recommended to re-parse
the result and compare it to the original AST using <ALGO>.
the result and compare it to the original AST using
:meth:`pyparser.algorithm.compare`.

For some applications, e.g. syntax highlighting,
:class:`pyparser.lexer.Lexer` will be able to provide a raw
@@ -70,7 +72,7 @@ stream of tokens.
excepthandler, ExceptHandler,
expr, BinOp, BoolOp, Call, Compare, Dict, DictComp, Ellipsis, GeneratorExp, IfExp, Lambda,
List, ListComp, Name, Num, Repr, Set, SetComp, Str, Subscript, Tuple, UnaryOp,
Yield, YieldFrom
Yield, YieldFrom,
keyword,
mod, Expression, Interactive, Module,
operator, Add, BitAnd, BitOr, BitXor, Div, FloorDiv, LShift, MatMult, Mod, Mult,
@@ -82,3 +84,10 @@ stream of tokens.
unaryop, Invert, Not, UAdd, USub,
withitem
:show-inheritance:

:mod:`pyparser.algorithm` Module
--------------------------------

.. automodule:: pyparser.algorithm
:members:
:show-inheritance:
11 changes: 6 additions & 5 deletions pyparser/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import sys, pyparser.source, pyparser.lexer, pyparser.parser
import sys, pyparser.source, pyparser.lexer, pyparser.parser, pyparser.diagnostic

def parse(source, filename='<unknown>', mode='exec',
flags=[], version=None):
flags=[], version=None, engine=pyparser.diagnostic.Engine()):
"""
Parse a string into an abstract syntax tree.
This is the replacement for the built-in :meth:`..ast.parse`.
@@ -18,20 +18,21 @@ def parse(source, filename='<unknown>', mode='exec',
Equivalent to ``from __future__ import <flags>``.
:param version: (2-tuple of int) Major and minor version of Python
syntax to recognize, ``sys.version_info[0:2]`` by default.
:param engine: :class:`diagnostic.Engine` Diagnostic engine
:return: (:class:`ast.AST`) abstract syntax tree
:raise: :class:`diagnostic.DiagnosticException`
:raise: :class:`diagnostic.Error`
if the source code is not well-formed
"""
if version is None:
version = sys.version_info[0:2]

buffer = pyparser.source.Buffer(source, filename)

lexer = pyparser.lexer.Lexer(buffer, version)
lexer = pyparser.lexer.Lexer(buffer, version, engine)
if mode in ('single', 'eval'):
lexer.interactive = True

parser = pyparser.parser.Parser(lexer, version)
parser = pyparser.parser.Parser(lexer, version, engine)
parser.add_flags(flags)

if mode == 'exec':
72 changes: 72 additions & 0 deletions pyparser/algorithm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""
The :mod:`Diagnostic` module provides several commonly useful
algorithms that operate on abstract syntax trees.
"""

from __future__ import absolute_import, division, print_function, unicode_literals
from . import ast

class Visitor:
"""
A node visitor base class that does a pre-order traversal
of the abstract syntax tree.
This class is meant to be subclassed, with the subclass adding visitor
methods.
The visitor functions for the nodes are ``'visit_'`` +
class name of the node. So a `Try` node visit function would
be `visit_Try`.
"""

def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node."""
pass

def visit(self, node):
"""Visit a node."""
visit_attr = 'visit_' + type(node).__name__
if hasattr(self, visit_attr):
getattr(self, visit_attr)(node)
else:
self.generic_visit(node)

for field_name in node._fields:
field_val = getattr(node, field_name)
if isinstance(field_val, ast.AST):
self.visit(field_val)
elif isinstance(field_val, list):
for field_val_elt in field_val:
self.visit(field_val_elt)

def compare(left, right, compare_locs=False):
"""
An AST comparison function. Returns ``True`` if all fields in
``left`` are equal to fields in ``right``; if ``compare_locs`` is
true, all locations should match as well.
"""
if type(left) != type(right):
return False

if isinstance(left, ast.AST):
for field in left._fields:
if not compare(getattr(left, field), getattr(right, field)):
return False

if compare_locs:
for loc in left._locs:
if getattr(left, loc) != getattr(right, loc):
return False

return True
elif isinstance(left, list):
if len(left) != len(right):
return False

for left_elt, right_elt in zip(left, right):
if not compare(left_elt, right_elt):
return False

return True
else:
return left == right
1 change: 1 addition & 0 deletions pyparser/diagnostic.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
"""

from __future__ import absolute_import, division, print_function, unicode_literals
from functools import reduce

class Diagnostic:
"""
29 changes: 29 additions & 0 deletions pyparser/test/test_algorithm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from .. import parse, algorithm
import unittest

class AlgorithmTestCase(unittest.TestCase):

def test_visitor(self):
class FooVisitor(algorithm.Visitor):
def __init__(self):
self.num_count = self.other_count = 0

def visit_Num(self, node):
self.num_count += 1

def generic_visit(self, node):
self.other_count += 1

visitor = FooVisitor()
visitor.visit(parse("[1,2,x,y]\n"))

self.assertEqual(2, visitor.num_count)
self.assertEqual(5, visitor.other_count)

def test_compare(self):
self.assertFalse(algorithm.compare(parse("1 + 2\n"), parse("1 + 3\n")))
self.assertTrue( algorithm.compare(parse("1 + 2\n"), parse("1 + 2\n")))
self.assertTrue( algorithm.compare(parse("1 + 2\n"), parse("1 + 2\n")))
self.assertFalse(algorithm.compare(parse("1 + 2\n"), parse("1 + 2\n"),
compare_locs=True))