Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 35749e24fd53
Choose a base ref
...
head repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 11ec15762d83
Choose a head ref
  • 2 commits
  • 8 files changed
  • 1 contributor

Commits on May 10, 2015

  1. Normalize ast to 3.4.

    whitequark committed May 10, 2015
    Copy the full SHA
    6f0a179 View commit details
  2. Add diagnostic.Engine.

    whitequark committed May 10, 2015
    Copy the full SHA
    11ec157 View commit details
Showing with 169 additions and 110 deletions.
  1. +1 −1 doc/index.rst
  2. +14 −22 pyparser/ast.py
  3. +2 −2 pyparser/coverage/__init__.py
  4. +24 −2 pyparser/diagnostic.py
  5. +17 −14 pyparser/lexer.py
  6. +46 −36 pyparser/parser.py
  7. +3 −2 pyparser/test/test_lexer.py
  8. +62 −31 pyparser/test/test_parser.py
2 changes: 1 addition & 1 deletion doc/index.rst
Original file line number Diff line number Diff line change
@@ -78,7 +78,7 @@ stream of tokens.
slice, ExtSlice, Index, Slice,
stmt, Assert, Assign, AugAssign, Break, ClassDef, Continue, Delete, Exec, Expr, For,
FunctionDef, Global, If, Import, ImportFrom, Nonlocal, Pass, Print, Raise, Return,
TryExcept, TryFinally, While, With,
Try, While, With,
unaryop, Invert, Not, UAdd, USub,
withitem
:show-inheritance:
36 changes: 14 additions & 22 deletions pyparser/ast.py
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@
In particular this affects:
* :class:`With`: on 2.6-2.7 it uses the 3.0 format.
* :class:`TryExcept` and :class:`TryFinally`: on 2.6-2.7 they're replaced with
:class:`Try` from 3.0.
* :class:`arguments`: on 2.6-3.1 it uses the 3.2 format, with dedicated
:class:`arg` in ``vararg`` and ``kwarg`` slots.
"""
@@ -188,14 +190,16 @@ class ExceptHandler(excepthandler):
An exception handler, e.g. ``except x as y:· z``.
:ivar type: (:class:`AST`) type of handled exception, if any
:ivar name: (assignable :class:`AST`) variable bound to exception, if any
:ivar name: (assignable :class:`AST` **until 3.0**, string **since 3.0**)
variable bound to exception, if any
:ivar body: (list of :class:`AST`) code to execute when exception is caught
:ivar except_loc: location of ``except``
:ivar as_loc: location of ``as``, if any
:ivar name_loc: location of variable name
:ivar colon_loc: location of ``:``
"""
_fields = ('type', 'name', 'body')
_locs = excepthandler._locs + ('except_loc', 'as_loc', 'colon_loc')
_locs = excepthandler._locs + ('except_loc', 'as_loc', 'name_loc', 'colon_loc')

class expr(AST, commonloc):
"""Base class for expression nodes."""
@@ -384,7 +388,7 @@ class Starred(expr):
:ivar value: (:class:`AST`) expression
:ivar star_loc: location of ``*``
"""
_fields = ('value',)
_fields = ('value', 'ctx')
_locs = expr._locs + ('star_loc',)
class Subscript(expr, beginendloc):
"""
@@ -728,37 +732,25 @@ class Return(stmt, keywordloc):
:ivar value: (:class:`AST`) return value, if any
"""
_fields = ('value',)
class TryExcept(stmt, keywordloc):
class Try(stmt, keywordloc):
"""
The ``try:· x·except y:· z·else:· t`` statement.
**Emitted until 3.0.**
The ``try:· x·except y:· z·else:· t`` or
``try:· x·finally:· y`` statement.
:ivar body: (list of :class:`AST`) code to try
:ivar handlers: (list of :class:`ExceptHandler`) exception handlers
:ivar orelse: (list of :class:`AST`) code if no exception
:ivar finalbody: (list of :class:`AST`) code to finalize
:ivar keyword_loc: location of ``try``
:ivar try_colon_loc: location of ``:`` after ``try``
:ivar else_loc: location of ``else``
:ivar else_colon_loc: location of ``:`` after ``else``
"""
_fields = ('body', 'handlers', 'orelse')
_locs = keywordloc._locs + ('try_colon_loc', 'else_loc', 'else_colon_loc',)
class TryFinally(stmt, keywordloc):
"""
The ``try:· x·finally:· y`` statement.
**Emitted until 3.0.**
:ivar body: (list of :class:`AST`) code to try
:ivar finalbody: (list of :class:`AST`) code to finalize
:ivar keyword_loc: location of ``try``
:ivar try_colon_loc: location of ``:`` after ``try``
:ivar finally_loc: location of ``finally``
:ivar finally_colon_loc: location of ``:`` after ``finally``
"""
_fields = ('body', 'finalbody')
_locs = keywordloc._locs + ('try_colon_loc', 'finally_loc', 'finally_colon_loc',)
_fields = ('body', 'handlers', 'orelse', 'finalbody')
_locs = keywordloc._locs + ('try_colon_loc', 'else_loc', 'else_colon_loc',
'finally_loc', 'finally_colon_loc',)
class While(stmt, keywordloc):
"""
The ``while x:· y·else:· z`` statement.
4 changes: 2 additions & 2 deletions pyparser/coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from .. import source, lexer
from .. import source, lexer, diagnostic
import os, codecs

_buf = None
@@ -13,7 +13,7 @@
# but the parser doesn't work yet at the time of writing.
def instrument():
rewriter = source.Rewriter(_buf)
lex = lexer.Lexer(_buf, (3, 4))
lex = lexer.Lexer(_buf, (3, 4), diagnostic.Engine())
in_grammar = False
paren_stack = []
stmt_stack = [None]
26 changes: 24 additions & 2 deletions pyparser/diagnostic.py
Original file line number Diff line number Diff line change
@@ -87,9 +87,9 @@ def render(self):
]


class DiagnosticException(Exception):
class Error(Exception):
"""
:class:`Exception` is an exception which carries a :class:`Diagnostic`.
:class:`Error` is an exception which carries a :class:`Diagnostic`.
:ivar diagnostic: (:class:`Diagnostic`) the diagnostic
"""
@@ -99,3 +99,25 @@ def __init__(self, diagnostic):
def __str__(self):
return "\n".join(self.diagnostic.render() +
reduce(list.__add__, map(Diagnostic.render, self.diagnostic.notes), []))

class Engine:
"""
:class:`Engine` is a single point through which diagnostics from
lexer, parser and any AST consumer are dispatched.
:ivar all_errors_are_fatal: if true, an exception is raised not only
for ``fatal`` diagnostic level, but also ``error``
"""
def __init__(self, all_errors_are_fatal=False):
self.all_errors_are_fatal = all_errors_are_fatal

def process(self, diagnostic):
"""
The default implementation of :meth:`process` renders non-fatal
diagnostics to ``sys.stderr``, and raises fatal ones as a :class:`Error`.
"""
if diagnostic.level == 'fatal' or \
(self.all_errors_are_fatal and diagnostic.level == 'error'):
raise Error(diagnostic)
else:
sys.stderr.puts("\n".join(diagnostic.render()))
31 changes: 17 additions & 14 deletions pyparser/lexer.py
Original file line number Diff line number Diff line change
@@ -37,6 +37,8 @@ class Lexer:
the version of Python, determining the grammar used
:ivar source_buffer: (:class:`pyparser.source.Buffer`)
the source buffer
:ivar diagnostic_engine: (:class:`pyparser.diagnostic.Engine`)
the diagnostic engine
:ivar offset: (integer) character offset into ``source_buffer``
indicating where the next token will be recognized
:ivar interactive: (boolean) whether a completely empty line
@@ -97,9 +99,10 @@ class Lexer:
:class:`frozenset`\s of string prefixes.
"""

def __init__(self, source_buffer, version, interactive=False):
def __init__(self, source_buffer, version, diagnostic_engine, interactive=False):
self.source_buffer = source_buffer
self.version = version
self.diagnostic_engine = diagnostic_engine
self.interactive = interactive
self.print_function = False

@@ -253,7 +256,7 @@ def _refill(self, eof_token):
"fatal", "unexpected {character}",
{"character": repr(self.source_buffer.source[self.offset]).lstrip("u")},
source.Range(self.source_buffer, self.offset, self.offset + 1))
raise diagnostic.DiagnosticException(diag)
self.diagnostic_engine.process(diag)

# Should we emit indent/dedent?
if self.new_line and \
@@ -279,12 +282,12 @@ def _refill(self, eof_token):
error = diagnostic.Diagnostic(
"fatal", "inconsistent indentation", {},
range, notes=[note])
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)
elif whitespace != self.indent[-1][2] and self.version >= (3, 0):
error = diagnostic.Diagnostic(
"error", "inconsistent use of tabs and spaces in indentation", {},
range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

# Prepare for next token.
self.offset = match.end(0)
@@ -348,7 +351,7 @@ def _refill(self, eof_token):
error = diagnostic.Diagnostic(
"error", "in Python 3, decimal literals must not start with a zero", {},
source.Range(self.source_buffer, tok_range.begin_pos, tok_range.begin_pos + 1))
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)
self.queue.append(Token(tok_range, "int", int(literal, 8)))

elif match.group(14) is not None: # long string literal
@@ -367,7 +370,7 @@ def _refill(self, eof_token):
error = diagnostic.Diagnostic(
"fatal", "unterminated string", {},
tok_range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

elif match.group(21) is not None: # keywords and operators
kwop = match.group(21)
@@ -385,7 +388,7 @@ def _refill(self, eof_token):
error = diagnostic.Diagnostic(
"error", "in Python 2, Unicode identifiers are not allowed", {},
tok_range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)
self.queue.append(Token(tok_range, "ident", match.group(23)))

elif match.group(24) is not None: # end-of-file
@@ -405,7 +408,7 @@ def _string_literal(self, options, begin_span, data, data_span, end_span):
"error", "string prefix '{prefix}' is not available in Python {major}.{minor}",
{'prefix': options, 'major': self.version[0], 'minor': self.version[1]},
begin_range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

self.queue.append(Token(begin_range, 'strbegin', options))
self.queue.append(Token(data_range,
@@ -425,7 +428,7 @@ def _replace_escape(self, range, mode, value):
error = diagnostic.Diagnostic(
"error", "non-7-bit character in a byte literal", {},
tok_range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

if is_unicode or self.version >= (3, 0):
re = self._lex_escape_unicode_re
@@ -481,7 +484,7 @@ def _replace_escape(self, range, mode, value):
source.Range(self.source_buffer,
range.begin_pos + match.start(0),
range.begin_pos + match.end(0)))
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)
elif match.group(6) is not None: # unicode-name
try:
chunks.append(unicodedata.lookup(match.group(6)))
@@ -491,7 +494,7 @@ def _replace_escape(self, range, mode, value):
source.Range(self.source_buffer,
range.begin_pos + match.start(0),
range.begin_pos + match.end(0)))
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

return ''.join(chunks)

@@ -500,7 +503,7 @@ def _check_long_literal(self, range, literal):
error = diagnostic.Diagnostic(
"error", "in Python 3, long integer literals were removed", {},
source.Range(self.source_buffer, range.end_pos - 1, range.end_pos))
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

def _match_pair_delim(self, range, kwop):
if kwop == "(":
@@ -540,13 +543,13 @@ def _check_innermost_pair_delim(self, range, expected):
"fatal", "mismatched '{delimiter}'",
{"delimiter": range.source()},
range, notes=[note])
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)
else:
error = diagnostic.Diagnostic(
"fatal", "mismatched '{delimiter}'",
{"delimiter": range.source()},
range)
raise diagnostic.DiagnosticException(error)
self.diagnostic_engine.process(error)

def __iter__(self):
return self
Loading