Skip to content

Commit 8b9a887

Browse files
author
whitequark
committedApr 23, 2015
Add parser module.
Currently this only implements 2.6 grammar, but in full. There are possible location bugs and semantic action bugs-- a test suite will be necessary to eradicate these-- but probably no grammar bugs, as it is mostly a straightforward translation of the official Python grammar. The latter also ensures it will be easy to update it for newer Python versions.
·
1.41.0
1 parent 09e9545 commit 8b9a887

File tree

7 files changed

+1057
-64
lines changed

7 files changed

+1057
-64
lines changed
 

Diff for: ‎pyparser/__init__.py

-1
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
from . import source, diagnostic, lexer

Diff for: ‎pyparser/ast.py

+18-14
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,13 @@ class arguments(commonloc):
7575
Function definition arguments, e.g. in ``def f(x, y=1, *z, **t)``.
7676
7777
:ivar args: (list of assignable node) regular formal arguments
78-
:ivar vararg: (node) splat formal argument (if any), e.g. in ``*x``
79-
:ivar kwarg: (node) keyword splat formal argument (if any), e.g. in ``**x``
80-
:ivar defaults: (node) values of default arguments
78+
:ivar vararg: (string) splat formal argument (if any), e.g. in ``*x``
79+
:ivar kwarg: (string) keyword splat formal argument (if any), e.g. in ``**x``
80+
:ivar defaults: (list of node) values of default arguments
8181
:ivar star_loc: location of ``*``, if any
82+
:ivar vararg_loc: location of splat formal argument, if any
8283
:ivar dstar_loc: location of ``**``, if any
84+
:ivar kwarg_loc: location of keyword splat formal argument, if any
8385
:ivar default_equals_locs: locations of ``=``
8486
"""
8587

@@ -132,13 +134,14 @@ class ExceptHandler(excepthandler, ast.ExceptHandler):
132134
"""
133135
An exception handler, e.g. ``except x as y:· z``.
134136
135-
:ivar type: (node) type of handled exception
137+
:ivar type: (node) type of handled exception, if any
136138
:ivar name: (assignable node) variable bound to exception, if any
137139
:ivar body: (list of node) code to execute when exception is caught
138140
:ivar except_loc: location of ``except``
139141
:ivar as_loc: location of ``as``, if any
142+
:ivar colon_loc: location of ``:``
140143
"""
141-
_locs = excepthandler._locs + ('except_loc', 'as_loc')
144+
_locs = excepthandler._locs + ('except_loc', 'as_loc', 'colon_loc')
142145

143146
class expr(commonloc):
144147
"""Base class for expression nodes."""
@@ -328,7 +331,6 @@ class keyword(commonloc, ast.keyword):
328331

329332
class mod(commonloc):
330333
"""Base class for modules (groups of statements)."""
331-
_locs = commonloc._locs + ('body',)
332334
class Expression(mod, ast.Expression):
333335
"""A group of statements parsed as if for :func:`eval`."""
334336
class Interactive(mod, ast.Interactive):
@@ -435,8 +437,9 @@ class ClassDef(keywordloc, stmt, ast.ClassDef):
435437
:ivar name_loc: location of name
436438
:ivar lparen_loc: location of ``(``, if any
437439
:ivar rparen_loc: location of ``)``, if any
440+
:ivar at_locs: locations of decorator ``@``
438441
"""
439-
_locs = keywordloc._locs + ('name_loc', 'lparen_loc', 'rparen_loc')
442+
_locs = keywordloc._locs + ('name_loc', 'lparen_loc', 'rparen_loc', 'at_loc')
440443
class Continue(keywordloc, stmt, ast.Continue):
441444
"""The ``continue`` statement."""
442445
class Delete(keywordloc, stmt, ast.Delete):
@@ -488,8 +491,9 @@ class FunctionDef(keywordloc, stmt, ast.FunctionDef):
488491
:ivar decorator_list: (list of node) decorators
489492
:ivar name_loc: location of name
490493
:ivar colon_loc: location of ``:``, if any
494+
:ivar at_locs: locations of decorator ``@``
491495
"""
492-
_locs = keywordloc._locs + ('keyword_loc' 'name_loc', 'colon_loc')
496+
_locs = keywordloc._locs + ('keyword_loc' 'name_loc', 'colon_loc', 'at_loc')
493497
class Global(keywordloc, stmt, ast.Global):
494498
"""
495499
The ``global x, y`` statement.
@@ -516,17 +520,17 @@ class Import(keywordloc, stmt, ast.Import):
516520
"""
517521
class ImportFrom(keywordloc, stmt, ast.Import):
518522
"""
519-
The ``from ...x import y, z`` statement.
523+
The ``from ...x import y, z`` or ``from x import *`` statement.
520524
521525
:ivar names: (list of :class:`alias`) names
522-
:ivar module: (string) module name
526+
:ivar module: (string) module name, if any
523527
:ivar level: (integer) amount of dots before module name
524528
:ivar keyword_loc: location of ``from``
525-
:ivar dots_loc: location of dots
526-
:ivar module_loc: location of module name
529+
:ivar dots_loc: location of dots, if any
530+
:ivar module_loc: location of module name, if any
527531
:ivar import_loc: location of ``import``
528532
"""
529-
_locs = keywordloc._locs + ('dots_loc', 'module_loc', 'import_loc')
533+
_locs = keywordloc._locs + ('module_loc', 'import_loc')
530534
class Pass(keywordloc, stmt, ast.Pass):
531535
"""The ``pass`` statement."""
532536
class Print(keywordloc, stmt, ast.Print):
@@ -554,7 +558,7 @@ class TryExcept(keywordloc, stmt, ast.TryExcept):
554558
The ``try:· x·except y:· z·else:· t`` statement.
555559
556560
:ivar body: (list of node) code to try
557-
:ivar handlers:
561+
:ivar handlers: (list of :class:`ExceptHandler`) exception handlers
558562
:ivar orelse: (list of node) code if no exception
559563
:ivar keyword_loc: location of ``try``
560564
:ivar try_colon_loc: location of ``:`` after ``try``

Diff for: ‎pyparser/parser.py

+1,031-37
Large diffs are not rendered by default.

Diff for: ‎pyparser/source.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,8 @@ def __str__(self):
152152
"""
153153
Returns a Clang-style string representation of the beginning of this range.
154154
"""
155-
return ":".join([self.source_buffer.name,
156-
str(self.line()), str(self.column() + 1)])
155+
return "%s:%d:%d-%d" % (self.source_buffer.name,
156+
self.line(), self.column() + 1, self.end().column() + 1)
157157

158158
def __eq__(self, other):
159159
"""

Diff for: ‎pyparser/test/test_diagnostic.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def test_render(self):
2121
[source.Range(self.buffer, 5, 6),
2222
source.Range(self.buffer, 9, 12)])
2323
self.assertEqual(
24-
["<input>:1:8: error: cannot add integer and string",
24+
["<input>:1:8-9: error: cannot add integer and string",
2525
"x + (1 + 'a')",
2626
" ~ ^ ~~~ "],
2727
diag.render())

Diff for: ‎pyparser/test/test_parser.py

+4-8
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ def lexer_next(**args):
2121
return self.parser
2222

2323
def assertParses(self, ast, code):
24-
self.assertEqual(ast, self.parser_for(code).expr())
24+
self.assertEqual(ast, self.parser_for(code).eval_input())
2525

2626
def assertDiagnoses(self, code, diag):
2727
try:
28-
self.parser_for(code).expr()
28+
self.parser_for(code).eval_input()
2929
self.fail("Expected a diagnostic")
3030
except diagnostic.DiagnosticException as e:
3131
level, reason, args, loc = diag
@@ -42,10 +42,6 @@ def assertDiagnosesUnexpected(self, code, err_token, loc):
4242
self.assertDiagnoses(code,
4343
("error", "unexpected {actual}: expected {expected}", {'actual': err_token}, loc))
4444

45-
def test_int(self):
46-
self.assertParses(1, "1")
47-
self.assertParses(1.0, "1.0")
48-
self.assertParses(1.0, "(1.0)")
49-
self.assertParses(1.0, "((1.0))")
50-
self.assertDiagnosesUnexpected("()", ")", (1, 2))
45+
def test_pass(self):
46+
self.assertParses(None, "pass")
5147

Diff for: ‎pyparser/test/test_source.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ def test_source_line(self):
8686
self.assertEqual("line two\n", self.range(9, 9).source_line())
8787

8888
def test___str__(self):
89-
self.assertEqual("<input>:2:1", str(self.range(9, 9)))
89+
self.assertEqual("<input>:2:1-1", str(self.range(9, 9)))
9090

9191
def test___ne__(self):
9292
self.assertTrue(self.range(0,0) != self.range(0,1))

0 commit comments

Comments
 (0)
Please sign in to comment.