Skip to content

Commit

Permalink
Factor out pyparser.parse.
Browse files Browse the repository at this point in the history
whitequark committed May 8, 2015
1 parent 3f61113 commit ab67bb8
Showing 4 changed files with 67 additions and 33 deletions.
22 changes: 14 additions & 8 deletions doc/index.rst
Original file line number Diff line number Diff line change
@@ -6,29 +6,35 @@ It parses source code into an AST that is a superset of Python's
built-in :mod:`ast` module, but returns precise location information
for every token.

:mod:`source` Module
--------------------
:mod:`pyparser` Module
----------------------

.. automodule:: pyparser
:members:

:mod:`pyparser.source` Module
-----------------------------

.. automodule:: pyparser.source
:members:
:show-inheritance:

:mod:`diagnostic` Module
------------------------
:mod:`pyparser.diagnostic` Module
---------------------------------

.. automodule:: pyparser.diagnostic
:members:
:show-inheritance:

:mod:`lexer` Module
-------------------
:mod:`pyparser.lexer` Module
----------------------------

.. automodule:: pyparser.lexer
:members:
:show-inheritance:

:mod:`ast` Module
-----------------
:mod:`pyparser.ast` Module
--------------------------

.. automodule:: pyparser.ast
:members: commonloc, beginendloc, keywordloc,
35 changes: 35 additions & 0 deletions pyparser/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import sys, pyparser.source, pyparser.lexer, pyparser.parser

def parse(source, filename='<unknown>', mode='exec',
flags=[], version=sys.version_info[0:2]):
"""
Parse a string into an abstract syntax tree.
This is the replacement for the built-in :meth:`..ast.parse`.
:param str source: Source code in the correct encoding
:param filename: Filename of the source (used in diagnostics)
:param mode: Execution mode. Pass ``"exec"`` to parse a module,
``"single"`` to parse a single (interactive) statement,
and ``"eval"`` to parse an expression. In the last two cases,
``source`` must be terminated with an empty line
(i.e. end with ``"\\n\\n"``).
:param flags: Future flags. Equivalent to ``from __future__ import <flags>``.
:param int,int version: A tuple of the major and minor version
of Python syntax to recognize.
:return ast.AST: abstract syntax tree
:raise diagnostic.DiagnosticException:
if the source code is not well-formed
"""
buffer = pyparser.source.Buffer(source, filename)
lexer = pyparser.lexer.Lexer(buffer, version)
parser = pyparser.parser.Parser(lexer)

parser.add_flags(flags)

if mode == 'exec':
return parser.file_input()
elif mode == 'single':
return parser.single_input()
elif mode == 'eval':
return parser.eval_input()
18 changes: 18 additions & 0 deletions pyparser/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from . import parse, diagnostic
import sys, time, codecs

for filename in sys.argv[1:]:
with codecs.open(filename, encoding='utf-8') as f:
input = f.read()
try:
start = time.time()
root = parse(input, filename)
interval = time.time() - start

print(root)
print("elapsed: %.2f (%.2f kb/s)" % (interval, len(input)/interval/1000),
file=sys.stderr)
except diagnostic.DiagnosticException as e:
print("\n".join(e.diagnostic.render()),
file=sys.stderr)
25 changes: 0 additions & 25 deletions pyparser/parser.py
Original file line number Diff line number Diff line change
@@ -1409,28 +1409,3 @@ def yield_expr(self, stmt_loc, exprs):
"""yield_expr: 'yield' [testlist]"""
return ast.Yield(value=exprs,
yield_loc=stmt_loc, loc=stmt_loc.join(exprs.loc))

import sys

def for_code(code, version=sys.version_info[0:2]):
return Parser(lexer.Lexer(source.Buffer(code), version))

def main():
import time, codecs
for filename in sys.argv[1:]:
with codecs.open(filename, encoding='utf-8') as f:
input = f.read()
try:
start = time.time()
root = for_code(input).file_input()
interval = time.time() - start

print(root)
print("elapsed: %.2f (%.2f kb/s)" % (interval, len(input)/interval/1000),
file=sys.stderr)
except diagnostic.DiagnosticException as e:
print(e.render,
file=sys.stderr)

if __name__ == "__main__":
main()

0 comments on commit ab67bb8

Please sign in to comment.