-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
whitequark
committed
Mar 30, 2015
0 parents
commit 58559cf
Showing
11 changed files
with
323 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
*.pyc | ||
__pycache__/ | ||
_build/ | ||
*.egg-info/ | ||
|
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
master_doc = 'index' | ||
|
||
extensions = [ | ||
'sphinx.ext.autodoc', | ||
'sphinx.ext.autosummary', | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
PyParser documentation | ||
====================== | ||
|
||
PyParser is a Python parser written specifically for use in tooling. | ||
It parses source code into an AST that is a superset of Python's | ||
built-in :mod:`ast` module, but returns precise location information | ||
for every token. | ||
|
||
:mod:`source` Module | ||
-------------------- | ||
|
||
.. automodule:: pyparser.source | ||
:members: | ||
:show-inheritance: | ||
|
||
:mod:`lexer` Module | ||
-------------------- | ||
|
||
.. automodule:: pyparser.lexer | ||
:members: | ||
:show-inheritance: |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
class Lexer: | ||
keywords = [] | ||
|
||
def __init__(self, source, filename="<input>", line=1): | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,170 @@ | ||
""" | ||
The :mod:`source` module concerns itself with manipulating | ||
buffers of source code: creating ranges of characters corresponding | ||
to a token, combining these ranges, extracting human-readable | ||
location information and original source from a range. | ||
""" | ||
|
||
import bisect | ||
|
||
class Buffer: | ||
""" | ||
A buffer containing source code and location information. | ||
:ivar source: (string) source code | ||
:ivar name: (string) input filename or another description | ||
of the input (e.g. ``<stdin>``). | ||
:ivar line: (integer) first line of the input | ||
""" | ||
def __init__(self, source, name="<input>", first_line=1): | ||
self.source = source | ||
self.name = name | ||
self.first_line = first_line | ||
self._line_begins = None | ||
|
||
def __repr__(self): | ||
return r'Buffer("%s")' % self.name | ||
|
||
def source_line(self, lineno): | ||
""" | ||
Returns line ``lineno`` from source, taking ``first_line`` into account, | ||
or raises :exc:`IndexError` if ``lineno`` is out of range. | ||
""" | ||
line_begins = self._extract_line_begins() | ||
lineno = lineno - self.first_line | ||
if lineno >= 0 and lineno + 1 < len(line_begins): | ||
first, last = line_begins[lineno:lineno + 2] | ||
return self.source[first:last] | ||
elif lineno >= 0 and lineno < len(line_begins): | ||
return self.source[line_begins[-1]:] | ||
else: | ||
raise IndexError | ||
|
||
def decompose_position(self, offset): | ||
""" | ||
Returns a ``line, column`` tuple for a character offset into the source, | ||
orraises :exc:`IndexError` if ``lineno`` is out of range. | ||
""" | ||
line_begins = self._extract_line_begins() | ||
lineno = bisect.bisect_right(line_begins, offset) - 1 | ||
if offset >= 0 and offset < len(self.source): | ||
return lineno + self.first_line, offset - line_begins[lineno] | ||
else: | ||
raise IndexError | ||
|
||
def _extract_line_begins(self): | ||
if self._line_begins: | ||
return self._line_begins | ||
|
||
self._line_begins = [0] | ||
index = None | ||
while True: | ||
index = self.source.find(u"\n", index) + 1 | ||
if index == 0: | ||
return self._line_begins | ||
self._line_begins.append(index) | ||
|
||
class Range: | ||
""" | ||
Location of an exclusive range of characters [*begin_pos*, *end_pos*) | ||
in a :class:`Buffer`. | ||
:ivar begin_pos: (integer) offset of the first character | ||
:ivar end_pos: (integer) offset of the character before the last | ||
""" | ||
def __init__(self, source_buffer, begin_pos, end_pos): | ||
self.source_buffer = source_buffer | ||
self.begin_pos = begin_pos | ||
self.end_pos = end_pos | ||
|
||
def __repr__(self): | ||
""" | ||
Returns a human-readable representation of this range. | ||
""" | ||
return r'Range("%s", %d, %d)' % \ | ||
(self.source_buffer.name, self.begin_pos, self.end_pos) | ||
|
||
def begin(self): | ||
""" | ||
Returns a zero-length range located just before the beginning of this range. | ||
""" | ||
return Range(self.source_buffer, self.begin_pos, self.begin_pos) | ||
|
||
def end(self): | ||
""" | ||
Returns a zero-length range located just after the end of this range. | ||
""" | ||
return Range(self.source_buffer, self.end_pos, self.end_pos) | ||
|
||
def size(self): | ||
""" | ||
Returns the amount of characters spanned by the range. | ||
""" | ||
return self.end_pos - self.begin_pos | ||
|
||
def column(self): | ||
""" | ||
Returns a zero-based column number of the beginning of this range. | ||
""" | ||
line, column = self.source_buffer.decompose_position(self.begin_pos) | ||
return column | ||
|
||
def column_range(self): | ||
""" | ||
Returns a [*begin*, *end*) tuple describing the range of columns spanned | ||
by this range. | ||
""" | ||
return self.begin().column(), self.end().column() | ||
|
||
def line(self): | ||
""" | ||
Returns the line number of the beginning of this range. | ||
""" | ||
line, column = self.source_buffer.decompose_position(self.begin_pos) | ||
return line | ||
|
||
def join(self, other): | ||
""" | ||
Returns the smallest possible range spanning both this range and other. | ||
Raises :exc:`ValueError` if the ranges do not belong to the same | ||
:class:`Buffer`. | ||
""" | ||
if self.source_buffer != other.source_buffer: | ||
raise ValueError | ||
return Range(self.source_buffer, | ||
min(self.begin_pos, other.begin_pos), | ||
max(self.end_pos, other.end_pos)) | ||
|
||
def source(self): | ||
""" | ||
Returns the source code covered by this range. | ||
""" | ||
return self.source_buffer.source[self.begin_pos:self.end_pos] | ||
|
||
def source_line(self): | ||
""" | ||
Returns the line of source code containing the beginning of this range. | ||
""" | ||
return self.source_buffer.source_line(self.line()) | ||
|
||
def __str__(self): | ||
""" | ||
Returns a Clang-style string representation of the beginning of this range. | ||
""" | ||
return ':'.join([self.source_buffer.name, | ||
str(self.line()), str(self.column() + 1)]) | ||
|
||
def __eq__(self, other): | ||
""" | ||
Returns true if the ranges have the same source buffer, start and end position. | ||
""" | ||
return (type(self) == type(other) and | ||
self.source_buffer == other.source_buffer and | ||
self.begin_pos == other.begin_pos and | ||
self.end_pos == other.end_pos) | ||
|
||
def __ne__(self, other): | ||
""" | ||
Inverse of :meth:`__eq__`. | ||
""" | ||
return not (self == other) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
import unittest | ||
import pyparser.source as source | ||
|
||
class BufferTestCase(unittest.TestCase): | ||
|
||
def setUp(self): | ||
self.buffer = source.Buffer("line one\nline two\n\nline four") | ||
|
||
def test_repr(self): | ||
self.assertEqual(r'Buffer("<input>")', repr(self.buffer)) | ||
|
||
def test_source_line(self): | ||
self.assertEqual("line one\n", self.buffer.source_line(1)) | ||
self.assertEqual("line two\n", self.buffer.source_line(2)) | ||
self.assertEqual("\n", self.buffer.source_line(3)) | ||
self.assertEqual("line four", self.buffer.source_line(4)) | ||
self.assertRaises(IndexError, lambda: self.buffer.source_line(0)) | ||
|
||
def test_decompose_position(self): | ||
self.assertEqual((1,0), self.buffer.decompose_position(0)) | ||
self.assertEqual((1,2), self.buffer.decompose_position(2)) | ||
self.assertEqual((1,8), self.buffer.decompose_position(8)) | ||
self.assertEqual((2,0), self.buffer.decompose_position(9)) | ||
self.assertRaises(IndexError, lambda: self.buffer.decompose_position(90)) | ||
|
||
class RangeTestCase(unittest.TestCase): | ||
|
||
def setUp(self): | ||
self.buffer = source.Buffer("line one\nline two\n\nline four") | ||
|
||
def range(self, lft, rgt): | ||
return source.Range(self.buffer, lft, rgt) | ||
|
||
def test_repr(self): | ||
self.assertEqual(r'Range("<input>", 0, 2)', | ||
repr(self.range(0, 2))) | ||
|
||
def test_begin(self): | ||
self.assertEqual(self.range(1, 1), | ||
self.range(1, 2).begin()) | ||
|
||
def test_end(self): | ||
self.assertEqual(self.range(2, 2), | ||
self.range(1, 2).end()) | ||
|
||
def test_size(self): | ||
self.assertEqual(1, self.range(2, 3).size()) | ||
|
||
def test_column(self): | ||
self.assertEqual(2, self.range(2, 2).column()) | ||
self.assertEqual(0, self.range(9, 11).column()) | ||
self.assertEqual(2, self.range(11, 11).column()) | ||
|
||
def test_column_range(self): | ||
self.assertEqual((2,2), self.range(2, 2).column_range()) | ||
self.assertEqual((0,2), self.range(9, 11).column_range()) | ||
self.assertEqual((2,2), self.range(11, 11).column_range()) | ||
|
||
def test_line(self): | ||
self.assertEqual(1, self.range(2, 2).line()) | ||
self.assertEqual(2, self.range(9, 11).line()) | ||
self.assertEqual(2, self.range(11, 11).line()) | ||
|
||
def test_line(self): | ||
self.assertEqual(1, self.range(2, 2).line()) | ||
self.assertEqual(2, self.range(9, 11).line()) | ||
self.assertEqual(2, self.range(11, 11).line()) | ||
|
||
def test_join(self): | ||
self.assertEqual(self.range(1, 6), | ||
self.range(2, 6).join(self.range(1, 5))) | ||
self.assertRaises(ValueError, | ||
lambda: self.range(0, 0).join( | ||
source.Range(source.Buffer(""), 0, 0))) | ||
|
||
def test_source(self): | ||
self.assertEqual("one", self.range(5, 8).source()) | ||
|
||
def test_source_line(self): | ||
self.assertEqual("line two\n", self.range(9, 9).source_line()) | ||
|
||
def test___str__(self): | ||
self.assertEqual("<input>:2:1", str(self.range(9, 9))) | ||
|
||
def test___ne__(self): | ||
self.assertTrue(self.range(0,0) != self.range(0,1)) | ||
self.assertFalse(self.range(0,0) != self.range(0,0)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
[bdist_wheel] | ||
universal=1 | ||
|
||
[build_sphinx] | ||
source-dir = doc/ | ||
build-dir = doc/_build | ||
all_files = 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
from setuptools import setup, find_packages | ||
import os | ||
|
||
setup( | ||
name="artiq", | ||
version="0.0+dev", | ||
author="whitequark", | ||
author_email="whitequark@whitequark.org", | ||
url="https://github.com/whitequark/parser", | ||
This comment has been minimized.
Sorry, something went wrong. |
||
description="A Python parser intended for use in tooling", | ||
long_description=open("README.rst").read(), | ||
license="MIT", | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong. |
||
install_requires=[], | ||
extras_require={}, | ||
dependency_links=[], | ||
packages=find_packages(exclude=['tests*']), | ||
namespace_packages=[], | ||
test_suite="pyparser.test", | ||
package_data={}, | ||
ext_modules=[], | ||
entry_points={} | ||
) |
need updating