Initial commit.

m-labs · Mar 30, 2015 · 58559cf · sbourdeauducq · Mar 30, 2015 · sbourdeauducq
commit 58559cf
Show file tree

Hide file tree

Showing 11 changed files with 323 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+*.pyc
+__pycache__/
+_build/
+*.egg-info/
+
diff --git a/README.rst b/README.rst
diff --git a/doc/conf.py b/doc/conf.py
@@ -0,0 +1,6 @@
+master_doc = 'index'
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+]
diff --git a/doc/index.rst b/doc/index.rst
@@ -0,0 +1,21 @@
+PyParser documentation
+======================
+
+PyParser is a Python parser written specifically for use in tooling.
+It parses source code into an AST that is a superset of Python's
+built-in :mod:`ast` module, but returns precise location information
+for every token.
+
+:mod:`source` Module
+--------------------
+
+.. automodule:: pyparser.source
+    :members:
+    :show-inheritance:
+
+:mod:`lexer` Module
+--------------------
+
+.. automodule:: pyparser.lexer
+    :members:
+    :show-inheritance:
diff --git a/pyparser/__init__.py b/pyparser/__init__.py
diff --git a/pyparser/lexer.py b/pyparser/lexer.py
@@ -0,0 +1,5 @@
+class Lexer:
+    keywords = []
+
+    def __init__(self, source, filename="<input>", line=1):
+        pass
diff --git a/pyparser/source.py b/pyparser/source.py
@@ -0,0 +1,170 @@
+"""
+The :mod:`source` module concerns itself with manipulating
+buffers of source code: creating ranges of characters corresponding
+to a token, combining these ranges, extracting human-readable
+location information and original source from a range.
+"""
+
+import bisect
+
+class Buffer:
+    """
+    A buffer containing source code and location information.
+
+    :ivar source: (string) source code
+    :ivar name: (string) input filename or another description
+        of the input (e.g. ``<stdin>``).
+    :ivar line: (integer) first line of the input
+    """
+    def __init__(self, source, name="<input>", first_line=1):
+        self.source = source
+        self.name = name
+        self.first_line = first_line
+        self._line_begins = None
+
+    def __repr__(self):
+        return r'Buffer("%s")' % self.name
+
+    def source_line(self, lineno):
+        """
+        Returns line ``lineno`` from source, taking ``first_line`` into account,
+        or raises :exc:`IndexError` if ``lineno`` is out of range.
+        """
+        line_begins = self._extract_line_begins()
+        lineno = lineno - self.first_line
+        if lineno >= 0 and lineno + 1 < len(line_begins):
+            first, last = line_begins[lineno:lineno + 2]
+            return self.source[first:last]
+        elif lineno >= 0 and lineno < len(line_begins):
+            return self.source[line_begins[-1]:]
+        else:
+            raise IndexError
+
+    def decompose_position(self, offset):
+        """
+        Returns a ``line, column`` tuple for a character offset into the source,
+        orraises :exc:`IndexError` if ``lineno`` is out of range.
+        """
+        line_begins = self._extract_line_begins()
+        lineno = bisect.bisect_right(line_begins, offset) - 1
+        if offset >= 0 and offset < len(self.source):
+            return lineno + self.first_line, offset - line_begins[lineno]
+        else:
+            raise IndexError
+
+    def _extract_line_begins(self):
+        if self._line_begins:
+            return self._line_begins
+
+        self._line_begins = [0]
+        index = None
+        while True:
+            index = self.source.find(u"\n", index) + 1
+            if index == 0:
+                return self._line_begins
+            self._line_begins.append(index)
+
+class Range:
+    """
+    Location of an exclusive range of characters [*begin_pos*, *end_pos*)
+    in a :class:`Buffer`.
+
+    :ivar begin_pos: (integer) offset of the first character
+    :ivar end_pos: (integer) offset of the character before the last
+    """
+    def __init__(self, source_buffer, begin_pos, end_pos):
+        self.source_buffer = source_buffer
+        self.begin_pos = begin_pos
+        self.end_pos = end_pos
+
+    def __repr__(self):
+        """
+        Returns a human-readable representation of this range.
+        """
+        return r'Range("%s", %d, %d)' % \
+            (self.source_buffer.name, self.begin_pos, self.end_pos)
+
+    def begin(self):
+        """
+        Returns a zero-length range located just before the beginning of this range.
+        """
+        return Range(self.source_buffer, self.begin_pos, self.begin_pos)
+
+    def end(self):
+        """
+        Returns a zero-length range located just after the end of this range.
+        """
+        return Range(self.source_buffer, self.end_pos, self.end_pos)
+
+    def size(self):
+        """
+        Returns the amount of characters spanned by the range.
+        """
+        return self.end_pos - self.begin_pos
+
+    def column(self):
+        """
+        Returns a zero-based column number of the beginning of this range.
+        """
+        line, column = self.source_buffer.decompose_position(self.begin_pos)
+        return column
+
+    def column_range(self):
+        """
+        Returns a [*begin*, *end*) tuple describing the range of columns spanned
+        by this range.
+        """
+        return self.begin().column(), self.end().column()
+
+    def line(self):
+        """
+        Returns the line number of the beginning of this range.
+        """
+        line, column = self.source_buffer.decompose_position(self.begin_pos)
+        return line
+
+    def join(self, other):
+        """
+        Returns the smallest possible range spanning both this range and other.
+        Raises :exc:`ValueError` if the ranges do not belong to the same
+        :class:`Buffer`.
+        """
+        if self.source_buffer != other.source_buffer:
+            raise ValueError
+        return Range(self.source_buffer,
+                     min(self.begin_pos, other.begin_pos),
+                     max(self.end_pos, other.end_pos))
+
+    def source(self):
+        """
+        Returns the source code covered by this range.
+        """
+        return self.source_buffer.source[self.begin_pos:self.end_pos]
+
+    def source_line(self):
+        """
+        Returns the line of source code containing the beginning of this range.
+        """
+        return self.source_buffer.source_line(self.line())
+
+    def __str__(self):
+        """
+        Returns a Clang-style string representation of the beginning of this range.
+        """
+        return ':'.join([self.source_buffer.name,
+                         str(self.line()), str(self.column() + 1)])
+
+    def __eq__(self, other):
+        """
+        Returns true if the ranges have the same source buffer, start and end position.
+        """
+        return (type(self) == type(other) and
+            self.source_buffer == other.source_buffer and
+            self.begin_pos == other.begin_pos and
+            self.end_pos == other.end_pos)
+
+    def __ne__(self, other):
+        """
+        Inverse of :meth:`__eq__`.
+        """
+        return not (self == other)
diff --git a/pyparser/test/__init__.py b/pyparser/test/__init__.py
diff --git a/pyparser/test/test_source.py b/pyparser/test/test_source.py
@@ -0,0 +1,87 @@
+import unittest
+import pyparser.source as source
+
+class BufferTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.buffer = source.Buffer("line one\nline two\n\nline four")
+
+    def test_repr(self):
+        self.assertEqual(r'Buffer("<input>")', repr(self.buffer))
+
+    def test_source_line(self):
+        self.assertEqual("line one\n", self.buffer.source_line(1))
+        self.assertEqual("line two\n", self.buffer.source_line(2))
+        self.assertEqual("\n", self.buffer.source_line(3))
+        self.assertEqual("line four", self.buffer.source_line(4))
+        self.assertRaises(IndexError, lambda: self.buffer.source_line(0))
+
+    def test_decompose_position(self):
+        self.assertEqual((1,0), self.buffer.decompose_position(0))
+        self.assertEqual((1,2), self.buffer.decompose_position(2))
+        self.assertEqual((1,8), self.buffer.decompose_position(8))
+        self.assertEqual((2,0), self.buffer.decompose_position(9))
+        self.assertRaises(IndexError, lambda: self.buffer.decompose_position(90))
+
+class RangeTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.buffer = source.Buffer("line one\nline two\n\nline four")
+
+    def range(self, lft, rgt):
+        return source.Range(self.buffer, lft, rgt)
+
+    def test_repr(self):
+        self.assertEqual(r'Range("<input>", 0, 2)',
+                         repr(self.range(0, 2)))
+
+    def test_begin(self):
+        self.assertEqual(self.range(1, 1),
+                         self.range(1, 2).begin())
+
+    def test_end(self):
+        self.assertEqual(self.range(2, 2),
+                         self.range(1, 2).end())
+
+    def test_size(self):
+        self.assertEqual(1, self.range(2, 3).size())
+
+    def test_column(self):
+        self.assertEqual(2, self.range(2, 2).column())
+        self.assertEqual(0, self.range(9, 11).column())
+        self.assertEqual(2, self.range(11, 11).column())
+
+    def test_column_range(self):
+        self.assertEqual((2,2), self.range(2, 2).column_range())
+        self.assertEqual((0,2), self.range(9, 11).column_range())
+        self.assertEqual((2,2), self.range(11, 11).column_range())
+
+    def test_line(self):
+        self.assertEqual(1, self.range(2, 2).line())
+        self.assertEqual(2, self.range(9, 11).line())
+        self.assertEqual(2, self.range(11, 11).line())
+
+    def test_line(self):
+        self.assertEqual(1, self.range(2, 2).line())
+        self.assertEqual(2, self.range(9, 11).line())
+        self.assertEqual(2, self.range(11, 11).line())
+
+    def test_join(self):
+        self.assertEqual(self.range(1, 6),
+                         self.range(2, 6).join(self.range(1, 5)))
+        self.assertRaises(ValueError,
+                          lambda: self.range(0, 0).join(
+                                source.Range(source.Buffer(""), 0, 0)))
+
+    def test_source(self):
+        self.assertEqual("one", self.range(5, 8).source())
+
+    def test_source_line(self):
+        self.assertEqual("line two\n", self.range(9, 9).source_line())
+
+    def test___str__(self):
+        self.assertEqual("<input>:2:1", str(self.range(9, 9)))
+
+    def test___ne__(self):
+        self.assertTrue(self.range(0,0) != self.range(0,1))
+        self.assertFalse(self.range(0,0) != self.range(0,0))
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,7 @@
+[bdist_wheel]
+universal=1
+
+[build_sphinx]
+source-dir = doc/
+build-dir = doc/_build
+all_files = 1
diff --git a/setup.py b/setup.py
@@ -0,0 +1,22 @@
+from setuptools import setup, find_packages
+import os
+
+setup(
+    name="artiq",
+    version="0.0+dev",
+    author="whitequark",
+    author_email="whitequark@whitequark.org",
+    url="https://github.com/whitequark/parser",
+    description="A Python parser intended for use in tooling",
+    long_description=open("README.rst").read(),
+    license="MIT",
+    install_requires=[],
+    extras_require={},
+    dependency_links=[],
+    packages=find_packages(exclude=['tests*']),
+    namespace_packages=[],
+    test_suite="pyparser.test",
+    package_data={},
+    ext_modules=[],
+    entry_points={}
+)