Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: ab67bb8ddcc5
Choose a base ref
...
head repository: m-labs/pythonparser
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 0266a52ae263
Choose a head ref
  • 2 commits
  • 3 files changed
  • 1 contributor

Commits on May 8, 2015

  1. More consistent documentation.

    whitequark committed May 8, 2015
    Copy the full SHA
    3650f5c View commit details
  2. Lex comments as whitespace.

    whitequark committed May 8, 2015
    Copy the full SHA
    0266a52 View commit details
Showing with 23 additions and 13 deletions.
  1. +9 −8 pyparser/__init__.py
  2. +5 −5 pyparser/lexer.py
  3. +9 −0 pyparser/test/test_lexer.py
17 changes: 9 additions & 8 deletions pyparser/__init__.py
Original file line number Diff line number Diff line change
@@ -7,18 +7,19 @@ def parse(source, filename='<unknown>', mode='exec',
Parse a string into an abstract syntax tree.
This is the replacement for the built-in :meth:`..ast.parse`.
:param str source: Source code in the correct encoding
:param filename: Filename of the source (used in diagnostics)
:param mode: Execution mode. Pass ``"exec"`` to parse a module,
:param source: (string) Source code in the correct encoding
:param filename: (string) Filename of the source (used in diagnostics)
:param mode: (string) Execution mode. Pass ``"exec"`` to parse a module,
``"single"`` to parse a single (interactive) statement,
and ``"eval"`` to parse an expression. In the last two cases,
``source`` must be terminated with an empty line
(i.e. end with ``"\\n\\n"``).
:param flags: Future flags. Equivalent to ``from __future__ import <flags>``.
:param int,int version: A tuple of the major and minor version
of Python syntax to recognize.
:return ast.AST: abstract syntax tree
:raise diagnostic.DiagnosticException:
:param flags: (list of string) Future flags.
Equivalent to ``from __future__ import <flags>``.
:param version: (2-tuple of int) Major and minor version of Python
syntax to recognize.
:return: (:class:`ast.AST`) abstract syntax tree
:raise: :class:`diagnostic.DiagnosticException`
if the source code is not well-formed
"""
buffer = pyparser.source.Buffer(source, filename)
10 changes: 5 additions & 5 deletions pyparser/lexer.py
Original file line number Diff line number Diff line change
@@ -299,14 +299,14 @@ def _refill(self, eof_token):
self.queue.append(Token(tok_range, "newline"))
return

# Lexing non-whitespace now.
self.new_line = False

if match.group(4) is not None: # comment
self.comments.append((tok_range, match.group(4)))
self._refill(eof_token)
return self._refill(eof_token)

# Lexing non-whitespace now.
self.new_line = False

elif match.group(5) is not None: # floating point or complex literal
if match.group(5) is not None: # floating point or complex literal
if match.group(6) is None:
self.queue.append(Token(tok_range, "float", float(match.group(5))))
else:
9 changes: 9 additions & 0 deletions pyparser/test/test_lexer.py
Original file line number Diff line number Diff line change
@@ -65,6 +65,15 @@ def test_comment(self):
self.assertEqual([(source.Range(self.buffer, 0, 5), "# foo")],
self.lexer.comments)

self.assertLexes("class x:\n # foo\n pass",
"class", None,
"ident", "x",
":", None,
"newline", None,
"indent", None,
"pass", None,
"dedent", None)

def test_float(self):
self.assertLexes("0.0",
"float", 0.0)