[pypy-svn] r78320 - in pypy/branch/fast-forward/pypy/interpreter/pyparser: . test

[afa at codespeak.net]

Author: afa
Date: Tue Oct 26 22:09:48 2010
New Revision: 78320

Modified:
   pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
Allow \r as a newline;
also tokens that go to the end of the line
should stop on either \r or \n.

This also fixes a cpython test in test_compile.py


Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py	Tue Oct 26 22:09:48 2010
@@ -17,12 +17,17 @@
 def makePyPseudoDFA ():
     import string
     states = []
+    def makeEOL():
+        return group(states,
+                     newArcPair(states, "\n"),
+                     chain(states,
+                           newArcPair(states, "\r"),
+                           maybe(states, newArcPair(states, "\n"))))
     # ____________________________________________________________
     def makeLineCont ():
         return chain(states,
                      newArcPair(states, "\\"),
-                     maybe(states, newArcPair(states, "\r")),
-                     newArcPair(states, "\n"))
+                     makeEOL())
     # ____________________________________________________________
     # Ignore stuff
     def makeWhitespace ():
@@ -130,9 +135,7 @@
                      newArcPair(states, "~"))
     bracket = groupStr(states, "[](){}")
     special = group(states,
-                    chain(states,
-                          maybe(states, newArcPair(states, "\r")),
-                          newArcPair(states, "\n")),
+                    makeEOL(),
                     groupStr(states, "@:;.,`"))
     funny = group(states, operator, bracket, special)
     # ____________________________________________________________
@@ -146,13 +149,13 @@
                           makeStrPrefix(),
                           newArcPair(states, "'"),
                           any(states,
-                              notGroupStr(states, "\n'\\")),
+                              notGroupStr(states, "\r\n'\\")),
                           any(states,
                               chain(states,
                                     newArcPair(states, "\\"),
                                     newArcPair(states, DEFAULT),
                                     any(states,
-                                        notGroupStr(states, "\n'\\")))),
+                                        notGroupStr(states, "\r\n'\\")))),
                           group(states,
                                 newArcPair(states, "'"),
                                 makeLineCont())),
@@ -160,13 +163,13 @@
                           makeStrPrefix(),
                           newArcPair(states, '"'),
                           any(states,
-                              notGroupStr(states, '\n"\\')),
+                              notGroupStr(states, '\r\n"\\')),
                           any(states,
                               chain(states,
                                     newArcPair(states, "\\"),
                                     newArcPair(states, DEFAULT),
                                     any(states,
-                                        notGroupStr(states, '\n"\\')))),
+                                        notGroupStr(states, '\r\n"\\')))),
                           group(states,
                                 newArcPair(states, '"'),
                                 makeLineCont())))

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py	Tue Oct 26 22:09:48 2010
@@ -24,7 +24,7 @@
 # Automatically generated DFA's
 
 accepts = [True, True, True, True, True, True, True, True,
-           True, True, False, True, True, True, False, False,
+           True, True, False, True, True, True, True, False,
            False, False, True, True, True, True, True, False,
            True, False, True, False, True, False, False,
            True, False, False, False, False, True, False,
@@ -143,9 +143,11 @@
     # 14
     {'\n': 13},
     # 15
-    {automata.DEFAULT: 30, '\n': 27, "'": 28, '\\': 29},
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 28, '\\': 29},
     # 16
-    {automata.DEFAULT: 33, '\n': 27, '"': 31, '\\': 32},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 31, '\\': 32},
     # 17
     {'\n': 13, '\r': 14},
     # 18
@@ -195,13 +197,15 @@
     # 29
     {automata.DEFAULT: 37, '\n': 13, '\r': 14},
     # 30
-    {automata.DEFAULT: 30, '\n': 27, "'": 13, '\\': 29},
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
     # 31
     {'"': 13},
     # 32
     {automata.DEFAULT: 38, '\n': 13, '\r': 14},
     # 33
-    {automata.DEFAULT: 33, '\n': 27, '"': 13, '\\': 32},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
     # 34
     {'+': 39, '-': 39, '0': 40, '1': 40,
      '2': 40, '3': 40, '4': 40, '5': 40,
@@ -215,9 +219,11 @@
      '4': 36, '5': 36, '6': 36, '7': 36,
      '8': 36, '9': 36, 'J': 13, 'j': 13},
     # 37
-    {automata.DEFAULT: 37, '\n': 27, "'": 13, '\\': 29},
+    {automata.DEFAULT: 37, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
     # 38
-    {automata.DEFAULT: 38, '\n': 27, '"': 13, '\\': 32},
+    {automata.DEFAULT: 38, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
     # 39
     {'0': 40, '1': 40, '2': 40, '3': 40,
      '4': 40, '5': 40, '6': 40, '7': 40,

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py	Tue Oct 26 22:09:48 2010
@@ -96,6 +96,9 @@
         exc = py.test.raises(IndentationError, parse, input).value
         assert exc.msg == "unindent does not match any outer indentation level"
 
+    def test_mac_newline(self):
+        self.parse("this_is\ra_mac\rfile")
+
     def test_mode(self):
         assert self.parse("x = 43*54").type == syms.file_input
         tree = self.parse("43**54", "eval")