[Python-checkins] closes bpo-34515: Support non-ASCII identifiers in lib2to3. (GH-8950)
Miss Islington (bot)
webhook-mailer at python.org
Sat Sep 15 13:53:02 EDT 2018
https://github.com/python/cpython/commit/51dbae867e82014f9af89662977e4981463c51e8
commit: 51dbae867e82014f9af89662977e4981463c51e8
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2018-09-15T10:52:57-07:00
summary:
closes bpo-34515: Support non-ASCII identifiers in lib2to3. (GH-8950)
(cherry picked from commit 10a428b64b3f224e2ccd40ff2afb141b9b3425b1)
Co-authored-by: Monson Shao <holymonson at gmail.com>
files:
A Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
M Lib/lib2to3/pgen2/tokenize.py
M Lib/lib2to3/tests/test_parser.py
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index bf83ab8c7d9d..c07b34f7c645 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -56,7 +56,7 @@ def _combinations(*l):
Whitespace = r'[ \f\t]*'
Comment = r'#[^\r\n]*'
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'[a-zA-Z_]\w*'
+Name = r'\w+'
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
@@ -107,8 +107,8 @@ def _combinations(*l):
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-tokenprog, pseudoprog, single3prog, double3prog = list(map(
- re.compile, (Token, PseudoToken, Single3, Double3)))
+tokenprog, pseudoprog, single3prog, double3prog = map(
+ re.compile, (Token, PseudoToken, Single3, Double3))
_strprefixes = (
_combinations('r', 'R', 'f', 'F') |
@@ -349,7 +349,6 @@ def generate_tokens(readline):
logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
- namechars, numchars = string.ascii_letters + '_', '0123456789'
contstr, needcont = '', 0
contline = None
indents = [0]
@@ -451,7 +450,7 @@ def generate_tokens(readline):
spos, epos, pos = (lnum, start), (lnum, end), end
token, initial = line[start:end], line[start]
- if initial in numchars or \
+ if initial in string.digits or \
(initial == '.' and token != '.'): # ordinary number
yield (NUMBER, token, spos, epos, line)
elif initial in '\r\n':
@@ -501,7 +500,7 @@ def generate_tokens(readline):
yield stashed
stashed = None
yield (STRING, token, spos, epos, line)
- elif initial in namechars: # ordinary name
+ elif initial.isidentifier(): # ordinary name
if token in ('async', 'await'):
if async_def:
yield (ASYNC if token == 'async' else AWAIT,
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 7eb9afac46e0..829e5a72924a 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -529,6 +529,16 @@ def test_4(self):
self.validate("""x = {2, 3, 4,}""")
+# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and
+# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
+class TestIdentfier(GrammarTest):
+ def test_non_ascii_identifiers(self):
+ self.validate("Örter = 'places'\ngrün = 'green'")
+ self.validate("蟒 = a蟒 = 锦蛇 = 1")
+ self.validate("µ = aµ = µµ = 1")
+ self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1")
+
+
class TestNumericLiterals(GrammarTest):
def test_new_octal_notation(self):
self.validate("""0o7777777777777""")
diff --git a/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
new file mode 100644
index 000000000000..7ace7ba8218c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst
@@ -0,0 +1 @@
+Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP 3131).
More information about the Python-checkins
mailing list