[Python-checkins] Remove mojibake in the locale aliases mapping. (GH-6716)

[Serhiy Storchaka]

https://github.com/python/cpython/commit/a3f19c3f52ddff85dd52eaa01b77b2d50cc9af3f
commit: a3f19c3f52ddff85dd52eaa01b77b2d50cc9af3f
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2018-05-06T10:52:38+03:00
summary:

Remove mojibake in the locale aliases mapping. (GH-6716)

files:
M Lib/locale.py
M Tools/i18n/makelocalealias.py

diff --git a/Lib/locale.py b/Lib/locale.py
index 876d2eb1e123..f3d3973d038c 100644
--- a/Lib/locale.py
+++ b/Lib/locale.py
@@ -923,7 +923,6 @@ def getpreferredencoding(do_setlocale = True):
     'bo_in':                                'bo_IN.UTF-8',
     'bokmal':                               'nb_NO.ISO8859-1',
     'bokm\xe5l':                            'nb_NO.ISO8859-1',
-    'bokm\xef\xbf\xbd':                     'nb_NO.ISO8859-1',
     'br':                                   'br_FR.ISO8859-1',
     'br_fr':                                'br_FR.ISO8859-1',
     'brx_in':                               'brx_IN.UTF-8',
@@ -1072,7 +1071,6 @@ def getpreferredencoding(do_setlocale = True):
     'fr_fr':                                'fr_FR.ISO8859-1',
     'fr_lu':                                'fr_LU.ISO8859-1',
     'fran\xe7ais':                          'fr_FR.ISO8859-1',
-    'fran\xef\xbf\xbdis':                   'fr_FR.ISO8859-1',
     'fre_fr':                               'fr_FR.ISO8859-1',
     'french':                               'fr_FR.ISO8859-1',
     'french.iso88591':                      'fr_CH.ISO8859-1',
diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py
index c7ecacec3877..b407a8a643be 100755
--- a/Tools/i18n/makelocalealias.py
+++ b/Tools/i18n/makelocalealias.py
@@ -19,6 +19,9 @@ def parse(filename):
 
     with open(filename, encoding='latin1') as f:
         lines = list(f)
+    # Remove mojibake in /usr/share/X11/locale/locale.alias.
+    # b'\xef\xbf\xbd' == '\ufffd'.encode('utf-8')
+    lines = [line for line in lines if '\xef\xbf\xbd' not in line]
     data = {}
     for line in lines:
         line = line.strip()