[Python-checkins] gh-82927: Update files related to HTML entities. (GH-92504)
ambv
webhook-mailer at python.org
Tue Jun 21 16:03:20 EDT 2022
https://github.com/python/cpython/commit/f28ec34c5c69f45dc0db8bc26df69bc7defe9c9d
commit: f28ec34c5c69f45dc0db8bc26df69bc7defe9c9d
branch: main
author: Ezio Melotti <ezio.melotti at gmail.com>
committer: ambv <lukasz at langa.pl>
date: 2022-06-21T22:03:12+02:00
summary:
gh-82927: Update files related to HTML entities. (GH-92504)
files:
A Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst
D Tools/scripts/parseentities.py
M .github/CODEOWNERS
M Doc/library/html.entities.rst
M Lib/html/entities.py
M Tools/scripts/parse_html5_entities.py
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 58023b63833ee..013e1cbd7241d 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -53,6 +53,7 @@ Python/pythonrun.c @iritkatriel
/Lib/html/ @ezio-melotti
/Lib/_markupbase.py @ezio-melotti
/Lib/test/test_html*.py @ezio-melotti
+/Tools/scripts/*html5* @ezio-melotti
# Import (including importlib).
# Ignoring importlib.h so as to not get flagged on
diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst
index cb8b623781c5e..10529561a92cd 100644
--- a/Doc/library/html.entities.rst
+++ b/Doc/library/html.entities.rst
@@ -34,12 +34,12 @@ This module defines four dictionaries, :data:`html5`,
.. data:: name2codepoint
- A dictionary that maps HTML entity names to the Unicode code points.
+ A dictionary that maps HTML4 entity names to the Unicode code points.
.. data:: codepoint2name
- A dictionary that maps Unicode code points to HTML entity names.
+ A dictionary that maps Unicode code points to HTML4 entity names.
.. rubric:: Footnotes
diff --git a/Lib/html/entities.py b/Lib/html/entities.py
index dc508631ac478..cc59bc314499a 100644
--- a/Lib/html/entities.py
+++ b/Lib/html/entities.py
@@ -3,8 +3,7 @@
__all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs']
-# maps the HTML entity name to the Unicode code point
-# from https://html.spec.whatwg.org/multipage/named-characters.html
+# maps HTML4 entity name to the Unicode code point
name2codepoint = {
'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -261,7 +260,11 @@
}
-# maps the HTML5 named character references to the equivalent Unicode character(s)
+# HTML5 named character references
+# Generated by 'Tools/scripts/parse_html5_entities.py'
+# from https://html.spec.whatwg.org/entities.json and
+# https://html.spec.whatwg.org/multipage/named-characters.html.
+# Map HTML5 named character references to the equivalent Unicode character(s).
html5 = {
'Aacute': '\xc1',
'aacute': '\xe1',
diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst
new file mode 100644
index 0000000000000..a10d4c7688452
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst
@@ -0,0 +1,2 @@
+The ``Tools/scripts/parseentities.py`` script used to parse HTML4 entities
+has been removed.
diff --git a/Tools/scripts/parse_html5_entities.py b/Tools/scripts/parse_html5_entities.py
index c011328b0101b..1e5bdad216554 100755
--- a/Tools/scripts/parse_html5_entities.py
+++ b/Tools/scripts/parse_html5_entities.py
@@ -2,10 +2,14 @@
"""
Utility for parsing HTML5 entity definitions available from:
- http://dev.w3.org/html5/spec/entities.json
+ https://html.spec.whatwg.org/entities.json
+ https://html.spec.whatwg.org/multipage/named-characters.html
-Written by Ezio Melotti and Iuliia Proskurnia.
+The page now contains the following note:
+
+ "This list is static and will not be expanded or changed in the future."
+Written by Ezio Melotti and Iuliia Proskurnia.
"""
import os
@@ -14,7 +18,9 @@
from urllib.request import urlopen
from html.entities import html5
-entities_url = 'http://dev.w3.org/html5/spec/entities.json'
+PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html'
+ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json'
+HTML5_SECTION_START = '# HTML5 named character references'
def get_json(url):
"""Download the json file from the url and returns a decoded object."""
@@ -62,9 +68,15 @@ def write_items(entities, file=sys.stdout):
# be before their equivalent lowercase version.
keys = sorted(entities.keys())
keys = sorted(keys, key=str.lower)
+ print(HTML5_SECTION_START, file=file)
+ print(f'# Generated by {sys.argv[0]!r}\n'
+ f'# from {ENTITIES_URL} and\n'
+ f'# {PAGE_URL}.\n'
+ f'# Map HTML5 named character references to the '
+ f'equivalent Unicode character(s).', file=file)
print('html5 = {', file=file)
for name in keys:
- print(' {!r}: {!a},'.format(name, entities[name]), file=file)
+ print(f' {name!r}: {entities[name]!a},', file=file)
print('}', file=file)
@@ -72,11 +84,8 @@ def write_items(entities, file=sys.stdout):
# without args print a diff between html.entities.html5 and new_html5
# with --create print the new html5 dict
# with --patch patch the Lib/html/entities.py file
- new_html5 = create_dict(get_json(entities_url))
+ new_html5 = create_dict(get_json(ENTITIES_URL))
if '--create' in sys.argv:
- print('# map the HTML5 named character references to the '
- 'equivalent Unicode character(s)')
- print('# Generated by {}. Do not edit manually.'.format(__file__))
write_items(new_html5)
elif '--patch' in sys.argv:
fname = 'Lib/html/entities.py'
@@ -84,7 +93,7 @@ def write_items(entities, file=sys.stdout):
with open(fname) as f1, open(temp_fname, 'w') as f2:
skip = False
for line in f1:
- if line.startswith('html5 = {'):
+ if line.startswith(HTML5_SECTION_START):
write_items(new_html5, file=f2)
skip = True
continue
diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py
deleted file mode 100755
index 0229d3af86ba7..0000000000000
--- a/Tools/scripts/parseentities.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-""" Utility for parsing HTML entity definitions available from:
-
- http://www.w3.org/ as e.g.
- http://www.w3.org/TR/REC-html40/HTMLlat1.ent
-
- Input is read from stdin, output is written to stdout in form of a
- Python snippet defining a dictionary "entitydefs" mapping literal
- entity name to character or numeric entity.
-
- Marc-Andre Lemburg, mal at lemburg.com, 1999.
- Use as you like. NO WARRANTIES.
-
-"""
-import re,sys
-
-entityRE = re.compile(r'<!ENTITY +(\w+) +CDATA +"([^"]+)" +-- +((?:.|\n)+?) *-->')
-
-def parse(text,pos=0,endpos=None):
-
- pos = 0
- if endpos is None:
- endpos = len(text)
- d = {}
- while 1:
- m = entityRE.search(text,pos,endpos)
- if not m:
- break
- name,charcode,comment = m.groups()
- d[name] = charcode,comment
- pos = m.end()
- return d
-
-def writefile(f,defs):
-
- f.write("entitydefs = {\n")
- items = sorted(defs.items())
- for name, (charcode,comment) in items:
- if charcode[:2] == '&#':
- code = int(charcode[2:-1])
- if code < 256:
- charcode = r"'\%o'" % code
- else:
- charcode = repr(charcode)
- else:
- charcode = repr(charcode)
- comment = ' '.join(comment.split())
- f.write(" '%s':\t%s, \t# %s\n" % (name,charcode,comment))
- f.write('\n}\n')
-
-if __name__ == '__main__':
- if len(sys.argv) > 1:
- with open(sys.argv[1]) as infile:
- text = infile.read()
- else:
- text = sys.stdin.read()
-
- defs = parse(text)
-
- if len(sys.argv) > 2:
- with open(sys.argv[2],'w') as outfile:
- writefile(outfile, defs)
- else:
- writefile(sys.stdout, defs)
More information about the Python-checkins
mailing list