[Python-checkins] cpython (3.2): #13960: HTMLParser is now able to handle broken comments when strict=False.

ezio.melotti python-checkins at python.org
Fri Feb 10 09:51:05 CET 2012


http://hg.python.org/cpython/rev/242b697449d8
changeset:   74859:242b697449d8
branch:      3.2
parent:      74848:a352e24b9907
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Fri Feb 10 10:45:44 2012 +0200
summary:
  #13960: HTMLParser is now able to handle broken comments when strict=False.

files:
  Lib/html/parser.py          |  25 ++++++++++++++++++++-
  Lib/test/test_htmlparser.py |  30 +++++++++++++++++++++++++
  Misc/NEWS                   |   5 +++-
  3 files changed, 58 insertions(+), 2 deletions(-)


diff --git a/Lib/html/parser.py b/Lib/html/parser.py
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -184,7 +184,17 @@
                 elif startswith("<?", i):
                     k = self.parse_pi(i)
                 elif startswith("<!", i):
-                    k = self.parse_declaration(i)
+                    # this might fail with things like <! not a comment > or
+                    # <! -- space before '--' -->.  When strict is True an
+                    # error is raised, when it's False they will be considered
+                    # as bogus comments and parsed (see parse_bogus_comment).
+                    if self.strict:
+                        k = self.parse_declaration(i)
+                    else:
+                        try:
+                            k = self.parse_declaration(i)
+                        except HTMLParseError:
+                            k = self.parse_bogus_comment(i)
                 elif (i + 1) < n:
                     self.handle_data("<")
                     k = i + 1
@@ -256,6 +266,19 @@
             i = self.updatepos(i, n)
         self.rawdata = rawdata[i:]
 
+    # Internal -- parse bogus comment, return length or -1 if not terminated
+    # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
+    def parse_bogus_comment(self, i, report=1):
+        rawdata = self.rawdata
+        if rawdata[i:i+2] != '<!':
+            self.error('unexpected call to parse_comment()')
+        pos = rawdata.find('>', i+2)
+        if pos == -1:
+            return -1
+        if report:
+            self.handle_comment(rawdata[i+2:pos])
+        return pos + 1
+
     # Internal -- parse processing instr, return end or -1 if not terminated
     def parse_pi(self, i):
         rawdata = self.rawdata
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -323,6 +323,23 @@
                                 ("endtag", element_lower)],
                             collector=Collector())
 
+    def test_comments(self):
+        html = ("<!-- I'm a valid comment -->"
+                '<!--me too!-->'
+                '<!------>'
+                '<!---->'
+                '<!----I have many hyphens---->'
+                '<!-- I have a > in the middle -->'
+                '<!-- and I have -- in the middle! -->')
+        expected = [('comment', " I'm a valid comment "),
+                    ('comment', 'me too!'),
+                    ('comment', '--'),
+                    ('comment', ''),
+                    ('comment', '--I have many hyphens--'),
+                    ('comment', ' I have a > in the middle '),
+                    ('comment', ' and I have -- in the middle! ')]
+        self._run_check(html, expected)
+
     def test_condcoms(self):
         html = ('<!--[if IE & !(lte IE 8)]>aren\'t<![endif]-->'
                 '<!--[if IE 8]>condcoms<![endif]-->'
@@ -426,6 +443,19 @@
         # see #12888
         self.assertEqual(p.unescape('&#123; ' * 1050), '{ ' * 1050)
 
+    def test_broken_comments(self):
+        html = ('<! not really a comment >'
+                '<! not a comment either -->'
+                '<! -- close enough -->'
+                '<!!! another bogus comment !!!>')
+        expected = [
+            ('comment', ' not really a comment '),
+            ('comment', ' not a comment either --'),
+            ('comment', ' -- close enough --'),
+            ('comment', '!! another bogus comment !!!'),
+        ]
+        self._run_check(html, expected)
+
     def test_broken_condcoms(self):
         # these condcoms are missing the '--' after '<!' and before the '>'
         html = ('<![if !(IE)]>broken condcom<![endif]>'
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -113,6 +113,9 @@
 Library
 -------
 
+- Issue #13960: HTMLParser is now able to handle broken comments when
+  strict=False.
+
 - Issue #9021: Add an introduction to the copy module documentation.
 
 - Issue #6005: Examples in the socket library documentation use sendall, where
@@ -123,7 +126,7 @@
 
 - Issue #10881: Fix test_site failure with OS X framework builds.
 
-- Issue #964437 Make IDLE help window non-modal.
+- Issue #964437: Make IDLE help window non-modal.
   Patch by Guilherme Polo and Roger Serwy.
 
 - Issue #2945: Make the distutils upload command aware of bdist_rpm products.

-- 
Repository URL: http://hg.python.org/cpython


More information about the Python-checkins mailing list