#670664: Fix HTMLParser to correctly handle the content of...

#670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.

Doc/library/html.parser.rst

@@ -115,7 +115,8 @@ An exception is defined as well:
 
 .. method:: HTMLParser.handle_data(data)
 
-   This method is called to process arbitrary data.  It is intended to be
+   This method is called to process arbitrary data (e.g. the content of
+   ``<script>...</script>`` and ``<style>...</style>``).  It is intended to be
    overridden by a derived class; the base class implementation does nothing.
 
 

Lib/html/parser.py

@@ -62,6 +62,8 @@ locatestarttagend_tolerant = re.compile(r"""
   \s*                                # trailing whitespace
 """, re.VERBOSE)
 endendtag = re.compile('>')
+# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between
+# </ and the tag name, so maybe this should be fixed
 endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
 
 
@@ -121,6 +123,7 @@ class HTMLParser(_markupbase.ParserBase):
         self.rawdata = ''
         self.lasttag = '???'
         self.interesting = interesting_normal
+        self.cdata_elem = None
         _markupbase.ParserBase.reset(self)
 
     def feed(self, data):
@@ -145,11 +148,13 @@ class HTMLParser(_markupbase.ParserBase):
         """Return full source of start tag: '<...>'."""
         return self.__starttag_text
 
-    def set_cdata_mode(self):
+    def set_cdata_mode(self, elem):
         self.interesting = interesting_cdata
+        self.cdata_elem = elem.lower()
 
     def clear_cdata_mode(self):
         self.interesting = interesting_normal
+        self.cdata_elem = None
 
     # Internal -- handle data as far as reasonable.  May leave state
     # and data to be processed by a subsequent call.  If 'end' is
@@ -314,7 +319,7 @@ class HTMLParser(_markupbase.ParserBase):
         else:
             self.handle_starttag(tag, attrs)
             if tag in self.CDATA_CONTENT_ELEMENTS:
-                self.set_cdata_mode()
+                self.set_cdata_mode(tag)
         return endpos
 
     # Internal -- check to see if we have a complete starttag; return end
@@ -371,6 +376,9 @@ class HTMLParser(_markupbase.ParserBase):
         j = match.end()
         match = endtagfind.match(rawdata, i) # </ + tag + >
         if not match:
+            if self.cdata_elem is not None:
+                self.handle_data(rawdata[i:j])
+                return j
             if self.strict:
                 self.error("bad end tag: %r" % (rawdata[i:j],))
             k = rawdata.find('<', i + 1, j)
@@ -380,8 +388,14 @@ class HTMLParser(_markupbase.ParserBase):
                 j = i + 1
             self.handle_data(rawdata[i:j])
             return j
-        tag = match.group(1)
-        self.handle_endtag(tag.lower())
+
+        elem = match.group(1).lower() # script or style
+        if self.cdata_elem is not None:
+            if elem != self.cdata_elem:
+                self.handle_data(rawdata[i:j])
+                return j
+
+        self.handle_endtag(elem.lower())
         self.clear_cdata_mode()
         return j
 

Lib/test/test_htmlparser.py

@@ -321,18 +321,36 @@ DOCTYPE html [
             ("starttag_text", s)])
 
     def test_cdata_content(self):
-        s = """<script> <!-- not a comment --> &not-an-entity-ref; </script>"""
-        self._run_check(s, [
-            ("starttag", "script", []),
-            ("data", " <!-- not a comment --> &not-an-entity-ref; "),
-            ("endtag", "script"),
-            ])
-        s = """<script> <not a='start tag'> </script>"""
-        self._run_check(s, [
-            ("starttag", "script", []),
-            ("data", " <not a='start tag'> "),
-            ("endtag", "script"),
-            ])
+        contents = [
+            '<!-- not a comment --> &not-an-entity-ref;',
+            "<not a='start tag'>",
+            '<a href="" /> <p> <span></span>',
+            'foo = "</scr" + "ipt>";',
+            'foo = "</SCRIPT" + ">";',
+            'foo = <\n/script> ',
+            '<!-- document.write("</scr" + "ipt>"); -->',
+            ('\n//<![CDATA[\n'
+             'document.write(\'<s\'+\'cript type="text/javascript" '
+             'src="http://www.example.org/r=\'+new '
+             'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'),
+            '\n<!-- //\nvar foo = 3.14;\n// -->\n',
+            'foo = "</sty" + "le>";',
+            '<!-- \u2603 -->',
+            # these two should be invalid according to the HTML 5 spec,
+            # section 8.1.2.2
+            #'foo = </\nscript>',
+            #'foo = </ script>',
+        ]
+        elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style']
+        for content in contents:
+            for element in elements:
+                element_lower = element.lower()
+                s = '<{element}>{content}</{element}>'.format(element=element,
+                                                               content=content)
+                self._run_check(s, [("starttag", element_lower, []),
+                                    ("data", content),
+                                    ("endtag", element_lower)])
+
 
     def test_entityrefs_in_attributes(self):
         self._run_check("<html foo='&euro;&amp;&#97;&#x61;&unsupported;'>", [

Misc/NEWS

@@ -66,10 +66,13 @@ Core and Builtins
 Library
 -------
 
-- Issue 10817: Fix urlretrieve function to raise ContentTooShortError even
+- Issue #670664: Fix HTMLParser to correctly handle the content of
+  ``<script>...</script>`` and ``<style>...</style>``.
+
+- Issue #10817: Fix urlretrieve function to raise ContentTooShortError even
   when reporthook is None. Patch by Jyrki Pulliainen.
 
-- Issue 13296: Fix IDLE to clear compile __future__ flags on shell restart.
+- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart.
   (Patch by Roger Serwy)
 
 - Issue #13293: Better error message when trying to marshal bytes using