Fix the suspicious builder not to write CRLF, update suspicious file and add...

Fix the suspicious builder not to write CRLF, update suspicious file and add instructions to Makefile.

Doc/Makefile

@@ -87,13 +87,14 @@ changes: build
 
 linkcheck: BUILDER = linkcheck
 linkcheck: build
-	@echo "Link check complete; look for any errors in the above output " \
+	@echo "Link check complete; look for any errors in the above output" \
 	      "or in build/$(BUILDER)/output.txt"
 
 suspicious: BUILDER = suspicious
 suspicious: build
-	@echo "Suspicious check complete; look for any errors in the above output " \
-	      "or in build/$(BUILDER)/suspicious.txt"
+	@echo "Suspicious check complete; look for any errors in the above output" \
+	      "or in build/$(BUILDER)/suspicious.csv.  If all issues are false" \
+	      "positives, append that file to tools/sphinxext/susp-ignored.csv."
 
 coverage: BUILDER = coverage
 coverage: build
@@ -101,12 +102,12 @@ coverage: build
 
 doctest: BUILDER = doctest
 doctest: build
-	@echo "Testing of doctests in the sources finished, look at the " \
+	@echo "Testing of doctests in the sources finished, look at the" \
 	      "results in build/doctest/output.txt"
 
 pydoc-topics: BUILDER = pydoc-topics
 pydoc-topics: build
-	@echo "Building finished; now copy build/pydoc-topics/topics.py " \
+	@echo "Building finished; now copy build/pydoc-topics/topics.py" \
 	      "to Lib/pydoc_data/topics.py"
 
 htmlview: html

Doc/tools/sphinxext/susp-ignored.csv

@@ -164,3 +164,160 @@ whatsnew/2.4,,:System,
 whatsnew/2.5,,:memory,:memory:
 whatsnew/2.5,,:step,[start:stop:step]
 whatsnew/2.5,,:stop,[start:stop:step]
+distutils/examples,267,`,This is the description of the ``foobar`` package.
+documenting/fromlatex,39,:func,:func:`str(object)`
+documenting/fromlatex,39,`,:func:`str(object)`
+documenting/fromlatex,39,`,``str(object)``
+documenting/fromlatex,55,.. deprecated:,.. deprecated:: 2.5
+documenting/fromlatex,66,.. note:,.. note::
+documenting/fromlatex,76,:samp,":samp:`open({filename}, {mode})`"
+documenting/fromlatex,76,`,":samp:`open({filename}, {mode})`"
+documenting/fromlatex,80,`,``'c'``
+documenting/fromlatex,80,`,`Title <URL>`_
+documenting/fromlatex,80,`,``code``
+documenting/fromlatex,80,`,`Title <URL>`_
+documenting/fromlatex,99,:file,:file:`C:\\Temp\\my.tmp`
+documenting/fromlatex,99,`,:file:`C:\\Temp\\my.tmp`
+documenting/fromlatex,99,`,"``open(""C:\Temp\my.tmp"")``"
+documenting/fromlatex,129,.. function:,.. function:: do_foo(bar)
+documenting/fromlatex,141,.. function:,".. function:: open(filename[, mode[, buffering]])"
+documenting/fromlatex,152,.. function:,.. function:: foo_*
+documenting/fromlatex,152,:noindex,:noindex:
+documenting/fromlatex,162,.. describe:,.. describe:: a == b
+documenting/fromlatex,168,.. cmdoption:,.. cmdoption:: -O
+documenting/fromlatex,168,.. envvar:,.. envvar:: PYTHONINSPECT
+documenting/markup,33,.. sectionauthor:,.. sectionauthor:: Guido van Rossum <guido@python.org>
+documenting/markup,42,:mod,:mod:`parrot` -- Dead parrot access
+documenting/markup,42,`,:mod:`parrot` -- Dead parrot access
+documenting/markup,42,.. module:,.. module:: parrot
+documenting/markup,42,:platform,":platform: Unix, Windows"
+documenting/markup,42,:synopsis,:synopsis: Analyze and reanimate dead parrots.
+documenting/markup,42,.. moduleauthor:,.. moduleauthor:: Eric Cleese <eric@python.invalid>
+documenting/markup,42,.. moduleauthor:,.. moduleauthor:: John Idle <john@python.invalid>
+documenting/markup,88,:noindex,:noindex:
+documenting/markup,95,.. function:,.. function:: spam(eggs)
+documenting/markup,95,:noindex,:noindex:
+documenting/markup,101,.. method:,.. method:: FileInput.input(...)
+documenting/markup,121,:function,c:function
+documenting/markup,121,.. c:,".. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)"
+documenting/markup,121,::,".. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)"
+documenting/markup,131,:member,c:member
+documenting/markup,131,.. c:,.. c:member:: PyObject* PyTypeObject.tp_bases
+documenting/markup,131,::,.. c:member:: PyObject* PyTypeObject.tp_bases
+documenting/markup,139,:macro,c:macro
+documenting/markup,143,:type,c:type
+documenting/markup,150,:var,c:var
+documenting/markup,150,.. cvar:,.. cvar:: PyObject* PyClass_Type
+documenting/markup,179,.. function:,".. function:: Timer.repeat([repeat=3[, number=1000000]])"
+documenting/markup,210,.. decorator:,.. decorator:: removename
+documenting/markup,210,.. decorator:,.. decorator:: setnewname(name)
+documenting/markup,210,:func,:func:
+documenting/markup,233,:meth,:meth:
+documenting/markup,246,.. cmdoption:,.. cmdoption:: -m <module>
+documenting/markup,264,.. describe:,.. describe:: opcode
+documenting/markup,293,.. highlightlang:,.. highlightlang:: c
+documenting/markup,313,.. literalinclude:,.. literalinclude:: example.py
+documenting/markup,328,:rolename,:rolename:`content`
+documenting/markup,328,`,:rolename:`content`
+documenting/markup,333,:role,:role:`title <target>`
+documenting/markup,333,`,:role:`title <target>`
+documenting/markup,339,:meth,:meth:`~Queue.Queue.get`
+documenting/markup,339,`,:meth:`~Queue.Queue.get`
+documenting/markup,387,:func,:func:`filter`
+documenting/markup,387,`,:func:`filter`
+documenting/markup,387,:func,:func:`foo.filter`
+documenting/markup,387,`,:func:`foo.filter`
+documenting/markup,393,:func,:func:`open`
+documenting/markup,393,`,:func:`open`
+documenting/markup,393,:func,:func:`.open`
+documenting/markup,393,`,:func:`.open`
+documenting/markup,409,:data,c:data
+documenting/markup,413,:func,c:func
+documenting/markup,417,:macro,c:macro
+documenting/markup,421,:type,c:type
+documenting/markup,426,:member,c:member
+documenting/markup,476,:file,... is installed in :file:`/usr/lib/python2.{x}/site-packages` ...
+documenting/markup,476,`,... is installed in :file:`/usr/lib/python2.{x}/site-packages` ...
+documenting/markup,495,:kbd,:kbd:`C-x C-f`
+documenting/markup,495,`,:kbd:`C-x C-f`
+documenting/markup,495,:kbd,:kbd:`Control-x Control-f`
+documenting/markup,495,`,:kbd:`Control-x Control-f`
+documenting/markup,509,:mailheader,:mailheader:`Content-Type`
+documenting/markup,509,`,:mailheader:`Content-Type`
+documenting/markup,518,:manpage,:manpage:`ls(1)`
+documenting/markup,518,`,:manpage:`ls(1)`
+documenting/markup,534,:menuselection,:menuselection:`Start --> Programs`
+documenting/markup,534,`,:menuselection:`Start --> Programs`
+documenting/markup,549,`,``code``
+documenting/markup,567,:file,:file:
+documenting/markup,567,`,``code``
+documenting/markup,602,:ref,:ref:`label-name`
+documenting/markup,602,`,:ref:`label-name`
+documenting/markup,606,:ref,"It refers to the section itself, see :ref:`my-reference-label`."
+documenting/markup,606,`,"It refers to the section itself, see :ref:`my-reference-label`."
+documenting/markup,615,:ref,:ref:
+documenting/markup,636,.. note:,.. note::
+documenting/markup,663,.. versionadded:,.. versionadded:: 3.1
+documenting/markup,688,::,.. impl-detail::
+documenting/markup,688,::,.. impl-detail:: This shortly mentions an implementation detail.
+documenting/markup,708,.. seealso:,.. seealso::
+documenting/markup,708,:mod,Module :mod:`zipfile`
+documenting/markup,708,`,Module :mod:`zipfile`
+documenting/markup,708,:mod,Documentation of the :mod:`zipfile` standard module.
+documenting/markup,708,`,Documentation of the :mod:`zipfile` standard module.
+documenting/markup,708,`,"`GNU tar manual, Basic Tar Format <http://link>`_"
+documenting/markup,722,.. centered:,.. centered::
+documenting/markup,767,.. toctree:,.. toctree::
+documenting/markup,767,:maxdepth,:maxdepth: 2
+documenting/markup,783,.. index:,.. index::
+documenting/markup,813,.. index:,".. index:: BNF, grammar, syntax, notation"
+documenting/markup,844,`,"unaryneg ::= ""-"" `integer`"
+documenting/markup,849,.. productionlist:,.. productionlist::
+documenting/markup,849,`,"try1_stmt: ""try"" "":"" `suite`"
+documenting/markup,849,`,": (""except"" [`expression` ["","" `target`]] "":"" `suite`)+"
+documenting/markup,849,`,": [""else"" "":"" `suite`]"
+documenting/markup,849,`,": [""finally"" "":"" `suite`]"
+documenting/markup,849,`,"try2_stmt: ""try"" "":"" `suite`"
+documenting/markup,849,`,": ""finally"" "":"" `suite`"
+documenting/rest,33,`,``text``
+documenting/rest,47,:rolename,:rolename:`content`
+documenting/rest,47,`,:rolename:`content`
+documenting/rest,103,::,This is a normal text paragraph. The next paragraph is a code sample::
+documenting/rest,130,`,`Link text <http://target>`_
+documenting/rest,187,.. function:,.. function:: foo(x)
+documenting/rest,187,:bar,:bar: no
+documenting/rest,208,.. rubric:,.. rubric:: Footnotes
+faq/programming,762,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y,"
+faq/programming,762,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro,"
+faq/programming,762,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr("
+faq/programming,1047,::,for x in sequence[::-1]:
+faq/windows,229,:EOF,@setlocal enableextensions & python -x %~f0 %* & goto :EOF
+faq/windows,393,:REG,.py :REG_SZ: c:\<path to python>\python.exe -u %s %s
+library/bisect,32,:hi,all(val >= x for val in a[i:hi])
+library/bisect,42,:hi,all(val > x for val in a[i:hi])
+library/http.client,52,:port,host:port
+library/nntplib,272,:bytes,:bytes
+library/nntplib,272,:lines,:lines
+library/nntplib,272,:lines,"['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']"
+library/nntplib,272,:bytes,"['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']"
+library/pickle,567,:memory,"conn = sqlite3.connect("":memory:"")"
+library/profile,293,:lineno,"(sort by filename:lineno),"
+library/socket,261,::,"(10, 1, 6, '', ('2001:888:2000:d::a2', 80, 0, 0))]"
+library/stdtypes,1026,:end,s[start:end]
+library/stdtypes,1195,:end,s[start:end]
+library/urllib.request,64,:close,Connection:close
+library/urllib.request,901,:password,"""joe:password@python.org"""
+library/urllib.request,1064,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n<head>\n"
+library/xmlrpc.client,103,:pass,http://user:pass@host:port/path
+library/xmlrpc.client,103,:port,http://user:pass@host:port/path
+library/xmlrpc.client,103,:pass,user:pass
+license,717,`,* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+license,717,`,* THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+license,879,`,"``Software''), to deal in the Software without restriction, including"
+license,879,`,"THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,"
+reference/lexical_analysis,704,`,$       ?       `
+whatsnew/2.7,735,:Sunday,'2009:4:Sunday'
+whatsnew/2.7,862,::,"export PYTHONWARNINGS=all,error:::Cookie:0"
+whatsnew/2.7,862,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0"
+whatsnew/2.7,1619,::,>>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo')
+whatsnew/2.7,1619,::,"ParseResult(scheme='http', netloc='[1080::8:800:200C:417A]',"

Doc/tools/sphinxext/suspicious.py

@@ -41,9 +41,11 @@ Copyright 2009 Gabriel A. Genellina
 
 """
 
-import os, sys
-import csv
+import os
 import re
+import csv
+import sys
+
 from docutils import nodes
 from sphinx.builders import Builder
 
@@ -54,9 +56,10 @@ detect_all = re.compile(ur'''
     (?<!\.)\.\.[ \t]*\w+:  # .. foo: (but NOT ... else:)
     ''', re.UNICODE | re.VERBOSE).finditer
 
+
 class Rule:
     def __init__(self, docname, lineno, issue, line):
-        "A rule for ignoring issues"
+        """A rule for ignoring issues"""
         self.docname = docname # document to which this rule applies
         self.lineno = lineno   # line number in the original source;
                                # this rule matches only near that.
@@ -65,9 +68,15 @@ class Rule:
         self.line = line       # text of the container element (single line only)
 
 
+
+class dialect(csv.excel):
+    """Our dialect: uses only linefeed as newline."""
+    lineterminator = '\n'
+
+
 class CheckSuspiciousMarkupBuilder(Builder):
     """
-    Checks for possibly invalid markup that may leak into the output
+    Checks for possibly invalid markup that may leak into the output.
     """
     name = 'suspicious'
 
@@ -76,7 +85,8 @@ class CheckSuspiciousMarkupBuilder(Builder):
         self.log_file_name = os.path.join(self.outdir, 'suspicious.csv')
         open(self.log_file_name, 'w').close()
         # load database of previously ignored issues
-        self.load_rules(os.path.join(os.path.dirname(__file__), 'susp-ignored.csv'))
+        self.load_rules(os.path.join(os.path.dirname(__file__),
+                                     'susp-ignored.csv'))
 
     def get_outdated_docs(self):
         return self.env.found_docs
@@ -85,14 +95,11 @@ class CheckSuspiciousMarkupBuilder(Builder):
         return ''
 
     def prepare_writing(self, docnames):
-        ### PYTHON PROJECT SPECIFIC ###
-        for name in set(docnames):
-            if name.split('/', 1)[0] == 'documenting':
-                docnames.remove(name)
-        ### PYTHON PROJECT SPECIFIC ###
+        pass
 
     def write_doc(self, docname, doctree):
-        self.any_issue = False # set when any issue is encountered in this document
+        # set when any issue is encountered in this document
+        self.any_issue = False
         self.docname = docname
         visitor = SuspiciousVisitor(doctree, self)
         doctree.walk(visitor)
@@ -105,8 +112,7 @@ class CheckSuspiciousMarkupBuilder(Builder):
             self.report_issue(line, lineno, issue)
 
     def is_ignored(self, line, lineno, issue):
-        """Determine whether this issue should be ignored.
-        """
+        """Determine whether this issue should be ignored."""
         docname = self.docname
         for rule in self.rules:
             if rule.docname != docname: continue
@@ -139,12 +145,11 @@ class CheckSuspiciousMarkupBuilder(Builder):
 
     def write_log_entry(self, lineno, issue, text):
         f = open(self.log_file_name, 'ab')
-        writer = csv.writer(f)
+        writer = csv.writer(f, dialect)
         writer.writerow([self.docname.encode('utf-8'),
                          lineno,
                          issue.encode('utf-8'),
                          text.strip().encode('utf-8')])
-        del writer
         f.close()
 
     def load_rules(self, filename):
@@ -159,7 +164,8 @@ class CheckSuspiciousMarkupBuilder(Builder):
         except IOError: return
         for i, row in enumerate(csv.reader(f)):
             if len(row) != 4:
-                raise ValueError, "wrong format in %s, line %d: %s" % (filename, i+1, row)
+                raise ValueError(
+                    "wrong format in %s, line %d: %s" % (filename, i+1, row))
             docname, lineno, issue, text = row
             docname = docname.decode('utf-8')
             if lineno: lineno = int(lineno)
@@ -173,7 +179,7 @@ class CheckSuspiciousMarkupBuilder(Builder):
 
 
 def get_lineno(node):
-    "Obtain line number information for a node"
+    """Obtain line number information for a node."""
     lineno = None
     while lineno is None and node:
         node = node.parent
@@ -198,7 +204,8 @@ def extract_line(text, index):
     """
     p = text.rfind('\n', 0, index) + 1
     q = text.find('\n', index)
-    if q<0: q = len(text)
+    if q < 0:
+        q = len(text)
     return text[p:q]
 
 
@@ -217,7 +224,6 @@ class SuspiciousVisitor(nodes.GenericNodeVisitor):
             self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno)
             seen = set() # don't report the same issue more than only once per line
             for match in detect_all(text):
-                #import pdb; pdb.set_trace()
                 issue = match.group()
                 line = extract_line(text, match.start())
                 if (issue, line) not in seen: