Support showing the complete C code in the annotated html-file (GH-2858)

Cython/Build/Cythonize.py

@@ -168,9 +168,10 @@ def create_args_parser():
                       help='use Python 3 syntax mode by default')
     parser.add_argument('--3str', dest='language_level', action='store_const', const='3str',
                       help='use Python 3 syntax mode by default')
-    parser.add_argument('-a', '--annotate', dest='annotate', action='store_true', default=None,
-                      help='generate annotated HTML page for source files')
-
+    parser.add_argument('-a', '--annotate', nargs='?', const='default', type=str, choices={'default','fullc'},
+                      help='Produce a colorized HTML version of the source. '
+                           'Use --annotate=fullc to include entire '
+                           'generated C/C++-code.')
     parser.add_argument('-x', '--exclude', metavar='PATTERN', dest='excludes',
                       action='append', default=[],
                       help='exclude certain file patterns from the compilation')

Cython/Build/IpythonMagic.py

@@ -179,8 +179,11 @@ class CythonMagics(Magics):
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
-        '-a', '--annotate', action='store_true', default=False,
-        help="Produce a colorized HTML version of the source."
+        '-a', '--annotate', nargs='?', const="default", type=str,
+        choices={"default","fullc"},
+        help="Produce a colorized HTML version of the source. "
+             "Use --annotate=fullc to include entire "
+             "generated C/C++-code."
     )
     @magic_arguments.argument(
         '-+', '--cplus', action='store_true', default=False,

Cython/Build/Tests/TestCythonizeArgsParser.py

 from Cython.Build.Cythonize import create_args_parser, parse_args_raw, parse_args
 from unittest import TestCase
 
+import argparse
 
 class TestCythonizeArgsParser(TestCase):
 
@@ -233,13 +234,25 @@ class TestCythonizeArgsParser(TestCase):
         options, args =  self.parse_args(['-a'])
         self.assertFalse(args)
         self.assertTrue(self.are_default(options, ['annotate']))
-        self.assertEqual(options.annotate, True)
+        self.assertEqual(options.annotate, 'default')
 
     def test_annotate_long(self):
         options, args =  self.parse_args(['--annotate'])
         self.assertFalse(args)
         self.assertTrue(self.are_default(options, ['annotate']))
-        self.assertEqual(options.annotate, True)
+        self.assertEqual(options.annotate, 'default')
+
+    def test_annotate_fullc(self):
+        options, args =  self.parse_args(['--annotate=fullc'])
+        self.assertFalse(args)
+        self.assertTrue(self.are_default(options, ['annotate']))
+        self.assertEqual(options.annotate, 'fullc')
+
+    def test_annotate_fullc(self):
+        options, args =  self.parse_args(['-a=default'])
+        self.assertFalse(args)
+        self.assertTrue(self.are_default(options, ['annotate']))
+        self.assertEqual(options.annotate, 'default')
 
     def test_exclude_short(self):
         options, args =  self.parse_args(['-x', '*.pyx'])
@@ -360,7 +373,7 @@ class TestCythonizeArgsParser(TestCase):
         options, args =  self.parse_args(['-i', 'file.pyx', '-a'])
         self.assertEqual(args, ['file.pyx'])
         self.assertEqual(options.build_inplace, True)
-        self.assertEqual(options.annotate, True)
+        self.assertEqual(options.annotate, 'default')
         self.assertTrue(self.are_default(options, ['build_inplace', 'annotate']))
 
     def test_option_trailing(self):

Cython/Build/Tests/TestIpythonMagic.py

@@ -10,6 +10,7 @@ import sys
 from contextlib import contextmanager
 from Cython.Build import IpythonMagic
 from Cython.TestUtils import CythonTest
+from Cython.Compiler.Annotate import AnnotationCCodeWriter
 
 try:
     import IPython.testing.globalipapp
@@ -210,3 +211,29 @@ x = sin(0.0)
             ip.ex('g = f(10)')
         self.assertEqual(ip.user_ns['g'], 20.0)
         self.assertEqual([normal_log.INFO], normal_log.thresholds)
+
+    def test_cython_no_annotate(self):
+        ip = self._ip
+        html = ip.run_cell_magic('cython', '', code)
+        self.assertTrue(html is None)
+
+    def test_cython_annotate(self):
+        ip = self._ip
+        html = ip.run_cell_magic('cython', '--annotate', code)
+        # somewhat brittle way to differentiate between annotated htmls
+        # with/without complete source code:
+        self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data)
+
+    def test_cython_annotate_default(self):
+        ip = self._ip
+        html = ip.run_cell_magic('cython', '--a=default', code)
+        # somewhat brittle way to differentiate between annotated htmls
+        # with/without complete source code:
+        self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data)
+
+    def test_cython_annotate_complete_c_code(self):
+        ip = self._ip
+        html = ip.run_cell_magic('cython', '--a=fullc', code)
+        # somewhat brittle way to differentiate between annotated htmls
+        # with/without complete source code:
+        self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html.data)

Cython/Compiler/Annotate.py

@@ -23,8 +23,12 @@ from .. import Utils
 
 class AnnotationCCodeWriter(CCodeWriter):
  
-    def __init__(self, create_from=None, buffer=None, copy_formatting=True):
+    # also used as marker for detection of complete code emission in tests
+    COMPLETE_CODE_TITLE = "Complete cythonized code"
+
+    def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False):
         CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting)
+        self.show_entire_c_code = show_entire_c_code
         if create_from is None:
             self.annotation_buffer = StringIO()
             self.last_annotated_pos = None
@@ -198,17 +202,24 @@ class AnnotationCCodeWriter(CCodeWriter):
             for line in coverage_data.iterfind('lines/line')
         )
 
-    def _htmlify_code(self, code):
+    def _htmlify_code(self, code, language):
         try:
             from pygments import highlight
-            from pygments.lexers import CythonLexer
+            from pygments.lexers import CythonLexer, CppLexer
             from pygments.formatters import HtmlFormatter
         except ImportError:
             # no Pygments, just escape the code
             return html_escape(code)
 
+        if language == "cython":
+            lexer = CythonLexer(stripnl=False, stripall=False)
+        elif language == "c/cpp":
+            lexer = CppLexer(stripnl=False, stripall=False)
+        else:
+            # unknown language, use fallback
+            return html_escape(code)
         html_code = highlight(
-            code, CythonLexer(stripnl=False, stripall=False),
+            code, lexer,
             HtmlFormatter(nowrap=True))
         return html_code
 
@@ -228,7 +239,7 @@ class AnnotationCCodeWriter(CCodeWriter):
             return u"<span class='%s'>%s</span>" % (
                 group_name, match.group(group_name))
 
-        lines = self._htmlify_code(cython_code).splitlines()
+        lines = self._htmlify_code(cython_code, "cython").splitlines()
         lineno_width = len(str(len(lines)))
         if not covered_lines:
             covered_lines = None
@@ -279,6 +290,19 @@ class AnnotationCCodeWriter(CCodeWriter):
                 outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format(
                     score=score, covered=covered, code=c_code))
         outlist.append(u"</div>")
+
+        # now the whole c-code if needed:
+        if self.show_entire_c_code:
+            outlist.append(u'<p><div class="cython">')
+            onclick_title = u"<pre class='cython line'{onclick}>+ {title}</pre>\n";
+            outlist.append(onclick_title.format(
+                              onclick=self._onclick_attr,
+                              title=AnnotationCCodeWriter.COMPLETE_CODE_TITLE,
+                           ))
+            complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp")
+            outlist.append(u"<pre class='cython code'>{code}</pre>".format(code=complete_code_as_html))
+            outlist.append(u"</div></p>")
+
         return outlist
 
 

Cython/Compiler/CmdLine.py

@@ -34,6 +34,7 @@ Options:
 
   -D, --no-docstrings            Strip docstrings from the compiled module.
   -a, --annotate                 Produce a colorized HTML version of the source.
+                                 Use --annotate=fullc to include entire generated C/C++-code.
   --annotate-coverage <cov.xml>  Annotate and include coverage information from cov.xml.
   --line-directives              Produce #line directives pointing to the .pyx source
   --cplus                        Output a C++ rather than C file.
@@ -128,7 +129,7 @@ def parse_command_line(args):
             elif option in ("-D", "--no-docstrings"):
                 Options.docstrings = False
             elif option in ("-a", "--annotate"):
-                Options.annotate = True
+                Options.annotate = pop_value('default')
             elif option == "--annotate-coverage":
                 Options.annotate = True
                 Options.annotate_coverage_xml = pop_value()

Cython/Compiler/ModuleNode.py

@@ -341,7 +341,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         modules = self.referenced_modules
 
         if Options.annotate or options.annotate:
-            rootwriter = Annotate.AnnotationCCodeWriter()
+            show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc"
+            rootwriter = Annotate.AnnotationCCodeWriter(show_entire_c_code=show_entire_c_code)
         else:
             rootwriter = Code.CCodeWriter()
 

Cython/Compiler/Tests/TestCmdLine.py

@@ -98,6 +98,33 @@ class CmdLineParserTest(TestCase):
         self.assertTrue(options.gdb_debug)
         self.assertEqual(options.output_dir, '/gdb/outdir')
 
+    def test_no_annotate(self):
+        options, sources = parse_command_line([
+            '--embed=huhu', 'source.pyx'
+        ])
+        self.assertFalse(Options.annotate)
+
+    def test_annotate_simple(self):
+        options, sources = parse_command_line([
+            '-a',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'default')
+
+    def test_annotate_default(self):
+        options, sources = parse_command_line([
+            '--annotate=default',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'default')
+
+    def test_annotate_fullc(self):
+        options, sources = parse_command_line([
+            '--annotate=fullc',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'fullc')
+
     def test_errors(self):
         def error(*args):
             old_stderr = sys.stderr

docs/src/userguide/source_files_and_compilation.rst

@@ -621,7 +621,7 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook.
 
 ============================================  =======================================================================================================================================
 
--a, --annotate                                Produce a colorized HTML version of the source.
+-a, --annotate                                Produce a colorized HTML version of the source. Use ``--annotate=fullc`` to include the complete generated C/C++-code as well.
 
 -+, --cplus                                   Output a C++ rather than C file.
 
@@ -648,6 +648,7 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook.
 --pgo                                         Enable profile guided optimisation in the C compiler. Compiles the cell twice and executes it in between to generate a runtime profile.
 
 --verbose                                     Print debug information like generated .c/.cpp file location and exact gcc/g++ command invoked.
+
 ============================================  =======================================================================================================================================
 
 

tests/build/cythonize_with_annotate.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import not_annotated; not_annotated.check()"
+PYTHON -c "import default_annotated; default_annotated.check()"
+PYTHON -c "import fullc_annotated; fullc_annotated.check()"
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize(["not_annotated.pyx"], language_level=3) +
+                cythonize(["default_annotated.pyx"], annotate=True, language_level=3) + 
+                cythonize(["fullc_annotated.pyx"], annotate='fullc', language_level=3)
+)
+######## not_annotated.pyx ########
+# check that html-file doesn't exist:
+def check():
+    import os.path as os_path
+    assert not os_path.isfile(__name__+'.html')
+
+
+
+######## default_annotated.pyx ########
+# load html-site and check that the marker isn't there:
+def check():
+    from codecs import open
+    with open(__name__+'.html', 'r', 'utf8') as html_file:
+        html = html_file.read()
+
+    from Cython.Compiler.Annotate import AnnotationCCodeWriter
+    assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code
+
+
+
+######## fullc_annotated.pyx ########
+# load html-site and check that the marker is there:
+def check():
+    from codecs import open
+    with open(__name__+'.html', 'r', 'utf8') as html_file:
+        html = html_file.read()
+
+    from Cython.Compiler.Annotate import AnnotationCCodeWriter
+    assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html)
+

tests/build/cythonize_with_annotate_via_Options.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import fullc_annotated; fullc_annotated.check()"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from Cython.Compiler import Options
+
+Options.annotate = 'fullc'
+
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize(["fullc_annotated.pyx"], language_level=3)
+)
+
+######## fullc_annotated.pyx ########
+# load html-site and check that the marker is there:
+
+def check():
+    from codecs import open
+    with open(__name__+'.html', 'r', 'utf8') as html_file:
+        html = html_file.read()
+
+    from Cython.Compiler.Annotate import AnnotationCCodeWriter
+    assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html)
+

tests/run/annotate_html.pyx

@@ -11,6 +11,8 @@
 
 >>> import re
 >>> assert re.search('<pre .*def.* .*mixed_test.*</pre>', html)
+>>> from Cython.Compiler.Annotate import AnnotationCCodeWriter
+>>> assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code
 """