Minor algorithmic tweaks, better factoring, improved docstrings and variable names

Tools/scripts/highlight.py

 #!/usr/bin/env python3
-'''Add syntax highlighting to Python source code
+'''Add syntax highlighting to Python source code'''
 
-Example command-line calls:
+__all__ = ['analyze_python', 'ansi_highlight', 'default_ansi',
+           'html_highlight', 'build_html_page', 'default_css', 'default_html']
 
-    # Show syntax highlighted code in the terminal window
-    $ ./highlight.py myfile.py
-
-    # Colorize myfile.py and display in a browser
-    $ ./highlight.py -b myfile.py
-
-    # Create an HTML section that can be embedded in an existing webpage
-    ./highlight.py -s myfile.py
-
-    # Create a complete HTML file
-    $ ./highlight.py -c myfile.py > myfile.html
-
-'''
-
-__all__ = ['colorize_html', 'build_page', 'default_css', 'default_html',
-           'colorize_ansi', 'default_ansi']
 __author__ = 'Raymond Hettinger'
 
 import keyword, tokenize, cgi, functools
@@ -31,13 +16,16 @@ def combine_range(lines, start, end):
     'Join content from a range of lines between start and end'
     (srow, scol), (erow, ecol) = start, end
     if srow == erow:
-        rows = [lines[srow-1][scol:ecol]]
-    else:
-        rows = [lines[srow-1][scol:]] + lines[srow: erow-1] + [lines[erow-1][:ecol]]
+        return lines[srow-1][scol:ecol], end
+    rows = [lines[srow-1][scol:]] + lines[srow: erow-1] + [lines[erow-1][:ecol]]
     return ''.join(rows), end
 
-def isolate_tokens(source):
-    'Generate chunks of source and identify chunks to be highlighted'
+def analyze_python(source):
+    '''Generate and classify chunks of Python for syntax highlighting.
+       Yields tuples in the form: (leadin_text, category, categorized_text).
+       The final tuple has empty strings for the category and categorized text.
+
+    '''
     lines = source.splitlines(True)
     lines.append('')
     readline = functools.partial(next, iter(lines), '')
@@ -65,36 +53,37 @@ def isolate_tokens(source):
                 kind = 'keyword'
             elif is_builtin(tok_str) and prev_tok_str != '.':
                 kind = 'builtin'
-        line_upto_token, written = combine_range(lines, written, (srow, scol))
-        line_thru_token, written = combine_range(lines, written, (erow, ecol))
-        yield kind, line_upto_token, line_thru_token
+        if kind:
+            line_upto_token, written = combine_range(lines, written, (srow, scol))
+            line_thru_token, written = combine_range(lines, written, (erow, ecol))
+            yield line_upto_token, kind, line_thru_token
+    line_upto_token, written = combine_range(lines, written, (erow, ecol))
+    yield line_upto_token, '', ''
 
 default_ansi = {
-    'comment': '\033[0;31m',
-    'string': '\033[0;32m',
-    'docstring': '\033[0;32m',
-    'keyword': '\033[0;33m',
-    'builtin': '\033[0;35m',
-    'definition': '\033[0;33m',
-    'defname': '\033[0;34m',
-    'operator': '\033[0;33m',
+    'comment': ('\033[0;31m', '\033[0m'),
+    'string': ('\033[0;32m', '\033[0m'),
+    'docstring': ('\033[0;32m', '\033[0m'),
+    'keyword': ('\033[0;33m', '\033[0m'),
+    'builtin': ('\033[0;35m', '\033[0m'),
+    'definition': ('\033[0;33m', '\033[0m'),
+    'defname': ('\033[0;34m', '\033[0m'),
+    'operator': ('\033[0;33m', '\033[0m'),
 }
 
-def colorize_ansi(source, colors=default_ansi):
-    'Add syntax highlighting to Python source code using ANSI escape sequences'
+def ansi_highlight(classified_text, colors=default_ansi):
+    'Add syntax highlighting to source code using ANSI escape sequences'
     # http://en.wikipedia.org/wiki/ANSI_escape_code
     result = []
-    for kind, line_upto_token, line_thru_token in isolate_tokens(source):
-        if kind:
-            result += [line_upto_token, colors[kind], line_thru_token, '\033[0m']
-        else:
-            result += [line_upto_token, line_thru_token]
+    for line_upto_token, kind, line_thru_token in classified_text:
+        opener, closer = colors.get(kind, ('', ''))
+        result += [line_upto_token, opener, line_thru_token, closer]
     return ''.join(result)
 
-def colorize_html(source):
-    'Convert Python source code to an HTML fragment with colorized markup'
-    result = ['<pre class="python">\n']
-    for kind, line_upto_token, line_thru_token in isolate_tokens(source):
+def html_highlight(classified_text,opener='<pre class="python">\n', closer='</pre>\n'):
+    'Convert classified text to an HTML fragment'
+    result = [opener]
+    for line_upto_token, kind, line_thru_token in classified_text:
         if kind:
             result += [cgi.escape(line_upto_token),
                        '<span class="%s">' % kind,
@@ -103,7 +92,7 @@ def colorize_html(source):
         else:
             result += [cgi.escape(line_upto_token),
                        cgi.escape(line_thru_token)]
-    result += ['</pre>\n']
+    result += [closer]
     return ''.join(result)
 
 default_css = {
@@ -134,21 +123,38 @@ default_html = '''\
 </html>
 '''
 
-def build_page(source, title='python', css=default_css, html=default_html):
-    'Create a complete HTML page with colorized Python source code'
+def build_html_page(classified_text, title='python',
+                    css=default_css, html=default_html):
+    'Create a complete HTML page with colorized source code'
     css_str = '\n'.join(['%s %s' % item for item in css.items()])
-    result = colorize_html(source)
+    result = html_highlight(classified_text)
     title = cgi.escape(title)
     return html.format(title=title, css=css_str, body=result)
 
 
 if __name__ == '__main__':
-    import sys, argparse, webbrowser, os
+    import sys, argparse, webbrowser, os, textwrap
 
     parser = argparse.ArgumentParser(
-            description = 'Add syntax highlighting to Python source')
+            description = 'Add syntax highlighting to Python source code',
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            epilog = textwrap.dedent('''
+                examples:
+
+                  # Show syntax highlighted code in the terminal window
+                  $ ./highlight.py myfile.py
+
+                  # Colorize myfile.py and display in a browser
+                  $ ./highlight.py -b myfile.py
+
+                  # Create an HTML section to embed in an existing webpage
+                  ./highlight.py -s myfile.py
+
+                  # Create a complete HTML file
+                  $ ./highlight.py -c myfile.py > myfile.html
+            '''))
     parser.add_argument('sourcefile', metavar = 'SOURCEFILE',
-            help = 'File containing Python sourcecode')
+            help = 'file containing Python sourcecode')
     parser.add_argument('-b', '--browser', action = 'store_true',
             help = 'launch a browser to show results')
     parser.add_argument('-c', '--complete', action = 'store_true',
@@ -164,13 +170,14 @@ if __name__ == '__main__':
     sourcefile = args.sourcefile
     with open(sourcefile) as f:
         source = f.read()
+    classified_text = analyze_python(source)
 
     if args.complete or args.browser:
-        encoded = build_page(source, title=sourcefile)
+        encoded = build_html_page(classified_text, title=sourcefile)
     elif args.section:
-        encoded = colorize_html(source)
+        encoded = html_highlight(classified_text)
     else:
-        encoded = colorize_ansi(source)
+        encoded = ansi_highlight(classified_text)
 
     if args.browser:
         htmlfile = os.path.splitext(os.path.basename(sourcefile))[0] + '.html'