Added numpy tutorial, and Sphinx extension for Cython and IPython highlighting

README

 This is a collection of all the assorted pyrex/cython documentation assembled into
 a unified Python style Sphinx project. It is hoped that this will become the default
 documentation for the Cython tool set.
+
+Curently the files are hosted at:
+http://hg.cython.org/cython-docs

conf.py

@@ -14,14 +14,22 @@
 import sys
 
 # If your extensions are in another directory, add it here.
-#sys.path.append('some/directory')
+sys.path.append('sphinxext')
+
+# Import support for ipython console session syntax highlighting (lives
+# in the sphinxext directory defined above)
+import ipython_console_highlighting
+
+# support cython highlighting
+import cython_highlighting
+
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = []
+extensions = ['ipython_console_highlighting', 'cython_highlighting']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['.templates']

index.rst

@@ -12,6 +12,7 @@ Contents:
 
    docs/overview
    docs/tutorial
+   docs/numpy_tutorial
    docs/language_basics
    docs/extension_types
    docs/sharing_declarations

sphinxext/cython_highlighting.py

+import re
+
+from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, \
+     LexerContext, include, combined, do_insertions, bygroups, using
+from pygments.token import Error, Text, \
+     Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation
+from pygments.util import get_bool_opt, get_list_opt, shebang_matches
+from pygments import unistring as uni
+
+from sphinx import highlighting
+
+
+line_re  = re.compile('.*?\n')
+
+class CythonLexer(RegexLexer):
+    """
+    For `Cython <http://cython.org>`_ source code.
+    """
+
+    name = 'Cython'
+    aliases = ['cython', 'pyx']
+    filenames = ['*.pyx', '*.pxd', '*.pxi']
+    mimetypes = ['text/x-cython', 'application/x-cython']
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)),
+            (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)),
+            (r'[^\S\n]+', Text),
+            (r'#.*$', Comment),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\\\n', Text),
+            (r'\\', Text),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
+            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', bygroups(Keyword, Number.Integer, Operator, Name, Operator, Name, Punctuation)), 
+            include('keywords'),
+            (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
+            (r'(cdef)(\s+)', bygroups(Keyword, Text), 'cfuncname'),
+            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
+            (r'(import)(\s+)', bygroups(Keyword, Text), 'import'),
+            include('builtins'),
+            include('backtick'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
+            ('[uU]?"""', String, combined('stringescape', 'tdqs')),
+            ("[uU]?'''", String, combined('stringescape', 'tsqs')),
+            ('[uU]?"', String, combined('stringescape', 'dqs')),
+            ("[uU]?'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (r'(assert|break|continue|del|elif|else|except|exec|'
+             r'finally|for|global|if|lambda|pass|print|raise|'
+             r'return|try|while|yield|as|with)\b', Keyword),
+        ],
+        'builtins': [
+            (r'(?<!\.)(__import__|abs|apply|basestring|bool|buffer|callable|'
+             r'chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|'
+             r'divmod|double|enumerate|eval|execfile|exit|file|filter|float|getattr|'
+             r'globals|hasattr|hash|hex|id|input|int|intern|isinstance|'
+             r'issubclass|iter|len|list|locals|long|map|max|min|object|oct|'
+             r'open|ord|pow|property|range|raw_input|reduce|reload|repr|'
+             r'round|setattr|slice|staticmethod|str|sum|super|tuple|type|'
+             r'unichr|unicode|unsigned|vars|xrange|zip)\b', Name.Builtin),
+            (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True'
+             r')\b', Name.Builtin.Pseudo),
+            (r'(?<!\.)(ArithmeticError|AssertionError|AttributeError|'
+             r'BaseException|DeprecationWarning|EOFError|EnvironmentError|'
+             r'Exception|FloatingPointError|FutureWarning|GeneratorExit|IOError|'
+             r'ImportError|ImportWarning|IndentationError|IndexError|KeyError|'
+             r'KeyboardInterrupt|LookupError|MemoryError|NameError|'
+             r'NotImplemented|NotImplementedError|OSError|OverflowError|'
+             r'OverflowWarning|PendingDeprecationWarning|ReferenceError|'
+             r'RuntimeError|RuntimeWarning|StandardError|StopIteration|'
+             r'SyntaxError|SyntaxWarning|SystemError|SystemExit|TabError|'
+             r'TypeError|UnboundLocalError|UnicodeDecodeError|'
+             r'UnicodeEncodeError|UnicodeError|UnicodeTranslateError|'
+             r'UnicodeWarning|UserWarning|ValueError|Warning|ZeroDivisionError'
+             r')\b', Name.Exception),
+        ],
+        'numbers': [
+            (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'0\d+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+L', Number.Integer.Long),
+            (r'\d+', Number.Integer)
+        ],
+        'backtick': [
+            ('`.*?`', String.Backtick),
+        ],
+        'name': [
+            (r'@[a-zA-Z0-9_]+', Name.Decorator),
+            ('[a-zA-Z_][a-zA-Z0-9_]*', Name),
+        ],
+        'funcname': [
+            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop')
+        ],
+        'cfuncname': [
+            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop')
+        ],
+        'classname': [
+            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop')
+        ],
+        'import': [
+            (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
+            (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace),
+            (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
+            (r'', Text, '#pop') # all else: go back
+        ],
+        'fromimport': [
+            (r'(\s+)((c)?import)\b', bygroups(Text, Keyword), '#pop'),
+            (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace),
+        ],
+        'stringescape': [
+            (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
+             r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+        ],
+        'strings': [
+            (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+             '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol),
+            (r'[^\\\'"%\n]+', String),
+            # quotes, percents and backslashes must be parsed one at a time
+            (r'[\'"\\]', String),
+            # unhandled string formatting sign
+            (r'%', String)
+            # newlines are an error (use "nl" state)
+        ],
+        'nl': [
+            (r'\n', String)
+        ],
+        'dqs': [
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings
+            include('strings')
+        ],
+        'sqs': [
+            (r"'", String, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings
+            include('strings')
+        ],
+        'tdqs': [
+            (r'"""', String, '#pop'),
+            include('strings'),
+            include('nl')
+        ],
+        'tsqs': [
+            (r"'''", String, '#pop'),
+            include('strings'),
+            include('nl')
+        ],
+    }
+
+    def analyse_text(text):
+        return shebang_matches(text, r'pythonw?(2\.\d)?')
+
+highlighting.lexers['cython'] = CythonLexer()

sphinxext/ipython_console_highlighting.py

+from pygments.lexer import Lexer, do_insertions
+from pygments.lexers.agile import PythonConsoleLexer, PythonLexer, \
+    PythonTracebackLexer
+from pygments.token import Comment, Generic
+from sphinx import highlighting
+import re
+
+line_re = re.compile('.*?\n')
+
+class IPythonConsoleLexer(Lexer):
+    """
+    For IPython console output or doctests, such as:
+
+    Tracebacks are not currently supported.
+
+    .. sourcecode:: ipython
+
+      In [1]: a = 'foo'
+
+      In [2]: a
+      Out[2]: 'foo'
+
+      In [3]: print a
+      foo
+
+      In [4]: 1 / 0
+    """
+    name = 'IPython console session'
+    aliases = ['ipython']
+    mimetypes = ['text/x-ipython-console']
+    input_prompt = re.compile("(In \[[0-9]+\]: )|(   \.\.\.+:)")
+    output_prompt = re.compile("(Out\[[0-9]+\]: )|(   \.\.\.+:)")
+    continue_prompt = re.compile("   \.\.\.+:")
+    tb_start = re.compile("\-+")
+
+    def get_tokens_unprocessed(self, text):
+        pylexer = PythonLexer(**self.options)
+        tblexer = PythonTracebackLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            input_prompt = self.input_prompt.match(line)
+            continue_prompt = self.continue_prompt.match(line.rstrip())
+            output_prompt = self.output_prompt.match(line)
+            if line.startswith("#"):
+                insertions.append((len(curcode),
+                                   [(0, Comment, line)]))
+            elif input_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, input_prompt.group())]))
+                curcode += line[input_prompt.end():]
+            elif continue_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, continue_prompt.group())]))
+                curcode += line[continue_prompt.end():]
+            elif output_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Output, output_prompt.group())]))
+                curcode += line[output_prompt.end():]
+            else:
+                if curcode:
+                    for item in do_insertions(insertions,
+                                              pylexer.get_tokens_unprocessed(curcode)):
+                        yield item
+                        curcode = ''
+                        insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            for item in do_insertions(insertions,
+                                      pylexer.get_tokens_unprocessed(curcode)):
+                yield item
+
+highlighting.lexers['ipython'] = IPythonConsoleLexer()