Improve the CythonLexer so that it covers all code shown in the docs.

docs/extension_types.rst

@@ -283,6 +283,8 @@ when it is deleted.::
     del shop.cheese
     print shop.cheese
 
+.. sourcecode:: text
+
     # Test output
     We don't have: []
     We don't have: ['camembert']
@@ -342,6 +344,8 @@ extension types.::
     print "p2:"
     p2.describe()
 
+.. sourcecode:: text
+
     # Output
     p1:
     This parrot is resting.
@@ -437,7 +441,9 @@ built-in complex object.::
 
     1. In this example, :keyword:`ctypedef` class has been used. This is
        because, in the Python header files, the ``PyComplexObject`` struct is
-       declared with::
+       declared with:
+
+       .. sourcecode:: c
 
         ctypedef struct {
             ...

docs/external_C_code.rst

@@ -179,19 +179,19 @@ same applies equally to union and enum declarations.
 +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+
 | C code                  | Possibilities for corresponding Cython Code | Comments                                                              |
 +=========================+=============================================+=======================================================================+
-| ::                      | ::                                          | Cython will refer to the as ``struct Foo`` in the generated C code.   |
+| .. sourcecode:: c       | ::                                          | Cython will refer to the as ``struct Foo`` in the generated C code.   |
 |                         |                                             |                                                                       |
 |   struct Foo {          |   cdef struct Foo:                          |                                                                       |
 |     ...                 |     ...                                     |                                                                       |
 |   };                    |                                             |                                                                       |
 +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+
-| ::                      | ::                                          | Cython will refer to the type simply as ``Foo`` in                    |
+| .. sourcecode:: c       | ::                                          | Cython will refer to the type simply as ``Foo`` in                    |
 |                         |                                             | the generated C code.                                                 |
 |   typedef struct {      |   ctypedef struct Foo:                      |                                                                       |
 |     ...                 |     ...                                     |                                                                       |
 |   } Foo;                |                                             |                                                                       |
 +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+
-| ::                      | ::                                          | If the C header uses both a tag and a typedef with *different*        |
+| .. sourcecode:: c       | ::                                          | If the C header uses both a tag and a typedef with *different*        |
 |                         |                                             | names, you can use either form of declaration in Cython               |
 |   typedef struct foo {  |   cdef struct foo:                          | (although if you need to forward reference the type,                  |
 |     ...                 |     ...                                     | you'll have to use the first form).                                   |
@@ -202,7 +202,7 @@ same applies equally to union and enum declarations.
 |                         |   ctypedef struct Foo:                      |                                                                       |
 |                         |     ...                                     |                                                                       |
 +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+
-| ::                      | ::                                          | If the header uses the *same* name for the tag and typedef, you       |
+| .. sourcecode:: c       | ::                                          | If the header uses the *same* name for the tag and typedef, you       |
 |                         |                                             | won't be able to include a :keyword:`ctypedef` for it -- but then,    |
 |   typedef struct Foo {  |   cdef struct Foo:                          | it's not necessary.                                                   |
 |     ...                 |     ...                                     |                                                                       |
@@ -352,6 +352,8 @@ made available when you include :file:`modulename_api.h`.::
         if v.speed >= 88 and v.power >= 1.21:
             print "Time travel achieved"
 
+.. sourcecode:: c
+            
     # marty.c
     #include "delorean_api.h"
 

docs/source_files_and_compilation.rst

@@ -12,7 +12,9 @@ file named :file:`primes.pyx`.
 
 Once you have written your ``.pyx`` file, there are a couple of ways of turning it
 into an extension module. One way is to compile it manually with the Cython
-compiler, e.g.::
+compiler, e.g.:
+
+.. sourcecode:: text
 
     $ cython primes.pyx
 
@@ -45,7 +47,9 @@ would be::
 
 To understand the :file:`setup.py` more fully look at the official
 :mod:`distutils` documentation. To compile the extension for use in the
-current directory use::
+current directory use:
+
+.. sourcecode:: text
 
     $ python setup.py build_ext --inplace
 
@@ -135,7 +139,9 @@ If you would always like to import Cython files without building them
 specially, you can also the first line above to your :file:`sitecustomize.py`.
 That will install the hook every time you run Python. Then you can use
 Cython modules just with simple import statements. I like to test my
-Cython modules like this::
+Cython modules like this:
+
+.. sourcecode:: text
 
     $ python -c "import foo"
 

docs/tutorial.rst

@@ -50,7 +50,9 @@ information see :ref:`compilation`)::
         ext_modules = [Extension("helloworld", ["helloworld.pyx"])]
     ) 
 
-To use this to build your Cython file use the commandline options::
+To use this to build your Cython file use the commandline options:
+
+.. sourcecode:: text
 
     $ python setup.py build_ext --inplace
 
@@ -93,7 +95,9 @@ module name, doing this we have:
 
 .. literalinclude:: ../examples/tutorial/fib1/setup.py
 
-Build the extension with the same command used for the helloworld.pyx::
+Build the extension with the same command used for the helloworld.pyx:
+
+.. sourcecode:: text
 
     $ python setup.py build_ext --inplace
 

docs/wrapping_CPlusPlus.rst

@@ -39,7 +39,9 @@ An example C++ API
 
 Here is a tiny C++ API which we will use as an example throughout this
 document. Let's assume it will be in a header file called
-:file:`Rectangle.h`::
+:file:`Rectangle.h`:
+
+.. sourcecode:: c++
 
     class Rectangle {
     public:
@@ -116,7 +118,9 @@ Add class methods
 
 Now, let's add the class methods. You can circumvent Cython syntax
 limitations by declaring these as function pointers. Recall that in the C++
-class we have::
+class we have:
+
+.. sourcecode:: c++
 
   int getLength();
   int getHeight();
@@ -278,7 +282,9 @@ Overloading
 ^^^^^^^^^^^^
 
 To support function overloading simply add a different alias to each
-signature, so if you have e.g. ::
+signature, so if you have e.g.
+
+.. sourcecode:: c++
 
     int foo(int a);
     int foo(int a, int b);
@@ -294,7 +300,7 @@ Operators
 Some operators (e.g. +,-,...) can be accessed from Cython like this::
 
     ctypedef struct c_Rectangle "Rectangle":
-    c_Rectangle add "operator+"(c_Rectangle right)
+        c_Rectangle add "operator+"(c_Rectangle right)
 
 Declaring/Using References
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^

sphinxext/cython_highlighting.py

@@ -33,14 +33,18 @@ class CythonLexer(RegexLexer):
             (r'\\\n', Text),
             (r'\\', Text),
             (r'(in|is|and|or|not)\b', Operator.Word),
-            (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
-            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', bygroups(Keyword, Number.Integer, Operator, Name, Operator, Name, Punctuation)), 
+            (r'(<)([a-zA-Z0-9.?]+)(>)',
+             bygroups(Punctuation, Keyword.Type, Punctuation)),
+            (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
+            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
+             bygroups(Keyword, Number.Integer, Operator, Name, Operator,
+                      Name, Punctuation)),
             include('keywords'),
-            (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
-            (r'(cdef)(\s+)', bygroups(Keyword, Text), 'cfuncname'),
-            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
+            (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
+            (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
             (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
-            (r'(import)(\s+)', bygroups(Keyword, Text), 'import'),
+            (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
             include('builtins'),
             include('backtick'),
             ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
@@ -55,20 +59,22 @@ class CythonLexer(RegexLexer):
             include('numbers'),
         ],
         'keywords': [
-            (r'(assert|break|continue|del|elif|else|except|exec|'
-             r'finally|for|global|if|lambda|pass|print|raise|'
+            (r'(assert|break|by|continue|ctypedef|del|elif|else|except\??|exec|'
+             r'finally|for|gil|global|if|include|lambda|nogil|pass|print|raise|'
              r'return|try|while|yield|as|with)\b', Keyword),
+            (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
         ],
         'builtins': [
-            (r'(?<!\.)(__import__|abs|apply|basestring|bool|buffer|callable|'
-             r'chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|'
-             r'divmod|double|enumerate|eval|execfile|exit|file|filter|float|getattr|'
-             r'globals|hasattr|hash|hex|id|input|int|intern|isinstance|'
-             r'issubclass|iter|len|list|locals|long|map|max|min|object|oct|'
-             r'open|ord|pow|property|range|raw_input|reduce|reload|repr|'
-             r'round|setattr|slice|staticmethod|str|sum|super|tuple|type|'
-             r'unichr|unicode|unsigned|vars|xrange|zip)\b', Name.Builtin),
-            (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True'
+            (r'(?<!\.)(__import__|abs|all|any|apply|basestring|bin|bool|buffer|'
+             r'bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|'
+             r'complex|delattr|dict|dir|divmod|enumerate|eval|execfile|exit|'
+             r'file|filter|float|frozenset|getattr|globals|hasattr|hash|hex|id|'
+             r'input|int|intern|isinstance|issubclass|iter|len|list|locals|'
+             r'long|map|max|min|next|object|oct|open|ord|pow|property|range|'
+             r'raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|'
+             r'sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|'
+             r'vars|xrange|zip)\b', Name.Builtin),
+            (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
              r')\b', Name.Builtin.Pseudo),
             (r'(?<!\.)(ArithmeticError|AssertionError|AttributeError|'
              r'BaseException|DeprecationWarning|EOFError|EnvironmentError|'
@@ -101,8 +107,19 @@ class CythonLexer(RegexLexer):
         'funcname': [
             ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop')
         ],
-        'cfuncname': [
-            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop')
+        'cdef': [
+            (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
+            (r'(struct|enum|union|class)\b', Keyword),
+            (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(?=[(:#=]|$)',
+             bygroups(Name.Function, Text), '#pop'),
+            (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(,)',
+             bygroups(Name.Function, Text, Punctuation)),
+            (r'from\b', Keyword, '#pop'),
+            (r'as\b', Keyword),
+            (r':', Punctuation, '#pop'),
+            (r'(?=["\'])', Text, '#pop'),
+            (r'[a-zA-Z_][a-zA-Z0-9_]*', Keyword.Type),
+            (r'.', Text),
         ],
         'classname': [
             ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop')
@@ -114,8 +131,10 @@ class CythonLexer(RegexLexer):
             (r'', Text, '#pop') # all else: go back
         ],
         'fromimport': [
-            (r'(\s+)((c)?import)\b', bygroups(Text, Keyword), '#pop'),
+            (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
             (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace),
+            # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
+            (r'', Text, '#pop'),
         ],
         'stringescape': [
             (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'