Commit 31bb39f0 authored by scoder's avatar scoder

Merge pull request #286 from mgaitan/ipython

IPython magics
parents 34a3cff9 4a64dcb8
# -*- coding: utf-8 -*-
"""
=====================
Cython related magics
=====================
Magic command interface for interactive work with Cython
.. note::
The ``Cython`` package needs to be installed separately. It
can be obtained using ``easy_install`` or ``pip``.
Usage
=====
To enable the magics below, execute ``%load_ext cythonmagic``.
``%%cython``
{CYTHON_DOC}
``%%cython_inline``
{CYTHON_INLINE_DOC}
``%%cython_pyximport``
{CYTHON_PYXIMPORT_DOC}
Author:
* Brian Granger
Code moved from IPython and adapted by:
* Martín Gaitán
Parts of this code were taken from Cython.inline.
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2010-2011, IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
from __future__ import print_function
import imp
import io
import os
import re
import sys
import time
try:
reload
except NameError: # Python 3
from imp import reload
try:
import hashlib
except ImportError:
import md5 as hashlib
from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext
from IPython.core import display
from IPython.core import magic_arguments
from IPython.core.magic import Magics, magics_class, cell_magic
from IPython.utils import py3compat
from IPython.utils.path import get_ipython_cache_dir
from IPython.utils.text import dedent
import Cython
from Cython.Compiler.Errors import CompileError
from Cython.Build.Dependencies import cythonize
@magics_class
class CythonMagics(Magics):
def __init__(self, shell):
super(CythonMagics,self).__init__(shell)
self._reloads = {}
self._code_cache = {}
def _import_all(self, module):
for k,v in module.__dict__.items():
if not k.startswith('__'):
self.shell.push({k:v})
@cell_magic
def cython_inline(self, line, cell):
"""Compile and run a Cython code cell using Cython.inline.
This magic simply passes the body of the cell to Cython.inline
and returns the result. If the variables `a` and `b` are defined
in the user's namespace, here is a simple example that returns
their sum::
%%cython_inline
return a+b
For most purposes, we recommend the usage of the `%%cython` magic.
"""
locs = self.shell.user_global_ns
globs = self.shell.user_ns
return Cython.inline(cell, locals=locs, globals=globs)
@cell_magic
def cython_pyximport(self, line, cell):
"""Compile and import a Cython code cell using pyximport.
The contents of the cell are written to a `.pyx` file in the current
working directory, which is then imported using `pyximport`. This
magic requires a module name to be passed::
%%cython_pyximport modulename
def f(x):
return 2.0*x
The compiled module is then imported and all of its symbols are
injected into the user's namespace. For most purposes, we recommend
the usage of the `%%cython` magic.
"""
module_name = line.strip()
if not module_name:
raise ValueError('module name must be given')
fname = module_name + '.pyx'
with io.open(fname, 'w', encoding='utf-8') as f:
f.write(cell)
if 'pyximport' not in sys.modules:
import pyximport
pyximport.install(reload_support=True)
if module_name in self._reloads:
module = self._reloads[module_name]
reload(module)
else:
__import__(module_name)
module = sys.modules[module_name]
self._reloads[module_name] = module
self._import_all(module)
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'-c', '--compile-args', action='append', default=[],
help="Extra flags to pass to compiler via the `extra_compile_args` "
"Extension flag (can be specified multiple times)."
)
@magic_arguments.argument(
'--link-args', action='append', default=[],
help="Extra flags to pass to linker via the `extra_link_args` "
"Extension flag (can be specified multiple times)."
)
@magic_arguments.argument(
'-l', '--lib', action='append', default=[],
help="Add a library to link the extension against (can be specified "
"multiple times)."
)
@magic_arguments.argument(
'-n', '--name',
help="Specify a name for the Cython module."
)
@magic_arguments.argument(
'-L', dest='library_dirs', metavar='dir', action='append', default=[],
help="Add a path to the list of libary directories (can be specified "
"multiple times)."
)
@magic_arguments.argument(
'-I', '--include', action='append', default=[],
help="Add a path to the list of include directories (can be specified "
"multiple times)."
)
@magic_arguments.argument(
'-+', '--cplus', action='store_true', default=False,
help="Output a C++ rather than C file."
)
@magic_arguments.argument(
'-f', '--force', action='store_true', default=False,
help="Force the compilation of a new module, even if the source has been "
"previously compiled."
)
@magic_arguments.argument(
'-a', '--annotate', action='store_true', default=False,
help="Produce a colorized HTML version of the source."
)
@cell_magic
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the
code. This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
To compile OpenMP codes, pass the required `--compile-args`
and `--link-args`. For example with gcc::
%%cython --compile-args=-fopenmp --link-args=-fopenmp
...
"""
args = magic_arguments.parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell+'\n'
lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
quiet = True
key = code, sys.version_info, sys.executable, Cython.__version__
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
if args.name:
module_name = py3compat.unicode_to_str(args.name)
else:
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name + self.so_ext)
have_module = os.path.isfile(module_path)
need_cythonize = not have_module
if args.annotate:
html_file = os.path.join(lib_dir, module_name + '.html')
if not os.path.isfile(html_file):
need_cythonize = True
if need_cythonize:
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
library_dirs = args.library_dirs,
extra_compile_args = args.compile_args,
extra_link_args = args.link_args,
libraries = args.lib,
language = 'c++' if args.cplus else 'c',
)
build_extension = self._get_build_extension()
try:
opts = dict(
quiet=quiet,
annotate = args.annotate,
force = True,
)
build_extension.extensions = cythonize([extension], **opts)
except CompileError:
return
if not have_module:
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
if args.annotate:
try:
with io.open(html_file, encoding='utf-8') as f:
annotated_html = f.read()
except IOError as e:
# File could not be opened. Most likely the user has a version
# of Cython before 0.15.1 (when `cythonize` learned the
# `force` keyword argument) and has already compiled this
# exact source without annotation.
print('Cython completed successfully but the annotated '
'source could not be read.', file=sys.stderr)
print(e, file=sys.stderr)
else:
return display.HTML(self.clean_annotated_html(annotated_html))
@property
def so_ext(self):
"""The extension suffix for compiled modules."""
try:
return self._so_ext
except AttributeError:
self._so_ext = self._get_build_extension().get_ext_filename('')
return self._so_ext
def _clear_distutils_mkpath_cache(self):
"""clear distutils mkpath cache
prevents distutils from skipping re-creation of dirs that have been removed
"""
try:
from distutils.dir_util import _path_created
except ImportError:
pass
else:
_path_created.clear()
def _get_build_extension(self):
self._clear_distutils_mkpath_cache()
dist = Distribution()
config_files = dist.find_config_files()
try:
config_files.remove('setup.cfg')
except ValueError:
pass
dist.parse_config_files(config_files)
build_extension = build_ext(dist)
build_extension.finalize_options()
return build_extension
@staticmethod
def clean_annotated_html(html):
"""Clean up the annotated HTML source.
Strips the link to the generated C or C++ file, which we do not
present to the user.
"""
r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>')
html = '\n'.join(l for l in html.splitlines() if not r.match(l))
return html
__doc__ = __doc__.format(
# rST doesn't see the -+ flag as part of an option list, so we
# hide it from the module-level docstring.
CYTHON_DOC = dedent(CythonMagics.cython.__doc__\
.replace('-+, --cplus','--cplus ')),
CYTHON_INLINE_DOC = dedent(CythonMagics.cython_inline.__doc__),
CYTHON_PYXIMPORT_DOC = dedent(CythonMagics.cython_pyximport.__doc__),
)
# -*- coding: utf-8 -*-
# tag: ipython
"""Tests for the Cython magics extension."""
import os
try:
from IPython.testing.globalipapp import get_ipython
from IPython.testing import decorators as dec
from IPython.utils import py3compat
except:
__test__ = False
from Cython.TestUtils import CythonTest
ip = get_ipython()
code = py3compat.str_to_unicode("""def f(x):
return 2*x
""")
class TestIPythonMagic(CythonTest):
def setUp(self):
CythonTest.setUp(self)
ip.extension_manager.load_extension('cython')
def test_cython_inline(self):
ip.ex('a=10; b=20')
result = ip.run_cell_magic('cython_inline', '', 'return a+b')
self.assertEqual(result, 30)
@dec.skip_win32
def test_cython_pyximport(self):
module_name = '_test_cython_pyximport'
ip.run_cell_magic('cython_pyximport', module_name, code)
ip.ex('g = f(10)')
self.assertEqual(ip.user_ns['g'], 20.0)
ip.run_cell_magic('cython_pyximport', module_name, code)
ip.ex('h = f(-10)')
self.assertEqual(ip.user_ns['h'], -20.0)
try:
os.remove(module_name + '.pyx')
except OSError:
pass
def test_cython(self):
ip.run_cell_magic('cython', '', code)
ip.ex('g = f(10)')
self.assertEqual(ip.user_ns['g'], 20.0)
def test_cython_name(self):
# The Cython module named 'mymodule' defines the function f.
ip.run_cell_magic('cython', '--name=mymodule', code)
# This module can now be imported in the interactive namespace.
ip.ex('import mymodule; g = mymodule.f(10)')
self.assertEqual(ip.user_ns['g'], 20.0)
@dec.skip_win32
def test_extlibs(self):
code = py3compat.str_to_unicode("""
from libc.math cimport sin
x = sin(0.0)
""")
ip.user_ns['x'] = 1
ip.run_cell_magic('cython', '-l m', code)
self.assertEqual(ip.user_ns['x'], 0)
=============================
The IPython licensing terms
=============================
IPython is licensed under the terms of the Modified BSD License (also known as
New or Revised or 3-Clause BSD), as follows:
- Copyright (c) 2008-2014, IPython Development Team
- Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
- Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
- Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
Neither the name of the IPython Development Team nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
About the IPython Development Team
----------------------------------
Fernando Perez began IPython in 2001 based on code from Janko Hauser
<jhauser@zscout.de> and Nathaniel Gray <n8gray@caltech.edu>. Fernando is still
the project lead.
The IPython Development Team is the set of all contributors to the IPython
project. This includes all of the IPython subprojects. A full list with
details is kept in the documentation directory, in the file
``about/credits.txt``.
The core team that coordinates development on GitHub can be found here:
https://github.com/ipython/.
Our Copyright Policy
--------------------
IPython uses a shared copyright model. Each contributor maintains copyright
over their contributions to IPython. But, it is important to note that these
contributions are typically only changes to the repositories. Thus, the IPython
source code, in its entirety is not the copyright of any single person or
institution. Instead, it is the collective copyright of the entire IPython
Development Team. If individual contributors want to maintain a record of what
changes/contributions they have specific copyright on, they should indicate
their copyright in the commit message of the change, when they commit the
change to one of the IPython repositories.
With this in mind, the following banner should be used in any source code file
to indicate the copyright and license terms:
::
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
......@@ -2,3 +2,9 @@ from Cython.Shadow import __version__
# Void cython.* directives (for case insensitive operating systems).
from Cython.Shadow import *
def load_ipython_extension(ip):
"""Load the extension in IPython."""
from Cython.Build.IpythonMagic import CythonMagics
ip.register_magics(CythonMagics)
......@@ -21,3 +21,4 @@ else:
from Cython.Shadow import *
## and bring in the __version__
from Cython import __version__
from Cython import load_ipython_extension
{
"metadata": {
"name": "Cython Magics",
"signature": "sha256:c357b93e9480d6347c6677862bf43750745cef4b30129c5bc53cb879a19d4074"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Cython Magic Functions"
]
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Loading the extension"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Cython has an IPython extension that contains a number of magic functions for working with Cython code. This extension can be loaded using the `%load_ext` magic as follows:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%load_ext cython"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"The %cython_inline magic"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The `%%cython_inline` magic uses `Cython.inline` to compile a Cython expression. This allows you to enter and run a function body with Cython code. Use a bare `return` statement to return values. "
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"a = 10\n",
"b = 20"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%cython_inline\n",
"return a+b"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 3,
"text": [
"30"
]
}
],
"prompt_number": 3
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"The %cython_pyximport magic"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The `%%cython_pyximport` magic allows you to enter arbitrary Cython code into a cell. That Cython code is written as a `.pyx` file in the current working directory and then imported using `pyximport`. You have the specify the name of the module that the Code will appear in. All symbols from the module are imported automatically by the magic function."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%cython_pyximport foo\n",
"def f(x):\n",
" return 4.0*x"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"f(10)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 5,
"text": [
"40.0"
]
}
],
"prompt_number": 5
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"The %cython magic"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Probably the most important magic is the `%cython` magic. This is similar to the `%%cython_pyximport` magic, but doesn't require you to specify a module name. Instead, the `%%cython` magic uses manages everything using temporary files in the `~/.cython/magic` directory. All of the symbols in the Cython module are imported automatically by the magic.\n",
"\n",
"Here is a simple example of a Black-Scholes options pricing algorithm written in Cython. Please note that this example might not compile on non-POSIX systems (e.g., Windows) because of a missing `erf` symbol."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%cython\n",
"cimport cython\n",
"from libc.math cimport exp, sqrt, pow, log, erf\n",
"\n",
"@cython.cdivision(True)\n",
"cdef double std_norm_cdf_cy(double x) nogil:\n",
" return 0.5*(1+erf(x/sqrt(2.0)))\n",
"\n",
"@cython.cdivision(True)\n",
"def black_scholes_cy(double s, double k, double t, double v,\n",
" double rf, double div, double cp):\n",
" \"\"\"Price an option using the Black-Scholes model.\n",
" \n",
" s : initial stock price\n",
" k : strike price\n",
" t : expiration time\n",
" v : volatility\n",
" rf : risk-free rate\n",
" div : dividend\n",
" cp : +1/-1 for call/put\n",
" \"\"\"\n",
" cdef double d1, d2, optprice\n",
" with nogil:\n",
" d1 = (log(s/k)+(rf-div+0.5*pow(v,2))*t)/(v*sqrt(t))\n",
" d2 = d1 - v*sqrt(t)\n",
" optprice = cp*s*exp(-div*t)*std_norm_cdf_cy(cp*d1) - \\\n",
" cp*k*exp(-rf*t)*std_norm_cdf_cy(cp*d2)\n",
" return optprice"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"black_scholes_cy(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 7,
"text": [
"10.327861752731728"
]
}
],
"prompt_number": 7
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For comparison, the same code is implemented here in pure python."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from math import exp, sqrt, pow, log, erf\n",
"\n",
"def std_norm_cdf_py(x):\n",
" return 0.5*(1+erf(x/sqrt(2.0)))\n",
"\n",
"def black_scholes_py(s, k, t, v, rf, div, cp):\n",
" \"\"\"Price an option using the Black-Scholes model.\n",
" \n",
" s : initial stock price\n",
" k : strike price\n",
" t : expiration time\n",
" v : volatility\n",
" rf : risk-free rate\n",
" div : dividend\n",
" cp : +1/-1 for call/put\n",
" \"\"\"\n",
" d1 = (log(s/k)+(rf-div+0.5*pow(v,2))*t)/(v*sqrt(t))\n",
" d2 = d1 - v*sqrt(t)\n",
" optprice = cp*s*exp(-div*t)*std_norm_cdf_py(cp*d1) - \\\n",
" cp*k*exp(-rf*t)*std_norm_cdf_py(cp*d2)\n",
" return optprice"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"black_scholes_py(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 9,
"text": [
"10.327861752731728"
]
}
],
"prompt_number": 9
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Below we see the runtime of the two functions: the Cython version is nearly a factor of 10 faster."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%timeit black_scholes_cy(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"1000000 loops, best of 3: 319 ns per loop\n"
]
}
],
"prompt_number": 10
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%timeit black_scholes_py(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"100000 loops, best of 3: 2.28 \u00b5s per loop\n"
]
}
],
"prompt_number": 11
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"External libraries"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Cython allows you to specify additional libraries to be linked with your extension, you can do so with the `-l` flag (also spelled `--lib`). Note that this flag can be passed more than once to specify multiple libraries, such as `-lm -llib2 --lib lib3`. Here's a simple example of how to access the system math library:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%cython -lm\n",
"from libc.math cimport sin\n",
"print 'sin(1)=', sin(1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"sin(1)= 0.841470984808\n"
]
}
],
"prompt_number": 12
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can similarly use the `-I/--include` flag to add include directories to the search path, and `-c/--compile-args` to add extra flags that are passed to Cython via the `extra_compile_args` of the distutils `Extension` class. Please see [the Cython docs on C library usage](http://docs.cython.org/src/tutorial/clibraries.html) for more details on the use of these flags."
]
}
],
"metadata": {}
}
]
}
......@@ -95,6 +95,7 @@ EXT_DEP_MODULES = {
'tag:pstats': 'pstats',
'tag:posix' : 'posix',
'tag:array' : 'array',
'tag:ipython': 'IPython'
}
def patch_inspect_isfunction():
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment