merge

CHANGES.rst

@@ -93,6 +93,8 @@ Bugs fixed
   runtime but not as global module names.  Trying both lookups helps with
   globals() manipulation.
 
+* Fixed stl container conversion for typedef element types.
+
 * ``obj.pop(x)`` truncated large C integer values of x to ``Py_ssize_t``.
 
 * ``__init__.pyc`` is recognised as marking a package directory

COPYING.txt

@@ -10,3 +10,10 @@ made to go from Pyrex to Cython are so licensed.
 
 See LICENSE.txt for more details.
 
+------------------
+
+The output of a Cython compilation is NOT considered a derivative
+work of Cython.  Specifically, though the compilation process may
+embed snippets of varying lengths into the final output, these
+snippets, as embedded in the output, do not encumber the resulting
+output with any license restrictions.

Cython/Compiler/JediTyper.py

+
+from __future__ import absolute_import
+
+from io import open
+from collections import defaultdict
+
+from jedi import Script
+from jedi.parser.representation import Function, Module, Import
+
+from Cython.Utils import open_source_file
+
+
+default_type_map = {
+    'float': 'double',
+    'int': 'long',
+}
+
+
+def analyse(source_path):
+    """
+    Analyse a Python source code file with Jedi.
+    Returns a mapping from (scope-name, (line, column)) pairs to a name-types mapping.
+    """
+    script = Script(path=source_path)
+    evaluator = script._evaluator
+    scoped_names = {}
+    for statements in script._parser.module().used_names.values():
+        for statement in statements:
+            scope = statement.parent
+            while not isinstance(scope, (Function, Module)):
+                scope = scope.parent
+            # hack: work around current Jedi problem with global module variables
+            if not hasattr(scope, 'scope_names_generator'):
+                continue
+            statement_names = statement.get_defined_names()
+            if not statement_names:
+                continue
+            key = (None if isinstance(scope, Module) else str(scope.name), scope.start_pos)
+            try:
+                names = scoped_names[key]
+            except KeyError:
+                names = scoped_names[key] = defaultdict(set)
+            for name in statement_names:
+                for name_type in evaluator.find_types(scope, name):
+                    if isinstance(name_type, Import):
+                        type_name = 'object'
+                    else:
+                        type_name = name_type.name
+                    names[str(name)].add(type_name)
+
+    return scoped_names
+
+
+def inject_types(source_path, types, type_map=default_type_map, mode='python'):
+    """
+    Hack type declarations into source code file.
+
+    @param mode is currently 'python', which means that the generated type declarations use pure Python syntax.
+    """
+    col_and_types_by_line = dict(
+        # {line: (column, scope_name or None, [(name, type)])}
+        (k[-1][0], (k[-1][1], k[0], [(n, next(iter(t))) for (n, t) in v.items() if len(t) == 1]))
+        for (k, v) in types.items())
+
+    lines = [u'import cython\n']
+    with open_source_file(source_path) as f:
+        for line_no, line in enumerate(f, 1):
+            if line_no in col_and_types_by_line:
+                col, scope, types = col_and_types_by_line[line_no]
+                types = ', '.join('%s=%s' % (name, type_map.get(type_name, type_name))
+                                  for name, type_name in types)
+                if scope is None:
+                    type_decl = u'{indent}cython.declare({types})\n'
+                else:
+                    type_decl = u'{indent}@cython.locals({types})\n'
+                lines.append(type_decl.format(indent=' '*col, types=types))
+            lines.append(line)
+
+    return lines
+
+
+def main(file_paths=None, overwrite=False):
+    """
+    Main entry point to process a list of .py files and inject type inferred declarations.
+    """
+    if file_paths is None:
+        import sys
+        file_paths = sys.argv[1:]
+
+    for source_path in file_paths:
+        types = analyse(source_path)
+        lines = inject_types(source_path, types)
+        target_path = source_path + ('' if overwrite else '_typed.py')
+        with open(target_path, 'w', encoding='utf8') as f:
+            for line in lines:
+                f.write(line)
+
+
+if __name__ == '__main__':
+    main()

Cython/Compiler/ParseTreeTransforms.py

@@ -1478,7 +1478,7 @@ if VALUE is not None:
 
     def visit_CClassDefNode(self, node):
         node = self.visit_ClassDefNode(node)
-        if node.scope and node.scope.implemented:
+        if node.scope and node.scope.implemented and node.body:
             stats = []
             for entry in node.scope.var_entries:
                 if entry.needs_property:

Cython/Compiler/PyrexTypes.py

@@ -3023,12 +3023,30 @@ class CppClassType(CType):
                     return False
                 tags.append(T.specialization_name())
                 if T.exception_value is not None:
-                    except_clause = T.exception_value
+                    # This is a hack due to the except value clause
+                    # requiring a const (literal) value of the right
+                    # (visible) type.
+                    def guess_type(value):
+                        if not T.is_typedef and (T.is_numeric or T.is_ptr):
+                            return T
+                        try:
+                            int(value)
+                            return c_longlong_type
+                        except ValueError:
+                            pass
+                        try:
+                            float(value)
+                            return c_double_type
+                        except ValueError:
+                            pass
+                        return T
+                    except_type = guess_type(T.exception_value)
+                    except_clause = "%s " % T.exception_value
                     if T.exception_check:
                         except_clause = "? %s" % except_clause
                     declarations.append(
                         "    ctypedef %s %s '%s'" % (
-                             T.declaration_code("", for_display=True), X[ix], T.declaration_code("")))
+                             except_type.declaration_code("", for_display=True), X[ix], T.declaration_code("")))
                 else:
                     except_clause = "*"
                     declarations.append(

Cython/Includes/libcpp/map.pxd

@@ -8,14 +8,18 @@ cdef extern from "<map>" namespace "std" nogil:
             iterator operator--()
             bint operator==(iterator)
             bint operator!=(iterator)
+        cppclass const_iterator:
+            pair[const T, U]& operator*()
+            const_iterator operator++()
+            const_iterator operator--()
+            bint operator==(const_iterator)
+            bint operator!=(const_iterator)
         cppclass reverse_iterator:
             pair[T, U]& operator*()
             iterator operator++()
             iterator operator--()
             bint operator==(reverse_iterator)
             bint operator!=(reverse_iterator)
-        #cppclass const_iterator(iterator):
-        #    pass
         #cppclass const_reverse_iterator(reverse_iterator):
         #    pass
         map() except +
@@ -31,19 +35,19 @@ cdef extern from "<map>" namespace "std" nogil:
         bint operator>=(map&, map&)
         U& at(T&)
         iterator begin()
-        #const_iterator begin()
+        const_iterator const_begin "begin" ()
         void clear()
         size_t count(T&)
         bint empty()
         iterator end()
-        #const_iterator end()
+        const_iterator const_end "end" ()
         pair[iterator, iterator] equal_range(T&)
         #pair[const_iterator, const_iterator] equal_range(key_type&)
         void erase(iterator)
         void erase(iterator, iterator)
         size_t erase(T&)
         iterator find(T&)
-        #const_iterator find(key_type&)
+        const_iterator const_find "find" (T&)
         pair[iterator, bint] insert(pair[T, U]) # XXX pair[T,U]&
         iterator insert(iterator, pair[T, U]) # XXX pair[T,U]&
         #void insert(input_iterator, input_iterator)

Cython/Includes/posix/resource.pxd

+# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html
+
+from posix.time cimport timeval
+from posix.types cimport id_t
+
+cdef extern from "sys/resource.h" nogil:
+
+    enum: PRIO_PROCESS
+    enum: PRIO_PGRP
+    enum: PRIO_USER
+
+    enum: RLIM_INFINITY
+    enum: RLIM_SAVED_MAX
+    enum: RLIM_SAVED_CUR
+
+    enum: RUSAGE_SELF
+    enum: RUSAGE_CHILDREN
+
+    enum: RLIMIT_CORE
+    enum: RLIMIT_CPU
+    enum: RLIMIT_DATA
+    enum: RLIMIT_FSIZE
+    enum: RLIMIT_NOFILE
+    enum: RLIMIT_STACK
+    enum: RLIMIT_AS
+
+    ctypedef unsigned long rlim_t
+
+    cdef struct rlimit:
+        rlim_t rlim_cur
+        rlim_t rlim_max
+
+    cdef struct rusage:
+        timeval ru_utime
+        timeval ru_stime
+        long    ru_maxrss
+        long    ru_ixrss
+        long    ru_idrss
+        long    ru_isrss
+        long    ru_minflt
+        long    ru_majflt
+        long    ru_nswap
+        long    ru_inblock
+        long    ru_oublock
+        long    ru_msgsnd
+        long    ru_msgrcv
+        long    ru_nsignals
+        long    ru_nvcsw
+        long    ru_nivcsw
+
+    int  getpriority(int, id_t)
+    int  getrlimit(int, rlimit *)
+    int  getrusage(int, rusage *)
+    int  setpriority(int, id_t, int)
+    int  setrlimit(int, const rlimit *)

Cython/Includes/posix/signal.pxd

+# 7.14 Signal handling <signal.h>
+
+from posix.types cimport pid_t, sigset_t, uid_t
+
+cdef extern from "signal.h" nogil:
+
+    cdef union sigval:
+        int  sival_int
+        void *sival_ptr
+
+    cdef struct sigevent:
+        int    sigev_notify
+        int    sigev_signo
+        sigval sigev_value
+        void   *sigev_notify_function(sigval)
+
+    ctypedef struct siginfo_t:
+        int    si_signo
+        int    si_code
+        int    si_errno
+        pid_t  si_pid
+        uid_t  si_uid
+        void   *si_addr
+        int    si_status
+        long   si_band
+        sigval si_value
+
+    cdef struct sigaction_t "sigaction":
+        void     *sa_handler(int)
+        sigset_t sa_mask
+        int      sa_flags
+        void     sa_sigaction(int, siginfo_t *, void *)
+
+    enum: SA_NOCLDSTOP
+    enum: SIG_BLOCK
+    enum: SIG_UNBLOCK
+    enum: SIG_SETMASK
+    enum: SA_ONSTACK
+    enum: SA_RESETHAND
+    enum: SA_RESTART
+    enum: SA_SIGINFO
+    enum: SA_NOCLDWAIT
+    enum: SA_NODEFER
+    enum: SS_ONSTACK
+    enum: SS_DISABLE
+    enum: MINSIGSTKSZ
+    enum: SIGSTKSZ
+
+    enum: SIGEV_NONE
+    enum: SIGEV_SIGNAL
+    enum: SIGEV_THREAD
+    enum: SIGEV_THREAD_ID
+
+
+    int          kill          (pid_t, int)
+    int          killpg        (pid_t, int)
+    int          sigaction     (int, const sigaction_t *, sigaction_t *)
+    int          sigpending    (sigset_t *)
+    int          sigprocmask   (int, const sigset_t *, sigset_t *)
+    int          sigsuspend    (const sigset_t *)
+
+    int          sigaddset     (sigset_t *, int)
+    int          sigdelset     (sigset_t *, int)
+    int          sigemptyset   (sigset_t *)
+    int          sigfillset    (sigset_t *)
+    int          sigismember   (const sigset_t *)

Cython/Includes/posix/time.pxd

+# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html
+
+from posix.signal cimport sigevent
+from posix.types cimport clock_t, clockid_t, suseconds_t, time_t, timer_t
+
+cdef extern from "sys/time.h" nogil:
+
+    enum: CLOCKS_PER_SEC
+    enum: CLOCK_PROCESS_CPUTIME_ID
+    enum: CLOCK_THREAD_CPUTIME_ID
+
+    enum: CLOCK_REALTIME
+    enum: TIMER_ABSTIME
+    enum: CLOCK_MONOTONIC
+
+    # FreeBSD-specific clocks
+    enum: CLOCK_UPTIME
+    enum: CLOCK_UPTIME_PRECISE
+    enum: CLOCK_UPTIME_FAST
+    enum: CLOCK_REALTIME_PRECISE
+    enum: CLOCK_REALTIME_FAST
+    enum: CLOCK_MONOTONIC_PRECISE
+    enum: CLOCK_MONOTONIC_FAST
+    enum: CLOCK_SECOND
+
+    # Linux-specific clocks
+    enum: CLOCK_PROCESS_CPUTIME_ID
+    enum: CLOCK_THREAD_CPUTIME_ID
+    enum: CLOCK_MONOTONIC_RAW
+    enum: CLOCK_REALTIME_COARSE
+    enum: CLOCK_MONOTONIC_COARSE
+    enum: CLOCK_BOOTTIME
+    enum: CLOCK_REALTIME_ALARM
+    enum: CLOCK_BOOTTIME_ALARM
+
+    enum: ITIMER_REAL
+    enum: ITIMER_VIRTUAL
+    enum: ITIMER_PROF
+
+    cdef struct timeval:
+        time_t      tv_sec
+        suseconds_t tv_usec
+
+    cdef struct itimerval:
+        timeval it_interval
+        timeval it_value
+
+    cdef struct timezone:
+        int tz_minuteswest
+        int dsttime
+
+    cdef struct timespec:
+        time_t tv_sec
+        long   tv_nsec
+
+    cdef struct itimerspec:
+        timespec it_interval
+        timespec it_value
+
+    cdef struct tm:
+        int  tm_sec
+        int  tm_min
+        int  tm_hour
+        int  tm_mday
+        int  tm_mon
+        int  tm_year
+        int  tm_wday
+        int  tm_yday
+        int  tm_isdst
+        char *tm_zone
+        long tm_gmtoff
+
+    char    *asctime(const tm *)
+    char    *asctime_r(const tm *, char *)
+    clock_t clock()
+    int     clock_getcpuclockid(pid_t, clockid_t *)
+    int     clock_getres(clockid_t, timespec *)
+    int     clock_gettime(clockid_t, timespec *)
+    int     clock_nanosleep(clockid_t, int, const timespec *, timespec *)
+    int     clock_settime(clockid_t, const timespec *)
+    char    *ctime(const time_t *)
+    char    *ctime_r(const time_t *, char *)
+    double  difftime(time_t, time_t)
+    tm      *getdate(const char *)
+    int     getitimer(int, itimerval *)
+    int     gettimeofday(timeval *tp, timezone *tzp)
+    tm      *gmtime(const time_t *)
+    tm      *gmtime_r(const time_t *, tm *)
+    tm      *localtime(const time_t *)
+    tm      *localtime_r(const time_t *, tm *)
+    time_t  mktime(tm *)
+    int     nanosleep(const timespec *, timespec *)
+    int     setitimer(int, const itimerval *, itimerval *)
+    size_t  strftime(char *, size_t, const char *, const tm *)
+    char    *strptime(const char *, const char *, tm *)
+    time_t  time(time_t *)
+    int     timer_create(clockid_t, sigevent *, timer_t *)
+    int     timer_delete(timer_t)
+    int     timer_gettime(timer_t, itimerspec *)
+    int     timer_getoverrun(timer_t)
+    int     timer_settime(timer_t, int, const itimerspec *, itimerspec *)
+    void    tzset()
+
+    int daylight
+    long timezone
+    char *tzname[2]

Cython/Includes/posix/types.pxd

 cdef extern from "sys/types.h":
     ctypedef long blkcnt_t
     ctypedef long blksize_t
+    ctypedef long clock_t
+    ctypedef long clockid_t
     ctypedef long dev_t
     ctypedef long gid_t
+    ctypedef long id_t
     ctypedef long ino_t
     ctypedef long mode_t
     ctypedef long nlink_t
     ctypedef long off_t
     ctypedef long pid_t
+    ctypedef long sigset_t
+    ctypedef long suseconds_t
     ctypedef long time_t
+    ctypedef long timer_t
     ctypedef long uid_t

Cython/Utility/CppConvert.pyx

@@ -213,18 +213,18 @@ cdef extern from *:
         cppclass value_type:
             T first
             U second
-        cppclass iterator:
+        cppclass const_iterator:
             value_type& operator*()
-            iterator operator++()
-            bint operator!=(iterator)
-        iterator begin()
-        iterator end()
+            const_iterator operator++()
+            bint operator!=(const_iterator)
+        const_iterator begin()
+        const_iterator end()
 
 @cname("{{cname}}")
-cdef object {{cname}}(map[X,Y] s):
+cdef object {{cname}}(const map[X,Y]& s):
     o = {}
-    cdef map[X,Y].value_type *key_value
-    cdef map[X,Y].iterator iter = s.begin()
+    cdef const map[X,Y].value_type *key_value
+    cdef map[X,Y].const_iterator iter = s.begin()
     while iter != s.end():
         key_value = &cython.operator.dereference(iter)
         o[X_to_py(key_value.first)] = Y_to_py(key_value.second)

tests/compile/posix_pxds.pyx

@@ -15,6 +15,10 @@ cimport posix.types
 from posix cimport types
 from posix.types cimport *
 
+cimport posix.signal
+from posix cimport signal
+from posix.signal cimport *
+
 cimport posix.stat
 from posix cimport stat
 from posix.stat cimport *
@@ -22,3 +26,11 @@ from posix.stat cimport *
 cimport posix.stdlib
 from posix cimport stdlib
 from posix.stdlib cimport *
+
+cimport posix.time
+from posix cimport time
+from posix.time cimport *
+
+cimport posix.resource
+from posix cimport resource
+from posix.resource cimport *

tests/run/cpp_stl_conversion.pyx

@@ -116,6 +116,24 @@ def test_double_vector(o):
     cdef vector[double] v = o
     return v
 
+ctypedef int my_int
+
+def test_typedef_vector(o):
+    """
+    >>> test_typedef_vector([1, 2, 3])
+    [1, 2, 3]
+    >>> test_typedef_vector([1, 2, 3**100])       #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    OverflowError: ...
+    >>> test_typedef_vector([1, 2, None])       #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: an integer is required
+    """
+    cdef vector[my_int] v = o
+    return v
+
 def test_pair(o):
     """
     >>> test_pair((1, 2))

tests/run/posix_resource.pyx

+# tag: posix
+from posix.unistd    cimport *
+from posix.resource cimport *
+
+def test_getpriority():
+    """
+    >>> test_getpriority()
+    """
+    ret = getpriority(PRIO_PROCESS, getpid())
+    assert ret == 0
+
+def test_getrlimit():
+    """
+    >>> test_getrlimit()
+    """
+    cdef rlimit rlim
+    ret = getrlimit(RLIMIT_CPU, &rlim)
+    assert ret == 0
+    assert rlim.rlim_cur != 0
+
+def test_getrusage():
+    """
+    >>> test_getrusage()
+    """
+    cdef rusage r
+    ret = getrusage(RUSAGE_SELF, &r)
+    assert ret == 0

tests/run/posix_time.pyx

+# tag: posix
+from libc.stdlib  cimport getenv
+from posix.stdlib cimport setenv, unsetenv
+from posix.time cimport *
+
+def test_itimer(sec, usec):
+    """
+    >>> test_itimer(10, 2)
+    (10, 2)
+    """
+    cdef itimerval t, gtime
+
+    t.it_interval.tv_sec = sec
+    t.it_interval.tv_usec = usec
+    t.it_value.tv_sec = sec
+    t.it_value.tv_usec = usec
+    ret = setitimer(ITIMER_REAL, &t, NULL)
+    assert ret == 0
+    ret = getitimer(ITIMER_REAL, &gtime)
+    assert ret == 0
+    t.it_interval.tv_sec = 0
+    t.it_interval.tv_usec = 0
+    t.it_value.tv_sec = 0
+    t.it_value.tv_usec = 0
+    ret = setitimer(ITIMER_REAL, &t, NULL)
+    return gtime.it_interval.tv_sec, gtime.it_interval.tv_usec
+
+def test_gettimeofday():
+    """
+    >>> test_gettimeofday()
+    """
+    cdef timeval t
+    ret = gettimeofday(&t, NULL)
+    assert ret == 0
+
+def test_time():
+    """
+    >>> test_time()
+    """
+    cdef time_t t1, t2
+    t1 = time(NULL)
+    assert t1 != 0
+    t1 = time(&t2)
+    assert t1 == t2
+
+def test_mktime():
+    """
+tests/run/posix_time.pyx
+    >>> test_mktime()  # doctest:+ELLIPSIS
+    (986138177, ...'Sun Apr  1 15:16:17 2001\\n')
+    """
+    cdef tm t, gmt
+    cdef time_t tt
+    cdef char *ct
+    cdef char *tz
+
+    tz = getenv("TZ")
+    setenv("TZ", "UTC", 1)
+    tzset()
+    t.tm_sec = 17
+    t.tm_min = 16
+    t.tm_hour = 15
+    t.tm_year = 101
+    t.tm_mon = 3
+    t.tm_mday = 1
+    t.tm_isdst = 0
+    tt = mktime(&t)
+    assert tt != -1
+    ct = ctime(&tt)
+    assert ct != NULL
+    if tz:
+        setenv("TZ", tz, 1)
+    else:
+        unsetenv("TZ")
+    tzset()
+    return tt, ct