runtests.py 101 KB
Newer Older
1
#!/usr/bin/env python
2

3 4
from __future__ import print_function

5
import atexit
6
import base64
7 8 9
import os
import sys
import re
10
import gc
11
import heapq
12
import locale
13
import shutil
14
import time
15 16
import unittest
import doctest
17
import operator
18
import subprocess
19
import tempfile
20
import traceback
21
import warnings
22
import zlib
23
import glob
24
from contextlib import contextmanager
25
from collections import defaultdict
26

27 28 29 30 31 32 33
try:
    import platform
    IS_PYPY = platform.python_implementation() == 'PyPy'
    IS_CPYTHON = platform.python_implementation() == 'CPython'
except (ImportError, AttributeError):
    IS_CPYTHON = True
    IS_PYPY = False
34
IS_PY2 = sys.version_info[0] < 3
35

36 37 38 39 40
from io import open as io_open
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO  # doesn't accept 'str' in Py2
41 42 43 44 45 46

try:
    import cPickle as pickle
except ImportError:
    import pickle

47 48 49 50 51
try:
    import threading
except ImportError: # No threads, no problems
    threading = None

52 53 54
try:
    from unittest import SkipTest
except ImportError:
55 56
    class SkipTest(Exception):  # don't raise, only provided to allow except-ing it!
        pass
57
    def skip_test(reason):
58
        sys.stderr.write("Skipping test: %s\n" % reason)
59 60 61 62
else:
    def skip_test(reason):
        raise SkipTest(reason)

Stefan Behnel's avatar
Stefan Behnel committed
63 64 65 66 67
try:
    basestring
except NameError:
    basestring = str

68 69 70
WITH_CYTHON = True

from distutils.command.build_ext import build_ext as _build_ext
71
from distutils import sysconfig
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
_to_clean = []

@atexit.register
def _cleanup_files():
    """
    This is only used on Cygwin to clean up shared libraries that are unsafe
    to delete while the test suite is running.
    """

    for filename in _to_clean:
        if os.path.isdir(filename):
            shutil.rmtree(filename, ignore_errors=True)
        else:
            try:
                os.remove(filename)
            except OSError:
                pass

90

91 92 93
def get_distutils_distro(_cache=[]):
    if _cache:
        return _cache[0]
Unknown's avatar
Unknown committed
94
    # late import to accommodate for setuptools override
95 96 97 98
    from distutils.dist import Distribution
    distutils_distro = Distribution()

    if sys.platform == 'win32':
99
        # TODO: Figure out why this hackery (see https://thread.gmane.org/gmane.comp.python.cython.devel/8280/).
100
        config_files = distutils_distro.find_config_files()
101 102 103 104
        try:
            config_files.remove('setup.cfg')
        except ValueError:
            pass
105 106 107
        distutils_distro.parse_config_files(config_files)

        cfgfiles = distutils_distro.find_config_files()
108 109 110 111
        try:
            cfgfiles.remove('setup.cfg')
        except ValueError:
            pass
112 113 114
        distutils_distro.parse_config_files(cfgfiles)
    _cache.append(distutils_distro)
    return distutils_distro
115 116


117
EXT_DEP_MODULES = {
118
    'tag:numpy':     'numpy',
119
    'tag:pythran':  'pythran',
120
    'tag:setuptools':  'setuptools.sandbox',
121 122 123 124
    'tag:asyncio':  'asyncio',
    'tag:pstats':   'pstats',
    'tag:posix':    'posix',
    'tag:array':    'array',
125
    'tag:coverage': 'Cython.Coverage',
126
    'Coverage':     'Cython.Coverage',
127
    'tag:ipython':  'IPython.testing.globalipapp',
128
    'tag:jedi':     'jedi_BROKEN_AND_DISABLED',
129
    'tag:test.support': 'test.support',  # support module for CPython unit tests
130 131
}

132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
def patch_inspect_isfunction():
    import inspect
    orig_isfunction = inspect.isfunction
    def isfunction(obj):
        return orig_isfunction(obj) or type(obj).__name__ == 'cython_function_or_method'
    isfunction._orig_isfunction = orig_isfunction
    inspect.isfunction = isfunction

def unpatch_inspect_isfunction():
    import inspect
    try:
        orig_isfunction = inspect.isfunction._orig_isfunction
    except AttributeError:
        pass
    else:
        inspect.isfunction = orig_isfunction

149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
def def_to_cdef(source):
    '''
    Converts the module-level def methods into cdef methods, i.e.

        @decorator
        def foo([args]):
            """
            [tests]
            """
            [body]

    becomes

        def foo([args]):
            """
            [tests]
            """
            return foo_c([args])

        cdef foo_c([args]):
            [body]
    '''
    output = []
    skip = False
    def_node = re.compile(r'def (\w+)\(([^()*]*)\):').match
    lines = iter(source.split('\n'))
    for line in lines:
        if not line.strip():
            output.append(line)
            continue

        if skip:
            if line[0] != ' ':
                skip = False
            else:
                continue

        if line[0] == '@':
            skip = True
            continue

        m = def_node(line)
        if m:
            name = m.group(1)
            args = m.group(2)
            if args:
                args_no_types = ", ".join(arg.split()[-1] for arg in args.split(','))
            else:
                args_no_types = ""
            output.append("def %s(%s):" % (name, args_no_types))
199
            line = next(lines)
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
            if '"""' in line:
                has_docstring = True
                output.append(line)
                for line in lines:
                    output.append(line)
                    if '"""' in line:
                        break
            else:
                has_docstring = False
            output.append("    return %s_c(%s)" % (name, args_no_types))
            output.append('')
            output.append("cdef %s_c(%s):" % (name, args))
            if not has_docstring:
                output.append(line)

        else:
            output.append(line)

    return '\n'.join(output)

220 221 222 223 224 225 226

def exclude_extension_in_pyver(*versions):
    def check(ext):
        return EXCLUDE_EXT if sys.version_info[:2] in versions else ext
    return check


227 228 229 230 231 232
def exclude_extension_on_platform(*platforms):
    def check(ext):
        return EXCLUDE_EXT if sys.platform in platforms else ext
    return check


233 234 235 236
def update_linetrace_extension(ext):
    ext.define_macros.append(('CYTHON_TRACE', 1))
    return ext

237

238
def update_numpy_extension(ext, set_api17_macro=True):
239
    import numpy
240 241
    from numpy.distutils.misc_util import get_info

242
    ext.include_dirs.append(numpy.get_include())
243

244 245 246
    if set_api17_macro:
        ext.define_macros.append(('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION'))

247 248 249 250 251
    # We need the npymath library for numpy.math.
    # This is typically a static-only library.
    for attr, value in get_info('npymath').items():
        getattr(ext, attr).extend(value)

252

253
def update_openmp_extension(ext):
254
    ext.openmp = True
255 256
    language = ext.language

257 258 259 260
    if sys.platform == 'win32' and sys.version_info[:2] == (3,4):
        # OpenMP tests fail in appveyor in Py3.4 -> just ignore them, EoL of Py3.4 is early 2019...
        return EXCLUDE_EXT

261 262 263 264 265 266 267 268 269 270
    if language == 'cpp':
        flags = OPENMP_CPP_COMPILER_FLAGS
    else:
        flags = OPENMP_C_COMPILER_FLAGS

    if flags:
        compile_flags, link_flags = flags
        ext.extra_compile_args.extend(compile_flags.split())
        ext.extra_link_args.extend(link_flags.split())
        return ext
271 272
    elif sys.platform == 'win32':
        return ext
273 274 275

    return EXCLUDE_EXT

276

277
def update_cpp11_extension(ext):
278
    """
279 280 281
        update cpp11 extensions that will run on versions of gcc >4.8
    """
    gcc_version = get_gcc_version(ext.language)
282
    if gcc_version:
283 284
        compiler_version = gcc_version.group(1)
        if float(compiler_version) > 4.8:
285
            ext.extra_compile_args.append("-std=c++11")
286
        return ext
287

Stefan Behnel's avatar
Stefan Behnel committed
288 289
    clang_version = get_clang_version(ext.language)
    if clang_version:
290
        ext.extra_compile_args.append("-std=c++11")
291 292 293
        if sys.platform == "darwin":
          ext.extra_compile_args.append("-stdlib=libc++")
          ext.extra_compile_args.append("-mmacosx-version-min=10.7")
294 295
        return ext

296
    return EXCLUDE_EXT
297

298 299 300 301 302 303 304 305
def update_pthread_extension(ext):
    """
      use pthread to link and compile on platform which supports it
    """
    if sys.platform in ('linux', 'linux2', 'freebsd', 'netbsd', 'openbsd'):
        # FIXME: this check on the platform is hacky
        ext.extra_compile_args.append("-pthread")
        ext.extra_link_args.append("-pthread")
306

307
def get_cc_version(language):
308 309
    """
        finds gcc version using Popen
310 311 312 313 314
    """
    if language == 'cpp':
        cc = sysconfig.get_config_var('CXX')
    else:
        cc = sysconfig.get_config_var('CC')
315
    if not cc:
316 317
        from distutils import ccompiler
        cc = ccompiler.get_default_compiler()
318 319

    if not cc:
320
        return ''
321

322 323 324
    # For some reason, cc can be e.g. 'gcc -pthread'
    cc = cc.split()[0]

325 326 327
    # Force english output
    env = os.environ.copy()
    env['LC_MESSAGES'] = 'C'
328
    try:
329
        p = subprocess.Popen([cc, "-v"], stderr=subprocess.PIPE, env=env)
330 331 332 333
    except EnvironmentError:
        # Be compatible with Python 3
        warnings.warn("Unable to find the %s compiler: %s: %s" %
                      (language, os.strerror(sys.exc_info()[1].errno), cc))
334
        return ''
335
    _, output = p.communicate()
336 337 338 339 340 341 342 343 344
    return output.decode(locale.getpreferredencoding() or 'ASCII', 'replace')


def get_gcc_version(language):
    matcher = re.compile(r"gcc version (\d+\.\d+)").search
    return matcher(get_cc_version(language))


def get_clang_version(language):
345
    matcher = re.compile(r"clang(?:-|\s+version\s+)(\d+\.\d+)").search
346
    return matcher(get_cc_version(language))
347 348 349 350 351 352 353 354 355 356 357


def get_openmp_compiler_flags(language):
    """
    As of gcc 4.2, it supports OpenMP 2.5. Gcc 4.4 implements 3.0. We don't
    (currently) check for other compilers.

    returns a two-tuple of (CFLAGS, LDFLAGS) to build the OpenMP extension
    """
    gcc_version = get_gcc_version(language)

358
    if not gcc_version:
359 360 361 362
        if sys.platform == 'win32':
            return '/openmp', ''
        else:
            return None # not gcc - FIXME: do something about other compilers
363

364 365 366 367
    # gcc defines "__int128_t", assume that at least all 64 bit architectures have it
    global COMPILER_HAS_INT128
    COMPILER_HAS_INT128 = getattr(sys, 'maxsize', getattr(sys, 'maxint', 0)) > 2**60

368
    compiler_version = gcc_version.group(1)
369 370 371
    if compiler_version and compiler_version.split('.') >= ['4', '2']:
        return '-fopenmp', '-fopenmp'

372 373 374 375
try:
    locale.setlocale(locale.LC_ALL, '')
except locale.Error:
    pass
376

377 378
COMPILER = None
COMPILER_HAS_INT128 = False
379 380 381 382 383 384
OPENMP_C_COMPILER_FLAGS = get_openmp_compiler_flags('c')
OPENMP_CPP_COMPILER_FLAGS = get_openmp_compiler_flags('cpp')

# Return this from the EXT_EXTRAS matcher callback to exclude the extension
EXCLUDE_EXT = object()

385 386
EXT_EXTRAS = {
    'tag:numpy' : update_numpy_extension,
387
    'tag:openmp': update_openmp_extension,
388
    'tag:cpp11': update_cpp11_extension,
389
    'tag:trace' : update_linetrace_extension,
390
    'tag:pthread': update_pthread_extension,
391
    'tag:bytesformat':  exclude_extension_in_pyver((3, 3), (3, 4)),  # no %-bytes formatting
392
    'tag:no-macos':  exclude_extension_on_platform('darwin'),
393
    'tag:py3only':  exclude_extension_in_pyver((2, 7)),
394
}
395

396

397
# TODO: use tags
398
VER_DEP_MODULES = {
399
    # tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e.
Stefan Behnel's avatar
Stefan Behnel committed
400
    # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x
401

402 403 404 405
    # The next line should start (3,); but this is a dictionary, so
    # we can only have one (3,) key.  Since 2.7 is supposed to be the
    # last 2.x release, things would have to change drastically for this
    # to be unsafe...
406 407
    (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3',
                                           'run.test_raisefrom',
408
                                           'run.different_package_names',
409
                                           'run.unicode_imports',  # encoding problems on appveyor in Py2
410
                                           'run.reimport_failure',  # reimports don't do anything in Py2
411
                                           ]),
412
    (3,): (operator.ge, lambda x: x in ['run.non_future_division',
Stefan Behnel's avatar
Stefan Behnel committed
413
                                        'compile.extsetslice',
414
                                        'compile.extdelslice',
415
                                        'run.special_methods_T561_py2',
416
                                        ]),
417
    (3,3) : (operator.lt, lambda x: x in ['build.package_compilation',
418
                                          'build.cythonize_pep420_namespace',
419
                                          'run.yield_from_py33',
420
                                          'pyximport.pyximport_namespace',
421
                                          'run.qualname',
422
                                          ]),
423
    (3,4): (operator.lt, lambda x: x in ['run.py34_signature',
424
                                         'run.test_unicode',  # taken from Py3.7, difficult to backport
425
                                         ]),
426 427
    (3,4,999): (operator.gt, lambda x: x in ['run.initial_file_path',
                                             ]),
428
    (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop',
429
                                         'run.py35_asyncio_async_def',
430
                                         'run.mod__spec__',
431
                                         'run.pep526_variable_annotations',  # typing module
432
                                         'run.test_exceptions',  # copied from Py3.7+
433
                                         ]),
434 435
}

436
INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
437
CFLAGS = os.getenv('CFLAGS', '').split()
438
CCACHE = os.getenv('CYTHON_RUNTESTS_CCACHE', '').split()
439
TEST_SUPPORT_DIR = 'testsupport'
440

441
BACKENDS = ['c', 'cpp']
442

443 444 445
UTF8_BOM_BYTES = r'\xef\xbb\xbf'.encode('ISO-8859-1').decode('unicode_escape')


446 447 448 449 450 451 452 453 454 455
def memoize(f):
    uncomputed = object()
    f._cache = {}
    def func(*args):
        res = f._cache.get(args, uncomputed)
        if res is uncomputed:
            res = f._cache[args] = f(*args)
        return res
    return func

456

Robert Bradshaw's avatar
Robert Bradshaw committed
457
@memoize
Robert Bradshaw's avatar
Robert Bradshaw committed
458 459
def parse_tags(filepath):
    tags = defaultdict(list)
460
    parse_tag = re.compile(r'#\s*(\w+)\s*:(.*)$').match
461
    with io_open(filepath, encoding='ISO-8859-1', errors='ignore') as f:
462
        for line in f:
463 464
            # ignore BOM-like bytes and whitespace
            line = line.lstrip(UTF8_BOM_BYTES).strip()
465
            if not line:
466 467 468 469
                if tags:
                    break  # assume all tags are in one block
                else:
                    continue
470 471
            if line[0] != '#':
                break
472 473 474 475 476 477 478 479
            parsed = parse_tag(line)
            if parsed:
                tag, values = parsed.groups()
                if tag in ('coding', 'encoding'):
                    continue
                if tag == 'tags':
                    tag = 'tag'
                    print("WARNING: test tags use the 'tag' directive, not 'tags' (%s)" % filepath)
480
                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils', 'preparse'):
481 482 483 484 485
                    print("WARNING: unknown test directive '%s' found (%s)" % (tag, filepath))
                values = values.split(',')
                tags[tag].extend(filter(None, [value.strip() for value in values]))
            elif tags:
                break  # assume all tags are in one block
Robert Bradshaw's avatar
Robert Bradshaw committed
486 487
    return tags

488

Stefan Behnel's avatar
Stefan Behnel committed
489
list_unchanging_dir = memoize(lambda x: os.listdir(x))  # needs lambda to set function attribute
490

Stefan Behnel's avatar
Stefan Behnel committed
491

492 493
@memoize
def _list_pyregr_data_files(test_directory):
494 495 496 497
    is_data_file = re.compile('(?:[.](txt|pem|db|html)|^bad.*[.]py)$').search
    return ['__init__.py'] + [
        filename for filename in list_unchanging_dir(test_directory)
        if is_data_file(filename)]
498 499


500 501 502 503 504 505 506 507 508 509 510 511 512 513
def import_ext(module_name, file_path=None):
    if file_path:
        import imp
        return imp.load_dynamic(module_name, file_path)
    else:
        try:
            from importlib import invalidate_caches
        except ImportError:
            pass
        else:
            invalidate_caches()
        return __import__(module_name, globals(), locals(), ['*'])


514 515
class build_ext(_build_ext):
    def build_extension(self, ext):
516 517 518 519 520 521
        try:
            try: # Py2.7+ & Py3.2+
                compiler_obj = self.compiler_obj
            except AttributeError:
                compiler_obj = self.compiler
            if ext.language == 'c++':
522
                compiler_obj.compiler_so.remove('-Wstrict-prototypes')
523 524
            if CCACHE:
                compiler_obj.compiler_so = CCACHE + compiler_obj.compiler_so
525 526
            if getattr(ext, 'openmp', None) and compiler_obj.compiler_type == 'msvc':
                ext.extra_compile_args.append('/openmp')
527 528
        except Exception:
            pass
529
        _build_ext.build_extension(self, ext)
530

531

532
class ErrorWriter(object):
533
    match_error = re.compile(r'(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
534

535
    def __init__(self, encoding=None):
536
        self.output = []
537 538 539 540 541 542
        self.encoding = encoding

    def write(self, value):
        if self.encoding:
            value = value.encode('ISO-8859-1').decode(self.encoding)
        self.output.append(value)
543

544
    def _collect(self):
545
        s = ''.join(self.output)
546 547
        results = {'errors': [], 'warnings': []}
        for line in s.splitlines():
548 549
            match = self.match_error(line)
            if match:
550
                is_warning, line, column, message = match.groups()
551 552 553
                results['warnings' if is_warning else 'errors'].append((int(line), int(column), message.strip()))

        return [["%d:%d: %s" % values for values in sorted(results[key])] for key in ('errors', 'warnings')]
554 555

    def geterrors(self):
556
        return self._collect()[0]
557 558

    def getwarnings(self):
559
        return self._collect()[1]
560 561

    def getall(self):
562 563 564 565 566
        return self._collect()

    def close(self):
        pass  # ignore, only to match file-like interface

567

568
class Stats(object):
569 570
    def __init__(self, top_n=8):
        self.top_n = top_n
571 572
        self.test_counts = defaultdict(int)
        self.test_times = defaultdict(float)
573
        self.top_tests = defaultdict(list)
574

575
    def add_time(self, name, language, metric, t):
576 577
        self.test_counts[metric] += 1
        self.test_times[metric] += t
578
        top = self.top_tests[metric]
579 580
        push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush
        # min-heap => pop smallest/shortest until longest times remain
581
        push(top, (t, name, language))
582 583

    @contextmanager
584
    def time(self, name, language, metric):
585 586 587
        t = time.time()
        yield
        t = time.time() - t
588
        self.add_time(name, language, metric, t)
589

590 591 592 593 594 595 596 597 598 599
    def update(self, stats):
        # type: (Stats) -> None
        for metric, t in stats.test_times.items():
            self.test_times[metric] += t
            self.test_counts[metric] += stats.test_counts[metric]
            top = self.top_tests[metric]
            for entry in stats.top_tests[metric]:
                push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush
                push(top, entry)

600
    def print_stats(self, out=sys.stderr):
601 602
        if not self.test_times:
            return
603
        lines = ['Times:\n']
604
        for metric, t in sorted(self.test_times.items()):
605
            count = self.test_counts[metric]
606 607 608
            top = self.top_tests[metric]
            lines.append("%-12s: %8.2f sec  (%4d, %6.3f / run) - slowest: %s\n" % (
                metric, t, count, t / count,
609
                ', '.join("'{2}:{1}' ({0:.2f}s)".format(*item) for item in heapq.nlargest(self.top_n, top))))
610 611 612
        out.write(''.join(lines))


613
class TestBuilder(object):
614 615
    def __init__(self, rootdir, workdir, selectors, exclude_selectors, options,
                 with_pyregr, languages, test_bugs, language_level,
616
                 common_utility_dir, pythran_dir=None,
617
                 default_mode='run', stats=None,
618
                 add_embedded_test=False):
619 620
        self.rootdir = rootdir
        self.workdir = workdir
621
        self.selectors = selectors
622
        self.exclude_selectors = exclude_selectors
623 624 625 626
        self.annotate = options.annotate_source
        self.cleanup_workdir = options.cleanup_workdir
        self.cleanup_sharedlibs = options.cleanup_sharedlibs
        self.cleanup_failures = options.cleanup_failures
627
        self.with_pyregr = with_pyregr
628
        self.cython_only = options.cython_only
629
        self.doctest_selector = re.compile(options.only_pattern).search if options.only_pattern else None
630
        self.languages = languages
631
        self.test_bugs = test_bugs
632
        self.fork = options.fork
633
        self.language_level = language_level
634
        self.test_determinism = options.test_determinism
635
        self.common_utility_dir = common_utility_dir
636
        self.pythran_dir = pythran_dir
637
        self.default_mode = default_mode
638
        self.stats = stats
639
        self.add_embedded_test = add_embedded_test
640
        self.capture = options.capture
641 642 643

    def build_suite(self):
        suite = unittest.TestSuite()
644 645
        filenames = os.listdir(self.rootdir)
        filenames.sort()
Stefan Behnel's avatar
Stefan Behnel committed
646
        # TODO: parallelise I/O with a thread pool for the different directories once we drop Py2 support
647
        for filename in filenames:
648
            path = os.path.join(self.rootdir, filename)
649
            if os.path.isdir(path) and filename != TEST_SUPPORT_DIR:
650 651
                if filename == 'pyregr' and not self.with_pyregr:
                    continue
652 653
                if filename == 'broken' and not self.test_bugs:
                    continue
654
                suite.addTest(
655
                    self.handle_directory(path, filename))
656
        if sys.platform not in ['win32'] and self.add_embedded_test:
657
            # Non-Windows makefile.
658
            if [1 for selector in self.selectors if selector("embedded")] \
659
                    and not [1 for selector in self.exclude_selectors if selector("embedded")]:
660
                suite.addTest(unittest.makeSuite(EmbedTest))
661 662
        return suite

663
    def handle_directory(self, path, context):
664 665 666 667
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)

668
        suite = unittest.TestSuite()
669
        filenames = list_unchanging_dir(path)
670 671
        filenames.sort()
        for filename in filenames:
672 673 674
            filepath = os.path.join(path, filename)
            module, ext = os.path.splitext(filename)
            if ext not in ('.py', '.pyx', '.srctree'):
675
                continue
676 677
            if filename.startswith('.'):
                continue # certain emacs backup files
678 679 680 681
            if context == 'pyregr':
                tags = defaultdict(list)
            else:
                tags = parse_tags(filepath)
Robert Bradshaw's avatar
Robert Bradshaw committed
682
            fqmodule = "%s.%s" % (context, module)
683
            if not [ 1 for match in self.selectors
Robert Bradshaw's avatar
Robert Bradshaw committed
684
                     if match(fqmodule, tags) ]:
685
                continue
686
            if self.exclude_selectors:
687
                if [1 for match in self.exclude_selectors
Robert Bradshaw's avatar
Robert Bradshaw committed
688
                        if match(fqmodule, tags)]:
689
                    continue
690

691
            mode = self.default_mode
692 693 694 695 696 697
            if tags['mode']:
                mode = tags['mode'][0]
            elif context == 'pyregr':
                mode = 'pyregr'

            if ext == '.srctree':
698
                if 'cpp' not in tags['tag'] or 'cpp' in self.languages:
699 700 701
                    suite.addTest(EndToEndTest(filepath, workdir,
                             self.cleanup_workdir, stats=self.stats,
                             capture=self.capture))
702 703 704 705 706 707
                continue

            # Choose the test suite.
            if mode == 'pyregr':
                if not filename.startswith('test_'):
                    continue
708
                test_class = CythonPyregrTestCase
709
            elif mode == 'run':
710
                if module.startswith("test_"):
711
                    test_class = CythonUnitTestCase
712
                else:
713
                    test_class = CythonRunTestCase
714
            elif mode in ['compile', 'error']:
715
                test_class = CythonCompileTestCase
716 717
            else:
                raise KeyError('Invalid test mode: ' + mode)
718

719
            for test in self.build_tests(test_class, path, workdir,
Robert Bradshaw's avatar
Robert Bradshaw committed
720
                                         module, mode == 'error', tags):
721
                suite.addTest(test)
722

723
            if mode == 'run' and ext == '.py' and not self.cython_only and not filename.startswith('test_'):
724
                # additionally test file in real Python
725 726 727 728 729 730
                min_py_ver = [
                    (int(pyver.group(1)), int(pyver.group(2)))
                    for pyver in map(re.compile(r'pure([0-9]+)[.]([0-9]+)').match, tags['tag'])
                    if pyver
                ]
                if not min_py_ver or any(sys.version_info >= min_ver for min_ver in min_py_ver):
731
                    suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename), tags, stats=self.stats))
732

733 734
        return suite

Robert Bradshaw's avatar
Robert Bradshaw committed
735
    def build_tests(self, test_class, path, workdir, module, expect_errors, tags):
736 737
        warning_errors = 'werror' in tags['tag']
        expect_warnings = 'warnings' in tags['tag']
Vitja Makarov's avatar
Vitja Makarov committed
738

739
        if expect_errors:
740
            if skip_c(tags) and 'cpp' in self.languages:
Robert Bradshaw's avatar
Robert Bradshaw committed
741 742 743
                languages = ['cpp']
            else:
                languages = self.languages[:1]
744 745
        else:
            languages = self.languages
Robert Bradshaw's avatar
Robert Bradshaw committed
746

747
        if 'c' in languages and skip_c(tags):
748 749
            languages = list(languages)
            languages.remove('c')
750
        if 'cpp' in languages and 'no-cpp' in tags['tag']:
Robert Bradshaw's avatar
Robert Bradshaw committed
751 752
            languages = list(languages)
            languages.remove('cpp')
753 754 755
        if not languages:
            return []

756
        language_levels = [2, 3] if 'all_language_levels' in tags['tag'] else [None]
757

758 759 760
        pythran_dir = self.pythran_dir
        if 'pythran' in tags['tag'] and not pythran_dir and 'cpp' in languages:
            import pythran.config
761 762 763 764
            try:
                pythran_ext = pythran.config.make_extension(python=True)
            except TypeError:  # old pythran version syntax
                pythran_ext = pythran.config.make_extension()
765 766
            pythran_dir = pythran_ext['include_dirs'][0]

767
        preparse_list = tags.get('preparse', ['id'])
768
        tests = [ self.build_test(test_class, path, workdir, module, tags, language, language_level,
769
                                  expect_errors, expect_warnings, warning_errors, preparse,
770
                                  pythran_dir if language == "cpp" else None)
771
                  for language in languages
772 773 774
                  for preparse in preparse_list
                  for language_level in language_levels
        ]
775 776
        return tests

777
    def build_test(self, test_class, path, workdir, module, tags, language, language_level,
778
                   expect_errors, expect_warnings, warning_errors, preparse, pythran_dir):
779 780 781 782
        language_workdir = os.path.join(workdir, language)
        if not os.path.exists(language_workdir):
            os.makedirs(language_workdir)
        workdir = os.path.join(language_workdir, module)
783
        if preparse != 'id':
784 785 786
            workdir += '_%s' % (preparse,)
        if language_level:
            workdir += '_cy%d' % (language_level,)
787
        return test_class(path, workdir, module, tags,
788
                          language=language,
789
                          preparse=preparse,
790
                          expect_errors=expect_errors,
791
                          expect_warnings=expect_warnings,
792 793 794
                          annotate=self.annotate,
                          cleanup_workdir=self.cleanup_workdir,
                          cleanup_sharedlibs=self.cleanup_sharedlibs,
795
                          cleanup_failures=self.cleanup_failures,
796
                          cython_only=self.cython_only,
797
                          doctest_selector=self.doctest_selector,
798
                          fork=self.fork,
799
                          language_level=language_level or self.language_level,
800
                          warning_errors=warning_errors,
801
                          test_determinism=self.test_determinism,
802
                          common_utility_dir=self.common_utility_dir,
803 804
                          pythran_dir=pythran_dir,
                          stats=self.stats)
805

806

807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
def skip_c(tags):
    if 'cpp' in tags['tag']:
        return True

    # We don't want to create a distutils key in the
    # dictionary so we check before looping.
    if 'distutils' in tags:
        for option in tags['distutils']:
            splitted = option.split('=')
            if len(splitted) == 2:
                argument, value = splitted
                if argument.strip() == 'language' and value.strip() == 'c++':
                    return True
    return False


823 824 825 826 827 828 829 830 831 832 833 834
def filter_stderr(stderr_bytes):
    """
    Filter annoying warnings from output.
    """
    if b"Command line warning D9025" in stderr_bytes:
        # MSCV: cl : Command line warning D9025 : overriding '/Ox' with '/Od'
        stderr_bytes = b'\n'.join(
            line for line in stderr_bytes.splitlines()
            if b"Command line warning D9025" not in line)
    return stderr_bytes


835
class CythonCompileTestCase(unittest.TestCase):
836
    def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id',
837
                 expect_errors=False, expect_warnings=False, annotate=False, cleanup_workdir=True,
838
                 cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False, doctest_selector=None,
839
                 fork=True, language_level=2, warning_errors=False,
840
                 test_determinism=False,
841
                 common_utility_dir=None, pythran_dir=None, stats=None):
842
        self.test_directory = test_directory
843
        self.tags = tags
844 845
        self.workdir = workdir
        self.module = module
846
        self.language = language
847 848
        self.preparse = preparse
        self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse)
849
        self.expect_errors = expect_errors
850
        self.expect_warnings = expect_warnings
851
        self.annotate = annotate
852
        self.cleanup_workdir = cleanup_workdir
853
        self.cleanup_sharedlibs = cleanup_sharedlibs
854
        self.cleanup_failures = cleanup_failures
855
        self.cython_only = cython_only
856
        self.doctest_selector = doctest_selector
857
        self.fork = fork
858
        self.language_level = language_level
Vitja Makarov's avatar
Vitja Makarov committed
859
        self.warning_errors = warning_errors
860
        self.test_determinism = test_determinism
861
        self.common_utility_dir = common_utility_dir
862
        self.pythran_dir = pythran_dir
863
        self.stats = stats
864 865 866
        unittest.TestCase.__init__(self)

    def shortDescription(self):
867 868 869 870
        return "compiling (%s%s%s) %s" % (
            self.language,
            "/cy2" if self.language_level == 2 else "/cy3" if self.language_level == 3 else "",
            "/pythran" if self.pythran_dir is not None else "",
871
            self.description_name()
872 873 874 875
        )

    def description_name(self):
        return self.name
876

Stefan Behnel's avatar
Stefan Behnel committed
877
    def setUp(self):
Vitja Makarov's avatar
Vitja Makarov committed
878
        from Cython.Compiler import Options
879 880
        self._saved_options = [
            (name, getattr(Options, name))
881 882 883 884 885
            for name in (
                'warning_errors',
                'clear_to_none',
                'error_on_unknown_names',
                'error_on_uninitialized',
886
                # 'cache_builtins',  # not currently supported due to incorrect global caching
887
            )
888
        ]
889
        self._saved_default_directives = list(Options.get_directive_defaults().items())
Vitja Makarov's avatar
Vitja Makarov committed
890
        Options.warning_errors = self.warning_errors
891
        if sys.version_info >= (3, 4):
892
            Options._directive_defaults['autotestdict'] = False
Vitja Makarov's avatar
Vitja Makarov committed
893

894 895
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)
Stefan Behnel's avatar
Stefan Behnel committed
896 897 898
        if self.workdir not in sys.path:
            sys.path.insert(0, self.workdir)

899
    def tearDown(self):
Vitja Makarov's avatar
Vitja Makarov committed
900
        from Cython.Compiler import Options
901 902
        for name, value in self._saved_options:
            setattr(Options, name, value)
903
        Options._directive_defaults = dict(self._saved_default_directives)
904
        unpatch_inspect_isfunction()
Vitja Makarov's avatar
Vitja Makarov committed
905

Stefan Behnel's avatar
Stefan Behnel committed
906 907 908 909 910 911 912 913
        try:
            sys.path.remove(self.workdir)
        except ValueError:
            pass
        try:
            del sys.modules[self.module]
        except KeyError:
            pass
914 915 916
        cleanup = self.cleanup_failures or self.success
        cleanup_c_files = WITH_CYTHON and self.cleanup_workdir and cleanup
        cleanup_lib_files = self.cleanup_sharedlibs and cleanup
917 918
        is_cygwin = sys.platform == 'cygwin'

919
        if os.path.exists(self.workdir):
920
            if cleanup_c_files and cleanup_lib_files and not is_cygwin:
921 922 923
                shutil.rmtree(self.workdir, ignore_errors=True)
            else:
                for rmfile in os.listdir(self.workdir):
924
                    ext = os.path.splitext(rmfile)[1]
925
                    if not cleanup_c_files:
926 927 928 929 930 931
                        # Keep C, C++ files, header files, preprocessed sources
                        # and assembly sources (typically the .i and .s files
                        # are intentionally generated when -save-temps is given)
                        if ext in (".c", ".cpp", ".h", ".i", ".ii", ".s"):
                            continue
                        if ext == ".html" and rmfile.startswith(self.module):
932
                            continue
933

934
                    is_shared_obj = ext in (".so", ".dll")
935 936

                    if not cleanup_lib_files and is_shared_obj:
937
                        continue
938

939 940 941 942
                    try:
                        rmfile = os.path.join(self.workdir, rmfile)
                        if os.path.isdir(rmfile):
                            shutil.rmtree(rmfile, ignore_errors=True)
943 944 945
                        elif is_cygwin and is_shared_obj:
                            # Delete later
                            _to_clean.append(rmfile)
946 947 948 949
                        else:
                            os.remove(rmfile)
                    except IOError:
                        pass
950

951 952 953 954
                if cleanup_c_files and cleanup_lib_files and is_cygwin:
                    # Finally, remove the work dir itself
                    _to_clean.append(self.workdir)

955 956 957 958
        if cleanup_c_files and os.path.exists(self.workdir + '-again'):
            shutil.rmtree(self.workdir + '-again', ignore_errors=True)


959
    def runTest(self):
960
        self.success = False
961
        self.runCompileTest()
962
        self.success = True
963 964

    def runCompileTest(self):
965 966
        return self.compile(
            self.test_directory, self.module, self.workdir,
967
            self.test_directory, self.expect_errors, self.expect_warnings, self.annotate)
968

969 970 971 972 973
    def find_module_source_file(self, source_file):
        if not os.path.exists(source_file):
            source_file = source_file[:-1]
        return source_file

Stefan Behnel's avatar
Stefan Behnel committed
974 975 976
    def build_target_filename(self, module_name):
        target = '%s.%s' % (module_name, self.language)
        return target
977

978
    def related_files(self, test_directory, module_name):
979
        is_related = re.compile('%s_.*[.].*' % module_name).match
980
        return [filename for filename in list_unchanging_dir(test_directory)
981
                if is_related(filename)]
982 983

    def copy_files(self, test_directory, target_directory, file_list):
984 985 986
        if self.preparse and self.preparse != 'id':
            preparse_func = globals()[self.preparse]
            def copy(src, dest):
987 988 989
                with open(src) as fin:
                    with open(dest, 'w') as fout:
                        fout.write(preparse_func(fin.read()))
990 991 992 993 994 995
        else:
            # use symlink on Unix, copy on Windows
            try:
                copy = os.symlink
            except AttributeError:
                copy = shutil.copy
996 997

        join = os.path.join
998
        for filename in file_list:
999
            file_path = join(test_directory, filename)
1000
            if os.path.exists(file_path):
1001
                copy(file_path, join(target_directory, filename))
1002

1003 1004 1005
    def source_files(self, workdir, module_name, file_list):
        return ([self.build_target_filename(module_name)] +
            [filename for filename in file_list
1006
             if not os.path.isfile(os.path.join(workdir, filename))])
1007 1008

    def split_source_and_output(self, test_directory, module, workdir):
1009
        source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx')
1010 1011 1012 1013 1014 1015 1016

        from Cython.Utils import detect_opened_file_encoding
        with io_open(source_file, 'rb') as f:
            # encoding is passed to ErrorWriter but not used on the source
            # since it is sometimes deliberately wrong
            encoding = detect_opened_file_encoding(f, default=None)

1017 1018
        with io_open(source_file, 'r', encoding='ISO-8859-1') as source_and_output:
            error_writer = warnings_writer = None
1019
            out = io_open(os.path.join(workdir, module + os.path.splitext(source_file)[1]),
1020
                          'w', encoding='ISO-8859-1')
1021 1022 1023 1024
            try:
                for line in source_and_output:
                    if line.startswith("_ERRORS"):
                        out.close()
1025
                        out = error_writer = ErrorWriter(encoding=encoding)
1026 1027
                    elif line.startswith("_WARNINGS"):
                        out.close()
1028
                        out = warnings_writer = ErrorWriter(encoding=encoding)
1029 1030 1031 1032
                    else:
                        out.write(line)
            finally:
                out.close()
1033

1034 1035
        return (error_writer.geterrors() if error_writer else [],
                warnings_writer.geterrors() if warnings_writer else [])
1036

1037 1038
    def run_cython(self, test_directory, module, targetdir, incdir, annotate,
                   extra_compile_options=None):
1039
        include_dirs = INCLUDE_DIRS + [os.path.join(test_directory, '..', TEST_SUPPORT_DIR)]
1040 1041
        if incdir:
            include_dirs.append(incdir)
1042

1043 1044 1045 1046 1047 1048
        if self.preparse == 'id':
            source = self.find_module_source_file(
                os.path.join(test_directory, module + '.pyx'))
        else:
            self.copy_files(test_directory, targetdir, [module + '.pyx'])
            source = os.path.join(targetdir, module + '.pyx')
Stefan Behnel's avatar
Stefan Behnel committed
1049
        target = os.path.join(targetdir, self.build_target_filename(module))
1050

1051 1052
        if extra_compile_options is None:
            extra_compile_options = {}
1053

1054 1055 1056 1057
        if 'allow_unknown_names' in self.tags['tag']:
            from Cython.Compiler import Options
            Options.error_on_unknown_names = False

1058 1059 1060
        try:
            CompilationOptions
        except NameError:
1061
            from Cython.Compiler.Options import CompilationOptions
1062
            from Cython.Compiler.Main import compile as cython_compile
1063
            from Cython.Compiler.Options import default_options
1064
        common_utility_include_dir = self.common_utility_dir
1065

1066
        options = CompilationOptions(
1067
            default_options,
1068 1069
            include_path = include_dirs,
            output_file = target,
1070
            annotate = annotate,
1071 1072
            use_listing_file = False,
            cplus = self.language == 'cpp',
1073
            np_pythran = self.pythran_dir is not None,
1074
            language_level = self.language_level,
1075
            generate_pxi = False,
1076
            evaluate_tree_assertions = True,
1077
            common_utility_include_dir = common_utility_include_dir,
1078
            **extra_compile_options
1079
            )
1080 1081 1082
        cython_compile(source, options=options,
                       full_module_name=module)

1083
    def run_distutils(self, test_directory, module, workdir, incdir,
1084
                      extra_extension_args=None):
1085 1086 1087
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
1088
            build_extension = build_ext(get_distutils_distro())
1089 1090 1091 1092
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()
1093 1094
            if COMPILER:
                build_extension.compiler = COMPILER
1095

1096
            ext_compile_flags = CFLAGS[:]
1097

1098 1099
            if  build_extension.compiler == 'mingw32':
                ext_compile_flags.append('-Wno-format')
1100 1101
            if extra_extension_args is None:
                extra_extension_args = {}
1102

1103 1104
            related_files = self.related_files(test_directory, module)
            self.copy_files(test_directory, workdir, related_files)