Merge pull request #1077 from gevent/reduce-use-of-cpp-in-corecext

Update corecext.ppyx

.gitignore

@@ -6,7 +6,6 @@ build/
 *.o
 *.egg-info
 gevent.*.[ch]
-src/gevent/libev/corecext.pyx
 src/gevent/__pycache__
 src/gevent/libev/_corecffi.c
 src/gevent/libev/_corecffi.o

CHANGES.rst

@@ -162,10 +162,11 @@ libuv
   - Using negative timeouts may behave differently from libev.
 
   - libuv blocks delivery of all signals, so signals are handled using
-    an (arbitrary) 1 second timer. This means that signal handling
+    an (arbitrary) 0.3 second timer. This means that signal handling
     will be delayed by up to that amount, and that the longest the
     event loop can sleep in the operating system's ``poll`` call is
-    that amount.
+    that amount. Note that this is what gevent does for libev on
+    Windows too.
 
   - libuv only supports one io watcher per file descriptor, whereas
     libev and gevent have always supported many watchers using
@@ -243,6 +244,27 @@ libuv
 
   See :issue:`790` for history and more in-depth discussion.
 
+libev
+-----
+
+- The C extension has been updated to use more modern Cython idioms
+  and generate less code for simplicity, faster compilation and better
+  cache usage. See :pr:`1077`.
+
+  - Watcher objects may be slightly larger. On a 64-bit platform, a
+    typical watcher may be 16 bytes (2 pointers) larger. This is
+    offset by slight performance gains.
+
+  - Cython is no longer preprocessed. Certain attributes that were
+    previously only defined in certain compilation modes (notably
+    LIBEV_EMBED) are now always defined, but will raise ``AttributeError``
+    or have a negative value when not available. In general these
+    attributes are not portable and not implemented by libuv or the
+    CFFI backend. See :issue:`1076`.
+
+  - The ``sigfd`` property that was only conditionally available on
+    certain platforms and when libev was embedded, and only in the C
+    implementation, not the CFFI implementation, has been removed.
 
 1.2.2 (2017-06-05)
 ==================

Makefile

@@ -14,8 +14,8 @@ export LC_ALL=C.UTF-8
 
 all: src/gevent/libev/gevent.corecext.c src/gevent/gevent.ares.c src/gevent/gevent._semaphore.c src/gevent/gevent._local.c
 
-src/gevent/libev/gevent.corecext.c: src/gevent/libev/corecext.ppyx src/gevent/libev/libev.pxd util/cythonpp.py
-	$(PYTHON) util/cythonpp.py -o gevent.corecext.c --module-name gevent.libev.corecext.pyx src/gevent/libev/corecext.ppyx
+src/gevent/libev/gevent.corecext.c: src/gevent/libev/corecext.pyx src/gevent/libev/libev.pxd src/gevent/libev/libev.h
+	$(CYTHON) -o gevent.corecext.c src/gevent/libev/corecext.pyx
 	echo '#include "callbacks.c"' >> gevent.corecext.c
 	mv gevent.corecext.* src/gevent/libev/
 
@@ -40,7 +40,6 @@ src/gevent/gevent._local.c: src/gevent/local.py
 
 
 clean:
-	rm -f corecext.pyx src/gevent/libev/corecext.pyx
 	rm -f gevent.corecext.c gevent.corecext.h src/gevent/libev/gevent.corecext.c src/gevent/libev/gevent.corecext.h
 	rm -f gevent.ares.c gevent.ares.h src/gevent/gevent.ares.c src/gevent/gevent.ares.h
 	rm -f gevent._semaphore.c gevent._semaphore.h src/gevent/gevent._semaphore.c src/gevent/gevent._semaphore.h

appveyor/make.cmd

 IF "%PYTHON_EXE%" == "python" (
-    %PYEXE% util\cythonpp.py -o gevent.corecext.c --module-name gevent.libev.corecext.pyx src\gevent\libev\corecext.ppyx
+    cython -o gevent.corecext.c src\gevent\libev\corecext.pyx
 	type src\gevent\libev\callbacks.c >> gevent.corecext.c
     move gevent.corecext.* src\gevent\libev
 )

src/gevent/ares.pyx

@@ -6,7 +6,10 @@
 # cython: auto_pickle=False
 cimport cares
 import sys
-from python cimport *
+
+from cpython.ref cimport Py_INCREF
+from cpython.ref cimport Py_DECREF
+
 from _socket import gaierror
 
 
@@ -207,7 +210,7 @@ cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent
     cdef channel channel
     cdef object callback
     channel, callback = <tuple>arg
-    Py_DECREF(<PyObjectPtr>arg)
+    Py_DECREF(<tuple>arg)
     cdef object host_result
     try:
         if status or not host:
@@ -227,7 +230,7 @@ cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, cha
     cdef channel channel
     cdef object callback
     channel, callback = <tuple>arg
-    Py_DECREF(<PyObjectPtr>arg)
+    Py_DECREF(<tuple>arg)
     cdef object node
     cdef object service
     try:
@@ -408,7 +411,7 @@ cdef public class channel [object PyGeventAresChannelObject, type PyGeventAresCh
             raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
         # note that for file lookups still AF_INET can be returned for AF_INET6 request
         cdef object arg = (self, callback)
-        Py_INCREF(<PyObjectPtr>arg)
+        Py_INCREF(arg)
         cares.ares_gethostbyname(self.channel, name, family, <void*>gevent_ares_host_callback, <void*>arg)
 
     def gethostbyaddr(self, object callback, char* addr):
@@ -427,7 +430,7 @@ cdef public class channel [object PyGeventAresChannelObject, type PyGeventAresCh
         else:
             raise InvalidIP(repr(addr))
         cdef object arg = (self, callback)
-        Py_INCREF(<PyObjectPtr>arg)
+        Py_INCREF(arg)
         cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, <void*>gevent_ares_host_callback, <void*>arg)
 
     cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags):
@@ -447,7 +450,7 @@ cdef public class channel [object PyGeventAresChannelObject, type PyGeventAresCh
         if length <= 0:
             raise InvalidIP(repr(hostp))
         cdef object arg = (self, callback)
-        Py_INCREF(<PyObjectPtr>arg)
+        Py_INCREF(arg)
         cdef sockaddr_t* x = <sockaddr_t*>&sa6
         cares.ares_getnameinfo(self.channel, x, length, flags, <void*>gevent_ares_nameinfo_callback, <void*>arg)
 

src/gevent/libev/callbacks.c

@@ -179,11 +179,16 @@ static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallback
     Py_DECREF(loop);
 }
 
+/*
+ * PyGeventWatcherObject is the first member of all the structs, so
+ * it is the same in all of them and they can all safely be cast to
+ * it. We could also use the *data member of the libev watcher objects.
+ */
 
 #undef DEFINE_CALLBACK
 #define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \
     static void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) {                  \
-        struct PyGevent##WATCHER_TYPE##Object* watcher = GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher);    \
+        struct PyGeventWatcherObject* watcher = (struct PyGeventWatcherObject*)GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher);    \
         gevent_callback(watcher->loop, watcher->_callback, watcher->args, (PyObject*)watcher, c_watcher, revents); \
     }
 
@@ -211,7 +216,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
     GIL_RELEASE;
 }
 
-#if defined(_WIN32)
+/* This is only used on Win32 */
 
 static void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) {
     GIL_DECLARE;
@@ -220,6 +225,5 @@ static void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, i
     GIL_RELEASE;
 }
 
-#endif  /* _WIN32 */
 
 #endif  /* Py_PYTHON_H */

src/gevent/libev/callbacks.h

@@ -11,21 +11,12 @@
     DEFINE_CALLBACK(check, Check);     \
     DEFINE_CALLBACK(fork, Fork);       \
     DEFINE_CALLBACK(async, Async);     \
-    DEFINE_CALLBACK(stat, Stat);
+    DEFINE_CALLBACK(stat, Stat);       \
+    DEFINE_CALLBACK(child, Child);
 
 
-#ifndef _WIN32
-
-#define DEFINE_CALLBACKS               \
-    DEFINE_CALLBACKS0                  \
-    DEFINE_CALLBACK(child, Child)
-
-#else
-
 #define DEFINE_CALLBACKS DEFINE_CALLBACKS0
 
-#endif
-
 
 DEFINE_CALLBACKS
 
@@ -36,8 +27,8 @@ static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* conte
 struct PyGeventCallbackObject;
 static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
 
-#if defined(_WIN32)
-static void gevent_periodic_signal_check(struct ev_loop *, void *, int);
-#endif
+static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) {
+}
 
-static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) { }
+/* Only used on Win32 */
+static void gevent_periodic_signal_check(struct ev_loop *, void *, int);

src/gevent/libev/libev.h

 #if defined(LIBEV_EMBED)
 #include "ev.c"
-#else
+#undef LIBEV_EMBED
+#define LIBEV_EMBED 1
+#define gevent_ev_loop_origflags(loop) ((loop)->origflags)
+#define gevent_ev_loop_sig_pending(loop) ((loop))->sig_pending
+#define gevent_ev_loop_backend_fd(loop) ((loop))->backend_fd
+#define gevent_ev_loop_activecnt(loop) ((loop))->activecnt
+#else /* !LIBEV_EMBED */
 #include "ev.h"
 
+#define gevent_ev_loop_origflags(loop) -1
+#define gevent_ev_loop_sig_pending(loop) -1
+#define gevent_ev_loop_backend_fd(loop) -1
+#define gevent_ev_loop_activecnt(loop) -1
+
+#define LIBEV_EMBED 0
+#define EV_USE_FLOOR -1
+#define EV_USE_CLOCK_SYSCALL -1
+#define EV_USE_REALTIME -1
+#define EV_USE_MONOTONIC -1
+#define EV_USE_NANOSLEEP -1
+#define EV_USE_INOTIFY -1
+#define EV_USE_SIGNALFD -1
+#define EV_USE_EVENTFD -1
+#define EV_USE_4HEAP -1
+
+
 #ifndef _WIN32
 #include <signal.h>
-#endif
+#endif /* !_WIN32 */
 
-#endif
+#endif /* LIBEV_EMBED */
 
 #ifndef _WIN32
 
@@ -58,9 +81,14 @@ static void gevent_reset_sigchld_handler(void) {
    }
 }
 
-#else
+#else /* !_WIN32 */
 
 #define gevent_ev_default_loop ev_default_loop
 static void gevent_install_sigchld_handler(void) { }
+static void gevent_reset_sigchld_handler(void) { }
+
+// Fake child functions that we can link to.
+static void ev_child_start(struct ev_loop* loop, ev_child* w) {};
+static void ev_child_stop(struct ev_loop* loop, ev_child* w) {};
 
-#endif
+#endif /* _WIN32 */

src/gevent/libev/libev.pxd

+# From cython/includes/libc/stdint.pxd
+#   Longness only used for type promotion.
+#   Actual compile time size used for conversions.
+# We don't have stdint.h on visual studio 9.0 (2008) on windows, sigh,
+# so go with Py_ssize_t
+# ssize_t -> intptr_t
+
 cdef extern from "libev_vfd.h":
-#ifdef _WIN32
-#ifdef _WIN64
-    ctypedef long long vfd_socket_t
-#else
-    ctypedef long vfd_socket_t
-#endif
-#else
-    ctypedef int vfd_socket_t
-#endif
-    long vfd_get(int)
+# cython doesn't process pre-processor directives, so they
+# don't matter in this file. It just takes the last definition it sees.
+    ctypedef Py_ssize_t intptr_t
+    ctypedef intptr_t vfd_socket_t
+
+    vfd_socket_t vfd_get(int)
     int vfd_open(long) except -1
     void vfd_free(int)
 
-cdef extern from "libev.h":
+cdef extern from "libev.h" nogil:
+    int LIBEV_EMBED
     int EV_MINPRI
     int EV_MAXPRI
 
@@ -87,6 +91,9 @@ cdef extern from "libev.h":
         int sigfd
         unsigned int origflags
 
+    struct ev_watcher:
+        void* data;
+
     struct ev_io:
         int fd
         int events
@@ -125,6 +132,13 @@ cdef extern from "libev.h":
         stat prev
         double interval
 
+    union ev_any_watcher:
+        ev_watcher w
+        ev_io io
+        ev_timer timer
+        ev_signal signal
+        ev_idle idle
+
     int ev_version_major()
     int ev_version_minor()
 
@@ -203,6 +217,14 @@ cdef extern from "libev.h":
     void ev_break(ev_loop*, int)
     unsigned int ev_pending_count(ev_loop*)
 
+    # gevent extra functions. These are defined in libev.h.
     ev_loop* gevent_ev_default_loop(unsigned int flags)
     void gevent_install_sigchld_handler()
     void gevent_reset_sigchld_handler()
+
+    # These compensate for lack of access to ev_loop struct definition
+    # when LIBEV_EMBED is false.
+    unsigned int gevent_ev_loop_origflags(ev_loop*);
+    int gevent_ev_loop_sig_pending(ev_loop*);
+    int gevent_ev_loop_backend_fd(ev_loop*);
+    int gevent_ev_loop_activecnt(ev_loop*);

src/gevent/libev/libev_vfd.h

 #ifdef _WIN32
-#ifdef _WIN64
-typedef PY_LONG_LONG vfd_socket_t;
+/* see discussion in the libuv directory: this is a SOCKET which is a
+   HANDLE which is a PVOID (even though they're really small ints),
+   and CPython and PyPy return that SOCKET cast to an int from
+   fileno()
+*/
+typedef intptr_t vfd_socket_t;
 #define vfd_socket_object PyLong_FromLongLong
-#else
-typedef long vfd_socket_t;
-#define vfd_socket_object PyInt_FromLong
-#endif
+
 #ifdef LIBEV_EMBED
 /*
  * If libev on win32 is embedded, then we can use an
@@ -53,13 +54,13 @@ static CRITICAL_SECTION* vfd_make_lock()
 #define VFD_GIL_DECLARE PyGILState_STATE ___save
 #define VFD_GIL_ENSURE  ___save = PyGILState_Ensure()
 #define VFD_GIL_RELEASE PyGILState_Release(___save)
-#else
+#else /* ! WITH_THREAD */
 #define VFD_LOCK_ENTER
 #define VFD_LOCK_LEAVE
 #define VFD_GIL_DECLARE
 #define VFD_GIL_ENSURE
 #define VFD_GIL_RELEASE
-#endif
+#endif /*_WITH_THREAD */
 
 /*
  * Given a virtual fd returns an OS handle or -1
@@ -67,7 +68,7 @@ static CRITICAL_SECTION* vfd_make_lock()
  */
 static vfd_socket_t vfd_get(int fd)
 {
-	int handle = -1;
+	vfd_socket_t handle = -1;
 	VFD_LOCK_ENTER;
 	if (vfd_entries != NULL && fd >= 0 && fd < vfd_num)
 		handle = vfd_entries[fd].handle;
@@ -201,7 +202,7 @@ done:
 #define vfd_free(fd) vfd_free_((fd), 0)
 #define EV_WIN32_CLOSE_FD(fd) vfd_free_((fd), 1)
 
-#else
+#else /* !LIBEV_EMBED */
 /*
  * If libev on win32 is not embedded in gevent, then
  * the only way to map vfds is to use the default of
@@ -211,13 +212,14 @@ done:
 #define vfd_get(fd) _get_osfhandle((fd))
 #define vfd_open(fd) _open_osfhandle((fd), 0)
 #define vfd_free(fd)
-#endif
-#else
+#endif /* LIBEV_EMBED */
+
+#else /* !_WIN32 */
 /*
  * On non-win32 platforms vfd_* are noop macros
  */
 typedef int vfd_socket_t;
 #define vfd_get(fd) (fd)
-#define vfd_open(fd) ((int)(fd))
+#define vfd_open(fd) (fd)
 #define vfd_free(fd)
-#endif
+#endif /* _WIN32 */

src/gevent/libuv/loop.py

@@ -124,12 +124,13 @@ class loop(AbstractLoop):
         # XXX: Perhaps we could optimize this to notice when there are other
         # timers in the loop and start/stop it then. When we have a callback
         # scheduled, this should also be the same and unnecessary?
+        # libev does takes this basic approach on Windows.
         self._signal_idle = ffi.new("uv_timer_t*")
         libuv.uv_timer_init(self._ptr, self._signal_idle)
         self._signal_idle.data = self._handle_to_self
         libuv.uv_timer_start(self._signal_idle, libuv.python_check_callback,
-                             1000,
-                             1000)
+                             300,
+                             300)
         libuv.uv_unref(self._signal_idle)
 
     def _run_callbacks(self):

src/gevent/python.pxd

-cdef extern from "Python.h":
-    struct PyObject:
-        pass
-    ctypedef PyObject* PyObjectPtr "PyObject*"
-    void   Py_INCREF(PyObjectPtr)
-    void   Py_DECREF(PyObjectPtr)
-    void   Py_XDECREF(PyObjectPtr)
-    int    Py_ReprEnter(PyObjectPtr)
-    void   Py_ReprLeave(PyObjectPtr)
-    int    PyCallable_Check(PyObjectPtr)
-
-cdef extern from "frameobject.h":
-    ctypedef struct PyThreadState:
-        PyObjectPtr exc_type
-        PyObjectPtr exc_value
-        PyObjectPtr exc_traceback
-    PyThreadState* PyThreadState_GET()

util/cythonpp.py

 #!/usr/bin/env python
 # Copyright (C) 2011-2012 Denis Bilenko (http://denisbilenko.com)
 # Copyright (C) 2015-2016 gevent contributors
+
+######################################
+######################################
+######################################
+### WARNING WARNING WARNING WARNING
+##
+## This script is unmaintained and no
+## longer in use in this project due to
+## bugs.
+## See https://github.com/gevent/gevent/issues/1076
+##
+### WARNING WARNING WARNING WARNING
+######################################
+######################################
+######################################
+
+
 from __future__ import print_function
 import sys
 import os
@@ -15,6 +32,7 @@ import subprocess
 import multiprocessing
 import tempfile
 import shutil
+from collections import OrderedDict
 
 import threading
 
@@ -39,6 +57,7 @@ else:
 
 CYTHON = os.environ.get('CYTHON') or 'cython'
 DEBUG = os.environ.get('CYTHONPP_DEBUG', False)
+TRACE = DEBUG == 'trace'
 WRITE_OUTPUT = False
 
 if os.getenv('READTHEDOCS'):
@@ -59,6 +78,9 @@ cython_header_re = re.compile(r'^/\* (generated by cython [^\s*]+)[^*]+\*/$', re
 #assert cython_header_re.match('/* Generated by Cython 0.21.1 */').group(1) == 'Generated by Cython 0.21.1'
 #assert cython_header_re.match('/* Generated by Cython 0.19 on 55-555-555 */').group(1) == 'Generated by Cython 0.19'
 
+class EmptyConfigurationError(TypeError):
+    pass
+
 class Configuration(frozenset):
     """
     A set of CPP conditions that apply to a given sequence
@@ -72,11 +94,12 @@ class Configuration(frozenset):
 
     def __new__(cls, iterable):
         sorted_iterable = tuple(sorted(frozenset(iterable)))
+        if not sorted_iterable:
+            raise EmptyConfigurationError("Empty configurations not allowed")
+
         if sorted_iterable not in cls._cache:
             if not all(isinstance(x, Condition) for x in sorted_iterable):
                 raise TypeError("Must be iterable of conditions")
-            if not sorted_iterable:
-                raise TypeError("Empty configurations not allowed")
             self = frozenset.__new__(cls, sorted_iterable)
             self._sorted = sorted_iterable
             cls._cache[sorted_iterable] = self
@@ -90,7 +113,11 @@ class Configuration(frozenset):
         return self.union(conditions)
 
     def difference(self, other):
-        return Configuration(frozenset.difference(self, other))
+        try:
+            return Configuration(frozenset.difference(self, other))
+        except EmptyConfigurationError:
+            raise EmptyConfigurationError(
+                "Couldn't subtract %r from %r" % (self, other))
 
     def __sub__(self, other):
         return self.difference(other)
@@ -375,7 +402,14 @@ class ConfigurationGroups(tuple):
         if self._simplified:
             return self
 
-        for tag1, tag2 in combinations(self, 2):
+        if (len(self) == 2
+                and len(self[0]) == len(self[1]) == 1
+                and list(self[0])[0] == list(self[1])[0].inverted()):
+            # This trivially simplifies to the empty group
+            # Its defined(foo, True) || defined(foo, False)
+            return ConfigurationGroups(()).simplify_tags()
+
+        for tag1, tag2 in sorted(combinations(self, 2)):
             if tag1 == tag2:
                 tags = list(self)
                 tags.remove(tag1)
@@ -383,6 +417,8 @@ class ConfigurationGroups(tuple):
 
             for condition in tag1:
                 inverted_condition = condition.inverted()
+                if inverted_condition == tag2:
+                    continue
                 if inverted_condition in tag2:
                     tag1_copy = tag1 - {inverted_condition}
                     tag2_copy = tag2 - {inverted_condition}
@@ -438,7 +474,8 @@ def _run_cython_on_file(configuration, pyx_filename,
         if WRITE_OUTPUT:
             atomic_write(unique_output_filename + '.deb', output)
     finally:
-        shutil.rmtree(tempdir, True)
+        if not DEBUG:
+            shutil.rmtree(tempdir, True)
 
     return configuration.attach_tags(output), configuration, sourcehash
 
@@ -462,6 +499,10 @@ def _run_cython_on_files(pyx_filename, py_banner, banner, output_filename, prepr
 
     same_results = {} # {sourcehash: tagged_str}
     for t in threads:
+        if not t.value:
+            log("Thread %s failed.", t)
+            return
+
         sourcehash = t.value[2]
         tagged_output = t.value[0]
         if sourcehash not in same_results:
@@ -518,6 +559,9 @@ def process_filename(filename, output_filename=None, module_name=None):
 
     sources = _run_cython_on_files(pyx_filename, py_banner, banner, output_filename,
                                    preprocessed, module_name)
+    if sources is None:
+        log("At least one thread failed to run")
+        sys.exit(1)
 
     log('Generating %s ',  output_filename)
     result = generate_merged(sources)
@@ -546,7 +590,7 @@ def preprocess_filename(filename, config):
     """
     linecount = 0
     current_name = None
-    definitions = {}
+    definitions = OrderedDict()
     result = []
     including_section = []
     with open(filename) as f:
@@ -576,10 +620,10 @@ def preprocess_filename(filename, config):
                             current_name = name
                         definitions[name] = {'lines': [value]}
                         if params is None:
-                            dbg('Adding definition for %r', name)
+                            trace('Adding definition for %r', name)
                         else:
                             definitions[name]['params'] = parse_parameter_names(params)
-                            dbg('Adding definition for %r: %s', name, definitions[name]['params'])
+                            trace('Adding definition for %r: %s', name, definitions[name]['params'])
                     else:
                         m = Condition.match_condition(stripped)
                         if m is not None and config is not None:
@@ -734,7 +778,7 @@ def expand_to_match(items):
 
 def produce_preprocessor(iterable):
 
-    if DEBUG:
+    if TRACE:
         current_line = [0]
 
         def wrap(line):
@@ -853,9 +897,9 @@ def expand_definitions(code, definitions):
                 arguments = None
             if arguments and len(params) == len(arguments):
                 local_definitions = {}
-                dbg('Macro %r params=%r arguments=%r source=%r', token, params, arguments, m.groups())
+                trace('Macro %r params=%r arguments=%r source=%r', token, params, arguments, m.groups())
                 for key, value in zip(params, arguments):
-                    dbg('Adding argument %r=%r', key, value)
+                    trace('Adding argument %r=%r', key, value)
                     local_definitions[key] = {'lines': [value]}
                 result = expand_definitions('\n'.join(definition['lines']), local_definitions)
             else:
@@ -868,7 +912,7 @@ def expand_definitions(code, definitions):
                 result += m.group(3)
         if m.group(1) != '##':
             result = m.group(1) + result
-        dbg('Replace %r with %r', m.group(0), result)
+        trace('Replace %r with %r', m.group(0), result)
         return result
 
     for _ in range(20000):
@@ -936,7 +980,8 @@ def system(command, comment):
     try:
         subprocess.check_call(command)
         dbg('\tDone running %s # %s', command_str, comment)
-    except subprocess.CalledProcessError:
+    except (subprocess.CalledProcessError, OSError):
+        # Python 2 can raise OSError: No such file or directory
         # debugging code
         log("Path: %s", os.getenv("PATH"))
         bin_dir = os.path.dirname(sys.executable)
@@ -1012,11 +1057,16 @@ def log(message, *args):
     else:
         print(string, file=sys.stderr)
 
-
-def dbg(*args):
-    if not DEBUG:
+if DEBUG:
+    dbg = log
+else:
+    def dbg(*_):
+        return
+if TRACE:
+    trace = log
+else:
+    def trace(*_):
         return
-    return log(*args)
 
 
 def main():