Add cython version of filesystem scanning program

cython/Makefile

+INCLUDE_PYTHON = -I/srv/slapgrid/slappart6/srv/runner/shared/python3/2e435adf7e2cb7d97668da52532ac7f3/include/python3.7m
+
+OPENSSL_PATH = /srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75
+INCLUDE_OPENSSL = -I$(OPENSSL_PATH)/include
+LIBRARY_OPENSSL = -L$(OPENSSL_PATH)/lib
+RUNPATH_OPENSSL = -Wl,-rpath=$(OPENSSL_PATH)/lib
+
+FMTLIB_PATH = /srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c
+INCLUDE_FMTLIB = -I$(FMTLIB_PATH)/include
+LIBRARY_FMTLIB = -L$(FMTLIB_PATH)/lib
+RUNPATH_FMTLIB = -Wl,-rpath=$(FMTLIB_PATH)/lib
+
+INCLUDE_PATHS = $(INCLUDE_PYTHON) $(INCLUDE_OPENSSL) $(INCLUDE_FMTLIB)
+LIBRARY_PATHS = $(LIBRARY_OPENSSL) $(LIBRARY_FMTLIB)
+LIBRARIES = -lcrypto -lfmt
+RUNPATHS = $(RUNPATH_OPENSSL) $(RUNPATH_FMTLIB)
+
+CC_FLAGS = -Wno-unused-result -Wsign-compare
+
+cython: main.exe
+
+run: main.exe
+	./main.exe 2>/dev/null
+
+%.cpp: %.pyx
+	@echo "[Cython Compiling $^ -> $@]"
+	-@python3 -c "from Cython.Compiler.Main import main; main(command_line=1)" $^ --cplus -3
+
+%.exe: %.cpp
+	@echo "[C++ Compiling $^ -> $@]"
+	g++ $^ $(CC_FLAGS) -pthread $(INCLUDE_PATHS) $(LIBRARY_PATHS) $(LIBRARIES) $(RUNPATHS) -Wl,--unresolved-symbols=ignore-all -o $@
+
+python: setup.py
+	-python3 setup.py build_ext --inplace
+
+pythonrun:
+	python3 -c "import main; main.python_main()" 2>/dev/null
+
+clean:
+	rm -f *.c *.cpp *.html
+	rm -f *.h
+	rm -f *.so
+	rm -f *.exe
+	rm -f *.o
+	rm -f -r build
+	rm -f *.json
+
+.PHONY: cython run python pythonrun clean
+.PRECIOUS: %.cpp

cython/main.pyx

+# distutils: language = c++
+
+from libcythonplus.list cimport cyplist
+
+from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, printf, ferror
+
+from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
+
+from stdlib.stat cimport Stat, dev_t
+from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
+from stdlib.fmt cimport sprintf
+from stdlib.string cimport string
+from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
+
+from posix.unistd cimport readlink
+
+
+cdef Scheduler scheduler
+
+
+cdef cypclass Node nolock activable:
+    string path
+    string name
+    Stat st
+    string formatted
+
+    __init__(self, string path, string name, Stat st):
+        self._active_result_class = NullResult
+        self._active_queue_class = BatchMailBox(scheduler)
+        self.path = path
+        self.name = name
+        self.st = st
+
+    void build_node(self, const cyplist[dev_t] dev_whitelist, const cyplist[string] ignore_paths):
+        # abstract
+        pass
+
+    void format_node(self):
+        self.formatted = sprintf("""\
+  {
+    "%s": {
+      "stat": %s
+    }
+  },
+""",
+            self.path,
+            self.st.to_json(),
+        )
+
+    void write_node(self, FILE * stream):
+        # abstract
+        pass
+
+
+cdef Node make_node(string path, string name) nogil:
+    s = Stat(path)
+    if s is NULL:
+        return NULL
+    elif s.is_symlink():
+        return SymlinkNode(path, name, s)
+    elif s.is_dir():
+        return DirNode(path, name, s)
+    elif s.is_regular():
+        return FileNode(path, name, s)
+    return NULL
+
+
+cdef cypclass DirNode(Node) nolock:
+    cyplist[Node] children
+
+    __init__(self, string path, string name, Stat st):
+        Node.__init__(self, path, name, st)
+        self.children = cyplist[Node]()
+
+    void build_node(self, const cyplist[dev_t] dev_whitelist, const cyplist[string] ignore_paths):
+        cdef DIR *d
+        cdef struct_dirent *entry
+        cdef string entry_name
+        cdef string entry_path
+
+        if ignore_paths is not NULL:
+            if self.path in ignore_paths:
+                return
+
+        if dev_whitelist is not NULL:
+            if self.st is NULL:
+                return
+            elif not self.st.st_data.st_dev in dev_whitelist:
+                return
+
+        d = opendir(self.path.c_str())
+        if d is not NULL:
+            while 1:
+                entry = readdir(d)
+                if entry is NULL:
+                    break
+                entry_name = entry.d_name
+                if entry_name == b'.' or entry_name == b'..':
+                    continue
+                entry_path = self.path
+                if entry_path != b'/':
+                    entry_path += b'/'
+                entry_path += entry_name
+                entry_node = make_node(entry_path, entry_name)
+                if entry_node is NULL:
+                    continue
+                self.children.append(entry_node)
+            closedir(d)
+
+        self.format_node()
+
+        for child in self.children:
+            child.__activate__().build_node(NULL, dev_whitelist, ignore_paths)
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+        for child in self.children:
+            child.write_node(stream)
+
+
+cdef enum:
+    BUFSIZE = 64 * 1024
+
+
+cdef cypclass FileNode(Node) nolock:
+    string md5_data
+    string sha1_data
+    string sha256_data
+    string sha512_data
+    bint error
+
+    __init__(self, string path, string name, Stat st):
+        Node.__init__(self, path, name, st)
+        self.error = False
+
+    void build_node(self, const cyplist[dev_t] dev_whitelist, const cyplist[string] ignore_paths):
+        cdef unsigned char buffer[BUFSIZE]
+        cdef bint eof = False
+        cdef bint md5_ok
+        cdef bint sha1_ok
+        cdef bint sha256_ok
+        cdef bint sha512_ok
+
+        cdef FILE * file = fopen(self.path.c_str(), 'rb')
+
+        if file is NULL:
+            self.error = True
+            self.format_node()
+            return
+
+        md5 = MessageDigest(md5sum())
+        sha1 = MessageDigest(sha1sum())
+        sha256 = MessageDigest(sha256sum())
+        sha512 = MessageDigest(sha512sum())
+
+        md5_ok = md5 is not NULL
+        sha1_ok = sha1 is not NULL
+        sha256_ok = sha256 is not NULL
+        sha512_ok = sha512 is not NULL
+
+        while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
+            size = fread(buffer, 1, BUFSIZE, file)
+            if size != BUFSIZE:
+                self.error = ferror(file)
+                if self.error:
+                    break
+                eof = True
+
+            if md5_ok: md5_ok = md5.update(buffer, size) == 0
+            if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
+            if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
+            if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
+
+        fclose(file)
+
+        if not self.error:
+            if md5_ok: self.md5_data = md5.hexdigest()
+            if sha1_ok: self.sha1_data = sha1.hexdigest()
+            if sha256_ok: self.sha256_data = sha256.hexdigest()
+            if sha512_ok: self.sha512_data = sha512.hexdigest()
+
+        self.format_node()
+
+    void format_node(self):
+        if self.error:
+            Node.format_node(self)
+        else:
+            self.formatted = sprintf("""\
+  {
+    "%s": {
+      "stat": %s,
+      "digests": {
+        "md5": "%s",
+        "sha1": "%s",
+        "sha256": "%s",
+        "sha512": "%s"
+      }
+    }
+  },
+""",
+                self.path,
+                self.st.to_json(),
+                self.md5_data,
+                self.sha1_data,
+                self.sha256_data,
+                self.sha512_data,
+            )
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+
+
+cdef cypclass SymlinkNode(Node) nolock:
+    string target
+    int error
+
+    void build_node(self, const cyplist[dev_t] dev_whitelist, const cyplist[string] ignore_paths):
+        size = self.st.st_data.st_size + 1
+        self.target.resize(size)
+        real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
+        self.error = not (0 < real_size < size)
+        self.target.resize(real_size)
+        self.format_node()
+
+    void format_node(self):
+        if self.error:
+            Node.format_node(self)
+        else:
+            self.formatted = sprintf("""\
+  {
+    "%s": {
+      "stat": %s,
+      "target": "%s"
+    }
+  },
+""",
+            self.path,
+            self.st.to_json(),
+            self.target,
+        )
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+
+
+cdef int start(string path) nogil:
+    global scheduler
+    scheduler = Scheduler()
+
+    ignore_paths = cyplist[string]()
+    ignore_paths.append(b'/opt/slapgrid')
+    ignore_paths.append(b'/srv/slapgrid')
+
+    dev_whitelist_paths = cyplist[string]()
+    dev_whitelist_paths.append(b'/')
+    dev_whitelist_paths.append(b'/boot')
+
+    dev_whitelist = cyplist[dev_t]()
+    for p in dev_whitelist_paths:
+        p_stat = Stat(p)
+        if p_stat is not NULL:
+            p_dev = p_stat.st_data.st_dev
+            dev_whitelist.append(p_dev)
+
+    node = make_node(path, path)
+    if node is NULL:
+        return -1
+
+    node.__activate__().build_node(NULL, dev_whitelist, ignore_paths)
+
+    scheduler.finish()
+
+    result = fopen('result.json', 'w')
+    if result is NULL:
+        return -1
+
+    fprintf(result, '[\n')
+    node.write_node(result)
+    fprintf(result, '  {}\n]\n')
+
+    fclose(result)
+
+    del scheduler
+
+    return 0
+
+cdef public int main() nogil:
+    return start(<char*>'/')
+
+def python_main():
+    start(<char*>'/')

cython/runtime/pthreads.pxd

+cdef extern from "<sys/types.h>" nogil:
+    ctypedef long unsigned int pthread_t
+
+    ctypedef union pthread_attr_t:
+        pass
+    ctypedef union pthread_mutex_t:
+        pass
+    ctypedef union pthread_mutexattr_t:
+        pass
+    ctypedef union pthread_barrier_t:
+        pass
+    ctypedef union pthread_barrierattr_t:
+        pass
+    ctypedef union pthread_cond_t:
+        pass
+    ctypedef union pthread_condattr_t:
+        pass
+
+cdef extern from "<pthread.h>" nogil:
+    int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), void *)
+    void pthread_exit(void *)
+    int pthread_join(pthread_t, void **)
+    int pthread_cancel(pthread_t thread)
+    int pthread_attr_init(pthread_attr_t *)
+    int pthread_attr_setdetachstate(pthread_attr_t *, int)
+    int pthread_attr_destroy(pthread_attr_t *)
+
+    int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *)
+    int pthread_mutex_destroy(pthread_mutex_t *)
+    int pthread_mutex_lock(pthread_mutex_t *)
+    int pthread_mutex_unlock(pthread_mutex_t *)
+    int pthread_mutex_trylock(pthread_mutex_t *)
+
+    int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, unsigned int)
+    int pthread_barrier_destroy(pthread_barrier_t *)
+    int pthread_barrier_wait(pthread_barrier_t *)
+
+    int pthread_cond_init(pthread_cond_t * cond, const pthread_condattr_t * attr)
+    int pthread_cond_destroy(pthread_cond_t *cond)
+    int pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
+    int pthread_cond_broadcast(pthread_cond_t *cond)
+    int pthread_cond_signal(pthread_cond_t *cond)
+
+    enum: PTHREAD_CREATE_JOINABLE
\ No newline at end of file

cython/runtime/runtime.pxd

+# distutils: language = c++
+
+from libcpp.deque cimport deque
+from libcpp.vector cimport vector
+from libcpp.atomic cimport atomic
+from libc.stdio cimport printf
+from libc.stdlib cimport rand
+from posix.unistd cimport sysconf
+from runtime.pthreads cimport *
+from runtime.semaphore cimport *
+
+cdef extern from "<unistd.h>" nogil:
+    enum: _SC_NPROCESSORS_ONLN # Seems to not be included in "posix.unistd".
+
+
+cdef cypclass Scheduler nolock
+cdef cypclass Worker nolock
+
+# The 'inline' qualifier on this function is a hack to convince Cython to allow a definition in a .pxd file.
+# The C compiler will dismiss it because we pass the function pointer to create a thread which prevents inlining.
+cdef inline void * worker_function(void * arg) nogil:
+    worker = <Worker> arg
+    sch = worker.scheduler
+    cdef int num_remaining_queues
+    # Wait until all the workers are ready.
+    pthread_barrier_wait(&sch.barrier)
+    while 1:
+        # Wait until a queue becomes available.
+        sem_wait(&sch.num_free_queues)
+        # If the scheduler is done there is nothing to do anymore.
+        if sch.is_done:
+            return <void*> 0
+        # Pop or steal a queue.
+        queue = worker.get_queue()
+        with wlocked queue:
+            # Do one task on the queue.
+            queue.activate()
+            if queue.is_empty():
+                # Mark the empty queue as not assigned to any worker.
+                del queue.worker
+                # Decrement the number of non-completed queues.
+                if sch.num_pending_queues.fetch_sub(1) == 1:
+                    # Signal that there are no more queues.
+                    sem_post(&sch.done)
+                # Discard the empty queue and continue the main loop.
+                continue
+        pthread_mutex_lock(&worker.lock)
+        # The queue is not empty: reinsert it in this worker's queues.
+        worker.queues.push_back(queue)
+        # Signal that the queue is available.
+        sem_post(&sch.num_free_queues)
+        pthread_mutex_lock(&worker.lock)
+
+
+cdef cypclass Worker nolock:
+    deque[SequentialMailBox] queues
+    Scheduler scheduler
+    pthread_t thread
+    pthread_mutex_t lock
+
+    Worker __new__(alloc, Scheduler scheduler):
+        instance = alloc()
+        instance.scheduler = scheduler
+        pthread_mutex_init(&instance.lock, NULL)
+        if not pthread_create(&instance.thread, NULL, worker_function, <void *> instance):
+            return instance
+        printf("pthread_create() failed\n")
+
+    __dealloc__(self):
+        pthread_mutex_destroy(&self.lock)
+
+    SequentialMailBox get_queue(self):
+        # Get the next queue in the worker's list or steal one.
+        pthread_mutex_lock(&self.lock)
+        if not self.queues.empty():
+            queue = self.queues.front()
+            self.queues.pop_front()
+            pthread_mutex_unlock(&self.lock)
+            return queue
+        pthread_mutex_unlock(&self.lock)
+        return self.steal_queue()
+
+    SequentialMailBox steal_queue(self):
+        # Steal a queue from another worker:
+        # - inspect each worker in order starting at a random offset
+        # - skip this worker and any worker with an empty queue list
+        # - return the last queue of the first worker with a non-empty list
+        cdef int i, index, num_workers, random_offset
+        sch = self.scheduler
+        num_workers = <int> sch.workers.size()
+        random_offset = rand() % num_workers
+        for i in range(num_workers):
+            index = (i + random_offset) % num_workers
+            victim = sch.workers[index]
+            if victim is self:
+                continue
+            pthread_mutex_lock(&victim.lock)
+            if not victim.queues.empty():
+                stolen_queue = victim.queues.back()
+                victim.queues.pop_back()
+                stolen_queue.worker = self
+                pthread_mutex_unlock(&victim.lock)
+                return stolen_queue
+            pthread_mutex_unlock(&victim.lock)
+        return NULL
+
+    int join(self):
+        # Join the worker thread.
+        return pthread_join(self.thread, NULL)
+
+
+cdef cypclass Scheduler nolock:
+    vector[Worker] workers
+    pthread_barrier_t barrier
+    sem_t num_free_queues
+    atomic[int] num_pending_queues
+    sem_t done
+    volatile bint is_done
+
+    __init__(self, int num_workers=0):
+        if num_workers == 0: num_workers = sysconf(_SC_NPROCESSORS_ONLN)
+        sem_init(&self.num_free_queues, 0, 0)
+        sem_init(&self.done, 0, 0)
+        self.num_pending_queues.store(0)
+        if pthread_barrier_init(&self.barrier, NULL, num_workers + 1):
+            printf("Could not allocate memory for the thread barrier\n")
+            # Signal that no work will be done.
+            sem_post(&self.done)
+            return
+        self.is_done = False
+        self.workers.reserve(num_workers)
+        for i in range(num_workers):
+            worker = Worker(self)
+            if worker is NULL:
+                # Signal that no work will be done.
+                sem_post(&self.done)
+                return
+            self.workers.push_back(worker)
+        # Wait until all the worker threads are ready.
+        pthread_barrier_wait(&self.barrier)
+
+    __dealloc__(self):
+        pthread_barrier_destroy(&self.barrier)
+        sem_destroy(&self.num_free_queues)
+        sem_destroy(&self.done)
+
+    void post_queue(self, SequentialMailBox queue):
+        # Add a queue to the first worker.
+        main_worker = self.workers[0]
+        pthread_mutex_lock(&main_worker.lock)
+        queue.worker = main_worker
+        main_worker.queues.push_back(queue)
+        pthread_mutex_unlock(&main_worker.lock)
+        # Increment the number of non-completed queues.
+        self.num_pending_queues.fetch_add(1)
+        # Signal that a queue is available.
+        sem_post(&self.num_free_queues)
+
+    void finish(self):
+        # Wait until there is no more work.
+        sem_wait(&self.done)
+        # Signal the worker threads that there is no more work.
+        self.is_done = True
+        # Pretend that there are new queues to wake up the workers.
+        for worker in self.workers:
+            sem_post(&self.num_free_queues)
+        # Clear the workers to break reference cycles.
+        self.workers.clear()
+
+
+cdef cypclass SequentialMailBox(ActhonQueueInterface):
+    deque[ActhonMessageInterface] messages
+    Scheduler scheduler
+    Worker worker
+
+    __init__(self, Scheduler scheduler):
+        self.scheduler = scheduler
+        self.worker = NULL
+
+    bint is_empty(const self):
+        return self.messages.empty()
+
+    void push(self, ActhonMessageInterface message):
+        # Add a task to the queue.
+        self.messages.push_back(message)
+        if message._sync_method is not NULL:
+            message._sync_method.insertActivity(message)
+        # If not worker is already assigned this queue
+        # register it with the scheduler.
+        if self.worker is NULL:
+            self.scheduler.post_queue(self)
+
+    bint activate(self):
+        # Try to process the first message in the queue.
+        cdef bint one_message_processed
+        if self.messages.empty():
+            return False
+        next_message = self.messages.front()
+        self.messages.pop_front()
+        one_message_processed = next_message.activate()
+        if one_message_processed:
+            if next_message._sync_method is not NULL:
+                next_message._sync_method.removeActivity(next_message)
+        else:
+            printf("Pushed front message to back :/\n")
+            self.messages.push_back(next_message)
+        return one_message_processed
+
+
+cdef cypclass BatchMailBox(SequentialMailBox):
+    bint activate(self):
+        # Process as many messages as possible.
+        while not self.messages.empty():
+            next_message = self.messages.front()
+            self.messages.pop_front()
+            if not next_message.activate():
+                printf("Pushed front message to back :/\n")
+                self.messages.push_back(next_message)
+                return False
+            if next_message._sync_method is not NULL:
+                next_message._sync_method.removeActivity(next_message)
+        return True
+
+
+cdef inline ActhonResultInterface NullResult() nogil:
+    return NULL

cython/runtime/semaphore.pxd

+cdef extern from "<semaphore.h>" nogil:
+    ctypedef struct sem_t:
+        pass
+    int sem_init(sem_t *sem, int pshared, unsigned int value)
+    int sem_wait(sem_t *sem)
+    int sem_post(sem_t *sem)
+    int sem_getvalue(sem_t *, int *)
+    int sem_destroy(sem_t* sem)

cython/setup.py

+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Build import cythonize
+
+extensions = [
+    Extension(
+        "main",
+        language="c++",
+        sources=["main.pyx"],
+        include_dirs = [
+            "/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/include",
+            "/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/include"
+        ],
+        library_dirs = [
+            "/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/lib",
+            "/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/lib"
+        ],
+        libraries = ["crypto", "fmt"],
+        extra_link_args=[
+            "-Wl,-rpath=/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/lib",
+            "-Wl,-rpath=/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/lib"
+        ], 
+    ),
+]
+
+setup(
+  ext_modules = cythonize(extensions)
+)

cython/stdlib/digest.pxd

+# distutils: language = c++
+
+from stdlib.string cimport string
+
+
+cdef extern from "<openssl/evp.h>" nogil:
+    ctypedef struct EVP_MD_CTX:
+        pass
+    ctypedef struct EVP_MD
+    ctypedef struct ENGINE
+
+    void EVP_MD_CTX_init(EVP_MD_CTX *ctx)
+    EVP_MD_CTX *EVP_MD_CTX_create()
+
+    int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl)
+    int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt)
+    int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s)
+
+    int EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx)
+    void EVP_MD_CTX_destroy(EVP_MD_CTX *ctx)
+
+    int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *_in)
+
+    int EVP_DigestInit(EVP_MD_CTX *ctx, const EVP_MD *type)
+    int EVP_DigestFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s)
+
+    int EVP_MD_CTX_copy(EVP_MD_CTX *out,EVP_MD_CTX *_in)
+
+    # Max size
+    const int EVP_MAX_MD_SIZE
+
+    # Algorithms
+    const EVP_MD *md5sum "EVP_md5"()
+    const EVP_MD *sha1sum "EVP_sha1"()
+    const EVP_MD *sha256sum "EVP_sha256"()
+    const EVP_MD *sha512sum "EVP_sha512"()
+
+    const EVP_MD *EVP_get_digestbyname(const char *name)
+
+
+cdef extern from * nogil:
+    '''
+    static const char hexdigits [] = "0123456789abcdef";
+    '''
+    cdef const char hexdigits[]
+
+
+cdef cypclass MessageDigest nolock:
+    EVP_MD_CTX * md_ctx
+
+    MessageDigest __new__(alloc, const EVP_MD * algo):
+        md_ctx = EVP_MD_CTX_create()
+        if md_ctx is NULL:
+            return NULL
+        if EVP_DigestInit_ex(md_ctx, algo, NULL) != 1:
+            return NULL
+        instance = alloc()
+        instance.md_ctx = md_ctx
+        return instance
+
+    __dealloc__(self):
+        EVP_MD_CTX_destroy(md_ctx)
+
+    int update(self, unsigned char * message, size_t size):
+        return EVP_DigestUpdate(self.md_ctx, message, size) - 1
+
+    string hexdigest(self):
+        cdef char result[EVP_MAX_MD_SIZE*2]
+        cdef unsigned char hashbuffer[EVP_MAX_MD_SIZE]
+        cdef unsigned int size
+        cdef unsigned int i
+        if EVP_DigestFinal_ex(self.md_ctx, hashbuffer, &size) == 1:
+            for i in range(size):
+                result[2*i] = hexdigits[hashbuffer[i] >> 4]
+                result[2*i+1] = hexdigits[hashbuffer[i] & 15]
+            return string(result, 2*size)

cython/stdlib/dirent.pxd

+from posix.types cimport ino_t
+
+cdef extern from "<sys/types.h>" nogil:
+    ctypedef struct DIR
+
+cdef extern from "<dirent.h>" nogil:
+    cdef struct struct_dirent "dirent":
+        ino_t           d_ino
+        char            d_name[256]
+
+    DIR *opendir(const char *name)
+    struct_dirent *readdir(DIR *dirp)
+    int readdir_r(DIR *dirp, struct_dirent *entry, struct_dirent **result)
+    int closedir(DIR *dirp)

cython/stdlib/fmt.pxd

+from stdlib.string cimport string
+
+cdef extern from "<fmt/printf.h>" namespace "fmt" nogil:
+
+    ctypedef struct FILE
+
+    int printf       (const char* template, ...)
+    int fprintf      (FILE *stream, const char* template, ...)
+    string sprintf   (const char* template, ...)
+

cython/stdlib/stat.pxd

+# distutils: language = c++
+
+# Differences with posix.stat:
+#
+# - the declaration for the non-standard field st_birthtime was removed
+#   because cypclass wrapping triggers the generation of a conversion
+#   function for the stat structure which references this field.
+#
+# - the absent declaration in posix.time of struct timespec was added.
+#
+# - the declarations for the time_t fields st_atime, st_mtime, st_ctime
+#   were replaced by the fields st_atim, st_mtim, st_ctim
+#   of type struct timespec.
+
+from stdlib.string cimport string
+from stdlib.fmt cimport sprintf
+
+from posix.types cimport (blkcnt_t, blksize_t, dev_t, gid_t, ino_t, mode_t,
+                          nlink_t, off_t, time_t, uid_t)
+
+cdef extern from "<sys/time.h>" nogil:
+    cdef struct struct_timespec "timespec":
+        time_t tv_sec
+        long int tv_nsec
+
+
+cdef extern from "<sys/stat.h>" nogil:
+    cdef struct struct_stat "stat":
+        dev_t   st_dev
+        ino_t   st_ino
+        mode_t  st_mode
+        nlink_t st_nlink
+        uid_t   st_uid
+        gid_t   st_gid
+        dev_t   st_rdev
+        off_t   st_size
+        blksize_t st_blksize
+        blkcnt_t st_blocks
+        struct_timespec  st_atim
+        struct_timespec  st_mtim
+        struct_timespec  st_ctim
+
+# POSIX prescribes including both <sys/stat.h> and <unistd.h> for these
+cdef extern from "<unistd.h>" nogil:
+    int fchmod(int, mode_t)
+    int chmod(const char *, mode_t)
+
+    int fstat(int, struct_stat *)
+    int lstat(const char *, struct_stat *)
+    int stat(const char *, struct_stat *)
+
+    # Macros for st_mode
+    mode_t S_ISREG(mode_t)
+    mode_t S_ISDIR(mode_t)
+    mode_t S_ISCHR(mode_t)
+    mode_t S_ISBLK(mode_t)
+    mode_t S_ISFIFO(mode_t)
+    mode_t S_ISLNK(mode_t)
+    mode_t S_ISSOCK(mode_t)
+
+    mode_t S_IFMT
+    mode_t S_IFREG
+    mode_t S_IFDIR
+    mode_t S_IFCHR
+    mode_t S_IFBLK
+    mode_t S_IFIFO
+    mode_t S_IFLNK
+    mode_t S_IFSOCK
+
+    # Permissions
+    mode_t S_ISUID
+    mode_t S_ISGID
+    mode_t S_ISVTX
+
+    mode_t S_IRWXU
+    mode_t S_IRUSR
+    mode_t S_IWUSR
+    mode_t S_IXUSR
+
+    mode_t S_IRWXG
+    mode_t S_IRGRP
+    mode_t S_IWGRP
+    mode_t S_IXGRP
+
+    mode_t S_IRWXO
+    mode_t S_IROTH
+    mode_t S_IWOTH
+    mode_t S_IXOTH
+
+
+# Cypclass to expose minimal stat support.
+
+cdef cypclass Stat nolock:
+    struct_stat st_data
+
+    Stat __new__(alloc, string path):
+        instance = alloc()
+        if not lstat(path.c_str(), &instance.st_data):
+            return instance
+
+    bint is_regular(self):
+        return S_ISREG(self.st_data.st_mode)
+
+    bint is_symlink(self):
+        return S_ISLNK(self.st_data.st_mode)
+
+    bint is_dir(self):
+        return S_ISDIR(self.st_data.st_mode)
+
+    string to_json(self):
+        return sprintf("""{
+        "st_dev": %lu,
+        "st_ino": %lu,
+        "st_mode": %lu,
+        "st_nlink": %lu,
+        "st_uid": %d,
+        "st_gid": %d,
+        "st_rdev": %lu,
+        "st_size": %ld,
+        "st_blksize": %ld,
+        "st_blocks": %ld,
+        "st_atime": %ld,
+        "st_mtime": %ld,
+        "st_ctime": %ld,
+        "st_atime_ns": %ld,
+        "st_mtime_ns": %ld,
+        "st_ctime_ns": %ld
+      }""",
+            self.st_data.st_dev,
+            self.st_data.st_ino,
+            self.st_data.st_mode,
+            self.st_data.st_nlink,
+            self.st_data.st_uid,
+            self.st_data.st_gid,
+            self.st_data.st_rdev,
+            self.st_data.st_size,
+            self.st_data.st_blksize,
+            self.st_data.st_blocks,
+            self.st_data.st_atim.tv_sec,
+            self.st_data.st_mtim.tv_sec,
+            self.st_data.st_ctim.tv_sec,
+            self.st_data.st_atim.tv_nsec,
+            self.st_data.st_mtim.tv_nsec,
+            self.st_data.st_ctim.tv_nsec,
+        )

cython/stdlib/string.pxd

+# Differences with libcpp.string:
+#
+# - declarations for operator+= have been added.
+
+
+cdef extern from "<string>" namespace "std" nogil:
+
+    size_t npos = -1
+
+    cdef cppclass string:
+        cppclass iterator:
+            iterator()
+            char& operator*()
+            iterator(iterator &)
+            iterator operator++()
+            iterator operator--()
+            bint operator==(iterator)
+            bint operator!=(iterator)
+        cppclass reverse_iterator:
+            char& operator*()
+            iterator operator++()
+            iterator operator--()
+            iterator operator+(size_t)
+            iterator operator-(size_t)
+            bint operator==(reverse_iterator)
+            bint operator!=(reverse_iterator)
+            bint operator<(reverse_iterator)
+            bint operator>(reverse_iterator)
+            bint operator<=(reverse_iterator)
+            bint operator>=(reverse_iterator)
+        cppclass const_iterator(iterator):
+            pass
+        cppclass const_reverse_iterator(reverse_iterator):
+            pass
+
+        string() except +
+        string(const char *) except +
+        string(const char *, size_t) except +
+        string(const string&) except +
+        # as a string formed by a repetition of character c, n times.
+        string(size_t, char) except +
+        # from a pair of iterators
+        string(iterator first, iterator last) except +
+
+        iterator begin()
+        const_iterator const_begin "begin"()
+        iterator end()
+        const_iterator const_end "end"()
+        reverse_iterator rbegin()
+        const_reverse_iterator const_rbegin "rbegin"()
+        reverse_iterator rend()
+        const_reverse_iterator const_rend "rend"()
+
+        const char* c_str()
+        const char* data()
+        size_t size()
+        size_t max_size()
+        size_t length()
+        void resize(size_t)
+        void resize(size_t, char c)
+        size_t capacity()
+        void reserve(size_t)
+        void clear()
+        bint empty()
+        iterator erase(iterator position)
+        iterator erase(const_iterator position)
+        iterator erase(iterator first, iterator last)
+        iterator erase(const_iterator first, const_iterator last)
+
+        char& at(size_t)
+        char& operator[](size_t)
+        char& front()  # C++11
+        char& back()   # C++11
+        int compare(const string&)
+
+        string& operator+=(const string&)
+        string& operator+=(char)
+        string& operator+=(const char *)
+
+        string& append(const string&)
+        string& append(const string&, size_t, size_t)
+        string& append(const char *)
+        string& append(const char *, size_t)
+        string& append(size_t, char)
+
+        void push_back(char c)
+
+        string& assign (const string&)
+        string& assign (const string&, size_t, size_t)
+        string& assign (const char *, size_t)
+        string& assign (const char *)
+        string& assign (size_t n, char c)
+
+        string& insert(size_t, const string&)
+        string& insert(size_t, const string&, size_t, size_t)
+        string& insert(size_t, const char* s, size_t)
+
+
+        string& insert(size_t, const char* s)
+        string& insert(size_t, size_t, char c)
+
+        size_t copy(char *, size_t, size_t)
+
+        size_t find(const string&)
+        size_t find(const string&, size_t)
+        size_t find(const char*, size_t pos, size_t)
+        size_t find(const char*, size_t pos)
+        size_t find(char, size_t pos)
+
+        size_t rfind(const string&, size_t)
+        size_t rfind(const char* s, size_t, size_t)
+        size_t rfind(const char*, size_t pos)
+        size_t rfind(char c, size_t)
+        size_t rfind(char c)
+
+        size_t find_first_of(const string&, size_t)
+        size_t find_first_of(const char* s, size_t, size_t)
+        size_t find_first_of(const char*, size_t pos)
+        size_t find_first_of(char c, size_t)
+        size_t find_first_of(char c)
+
+        size_t find_first_not_of(const string&, size_t)
+        size_t find_first_not_of(const char* s, size_t, size_t)
+        size_t find_first_not_of(const char*, size_t pos)
+        size_t find_first_not_of(char c, size_t)
+        size_t find_first_not_of(char c)
+
+        size_t find_last_of(const string&, size_t)
+        size_t find_last_of(const char* s, size_t, size_t)
+        size_t find_last_of(const char*, size_t pos)
+        size_t find_last_of(char c, size_t)
+        size_t find_last_of(char c)
+
+        size_t find_last_not_of(const string&, size_t)
+        size_t find_last_not_of(const char* s, size_t, size_t)
+        size_t find_last_not_of(const char*, size_t pos)
+
+        string substr(size_t, size_t)
+        string substr()
+        string substr(size_t)
+
+        size_t find_last_not_of(char c, size_t)
+        size_t find_last_not_of(char c)
+
+        #string& operator= (const string&)
+        #string& operator= (const char*)
+        #string& operator= (char)
+
+        string operator+ (const string& rhs)
+        string operator+ (const char* rhs)
+
+        bint operator==(const string&)
+        bint operator==(const char*)
+
+        bint operator!= (const string& rhs )
+        bint operator!= (const char* )
+
+        bint operator< (const string&)
+        bint operator< (const char*)
+
+        bint operator> (const string&)
+        bint operator> (const char*)
+
+        bint operator<= (const string&)
+        bint operator<= (const char*)
+
+        bint operator>= (const string&)
+        bint operator>= (const char*)