Merge branch 'master' into t

* master:
  bigfile: Note that ramh_alloc_page does not add the page to any lists
  bigfile: Unstub ram_close
  bigfile/test/test_ram: Don't forget to free allocated Page structs
  bigfile: RAM must be explicitly free'ed after close
  bigfile/test: Don't forget to close opened RAM
  tests: TSAN no longer fails on test_virtmem
  tox: Don't duplicate setup.py on which for-tests dependencies we need
  3rdparty/ccan: Update

ccan @ ce4660aa

-Subproject commit c0b655441804c0edace4f539fb9008b315eb1bd8
+Subproject commit ce4660aa49404530a608a37754fae16fbb3fde02

Makefile

@@ -76,11 +76,6 @@ test	: test.t test.py test.fault test.asan test.tsan test.vgmem test.vghel test.
 
 # TODO move XFAIL markers into *.c
 
-# TSAN fails on test_virtmem (http://code.google.com/p/thread-sanitizer/issues/detail?id=75)
-# NOTE the bug was fixed in compiler-rt 20140917 (6afe775d)
-#      -> we can remove this xfail when the fix propagates to gcc/clang release
-XFAIL_bigfile/tests/test_virtmem.tsanrun	:= y
-
 # Before calling our SIGSEGV handler, Memcheck first reports "invalid read|write" error.
 # A solution could be to tell memcheck via VALGRIND_MAKE_MEM_DEFINED that VMA
 # address space is ok to access _before_ handling pagefault.

bigfile/ram.c

 /* Wendelin.bigfile | Interfaces to work with RAM
- * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ * Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -87,8 +87,10 @@ RAMH *ramh_open(RAM *ram)
 
 void ram_close(RAM *ram)
 {
-    WARN("TODO ram_close()");   // XXX
-    //return ram->ram_ops->close(ram);
+    // TODO assert that
+    // - there are no ramh open left
+    // - there are no pages on ram->lru_list
+    return ram->ram_ops->close(ram);
 }
 
 

bigfile/ram_shmfs.c

 /* Wendelin.bigfile | shmfs (aka tmpfs) ram backend
- * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ * Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -243,11 +243,12 @@ out:
     return NULL;
 }
 
+static void shmfs_ram_close(RAM *ram0);
 
 static const struct ram_ops shmfs_ram_ops = {
     .get_current_maxsize    = shmfs_get_current_maxsize,
     .ramh_open              = shmfs_ramh_open,
-    //.close    = shmfs_ram_dtor
+    .close                  = shmfs_ram_close,
 };
 
 
@@ -266,8 +267,11 @@ static RAM *shmfs_ram_new(const char *arg)
     return ram;
 };
 
-
-// TODO shmfs_ram_dtor
+static void shmfs_ram_close(RAM *ram0)
+{
+    SHMFS_RAM *ram = upcast(SHMFS_RAM *, ram0);
+    free((void*)ram->prefix); ram->prefix = NULL;
+}
 
 
 static const struct ram_type shmfs_ram_type = {

bigfile/tests/bench_virtmem.c

 /* Wendelin.bigfile | virtual memory benchmarks
- * Copyright (C) 2017  Nexedi SA and Contributors.
- *                     Kirill Smelkov <kirr@nexedi.com>
+ * Copyright (C) 2017-2019  Nexedi SA and Contributors.
+ *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
  * it under the terms of the GNU General Public License version 3, or (at your
@@ -102,7 +102,7 @@ void bench_pagefault() {
     vma_unmap(vma);
     fileh_close(fh);
     ram_close(ram);
-
+    free(ram);
 }
 
 int main()

bigfile/tests/test_ram.c

 /* Wendelin.bigfile | ram tests
- * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ * Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -116,6 +116,10 @@ int main()
     xmunmap(p01, ps);
     xmunmap(p02, ps);
     xmunmap(p03, ps);
+    free(page0);
+    // page1: was not yet allocated
+    free(page2);
+    // page3: was not yet allocated
 
 
     /* ensure we get "no memory" when overallocating (not doing so would lead
@@ -145,8 +149,17 @@ int main()
     // TODO allocate memory amount = 2*ram_maxsize and touch it linearly
 
 
+    xmunmap(p0, ps);
+    xmunmap(p1, ps);
+    xmunmap(p2, ps);
+    free(page0);
+    free(page1);
+    free(page2);
+    ok1(!page3);
 
     ramh_close(ramh);
+    ram_close(ram);
+    free(ram);
 
     return 0;
 }

bigfile/tests/test_virtmem.c

 /* Wendelin.bigfile | virtual memory tests
- * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ * Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -908,8 +908,8 @@ void test_file_access_synthetic(void)
 
 
     /* free resources & restore SIGSEGV handler */
-    ram_close(ram);
-    ram_close(ram0);
+    ram_close(ram);  free(ram);
+    ram_close(ram0); free(ram0);
 
     ok1(!sigaction(SIGSEGV, &saveact, NULL));
 
@@ -1035,6 +1035,7 @@ void test_file_access_pagefault()
     fileh_close(fh);
     // ok1(list_empty(&ram->lru_list));
     ram_close(ram);
+    free(ram);
 }
 
 
@@ -1105,6 +1106,7 @@ void test_pagefault_savestate()
     vma_unmap(vma);
     fileh_close(fh);
     ram_close(ram);
+    free(ram);
 
 #undef  CHECK_PAGE
 #undef  CHECK_NOPAGE

include/wendelin/bigfile/ram.h

@@ -2,7 +2,7 @@
 #define _WENDELIN_BIGFILE_RAM_H_
 
 /* Wendelin.bigfile | Interfaces to work with RAM
- * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ * Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -92,9 +92,10 @@ size_t ram_get_current_maxsize(RAM *ram);
 RAMH *ramh_open(RAM *ram);
 
 
-/* close RAM
+/* ram_close releases resources associated with RAM.
  *
- * TODO text
+ * All RAM handles opened on this RAM must be closed.
+ * NOTE struct RAM itself is not released - it has to be explicitly freed by user.
  */
 void ram_close(RAM *ram);
 
@@ -144,7 +145,10 @@ struct ramh_ops {
  *
  * XXX write on how to free pages (= drop & free(struct Page) ?)
  *
- * NOTE after allocation, page->fileh & page->f_pgoffset are unset
+ * NOTE after allocation:
+ *
+ * - page->fileh & page->f_pgoffset are unset;
+ * - page is not added to ram->lru_list and fileh->dirty_pages lists.
  */
 Page *ramh_alloc_page(RAMH *ramh, pgoff_t pgoffset_hint);
 

t/t_utils.c

-/* Copyright (C) 2014-2015  Nexedi SA and Contributors.
+/* Copyright (C) 2014-2019  Nexedi SA and Contributors.
  *                          Kirill Smelkov <kirr@nexedi.com>
  *
  * This program is free software: you can Use, Study, Modify and Redistribute
@@ -95,8 +95,6 @@ void ram_limited_close(RAM *ram0)
 
     // XXX close if owning?
     // ram_close(ram->backend);
-
-    // TODO free(self) ?
 }
 
 

tox.ini

@@ -11,8 +11,7 @@ envlist = py27-{ZODB3,ZODB4,ZODB5}-{zblk0,zblk1}-{fs,zeo,neo}-{numpy115,numpy116
 
 [testenv]
 deps    =
-    # why tox does not get it from extras_require['test'] ?
-    pytest
+    .[test]
 
     # XXX temp hack
     cython