X Thoughts on how to avoid readahead touching pages of neighbour block

wcfs/internal/mm.pyx

@@ -35,6 +35,13 @@ cpdef enum:
     MCL_FUTURE      = mman.MCL_FUTURE
     MCL_ONFAULT     = mman.MCL_ONFAULT
 
+    # XXX there is no mman.MADV_{NORMAL,RANDOM} -> use POSIX_... as workaround
+    MADV_NORMAL     = mman.POSIX_MADV_NORMAL
+    MADV_RANDOM     = mman.POSIX_MADV_RANDOM
+    MADV_SEQUENTIAL = mman.POSIX_MADV_SEQUENTIAL
+    MADV_WILLNEED   = mman.POSIX_MADV_WILLNEED
+    MADV_DONTNEED   = mman.POSIX_MADV_DONTNEED
+
 # incore returns bytearray vector indicating whether page of mem is in core or not.
 #
 # mem start must be page-aligned.
@@ -96,7 +103,7 @@ def map_ro(int fd, off_t offset, size_t size):
     return <unsigned char[:size:1]>addr
 
 
-# unmap unmaps memory coverd by mem.
+# unmap unmaps memory covered by mem.
 def unmap(const unsigned char[::1] mem not None):
     cdef const void *addr = &mem[0]
     cdef size_t      size = mem.shape[0]
@@ -105,3 +112,16 @@ def unmap(const unsigned char[::1] mem not None):
     if err:
         PyErr_SetFromErrno(OSError)
     # ok
+
+
+# advise advises kernel about use of mem's memory.
+#
+# see madvise(2) for details.
+def advise(const unsigned char[::1] mem not None, int advice):
+    cdef const void *addr = &mem[0]
+    cdef size_t      size = mem.shape[0]
+
+    # XXX using posix_madvise, not madvise
+    cdef err = mman.posix_madvise(<void *>addr, size, advice)
+    if err:
+        PyErr_SetFromErrno(OSError)

wcfs/wcfs_test.py

@@ -428,13 +428,40 @@ class tFile:
         t.f   = tdb._open(zf, at=at)
         t.blksize = zf.blksize
 
-        # mmap the file past the end up to _max_tracked_pages and lock the
-        # pages with MLOCK_ONFAULT. This way when a page is read by mmap access
-        # we have the guarantee from kernel that the page will stay in
-        # pagecache. We rely on this to verify OS cache state.
+        # mmap the file past the end up to _max_tracked_pages and setup
+        # invariants on which we rely to verify OS cache state:
+        #
+        # 1. lock pages with MLOCK_ONFAULT: this way when a page is read by
+        #    mmap access we have the guarantee from kernel that the page will
+        #    stay in pagecache.
+        #
+        # 2. madvise in interleaved mode blocks memory to be either
+        #    MADV_NORMAL or MAD_RANDOM. This adjusts kernel readahead (which
+        #    triggers for MADV_NORMAL memory) to not go over to next block and
+        #    thus a read access to one block won't trigger implicit read access
+        #    to neighbour block.
+        #
+        #      https://www.quora.com/What-heuristics-does-the-adaptive-readahead-implementation-in-the-Linux-kernel-use
+        #      https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/mm/madvise.c?h=v5.2-rc4#n51
+        #
+        #    don't disable readahead universally, since enabled readahead helps
+        #    to test how wcfs handles simultaneous read vs wcfs uploading data
+        #    for the same block into OS cache.
         assert t.blksize % mm.PAGE_SIZE == 0
         t.fmmap = mm.map_ro(t.f.fileno(), 0, t._max_tracked_pages*t.blksize)
         mm.lock(t.fmmap, mm.MLOCK_ONFAULT)
+        for blk in range(t._max_tracked_pages):
+            blkmmap = t.fmmap[blk*t.blksize:(blk+1)*t.blksize]
+            # FIXME somehow does not completely prevent readahead to go into MADV_RANDOM page
+            # NOTE  with MADV_RANDOM the kernel issues 4K sized reads; wcfs
+            # starts uploading into cache almost immediately, but the kernel
+            # still issues many reads to read the full 2MB of the block. This
+            # works slow.
+            # XXX -> make read(while-uploading) wait for uploading to complete
+            # and only then return? (maybe it will help performance even in normal case)
+            mm.advise(blkmmap, (mm.MADV_NORMAL, mm.MADV_RANDOM)[blk%2])
+            #mm.advise(blkmmap, mm.MADV_NORMAL)
+            #mm.advise(blkmmap, mm.MADV_RANDOM)
 
         tdb._files.add(t)
 
@@ -490,6 +517,7 @@ class tFile:
     # pinokByWLink: {} tWatchLink -> {} blk -> at.
     # pinokByWLink can be None - in that case it is computed automatically.
     def assertBlk(t, blk, dataok, pinokByWLink=None):
+        print('assertBlk #%d' % blk)
         if not isinstance(dataok, bytes):
             dataok = dataok.encode('utf-8')
         assert len(dataok) <= t.blksize
@@ -499,7 +527,7 @@ class tFile:
         assert blk < t._sizeinblk()
 
         cached = t.cached()[blk]
-        assert cached in (0, 1) # XXX temp - breaks becuase of kernel readahead
+        assert cached in (0, 1), "blk #%d" % blk # XXX temp - breaks becuase of kernel readahead
         shouldPin = False   # whether at least one wlink should receive a pin
 
         # watches must be notified if access goes to @head/file; not if to @rev/file.   XXX text