X Transition to all VMA under 1 fileh to be either all based on wcfs or all based on !wcfs

This allows to make decision to drop a page right after writeout for
wcfs case. If vmas for one fileh were allowed to be mixed - we could not
know whether to drop the page & its memory from RAM or not - even if
there is no vma mapped in !wcfs mode, it could be created later, and by
dropping RAM cahce we hit severe slowness. NOTE: in between transactions
there is usually no vmas mapped, so this case is not artificial.

By requiring that all VMAs are of the same kind under one fileh we avoid
this ambiguity.

bigfile/virtmem.c

@@ -167,6 +167,9 @@ int fileh_open(BigFileH *fileh, BigFile *file, RAM *ram)
     fileh->writeout_inprogress = 0;
     pagemap_init(&fileh->pagemap, ilog2_exact(ram->pagesize));
 
+    // XXX hardcoded - allow user choice?
+    fileh->mmap_overlay = (file->file_ops->mmap_setup_read != NULL);
+
 out:
     virt_unlock();
     sigsegv_restore(&save_sigset);
@@ -243,10 +246,7 @@ int fileh_mmap(VMA *vma, BigFileH *fileh, pgoff_t pgoffset, pgoff_t pglen)
     if (!vma->page_ismappedv)
         goto fail;
 
-    // XXX hardcoded - allow user choice?
-    vma->mmap_overlay = (fops->mmap_setup_read != NULL);
-
-    if (vma->mmap_overlay) {
+    if (fileh->mmap_overlay) {
         /* wcfs: mmap(base, READ) */
         TODO (file->blksize != fileh->ramh->ram->pagesize);
         addr = fops->mmap_setup_read(file, pgoffset, pglen, vma);
@@ -262,7 +262,7 @@ int fileh_mmap(VMA *vma, BigFileH *fileh, pgoff_t pgoffset, pgoff_t pglen)
     vma->addr_stop   = vma->addr_start + len;
 
     /* wcfs: mmap(fileh->dirty_pages) over base */
-    if (vma->mmap_overlay) {
+    if (fileh->mmap_overlay) {
         Page* page;
         struct list_head *hpage;
 
@@ -448,7 +448,7 @@ int fileh_dirty_writeout(BigFileH *fileh, enum WriteoutFlags flags)
 
             list_for_each(hmmap, &fileh->mmaps) {
                 VMA *vma = list_entry(hmmap, typeof(*vma), same_fileh);
-                if (vma->mmap_overlay) {
+                if (fileh->mmap_overlay) {
                     /* wcfs:  RW -> base layer */
                     vma_page_ensure_unmapped(vma, page);
                 } else {
@@ -457,20 +457,16 @@ int fileh_dirty_writeout(BigFileH *fileh, enum WriteoutFlags flags)
                 }
             }
 
-            // XXX just relying on page->refcnt==0 is wrong - for !wcfs case
-            // when there are no mappings it will unnecessarily drop fileh cache.
-            // -> move mmap_overlay from vma to fileh and check:
-            // if (fileh->mmap_overlay) {
-            //  ASSERT(page->refcnt == 0);
-            //  page_drop(page);
-            // }
-#if 0
-            /* if noone is using the page (ex all vmas use base layer) - drop
-             * it completely without unnecessarily growing RSS and relying on reclaim. */
-            if (page->refcnt == 0) {
-
+            /* wcfs:  all vmas are using base layer now - drop page completely
+             *        without unnecessarily growing RSS and relying on reclaim.
+             * !wcfs: keep the page in RAM cache, even if it is not mapped anywhere */
+            if (fileh->mmap_overlay) {
+                ASSERT(page->refcnt == 0);
+                // XXX -> page_drop(page)
+                pagemap_del(&fileh->pagemap, page->f_pgoffset);
+                page_drop_memory(page);
+                page_del(page);
             }
-#endif
         }
     }
 
@@ -678,14 +674,14 @@ VMFaultResult vma_on_pagefault(VMA *vma, uintptr_t addr, int write)
 
     /* continuing on_pagefault() - see (1) there ... */
 
-    /* wcfs: we should get into SIGSEGV handler only on write access */
-    if (vma->mmap_overlay)
-        BUG_ON(!write);
-
     /* (2) vma, addr -> fileh, pagen    ;idx of fileh page covering addr */
     fileh = vma->fileh;
     pagen = vma_addr_fpgoffset(vma, addr);
 
+    /* wcfs: we should get into SIGSEGV handler only on write access */
+    if (fileh->mmap_overlay)
+        BUG_ON(!write);
+
     /* (3) fileh, pagen -> page  (via pagemap) */
     page = pagemap_get(&fileh->pagemap, pagen);
 
@@ -698,7 +694,7 @@ VMFaultResult vma_on_pagefault(VMA *vma, uintptr_t addr, int write)
      *   always keep all dirty pages mmapped on fileh_open and so pagefault
      *   handler must not see a PAGE_LOADED page. )
      */
-    if (vma->mmap_overlay && page)
+    if (fileh->mmap_overlay && page)
         ASSERT(page->state == PAGE_LOADED_FOR_WRITE || page->state == PAGE_LOADING);
 
     /* (4) no page found - allocate new from ram */
@@ -781,7 +777,7 @@ VMFaultResult vma_on_pagefault(VMA *vma, uintptr_t addr, int write)
         page->state = PAGE_LOADING;
         virt_unlock();
 
-        if (vma->mmap_overlay) {
+        if (fileh->mmap_overlay) {
             /* wcfs: copy block data from read-only base mmap.
              * NOTE we'll get SIGBUG here if wcfs returns EIO when loading block data */
             memcpy(pageram, vma_page_addr(vma, page), page_size(page));
@@ -868,13 +864,15 @@ VMFaultResult vma_on_pagefault(VMA *vma, uintptr_t addr, int write)
     // XXX mmap page to all vma with .mmap_overlay=1 of this fileh.
 
     vma_mmap_page(vma, page);
-    /* wcfs: mmap the page to all wcfs-backed vmas. If we don't the memory on
+    /* wcfs: mmap the page to all wcfs-backed vmas. If we don't, the memory on
      * those vmas will read with stale data */
+    if (fileh->mmap_overlay) {
         list_for_each(hmmap, &fileh->mmaps) {
             VMA *vma2 = list_entry(hmmap, typeof(*vma2), same_fileh);
-        if (vma2 != vma && vma2->mmap_overlay)
+            if (vma2 != vma)
                 vma_mmap_page(vma2, page);
         }
+    }
 
     /* mark page as used recently */
     // XXX = list_move_tail()
@@ -1097,7 +1095,7 @@ static void vma_page_ensure_unmapped(VMA *vma, Page *page)
     if (!vma_page_ismapped(vma, page))
         return;
 
-    if (vma->mmap_overlay) {
+    if (vma->fileh->mmap_overlay) {
         /* wcfs: remmap readonly to base image */
         BigFile *file = vma->fileh->file;
         int err;

include/wendelin/bigfile/virtmem.h

@@ -74,6 +74,10 @@ struct BigFileH {
 
     /* whether writeout is currently in progress */
     int writeout_inprogress;
+
+    /* whether base data for all VMAs of this fileh are taken as base-layer mmap XXX */
+    // XXX name
+    unsigned    mmap_overlay : 1;
 };
 typedef struct BigFileH BigFileH;
 
@@ -142,8 +146,8 @@ struct VMA {
     /* whether corresponding to pgoffset-f_offset page is mapped in this VMA */
     bitmap      *page_ismappedv;    /* len ~ Δaddr / pagesize */
 
-    // XXX name
-    unsigned    mmap_overlay : 1;   /* whether base data are taken as file mmap XXX */
+//    // XXX name
+//    unsigned    mmap_overlay : 1;   /* whether base data are taken as file mmap XXX */
 };