.

bigfile/file_zodb.py

@@ -18,7 +18,63 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-""" BigFile backed by ZODB objects
+"""Package file_zodb provides BigFile backed by ZODB.
+
+ZBigFile provides BigFile backend with data stored in ZODB. Files data are
+stored in ZODB in blocks with each block stored as separate ZODB object(s).
+This way only regular ZODB objects - not blobs - are used, and whenever file
+data is changed, δ(ZODB) is proportional to δ(data).
+
+Being BigFile ZBigFile can be memory-mapped. Created mappings provide lazy
+on-access block loading and dirtying. This way ZBigFile larger than RAM can be
+accessed transparently as if it was a regular array in RAM. Changes made to
+ZBigFile data will be either saved or discarded depending on current
+transaction completion - commit or abort. The amount of ZBigFile changes in one
+transaction is limited by available RAM.
+
+ZBigFile does not weaken ZODB ACID properties, in particular:
+
+  - (A) either none or all changes to file data will be committed;
+  - (I) file view in current transaction is isolated from simultaneous
+        changes to this file done by other database users.
+
+API for clients
+---------------
+
+    .fileh_open()   -> open new BigFileH-like object which can be mmaped
+
+
+Operating mode
+--------------
+
+Two operating modes are provided: "local-cache" and "shared-cache".
+
+Local-cache is the mode wendelin.core was originally implemented with in 2014.
+In this mode ZBigFile data is loaded from ZODB directly via current ZODB connection.
+It was relatively straight-forward to implement, but cached file data becomes
+duplicated in between ZODB connections of current process and in between
+several ZODB processes.
+
+In shared-cache mode file's data is accessed through special filesystem for
+which data cache is centrally maintained by OS kernel. This mode was added in
+2020 and reduces wendelin.core RAM consumption dramatically.
+
+The mode of operation can be selected via environment variable::
+
+  $WENDELIN_CORE_VIRTMEM
+      rw:uvmm         local-cache   (i.e. !wcfs)    (default)
+      r:wcfs+w:uvmm   shared-cache  (i.e.  wcfs)
+
+Please see wcfs/wcfs.go which describes shared-cache mode in detail.
+
+
+Data format
+-----------
+
+$WENDELIN_CORE_ZBLK_FMT
+    ZBlk0
+    ZBlk1
+
 
 To represent BigFile as ZODB objects, each file block is represented separately
 either as
@@ -455,9 +511,9 @@ class LivePersistent(Persistent):
         # thus we'll stay in non-ghost state.
         return
 
-
     # NOTE _p_invalidate() is triggered on invalidations. We do not override it.
 
+
 # ZBigFile implements BigFile backend with data stored in ZODB.
 #
 # NOTE Can't inherit from Persistent and BigFile at the same time - both are C
@@ -480,7 +536,7 @@ class ZBigFile(LivePersistent):
         LivePersistent.__init__(self)
         self.__setstate__((blksize, LOBTree()))     # NOTE L enough for blk_t
 
-        # TODO use custom class for .blktab with adjusted bucket size, simething like
+        # TODO use custom class for .blktab with adjusted bucket size, something like
         # class xLOBTree(LOBTree):
         #   __slots__ = ()
         #   max_leaf_size       = ...   # BTree's default =  60