Commit 23362204 authored by Kirill Smelkov's avatar Kirill Smelkov

bigfile/py: Allow PyBigFile backend to expose "mmap overlay" functionality

This patch logically continues previous change `bigfile/virtmem:
Introduce "mmap overlay" mode` and exposes mmap-overlay functionality to
Python: if PyBigFile backend provides .blkmmapper PyCapsule the
mmap-related methods will be extracted from it and passed on through to
virtmem - see _bigfile.h for details.

ZBigFile will use this to hook into using WCFS.
parent fae045cc
......@@ -425,6 +425,19 @@ PyFunc(pyfileh_invalidate_page, "invalidate_page(pgoffset) - invalidate fileh pa
}
PyFunc(pyfileh_uses_mmap_overlay, "uses_mmap_overlay() - whether base data for all VMAs"
" of this fileh are taken as base-layer mmap")
(PyObject *pyfileh0, PyObject *args)
{
PyBigFileH *pyfileh = container_of(pyfileh0, PyBigFileH, pyobj);
BigFileH *fileh = &pyfileh->fileh;
if (!PyArg_ParseTuple(args, ""))
return NULL;
return PyBool_FromLong(fileh->mmap_overlay);
}
/* pyfileh vs cyclic GC */
static int
......@@ -505,11 +518,12 @@ pyfileh_new(PyTypeObject *type, PyObject *args, PyObject *kw)
static /*const*/ PyMethodDef pyfileh_methods[] = {
{"mmap", pyfileh_mmap, METH_VARARGS, pyfileh_mmap_doc},
{"dirty_writeout", pyfileh_dirty_writeout, METH_VARARGS, pyfileh_dirty_writeout_doc},
{"dirty_discard", pyfileh_dirty_discard, METH_VARARGS, pyfileh_dirty_discard_doc},
{"isdirty", pyfileh_isdirty, METH_VARARGS, pyfileh_isdirty_doc},
{"invalidate_page", pyfileh_invalidate_page,METH_VARARGS, pyfileh_invalidate_page_doc},
{"mmap", pyfileh_mmap, METH_VARARGS, pyfileh_mmap_doc},
{"dirty_writeout", pyfileh_dirty_writeout, METH_VARARGS, pyfileh_dirty_writeout_doc},
{"dirty_discard", pyfileh_dirty_discard, METH_VARARGS, pyfileh_dirty_discard_doc},
{"isdirty", pyfileh_isdirty, METH_VARARGS, pyfileh_isdirty_doc},
{"invalidate_page", pyfileh_invalidate_page, METH_VARARGS, pyfileh_invalidate_page_doc},
{"uses_mmap_overlay", pyfileh_uses_mmap_overlay, METH_VARARGS, pyfileh_uses_mmap_overlay_doc},
{NULL}
};
......@@ -955,10 +969,42 @@ out:
}
/* PyBigFile: mmap methods.
* They redirect op X to type.blkmmapper.X without going to Python level */
static int
pybigfile_mmap_setup_read(VMA *vma, BigFile *file0, blk_t blk, size_t blklen)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->mmap_setup_read(vma, file0, blk, blklen);
}
static int
pybigfile_remmap_blk_read(VMA *vma, BigFile *file0, blk_t blk)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->remmap_blk_read(vma, file0, blk);
}
static int
pybigfile_munmap(VMA *vma, BigFile *file0)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->munmap(vma, file0);
}
static const struct bigfile_ops pybigfile_ops = {
.loadblk = pybigfile_loadblk,
.storeblk = pybigfile_storeblk,
//.release =
.mmap_setup_read = pybigfile_mmap_setup_read,
.remmap_blk_read = pybigfile_remmap_blk_read,
.munmap = pybigfile_munmap,
};
......@@ -972,16 +1018,23 @@ pyfileh_open(PyObject *pyfile0, PyObject *args)
RAM *ram = ram_get_default(NULL); // TODO get ram from args
int err;
if (!PyArg_ParseTuple(args, ""))
int mmap_overlay = -1; /* -1 means None; https://bugs.python.org/issue14705 */
if (!PyArg_ParseTuple(args, "|i", &mmap_overlay))
return NULL;
if (mmap_overlay == -1)
mmap_overlay = (pyfile->blkmmap_ops != NULL ? 1 : 0);
if (mmap_overlay && pyfile->blkmmap_ops == NULL)
return PyErr_Format(PyExc_TypeError,
"%s type does not provide blkmmapper", pyfile0->ob_type->tp_name);
pyfileh = PyType_New(PyBigFileH, &PyBigFileH_Type, NULL);
if (!pyfileh)
return NULL;
Py_INCREF(pyfile);
err = fileh_open(&pyfileh->fileh, &pyfile->file, ram, DONT_MMAP_OVERLAY);
err = fileh_open(&pyfileh->fileh, &pyfile->file, ram,
mmap_overlay ? MMAP_OVERLAY : DONT_MMAP_OVERLAY);
if (err) {
XPyErr_SetFromErrno();
Py_DECREF(pyfile);
......@@ -998,6 +1051,7 @@ pyfile_traverse(PyObject *pyfile0, visitproc visit, void *arg)
{
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
Py_VISIT(pyfile->blkmmapper);
return 0;
}
......@@ -1006,6 +1060,7 @@ pyfile_clear(PyObject *pyfile0)
{
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
Py_CLEAR(pyfile->blkmmapper);
return 0;
}
......@@ -1022,6 +1077,8 @@ pyfile_dealloc(PyObject *pyfile0)
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
pyfile->blkmmap_ops = NULL;
pyfile_clear(&pyfile->pyobj);
pyfile->pyobj.ob_type->tp_free(&pyfile->pyobj);
}
......@@ -1030,11 +1087,52 @@ pyfile_dealloc(PyObject *pyfile0)
static PyObject *
pyfile_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
PyBigFile *self;
PyBigFile *self;
PyObject *blkmmapper;
bigfile_ops *blkmmap_ops = NULL;
/* try to get type.blkmmapper and verify it provides IBlkMMapper interface */
blkmmapper = PyObject_GetAttrString((PyObject*)type, "blkmmapper");
PyErr_Clear(); /* GetAttr raises exception if there is no attribute */
if (blkmmapper) {
if (!PyCapsule_IsValid(blkmmapper, "wendelin.bigfile.IBlkMMapper")) {
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper is not a valid pycapsule with mmap methods", type->tp_name);
}
blkmmap_ops = PyCapsule_GetPointer(blkmmapper, "wendelin.bigfile.IBlkMMapper");
if (blkmmap_ops == NULL) { /* just in case - must not fail */
Py_DECREF(blkmmapper);
return NULL;
}
if (blkmmap_ops->loadblk ||
blkmmap_ops->storeblk)
{
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper: !mmap methods present", type->tp_name);
}
if (!(blkmmap_ops->mmap_setup_read &&
blkmmap_ops->remmap_blk_read &&
blkmmap_ops->munmap))
{
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper: not all mmap methods present", type->tp_name);
}
}
self = (PyBigFile *)PyType_GenericNew(type, args, kw);
if (!self)
if (!self) {
Py_XDECREF(blkmmapper);
return NULL;
}
self->blkmmapper = blkmmapper;
self->blkmmap_ops = blkmmap_ops;
// FIXME "k" = unsigned long - we need size_t
static char *kw_list[] = {"blksize", NULL};
......@@ -1054,7 +1152,7 @@ static PyMemberDef pyfile_members[] = {
};
static /*const*/ PyMethodDef pyfile_methods[] = {
{"fileh_open", pyfileh_open, METH_VARARGS, "fileh_open(ram=None) -> new file handle"},
{"fileh_open", pyfileh_open, METH_VARARGS, "fileh_open(ram=None, mmap_overlay=None) -> new file handle"},
{NULL}
};
......
......@@ -2,7 +2,7 @@
#define _WENDELIN_BIGFILE__BIGFILE_H
/* Wendelin.bigfile | Python interface to memory/files
* Copyright (C) 2014-2019 Nexedi SA and Contributors.
* Copyright (C) 2014-2021 Nexedi SA and Contributors.
* Kirill Smelkov <kirr@nexedi.com>
*
* This program is free software: you can Use, Study, Modify and Redistribute
......@@ -26,11 +26,14 @@
*
* - `BigFile` is base class that allows implementing BigFile backends in Python.
* Users can inherit from BigFile, implement loadblk/storeblk and this way
* provide access to data managed from Python to virtmem subsystem.
* provide access to data managed from Python to virtmem subsystem(*).
* - `BigFileH` represents virtmem file handle for opened BigFile.
* It can be mmap'ed and provides writeout control.
* - `VMA` represents mmap'ed part of a BigFileH.
* It provides buffer/memoryview interface for data access.
*
* (*) A subclass may additionally provide functionality to map file data into
* memory. Please see BigFile documentation for details.
*/
#include <Python.h>
......@@ -97,8 +100,18 @@ typedef struct PyBigFileH PyBigFileH;
/*
* BigFile that can be implemented in python
*
* Allows subclasses to implement .loadblk() (& friends) in python.
* Allows subclasses to implement .loadblk() and .storeblk() in python.
* For users .fileh_open() is exposed to get to file handles.
*
* A subclass may additionally provide functionality to map file data into
* memory: if subclass provides .blkmmapper attribute, it is treated as
* pycapsule with type "wendelin.bigfile.IBlkMMapper" and C-level bigfile_ops
* struct that provides .mmap_setup_read and other operations related to
* mmapping data. To avoid deadlocks all mmap-related functionality must be
* nogil and so cannot be implemented in Python.
*
* The primary user of .blkmmapper functionality will be _ZBigFile which uses WCFS
* and mmaps files from it to provide memory mappings for ZBigFile data.
*/
struct PyBigFile {
PyObject pyobj;
......@@ -106,6 +119,11 @@ struct PyBigFile {
* automatically adds support for weakrefs for in-python defined children */
BigFile file;
/* blkmmapper is PyCapsule object with type.blkmmapper if BigFile subclass has it | NULL */
PyObject *blkmmapper;
/* bigfile_ops extracted from ^^^ capsule | NULL */
bigfile_ops *blkmmap_ops;
};
typedef struct PyBigFile PyBigFile;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment