Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
Kirill Smelkov
wendelin.core
Commits
b07298d7
Commit
b07298d7
authored
Dec 18, 2019
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
c113d3c2
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
47 additions
and
41 deletions
+47
-41
bigfile/_bigfile.h
bigfile/_bigfile.h
+4
-4
bigfile/_file_zodb.pyx
bigfile/_file_zodb.pyx
+37
-15
bigfile/file_zodb.py
bigfile/file_zodb.py
+6
-22
No files found.
bigfile/_bigfile.h
View file @
b07298d7
...
...
@@ -25,8 +25,8 @@
/* Package _bigfile provides Python bindings to virtmem.
*
* - `BigFile` is base class that allows implementing BigFile backends in Python.
* Users can inherit from BigFile, implement loadblk/storeblk
and this way
* provide access to data managed from Python to virtmem subsystem.
* Users can inherit from BigFile, implement loadblk/storeblk
/blkmmapper and XXX review
*
this way
provide access to data managed from Python to virtmem subsystem.
* - `BigFileH` represents virtmem file handle for opened BigFile.
* It can be mmap'ed and provides writeout control.
* - `VMA` represents mmap'ed part of a BigFileH.
...
...
@@ -93,7 +93,7 @@ struct PyBigFileH {
/* if subclass, in addition to .loadblk/.storeblk, defines .blkmmapper XXX ... */
PyObject
*
pymmapper
;
// python object returned by .blkmmapper() that is holding virtmem_mapper pycapsule
//virt_mapper *virt_mmap_ops;
// XXX ok?
//virt_mapper *virt_mmap_ops;
// XXX ok?
};
typedef
struct
PyBigFileH
PyBigFileH
;
...
...
@@ -101,7 +101,7 @@ typedef struct PyBigFileH PyBigFileH;
/*
* BigFile that can be implemented in python
*
* Allows subclasses to implement .loadblk() (& friends) in python.
* Allows subclasses to implement .loadblk() (& friends) in python.
XXX blkmmapper doc
* For users .fileh_open() is exposed to get to file handles.
*/
struct
PyBigFile
{
...
...
bigfile/_file_zodb.pyx
View file @
b07298d7
...
...
@@ -73,22 +73,44 @@ cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
def
loadblk
(
self
,
blk
,
buf
):
return
self
.
zself
.
loadblk
(
blk
,
buf
)
def
storeblk
(
self
,
blk
,
buf
):
return
self
.
zself
.
storeblk
(
blk
,
buf
)
# blkmmapper returns pycapsule with virtmem mmapper for the file.
def
blkmmapper
(
_ZBigFile
zf
):
# it is called from under PyBigFile.fileh_open(mmap_overlay=True) and
# .zfile should be already associated with jar and have oid.
# make sure that WCFS file handle corresponding to ZBigFile is opened.
# fileh_open wraps BigFile.fileh_open and makes sure that WCFS file handle
# corresponding to ZBigFile is opened if use_wcfs=True.
def
fileh_open
(
_ZBigFile
zf
,
bint
use_wcfs
):
mmap_overlay
=
False
cdef
wcfs
.
PyFileH
pywfileh
if
zf
.
wfileh
==
nil
:
zconn
=
zf
.
zself
.
_p_jar
assert
zconn
is
not
None
# XXX locking? or rely on that ZODB objects for must be used from under 1 thread only?
# join zconn to wconn; link to wconn from _ZBigFile
pywconn
=
pywconnOf
(
zconn
)
pywfileh
=
pywconn
.
open
(
zf
.
zself
.
_p_oid
)
zf
.
wfileh
=
pywfileh
.
wfileh
if
use_wcfs
:
mmap_overlay
=
True
if
zf
.
wfileh
==
nil
:
zconn
=
zf
.
zself
.
_p_jar
assert
zconn
is
not
None
# XXX locking? or rely on that ZODB objects for must be used from under 1 thread only?
# join zconn to wconn; link to wconn from _ZBigFile
pywconn
=
pywconnOf
(
zconn
)
pywfileh
=
pywconn
.
open
(
zf
.
zself
.
_p_oid
)
zf
.
wfileh
=
pywfileh
.
wfileh
return
super
(
_ZBigFile
,
zf
).
fileh_open
(
mmap_overlay
)
# blkmmapper returns pycapsule with virtmem mmapper for the file.
@
staticmethod
def
blkmmapper
():
# def blkmmapper(_ZBigFile zf):
# # it is called from under PyBigFile.fileh_open(mmap_overlay=True) and
# # .zfile should be already associated with jar and have oid.
#
# # make sure that WCFS file handle corresponding to ZBigFile is opened.
# cdef wcfs.PyFileH pywfileh
# if zf.wfileh == nil:
# zconn = zf.zself._p_jar
# assert zconn is not None
# # XXX locking? or rely on that ZODB objects for must be used from under 1 thread only?
#
# # join zconn to wconn; link to wconn from _ZBigFile
# pywconn = pywconnOf(zconn)
# pywfileh = pywconn.open(zf.zself._p_oid)
# zf.wfileh = pywfileh.wfileh
# zf is ready to serve mmap requests via .wfileh
# return pycapsule with corresponding mmap methods.
...
...
@@ -99,7 +121,7 @@ cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
# _ZBigFile_mmap_pycapsule is pycapsule with functions that we give to
# PyBigFile for ZBigFile virtmem .mmap* ops.
# PyBigFile for ZBigFile virtmem .mmap* ops
(see _ZBigFile.blkmmapper)
.
cdef
extern
from
"<wendelin/bigfile/file.h>"
nogil
:
struct
bigfile_ops
:
pass
...
...
bigfile/file_zodb.py
View file @
b07298d7
...
...
@@ -624,19 +624,7 @@ class ZBigFile(LivePersistent):
if
_use_wcfs
is
None
:
_use_wcfs
=
self
.
_default_use_wcfs
()
wcfileh
=
None
if
_use_wcfs
:
# TODO maintain zconn -> wconn in sync (p_jar -> wconn)
# TODO close wconn on zconn del (NOTE not close, as zconn.close() just puts it into DB.poll)
# TODO close wcfileh on _ZBigFileH close
zconn
=
self
.
_p_jar
zstor
=
zconn
.
db
().
storage
zurl
=
wcfs
.
zstor_2zurl
(
zstor
)
wc
=
wcfs
.
join
(
zurl
)
wconn
=
wc
.
connect
(
zconn_at
(
zconn
))
wcfileh
=
wconn
.
open
(
self
.
_p_oid
)
fileh
=
_ZBigFileH
(
self
,
wcfileh
)
fileh
=
_ZBigFileH
(
self
,
_use_wcfs
)
self
.
_v_filehset
.
add
(
fileh
)
return
fileh
...
...
@@ -722,15 +710,11 @@ Connection.open = Connection_open
@
implementer
(
ISynchronizer
)
class
_ZBigFileH
(
object
):
# .zfile ZBigFile we were opened for
# # .wcfileh handle for ZBigFile@zconn.at view in wcfs | None
# .zfileh handle for ZBigFile in virtmem (overlayed over .wcfileh if .wcfileh != ø)
def
__init__
(
self
,
zfile
,
wcfileh
):
self
.
zfile
=
zfile
self
.
wcfileh
=
wcfileh
#self.zfileh = zfile._v_file.fileh_open(wcfileh) # XXX pass wcfileh in
# XXX no - BigFile should fetch wcfileh itself from ZBigFile by calling ZBigFile.blkmmapper()
self
.
zfileh
=
zfile
.
_v_file
.
fileh_open
()
# .zfileh handle for ZBigFile in virtmem
def
__init__
(
self
,
zfile
,
use_wcfs
):
self
.
zfile
=
zfile
self
.
zfileh
=
zfile
.
_v_file
.
fileh_open
(
use_wcfs
)
# FIXME zfile._p_jar could be None (ex. ZBigFile is newly created
# before first commit)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment