Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
Kirill Smelkov
wendelin.core
Commits
f723fb47
Commit
f723fb47
authored
Jun 28, 2018
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
4ff147d1
Changes
4
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
135 additions
and
85 deletions
+135
-85
wcfs/__init__.py
wcfs/__init__.py
+39
-16
wcfs/todo.dot
wcfs/todo.dot
+4
-0
wcfs/todo.svg
wcfs/todo.svg
+82
-63
wcfs/wcfs.go
wcfs/wcfs.go
+10
-6
No files found.
wcfs/__init__.py
View file @
f723fb47
...
...
@@ -20,7 +20,12 @@
"""Module wcfs.py provides python gateway for spawning and interoperating with wcfs server
XXX doc
Join(zurl) joins wcfs server. If wcfs server for zurl is not yet running, it
can be automatically started if `autostart=True` parameter is passed to join.
It will also be automatically started by default unless
$WENDELIN_CORE_WCFS_AUTOSTART=no is specified in environment.
XXX Conn.
"""
import
os
,
sys
,
hashlib
,
tempfile
,
subprocess
,
time
...
...
@@ -49,19 +54,19 @@ class Conn(object):
# serve starts and runs wcfs server for ZODB @ zurl.
#
# it mounts wcfs at a location that is with 1-1 corresponence with zurl.
# it mounts wcfs at a location that is with 1-1 correspon
d
ence with zurl.
# it then waits for wcfs to exit (either due to unmount or an error).
#
# it is an error if wcfs was already started.
#
# if exec_ is True, wcfs is not spawned, but exec
ed
to.
# if exec_ is True, wcfs is not spawned, but exec
uted in
to.
#
# serve(zurl, exec_=False).
def
serve
(
zurl
,
exec_
=
False
):
mntpt
=
_mntpt_4zurl
(
zurl
)
# try opening .wcfs - it is an error if we can do it.
# XXX -> option to wcfs itself
# XXX -> option to wcfs itself
?
try
:
f
=
open
(
mntpt
+
"/.wcfs"
)
except
IOError
as
e
:
...
...
@@ -80,14 +85,24 @@ def serve(zurl, exec_=False):
os
.
execv
(
argv
[
0
],
argv
)
# _default_autostart returns default autostart setting for join.
#
# Out-of-the-box we want wcfs to be automatically started, to ease developer
# experience when wendelin.core is standalone installed. However in environments
# like SlapOS, it is more preferable to start and monitor wcfs service explicitly.
# SlapOS & co. should thus set $WENDELIN_CORE_WCFS_AUTOSTART=no.
def
_default_autostart
():
autostart
=
os
.
environ
.
get
(
"WENDELIN_CORE_WCFS_AUTOSTART"
,
"yes"
)
autostart
=
autostart
.
lower
()
return
{
"yes"
:
True
,
"no"
:
False
}[
autostart
]
# join connects to wcfs server for ZODB @ zurl.
#
# if wcfs for that service was already started, join connects to it.
# otherwise it starts wcfs for zurl if autostart is True or (None and system
# default for autostart is True). XXX
# If wcfs for that service was already started, join connects to it.
# Otherwise it starts wcfs for zurl if autostart is True.
#
# join(zurl) -> Conn.
def
join
(
zurl
,
autostart
=
None
):
def
join
(
zurl
,
autostart
=
_default_autostart
()
):
mntpt
=
_mntpt_4zurl
(
zurl
)
# try opening .wcfs - if we succeed - it is already mounted.
...
...
@@ -101,16 +116,18 @@ def join(zurl, autostart=None):
# already have it
return
Conn
(
mntpt
,
f
)
# XXX autostart=None processing
if
not
autostart
:
raise
RuntimeError
(
"wcfs: join %s: server not started"
%
zurl
)
return
_start
(
zurl
)
# start wcfs with telling it to automatically exit when there is no client activity.
return
_start
(
zurl
,
"-autoexit"
)
# _start starts wcfs server for ZODB @ zurl.
#
# _start(zurl) -> Conn
def
_start
(
zurl
):
# optv can be optionally given to pass flags to wcfs.
#
# _start(zurl, *optv) -> Conn
def
_start
(
zurl
,
*
optv
):
mntpt
=
_mntpt_4zurl
(
zurl
)
log
.
info
(
"wcfs: starting for %s ..."
,
zurl
)
...
...
@@ -123,7 +140,8 @@ def _start(zurl):
def
spawn
():
err
=
None
try
:
p
=
subprocess
.
Popen
([
_wcfs_exe
(),
zurl
,
mntpt
],
close_fds
=
True
)
argv
=
[
_wcfs_exe
()]
+
list
(
optv
)
+
[
zurl
,
mntpt
]
p
=
subprocess
.
Popen
(
argv
,
close_fds
=
True
)
while
1
:
ret
=
p
.
poll
()
if
ret
is
not
None
:
...
...
@@ -140,7 +158,7 @@ def _start(zurl):
break
if
_
==
1
:
# startup was ok - don't monitor spawned wcfs anylonger
# startup was ok - don't monitor spawned wcfs any
longer
break
time
.
sleep
(
0.1
)
...
...
@@ -203,7 +221,13 @@ def _start(zurl):
if
_
==
1
:
if
isinstance
(
_rx
,
file
):
# mounted ok - return Conn object
# mounted ok - return Conn object.
#
# NOTE: we tell `spawn` thread to exit and stop monitoring spawned
# wcfs, because we want spawned wcfs to potentially overlive our
# process and to serve other processes. For the same reason we do
# not preserve cancel channel in returned Conn.
f
=
_rx
startedok
.
close
()
return
Conn
(
mntpt
,
f
)
...
...
@@ -212,7 +236,6 @@ def _start(zurl):
err
=
_rx
cancel
.
close
()
raise
RuntimeError
(
"wcfs: start: %s"
%
err
)
# XXX errctx
...
...
wcfs/todo.dot
View file @
f723fb47
...
...
@@ -3,9 +3,11 @@ digraph {
wcfs
->
ZODB_go_inv
;
wcfs
->
Sinvtree
;
wcfs
->
δ
R
;
wcfs
->
autoexit
;
wcfs_simple
->
Btree_read
;
wcfs_simple
->
ZBlk_read
;
wcfs_simple
->
autoexit
;
client
->
wcfs_spawn
;
client
->
δ
R
;
...
...
@@ -27,4 +29,6 @@ digraph {
test
[
label
=
"? tests"
]
zodburl
[
label
=
"zstor -> zurl"
,
style
=
filled
fillcolor
=
grey95
]
autoexit
[
label
=
"autoexit if\nautostart && !activity"
]
}
wcfs/todo.svg
View file @
f723fb47
This diff is collapsed.
Click to expand it.
wcfs/wcfs.go
View file @
f723fb47
...
...
@@ -119,7 +119,7 @@
// mmap(bigfile/<bigfileX>/head/data)
// mmap(bigfile/<bigfileX>/@<Cat>/data, δR(Cat,Sat), MAP_FIXED) # mmaped at addresses corresponding to δR(Cat,Sat)
//
// When client completes its initial
l
mmapping it sends ack back to the server:
// When client completes its initial mmapping it sends ack back to the server:
//
// C: ack
//
...
...
@@ -170,7 +170,7 @@
// This way there should be no possibility for a client to block wcfs
// indefinitely waiting for client's ack.
//
// Similarly for initial
l
mmapings client could first mmap head/data, then open
// Similarly for initial mmapings client could first mmap head/data, then open
// head/invalidations and tell the server that it wants Cat revision, with
// the server then remmaping blocks to get to Cat state via ptrace.
//
...
...
@@ -357,6 +357,8 @@ func (br *BigFileRoot) Mkdir(name string, mode uint32, _ *fuse.Context) (*nodefs
mkdir
(
br
,
name
,
bx
)
// XXX takes treeLock - ok under br.mu ?
mkdir
(
bx
,
"head"
,
bf
)
mkfile
(
bf
,
"data"
,
bf
.
data
)
// XXX mkfile(bf, "at", bf.at)
// XXX mkfile(bf, "invalidations", bf.inv)
return
bx
.
Inode
(),
fuse
.
OK
}
...
...
@@ -367,16 +369,15 @@ func (br *BigFileRoot) Mkdir(name string, mode uint32, _ *fuse.Context) (*nodefs
// XXX option to prevent starting if wcfs was already started ?
// XXX option to automatically exit/unmount if there are no requests for some t
// (UC: autospawned from join)
func
main
()
{
log
.
SetPrefix
(
"wcfs: "
)
debug
:=
flag
.
Bool
(
"d"
,
false
,
"debug"
)
autoexit
:=
flag
.
Bool
(
"autoexit"
,
false
,
"automatically stop service when there is no client activity"
)
flag
.
Parse
()
if
len
(
flag
.
Args
())
!=
2
{
log
.
Fatalf
(
"Usage: %s zurl mntpt"
,
os
.
Args
[
0
])
log
.
Fatalf
(
"Usage: %s
[OPTIONS]
zurl mntpt"
,
os
.
Args
[
0
])
}
zurl
:=
flag
.
Args
()[
0
]
mntpt
:=
flag
.
Args
()[
1
]
...
...
@@ -405,6 +406,9 @@ func main() {
mkfile
(
root
,
".wcfs"
,
NewStaticFile
([]
byte
(
zurl
)))
mkdir
(
root
,
"bigfile"
,
NewBigFileRoot
(
zstor
))
// TODO handle autoexit
_
=
autoexit
// serve client requests
server
.
Serve
()
// XXX Serve returns no eror
server
.
Serve
()
// XXX Serve returns no er
r
or
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment