Commit eb0593bc authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: Initial stub

Add initial stub for WCFS program and tests.
WCFS functionality will be added step-by-step in follow-up commits.

Some preliminary history:

0ae88a32       X .nxdtest: Verify Go bits with GOMAXPROCS=1,2,`nproc`
23528eb4       X wcfs: make it to use go modules for dependencies
parent 622fb217
......@@ -3,6 +3,10 @@
storv = ['fs', 'zeo', 'neo'] # storage backends to test against
# some bugs are only likely to trigger when there is only 1 or 2 main OS thread(s) in wcfs
# GOMAXPROCS='' means use `nproc`
gonprocv = ['1', '2', ''] # GOMAXPROCS=... to test against
# test.t & friends unit-test core of UVMM and are Go-, Python- and ZODB-storage independent.
# we don't run test.vg* because there is currently no valgrind on SlapOS.
......@@ -27,3 +31,17 @@ for stor in storv:
TestCase('test.py/%s-!wcfs' % stor, ['make', 'test.py'],
envadj={'WENDELIN_CORE_TEST_DB': '<%s>' % stor},
summaryf=PyTest.summary)
# test.go unit-tests Go bits in wcfs.
for nproc in gonprocv:
TestCase('test.go:%s' % nproc, ['make', 'test.go'],
envadj={'GOMAXPROCS': nproc})
# test.wcfs/<stor> runs unit tests for WCFS
for stor in storv:
envdb = {'WENDELIN_CORE_TEST_DB': '<%s>' % stor}
for nproc in gonprocv:
TestCase('test.wcfs/%s:%s' % (stor, nproc), ['make', 'test.wcfs'],
envadj=dict(GOMAXPROCS=nproc, **envdb), summaryf=PyTest.summary)
......@@ -23,6 +23,7 @@ PYTHON ?= python
PYTEST ?= $(PYTHON) -m pytest
PYBENCH ?= $(PYTHON) -m golang.cmd.pybench
VALGRIND?= valgrind
GO ?= go
# use the same C compiler as python
# (for example it could be `gcc -m64` on a 32bit userspace)
......@@ -33,7 +34,7 @@ ifeq ($(CC),)
$(error "Cannot defermine py-CC")
endif
all : bigfile/_bigfile.so
all : bigfile/_bigfile.so wcfs/wcfs
ccan_config := 3rdparty/ccan/config.h
......@@ -42,6 +43,9 @@ bigfile/_bigfile.so : $(ccan_config) FORCE
$(PYTHON) setup.py build_dso --inplace
$(PYTHON) setup.py ll_build_ext --inplace
wcfs/wcfs: FORCE
cd wcfs && $(GO) build
FORCE :
......@@ -69,7 +73,7 @@ CFLAGS := -g -Wall -D_GNU_SOURCE -std=gnu99 -fplan9-extensions \
# XXX hack ugly
LOADLIBES=lib/bug.c lib/utils.c 3rdparty/ccan/ccan/tap/tap.c
TESTS := $(patsubst %.c,%,$(wildcard bigfile/tests/test_*.c))
test : test.t test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd
test : test.t test.go test.wcfs test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd
# TODO move XFAIL markers into *.c
......@@ -165,17 +169,25 @@ test.vgdrd: $(TESTS:%=%.vgdrdrun)
# run python tests
PYTEST_IGNORE := --ignore=3rdparty --ignore=build --ignore=t
test.py : bigfile/_bigfile.so
$(PYTEST) $(PYTEST_IGNORE)
# wcfs unit-tests
test.wcfs : bigfile/_bigfile.so wcfs/wcfs
$(PYTEST) wcfs/
# unit/functional tests for whole wendelin.core
test.py : bigfile/_bigfile.so wcfs/wcfs
$(PYTEST) $(PYTEST_IGNORE) --ignore=wcfs
# test.py via Valgrind (very slow)
test.py.vghel: bigfile/_bigfile.so
test.py.vghel: bigfile/_bigfile.so wcfs/wcfs
$(call vgxrun,--tool=helgrind, $(PYTEST) $(PYTEST_IGNORE))
test.py.drd: bigfile/_bigfile.so
test.py.drd: bigfile/_bigfile.so wcfs/wcfs
$(call vgxrun,--tool=drd, $(PYTEST) $(PYTEST_IGNORE))
# run go tests
test.go :
cd wcfs && $(GO) test -count=1 ./... # -count=1 disables tests caching
# test pagefault for double/real faults - it should crash
tfault := bigfile/tests/tfault
......@@ -194,5 +206,5 @@ bench : bench.t bench.py
bench.t : $(BENCHV.C:%=%.trun)
bench.py: bigfile/_bigfile.so
bench.py: bigfile/_bigfile.so wcfs/wcfs
$(PYBENCH) --count=3 --forked $(PYTEST_IGNORE)
......@@ -21,11 +21,13 @@ from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ex
from setuptools import Command, find_packages
from setuptools.command.build_py import build_py as _build_py
from pkg_resources import working_set, EntryPoint
from distutils.file_util import copy_file
from distutils.errors import DistutilsExecError
from subprocess import Popen, PIPE
import os
import sys
from errno import EEXIST
# tell cython to resolve `cimport wendelin.*` modules hierarchy starting at top-level.
......@@ -172,11 +174,23 @@ def viamake(target, help_text):
# build_ext that
# - builds via Makefile (and thus pre-builds ccan) XXX hacky
# - integrates built wcfs/wcfs.go exe into wcfs/ python package
class build_ext(_build_ext):
def run(self):
runmake('all')
return _build_ext.run(self)
_build_ext.run(self)
# copy wcfs/wcfs built from wcfs.go by make into build/lib.linux-x86_64-2.7/...
# so that py packaging tools see built wcfs.go as part of wendelin/wcfs package.
dst = os.path.join(self.build_lib, 'wendelin/wcfs')
try:
os.makedirs(dst)
except OSError as e:
if e.errno != EEXIST:
raise
copy_file('wcfs/wcfs', dst, verbose=self.verbose, dry_run=self.dry_run)
......@@ -329,6 +343,9 @@ setup(
},
entry_points= {'console_scripts': [
# start wcfs for zurl
'wcfs = wendelin.wcfs:main',
# demo to test
'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
]
......
/wcfs
*.test
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Module wcfs.py provides python gateway for spawning wcfs server.
"""
from __future__ import print_function, absolute_import
import os, sys
from os.path import dirname
# _wcfs_exe returns path to wcfs executable.
def _wcfs_exe():
return '%s/wcfs' % dirname(__file__)
def main():
argv = sys.argv[1:]
os.execv(_wcfs_exe(), [_wcfs_exe()] + argv)
module lab.nexedi.com/nexedi/wendelin.core/wcfs
go 1.14
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
func main() {
// XXX stub
}
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""wcfs_test.py tests wcfs filesystem from outside as python client process.
"""
from __future__ import print_function, absolute_import
from wendelin import wcfs
# TODO
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment