Skip to content

S
scan-filesystem
Commit 27ace4a0 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

Move utils to stdlib and adapt and simplify scan

parent a563af18
Show whitespace changes
Inline Side-by-side
Showing with 73 additions and 387 deletions
cython/Makefile deleted 100644 → 0
View file @ a563af18
EXE = main
CXX = g++
CPPFLAGS = -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_DIRS)
LDFLAGS += -Wl,--unresolved-symbols=ignore-all
LDLIBS = -lcrypto -lfmt
EXT_SUFFIX := $(shell python3 -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
EXT = $(EXE)$(EXT_SUFFIX)
# Build without Python runtime
nopython: $(EXE)
%.cpp: %.pyx
@echo "[Cython Compiling $^ -> $@]"
python3 -c "from Cython.Compiler.Main import main; main(command_line=1)" $^ --cplus -3
@rm -f $(subst .cpp,.h,$@)
%: %.cpp
@echo "[C++ Compiling $^ -> $@]"
$(LINK.cpp) $^ $(LOADLIBES) $(LDLIBS) -o $@
clean:
-rm -f *.c *.cpp *.html
-rm -f *.h
-rm -f *.so
-rm -f $(EXE)
-rm -f *.o
-rm -f -r build
-rm -f *.json
.PHONY: all run nopython runnopython clean
.PRECIOUS: %.cpp
cython/main.pyx
View file @ 27ace4a0
# distutils: language = c++ # distutils: language = c++
# distutils: libraries = fmt crypto # distutils: libraries = fmt crypto
from runtime.runtime cimport Scheduler, BatchMailBox
from stdlib.list cimport List from stdlib.list cimport List
from stdlib.string cimport Str from stdlib.string cimport Str
from stdlib.format cimport format from stdlib.format cimport format
from runtime.runtime cimport Scheduler, BatchMailBox cimport stdlib.hashlib as hashlib
cimport util.hashlib as hashlib
from util.hashlib cimport Hash
cimport util.os as os cimport stdlib.os as os
from util.os cimport FILE, DIR, Stat from stdlib.os cimport FILE, DIR, Stat
cdef Str curdir = Str(".") cdef Str curdir = Str(".")
...@@ -20,13 +19,45 @@ cdef Str pardir = Str("..") ...@@ -20,13 +19,45 @@ cdef Str pardir = Str("..")
cdef Str sep = Str("/") cdef Str sep = Str("/")
cdef char * file_fmt = """\
[{}]
sha256 = {}
sha512 = {}
{}
"""
cdef char * dir_fmt = """\
[{}]
{}
"""
cdef char * symlink_fmt = """\
[{}]
target = {}
{}
"""
cdef char * stat_fmt = """\
dev = {}
ino = {}
mode = {}
nlink = {}
owner = {{ uid = {}, gid = {} }}
rdev = {}
sizes = {{ size = {}, blksize = {}, blocks = {} }}
times = {{ atim = [{}, {}], mtim = [{}, {}], ctim = [{}, {}] }}"""
cdef lock Scheduler scheduler = Scheduler() cdef lock Scheduler scheduler = Scheduler()
cdef cypclass Node activable: cdef cypclass Node activable:
Str path Str path
Stat stat Stat stat
Str json Str text
__init__(self, Str path, Stat stat): __init__(self, Str path, Stat stat):
self._active_queue_class = consume BatchMailBox(scheduler) self._active_queue_class = consume BatchMailBox(scheduler)
...@@ -35,14 +66,10 @@ cdef cypclass Node activable: ...@@ -35,14 +66,10 @@ cdef cypclass Node activable:
void build_node(self): pass void build_node(self): pass
void format_node(self): void format_node(self): pass
self.json = format("{}\n {}\n",
self.path,
self.format_stat(),
)
Str format_stat(self): Str format_stat(self):
return format("{} {} {} {} {} {} {} {} {} {} {} {} {} {} {} {}", return format(stat_fmt,
self.stat.st_dev, self.stat.st_dev,
self.stat.st_ino, self.stat.st_ino,
self.stat.st_mode, self.stat.st_mode,
...@@ -54,10 +81,10 @@ cdef cypclass Node activable: ...@@ -54,10 +81,10 @@ cdef cypclass Node activable:
self.stat.st_blksize, self.stat.st_blksize,
self.stat.st_blocks, self.stat.st_blocks,
self.stat.st_atim.tv_sec, self.stat.st_atim.tv_sec,
self.stat.st_mtim.tv_sec,
self.stat.st_ctim.tv_sec,
self.stat.st_atim.tv_nsec, self.stat.st_atim.tv_nsec,
self.stat.st_mtim.tv_sec,
self.stat.st_mtim.tv_nsec, self.stat.st_mtim.tv_nsec,
self.stat.st_ctim.tv_sec,
self.stat.st_ctim.tv_nsec, self.stat.st_ctim.tv_nsec,
) )
...@@ -109,8 +136,14 @@ cdef cypclass DirNode(Node): ...@@ -109,8 +136,14 @@ cdef cypclass DirNode(Node):
for active_child in self.children: for active_child in self.children:
active_child.build_node(NULL) active_child.build_node(NULL)
void format_node(self):
self.text = format(dir_fmt,
self.path,
self.format_stat(),
)
void write_node(self, FILE * stream): void write_node(self, FILE * stream):
os.write(self.json, stream) os.write(self.text, stream)
while self.children.__len__() > 0: while self.children.__len__() > 0:
active_child = self.children.pop() active_child = self.children.pop()
child = consume active_child child = consume active_child
...@@ -124,25 +157,22 @@ cdef enum: ...@@ -124,25 +157,22 @@ cdef enum:
cdef cypclass FileNode(Node): cdef cypclass FileNode(Node):
Str sha256 Str sha256
Str sha512 Str sha512
bint error
__init__(self, Str path, Stat stat): __init__(self, Str path, Stat stat):
Node.__init__(self, path, stat) Node.__init__(self, path, stat)
self.error = False
void build_node(self): void build_node(self):
cdef bint sha256_ok cdef bint sha256_ok
cdef bint sha512_ok cdef bint sha512_ok
cdef bint error = False
cdef FILE * file = os.open(self.path, 'rb') cdef FILE * file = os.open(self.path, 'rb')
if file is NULL: if file is NULL:
self.error = True
self.format_node() self.format_node()
return return
sha256 = Hash(hashlib.sha256()) sha256 = hashlib.Hash(hashlib.sha256())
sha512 = Hash(hashlib.sha512()) sha512 = hashlib.Hash(hashlib.sha512())
sha256_ok = sha256 is not NULL sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL sha512_ok = sha512 is not NULL
...@@ -150,7 +180,7 @@ cdef cypclass FileNode(Node): ...@@ -150,7 +180,7 @@ cdef cypclass FileNode(Node):
while (sha256_ok or sha512_ok): while (sha256_ok or sha512_ok):
s = os.read(file, CHUNK) s = os.read(file, CHUNK)
if s is NULL: if s is NULL:
self.error = True error = True
break break
if sha256_ok: if sha256_ok:
...@@ -163,25 +193,24 @@ cdef cypclass FileNode(Node): ...@@ -163,25 +193,24 @@ cdef cypclass FileNode(Node):
os.close(file) os.close(file)
if not self.error: if not error:
self.sha256 = sha256.hexdigest() if sha256_ok else Str("<errror>") self.sha256 = sha256.hexdigest() if sha256_ok else NULL
self.sha512 = sha512.hexdigest() if sha512_ok else Str("<errror>") self.sha512 = sha512.hexdigest() if sha512_ok else NULL
self.format_node() self.format_node()
void format_node(self): void format_node(self):
if self.error: sha256 = self.sha256 if self.sha256 else Str("<error>")
Node.format_node(self) sha512 = self.sha512 if self.sha512 else Str("<error>")
else: self.text = format(file_fmt,
self.json = format("{}\n {}\n {}\n {}\n",
self.path, self.path,
self.sha256, sha256,
self.sha512, sha512,
self.format_stat(), self.format_stat(),
) )
void write_node(self, FILE * stream): void write_node(self, FILE * stream):
os.write(self.json, stream) os.write(self.text, stream)
cdef cypclass SymlinkNode(Node): cdef cypclass SymlinkNode(Node):
...@@ -192,17 +221,15 @@ cdef cypclass SymlinkNode(Node): ...@@ -192,17 +221,15 @@ cdef cypclass SymlinkNode(Node):
self.format_node() self.format_node()
void format_node(self): void format_node(self):
if self.target is NULL: target = self.target if self.target is not NULL else Str("<error>")
Node.format_node(self) self.text = format(symlink_fmt,
else:
self.json = format("{} -> {}\n {}\n",
self.path, self.path,
self.target, target,
self.format_stat(), self.format_stat(),
) )
void write_node(self, FILE * stream): void write_node(self, FILE * stream):
os.write(self.json, stream) os.write(self.text, stream)
cdef int scan(iso Str root) nogil: cdef int scan(iso Str root) nogil:
......
cython/setup.py deleted 100644 → 0
View file @ a563af18
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
extensions = [
Extension(
"main",
language="c++",
sources=["main.pyx"],
include_dirs = [
"/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/include",
"/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/include"
],
library_dirs = [
"/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/lib",
"/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/lib"
],
libraries = ["crypto", "fmt"],
extra_link_args=[
"-Wl,-rpath=/srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75/lib",
"-Wl,-rpath=/srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c/lib"
],
),
]
setup(
ext_modules = cythonize(extensions)
)
cython/util/_hashlib.pxd deleted 100644 → 0
View file @ a563af18
cdef extern from "<openssl/evp.h>" nogil:
ctypedef struct EVP_MD_CTX:
pass
ctypedef struct EVP_MD
ctypedef struct ENGINE
void EVP_MD_CTX_init(EVP_MD_CTX *ctx)
EVP_MD_CTX *EVP_MD_CTX_create()
int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl)
int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt)
int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s)
int EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx)
void EVP_MD_CTX_destroy(EVP_MD_CTX *ctx)
int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *_in)
int EVP_DigestInit(EVP_MD_CTX *ctx, const EVP_MD *type)
int EVP_DigestFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s)
int EVP_MD_CTX_copy(EVP_MD_CTX *out,EVP_MD_CTX *_in)
# Max size
const int EVP_MAX_MD_SIZE
# Algorithms
const EVP_MD *md5 "EVP_md5"()
const EVP_MD *blake2b "EVP_blake2b512"
const EVP_MD *blake2s "EVP_blake2b256"
const EVP_MD *sha1 "EVP_sha1"()
const EVP_MD *sha224 "EVP_sha224"()
const EVP_MD *sha256 "EVP_sha256"()
const EVP_MD *sha384 "EVP_sha384"()
const EVP_MD *sha512 "EVP_sha512"()
const EVP_MD *EVP_get_digestbyname(const char *name)
cdef extern from * nogil:
'''
static const char hexdigits [] = "0123456789abcdef";
'''
cdef const char hexdigits[]
cython/util/_os.pxd deleted 100644 → 0
View file @ a563af18
from posix.types cimport (blkcnt_t,
blksize_t,
dev_t,
gid_t,
ino_t,
mode_t,
nlink_t,
off_t,
time_t,
uid_t)
# Differences with posix.stat:
#
# - the declaration for the non-standard field st_birthtime was removed
# because cypclass wrapping triggers the generation of a conversion
# function for the stat structure which references this field.
#
# - the absent declaration in posix.time of struct timespec was added.
#
# - the declarations for the time_t fields st_atime, st_mtime, st_ctime
# were replaced by the fields st_atim, st_mtim, st_ctim
# of type struct timespec.
cdef extern from "<sys/time.h>" nogil:
cdef struct struct_timespec "timespec":
time_t tv_sec
long int tv_nsec
cdef extern from "<sys/stat.h>" nogil:
cdef cppclass struct_stat "stat":
dev_t st_dev
ino_t st_ino
mode_t st_mode
nlink_t st_nlink
uid_t st_uid
gid_t st_gid
dev_t st_rdev
off_t st_size
blksize_t st_blksize
blkcnt_t st_blocks
struct_timespec st_atim
struct_timespec st_mtim
struct_timespec st_ctim
# POSIX dictates including both <sys/stat.h> and <unistd.h> for these
cdef extern from "<unistd.h>" nogil:
int fchmod(int, mode_t)
int chmod(const char *, mode_t)
int fstat(int, struct_stat *)
int lstat(const char *, struct_stat *)
int stat(const char *, struct_stat *)
# Macros for st_mode
mode_t S_ISREG(mode_t)
mode_t S_ISDIR(mode_t)
mode_t S_ISCHR(mode_t)
mode_t S_ISBLK(mode_t)
mode_t S_ISFIFO(mode_t)
mode_t S_ISLNK(mode_t)
mode_t S_ISSOCK(mode_t)
mode_t S_IFMT
mode_t S_IFREG
mode_t S_IFDIR
mode_t S_IFCHR
mode_t S_IFBLK
mode_t S_IFIFO
mode_t S_IFLNK
mode_t S_IFSOCK
# Permissions
mode_t S_ISUID
mode_t S_ISGID
mode_t S_ISVTX
mode_t S_IRWXU
mode_t S_IRUSR
mode_t S_IWUSR
mode_t S_IXUSR
mode_t S_IRWXG
mode_t S_IRGRP
mode_t S_IWGRP
mode_t S_IXGRP
mode_t S_IRWXO
mode_t S_IROTH
mode_t S_IWOTH
mode_t S_IXOTH
cdef extern from "<sys/types.h>" nogil:
ctypedef struct DIR
cdef extern from "<dirent.h>" nogil:
cdef struct struct_dirent "dirent":
ino_t d_ino
char d_name[256]
DIR *opendir(const char *name)
struct_dirent *readdir(DIR *dirp)
int readdir_r(DIR *dirp, struct_dirent *entry, struct_dirent **result)
int closedir(DIR *dirp)
cython/util/hashlib.pxd deleted 100644 → 0
View file @ a563af18
from stdlib.string cimport Str
from ._hashlib cimport *
cdef cypclass Hash:
EVP_MD_CTX * md_ctx
Hash __new__(alloc, const EVP_MD * algo):
md_ctx = EVP_MD_CTX_create()
if md_ctx is NULL:
return NULL
if EVP_DigestInit_ex(md_ctx, algo, NULL) != 1:
return NULL
instance = alloc()
instance.md_ctx = md_ctx
return instance
__dealloc__(self):
EVP_MD_CTX_destroy(md_ctx)
int update(self, Str message):
bytes = <unsigned char * > message.bytes()
return EVP_DigestUpdate(self.md_ctx, bytes, message._str.size()) - 1
Str hexdigest(self):
cdef char result[EVP_MAX_MD_SIZE*2 + 1]
cdef unsigned char hashbuffer[EVP_MAX_MD_SIZE]
cdef unsigned int size
cdef unsigned int i
if EVP_DigestFinal_ex(self.md_ctx, hashbuffer, &size) == 1:
for i in range(size):
result[2*i] = hexdigits[hashbuffer[i] >> 4]
result[2*i+1] = hexdigits[hashbuffer[i] & 15]
result[size] = 0
return Str(result)
cython/util/os.pxd deleted 100644 → 0
View file @ a563af18
from stdlib.string cimport Str
from stdlib.list cimport List
from libc.errno cimport errno
from libc.stdio cimport FILE, fopen, fclose, fread, fwrite, ferror
from libc.stdio cimport stdin, stdout, stderr
from posix cimport unistd
from ._os cimport DIR, struct_dirent, opendir, readdir, closedir
from ._os cimport struct_stat, lstat, S_ISREG, S_ISLNK, S_ISDIR
cdef enum:
_BUFSIZE = 64 * 1024
cdef inline FILE * open(Str path, char * mode) nogil:
return fopen(path.bytes(), mode)
cdef inline Str read(FILE * file, int nbytes) nogil:
s = Str()
s._str.append(_BUFSIZE, 0)
cdef int size
size = fread(s._str.data(), 1, nbytes, file)
if size != nbytes:
if ferror(file):
return NULL
s._str.resize(size)
return s
cdef inline int close(FILE * file) nogil:
return fclose(file)
cdef inline int write(Str data, FILE * file) nogil:
return fwrite(data.bytes(), 1, data._str.size(), file)
cdef inline List[Str] listdir(Str path) nogil:
d = opendir(path.bytes())
if d is NULL:
return NULL
entries = List[Str]()
while True:
errno = 0
entry = readdir(d)
if entry is NULL:
break
entries.append(Str(entry.d_name))
read_error = errno
closedir(d)
if read_error:
return NULL
return entries
cdef inline Str readlink(Str path, int max_size) nogil:
s = Str()
s._str.resize(max_size)
size = unistd.readlink(path.bytes(), <char*> s._str.data(), max_size)
if size == -1:
return NULL
s._str.resize(size)
return s
cdef cypclass Stat(struct_stat):
Stat __new__(alloc, Str path):
instance = alloc()
s = <struct_stat *> instance
if s is not NULL:
if not lstat(path.bytes(), s):
return instance
bint is_regular(self):
return S_ISREG(self.st_mode)
bint is_symlink(self):
return S_ISLNK(self.st_mode)
bint is_dir(self):
return S_ISDIR(self.st_mode)
cdef inline Stat stat(Str path) nogil:
return Stat(path)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7