Skip to content

M
metadata-collect-agent
Commit 3f6bcdda authored by test's avatar test
Browse files
Options

Delete two obsolete "main" files.

parent 5172b133
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 691 deletions
scan-filesystem/cython/stable.main.pyx deleted 100644 → 0
View file @ 5172b133
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""\
{
"%s": {
"stat": %s
}
},
""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
elif not self.st.st_data.st_dev in dev_whitelist:
return
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"digests": {
"md5": "%s",
"sha1": "%s",
"sha256": "%s",
"sha512": "%s"
}
}
},
""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"target": "%s"
}
},
""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(string path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
node = make_node(path, path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
return -1
printf("WRITE_NOTE STAGE\n\n") # DEBUG
fscanf(address_path, "%s", ip_address)
fclose(address_path)
fprintf(result, '[\n')
fprintf(result, 'MAC ADDRESS: %s', ip_address)
node.write_node(result)
fprintf(result, ' {}\n]\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
return 0
cdef public int main() nogil:
return start(<char*>'.')
#def python_main():
# start(<char*>'.')
scan-filesystem/cython/test.main.pyx deleted 100644 → 0
View file @ 5172b133
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""\
{
"%s": {
"stat": %s
}
},
""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
elif not self.st.st_data.st_dev in dev_whitelist:
return
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"digests": {
"md5": "%s",
"sha1": "%s",
"sha256": "%s",
"sha512": "%s"
}
}
},
""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"target": "%s"
}
},
""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "flb/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "flb/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
fprintf(stderr, "test 001\n") # DEBUG
fflush(stderr) # DEBUG
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
fprintf(stderr, "test 002\n") # DEBUG
fflush(stderr) # DEBUG
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
return -1
printf("WRITE_NOTE STAGE\n\n") # DEBUG
fscanf(address_path, "%s", ip_address)
fclose(address_path)
fprintf(result, '[\n')
fprintf(result, '"%s",\n', ip_address)
node.write_node(result)
fprintf(result, ' {}\n]\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
return 0
cdef public int main() nogil:
return start(<char*>'.')
#def python_main():
# start(<char*>'.')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7