Commit 8eb14c23 authored by Ophélie Gagnard's avatar Ophélie Gagnard

Send a better data structure to Wendelin. Send a string which is json-compliant again.

parent 3ef89618
......@@ -2,7 +2,7 @@
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
......@@ -14,6 +14,8 @@ from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
......@@ -40,9 +42,8 @@ cdef cypclass Node activable:
void format_node(self):
self.formatted = sprintf("""\
{
"%s": {
"path": "%s",
"stat": %s
}
},
""",
self.path,
......@@ -128,7 +129,6 @@ cdef cypclass DirNode(Node):
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
......@@ -194,15 +194,14 @@ cdef cypclass FileNode(Node):
else:
self.formatted = sprintf("""\
{
"%s": {
"path": "%s",
"stat": %s,
"digests": {
"hash": {
"md5": "%s",
"sha1": "%s",
"sha256": "%s",
"sha512": "%s"
}
}
},
""",
self.path,
......@@ -235,10 +234,9 @@ cdef cypclass SymlinkNode(Node):
else:
self.formatted = sprintf("""\
{
"%s": {
"path": "%s",
"stat": %s,
"target": "%s"
}
},
""",
self.path,
......@@ -250,21 +248,25 @@ cdef cypclass SymlinkNode(Node):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(string path) nogil:
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg2 = "flb/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef char* arg4 = "flb/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
......@@ -291,10 +293,19 @@ cdef int start(string path) nogil:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
node = make_node(path, path)
fprintf(stderr, "test 001\n") # DEBUG
fflush(stderr) # DEBUG
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
fprintf(stderr, "test 002\n") # DEBUG
fflush(stderr) # DEBUG
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
......@@ -310,7 +321,10 @@ cdef int start(string path) nogil:
printf("WRITE_NOTE STAGE\n\n") # DEBUG
fscanf(address_path, "%s", ip_address)
fclose(address_path)
fprintf(result, '[\n')
fprintf(result, '"%s",\n', ip_address)
node.write_node(result)
fprintf(result, ' {}\n]\n')
fprintf(result, 'fluentbit_end\n')
......@@ -319,12 +333,10 @@ cdef int start(string path) nogil:
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING [error version]\n\n")
printf("WAITING...\n\n")
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS [error version]\n\n")
printf("WAITING ENDS NOW!\n\n")
fprintf(stderr, "WAITING ENDS\n\n")
return 0
......
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""\
{
"%s": {
"stat": %s
}
},
""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
elif not self.st.st_data.st_dev in dev_whitelist:
return
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"digests": {
"md5": "%s",
"sha1": "%s",
"sha256": "%s",
"sha512": "%s"
}
}
},
""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"target": "%s"
}
},
""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(string path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
node = make_node(path, path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
return -1
printf("WRITE_NOTE STAGE\n\n") # DEBUG
fscanf(address_path, "%s", ip_address)
fclose(address_path)
fprintf(result, '[\n')
fprintf(result, 'MAC ADDRESS: %s', ip_address)
node.write_node(result)
fprintf(result, ' {}\n]\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
return 0
cdef public int main() nogil:
return start(<char*>'.')
#def python_main():
# start(<char*>'.')
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""\
{
"%s": {
"stat": %s
}
},
""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
elif not self.st.st_data.st_dev in dev_whitelist:
return
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"digests": {
"md5": "%s",
"sha1": "%s",
"sha256": "%s",
"sha512": "%s"
}
}
},
""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""\
{
"%s": {
"stat": %s,
"target": "%s"
}
},
""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "flb/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "flb/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
fprintf(stderr, "test 001\n") # DEBUG
fflush(stderr) # DEBUG
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
fprintf(stderr, "test 002\n") # DEBUG
fflush(stderr) # DEBUG
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
return -1
printf("WRITE_NOTE STAGE\n\n") # DEBUG
fscanf(address_path, "%s", ip_address)
fclose(address_path)
fprintf(result, '[\n')
fprintf(result, '"%s",\n', ip_address)
node.write_node(result)
fprintf(result, ' {}\n]\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
return 0
cdef public int main() nogil:
return start(<char*>'.')
#def python_main():
# start(<char*>'.')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment