Skip to content

M
metadata-collect-agent
Commit caaa89be authored by root's avatar root
Browse files
Options

Add a light data version in the Cython+ filesystem scanner. Change minor elements or various files.

parent 62ff37e4
Hide whitespace changes
Inline Side-by-side
Showing with 406 additions and 37 deletions
dracut.module/90metadata-collect/flb.conf
View file @ caaa89be
......@@ -14,4 +14,4 @@
buffer_type memory
flush_interval 60s
disable_retry_limit true
reference test_server
reference c-light-data_02
dracut.module/collect-sh-template.mk
View file @ caaa89be
......@@ -3,6 +3,8 @@ define collect_sh :=
. /lib/dracut-lib.sh
>&2 echo "$$NEWROOT"
sleep 10
/sbin/metadata-collect-agent "$$NEWROOT"
......
install.sh
View file @ caaa89be
......@@ -90,8 +90,8 @@ if [ ! -e /etc/uefi-key/db.crt ]; then
openssl req -newkey rsa:2048 -nodes -keyout /etc/uefi-key/db.key -new -x509 -sha256 -days 36500 -subj "/CN=TEST" -out /etc/uefi-key/db.crt
openssl x509 -outform DER -in /etc/uefi-key/db.crt -out /etc/uefi-key/db.cer
fi
'
source secure-boot-automation.sh
#'
#source secure-boot-automation.sh
rm -rf dracut_tmp
mkdir dracut_tmp
......@@ -113,4 +113,4 @@ cp /etc/uefi-key/db.cer /EFI/db.cer
cp /etc/uefi-key/db.cer /boot/efi/db.cer
uefi=$(ls /EFI/Linux)
efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI wrong keys' --loader /EFI/Linux/$uefi
efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI test_root' --loader /EFI/Linux/$uefi
scan-filesystem/cython/command-line.main.pyx
View file @ caaa89be
......@@ -80,13 +80,14 @@ cdef cypclass DirNode(Node):
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
#""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
#"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
......@@ -247,7 +248,7 @@ cdef int start(const char *path) nogil:
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
cdef char ip_address[100]
#cdef char ip_address[100]
#cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
......@@ -268,6 +269,9 @@ cdef int start(const char *path) nogil:
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist_paths.append(b'/sysroot')
dev_whitelist_paths.append(path)
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
......@@ -290,7 +294,7 @@ cdef int start(const char *path) nogil:
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
result = fopen('flb/metadata_collect.log', 'w')
if result is NULL:
fprintf(stderr, 'Error creating the log file.\n') # DEBUG
fflush(stderr)
......@@ -298,11 +302,11 @@ cdef int start(const char *path) nogil:
fprintf(stderr, 'Log opened successfully.\n') # DEBUG
fprintf(stderr, 'WRITE_NOTE STAGE\n\n') # DEBUG
fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
#fscanf(address_path, "%s", ip_address) # ADDRESS
#fclose(address_path) # ADDRESS
#fprintf(result, '{"mac_address": "%s"}\n', ip_address) # ADDRESS
#fscanf(address_path, "%s", ip_address)
#fclose(address_path)
#fprintf(result, '{"mac_address": "%s"}\n', ip_address)
node.write_node(result)
fprintf(result, '{}\n')
fprintf(result, 'fluentbit_end\n')
......@@ -319,8 +323,11 @@ cdef int start(const char *path) nogil:
return 0
cdef public int main() nogil:
return start(<char*>'/')
cdef public int main(int argc, char* argv[]) nogil:
if argc >= 2:
return start(<char*>argv[1])
else:
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
scan-filesystem/cython/main.pyx
View file @ caaa89be
# distutils: language = c++
# TODO:
# + take the needed hashes (and more broadly, the needed informations) as a parameter
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
......@@ -40,10 +43,17 @@ cdef cypclass Node activable:
pass
void format_node(self):
#''' light data version
self.formatted = sprintf("""{"path": "%s"}\n""",
self.path,
)
#'''
''' full data version
self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
#'''
void write_node(self, FILE * stream):
# abstract
......@@ -84,7 +94,7 @@ cdef cypclass DirNode(Node):
if dev_whitelist is not NULL:
if self.st is NULL:
return
#""" temporarily desabling white list
""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
#"""
......@@ -114,10 +124,17 @@ cdef cypclass DirNode(Node):
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void format_node(self):
#''' light data version
self.formatted = sprintf("""{"path": "%s/"}\n""",
self.path,
)
#'''
''' full data version
self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
#'''
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
......@@ -194,6 +211,13 @@ cdef cypclass FileNode(Node):
if self.error:
Node.format_node(self)
else:
#''' light data version
self.formatted = sprintf("""{"path: "%s", "hash": {"md5": "%s"}}\n""",
self.path,
self.md5_data,
)
#'''
''' full data version
self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
self.path,
self.st.to_json(),
......@@ -202,6 +226,7 @@ cdef cypclass FileNode(Node):
self.sha256_data,
self.sha512_data,
)
#'''
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
......@@ -223,11 +248,19 @@ cdef cypclass SymlinkNode(Node):
if self.error:
Node.format_node(self)
else:
#''' light data version
self.formatted = sprintf("""{"path": "%s", "target": "%s"}\n""",
self.path,
self.target,
)
#'''
''' full data version
self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
self.path,
self.st.to_json(),
self.target,
)
self.path,
self.st.to_json(),
self.target,
)
#'''
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
......
scan-filesystem/cython/new.uefi.main.pyx 0 → 100644
View file @ caaa89be
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void format_node(self):
self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
#cdef char ip_address[100]
#cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist_paths.append(b'/sysroot')
dev_whitelist_paths.append(path)
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
fprintf(stderr, 'Error creating the log file.\n') # DEBUG
fflush(stderr)
return -1
fprintf(stderr, 'Log opened successfully.\n') # DEBUG
fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
#fscanf(address_path, "%s", ip_address)
#fclose(address_path)
#fprintf(result, '{"mac_address": "%s"}\n', ip_address)
node.write_node(result)
fprintf(result, '{}\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
fflush(stderr) # DEBUG
return 0
cdef public int main(int argc, char* argv[]) nogil:
if argc >= 2:
return start(<char*>argv[1])
else:
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
scan-filesystem/cython/uefi.main.pyx
View file @ caaa89be
......@@ -80,13 +80,14 @@ cdef cypclass DirNode(Node):
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
#""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
#"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
......@@ -267,7 +268,8 @@ cdef int start(const char *path) nogil:
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
......@@ -276,8 +278,6 @@ cdef int start(const char *path) nogil:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
fprintf(stderr, "test 001\n") # DEBUG
fflush(stderr) # DEBUG
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
......@@ -286,18 +286,9 @@ cdef int start(const char *path) nogil:
if node is NULL:
return -1
fprintf(stderr, "test 002\n") # DEBUG
fflush(stderr) # DEBUG
active_node = activate(consume node)
fprintf(stderr, 'test 003\n') # DEBUG
fflush(stderr) # DEBUG
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
fprintf(stderr, 'test 004\n') # DEBUG
fflush(stderr) # DEBUG
scheduler.finish()
fprintf(stderr, 'test 005\n') # DEBUG
fflush(stderr) # DEBUG
node = consume active_node
#""" # DEBUG
......@@ -331,7 +322,7 @@ cdef int start(const char *path) nogil:
return 0
cdef public int main() nogil:
return start(<char*>'.')
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
secure-boot-automation.sh
View file @ caaa89be
......@@ -4,7 +4,7 @@
## KEYS CREATION
if false; then # DEBUG
rm -rf secure-boot-automation
mkdir -p secure-boot-automation
cd secure-boot-automation
......@@ -41,12 +41,13 @@ sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k KEK.key -c KEK.crt db DB.esl DB.auth
chmod 0600 *.key
fi # DEBUG
if [ ! -e /etc/uefi-key/db.crt ]; then
cp DB.crt /etc/uefi-key/db.crt
cp DB.cer /etc/uefi-key/db.cer
cp DB.key /etc/uefi-key/db.key
fi
if false; then # DEBUG
## SECURE BOOT SETUP
......@@ -54,3 +55,5 @@ fi
efi-updatevar -f PK.auth PK
efi-updatevar -a -c KEK.crt -k PK.key KEK
efi-updatevar -a -c DB.crt -k KEK.key db
fi # DEBUG
cd ..
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7