Add a light data version in the Cython+ filesystem scanner. Change minor elements or various files.

dracut.module/90metadata-collect/flb.conf

@@ -14,4 +14,4 @@
   buffer_type memory
   flush_interval 60s
   disable_retry_limit true
-  reference test_server
+  reference c-light-data_02

dracut.module/collect-sh-template.mk

@@ -3,6 +3,8 @@ define collect_sh :=
 
 . /lib/dracut-lib.sh
 
+>&2 echo "$$NEWROOT"
+sleep 10
 
 /sbin/metadata-collect-agent "$$NEWROOT"
 

install.sh

@@ -90,8 +90,8 @@ if [ ! -e /etc/uefi-key/db.crt ]; then
   openssl req -newkey rsa:2048 -nodes -keyout /etc/uefi-key/db.key -new -x509 -sha256 -days 36500 -subj "/CN=TEST" -out /etc/uefi-key/db.crt
   openssl x509 -outform DER -in /etc/uefi-key/db.crt -out /etc/uefi-key/db.cer
 fi
-'
-source secure-boot-automation.sh
+#'
+#source secure-boot-automation.sh
 
 rm -rf dracut_tmp
 mkdir dracut_tmp
@@ -113,4 +113,4 @@ cp /etc/uefi-key/db.cer /EFI/db.cer
 cp /etc/uefi-key/db.cer /boot/efi/db.cer
 uefi=$(ls /EFI/Linux)
 
-efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI wrong keys' --loader /EFI/Linux/$uefi
+efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI test_root' --loader /EFI/Linux/$uefi

scan-filesystem/cython/command-line.main.pyx

@@ -80,13 +80,14 @@ cdef cypclass DirNode(Node):
         if ignore_paths is not NULL:
             if self.path in ignore_paths:
                 return
-
+        
         if dev_whitelist is not NULL:
             if self.st is NULL:
                 return
+            #""" temporarily desabling white list
             elif not self.st.st_data.st_dev in dev_whitelist:
                 return
-
+            #"""
         d = opendir(self.path.c_str())
         if d is not NULL:
             while 1:
@@ -247,7 +248,7 @@ cdef int start(const char *path) nogil:
     cdef pid_t child_pid = -1 # DEBUG
     child_pid = fork() # DEBUG
     cdef int err
-    cdef char ip_address[100]
+    #cdef char ip_address[100]
     #cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
     if child_pid == 0:  # CHILD
         err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
@@ -268,6 +269,9 @@ cdef int start(const char *path) nogil:
         dev_whitelist_paths.append(b'.')
         dev_whitelist_paths.append(b'/')
         dev_whitelist_paths.append(b'/boot/efi')
+        dev_whitelist_paths.append(b'/root')
+        dev_whitelist_paths.append(b'/sysroot')
+        dev_whitelist_paths.append(path)
 
         dev_whitelist = cyplist[dev_t]()
         for p in dev_whitelist_paths:
@@ -290,7 +294,7 @@ cdef int start(const char *path) nogil:
         node = consume active_node
 
         #""" # DEBUG
-        result = fopen('/var/log/metadata_collect.log', 'w')
+        result = fopen('flb/metadata_collect.log', 'w')
         if result is NULL:
             fprintf(stderr, 'Error creating the log file.\n') # DEBUG
             fflush(stderr)
@@ -298,11 +302,11 @@ cdef int start(const char *path) nogil:
 
         fprintf(stderr, 'Log opened successfully.\n') # DEBUG
 
-        fprintf(stderr, 'WRITE_NOTE STAGE\n\n') # DEBUG
+        fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
 
-        #fscanf(address_path, "%s", ip_address) # ADDRESS
-        #fclose(address_path) # ADDRESS
-        #fprintf(result, '{"mac_address": "%s"}\n', ip_address) # ADDRESS
+        #fscanf(address_path, "%s", ip_address)
+        #fclose(address_path)
+        #fprintf(result, '{"mac_address": "%s"}\n', ip_address)
         node.write_node(result)
         fprintf(result, '{}\n')
         fprintf(result, 'fluentbit_end\n')
@@ -319,8 +323,11 @@ cdef int start(const char *path) nogil:
 
         return 0
 
-cdef public int main() nogil:
-    return start(<char*>'/')
+cdef public int main(int argc, char* argv[]) nogil:
+    if argc >= 2:
+        return start(<char*>argv[1])
+    else:
+        return start(<char*>'/')
 
 #def python_main():
 #    start(<char*>'.')

scan-filesystem/cython/main.pyx

 # distutils: language = c++
 
+# TODO:
+# + take the needed hashes (and more broadly, the needed informations) as a parameter
+
 from libcythonplus.list cimport cyplist
 
 from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
@@ -40,10 +43,17 @@ cdef cypclass Node activable:
         pass
 
     void format_node(self):
+        #''' light data version
+        self.formatted = sprintf("""{"path": "%s"}\n""",
+            self.path,
+        )
+        #'''
+        ''' full data version
         self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
             self.path,
             self.st.to_json(),
         )
+        #'''
 
     void write_node(self, FILE * stream):
         # abstract
@@ -84,7 +94,7 @@ cdef cypclass DirNode(Node):
         if dev_whitelist is not NULL:
             if self.st is NULL:
                 return
-            #""" temporarily desabling white list
+            """ temporarily desabling white list
             elif not self.st.st_data.st_dev in dev_whitelist:
                 return
             #"""
@@ -114,10 +124,17 @@ cdef cypclass DirNode(Node):
             active_child.build_node(NULL, dev_whitelist, ignore_paths)
 
     void format_node(self):
+        #''' light data version
+        self.formatted = sprintf("""{"path": "%s/"}\n""",
+            self.path,
+        )
+        #'''
+        ''' full data version
         self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
             self.path,
             self.st.to_json(),
         )
+        #'''
 
     void write_node(self, FILE * stream):
         fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
@@ -194,6 +211,13 @@ cdef cypclass FileNode(Node):
         if self.error:
             Node.format_node(self)
         else:
+            #''' light data version
+            self.formatted = sprintf("""{"path: "%s", "hash": {"md5": "%s"}}\n""",
+                self.path,
+                self.md5_data,
+            )
+            #'''
+            ''' full data version
             self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
                 self.path,
                 self.st.to_json(),
@@ -202,6 +226,7 @@ cdef cypclass FileNode(Node):
                 self.sha256_data,
                 self.sha512_data,
             )
+            #'''
 
     void write_node(self, FILE * stream):
         fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
@@ -223,11 +248,19 @@ cdef cypclass SymlinkNode(Node):
         if self.error:
             Node.format_node(self)
         else:
+            #''' light data version
+            self.formatted = sprintf("""{"path": "%s", "target": "%s"}\n""",
+                self.path,
+                self.target,
+            )
+            #'''
+            ''' full data version
             self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
-            self.path,
-            self.st.to_json(),
-            self.target,
-        )
+                self.path,
+                self.st.to_json(),
+                self.target,
+            )
+            #'''
 
     void write_node(self, FILE * stream):
         fwrite(self.formatted.data(), 1, self.formatted.size(), stream)

scan-filesystem/cython/new.uefi.main.pyx

+# distutils: language = c++
+
+from libcythonplus.list cimport cyplist
+
+from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
+
+from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
+from runtime.unistd cimport pid_t, execlp, fork, sleep
+from runtime.wait cimport wait, waitpid
+
+from stdlib.stat cimport Stat, dev_t
+from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
+from stdlib.fmt cimport sprintf
+from stdlib.string cimport string
+from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
+
+from posix.stdlib cimport realpath
+
+from posix.unistd cimport readlink
+
+
+cdef lock Scheduler scheduler
+
+
+cdef cypclass Node activable:
+    string path
+    string name
+    Stat st
+    string formatted
+
+    __init__(self, string path, string name, Stat st):
+        self._active_result_class = NullResult
+        self._active_queue_class = consume BatchMailBox(scheduler)
+        self.path = path
+        self.name = name
+        self.st = st
+
+    void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
+        # abstract
+        pass
+
+    void format_node(self):
+        self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
+            self.path,
+            self.st.to_json(),
+        )
+
+    void write_node(self, FILE * stream):
+        # abstract
+        pass
+
+
+cdef iso Node make_node(string path, string name) nogil:
+    s = Stat(path)
+    if s is NULL:
+        return NULL
+    elif s.is_symlink():
+        return consume SymlinkNode(path, name, consume s)
+    elif s.is_dir():
+        return consume DirNode(path, name, consume s)
+    elif s.is_regular():
+        return consume FileNode(path, name, consume s)
+    return NULL
+
+
+cdef cypclass DirNode(Node):
+    cyplist[active Node] children
+
+    __init__(self, string path, string name, Stat st):
+        Node.__init__(self, path, name, st)
+        self.children = new cyplist[active Node]()
+        self.children.__init__()
+
+    void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
+        cdef DIR *d
+        cdef struct_dirent *entry
+        cdef string entry_name
+        cdef string entry_path
+
+        if ignore_paths is not NULL:
+            if self.path in ignore_paths:
+                return
+        
+        if dev_whitelist is not NULL:
+            if self.st is NULL:
+                return
+            """ temporarily desabling white list
+            elif not self.st.st_data.st_dev in dev_whitelist:
+                return
+            """
+        d = opendir(self.path.c_str())
+        if d is not NULL:
+            while 1:
+                entry = readdir(d)
+                if entry is NULL:
+                    break
+                entry_name = entry.d_name
+                if entry_name == b'.' or entry_name == b'..':
+                    continue
+                entry_path = self.path
+                if entry_path != b'/':
+                    entry_path += b'/'
+                entry_path += entry_name
+                entry_node = make_node(entry_path, entry_name)
+                if entry_node is NULL:
+                    continue
+                active_entry = activate(consume entry_node)
+                self.children.append(active_entry)
+            closedir(d)
+
+        self.format_node()
+
+        for active_child in self.children:
+            active_child.build_node(NULL, dev_whitelist, ignore_paths)
+
+    void format_node(self):
+        self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
+            self.path,
+            self.st.to_json(),
+        )
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+        while self.children.__len__() > 0:
+            active_child = self.children[self.children.__len__() -1]
+            del self.children[self.children.__len__() -1]
+            child = consume active_child
+            child.write_node(stream)
+
+
+cdef enum:
+    BUFSIZE = 64 * 1024
+
+cdef cypclass FileNode(Node):
+    string md5_data
+    string sha1_data
+    string sha256_data
+    string sha512_data
+    bint error
+
+    __init__(self, string path, string name, Stat st):
+        Node.__init__(self, path, name, st)
+        self.error = False
+
+    void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
+        cdef unsigned char buffer[BUFSIZE]
+        cdef bint eof = False
+        cdef bint md5_ok
+        cdef bint sha1_ok
+        cdef bint sha256_ok
+        cdef bint sha512_ok
+
+        cdef FILE * file = fopen(self.path.c_str(), 'rb')
+
+        if file is NULL:
+            self.error = True
+            self.format_node()
+            return
+
+        md5 = MessageDigest(md5sum())
+        sha1 = MessageDigest(sha1sum())
+        sha256 = MessageDigest(sha256sum())
+        sha512 = MessageDigest(sha512sum())
+
+        md5_ok = md5 is not NULL
+        sha1_ok = sha1 is not NULL
+        sha256_ok = sha256 is not NULL
+        sha512_ok = sha512 is not NULL
+
+        while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
+            size = fread(buffer, 1, BUFSIZE, file)
+            if size != BUFSIZE:
+                self.error = ferror(file)
+                if self.error:
+                    break
+                eof = True
+
+            if md5_ok: md5_ok = md5.update(buffer, size) == 0
+            if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
+            if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
+            if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
+
+        fclose(file)
+
+        if not self.error:
+            if md5_ok: self.md5_data = md5.hexdigest()
+            if sha1_ok: self.sha1_data = sha1.hexdigest()
+            if sha256_ok: self.sha256_data = sha256.hexdigest()
+            if sha512_ok: self.sha512_data = sha512.hexdigest()
+
+        self.format_node()
+
+    void format_node(self):
+        if self.error:
+            Node.format_node(self)
+        else:
+            self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
+                self.path,
+                self.st.to_json(),
+                self.md5_data,
+                self.sha1_data,
+                self.sha256_data,
+                self.sha512_data,
+            )
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+
+
+cdef cypclass SymlinkNode(Node):
+    string target
+    int error
+
+    void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
+        size = self.st.st_data.st_size + 1
+        self.target.resize(size)
+        real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
+        self.error = not (0 < real_size < size)
+        self.target.resize(real_size)
+        self.format_node()
+
+    void format_node(self):
+        if self.error:
+            Node.format_node(self)
+        else:
+            self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
+            self.path,
+            self.st.to_json(),
+            self.target,
+        )
+
+    void write_node(self, FILE * stream):
+        fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
+
+
+cdef int start(const char *path) nogil:
+
+    printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
+
+    # TODO replace 4096 by PATH_MAX (yet it will not be perfect)
+    cdef char resolved_path[4096]
+    cdef pid_t wait_error = -1 # DEBUG
+    cdef char* program_name = "fluentbit"
+    cdef char* arg1 = "-e"
+    cdef char* arg2 = "/etc/fluentbit_wendelin.so"
+    cdef char* arg3 = "-c"
+    cdef char* arg4 = "/etc/flb.conf"
+    cdef pid_t child_pid = -1 # DEBUG
+    child_pid = fork() # DEBUG
+    cdef int err
+    #cdef char ip_address[100]
+    #cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
+    if child_pid == 0:  # CHILD
+        err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
+        fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
+    else:               # PARENT
+        printf("WELCOME TO PARENT\n\n") # DEBUG
+
+        sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
+
+        global scheduler
+        scheduler = Scheduler()
+
+        ignore_paths = cyplist[string]()
+        ignore_paths.append(b'/opt/slapgrid')
+        ignore_paths.append(b'/srv/slapgrid')
+
+        dev_whitelist_paths = cyplist[string]()
+        dev_whitelist_paths.append(b'.')
+        dev_whitelist_paths.append(b'/')
+        dev_whitelist_paths.append(b'/boot/efi')
+        dev_whitelist_paths.append(b'/root')
+        dev_whitelist_paths.append(b'/sysroot')
+        dev_whitelist_paths.append(path)
+
+        dev_whitelist = cyplist[dev_t]()
+        for p in dev_whitelist_paths:
+            p_stat = Stat(p)
+            if p_stat is not NULL:
+                p_dev = p_stat.st_data.st_dev
+                dev_whitelist.append(p_dev)
+       
+        realpath(path, resolved_path)
+        fprintf(stderr, resolved_path) # DEBUG
+        fprintf(stderr, "\n") # DEBUG
+        fflush(stderr) # DEBUG
+        node = make_node(resolved_path, resolved_path)
+        if node is NULL:
+            return -1
+
+        active_node = activate(consume node)
+        active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
+        scheduler.finish()
+        node = consume active_node
+
+        #""" # DEBUG
+        result = fopen('/var/log/metadata_collect.log', 'w')
+        if result is NULL:
+            fprintf(stderr, 'Error creating the log file.\n') # DEBUG
+            fflush(stderr)
+            return -1
+
+        fprintf(stderr, 'Log opened successfully.\n') # DEBUG
+
+        fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
+
+        #fscanf(address_path, "%s", ip_address)
+        #fclose(address_path)
+        #fprintf(result, '{"mac_address": "%s"}\n', ip_address)
+        node.write_node(result)
+        fprintf(result, '{}\n')
+        fprintf(result, 'fluentbit_end\n')
+
+        fclose(result)
+        #""" # DEBUG
+        del scheduler
+
+        fprintf(stderr, "WAITING for fluent-bit to end\n\n")
+        wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
+        #wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
+        fprintf(stderr, "WAITING ENDS\n\n")
+        fflush(stderr) # DEBUG
+
+        return 0
+
+cdef public int main(int argc, char* argv[]) nogil:
+    if argc >= 2:
+        return start(<char*>argv[1])
+    else:
+        return start(<char*>'/')
+
+#def python_main():
+#    start(<char*>'.')

scan-filesystem/cython/uefi.main.pyx

@@ -80,13 +80,14 @@ cdef cypclass DirNode(Node):
         if ignore_paths is not NULL:
             if self.path in ignore_paths:
                 return
-
+        
         if dev_whitelist is not NULL:
             if self.st is NULL:
                 return
+            #""" temporarily desabling white list
             elif not self.st.st_data.st_dev in dev_whitelist:
                 return
-
+            #"""
         d = opendir(self.path.c_str())
         if d is not NULL:
             while 1:
@@ -267,7 +268,8 @@ cdef int start(const char *path) nogil:
         dev_whitelist_paths = cyplist[string]()
         dev_whitelist_paths.append(b'.')
         dev_whitelist_paths.append(b'/')
-        dev_whitelist_paths.append(b'/boot')
+        dev_whitelist_paths.append(b'/boot/efi')
+        dev_whitelist_paths.append(b'/root')
 
         dev_whitelist = cyplist[dev_t]()
         for p in dev_whitelist_paths:
@@ -276,8 +278,6 @@ cdef int start(const char *path) nogil:
                 p_dev = p_stat.st_data.st_dev
                 dev_whitelist.append(p_dev)
        
-        fprintf(stderr, "test 001\n") # DEBUG
-        fflush(stderr) # DEBUG
         realpath(path, resolved_path)
         fprintf(stderr, resolved_path) # DEBUG
         fprintf(stderr, "\n") # DEBUG
@@ -286,18 +286,9 @@ cdef int start(const char *path) nogil:
         if node is NULL:
             return -1
 
-        fprintf(stderr, "test 002\n") # DEBUG
-        fflush(stderr) # DEBUG
-
         active_node = activate(consume node)
-        fprintf(stderr, 'test 003\n') # DEBUG
-        fflush(stderr) # DEBUG
         active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
-        fprintf(stderr, 'test 004\n') # DEBUG
-        fflush(stderr) # DEBUG
         scheduler.finish()
-        fprintf(stderr, 'test 005\n') # DEBUG
-        fflush(stderr) # DEBUG
         node = consume active_node
 
         #""" # DEBUG
@@ -331,7 +322,7 @@ cdef int start(const char *path) nogil:
         return 0
 
 cdef public int main() nogil:
-    return start(<char*>'.')
+    return start(<char*>'/')
 
 #def python_main():
 #    start(<char*>'.')

secure-boot-automation.sh

@@ -4,7 +4,7 @@
 
 
 ## KEYS CREATION
-
+if false; then # DEBUG
 rm -rf secure-boot-automation
 mkdir -p secure-boot-automation
 cd secure-boot-automation
@@ -41,12 +41,13 @@ sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
                   -k KEK.key -c KEK.crt db DB.esl DB.auth
 
 chmod 0600 *.key
-
+fi # DEBUG
 if [ ! -e /etc/uefi-key/db.crt ]; then
   cp DB.crt /etc/uefi-key/db.crt
   cp DB.cer /etc/uefi-key/db.cer
+  cp DB.key /etc/uefi-key/db.key
 fi
-
+if false; then # DEBUG
 
 ## SECURE BOOT SETUP
 
@@ -54,3 +55,5 @@ fi
 efi-updatevar -f PK.auth PK
 efi-updatevar -a -c KEK.crt -k PK.key KEK
 efi-updatevar -a -c DB.crt -k KEK.key db
+fi # DEBUG
+cd ..