Skip to content

M
metadata-collect-agent
Commit 4703ad1d authored by Ophélie Gagnard's avatar Ophélie Gagnard
Browse files
Options

WIP: Clean project and installation process.

parent 0f83f6e3
Expand all Show whitespace changes
Inline Side-by-side
Showing with 31 additions and 2132 deletions
Cargo.lock deleted 100644 → 0
View file @ 0f83f6e3
This diff is collapsed.
Cargo.toml deleted 100644 → 0
View file @ 0f83f6e3
[package]
name = "metadata-collect-agent"
version = "0.1.0"
authors = ["Leo Le Bouter <leo.le.bouter@nexedi.com>"]
edition = "2018"
[dependencies]
posix-acl = "1.0.0"
xattr = "0.2.2"
md-5 = "0.9.1"
sha-1 = "0.9.1"
sha2 = "0.9.1"
hex = "0.4.2"
anyhow = "1.0.32"
clap = "2.33.3"
psutil = { version = "3.2.0", features = ["serde"] }
reqwest = { version = "0.10.7", default-features = false, features = ["blocking", "rustls-tls"] }
rmp-serde = "0.14.4"
nix = "0.19.0"
serde = { version = "1.0.115", features = ["derive"] }
base64 = "0.13.0"
rayon = "1.4.0"
serde_json = "1.0.57"
[profile.release]
opt-level = 'z'
lto = true
codegen-units = 1
debian.package.unsafe/Makefile deleted 100644 → 0
View file @ 0f83f6e3
unsafe-boot-metadata-collect-agent.deb: unsafe-boot-metadata-collect-agent/usr/share/initramfs-tools/scripts/local-bottom/metadata-collect-agent
dpkg-deb --build unsafe-boot-metadata-collect-agent || rm -fv $@
include initramfs-script-template.mk
unsafe-boot-metadata-collect-agent/usr/share/initramfs-tools/scripts/local-bottom/metadata-collect-agent: unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent
@ if [ "${ERP5_USER}" = "" ]; then \
echo "Environment variable ERP5_USER not set"; \
exit 1; \
fi
@ if [ "${ERP5_PASS}" = "" ]; then \
echo "Environment variable ERP5_PASS not set"; \
exit 1; \
fi
@ if [ "${ERP5_BASE_URL}" = "" ]; then \
echo "Environment variable ERP5_BASE_URL not set"; \
exit 1; \
fi
mkdir -p unsafe-boot-metadata-collect-agent/usr/share/initramfs-tools/scripts/local-bottom
echo "$${initramfs_script}" >> $@
unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent:
cd ../ && ./rust-build-static.bash
.PHONY: clean
clean:
rm -fv unsafe-boot-metadata-collect-agent/usr/share/initramfs-tools/scripts/local-bottom/metadata-collect-agent unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent unsafe-boot-metadata-collect-agent.deb
\ No newline at end of file
debian.package.unsafe/initramfs-script-template.mk deleted 100644 → 0
View file @ 0f83f6e3
define initramfs_script :=
#!/bin/sh
PREREQ=""
prereqs()
{
echo "$$PREREQ"
}
case $$1 in
prereqs)
prereqs
exit 0
;;
esac
. /scripts/functions
configure_networking
DEFAULT_IF="$$(ip route show default | grep -Po "(?<=dev )\w+" | head -n 1)"
DEFAULT_IF_MAC="$$(ip link show "$$DEFAULT_IF" | grep -Po '(?<= )\w{2}:\w{2}:\w{2}:\w{2}:\w{2}:\w{2}(?= )' | head -n 1)"
ERP5_BASE_URL="$(ERP5_BASE_URL)"
ERP5_USER="$(ERP5_USER)"
ERP5_PASS="$(ERP5_PASS)"
REFERENCE="COMP-MAC-$$(echo "$$DEFAULT_IF_MAC" | sed s/:/-/g).Metadata.Snapshot"
/sbin/metadata-collect-agent "$$NEWROOT" "$$ERP5_USER" "$$ERP5_PASS" "$$REFERENCE" "$$ERP5_BASE_URL"
endef
export initramfs_script
\ No newline at end of file
debian.package.unsafe/unsafe-boot-metadata-collect-agent/DEBIAN/control deleted 100644 → 0
View file @ 0f83f6e3
Package: unsafe-boot-metadata-collect-agent
Version: 0.1
Section: custom
Priority: optional
Architecture: amd64
Essential: no
Maintainer: leo.le.bouter@nexedi.com
Description: metadata-collect-agent with initramfs scripts
debian.package.unsafe/unsafe-boot-metadata-collect-agent/usr/share/initramfs-tools/hooks/metadata-collect-agent deleted 100644 → 0
View file @ 0f83f6e3
#!/bin/sh
PREREQ=""
prereqs()
{
echo "$PREREQ"
}
case $1 in
prereqs)
prereqs
exit 0
;;
esac
. /usr/share/initramfs-tools/hook-functions
copy_exec /sbin/metadata-collect-agent /sbin
\ No newline at end of file
debian.package/Makefile deleted 100644 → 0
View file @ 0f83f6e3
SECBOOT_CRT=$(shell pwd)/db.crt
SECBOOT_KEY=$(shell pwd)/db.key
guard-%:
@ if [ "${${*}}" = "" ]; then \
echo "Environment variable $* not set"; \
exit 1; \
fi
uefi-boot-metadata-collect-agent.deb: uefi-boot-metadata-collect-agent/boot/efi/EFI/Nexedi/uefi-boot-metadata-collect-agent.efi
dpkg-deb --build uefi-boot-metadata-collect-agent || rm -fv $@
include dracut-conf-template.mk
uefi-boot-metadata-collect-agent/boot/efi/EFI/Nexedi/uefi-boot-metadata-collect-agent.efi: db.crt db.key db.cer uefi-boot-metadata-collect-agent/boot/efi/secboot.cer
mkdir -p /tmp/dracut-empty
echo "$${dracut_conf}" > dracut.conf
sudo dracut --force -c dracut.conf --confdir /tmp/dracut-empty --uefi-output $@
sudo chmod 755 $@
rm -rfv /tmp/dracut-empty
uefi-boot-metadata-collect-agent/boot/efi/secboot.cer: uefi-boot-metadata-collect-agent/boot/efi/EFI/Nexedi db.cer
cp -fv db.cer $@
sudo chown 0:0 $@
sudo chmod 755 $@
uefi-boot-metadata-collect-agent/boot/efi/EFI/Nexedi:
mkdir -p $@
db.crt db.key:
openssl req -newkey rsa:4096 -nodes -keyout db.key -new -x509 -sha256 -days 3650 -subj "/CN=Nexedi's UEFI Signature Database key/" -out db.crt
db.cer:
openssl x509 -outform DER -in db.crt -out db.cer
clean:
rm -fv uefi-boot-metadata-collect-agent.deb dracut.conf db.key db.crt db.cer uefi-boot-metadata-collect-agent/boot/efi/EFI/Nexedi/uefi-boot-metadata-collect-agent.efi uefi-boot-metadata-collect-agent/boot/efi/secboot.cer
\ No newline at end of file
debian.package/dracut-conf-template.mk deleted 100644 → 0
View file @ 0f83f6e3
define dracut_conf :=
add_dracutmodules="metadata-collect"
hostonly=no
hostonly_cmdline=no
kernel_cmdline="root=LABEL=ROOT rd.auto rd.auto=1 rd.neednet=1 ip=dhcp"
reproducible=yes
compress=xz
uefi=yes
uefi_stub=/usr/lib/systemd/boot/efi/linuxx64.efi.stub
uefi_secureboot_cert="$(SECBOOT_CRT)"
uefi_secureboot_key="$(SECBOOT_KEY)"
endef
export dracut_conf
\ No newline at end of file
debian.package/uefi-boot-metadata-collect-agent/DEBIAN/control deleted 100644 → 0
View file @ 0f83f6e3
Package: uefi-boot-metadata-collect-agent
Version: 0.1
Section: custom
Priority: optional
Architecture: amd64
Essential: no
Maintainer: leo.le.bouter@nexedi.com
Description: Signed UEFI boot application with embedded pre-configured metadata-collect-agent
debian.package/uefi-boot-metadata-collect-agent/DEBIAN/postinst deleted 100755 → 0
View file @ 0f83f6e3
#!/bin/bash
EFI_DEV=$(mount | grep -Po '^.+(?= on /boot/efi )')
ROOT_DEV=$(mount | grep -Po '^.+(?= on / )')
if ! efibootmgr --disk "$EFI_DEV" --verbose | grep "uefi-boot-metadata-collect-agent.efi"; then
efibootmgr --create --disk "$EFI_DEV" --label 'UEFI metadata-collect-agent' --loader /EFI/Nexedi/uefi-boot-metadata-collect-agent.efi
#BOOT_CUR=$(efibootmgr | grep -Po '(?<=BootCurrent: ).+')
#BOOT_AGENT=$(efibootmgr | grep -Po '(?<=Boot)\d+(?=.+UEFI metadata-collect-agent)')
#NEW_BOOT_ORDER=$(efibootmgr | grep -Po '(?<=BootOrder: ).+' | sed "s/$BOOT_CUR/$BOOT_AGENT,$BOOT_CUR/")
#efibootmgr -o "$NEW_BOOT_ORDER"
fi
e2label "$ROOT_DEV" "ROOT" || true
btrfs filesystem label / "ROOT" || true
reiserfstune -l "ROOT" "$ROOT_DEV" || true
xfs_io -c "label -s ROOT" / || true
cryptsetup config --label="ROOT" "$ROOT_DEV" || true
\ No newline at end of file
dracut.module/Makefile
View file @ 4703ad1d
dracut_module: 90metadata-collect/fluent-bit 90metadata-collect/fluentbit_wendelin.so 90metadata-collect/metadata-collect-agent 90metadata-collect/collect.sh .PHONY: all
all: 90metadata-collect/fluent-bit 90metadata-collect/fluentbit_wendelin.so 90metadata-collect/metadata-collect-agent 90metadata-collect/collect.sh
include collect-sh-template.mk include collect-sh-template.mk
90metadata-collect/collect.sh: 90metadata-collect/collect.sh:
echo "$${collect_sh}" >> 90metadata-collect/collect.sh echo "$${collect_sh}" >> 90metadata-collect/collect.sh
90metadata-collect/metadata-collect-agent: 90metadata-collect/metadata-collect-agent:
# cd ../ && ./rust-build-static.bash
cd ../scan-filesystem/cython/ && make nopython && cd ../../dracut.module cd ../scan-filesystem/cython/ && make nopython && cd ../../dracut.module
90metadata-collect/fluent-bit: 90metadata-collect/fluent-bit:
...@@ -19,7 +19,7 @@ clean: ...@@ -19,7 +19,7 @@ clean:
rm -fv 90metadata-collect/collect.sh 90metadata-collect/metadata-collect-agent rm -fv 90metadata-collect/collect.sh 90metadata-collect/metadata-collect-agent
.PHONY: install .PHONY: install
install: dracut_module install: all
cp -vLr "90metadata-collect" /usr/lib/dracut/modules.d cp -vLr "90metadata-collect" /usr/lib/dracut/modules.d
.PHONY: uninstall .PHONY: uninstall
......
fluent-bit-install.sh
View file @ 4703ad1d
apt-get install cmake flex bison build-essential apt-get -y install cmake flex bison build-essential
rm -rf fluent-bit rm -rf fluent-bit
git clone https://github.com/fluent/fluent-bit.git git clone https://github.com/fluent/fluent-bit.git
cd fluent-bit cd fluent-bit
......
guix-pkg.scm deleted 100644 → 0
View file @ 0f83f6e3
(use-modules
(guix packages)
(guix git-download)
(guix download)
(guix build python-build-system)
(guix build-system python)
(guix build utils)
(gnu packages python-web)
(gnu packages python-xyz)
(gnu packages python-crypto)
(gnu packages acl))
(define python-pylibacl
(package
(name "python-pylibacl")
(version "0.5.4")
(source
(origin
(method url-fetch)
(uri (pypi-uri "pylibacl" version))
(sha256
(base32
"0drvxb21y7p0aikcv3jx90vdcjk96kibf9x8qgxic2prxxd3f3q6"))))
(build-system python-build-system)
(arguments
'(#:tests? #f))
(propagated-inputs
`(("acl" ,acl)))
(home-page "http://pylibacl.k1024.org/")
(synopsis "POSIX.1e ACLs for python")
(description "POSIX.1e ACLs for python")
(license #f)))
(define metadata-collect-agent
(package
(name "metadata-collect-agent")
(version "0.1")
(source
(origin
(method git-fetch)
(uri (git-reference
(url "https://lab.nexedi.com/nexedi/metadata-collect-agent.git")
(commit "v0.1")))
(sha256
(base32
"")))) ;; Fill with "Actual hash" of GNU Guix error
(build-system python-build-system)
(arguments
'(#:tests? #f))
(propagated-inputs
`(("python-urllib3" ,python-urllib3)
("python-psutil" ,python-psutil)
("python-msgpack" ,python-msgpack)
("python-pylibacl" ,python-pylibacl)
("python-certifi" ,python-certifi)))
(description "")
(synopsis "")
(license #f)
(home-page "")))
(packages->manifest
(list metadata-collect-agent))
install.sh
View file @ 4703ad1d
...@@ -2,11 +2,15 @@ ...@@ -2,11 +2,15 @@
set -eux set -eux
# apt install git
# installing miscellaneous useful packages # installing miscellaneous useful packages
apt -y install make autopoint autoconf libtool libattr1-dev musl-tools mmv sbsigntool apt -y install make autopoint autoconf libtool libattr1-dev musl-tools mmv sbsigntool
# installing more necessary packages to activate Secure Boot with our own keys # installing more necessary packages to activate Secure Boot with our own keys
apt -y install efitools apt -y install efitools
# installing (download and compile)a specific version of python to make sure the whole installation process work # installing packages needed to build fluentbit
apt -y cmake flex bison build-essential
# installing (download and compile) a specific version of python to make sure the whole installation process work
: '
if [ ! -d "/opt/python-3.7.12/include/python3.7m" ]; then if [ ! -d "/opt/python-3.7.12/include/python3.7m" ]; then
wget https://www.python.org/ftp/python/3.7.12/Python-3.7.12.tgz wget https://www.python.org/ftp/python/3.7.12/Python-3.7.12.tgz
tar zxf Python-3.7.12.tgz tar zxf Python-3.7.12.tgz
...@@ -18,7 +22,9 @@ if [ ! -d "/opt/python-3.7.12/include/python3.7m" ]; then ...@@ -18,7 +22,9 @@ if [ ! -d "/opt/python-3.7.12/include/python3.7m" ]; then
fi fi
rm -f Python-3.7.12.tgz rm -f Python-3.7.12.tgz
rm -rf Python-3.7.12/ rm -rf Python-3.7.12/
#'
: '
## Install dracut ## Install dracut
if ! which dracut ; then if ! which dracut ; then
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-core_051-1_amd64.deb wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-core_051-1_amd64.deb
...@@ -27,12 +33,7 @@ if ! which dracut ; then ...@@ -27,12 +33,7 @@ if ! which dracut ; then
apt -y install ./dracut-core_051-1_amd64.deb ./dracut_051-1_all.deb ./dracut-network_051-1_all.deb apt -y install ./dracut-core_051-1_amd64.deb ./dracut_051-1_all.deb ./dracut-network_051-1_all.deb
rm dracut*.deb rm dracut*.deb
fi fi
#'
if [ ! -e "$HOME/.cargo/bin" ]; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
fi
export PATH=$PATH:$HOME/.cargo/bin
rustup target add x86_64-unknown-linux-musl
## Install others ## Install others
apt -y install libssl-dev apt -y install libssl-dev
...@@ -41,21 +42,23 @@ apt -y install cmake ...@@ -41,21 +42,23 @@ apt -y install cmake
## Install Go ## Install Go
# needed for dracut.module/ make install # needed for dracut.module/ make install
GOVERSION=1.16.5 #GOVERSION=1.16.5
if [ ! -e "/usr/local/go/VERSION" ] || [ go${GOVERSION} != $(cat /usr/local/go/VERSION) ]; then #if [ ! -e "/usr/local/go/VERSION" ] || [ go${GOVERSION} != $(cat /usr/local/go/VERSION) ]; then
wget https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz # wget https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz
rm -rf /usr/local/go # rm -rf /usr/local/go
tar -C /usr/local -xzf go${GOVERSION}.linux-amd64.tar.gz # tar -C /usr/local -xzf go${GOVERSION}.linux-amd64.tar.gz
rm go*.tar.gz* # rm go*.tar.gz*
# TODO: write in /etc/profile.d/ instead # # TODO: write in /etc/profile.d/ instead
echo -e "\n" >> /etc/profile # echo -e "\n" >> /etc/profile
echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile # echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile
source /etc/profile # source /etc/profile
fi #fi
apt -y install golang
## Install Cython+ ## Install Cython+
# needed for scan-filesystem/ make nopython # needed for scan-filesystem/ make nopython
# WARNING: it does not differentiate Cython+ from Cython # WARNING: it does not differentiate Cython+ from Cython
: '
if ! which cython3 ; then if ! which cython3 ; then
mkdir -p build_cythonplus && cd build_cythonplus mkdir -p build_cythonplus && cd build_cythonplus
wget https://lab.nexedi.com/nexedi/cython/raw/debian/build_cythonplus wget https://lab.nexedi.com/nexedi/cython/raw/debian/build_cythonplus
...@@ -64,7 +67,10 @@ if ! which cython3 ; then ...@@ -64,7 +67,10 @@ if ! which cython3 ; then
apt -y install ./cython3_* apt -y install ./cython3_*
cd .. cd ..
fi fi
'
apt -y install libfmt-dev
: '
## Install fmt library ## Install fmt library
# needed for scan-filesystem/ make nopython # needed for scan-filesystem/ make nopython
if [ ! -d "fmt" ]; then if [ ! -d "fmt" ]; then
...@@ -75,10 +81,13 @@ if [ ! -d "fmt" ]; then ...@@ -75,10 +81,13 @@ if [ ! -d "fmt" ]; then
make install make install
cd ../.. cd ../..
fi fi
'
: '
cd dracut.module cd dracut.module
make install make install
cd .. cd ..
'
apt -y autoremove apt -y autoremove
......
rust-build-static.bash deleted 100755 → 0
View file @ 0f83f6e3
#!/bin/bash
set -eux
ACLVERSION="2.2.53"
ACLTARGZSHA256=9e905397ac10d06768c63edd0579c34b8431555f2ea8e8f2cee337b31f856805
HOST_TARGET=${HOST_TARGET:-x86_64-unknown-linux-musl}
wget -O "acl-${ACLVERSION}.tar.gz" \
"https://git.savannah.nongnu.org/cgit/acl.git/snapshot/acl-${ACLVERSION}.tar.gz"
echo -n "$ACLTARGZSHA256 acl-${ACLVERSION}.tar.gz" | sha256sum -c /dev/stdin
tar -xvf "acl-${ACLVERSION}.tar.gz"
cd "acl-${ACLVERSION}"
export CFLAGS="-fPIE"
[ -f ./configure ] || ./autogen.sh
[ -f ./config.status ] || ./configure --host "${HOST_TARGET}"
make -j"$(nproc)"
cd -
RUSTFLAGS="-L native=$(pwd)/acl-${ACLVERSION}/.libs -l static=acl" PKG_CONFIG_ALLOW_CROSS=1 \
cargo build --target "$HOST_TARGET" --release
BINARY="$(pwd)/target/$HOST_TARGET/release/$(basename "$(pwd)")"
strip --strip-all "$BINARY"
objdump -T "$BINARY" || true
ln -sf "$BINARY" dracut.module/90metadata-collect/metadata-collect-agent || true
ln -sf "$BINARY" debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent || true
scan-filesystem/python/main.py deleted 100644 → 0
View file @ 0f83f6e3
#!/usr/bin/env python3
# Developed with Python 3.8.5
import sys
import os
import stat
import traceback
import hashlib
import io
import multiprocessing
import json
def compute_hashes(entry_path):
with open(entry_path, mode="rb") as f:
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
while True:
data = f.read(io.DEFAULT_BUFFER_SIZE)
md5.update(data)
sha1.update(data)
sha256.update(data)
sha512.update(data)
if len(data) < io.DEFAULT_BUFFER_SIZE:
break
return {"md5": md5.hexdigest(),
"sha1": sha1.hexdigest(),
"sha256": sha256.hexdigest(),
"sha512": sha512.hexdigest()}
def stat_result_to_dict(stat_result):
return {
"st_mode": getattr(stat_result, "st_mode", None),
"st_ino": getattr(stat_result, "st_ino", None),
"st_dev": getattr(stat_result, "st_dev", None),
"st_nlink": getattr(stat_result, "st_nlink", None),
"st_uid": getattr(stat_result, "st_uid", None),
"st_gid": getattr(stat_result, "st_gid", None),
"st_size": getattr(stat_result, "st_size", None),
"st_atime": getattr(stat_result, "st_atime", None),
"st_mtime": getattr(stat_result, "st_mtime", None),
"st_ctime": getattr(stat_result, "st_ctime", None),
"st_blocks": getattr(stat_result, "st_blocks", None),
"st_blksize": getattr(stat_result, "st_blksize", None),
"st_rdev": getattr(stat_result, "st_rdev", None),
"st_flags": getattr(stat_result, "st_flags", None),
"st_gen": getattr(stat_result, "st_gen", None),
"st_birthtime": getattr(stat_result, "st_birthtime", None),
}
def construct_fs_tree(mp_pool=None, mp_tasks=[], cur_dict=None, path="/", dev_whitelist=None, ignored_dirs=[]):
is_first_call = False
if mp_pool == None:
is_first_call = True
mp_pool = multiprocessing.Pool()
if cur_dict == None:
cur_dict = {"stat": stat_result_to_dict(os.stat(path, follow_symlinks=False)),
"childs": dict()}
if dev_whitelist != None:
path_stat = cur_dict["stat"]
if "st_dev" in path_stat:
if not path_stat["st_dev"] in dev_whitelist:
return cur_dict
for dir in ignored_dirs:
if path.startswith(dir):
cur_dict["ignored"] = True
return cur_dict
try:
with os.scandir(path) as it:
for entry in it:
try:
entry_path = os.fsdecode(entry.path)
entry_name = os.fsdecode(entry.name)
try:
entry_stat = os.stat(entry_path, follow_symlinks=False)
except Exception:
entry_stat = None
cur_dict["childs"][entry_name] = {"stat": stat_result_to_dict(entry_stat),
"childs": dict()}
if stat.S_ISDIR(entry_stat.st_mode):
construct_fs_tree(mp_pool=mp_pool, mp_tasks=mp_tasks, cur_dict=cur_dict["childs"][entry_name],
path=entry_path, dev_whitelist=dev_whitelist, ignored_dirs=ignored_dirs)
elif stat.S_ISREG(entry_stat.st_mode):
mp_tasks.append({"result": mp_pool.apply_async(compute_hashes, [entry_path]),
"merge_into": cur_dict["childs"][entry_name]})
elif stat.S_ISLNK(entry_stat.st_mode):
cur_dict["childs"][entry_name]["symlink_target"] = os.readlink(
entry_path)
except Exception:
pass
except Exception:
pass
if is_first_call == True:
mp_pool.close()
for task in mp_tasks:
try:
result = task["result"].get()
for k in iter(result):
task["merge_into"][k] = result[k]
except Exception:
pass
mp_pool.join()
return cur_dict
def main():
ignored_dirs = ["/opt/slapgrid", "/srv/slapgrid"]
dev_whitelist = list()
for path in [".", "/", "/boot"]:
try:
dev_whitelist.append(
os.stat(path, follow_symlinks=False).st_dev)
except FileNotFoundError:
pass
tree = construct_fs_tree(path=".", dev_whitelist=dev_whitelist, ignored_dirs=ignored_dirs)
with open('result.json', 'w') as fp:
json.dump(tree, fp, indent=2, separators=(',', ': '))
if __name__ == "__main__":
main()
scan-filesystem/rust/Cargo.toml deleted 100644 → 0
View file @ 0f83f6e3
[package]
name = "main"
version = "0.1.0"
authors = ["Leo Le Bouter <leo.le.bouter@nexedi.com>", "Xavier Thompson <xavier.thompson@nexedi.com>"]
edition = "2018"
[dependencies]
md-5 = "0.9.1"
sha-1 = "0.9.1"
sha2 = "0.9.1"
hex = "0.4.2"
anyhow = "1.0.32"
rmp-serde = "0.14.4"
nix = "0.18.0"
serde = { version = "1.0.115", features = ["derive"] }
base64 = "0.12.3"
rayon = "1.4.0"
serde_json = "1.0.57"
[profile.release]
opt-level = 'z'
lto = true
codegen-units = 1
scan-filesystem/rust/src/main.rs deleted 100644 → 0
View file @ 0f83f6e3
use anyhow::Result;
use rayon::prelude::*;
use serde::Serialize;
use std::collections::HashMap;
use std::{
fs::DirEntry,
fs::File,
io::{Read, Write},
path::PathBuf,
path::Path,
sync::{Arc, Mutex},
};
#[derive(Debug, Serialize)]
struct FileStat {
st_dev: u64,
st_ino: u64,
st_nlink: u64,
st_mode: u32,
st_uid: u32,
st_gid: u32,
st_rdev: u64,
st_size: i64,
st_blksize: i64,
st_blocks: i64,
st_atime: i64,
st_atime_nsec: i64,
st_mtime: i64,
st_mtime_nsec: i64,
st_ctime: i64,
st_ctime_nsec: i64,
}
impl From<nix::sys::stat::FileStat> for FileStat {
fn from(x: nix::sys::stat::FileStat) -> Self {
Self {
st_dev: x.st_dev,
st_ino: x.st_ino,
st_nlink: x.st_nlink,
st_mode: x.st_mode,
st_uid: x.st_uid,
st_gid: x.st_gid,
st_rdev: x.st_rdev,
st_size: x.st_size,
st_blksize: x.st_blksize,
st_blocks: x.st_blocks,
st_atime: x.st_atime,
st_atime_nsec: x.st_atime_nsec,
st_mtime: x.st_mtime,
st_mtime_nsec: x.st_mtime_nsec,
st_ctime: x.st_ctime,
st_ctime_nsec: x.st_ctime_nsec,
}
}
}
#[derive(Default, Debug, Serialize)]
struct Tree {
childs: HashMap<String, Tree>,
stat: Option<FileStat>,
ignored: bool,
symlink_target: Option<String>,
md5: Option<String>,
sha1: Option<String>,
sha256: Option<String>,
sha512: Option<String>,
}
fn construct_fs_tree(
cur_tree: Option<Tree>,
path: &PathBuf,
dev_whitelist: &Vec<u64>,
ignored_dirs: &Vec<PathBuf>,
) -> Result<Tree> {
let mut cur_tree = match cur_tree {
Some(cur_tree) => cur_tree,
None => Tree {
stat: Some(nix::sys::stat::lstat(path)?.into()),
..Tree::default()
},
};
if !dev_whitelist.iter().any(|x| match &cur_tree.stat {
Some(stat) if stat.st_dev == *x => true,
_ => false,
}) {
return Ok(cur_tree);
}
if ignored_dirs.iter().any(|x| path.starts_with(x)) {
cur_tree.ignored = true;
return Ok(cur_tree);
}
let entries: Vec<Result<DirEntry, _>> = match std::fs::read_dir(&path) {
Ok(x) => x,
_ => return Ok(cur_tree),
}
.collect();
let cur_tree = {
let cur_tree = Arc::new(Mutex::new(cur_tree));
entries.par_iter().for_each(|entry| match entry {
Ok(entry) => {
let mut tree = Tree {
stat: match nix::sys::stat::lstat(&entry.path()) {
Ok(s) => Some(s.into()),
_ => None,
},
..Tree::default()
};
match entry.file_type() {
Ok(file_type) if file_type.is_dir() => {
tree = construct_fs_tree(
Some(tree),
&entry.path(),
dev_whitelist,
ignored_dirs,
)
.unwrap();
}
Ok(file_type) if file_type.is_file() => {
if let Ok(mut file) = std::fs::File::open(&entry.path()) {
use md5::{Digest, Md5};
use sha1::Sha1;
use sha2::{Sha256, Sha512};
let mut md5 = Md5::new();
let mut sha1 = Sha1::new();
let mut sha256 = Sha256::new();
let mut sha512 = Sha512::new();
let buf: &mut [u8] = &mut [0; 64 * 1024];
loop {
match file.read(buf) {
Ok(0) => {
tree.md5 = Some(hex::encode(md5.finalize()));
tree.sha1 = Some(hex::encode(sha1.finalize()));
tree.sha256 = Some(hex::encode(sha256.finalize()));
tree.sha512 = Some(hex::encode(sha512.finalize()));
break;
}
Ok(n) => {
md5.update(&buf[0..n - 1]);
sha1.update(&buf[0..n - 1]);
sha256.update(&buf[0..n - 1]);
sha512.update(&buf[0..n - 1]);
}
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {
continue;
}
Err(e) => {
eprintln!("{:#?}", e);
break;
}
};
}
}
}
Ok(file_type) if file_type.is_symlink() => {
tree.symlink_target = match std::fs::read_link(&entry.path()) {
Ok(target) => match target.to_str() {
Some(target_str) => Some(String::from(target_str)),
None => Some(String::from("<invalid unicode error>")),
}
Err(_) => None,
}
}
_ => {}
};
{
let mut locked = cur_tree.lock().unwrap();
locked
.childs
.insert(entry.file_name().to_str().unwrap().to_string(), tree);
}
}
_ => {}
});
Arc::try_unwrap(cur_tree).unwrap().into_inner().unwrap()
};
Ok(cur_tree)
}
fn main() -> Result<()> {
let ignored_dirs = ["/opt/slapgrid", "/srv/slapgrid"]
.iter()
.map(PathBuf::from)
.collect();
let disk_partitions = [".", "/", "/boot"];
let dev_whitelist = disk_partitions
.iter()
.filter_map(|p| match nix::sys::stat::lstat(&PathBuf::from(p)) {
Ok(stat) => Some(stat.st_dev),
Err(_) => None,
})
.collect();
let tree = construct_fs_tree(
None,
&PathBuf::from("."),
&dev_whitelist,
&ignored_dirs,
)?;
let result = serde_json::to_string_pretty(&tree)?;
let path = Path::new("result.json");
let mut file = File::create(&path).unwrap();
file.write_all(result.as_bytes()).unwrap();
Ok(())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7