bcachefs: Compression levels

This allows including a compression level when specifying a compression
type, e.g.
  compression=zstd:15

Values from 1 through 15 indicate compression levels, 0 or unspecified
indicates the default.

For LZ4, values 3-15 specify that the HC algorithm should be used.

Note that for compatibility, extents themselves only include the
compression type, not the compression level. This means that specifying
the same compression algorithm but different compression levels for the
compression and background_compression options will have no effect.

XXX: perhaps we could add a warning for this
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/Kconfig

@@ -9,6 +9,8 @@ config BCACHEFS_FS
 	select FS_POSIX_ACL
 	select LZ4_COMPRESS
 	select LZ4_DECOMPRESS
+	select LZ4HC_COMPRESS
+	select LZ4HC_DECOMPRESS
 	select ZLIB_DEFLATE
 	select ZLIB_INFLATE
 	select ZSTD_COMPRESS

fs/bcachefs/checksum.h

@@ -120,12 +120,6 @@ static inline enum bch_csum_type bch2_meta_checksum_type(struct bch_fs *c)
 	return bch2_csum_opt_to_type(c->opts.metadata_checksum, false);
 }
 
-static const unsigned bch2_compression_opt_to_type[] = {
-#define x(t, n) [BCH_COMPRESSION_OPT_##t] = BCH_COMPRESSION_TYPE_##t,
-	BCH_COMPRESSION_OPTS()
-#undef x
-};
-
 static inline bool bch2_checksum_type_valid(const struct bch_fs *c,
 					   unsigned type)
 {

fs/bcachefs/compress.c

@@ -296,21 +296,32 @@ static int attempt_compress(struct bch_fs *c,
 			    void *workspace,
 			    void *dst, size_t dst_len,
 			    void *src, size_t src_len,
-			    enum bch_compression_type compression_type)
+			    struct bch_compression_opt compression)
 {
-	switch (compression_type) {
-	case BCH_COMPRESSION_TYPE_lz4: {
-		int len = src_len;
-		int ret = LZ4_compress_destSize(
-				src,		dst,
-				&len,		dst_len,
-				workspace);
-
-		if (len < src_len)
-			return -len;
+	enum bch_compression_type compression_type =
+		__bch2_compression_opt_to_type[compression.type];
 
-		return ret;
-	}
+	switch (compression_type) {
+	case BCH_COMPRESSION_TYPE_lz4:
+		if (compression.level < LZ4HC_MIN_CLEVEL) {
+			int len = src_len;
+			int ret = LZ4_compress_destSize(
+					src,		dst,
+					&len,		dst_len,
+					workspace);
+			if (len < src_len)
+				return -len;
+
+			return ret;
+		} else {
+			int ret = LZ4_compress_HC(
+					src,		dst,
+					src_len,	dst_len,
+					compression.level,
+					workspace);
+
+			return ret ?: -1;
+		}
 	case BCH_COMPRESSION_TYPE_gzip: {
 		z_stream strm = {
 			.next_in	= src,
@@ -320,7 +331,11 @@ static int attempt_compress(struct bch_fs *c,
 		};
 
 		zlib_set_workspace(&strm, workspace);
-		zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
+		zlib_deflateInit2(&strm,
+				  compression.level
+				  ? clamp_t(unsigned, compression.level,
+					    Z_BEST_SPEED, Z_BEST_COMPRESSION)
+				  : Z_DEFAULT_COMPRESSION,
 				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
 				  Z_DEFAULT_STRATEGY);
 
@@ -333,8 +348,14 @@ static int attempt_compress(struct bch_fs *c,
 		return strm.total_out;
 	}
 	case BCH_COMPRESSION_TYPE_zstd: {
+		/*
+		 * rescale:
+		 * zstd max compression level is 22, our max level is 15
+		 */
+		unsigned level = min((compression.level * 3) / 2, zstd_max_clevel());
+		ZSTD_parameters params = zstd_get_params(level, c->opts.encoded_extent_max);
 		ZSTD_CCtx *ctx = zstd_init_cctx(workspace,
-			zstd_cctx_workspace_bound(&c->zstd_params.cParams));
+			zstd_cctx_workspace_bound(&params.cParams));
 
 		/*
 		 * ZSTD requires that when we decompress we pass in the exact
@@ -365,10 +386,12 @@ static int attempt_compress(struct bch_fs *c,
 static unsigned __bio_compress(struct bch_fs *c,
 			       struct bio *dst, size_t *dst_len,
 			       struct bio *src, size_t *src_len,
-			       enum bch_compression_type compression_type)
+			       struct bch_compression_opt compression)
 {
 	struct bbuf src_data = { NULL }, dst_data = { NULL };
 	void *workspace;
+	enum bch_compression_type compression_type =
+		__bch2_compression_opt_to_type[compression.type];
 	unsigned pad;
 	int ret = 0;
 
@@ -400,7 +423,7 @@ static unsigned __bio_compress(struct bch_fs *c,
 		ret = attempt_compress(c, workspace,
 				       dst_data.b,	*dst_len,
 				       src_data.b,	*src_len,
-				       compression_type);
+				       compression);
 		if (ret > 0) {
 			*dst_len = ret;
 			ret = 0;
@@ -447,22 +470,24 @@ static unsigned __bio_compress(struct bch_fs *c,
 	BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
 	BUG_ON(*dst_len & (block_bytes(c) - 1));
 	BUG_ON(*src_len & (block_bytes(c) - 1));
+	ret = compression_type;
 out:
 	bio_unmap_or_unbounce(c, src_data);
 	bio_unmap_or_unbounce(c, dst_data);
-	return compression_type;
+	return ret;
 err:
-	compression_type = BCH_COMPRESSION_TYPE_incompressible;
+	ret = BCH_COMPRESSION_TYPE_incompressible;
 	goto out;
 }
 
 unsigned bch2_bio_compress(struct bch_fs *c,
 			   struct bio *dst, size_t *dst_len,
 			   struct bio *src, size_t *src_len,
-			   unsigned compression_type)
+			   unsigned compression_opt)
 {
 	unsigned orig_dst = dst->bi_iter.bi_size;
 	unsigned orig_src = src->bi_iter.bi_size;
+	unsigned compression_type;
 
 	/* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
 	src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
@@ -470,11 +495,9 @@ unsigned bch2_bio_compress(struct bch_fs *c,
 	/* Don't generate a bigger output than input: */
 	dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 
-	if (compression_type == BCH_COMPRESSION_TYPE_lz4_old)
-		compression_type = BCH_COMPRESSION_TYPE_lz4;
-
 	compression_type =
-		__bio_compress(c, dst, dst_len, src, src_len, compression_type);
+		__bio_compress(c, dst, dst_len, src, src_len,
+			       bch2_compression_decode(compression_opt));
 
 	dst->bi_iter.bi_size = orig_dst;
 	src->bi_iter.bi_size = orig_src;
@@ -521,8 +544,10 @@ static int __bch2_check_set_has_compressed_data(struct bch_fs *c, u64 f)
 }
 
 int bch2_check_set_has_compressed_data(struct bch_fs *c,
-				       unsigned compression_type)
+				       unsigned compression_opt)
 {
+	unsigned compression_type = bch2_compression_decode(compression_opt).type;
+
 	BUG_ON(compression_type >= ARRAY_SIZE(bch2_compression_opt_to_feature));
 
 	return compression_type
@@ -546,14 +571,16 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features)
 {
 	size_t decompress_workspace_size = 0;
 	bool decompress_workspace_needed;
-	ZSTD_parameters params = zstd_get_params(0, c->opts.encoded_extent_max);
+	ZSTD_parameters params = zstd_get_params(zstd_max_clevel(),
+						 c->opts.encoded_extent_max);
 	struct {
-		unsigned	feature;
-		unsigned	type;
-		size_t		compress_workspace;
-		size_t		decompress_workspace;
+		unsigned			feature;
+		enum bch_compression_type	type;
+		size_t				compress_workspace;
+		size_t				decompress_workspace;
 	} compression_types[] = {
-		{ BCH_FEATURE_lz4, BCH_COMPRESSION_TYPE_lz4, LZ4_MEM_COMPRESS, 0 },
+		{ BCH_FEATURE_lz4, BCH_COMPRESSION_TYPE_lz4,
+			max_t(size_t, LZ4_MEM_COMPRESS, LZ4HC_MEM_COMPRESS) },
 		{ BCH_FEATURE_gzip, BCH_COMPRESSION_TYPE_gzip,
 			zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),
 			zlib_inflate_workspacesize(), },
@@ -612,16 +639,74 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features)
 	return 0;
 }
 
+static u64 compression_opt_to_feature(unsigned v)
+{
+	unsigned type = bch2_compression_decode(v).type;
+	return 1ULL << bch2_compression_opt_to_feature[type];
+}
+
 int bch2_fs_compress_init(struct bch_fs *c)
 {
 	u64 f = c->sb.features;
 
-	if (c->opts.compression)
-		f |= 1ULL << bch2_compression_opt_to_feature[c->opts.compression];
-
-	if (c->opts.background_compression)
-		f |= 1ULL << bch2_compression_opt_to_feature[c->opts.background_compression];
+	f |= compression_opt_to_feature(c->opts.compression);
+	f |= compression_opt_to_feature(c->opts.background_compression);
 
 	return __bch2_fs_compress_init(c, f);
+}
+
+int bch2_opt_compression_parse(struct bch_fs *c, const char *_val, u64 *res,
+			       struct printbuf *err)
+{
+	char *val = kstrdup(_val, GFP_KERNEL);
+	char *p = val, *type_str, *level_str;
+	struct bch_compression_opt opt = { 0 };
+	int ret;
+
+	if (!val)
+		return -ENOMEM;
+
+	type_str = strsep(&p, ":");
+	level_str = p;
+
+	ret = match_string(bch2_compression_opts, -1, type_str);
+	if (ret < 0 && err)
+		prt_str(err, "invalid compression type");
+	if (ret < 0)
+		goto err;
+
+	opt.type = ret;
+
+	if (level_str) {
+		unsigned level;
+
+		ret = kstrtouint(level_str, 10, &level);
+		if (!ret && !opt.type && level)
+			ret = -EINVAL;
+		if (!ret && level > 15)
+			ret = -EINVAL;
+		if (ret < 0 && err)
+			prt_str(err, "invalid compression level");
+		if (ret < 0)
+			goto err;
+
+		opt.level = level;
+	}
+
+	*res = bch2_compression_encode(opt);
+err:
+	kfree(val);
+	return ret;
+}
+
+void bch2_opt_compression_to_text(struct printbuf *out,
+				  struct bch_fs *c,
+				  struct bch_sb *sb,
+				  u64 v)
+{
+	struct bch_compression_opt opt = bch2_compression_decode(v);
 
+	prt_str(out, bch2_compression_opts[opt.type]);
+	if (opt.level)
+		prt_printf(out, ":%u", opt.level);
 }

fs/bcachefs/compress.h

@@ -4,6 +4,35 @@
 
 #include "extents_types.h"
 
+struct bch_compression_opt {
+	u8		type:4,
+			level:4;
+};
+
+static inline struct bch_compression_opt bch2_compression_decode(unsigned v)
+{
+	return (struct bch_compression_opt) {
+		.type	= v & 15,
+		.level	= v >> 4,
+	};
+}
+
+static inline unsigned bch2_compression_encode(struct bch_compression_opt opt)
+{
+	return opt.type|(opt.level << 4);
+}
+
+static const unsigned __bch2_compression_opt_to_type[] = {
+#define x(t, n) [BCH_COMPRESSION_OPT_##t] = BCH_COMPRESSION_TYPE_##t,
+	BCH_COMPRESSION_OPTS()
+#undef x
+};
+
+static inline enum bch_compression_type bch2_compression_opt_to_type(unsigned v)
+{
+	return __bch2_compression_opt_to_type[bch2_compression_decode(v).type];
+}
+
 int bch2_bio_uncompress_inplace(struct bch_fs *, struct bio *,
 				struct bch_extent_crc_unpacked *);
 int bch2_bio_uncompress(struct bch_fs *, struct bio *, struct bio *,
@@ -15,4 +44,12 @@ int bch2_check_set_has_compressed_data(struct bch_fs *, unsigned);
 void bch2_fs_compress_exit(struct bch_fs *);
 int bch2_fs_compress_init(struct bch_fs *);
 
+int bch2_opt_compression_parse(struct bch_fs *, const char *, u64 *, struct printbuf *);
+void bch2_opt_compression_to_text(struct printbuf *, struct bch_fs *, struct bch_sb *, u64);
+
+#define bch2_opt_compression (struct bch_opt_fn) {		\
+	.parse		= bch2_opt_compression_parse,	\
+	.to_text	= bch2_opt_compression_to_text,	\
+}
+
 #endif /* _BCACHEFS_COMPRESS_H */

fs/bcachefs/data_update.c

@@ -455,9 +455,7 @@ int bch2_data_update_init(struct btree_trans *trans,
 		BCH_WRITE_DATA_ENCODED|
 		BCH_WRITE_MOVE|
 		m->data_opts.write_flags;
-	m->op.compression_type =
-		bch2_compression_opt_to_type[io_opts.background_compression ?:
-					     io_opts.compression];
+	m->op.compression_opt	= io_opts.background_compression ?: io_opts.compression;
 	m->op.watermark		= m->data_opts.btree_insert_flags & BCH_WATERMARK_MASK;
 
 	bkey_for_each_ptr(ptrs, ptr)

fs/bcachefs/io.c

@@ -1078,7 +1078,7 @@ static enum prep_encoded_ret {
 	/* Can we just write the entire extent as is? */
 	if (op->crc.uncompressed_size == op->crc.live_size &&
 	    op->crc.compressed_size <= wp->sectors_free &&
-	    (op->crc.compression_type == op->compression_type ||
+	    (op->crc.compression_type == bch2_compression_opt_to_type(op->compression_opt) ||
 	     op->incompressible)) {
 		if (!crc_is_compressed(op->crc) &&
 		    op->csum_type != op->crc.csum_type &&
@@ -1126,7 +1126,7 @@ static enum prep_encoded_ret {
 	/*
 	 * If we want to compress the data, it has to be decrypted:
 	 */
-	if ((op->compression_type ||
+	if ((op->compression_opt ||
 	     bch2_csum_type_is_encryption(op->crc.csum_type) !=
 	     bch2_csum_type_is_encryption(op->csum_type)) &&
 	    bch2_write_decrypt(op))
@@ -1173,7 +1173,7 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp,
 	}
 
 	if (ec_buf ||
-	    op->compression_type ||
+	    op->compression_opt ||
 	    (op->csum_type &&
 	     !(op->flags & BCH_WRITE_PAGES_STABLE)) ||
 	    (bch2_csum_type_is_encryption(op->csum_type) &&
@@ -1196,16 +1196,16 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp,
 		    dst->bi_iter.bi_size < c->opts.encoded_extent_max)
 			break;
 
-		BUG_ON(op->compression_type &&
+		BUG_ON(op->compression_opt &&
 		       (op->flags & BCH_WRITE_DATA_ENCODED) &&
 		       bch2_csum_type_is_encryption(op->crc.csum_type));
-		BUG_ON(op->compression_type && !bounce);
+		BUG_ON(op->compression_opt && !bounce);
 
 		crc.compression_type = op->incompressible
 			? BCH_COMPRESSION_TYPE_incompressible
-			: op->compression_type
+			: op->compression_opt
 			? bch2_bio_compress(c, dst, &dst_len, src, &src_len,
-					    op->compression_type)
+					    op->compression_opt)
 			: 0;
 		if (!crc_is_compressed(crc)) {
 			dst_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);

fs/bcachefs/io.h

@@ -86,7 +86,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
 	op->written		= 0;
 	op->error		= 0;
 	op->csum_type		= bch2_data_checksum_type(c, opts);
-	op->compression_type	= bch2_compression_opt_to_type[opts.compression];
+	op->compression_opt	= opts.compression;
 	op->nr_replicas		= 0;
 	op->nr_replicas_required = c->opts.data_replicas_required;
 	op->watermark		= BCH_WATERMARK_normal;

fs/bcachefs/io_types.h

@@ -115,8 +115,8 @@ struct bch_write_op {
 	u16			flags;
 	s16			error; /* dio write path expects it to hold -ERESTARTSYS... */
 
+	unsigned		compression_opt:8;
 	unsigned		csum_type:4;
-	unsigned		compression_type:4;
 	unsigned		nr_replicas:4;
 	unsigned		nr_replicas_required:4;
 	unsigned		watermark:3;

fs/bcachefs/opts.h

@@ -174,12 +174,12 @@ enum fsck_err_opts {
 	  NULL,		NULL)						\
 	x(compression,			u8,				\
 	  OPT_FS|OPT_INODE|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME,		\
-	  OPT_STR(bch2_compression_opts),				\
+	  OPT_FN(bch2_opt_compression),					\
 	  BCH_SB_COMPRESSION_TYPE,	BCH_COMPRESSION_OPT_none,	\
 	  NULL,		NULL)						\
 	x(background_compression,	u8,				\
 	  OPT_FS|OPT_INODE|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME,		\
-	  OPT_STR(bch2_compression_opts),				\
+	  OPT_FN(bch2_opt_compression),					\
 	  BCH_SB_BACKGROUND_COMPRESSION_TYPE,BCH_COMPRESSION_OPT_none,	\
 	  NULL,		NULL)						\
 	x(str_hash,			u8,				\

fs/bcachefs/rebalance.c

@@ -5,6 +5,7 @@
 #include "btree_iter.h"
 #include "buckets.h"
 #include "clock.h"
+#include "compress.h"
 #include "disk_groups.h"
 #include "errcode.h"
 #include "extents.h"
@@ -45,7 +46,7 @@ static bool rebalance_pred(struct bch_fs *c, void *arg,
 		bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
 			if (!p.ptr.cached &&
 			    p.crc.compression_type !=
-			    bch2_compression_opt_to_type[io_opts->background_compression])
+			    bch2_compression_opt_to_type(io_opts->background_compression))
 				data_opts->rewrite_ptrs |= 1U << i;
 			i++;
 		}