closes #5916 merge to main

git-svn-id: file:///svn/toku/tokudb@52402 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/make_tdb.cc

@@ -580,6 +580,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
     printf("    TOKU_ZLIB_METHOD    = 8,\n");  // RFC 1950 says use 8 for zlib.  It reserves 15 to allow more bytes.
     printf("    TOKU_QUICKLZ_METHOD = 9,\n");  // We use 9 for QUICKLZ (the QLZ compression level is stored int he high-order nibble).  I couldn't find any standard for any other numbers, so I just use 9. -Bradley
     printf("    TOKU_LZMA_METHOD    = 10,\n");  // We use 10 for LZMA.  (Note the compression level is stored in the high-order nibble).
+    printf("    TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD = 11,\n"); // We wrap a zlib without checksumming compression technique in our own checksummed metadata.
     printf("    TOKU_DEFAULT_COMPRESSION_METHOD = 1,\n");  // default is actually quicklz
     printf("    TOKU_FAST_COMPRESSION_METHOD = 2,\n");  // friendlier names
     printf("    TOKU_SMALL_COMPRESSION_METHOD = 3,\n");

ft/compress.cc

@@ -40,6 +40,8 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size)
         return size+400 + 1;  // quicklz manual says 400 bytes is enough.  We need one more byte for the rfc1950-style header byte.  bits 0-3 are 9, bits 4-7 are the QLZ_COMPRESSION_LEVEL.
     case TOKU_ZLIB_METHOD:
         return compressBound (size);
+    case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD:
+        return 2+deflateBound(nullptr, size); // We need one extra for the rfc1950-style header byte, and one extra to store windowBits (a bit over cautious about future upgrades maybe).
     default:
         break;
     }
@@ -47,14 +49,15 @@ size_t toku_compress_bound (enum toku_compression_method a, size_t size)
     assert(0); return 0;
 }
 
-static const int zlib_compression_level = 5;
-
 void toku_compress (enum toku_compression_method a,
                     // the following types and naming conventions come from zlib.h
                     Bytef       *dest,   uLongf *destLen,
                     const Bytef *source, uLong   sourceLen)
 // See compress.h for the specification of this function.
 {
+    static const int zlib_compression_level = 5;
+    static const int zlib_without_checksum_windowbits = -15;
+
     a = normalize_compression_method(a);
     assert(sourceLen < (1LL << 32));
     switch (a) {
@@ -107,6 +110,27 @@ void toku_compress (enum toku_compression_method a,
 
 	return;
     }
+    case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: {
+        z_stream strm;
+        strm.zalloc = Z_NULL;
+        strm.zfree = Z_NULL;
+        strm.opaque = Z_NULL;
+        strm.next_in = const_cast<Bytef *>(source);
+        strm.avail_in = sourceLen;
+        int r = deflateInit2(&strm, zlib_compression_level, Z_DEFLATED,
+                             zlib_without_checksum_windowbits, 8, Z_DEFAULT_STRATEGY);
+        lazy_assert(r == Z_OK);
+        strm.next_out = dest + 2;
+        strm.avail_out = *destLen - 2;
+        r = deflate(&strm, Z_FINISH);
+        lazy_assert(r == Z_STREAM_END);
+        r = deflateEnd(&strm);
+        lazy_assert(r == Z_OK);
+        *destLen = strm.total_out + 2;
+        dest[0] = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD + (zlib_compression_level << 4);
+        dest[1] = zlib_without_checksum_windowbits;
+        return;
+    }
     default:
         break;
     }
@@ -159,6 +183,24 @@ void toku_decompress (Bytef       *dest,   uLongf destLen,
 	}
 	return;
     }
+    case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: {
+        z_stream strm;
+        strm.next_in = const_cast<Bytef *>(source + 2);
+        strm.avail_in = sourceLen - 2;
+        strm.zalloc = Z_NULL;
+        strm.zfree = Z_NULL;
+        strm.opaque = Z_NULL;
+        char windowBits = source[1];
+        int r = inflateInit2(&strm, windowBits);
+        lazy_assert(r == Z_OK);
+        strm.next_out = dest;
+        strm.avail_out = destLen;
+        r = inflate(&strm, Z_FINISH);
+        lazy_assert(r == Z_STREAM_END);
+        r = inflateEnd(&strm);
+        lazy_assert(r == Z_OK);
+        return;
+    }
     }
     // default fall through to error.
     assert(0);

ft/tests/compress-test.cc

@@ -22,6 +22,7 @@ static void test_compress_buf_method (unsigned char *buf, int i, enum toku_compr
 
 static void test_compress_buf (unsigned char *buf, int i) {
     test_compress_buf_method(buf, i, TOKU_ZLIB_METHOD);
+    test_compress_buf_method(buf, i, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
     test_compress_buf_method(buf, i, TOKU_QUICKLZ_METHOD);
     test_compress_buf_method(buf, i, TOKU_LZMA_METHOD);
 }

ft/tests/subblock-test-checksum.cc

@@ -147,6 +147,7 @@ test_main (int argc, const char *argv[]) {
         for (int size = total_size - e; size <= total_size + e; size++) {
             run_test(size, n_cores, pool, TOKU_NO_COMPRESSION);
             run_test(size, n_cores, pool, TOKU_ZLIB_METHOD);
+            run_test(size, n_cores, pool, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
             run_test(size, n_cores, pool, TOKU_QUICKLZ_METHOD);
             run_test(size, n_cores, pool, TOKU_LZMA_METHOD);
         }

ft/tests/subblock-test-compression.cc

@@ -98,6 +98,7 @@ test_main (int argc, const char *argv[]) {
         for (int size = total_size - e; size <= total_size + e; size++) {
             run_test(size, n_cores, TOKU_NO_COMPRESSION);
             run_test(size, n_cores, TOKU_ZLIB_METHOD);
+            run_test(size, n_cores, TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
             run_test(size, n_cores, TOKU_QUICKLZ_METHOD);
             run_test(size, n_cores, TOKU_LZMA_METHOD);
         }

src/tests/test_compression_methods.cc

@@ -115,6 +115,7 @@ test_main(int argc, char *const argv[])
     parse_args(argc, argv);
     run_test(TOKU_NO_COMPRESSION);
     run_test(TOKU_ZLIB_METHOD);
+    run_test(TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD);
     run_test(TOKU_QUICKLZ_METHOD);
     run_test(TOKU_LZMA_METHOD);
     return 0;

src/tests/threaded_stress_test_helpers.h

@@ -2442,7 +2442,7 @@ static inline void parse_stress_test_args (int argc, char *const argv[], struct
         if (strcmp(compression_method_s, "quicklz") == 0) {
             args->compression_method = TOKU_QUICKLZ_METHOD;
         } else if (strcmp(compression_method_s, "zlib") == 0) {
-            args->compression_method = TOKU_ZLIB_METHOD;
+            args->compression_method = TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD;
         } else if (strcmp(compression_method_s, "lzma") == 0) {
             args->compression_method = TOKU_LZMA_METHOD;
         } else if (strcmp(compression_method_s, "none") == 0) {