Skip to content

Commit fb63d9b

Browse files
pdillingerfacebook-github-bot
pdillinger
authored and
facebook-github-bot
committed
Fix compression tests when snappy not available (facebook#11396)
Summary: Tweak some bounds and things, and reduce risk of surprise results by running on all supported compressions (mostly). Also improves the precise compressibility of CompressibleString by using RandomBinaryString. Pull Request resolved: facebook#11396 Test Plan: updated tests Reviewed By: ltamasi Differential Revision: D45211938 Pulled By: pdillinger fbshipit-source-id: 9dc1dd8574a60a9364efe18558be66d31a35598b
1 parent d79be3d commit fb63d9b

File tree

3 files changed

+145
-175
lines changed

3 files changed

+145
-175
lines changed

‎db/db_statistics_test.cc

+105-121
Original file line numberDiff line numberDiff line change
@@ -20,131 +20,115 @@ class DBStatisticsTest : public DBTestBase {
2020
};
2121

2222
TEST_F(DBStatisticsTest, CompressionStatsTest) {
23-
CompressionType type;
24-
25-
if (Snappy_Supported()) {
26-
type = kSnappyCompression;
27-
fprintf(stderr, "using snappy\n");
28-
} else if (Zlib_Supported()) {
29-
type = kZlibCompression;
30-
fprintf(stderr, "using zlib\n");
31-
} else if (BZip2_Supported()) {
32-
type = kBZip2Compression;
33-
fprintf(stderr, "using bzip2\n");
34-
} else if (LZ4_Supported()) {
35-
type = kLZ4Compression;
36-
fprintf(stderr, "using lz4\n");
37-
} else if (XPRESS_Supported()) {
38-
type = kXpressCompression;
39-
fprintf(stderr, "using xpress\n");
40-
} else if (ZSTD_Supported()) {
41-
type = kZSTD;
42-
fprintf(stderr, "using ZSTD\n");
43-
} else {
44-
fprintf(stderr, "skipping test, compression disabled\n");
45-
return;
46-
}
47-
48-
Options options = CurrentOptions();
49-
options.compression = type;
50-
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
51-
options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
52-
BlockBasedTableOptions bbto;
53-
bbto.enable_index_compression = false;
54-
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
55-
DestroyAndReopen(options);
56-
57-
auto PopStat = [&](Tickers t) -> uint64_t {
58-
return options.statistics->getAndResetTickerCount(t);
59-
};
60-
61-
int kNumKeysWritten = 100;
62-
double compress_to = 0.5;
63-
// About three KVs per block
64-
int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
65-
int uncomp_est = kNumKeysWritten * (len + 20);
66-
67-
Random rnd(301);
68-
std::string buf;
23+
for (CompressionType type : GetSupportedCompressions()) {
24+
if (type == kNoCompression) {
25+
continue;
26+
}
27+
if (type == kBZip2Compression) {
28+
// Weird behavior in this test
29+
continue;
30+
}
31+
SCOPED_TRACE("Compression type: " + std::to_string(type));
32+
33+
Options options = CurrentOptions();
34+
options.compression = type;
35+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
36+
options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
37+
BlockBasedTableOptions bbto;
38+
bbto.enable_index_compression = false;
39+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
40+
DestroyAndReopen(options);
41+
42+
auto PopStat = [&](Tickers t) -> uint64_t {
43+
return options.statistics->getAndResetTickerCount(t);
44+
};
45+
46+
int kNumKeysWritten = 100;
47+
double compress_to = 0.5;
48+
// About three KVs per block
49+
int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
50+
int uncomp_est = kNumKeysWritten * (len + 20);
51+
52+
Random rnd(301);
53+
std::string buf;
54+
55+
// Check that compressions occur and are counted when compression is turned
56+
// on
57+
for (int i = 0; i < kNumKeysWritten; ++i) {
58+
ASSERT_OK(
59+
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
60+
}
61+
ASSERT_OK(Flush());
62+
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
63+
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
64+
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
65+
uncomp_est / 10);
66+
67+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
68+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
69+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
70+
71+
// And decompressions
72+
for (int i = 0; i < kNumKeysWritten; ++i) {
73+
auto r = Get(Key(i));
74+
}
75+
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
76+
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
77+
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
78+
uncomp_est / 10);
6979

70-
// Check that compressions occur and are counted when compression is turned on
71-
for (int i = 0; i < kNumKeysWritten; ++i) {
72-
ASSERT_OK(
73-
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
74-
}
75-
ASSERT_OK(Flush());
76-
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
77-
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
78-
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
79-
uncomp_est / 10);
80-
81-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
82-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
83-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
84-
85-
// And decompressions
86-
for (int i = 0; i < kNumKeysWritten; ++i) {
87-
auto r = Get(Key(i));
88-
}
89-
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
90-
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
91-
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
92-
uncomp_est / 10);
93-
94-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
95-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
96-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
97-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
98-
99-
// Check when compression is rejected.
100-
compress_to = 0.95;
101-
DestroyAndReopen(options);
80+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
81+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
82+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
83+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
10284

103-
for (int i = 0; i < kNumKeysWritten; ++i) {
104-
ASSERT_OK(
105-
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
106-
}
107-
ASSERT_OK(Flush());
108-
for (int i = 0; i < kNumKeysWritten; ++i) {
109-
auto r = Get(Key(i));
110-
}
111-
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
112-
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
113-
uncomp_est / 10);
114-
115-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
116-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
117-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
118-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
119-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
120-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
121-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
122-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
123-
124-
// Check when compression is disabled.
125-
options.compression = kNoCompression;
126-
DestroyAndReopen(options);
85+
// Check when compression is rejected.
86+
DestroyAndReopen(options);
12787

128-
for (int i = 0; i < kNumKeysWritten; ++i) {
129-
ASSERT_OK(
130-
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
131-
}
132-
ASSERT_OK(Flush());
133-
for (int i = 0; i < kNumKeysWritten; ++i) {
134-
auto r = Get(Key(i));
88+
for (int i = 0; i < kNumKeysWritten; ++i) {
89+
ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
90+
}
91+
ASSERT_OK(Flush());
92+
for (int i = 0; i < kNumKeysWritten; ++i) {
93+
auto r = Get(Key(i));
94+
}
95+
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
96+
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
97+
uncomp_est / 10);
98+
99+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
100+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
101+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
102+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
103+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
104+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
105+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
106+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
107+
108+
// Check when compression is disabled.
109+
options.compression = kNoCompression;
110+
DestroyAndReopen(options);
111+
112+
for (int i = 0; i < kNumKeysWritten; ++i) {
113+
ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
114+
}
115+
ASSERT_OK(Flush());
116+
for (int i = 0; i < kNumKeysWritten; ++i) {
117+
auto r = Get(Key(i));
118+
}
119+
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
120+
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
121+
uncomp_est / 10);
122+
123+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
124+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
125+
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
126+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
127+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
128+
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
129+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
130+
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
135131
}
136-
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
137-
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
138-
uncomp_est / 10);
139-
140-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
141-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
142-
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
143-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
144-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
145-
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
146-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
147-
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
148132
}
149133

150134
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {

‎table/table_test.cc

+39-53
Original file line numberDiff line numberDiff line change
@@ -5081,63 +5081,49 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
50815081
}
50825082

50835083
TEST_P(BlockBasedTableTest, CompressionRatioThreshold) {
5084-
Options options;
5085-
if (Snappy_Supported()) {
5086-
options.compression = kSnappyCompression;
5087-
fprintf(stderr, "using snappy\n");
5088-
} else if (Zlib_Supported()) {
5089-
options.compression = kZlibCompression;
5090-
fprintf(stderr, "using zlib\n");
5091-
} else if (BZip2_Supported()) {
5092-
options.compression = kBZip2Compression;
5093-
fprintf(stderr, "using bzip2\n");
5094-
} else if (LZ4_Supported()) {
5095-
options.compression = kLZ4Compression;
5096-
fprintf(stderr, "using lz4\n");
5097-
} else if (XPRESS_Supported()) {
5098-
options.compression = kXpressCompression;
5099-
fprintf(stderr, "using xpress\n");
5100-
} else if (ZSTD_Supported()) {
5101-
options.compression = kZSTD;
5102-
fprintf(stderr, "using ZSTD\n");
5103-
} else {
5104-
fprintf(stderr, "skipping test, compression disabled\n");
5105-
return;
5106-
}
5107-
5108-
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
5109-
int len = 10000;
5110-
Random rnd(301);
5111-
std::vector<std::string> keys;
5112-
stl_wrappers::KVMap kvmap;
5084+
for (CompressionType type : GetSupportedCompressions()) {
5085+
if (type == kNoCompression) {
5086+
continue;
5087+
}
5088+
SCOPED_TRACE("Compression type: " + std::to_string(type));
51135089

5114-
// Test the max_compressed_bytes_per_kb option
5115-
for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) {
5116-
SCOPED_TRACE("threshold=" + std::to_string(threshold));
5117-
options.compression_opts.max_compressed_bytes_per_kb = threshold;
5118-
ImmutableOptions ioptions(options);
5119-
MutableCFOptions moptions(options);
5090+
Options options;
5091+
options.compression = type;
51205092

5121-
for (double compressible_to : {0.25, 0.75}) {
5122-
SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to));
5123-
TableConstructor c(BytewiseComparator(),
5124-
true /* convert_to_internal_key_ */);
5125-
std::string buf;
5126-
c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
5093+
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
5094+
int len = 10000;
5095+
Random rnd(301);
5096+
std::vector<std::string> keys;
5097+
stl_wrappers::KVMap kvmap;
51275098

5128-
// write an SST file
5129-
c.Finish(options, ioptions, moptions, table_options,
5130-
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
5099+
// Test the max_compressed_bytes_per_kb option
5100+
for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) {
5101+
SCOPED_TRACE("threshold=" + std::to_string(threshold));
5102+
options.compression_opts.max_compressed_bytes_per_kb = threshold;
5103+
ImmutableOptions ioptions(options);
5104+
MutableCFOptions moptions(options);
51315105

5132-
size_t table_file_size = c.TEST_GetSink()->contents().size();
5133-
size_t approx_sst_overhead = 1000;
5134-
if (compressible_to < threshold / 1024.0) {
5135-
// Should be compressed
5136-
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead,
5137-
table_file_size, len / 10);
5138-
} else {
5139-
// Should not be compressed
5140-
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10);
5106+
for (double compressible_to : {0.25, 0.75}) {
5107+
SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to));
5108+
TableConstructor c(BytewiseComparator(),
5109+
true /* convert_to_internal_key_ */);
5110+
std::string buf;
5111+
c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
5112+
5113+
// write an SST file
5114+
c.Finish(options, ioptions, moptions, table_options,
5115+
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
5116+
5117+
size_t table_file_size = c.TEST_GetSink()->contents().size();
5118+
size_t approx_sst_overhead = 1000;
5119+
if (compressible_to < threshold / 1024.0) {
5120+
// Should be compressed (substantial variance depending on algorithm)
5121+
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead,
5122+
table_file_size, len / 8);
5123+
} else {
5124+
// Should not be compressed
5125+
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10);
5126+
}
51415127
}
51425128
}
51435129
}

‎test_util/testutil.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ extern Slice CompressibleString(Random* rnd, double compressed_fraction,
7676
int len, std::string* dst) {
7777
int raw = static_cast<int>(len * compressed_fraction);
7878
if (raw < 1) raw = 1;
79-
std::string raw_data = rnd->RandomString(raw);
79+
std::string raw_data = rnd->RandomBinaryString(raw);
8080

8181
// Duplicate the random data until we have filled "len" bytes
8282
dst->clear();

0 commit comments

Comments
 (0)