|
13 | 13 | #include <string> |
14 | 14 |
|
15 | 15 | #include "db/db_test_util.h" |
| 16 | +#include "db/write_stall_stats.h" |
16 | 17 | #include "options/cf_options.h" |
17 | 18 | #include "port/stack_trace.h" |
18 | 19 | #include "rocksdb/listener.h" |
@@ -2106,6 +2107,180 @@ TEST_F(DBPropertiesTest, GetMapPropertyBlockCacheEntryStats) { |
2106 | 2107 | ASSERT_EQ(3 * kNumCacheEntryRoles + 4, values.size()); |
2107 | 2108 | } |
2108 | 2109 |
|
| 2110 | +TEST_F(DBPropertiesTest, WriteStallStatsSanityCheck) { |
| 2111 | + for (uint32_t i = 0; i < static_cast<uint32_t>(WriteStallCause::kNone); ++i) { |
| 2112 | + std::string str = kWriteStallCauseToHyphenString[i]; |
| 2113 | + ASSERT_TRUE(!str.empty()) |
| 2114 | + << "Please ensure mapping from `WriteStallCause` to " |
| 2115 | + "`kWriteStallCauseToHyphenString` is complete"; |
| 2116 | + WriteStallCause cause = static_cast<WriteStallCause>(i); |
| 2117 | + if (cause == WriteStallCause::kCFScopeWriteStallCauseEnumMax || |
| 2118 | + cause == WriteStallCause::kDBScopeWriteStallCauseEnumMax) { |
| 2119 | + ASSERT_EQ(str, kInvalidWriteStallCauseHyphenString) |
| 2120 | + << "Please ensure order in `kWriteStallCauseToHyphenString` is " |
| 2121 | + "consistent with `WriteStallCause`"; |
| 2122 | + } |
| 2123 | + } |
| 2124 | + |
| 2125 | + for (uint32_t i = 0; i < static_cast<uint32_t>(WriteStallCondition::kNormal); |
| 2126 | + ++i) { |
| 2127 | + std::string str = kWriteStallConditionToHyphenString[i]; |
| 2128 | + ASSERT_TRUE(!str.empty()) |
| 2129 | + << "Please ensure mapping from `WriteStallCondition` to " |
| 2130 | + "`kWriteStallConditionToHyphenString` is complete"; |
| 2131 | + } |
| 2132 | + |
| 2133 | + for (uint32_t i = 0; i < static_cast<uint32_t>(WriteStallCause::kNone); ++i) { |
| 2134 | + for (uint32_t j = 0; |
| 2135 | + j < static_cast<uint32_t>(WriteStallCondition::kNormal); ++j) { |
| 2136 | + WriteStallCause cause = static_cast<WriteStallCause>(i); |
| 2137 | + WriteStallCondition condition = static_cast<WriteStallCondition>(j); |
| 2138 | + |
| 2139 | + if (isCFScopeWriteStallCause(cause)) { |
| 2140 | + ASSERT_TRUE(InternalCFStat(cause, condition) != |
| 2141 | + InternalStats::INTERNAL_CF_STATS_ENUM_MAX) |
| 2142 | + << "Please ensure the combination of WriteStallCause(" + |
| 2143 | + std::to_string(static_cast<uint32_t>(cause)) + |
| 2144 | + ") + WriteStallCondition(" + |
| 2145 | + std::to_string(static_cast<uint32_t>(condition)) + |
| 2146 | + ") is correctly mapped to a valid `InternalStats` or bypass " |
| 2147 | + "its check in this test"; |
| 2148 | + } else if (isDBScopeWriteStallCause(cause)) { |
| 2149 | + InternalStats::InternalDBStatsType internal_db_stat = |
| 2150 | + InternalDBStat(cause, condition); |
| 2151 | + if (internal_db_stat == InternalStats::kIntStatsNumMax) { |
| 2152 | + ASSERT_TRUE(cause == WriteStallCause::kWriteBufferManagerLimit && |
| 2153 | + condition == WriteStallCondition::kDelayed) |
| 2154 | + << "Please ensure the combination of WriteStallCause(" + |
| 2155 | + std::to_string(static_cast<uint32_t>(cause)) + |
| 2156 | + ") + WriteStallCondition(" + |
| 2157 | + std::to_string(static_cast<uint32_t>(condition)) + |
| 2158 | + ") is correctly mapped to a valid `InternalStats` or " |
| 2159 | + "bypass its check in this test"; |
| 2160 | + } |
| 2161 | + } else if (cause != WriteStallCause::kCFScopeWriteStallCauseEnumMax && |
| 2162 | + cause != WriteStallCause::kDBScopeWriteStallCauseEnumMax) { |
| 2163 | + ASSERT_TRUE(false) << "Please ensure the WriteStallCause(" + |
| 2164 | + std::to_string(static_cast<uint32_t>(cause)) + |
| 2165 | + ") is either CF-scope or DB-scope write " |
| 2166 | + "stall cause in enum `WriteStallCause`"; |
| 2167 | + } |
| 2168 | + } |
| 2169 | + } |
| 2170 | +} |
| 2171 | +TEST_F(DBPropertiesTest, GetMapPropertyWriteStallStats) { |
| 2172 | + Options options = CurrentOptions(); |
| 2173 | + CreateAndReopenWithCF({"heavy_write_cf"}, options); |
| 2174 | + |
| 2175 | + for (auto test_cause : {WriteStallCause::kWriteBufferManagerLimit, |
| 2176 | + WriteStallCause::kMemtableLimit}) { |
| 2177 | + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { |
| 2178 | + options.write_buffer_manager.reset( |
| 2179 | + new WriteBufferManager(100000, nullptr, true)); |
| 2180 | + } else if (test_cause == WriteStallCause::kMemtableLimit) { |
| 2181 | + options.max_write_buffer_number = 2; |
| 2182 | + options.disable_auto_compactions = true; |
| 2183 | + } |
| 2184 | + ReopenWithColumnFamilies({"default", "heavy_write_cf"}, options); |
| 2185 | + |
| 2186 | + // Assert initial write stall stats are all 0 |
| 2187 | + std::map<std::string, std::string> db_values; |
| 2188 | + ASSERT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats, |
| 2189 | + &db_values)); |
| 2190 | + ASSERT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::CauseConditionCount( |
| 2191 | + WriteStallCause::kWriteBufferManagerLimit, |
| 2192 | + WriteStallCondition::kStopped)]), |
| 2193 | + 0); |
| 2194 | + |
| 2195 | + for (int cf = 0; cf <= 1; ++cf) { |
| 2196 | + std::map<std::string, std::string> cf_values; |
| 2197 | + ASSERT_TRUE(dbfull()->GetMapProperty( |
| 2198 | + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); |
| 2199 | + ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), 0); |
| 2200 | + ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), 0); |
| 2201 | + } |
| 2202 | + |
| 2203 | + // Pause flush thread to help coerce write stall |
| 2204 | + std::unique_ptr<test::SleepingBackgroundTask> sleeping_task( |
| 2205 | + new test::SleepingBackgroundTask()); |
| 2206 | + env_->SetBackgroundThreads(1, Env::HIGH); |
| 2207 | + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, |
| 2208 | + sleeping_task.get(), Env::Priority::HIGH); |
| 2209 | + sleeping_task->WaitUntilSleeping(); |
| 2210 | + |
| 2211 | + // Coerce write stall |
| 2212 | + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { |
| 2213 | + ASSERT_OK(dbfull()->Put( |
| 2214 | + WriteOptions(), handles_[1], Key(1), |
| 2215 | + DummyString(options.write_buffer_manager->buffer_size()))); |
| 2216 | + |
| 2217 | + WriteOptions wo; |
| 2218 | + wo.no_slowdown = true; |
| 2219 | + Status s = dbfull()->Put( |
| 2220 | + wo, handles_[1], Key(2), |
| 2221 | + DummyString(options.write_buffer_manager->buffer_size())); |
| 2222 | + ASSERT_TRUE(s.IsIncomplete()); |
| 2223 | + ASSERT_TRUE(s.ToString().find("Write stall") != std::string::npos); |
| 2224 | + } else if (test_cause == WriteStallCause::kMemtableLimit) { |
| 2225 | + FlushOptions fo; |
| 2226 | + fo.allow_write_stall = true; |
| 2227 | + fo.wait = false; |
| 2228 | + |
| 2229 | + ASSERT_OK( |
| 2230 | + dbfull()->Put(WriteOptions(), handles_[1], Key(1), DummyString(1))); |
| 2231 | + ASSERT_OK(dbfull()->Flush(fo, handles_[1])); |
| 2232 | + |
| 2233 | + ASSERT_OK( |
| 2234 | + dbfull()->Put(WriteOptions(), handles_[1], Key(2), DummyString(1))); |
| 2235 | + ASSERT_OK(dbfull()->Flush(fo, handles_[1])); |
| 2236 | + } |
| 2237 | + |
| 2238 | + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { |
| 2239 | + db_values.clear(); |
| 2240 | + EXPECT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats, |
| 2241 | + &db_values)); |
| 2242 | + EXPECT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::CauseConditionCount( |
| 2243 | + WriteStallCause::kWriteBufferManagerLimit, |
| 2244 | + WriteStallCondition::kStopped)]), |
| 2245 | + 1); |
| 2246 | + // `WriteStallCause::kWriteBufferManagerLimit` should not result in any |
| 2247 | + // CF-scope write stall stats changes |
| 2248 | + for (int cf = 0; cf <= 1; ++cf) { |
| 2249 | + std::map<std::string, std::string> cf_values; |
| 2250 | + EXPECT_TRUE(dbfull()->GetMapProperty( |
| 2251 | + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); |
| 2252 | + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), |
| 2253 | + 0); |
| 2254 | + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), |
| 2255 | + 0); |
| 2256 | + } |
| 2257 | + } else if (test_cause == WriteStallCause::kMemtableLimit) { |
| 2258 | + for (int cf = 0; cf <= 1; ++cf) { |
| 2259 | + std::map<std::string, std::string> cf_values; |
| 2260 | + EXPECT_TRUE(dbfull()->GetMapProperty( |
| 2261 | + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); |
| 2262 | + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), |
| 2263 | + cf == 1 ? 1 : 0); |
| 2264 | + EXPECT_EQ( |
| 2265 | + std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount( |
| 2266 | + WriteStallCause::kMemtableLimit, |
| 2267 | + WriteStallCondition::kStopped)]), |
| 2268 | + cf == 1 ? 1 : 0); |
| 2269 | + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), |
| 2270 | + 0); |
| 2271 | + EXPECT_EQ( |
| 2272 | + std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount( |
| 2273 | + WriteStallCause::kMemtableLimit, |
| 2274 | + WriteStallCondition::kDelayed)]), |
| 2275 | + 0); |
| 2276 | + } |
| 2277 | + } |
| 2278 | + |
| 2279 | + sleeping_task->WakeUp(); |
| 2280 | + sleeping_task->WaitUntilDone(); |
| 2281 | + } |
| 2282 | +} |
| 2283 | + |
2109 | 2284 | namespace { |
2110 | 2285 | std::string PopMetaIndexKey(InternalIterator* meta_iter) { |
2111 | 2286 | Status s = meta_iter->status(); |
|
0 commit comments