Skip to content

Consolidate CHECK and LOG implementations. #1753

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Consolidate CHECK and LOG implementations. #1753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 14 additions & 71 deletions centipede/BUILD
View file
Open in desktop

Large diffs are not rendered by default.

42 changes: 21 additions & 21 deletions centipede/analyze_corpora.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@

#include "absl/container/flat_hash_map.h"
#include "absl/container/flat_hash_set.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "./centipede/binary_info.h"
#include "./centipede/control_flow.h"
#include "./centipede/corpus.h"
Expand All @@ -48,17 +46,17 @@ std::vector<CorpusRecord> ReadCorpora(std::string_view binary_name,
WorkDir workdir(std::string(workdir_path), std::string(binary_name),
std::string(binary_hash), /*my_shard_index=*/0);
std::vector<std::string> corpus_paths;
CHECK_OK(
FUZZTEST_CHECK_OK(
RemoteGlobMatch(workdir.CorpusFilePaths().AllShardsGlob(), corpus_paths));
std::vector<std::string> features_paths;
CHECK_OK(RemoteGlobMatch(workdir.FeaturesFilePaths().AllShardsGlob(),
features_paths));
FUZZTEST_CHECK_OK(RemoteGlobMatch(workdir.FeaturesFilePaths().AllShardsGlob(),
features_paths));

CHECK_EQ(corpus_paths.size(), features_paths.size());
FUZZTEST_CHECK_EQ(corpus_paths.size(), features_paths.size());
std::vector<CorpusRecord> corpus;
for (int i = 0; i < corpus_paths.size(); ++i) {
LOG(INFO) << "Reading corpus at: " << corpus_paths[i];
LOG(INFO) << "Reading features at: " << features_paths[i];
FUZZTEST_LOG(INFO) << "Reading corpus at: " << corpus_paths[i];
FUZZTEST_LOG(INFO) << "Reading features at: " << features_paths[i];
ReadShard(corpus_paths[i], features_paths[i],
[&corpus](ByteArray input, FeatureVec features) {
corpus.push_back({std::move(input), std::move(features)});
Expand Down Expand Up @@ -126,9 +124,10 @@ AnalyzeCorporaResults AnalyzeCorpora(const BinaryInfo &binary_info,
a_only_pcs.insert(pc);
}
}
LOG(INFO) << VV(a.size()) << VV(b.size()) << VV(a_pcs.size())
<< VV(a_only_pcs.size()) << VV(b_only_pcs.size())
<< VV(b_shared_indices.size()) << VV(b_unique_indices.size());
FUZZTEST_LOG(INFO) << VV(a.size()) << VV(b.size()) << VV(a_pcs.size())
<< VV(a_only_pcs.size()) << VV(b_only_pcs.size())
<< VV(b_shared_indices.size())
<< VV(b_unique_indices.size());

// Sort PCs to put them in the canonical order, as in pc_table.
AnalyzeCorporaResults ret;
Expand Down Expand Up @@ -178,7 +177,7 @@ CoverageResults GetCoverage(std::string_view binary_name,

void DumpCoverageReport(const CoverageResults &coverage_results,
std::string_view coverage_report_path) {
LOG(INFO) << "Dump coverage to file: " << coverage_report_path;
FUZZTEST_LOG(INFO) << "Dump coverage to file: " << coverage_report_path;

const fuzztest::internal::PCTable &pc_table =
coverage_results.binary_info.pc_table;
Expand All @@ -187,7 +186,7 @@ void DumpCoverageReport(const CoverageResults &coverage_results,

fuzztest::internal::SymbolTable coverage_symbol_table;
for (const PCIndex pc : coverage_results.pcs) {
CHECK_LE(pc, symbols.size());
FUZZTEST_CHECK_LE(pc, symbols.size());
if (!pc_table[pc].has_flag(fuzztest::internal::PCInfo::kFuncEntry))
continue;
const SymbolTable::Entry entry = symbols.entry(pc);
Expand All @@ -197,7 +196,7 @@ void DumpCoverageReport(const CoverageResults &coverage_results,
std::ostringstream symbol_table_stream;
coverage_symbol_table.WriteToLLVMSymbolizer(symbol_table_stream);

CHECK_OK(
FUZZTEST_CHECK_OK(
RemoteFileSetContents(coverage_report_path, symbol_table_stream.str()));
}

Expand All @@ -210,8 +209,9 @@ AnalyzeCorporaResults AnalyzeCorpora(std::string_view binary_name,
BinaryInfo binary_info_b =
ReadBinaryInfo(binary_name, binary_hash, workdir_b);

CHECK_EQ(binary_info_a.pc_table.size(), binary_info_b.pc_table.size());
CHECK_EQ(binary_info_a.symbols.size(), binary_info_b.symbols.size());
FUZZTEST_CHECK_EQ(binary_info_a.pc_table.size(),
binary_info_b.pc_table.size());
FUZZTEST_CHECK_EQ(binary_info_a.symbols.size(), binary_info_b.symbols.size());

const std::vector<CorpusRecord> a =
ReadCorpora(binary_name, binary_hash, workdir_a);
Expand All @@ -235,26 +235,26 @@ void AnalyzeCorporaToLog(std::string_view binary_name,
CoverageLogger coverage_logger(pc_table, symbols);

// TODO(kcc): use frontier_a to show the most interesting b-only PCs.
// TODO(kcc): these cause a CHECK-fail
// TODO(kcc): these cause a FUZZTEST_CHECK-fail
// CoverageFrontier frontier_a(results.binary_info);
// frontier_a.Compute(a);

// First, print the newly covered functions (including partially covered).
LOG(INFO) << "B-only new functions:";
FUZZTEST_LOG(INFO) << "B-only new functions:";
absl::flat_hash_set<std::string_view> b_only_new_functions;
for (const auto pc : results.b_only_pcs) {
if (!pc_table[pc].has_flag(PCInfo::kFuncEntry)) continue;
auto str = coverage_logger.ObserveAndDescribeIfNew(pc);
if (!str.empty()) LOG(INFO).NoPrefix() << str;
if (!str.empty()) FUZZTEST_LOG(INFO).NoPrefix() << str;
b_only_new_functions.insert(symbols.func(pc));
}

// Now, print newly covered edges in functions that were covered in `a`.
LOG(INFO) << "B-only new edges:";
FUZZTEST_LOG(INFO) << "B-only new edges:";
for (const auto pc : results.b_only_pcs) {
if (b_only_new_functions.contains(symbols.func(pc))) continue;
auto str = coverage_logger.ObserveAndDescribeIfNew(pc);
if (!str.empty()) LOG(INFO).NoPrefix() << str;
if (!str.empty()) FUZZTEST_LOG(INFO).NoPrefix() << str;
}
}

Expand Down
2 changes: 1 addition & 1 deletion centipede/analyze_corpora.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ AnalyzeCorporaResults AnalyzeCorpora(std::string_view binary_name,
std::string_view workdir_a,
std::string_view workdir_b);

// Same as above but `LOG`s the results for human consumption.
// Same as above but `FUZZTEST_LOG`s the results for human consumption.
void AnalyzeCorporaToLog(std::string_view binary_name,
std::string_view binary_hash,
std::string_view workdir_a,
Expand Down
5 changes: 2 additions & 3 deletions centipede/analyze_corpora_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,11 @@

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "./centipede/binary_info.h"
#include "./centipede/environment.h"
#include "./centipede/symbol_table.h"
#include "./centipede/test_coverage_util.h"
#include "./common/logging.h"
#include "./common/remote_file.h"
#include "./common/test_util.h"

Expand All @@ -47,7 +46,7 @@ static std::string GetTargetPath() {
}

// TODO(ussuri): Implement.
TEST(AnalyzeCorpora, AnalyzeCorpora) { LOG(INFO) << "Unimplemented"; }
TEST(AnalyzeCorpora, AnalyzeCorpora) { FUZZTEST_LOG(INFO) << "Unimplemented"; }

TEST(GetCoverage, SimpleCoverageResults) {
Environment env;
Expand Down
36 changes: 18 additions & 18 deletions centipede/binary_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,13 @@
#include <utility>
#include <vector>

#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "./centipede/command.h"
#include "./centipede/control_flow.h"
#include "./centipede/pc_info.h"
#include "./centipede/util.h"
#include "./common/logging.h"
#include "./common/remote_file.h"

namespace fuzztest::internal {
Expand All @@ -45,18 +44,18 @@ void BinaryInfo::InitializeFromSanCovBinary(
std::string_view symbolizer_path, std::string_view tmp_dir_path) {
if (binary_path_with_args.empty()) {
// This usually happens in tests.
LOG(INFO) << __func__ << ": binary_path_with_args is empty";
FUZZTEST_LOG(INFO) << __func__ << ": binary_path_with_args is empty";
return;
}
// Compute names for temp files.
const std::filesystem::path tmp_dir = tmp_dir_path;
CHECK(std::filesystem::exists(tmp_dir) &&
std::filesystem::is_directory(tmp_dir));
FUZZTEST_CHECK(std::filesystem::exists(tmp_dir) &&
std::filesystem::is_directory(tmp_dir));
ScopedFile pc_table_path(tmp_dir_path, "pc_table_tmp");
ScopedFile cf_table_path(tmp_dir_path, "cf_table_tmp");
ScopedFile dso_table_path(tmp_dir_path, "dso_table_tmp");
ScopedFile log_path(tmp_dir_path, "binary_info_log_tmp");
LOG(INFO) << __func__ << ": tmp_dir: " << tmp_dir;
FUZZTEST_LOG(INFO) << __func__ << ": tmp_dir: " << tmp_dir;

Command::Options cmd_options;
cmd_options.env_add = {absl::StrCat(
Expand All @@ -66,7 +65,7 @@ void BinaryInfo::InitializeFromSanCovBinary(
Command cmd{binary_path_with_args, std::move(cmd_options)};
int exit_code = cmd.Execute();
if (exit_code != EXIT_SUCCESS) {
LOG(INFO) << __func__ << ": exit_code: " << exit_code;
FUZZTEST_LOG(INFO) << __func__ << ": exit_code: " << exit_code;
}

// Load PC Table.
Expand All @@ -80,24 +79,25 @@ void BinaryInfo::InitializeFromSanCovBinary(
dso_table = ReadDsoTableFromFile(dso_table_path.path());

if (pc_table.empty()) {
CHECK(dso_table.empty());
FUZZTEST_CHECK(dso_table.empty());
// Fallback to GetPcTableFromBinaryWithTracePC().
LOG(WARNING)
FUZZTEST_LOG(WARNING)
<< "Failed to dump PC table directly from binary using linked-in "
"runner; see target execution logs above; falling back to legacy PC "
"table extraction using trace-pc and objdump";
pc_table = GetPcTableFromBinaryWithTracePC(
binary_path_with_args, objdump_path, pc_table_path.path());
if (pc_table.empty()) {
LOG(ERROR) << "Failed to extract PC table from binary using objdump; see "
"objdump execution logs above";
FUZZTEST_LOG(ERROR)
<< "Failed to extract PC table from binary using objdump; see "
"objdump execution logs above";
}
// For the legacy trace-pc instrumentation, set the dso_table
// to 1-element array consisting of the binary name
const std::vector<std::string> args =
absl::StrSplit(binary_path_with_args, absl::ByAnyChar{" \t\n"},
absl::SkipWhitespace{});
CHECK(!args.empty());
FUZZTEST_CHECK(!args.empty());
dso_table.push_back({args[0], pc_table.size()});
uses_legacy_trace_pc_instrumentation = true;
} else {
Expand All @@ -110,7 +110,7 @@ void BinaryInfo::InitializeFromSanCovBinary(
for (const auto& dso : dso_table) {
num_instrumened_pcs_in_all_dsos += dso.num_instrumented_pcs;
}
CHECK_EQ(num_instrumened_pcs_in_all_dsos, pc_table.size());
FUZZTEST_CHECK_EQ(num_instrumened_pcs_in_all_dsos, pc_table.size());
}

// Load symbols, if there is a PC table.
Expand All @@ -125,14 +125,14 @@ void BinaryInfo::InitializeFromSanCovBinary(
void BinaryInfo::Read(std::string_view dir) {
std::string symbol_table_contents;
// TODO(b/295978603): move calculation of paths into WorkDir class.
CHECK_OK(RemoteFileGetContents(
FUZZTEST_CHECK_OK(RemoteFileGetContents(
(std::filesystem::path(dir) / kSymbolTableFileName).c_str(),
symbol_table_contents));
std::istringstream symbol_table_stream(symbol_table_contents);
symbols.ReadFromLLVMSymbolizer(symbol_table_stream);

std::string pc_table_contents;
CHECK_OK(RemoteFileGetContents(
FUZZTEST_CHECK_OK(RemoteFileGetContents(
(std::filesystem::path(dir) / kPCTableFileName).c_str(),
pc_table_contents));
std::istringstream pc_table_stream(pc_table_contents);
Expand All @@ -146,19 +146,19 @@ void BinaryInfo::Write(std::string_view dir) {
std::ostringstream symbol_table_stream;
symbols.WriteToLLVMSymbolizer(symbol_table_stream);
// TODO(b/295978603): move calculation of paths into WorkDir class.
CHECK_OK(RemoteFileSetContents(
FUZZTEST_CHECK_OK(RemoteFileSetContents(
(std::filesystem::path(dir) / kSymbolTableFileName).c_str(),
symbol_table_stream.str()));

std::ostringstream pc_table_stream;
WritePcTable(pc_table, pc_table_stream);
CHECK_OK(RemoteFileSetContents(
FUZZTEST_CHECK_OK(RemoteFileSetContents(
(std::filesystem::path(dir) / kPCTableFileName).c_str(),
pc_table_stream.str()));

std::ostringstream cf_table_stream;
WriteCfTable(cf_table, cf_table_stream);
CHECK_OK(RemoteFileSetContents(
FUZZTEST_CHECK_OK(RemoteFileSetContents(
(std::filesystem::path(dir) / kCfTableFileName).c_str(),
cf_table_stream.str()));
}
Expand Down
45 changes: 24 additions & 21 deletions centipede/blob_file_converter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@

#include "absl/base/nullability.h"
#include "absl/flags/flag.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/time/clock.h"
Expand Down Expand Up @@ -59,11 +57,11 @@ class StatsLogger {
const std::string stats = absl::StrFormat(
"blobs: %9lld | blobs/s: %5.0f | bytes: %12lld | bytes/s: %8.0f",
num_blobs_, num_blobs_ / secs, num_bytes_, num_bytes_ / secs);
if (ABSL_VLOG_IS_ON(3)) {
if (FUZZTEST_VLOG_IS_ON(3)) {
const RUsageProfiler::Snapshot& snapshot = RPROF_SNAPSHOT(stats);
LOG(INFO) << stats << " | " << snapshot.memory.ShortStr();
FUZZTEST_LOG(INFO) << stats << " | " << snapshot.memory.ShortStr();
} else {
LOG(INFO) << stats;
FUZZTEST_LOG(INFO) << stats;
}
}

Expand All @@ -90,45 +88,49 @@ class StatsLogger {
void Convert( //
const std::string& in, //
const std::string& out, const std::string& out_format) {
RPROF_THIS_FUNCTION_WITH_REPORT(/*enable=*/ABSL_VLOG_IS_ON(1));
RPROF_THIS_FUNCTION_WITH_REPORT(/*enable=*/FUZZTEST_VLOG_IS_ON(1));

LOG(INFO) << "Converting:\n" << VV(in) << "\n" << VV(out) << VV(out_format);
FUZZTEST_LOG(INFO) << "Converting:\n"
<< VV(in) << "\n"
<< VV(out) << VV(out_format);

const bool out_is_riegeli = out_format == "riegeli";

// Verify and prepare source and destination.

CHECK(RemotePathExists(in)) << VV(in);
CHECK_OK(RemoteMkdir(std::filesystem::path{out}.parent_path().c_str()));
FUZZTEST_CHECK(RemotePathExists(in)) << VV(in);
FUZZTEST_CHECK_OK(
RemoteMkdir(std::filesystem::path{out}.parent_path().c_str()));

// Open blob file reader and writer.

RPROF_START_TIMELAPSE( //
absl::Seconds(20), /*also_log=*/ABSL_VLOG_IS_ON(3), "Opening --in");
absl::Seconds(20), /*also_log=*/FUZZTEST_VLOG_IS_ON(3), "Opening --in");
const auto in_reader = DefaultBlobFileReaderFactory();
CHECK_OK(in_reader->Open(in)) << VV(in);
FUZZTEST_CHECK_OK(in_reader->Open(in)) << VV(in);
RPROF_STOP_TIMELAPSE();
RPROF_SNAPSHOT_AND_LOG("Opened --in; opening --out");
RPROF_SNAPSHOT_AND_FUZZTEST_LOG("Opened --in; opening --out");
const auto out_writer = DefaultBlobFileWriterFactory(out_is_riegeli);
CHECK_OK(out_writer->Open(out, "w")) << VV(out);
RPROF_SNAPSHOT_AND_LOG("Opened --out");
FUZZTEST_CHECK_OK(out_writer->Open(out, "w")) << VV(out);
RPROF_SNAPSHOT_AND_FUZZTEST_LOG("Opened --out");

// Read and write blobs one-by-one.

ByteSpan blob;
absl::Status read_status = absl::OkStatus();
StatsLogger stats_logger{
absl::Seconds(ABSL_VLOG_IS_ON(1) ? 20 : 60),
absl::Seconds(FUZZTEST_VLOG_IS_ON(1) ? 20 : 60),
FUNCTION_LEVEL_RPROF_NAME,
};
while ((read_status = in_reader->Read(blob)).ok()) {
CHECK_OK(out_writer->Write(blob));
FUZZTEST_CHECK_OK(out_writer->Write(blob));
stats_logger.UpdateStats(blob);
stats_logger.MaybeLogIfTime();
}
stats_logger.Log();
CHECK(read_status.ok() || absl::IsOutOfRange(read_status)) << VV(read_status);
CHECK_OK(out_writer->Close()) << VV(out);
FUZZTEST_CHECK(read_status.ok() || absl::IsOutOfRange(read_status))
<< VV(read_status);
FUZZTEST_CHECK_OK(out_writer->Close()) << VV(out);
}

} // namespace
Expand All @@ -138,11 +140,12 @@ int main(int argc, char** absl_nonnull argv) {
(void)fuzztest::internal::InitRuntime(argc, argv);

const std::string in = absl::GetFlag(FLAGS_in);
QCHECK(!in.empty());
FUZZTEST_QCHECK(!in.empty());
const std::string out = absl::GetFlag(FLAGS_out);
QCHECK(!out.empty());
FUZZTEST_QCHECK(!out.empty());
const std::string out_format = absl::GetFlag(FLAGS_out_format);
QCHECK(out_format == "legacy" || out_format == "riegeli") << VV(out_format);
FUZZTEST_QCHECK(out_format == "legacy" || out_format == "riegeli")
<< VV(out_format);

fuzztest::internal::Convert(in, out, out_format);

Expand Down
Loading
Loading