Skip to content

No public description #1759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

No public description #1759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion centipede/batch_fuzz_example/customized_centipede.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ bool CustomizedCallbacks::Execute(std::string_view binary,
cmd_options.stdout_file = tmp_log_filepath;
cmd_options.stderr_file = tmp_log_filepath;
Command cmd{env_.binary, std::move(cmd_options)};
const int retval = cmd.Execute();
const auto retval = cmd.Execute();

std::string tmp_log;
ReadFromLocalFile(tmp_log_filepath, tmp_log);
Expand Down
7 changes: 4 additions & 3 deletions centipede/binary_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,10 @@ void BinaryInfo::InitializeFromSanCovBinary(
":arg2=", cf_table_path.path(), ":arg3=", dso_table_path.path(), ":")};
cmd_options.stdout_file = std::string(log_path.path());
Command cmd{binary_path_with_args, std::move(cmd_options)};
int exit_code = cmd.Execute();
if (exit_code != EXIT_SUCCESS) {
LOG(INFO) << __func__ << ": exit_code: " << exit_code;
const auto exit_code = cmd.Execute();
PCHECK(exit_code.has_value()) << "no exit_code returned!";
if (*exit_code != EXIT_SUCCESS) {
LOG(INFO) << __func__ << ": exit_code: " << *exit_code;
}

// Load PC Table.
Expand Down
90 changes: 59 additions & 31 deletions centipede/centipede_callbacks.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <cstddef>
#include <cstdlib>
#include <filesystem> // NOLINT
#include <memory>
#include <string>
#include <string_view>
#include <system_error> // NOLINT
Expand All @@ -38,6 +39,7 @@
#include "./centipede/mutation_input.h"
#include "./centipede/runner_request.h"
#include "./centipede/runner_result.h"
#include "./centipede/shared_memory_blob_sequence.h"
#include "./centipede/util.h"
#include "./centipede/workdir.h"
#include "./common/blob_file.h"
Expand Down Expand Up @@ -143,6 +145,13 @@ std::string CentipedeCallbacks::ConstructRunnerFlags(
return absl::StrJoin(flags, ":");
}

void CentipedeCallbacks::OpenBlobSequences() {
inputs_blobseq_ = std::make_unique<SharedMemoryBlobSequence>(
shmem_name1_.c_str(), env_.shmem_size_mb << 20, env_.use_posix_shmem);
outputs_blobseq_ = std::make_unique<SharedMemoryBlobSequence>(
shmem_name2_.c_str(), env_.shmem_size_mb << 20, env_.use_posix_shmem);
}

Command &CentipedeCallbacks::GetOrCreateCommandForBinary(
std::string_view binary) {
for (auto &cmd : commands_) {
Expand All @@ -155,7 +164,7 @@ Command &CentipedeCallbacks::GetOrCreateCommandForBinary(

std::vector<std::string> env = {ConstructRunnerFlags(
absl::StrCat(":shmem:test=", env_.test_name, ":arg1=",
inputs_blobseq_.path(), ":arg2=", outputs_blobseq_.path(),
inputs_blobseq_->path(), ":arg2=", outputs_blobseq_->path(),
":failure_description_path=", failure_description_path_,
":failure_signature_path=", failure_signature_path_, ":"),
disable_coverage)};
Expand Down Expand Up @@ -191,8 +200,8 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
batch_result.ClearAndResize(inputs.size());

// Reset the blobseqs.
inputs_blobseq_.Reset();
outputs_blobseq_.Reset();
inputs_blobseq_->Reset();
outputs_blobseq_->Reset();

size_t num_inputs_written = 0;

Expand All @@ -202,7 +211,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
num_inputs_written = 1;
} else {
// Feed the inputs to inputs_blobseq_.
num_inputs_written = RequestExecution(inputs, inputs_blobseq_);
num_inputs_written = RequestExecution(inputs, *inputs_blobseq_);
}

if (num_inputs_written != inputs.size()) {
Expand All @@ -213,14 +222,18 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(

// Run.
Command &cmd = GetOrCreateCommandForBinary(binary);
int retval = cmd.Execute();
inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed.
const auto exit_code = cmd.Execute();
inputs_blobseq_->ReleaseSharedMemory(); // Inputs are already consumed.

// Get results.
batch_result.exit_code() = retval;
const bool read_success = batch_result.Read(outputs_blobseq_);
if (exit_code.has_value()) {
batch_result.exit_code() = *exit_code;
} else {
batch_result.exit_code() = EXIT_FAILURE;
}
const bool read_success = batch_result.Read(*outputs_blobseq_);
LOG_IF(ERROR, !read_success) << "Failed to read batch result!";
outputs_blobseq_.ReleaseSharedMemory(); // Outputs are already consumed.
outputs_blobseq_->ReleaseSharedMemory(); // Outputs are already consumed.

// We may have fewer feature blobs than inputs if
// * some inputs were not written (i.e. num_inputs_written < inputs.size).
Expand All @@ -229,7 +242,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
// * Will be logged by the caller.
// * some outputs were not written because the outputs_blobseq_ overflown.
// * Logged by the following code.
if (retval == 0 && read_success &&
if (batch_result.exit_code() == EXIT_SUCCESS && read_success &&
batch_result.num_outputs_read() != num_inputs_written) {
LOG(INFO) << "Read " << batch_result.num_outputs_read() << "/"
<< num_inputs_written
Expand All @@ -239,7 +252,10 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(

if (env_.print_runner_log) PrintExecutionLog();

if (retval != EXIT_SUCCESS) {
if (!exit_code.has_value()) {
batch_result.failure_description() = "IGNORED FAILURE: execution timed out";
OpenBlobSequences();
} else if (*exit_code != EXIT_SUCCESS) {
ReadFromLocalFile(execute_log_path_, batch_result.log());
ReadFromLocalFile(failure_description_path_,
batch_result.failure_description());
Expand All @@ -251,13 +267,13 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
// be removed.
batch_result.failure_signature() = batch_result.failure_description();
}
// Remove the failure description and signature files here so that they do
// not stay until another failed execution.
std::filesystem::remove(failure_description_path_);
std::filesystem::remove(failure_signature_path_);
}
// Remove the failure description and signature files here so that they do
// not stay until another failed execution.
std::filesystem::remove(failure_description_path_);
std::filesystem::remove(failure_signature_path_);
VLOG(1) << __FUNCTION__ << " took " << (absl::Now() - start_time);
return retval;
return batch_result.exit_code();
}

// See also: `DumpSeedsToDir()`.
Expand All @@ -283,10 +299,15 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary(
cmd_options.stderr_file = execute_log_path_;
cmd_options.temp_file_path = temp_input_file_path_;
Command cmd{binary, std::move(cmd_options)};
const int retval = cmd.Execute();
const auto exit_code = cmd.Execute();

if (env_.print_runner_log) {
LOG(INFO) << "Getting seeds via external binary returns " << retval;
if (exit_code.has_value()) {
LOG(INFO)
<< "Getting seeds via external binary failed without an exit code";
} else {
LOG(INFO) << "Getting seeds via external binary returns " << *exit_code;
}
PrintExecutionLog();
}

Expand All @@ -308,7 +329,7 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary(
seeds.resize(num_seeds_read);
std::filesystem::remove_all(output_dir);

return retval == 0;
return exit_code.has_value() && *exit_code == EXIT_SUCCESS;
}

// See also: `DumpSerializedTargetConfigToFile()`.
Expand Down Expand Up @@ -357,30 +378,37 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary(
<< "Standalone binary does not support custom mutator";

auto start_time = absl::Now();
inputs_blobseq_.Reset();
outputs_blobseq_.Reset();
inputs_blobseq_->Reset();
outputs_blobseq_->Reset();

size_t num_inputs_written =
RequestMutation(num_mutants, inputs, inputs_blobseq_);
RequestMutation(num_mutants, inputs, *inputs_blobseq_);
LOG_IF(INFO, num_inputs_written != inputs.size())
<< VV(num_inputs_written) << VV(inputs.size());

// Execute.
Command &cmd = GetOrCreateCommandForBinary(binary);
int retval = cmd.Execute();
inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed.
const auto exit_code = cmd.Execute();
inputs_blobseq_->ReleaseSharedMemory(); // Inputs are already consumed.

if (retval != EXIT_SUCCESS) {
LOG(WARNING) << "Custom mutator failed with exit code: " << retval;
}
if (env_.print_runner_log || retval != EXIT_SUCCESS) {
if (!exit_code.has_value()) {
LOG(WARNING) << "Custom mutator failed without exit code.";
PrintExecutionLog();
} else if (*exit_code != EXIT_SUCCESS) {
LOG(WARNING) << "Custom mutator failed with exit code: " << *exit_code;
PrintExecutionLog();
} else if (env_.print_runner_log) {
PrintExecutionLog();
}

MutationResult result;
result.exit_code() = retval;
result.Read(num_mutants, outputs_blobseq_);
outputs_blobseq_.ReleaseSharedMemory(); // Outputs are already consumed.
result.exit_code() = exit_code.value_or(EXIT_FAILURE);
result.Read(num_mutants, *outputs_blobseq_);
outputs_blobseq_->ReleaseSharedMemory(); // Outputs are already consumed.

if (!exit_code.has_value()) {
OpenBlobSequences();
}

VLOG(1) << __FUNCTION__ << " took " << (absl::Now() - start_time);
return result;
Expand Down
15 changes: 8 additions & 7 deletions centipede/centipede_callbacks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,8 @@ class CentipedeCallbacks {
CentipedeCallbacks(const Environment &env)
: env_(env),
byte_array_mutator_(env.knobs, GetRandomSeed(env.seed)),
fuzztest_mutator_(env.knobs, GetRandomSeed(env.seed)),
inputs_blobseq_(shmem_name1_.c_str(), env.shmem_size_mb << 20,
env.use_posix_shmem),
outputs_blobseq_(shmem_name2_.c_str(), env.shmem_size_mb << 20,
env.use_posix_shmem) {
fuzztest_mutator_(env.knobs, GetRandomSeed(env.seed)) {
OpenBlobSequences();
if (env.use_legacy_default_mutator)
CHECK(byte_array_mutator_.set_max_len(env.max_len));
else
Expand Down Expand Up @@ -158,6 +155,10 @@ class CentipedeCallbacks {
FuzzTestMutator fuzztest_mutator_;

private:
// (Re-)open inputs/outputs blob sequences. Note that this destroys previous
// blob sequence objects and thus resets the reference points on the file
// system level.
void OpenBlobSequences();
// Returns a Command object with matching `binary` from commands_,
// creates one if needed.
Command &GetOrCreateCommandForBinary(std::string_view binary);
Expand All @@ -179,8 +180,8 @@ class CentipedeCallbacks {
const std::string shmem_name1_ = ProcessAndThreadUniqueID("/ctpd-shm1-");
const std::string shmem_name2_ = ProcessAndThreadUniqueID("/ctpd-shm2-");

SharedMemoryBlobSequence inputs_blobseq_;
SharedMemoryBlobSequence outputs_blobseq_;
std::unique_ptr<SharedMemoryBlobSequence> inputs_blobseq_;
std::unique_ptr<SharedMemoryBlobSequence> outputs_blobseq_;

std::vector<Command> commands_;
};
Expand Down
54 changes: 43 additions & 11 deletions centipede/command.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
#include <cstdlib>
#include <filesystem> // NOLINT
#include <fstream>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <system_error> // NOLINT
Expand Down Expand Up @@ -122,6 +124,13 @@ struct Command::ForkServerProps {
// recycled by the OS.
std::string creation_stamp;

void CopyPaths(const ForkServerProps &other) {
for (int i = 0; i < 2; ++i) {
fifo_path_[i] = other.fifo_path_[i];
}
pid_file_path_ = other.pid_file_path_;
}

~ForkServerProps() {
for (int i = 0; i < 2; ++i) {
if (pipe_[i] >= 0 && close(pipe_[i]) != 0) {
Expand Down Expand Up @@ -208,9 +217,24 @@ bool Command::StartForkServer(std::string_view temp_dir_path,
.append(absl::StrCat(prefix, "_FIFO0"));
fork_server_->fifo_path_[1] = std::filesystem::path(temp_dir_path)
.append(absl::StrCat(prefix, "_FIFO1"));
const std::string pid_file_path =
fork_server_->pid_file_path_ =
std::filesystem::path(temp_dir_path).append("pid");

(void)std::filesystem::create_directory(temp_dir_path); // it may not exist.

if (!StartForkServerInternally()) {
LOG(ERROR) << "Failed to start the fork server. Will proceed without it.";
fork_server_.reset();
return false;
}
return true;
}

bool Command::StartForkServerInternally() {
CHECK(fork_server_ != nullptr);
PCHECK(fork_server_->pid_ == -1)
<< "Fork server is already started with PID " << fork_server_->pid_;

for (int i = 0; i < 2; ++i) {
PCHECK(mkfifo(fork_server_->fifo_path_[i].c_str(), 0600) == 0)
<< VV(i) << VV(fork_server_->fifo_path_[i]);
Expand All @@ -230,16 +254,14 @@ bool Command::StartForkServer(std::string_view temp_dir_path,
)sh";
const std::string fork_server_command = absl::StrFormat(
kForkServerCommandStub, fork_server_->fifo_path_[0],
fork_server_->fifo_path_[1], command_line_, pid_file_path);
fork_server_->fifo_path_[1], command_line_, fork_server_->pid_file_path_);
VLOG(2) << "Fork server command:" << fork_server_command;

const int exit_code = system(fork_server_command.c_str());

// Check if `system()` was able to parse and run the command at all.
if (exit_code != EXIT_SUCCESS) {
LogProblemInfo(
"Failed to parse or run command to launch fork server; will proceed "
"without it");
LogProblemInfo("Failed to parse or run command to launch fork server.");
return false;
}

Expand All @@ -259,14 +281,12 @@ bool Command::StartForkServer(std::string_view temp_dir_path,
O_RDWR | O_NONBLOCK)) < 0 ||
(fork_server_->pipe_[1] = open(fork_server_->fifo_path_[1].c_str(),
O_RDONLY | O_NONBLOCK)) < 0) {
LogProblemInfo(
"Failed to establish communication with fork server; will proceed "
"without it");
LogProblemInfo("Failed to establish communication with fork server.");
return false;
}

std::string pid_str;
ReadFromLocalFile(pid_file_path, pid_str);
ReadFromLocalFile(fork_server_->pid_file_path_, pid_str);
CHECK(absl::SimpleAtoi(pid_str, &fork_server_->pid_)) << VV(pid_str);
auto creation_stamp = GetProcessCreationStamp(fork_server_->pid_);
if (!creation_stamp.ok()) {
Expand Down Expand Up @@ -308,7 +328,7 @@ absl::Status Command::VerifyForkServerIsHealthy() {
return absl::OkStatus();
}

int Command::Execute() {
std::optional<int> Command::Execute() {
VLOG(1) << "Executing command '" << command_line_ << "'...";

int exit_code = EXIT_SUCCESS;
Expand Down Expand Up @@ -357,7 +377,19 @@ int Command::Execute() {
LogProblemInfo(absl::StrCat(
"Error while waiting for fork server: poll() returned ", poll_ret));
}
return EXIT_FAILURE;

auto new_fork_server = std::make_unique<ForkServerProps>();
new_fork_server->CopyPaths(*fork_server_);
fork_server_ = std::move(new_fork_server);
if (!StartForkServerInternally()) {
LOG(ERROR) << "Failed to restart the fork server, will disable it for "
"the next execution.";
fork_server_.reset();
} else {
LOG(INFO) << "Restarted fork server";
}

return std::nullopt;
}

// The fork server wrote the execution result to the pipe: read it.
Expand Down
Loading