Skip to content

Let persistent batch stop gracefully on cleanup if possible. #2035

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Apr 28, 2026
Merged

Let persistent batch stop gracefully on cleanup if possible. #2035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions centipede/centipede.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,14 +389,19 @@ bool Centipede::ExecuteAndReportCrash(std::string_view binary,
BatchResult& batch_result) {
bool success = user_callbacks_.Execute(binary, input_vec, batch_result);
if (success) return true;
if (batch_result.IsIgnoredFailure()) {
FUZZTEST_LOG(INFO) << "Skip reporting "
<< batch_result.failure_description();
return true;
}
if (ShouldStop()) {
FUZZTEST_LOG_FIRST_N(WARNING, 1)
<< "Crash found but the stop condition is met - not reporting further "
"possibly related crashes.";
return true;
}
ReportCrash(binary, input_vec, batch_result);
return batch_result.IsIgnoredFailure();
return false;
}

// *** Highly experimental and risky. May not scale well for large targets. ***
Expand Down Expand Up @@ -980,6 +985,8 @@ void Centipede::ReportCrash(std::string_view binary,
const BatchResult& batch_result) {
FUZZTEST_CHECK_EQ(input_vec.size(), batch_result.results().size());

if (batch_result.IsIgnoredFailure()) return;

const size_t suspect_input_idx = std::clamp<size_t>(
batch_result.num_outputs_read(), 0, input_vec.size() - 1);
auto log_execution_failure = [&](std::string_view log_prefix) {
Expand All @@ -1005,12 +1012,6 @@ void Centipede::ReportCrash(std::string_view binary,
FUZZTEST_LOG(INFO).NoPrefix() << "\n";
};

if (batch_result.IsIgnoredFailure()) {
FUZZTEST_LOG(INFO) << "Skip further processing of "
<< batch_result.failure_description();
return;
}

if (batch_result.IsSkippedTest()) {
log_execution_failure("Skipped Test: ");
FUZZTEST_LOG(INFO) << "Requesting early stop due to skipped test.";
Expand Down
106 changes: 67 additions & 39 deletions centipede/centipede_callbacks.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,32 +101,44 @@ class CentipedeCallbacks::PersistentModeServer {

const std::string& server_path() const { return server_path_; }

// Triggers the persistent mode to run one batch request. If the persistent
// mode finished normally, returns true with `exit_code` set to the return
// value of the handler (which would be returned as the command exit code if
// running without persistent mode, hence the name). Returns false otherwise.
bool RunBatch(absl::Time deadline, int& exit_code) {
// Requests the persistent mode client to run one batch request. Returns true
// if the request is sent successfully within `deadline`, false otherwise.
bool RequestBatch(absl::Time deadline) {
if (!EnsureConnection(deadline)) {
return false;
}
FUZZTEST_CHECK_NE(conn_socket_, -1);
FUZZTEST_CHECK(!in_batch_);
if (!WriteFd(conn_socket_, deadline, PersistentModeRequest::kRunBatch)) {
FUZZTEST_LOG(ERROR)
<< "Failed to request the persistent mode client to run a "
"batch - disconnecting.";
Disconnect();
"batch.";
return false;
}
in_batch_ = true;
return true;
}

// Waits for the persistent mode client to finish the batch (if previously
// requested) and gather the result. Returns true with `exit_code` set to the
// return value of the client (which would be returned as the command exit
// code if running without persistent mode, hence the name). Returns false
// otherwise.
bool WaitBatch(absl::Time deadline, int& exit_code) {
FUZZTEST_CHECK_NE(conn_socket_, -1);
FUZZTEST_CHECK(in_batch_);
if (!ReadFd(conn_socket_, deadline, exit_code)) {
FUZZTEST_LOG(ERROR)
<< "Failed to receive the batch response from the persistent "
"mode client - disconnecting.";
Disconnect();
"mode client.";
return false;
}
in_batch_ = false;
return true;
}

bool in_batch() const { return in_batch_; }

void RequestExit(absl::Time deadline) {
if (!EnsureConnection(deadline)) return;
FUZZTEST_CHECK_NE(conn_socket_, -1);
Expand Down Expand Up @@ -288,6 +300,7 @@ class CentipedeCallbacks::PersistentModeServer {
std::string server_path_;
int server_socket_ = -1;
int conn_socket_ = -1;
bool in_batch_ = false;
};

void CentipedeCallbacks::PopulateBinaryInfo(BinaryInfo& binary_info) {
Expand Down Expand Up @@ -468,44 +481,59 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
} else {
last_execute_log_path_ = cmd.stdout_file();
}
if (command_context.persistent_mode_server != nullptr &&
command_context.persistent_mode_server->RunBatch(deadline, exit_code)) {
if (command_context.persistent_mode_server != nullptr) {
if (!command_context.persistent_mode_server->RequestBatch(deadline)) {
return true;
}
if (!command_context.persistent_mode_server->WaitBatch(deadline,
exit_code)) {
return true;
}
return false;
}
const std::optional<int> ret = cmd.Wait(deadline);
if (!ret.has_value()) return true;
exit_code = *ret;
return false;
}();
if (should_clean_up) {
exit_code = [&] {
if (!cmd.is_executing()) return EXIT_FAILURE;
FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout: "
<< env_.runner_cleanup_timeout;
cmd.RequestStop();
const auto ret = cmd.Wait(absl::Now() + env_.runner_cleanup_timeout);
if (ret.has_value()) return *ret;
FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup.";
return EXIT_FAILURE;
}();
// We need to save any execution log before the destruction of the command.
std::error_code ec;
std::filesystem::rename(last_execute_log_path_, saved_execute_log_path_,
ec);
if (ec) {
FUZZTEST_LOG(ERROR) << "Failed to save the execution log "
<< last_execute_log_path_ << " to "
<< saved_execute_log_path_ << "(" << ec.message()
<< "). Left with an empty log ...";
(void)std::filesystem::remove(saved_execute_log_path_, ec);
};
last_execute_log_path_ = saved_execute_log_path_;
command_contexts_.erase(
std::find_if(command_contexts_.begin(), command_contexts_.end(),
[=](const auto& command_context) {
return command_context->cmd.path() == binary;
}));
if (!should_clean_up) return exit_code;
FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout: "
<< env_.runner_cleanup_timeout;
const auto cleanup_deadline = absl::Now() + env_.runner_cleanup_timeout;
if (cmd.is_executing() && command_context.persistent_mode_server != nullptr &&
command_context.persistent_mode_server->in_batch()) {
// Give a chance for the batch to gracefully stop. If succeeded, we can keep
// the Command.
cmd.RequestStop();
if (command_context.persistent_mode_server->WaitBatch(cleanup_deadline,
exit_code)) {
return exit_code;
}
}
exit_code = [&] {
if (!cmd.is_executing()) return EXIT_FAILURE;
cmd.RequestStop();
const auto ret = cmd.Wait(cleanup_deadline);
if (ret.has_value()) return *ret;
FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup.";
return EXIT_FAILURE;
}();
// We need to save any execution log before the destruction of the command.
std::error_code ec;
std::filesystem::rename(last_execute_log_path_, saved_execute_log_path_, ec);
if (ec) {
FUZZTEST_LOG(ERROR) << "Failed to save the execution log "
<< last_execute_log_path_ << " to "
<< saved_execute_log_path_ << "(" << ec.message()
<< "). Left with an empty log ...";
(void)std::filesystem::remove(saved_execute_log_path_, ec);
}
last_execute_log_path_ = saved_execute_log_path_;
command_contexts_.erase(
std::find_if(command_contexts_.begin(), command_contexts_.end(),
[=](const auto& command_context) {
return command_context->cmd.path() == binary;
}));
return exit_code;
}

Expand Down
Loading