Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/LogLoader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,11 @@ void LogLoader::upload_pending_logs(std::shared_ptr<ServerInterface> server)
} else if (result.status_code == 400) {
LOG("Log upload failed (" << result.status_code << "): " << result.message);

} else if (result.status_code == 503) {
// Server down (e.g. local flight-review not running). Already logged once
// with a cooldown in ServerInterface; do not walk the rest of the queue.
return;

} else {
LOG("Log upload TEMPORARILY FAILED (" << result.status_code << "): "
<< result.message << " - Will retry later");
Expand Down
38 changes: 34 additions & 4 deletions src/ServerInterface.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,9 @@ ServerInterface::UploadResult ServerInterface::upload(const std::string& filepat
}

if (!server_reachable()) {
return {false, 0, "Server unreachable: " + _settings.server_url};
// 503: treat as service unavailable so the upload loop can bail the batch
// instead of hammering a dead local flight-review for every pending log.
return {false, 503, "Server unreachable: " + _settings.server_url};
}

std::ifstream file(filepath, std::ios::binary);
Expand Down Expand Up @@ -470,24 +472,52 @@ ServerInterface::UploadResult ServerInterface::upload(const std::string& filepat

bool ServerInterface::server_reachable()
{
const auto now = std::chrono::steady_clock::now();

// Still inside the cooldown from a previous failure — do not re-probe or re-log.
if (now < _unreachable_until) {
return false;
}

httplib::Result res;

if (_protocol == Protocol::Https) {
httplib::SSLClient cli(_settings.server_url);
cli.set_connection_timeout(2, 0);
cli.set_read_timeout(2, 0);
res = cli.Get("/");

} else {
httplib::Client cli(_settings.server_url);
cli.set_connection_timeout(2, 0);
cli.set_read_timeout(2, 0);
res = cli.Get("/");
}

bool success = res && res->status == 200;
const bool success = res && res->status == 200;

if (!success) {
LOG("Connection to " << _settings.server_url << " failed: " << (res ? std::to_string(res->status) : "No response"));
_unreachable_until = now + kUnreachableCooldown;

if (!_reported_unreachable) {
const std::string detail = res ? ("HTTP " + std::to_string(res->status)) : "No response";
LOG("Upload server " << _settings.server_url << " unreachable (" << detail
<< "); skipping uploads for "
<< std::chrono::duration_cast<std::chrono::seconds>(kUnreachableCooldown).count()
<< "s");
_reported_unreachable = true;
}

return false;
}

return success;
if (_reported_unreachable) {
LOG("Upload server " << _settings.server_url << " is reachable again");
_reported_unreachable = false;
}

_unreachable_until = {};
return true;
}

bool ServerInterface::init_database()
Expand Down
12 changes: 11 additions & 1 deletion src/ServerInterface.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once

#include <chrono>
#include <cstdint>
#include <string>
#include <vector>
Expand All @@ -19,7 +20,9 @@ class ServerInterface

struct UploadResult {
bool success;
int status_code; // HTTP status code, or 0 if not applicable
// HTTP status, or 0 when not applicable. 503 means the server is unreachable
// (connectivity); callers should stop the current upload batch and retry later.
int status_code;
std::string message;
};

Expand Down Expand Up @@ -100,4 +103,11 @@ class ServerInterface
bool _should_exit = false;
bool _has_legacy_table = false;
sqlite3* _db = nullptr;

// When flight-review (or any upload target) is down, avoid probing and logging
// once per pending log. Probe at most every kUnreachableCooldown, and log only on
// the down/up transitions.
static constexpr auto kUnreachableCooldown = std::chrono::seconds(60);
std::chrono::steady_clock::time_point _unreachable_until {};
bool _reported_unreachable {false};
};