feat(logloader): download logs over MAVLink FTP only

CMakeLists.txt

@@ -33,6 +33,8 @@ include_directories(${SQLite3_INCLUDE_DIRS})
 add_executable(${PROJECT_NAME}
     src/main.cpp
     src/ServerInterface.cpp
+    src/FtpListClient.cpp
+    src/FtpLogFetcher.cpp
     src/LogLoader.cpp)
 
 target_link_libraries(${PROJECT_NAME}

README.md

@@ -1,12 +1,29 @@
 ![image](logloader_logo.png)
 
-Downloads PX4 log files (.ulg) and uploads them to a local server and optionally a remote server.
+Downloads flight logs from the vehicle and uploads them to a local server and optionally a remote server. Works with PX4 (`.ulg`) and ArduPilot (`.BIN`).
 
 The **config.toml** file is used to configure the program settings.
 
 ### Behavior
 Downloading and uploading will only occur while the vehicle is not armed. Downloading and uploading operations are performed in separate threads. An sqlite database per server is used to track log file download/upload status.
 
+### Log transport
+Everything runs over **MAVLink FTP** (`FILE_TRANSFER_PROTOCOL`, msg 110): the directory listing that finds the logs and the bulk transfer that downloads them. The classic log protocol (`LOG_REQUEST_LIST` / `LOG_ENTRY` / `LOG_DATA`, msgs 117-120) is not used at all.
+
+`FILE_TRANSFER_PROTOCOL` carries `target_system` and `target_component`, and both PX4 and ArduPilot address their replies back to the requesting sysid/compid, so a download is unicast between the vehicle and logloader. `LOG_DATA` has no target fields, which leaves a router in between — mavlink-router, for instance — no choice but to copy every chunk to every endpoint it serves, telemetry radio included. A log download over the old protocol saturates links that have no interest in it.
+
+The autopilot's MAVLink instance must have FTP enabled (`mavlink start -x` on PX4). If it does not, logloader says so and idles.
+
+### Log discovery
+At startup logloader lists the vehicle's log directory, trying `@MAV_LOG` first — the virtual log directory the MAVLink FTP specification defines, supported by PX4 v1.17 and newer — then the physical `/fs/microsd/log` (PX4) and `/APM/LOGS` (ArduPilot). Set `remote_log_directory` to skip the probe. The listing is what identifies a log: its path below the log root plus its size. PX4's nested `<date>/<time>.ulg` layout and ArduPilot's flat `<number>.BIN` layout are both handled, and the file extension follows whatever the vehicle actually has.
+
+Log timestamps come from the modification time in the listing when the vehicle supports the `ListDirectoryWithTime` opcode (PX4 v1.17 and newer). Otherwise PX4 logs fall back to the start time encoded in the path, and ArduPilot logs have no timestamp — nothing depends on having one.
+
+Downloads are staged in a temporary directory and only moved next to the finished logs once the transferred size matches the listing, so a partial file is never mistaken for a complete one.
+
+### Upgrading from a pre-FTP logloader
+Older versions identified logs by the timestamp `LOG_ENTRY` reported, which MAVLink FTP cannot reproduce. On first start the existing `logs` table is renamed to `logs_legacy` and its rows are matched against the FTP listing by size, so logs already downloaded and uploaded are not fetched or uploaded a second time. Nothing is deleted; `logs_legacy` stays behind for inspection.
+
 ### Build
 Install dependencies
 ```
@@ -85,7 +102,7 @@ sudo iftop -i wlo1
 ```
 Or use a Wireshark filter
 ```
-mavlink_proto.msgid == 117 || mavlink_proto.msgid == 118 || mavlink_proto.msgid == 119 || mavlink_proto.msgid == 120
+mavlink_proto.msgid == 110
 ```
 
 Watch your beautiful logs arrive

config.toml

@@ -4,3 +4,12 @@ remote_server = "https://review.px4.io"
 email = ""
 upload_enabled = false
 public_logs = false
+
+# Vehicle log directory. Empty probes "@MAV_LOG" (the virtual log directory from
+# the MAVLink FTP specification), then "/fs/microsd/log" (PX4) and "/APM/LOGS"
+# (ArduPilot). Set this when the logs live somewhere else.
+remote_log_directory = ""
+
+# FTP burst reads. Much faster, disable it if the flight stack's burst support
+# misbehaves.
+ftp_use_burst = true

src/FtpListClient.cpp

@@ -0,0 +1,251 @@
+#include "FtpListClient.hpp"
+#include "Log.hpp"
+
+#include <chrono>
+#include <cstring>
+
+namespace
+{
+
+constexpr int kMaxAttempts = 3;
+
+// Generous enough for a listing that has to come back over a telemetry link.
+// Servers answer a retransmitted request from their reply cache, so a retry
+// after a dropped packet is cheap.
+constexpr auto kReplyTimeout = std::chrono::milliseconds(2000);
+
+} // namespace
+
+FtpListClient::FtpListClient(std::shared_ptr<mavsdk::System> system)
+	: _passthrough(std::make_shared<mavsdk::MavlinkPassthrough>(system))
+{
+	_subscription = _passthrough->subscribe_message(MAVLINK_MSG_ID_FILE_TRANSFER_PROTOCOL,
+	[this](const mavlink_message_t& message) {
+		handle_message(message);
+	});
+}
+
+FtpListClient::~FtpListClient()
+{
+	_passthrough->unsubscribe_message(MAVLINK_MSG_ID_FILE_TRANSFER_PROTOCOL, _subscription);
+}
+
+void FtpListClient::stop()
+{
+	_should_exit = true;
+	_cv.notify_all();
+}
+
+void FtpListClient::reset_sessions()
+{
+	PayloadHeader reply;
+
+	if (transact(kOpcodeResetSessions, "", 0, reply)) {
+		LOG_DEBUG("FTP sessions reset");
+
+	} else {
+		LOG_DEBUG("FTP session reset went unanswered");
+	}
+}
+
+void FtpListClient::handle_message(const mavlink_message_t& message)
+{
+	mavlink_file_transfer_protocol_t ftp;
+	mavlink_msg_file_transfer_protocol_decode(&message, &ftp);
+
+	// The transfer runs unicast: the server addresses its replies to whoever
+	// asked. Anything not for us is another client's traffic.
+	if (ftp.target_system != _passthrough->get_our_sysid()) {
+		return;
+	}
+
+	if (ftp.target_component != _passthrough->get_our_compid() && ftp.target_component != 0) {
+		return;
+	}
+
+	PayloadHeader payload;
+	std::memcpy(&payload, ftp.payload, sizeof(payload));
+
+	if (payload.opcode != kOpcodeAck && payload.opcode != kOpcodeNak) {
+		return;
+	}
+
+	std::lock_guard<std::mutex> lock(_mutex);
+
+	// Match against the in-flight request so replies to MAVSDK's Ftp plugin
+	// (which shares our system and component id) are ignored.
+	if (!_expected_req_opcode.has_value() || payload.req_opcode != _expected_req_opcode.value()
+	    || payload.seq_number != _expected_seq) {
+		return;
+	}
+
+	_reply = payload;
+	_cv.notify_all();
+}
+
+void FtpListClient::send_request(const PayloadHeader& request)
+{
+	_passthrough->queue_message([this, request](MavlinkAddress address, uint8_t channel) {
+		mavlink_message_t message;
+		mavlink_msg_file_transfer_protocol_pack_chan(address.system_id, address.component_id, channel, &message,
+				0, // target_network
+				_passthrough->get_target_sysid(),
+				_passthrough->get_target_compid(),
+				reinterpret_cast<const uint8_t*>(&request));
+		return message;
+	});
+}
+
+bool FtpListClient::transact(uint8_t opcode, const std::string& path, uint32_t offset, PayloadHeader& reply)
+{
+	if (path.size() >= kMaxDataLength) {
+		return false;
+	}
+
+	PayloadHeader request {};
+	request.seq_number = ++_seq;
+	request.opcode = opcode;
+	request.offset = offset;
+	request.size = static_cast<uint8_t>(path.size() + 1);
+	std::memcpy(request.data, path.c_str(), path.size() + 1);
+
+	for (int attempt = 0; attempt < kMaxAttempts && !_should_exit; attempt++) {
+		{
+			std::lock_guard<std::mutex> lock(_mutex);
+			_reply.reset();
+			// Servers reply with the request sequence number plus one, and
+			// answer a retransmission (same sequence number) from their
+			// duplicate-reply cache, so retries reuse the packet as-is.
+			_expected_seq = static_cast<uint16_t>(request.seq_number + 1);
+			_expected_req_opcode = opcode;
+		}
+
+		send_request(request);
+
+		std::unique_lock<std::mutex> lock(_mutex);
+
+		if (_cv.wait_for(lock, kReplyTimeout, [this] { return _reply.has_value() || _should_exit.load(); })) {
+			_expected_req_opcode.reset();
+
+			if (_should_exit || !_reply.has_value()) {
+				return false;
+			}
+
+			reply = _reply.value();
+			return true;
+		}
+	}
+
+	std::lock_guard<std::mutex> lock(_mutex);
+	_expected_req_opcode.reset();
+
+	return false;
+}
+
+void FtpListClient::parse_entries(const PayloadHeader& reply, std::vector<Entry>& entries, uint32_t& next_offset)
+{
+	const size_t data_size = std::min(static_cast<size_t>(reply.size), kMaxDataLength);
+	size_t i = 0;
+
+	while (i < data_size) {
+		const char* raw = reinterpret_cast<const char*>(&reply.data[i]);
+		const size_t length = strnlen(raw, data_size - i);
+		i += length + 1;
+
+		// Every directory entry advances the server-side offset, including the
+		// "skip" placeholders it emits for entries it does not report.
+		next_offset++;
+
+		if (length < 2) {
+			continue;
+		}
+
+		const std::string body(raw + 1, length - 1);
+
+		if (raw[0] == 'D') {
+			Entry entry;
+			entry.name = body;
+			entry.is_directory = true;
+			entries.push_back(entry);
+
+		} else if (raw[0] == 'F') {
+			// "F<name>\t<size>" plus "\t<mtime>" from ListDirectoryWithTime
+			const size_t first_tab = body.find('\t');
+
+			if (first_tab == std::string::npos) {
+				continue;
+			}
+
+			Entry entry;
+			entry.name = body.substr(0, first_tab);
+
+			try {
+				const size_t second_tab = body.find('\t', first_tab + 1);
+				entry.size_bytes = static_cast<uint32_t>(std::stoul(body.substr(first_tab + 1)));
+
+				if (second_tab != std::string::npos) {
+					// The server reports zero when it does not know the time
+					const int64_t mtime = std::stoll(body.substr(second_tab + 1));
+
+					if (mtime > 0) {
+						entry.mtime_utc = mtime;
+					}
+				}
+
+			} catch (const std::exception&) {
+				continue;
+			}
+
+			entries.push_back(entry);
+		}
+	}
+}
+
+FtpListClient::Result FtpListClient::list_directory(const std::string& path, std::vector<Entry>& entries)
+{
+	entries.clear();
+
+	uint32_t offset = 0;
+
+	while (!_should_exit) {
+		const uint8_t opcode = _with_time_supported ? kOpcodeListDirectoryWithTime : kOpcodeListDirectory;
+
+		PayloadHeader reply;
+
+		if (!transact(opcode, path, offset, reply)) {
+			return _should_exit ? Result::Stopped : Result::Timeout;
+		}
+
+		if (reply.opcode == kOpcodeNak) {
+			const uint8_t error = reply.size > 0 ? reply.data[0] : 0;
+
+			if (error == kErrUnknownCommand && opcode == kOpcodeListDirectoryWithTime) {
+				LOG_DEBUG("ListDirectoryWithTime unsupported, falling back to ListDirectory");
+				_with_time_supported = false;
+				continue; // Same offset, plain opcode
+			}
+
+			if (error == kErrEOF) {
+				return Result::Success;
+			}
+
+			if (offset == 0 && (error == kErrFileNotFound || error == kErrFailErrno)) {
+				return Result::FileNotFound;
+			}
+
+			LOG_DEBUG("FTP list of " << path << " failed with error " << static_cast<int>(error)
+				  << " at offset " << offset);
+			return Result::ProtocolError;
+		}
+
+		const uint32_t previous_offset = offset;
+		parse_entries(reply, entries, offset);
+
+		// An ACK that advances nothing would loop forever; treat it as the end
+		if (offset == previous_offset) {
+			return Result::Success;
+		}
+	}
+
+	return Result::Stopped;
+}