diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1befa2b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,40 @@ +name: build + +on: + push: + branches: [main] + pull_request: + +jobs: + build: + runs-on: ubuntu-24.04 + env: + MAVSDK_VERSION: 3.17.1 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libsqlite3-dev libssl-dev + curl -sL -o mavsdk.deb \ + "https://github.com/mavlink/MAVSDK/releases/download/v${MAVSDK_VERSION}/libmavsdk-dev_${MAVSDK_VERSION}_ubuntu24.04_amd64.deb" + sudo dpkg -i mavsdk.deb + + - name: Configure + run: cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON + + - name: Build + run: cmake --build build -j"$(nproc)" + + - name: Test + run: ctest --test-dir build --output-on-failure + + format: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get update && sudo apt-get install -y astyle + - run: make check-format diff --git a/.gitmodules b/.gitmodules index 5b8b83f..13b4e2e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "third_party/tomlplusplus"] path = third_party/tomlplusplus url = https://github.com/marzer/tomlplusplus.git +[submodule "third_party/json"] + path = third_party/json + url = https://github.com/nlohmann/json.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 6529639..7c36215 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,41 +3,73 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -project(logloader VERSION 0.9 LANGUAGES CXX) +project(logloader VERSION 1.0 LANGUAGES CXX) -option(DEBUG_BUILD "Enable debug logging" OFF) -if(DEBUG_BUILD) - add_definitions(-DDEBUG_BUILD) - message(STATUS "Debug logging enabled") +# Without this the default is an empty build type, i.e. no optimisation at all. +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) endif() +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") add_compile_options(-Wall -Wextra -Werror -Wpedantic -Wunused) find_package(OpenSSL 3.0.2 REQUIRED) find_package(SQLite3 REQUIRED) +find_package(Threads REQUIRED) message(STATUS "OpenSSL version: ${OPENSSL_VERSION}") -message(STATUS "OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}") -message(STATUS "OpenSSL libraries: ${OPENSSL_LIBRARIES}") -message(STATUS "SQLite3 include dir: ${SQLite3_INCLUDE_DIRS}") -message(STATUS "SQLite3 libraries: ${SQLite3_LIBRARIES}") +message(STATUS "SQLite3 version: ${SQLite3_VERSION}") -# Assumes MAVSDK system wide install +# Assumes a system-wide MAVSDK install list(APPEND CMAKE_PREFIX_PATH "/usr/local/MAVSDK/install") find_package(MAVSDK REQUIRED) -include_directories(third_party/cpp-httplib/) -include_directories(third_party/tomlplusplus/) -include_directories(${SQLite3_INCLUDE_DIRS}) - add_executable(${PROJECT_NAME} src/main.cpp - src/ServerInterface.cpp - src/LogLoader.cpp) + src/ApiServer.cpp + src/Config.cpp + src/FtpListClient.cpp + src/FtpLogFetcher.cpp + src/LegacyImport.cpp + src/Log.cpp + src/LogDatabase.cpp + src/LogLoader.cpp + src/Sqlite.cpp + src/StatusBoard.cpp + src/UploadTarget.cpp) + +# Must be set for every translation unit that sees httplib.h: it changes the +# layout of httplib's socket types, so a per-file #define is an ODR violation. +target_compile_definitions(${PROJECT_NAME} PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT) + +# The vendored headers are SYSTEM so their warnings are not ours to fix. +target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE + third_party/cpp-httplib + third_party/tomlplusplus + third_party/json/single_include + ${SQLite3_INCLUDE_DIRS}) target_link_libraries(${PROJECT_NAME} - pthread + Threads::Threads OpenSSL::SSL OpenSSL::Crypto MAVSDK::mavsdk ${SQLite3_LIBRARIES}) + +# The database is the one module worth testing without a vehicle: it holds every +# decision logloader makes, and the legacy import runs once against real user +# data and cannot be un-run. +option(BUILD_TESTING "Build the tests" OFF) +if(BUILD_TESTING) + enable_testing() + add_executable(test_log_database + tests/test_log_database.cpp + src/LegacyImport.cpp + src/Log.cpp + src/LogDatabase.cpp + src/Sqlite.cpp) + target_include_directories(test_log_database PRIVATE src) + target_include_directories(test_log_database SYSTEM PRIVATE ${SQLite3_INCLUDE_DIRS}) + target_link_libraries(test_log_database Threads::Threads ${SQLite3_LIBRARIES}) + add_test(NAME log_database COMMAND test_log_database) +endif() diff --git a/Makefile b/Makefile index c3ea421..a4afc37 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,22 @@ PROJECT_NAME="logloader" all: - @astyle --quiet --options=astylerc src/*.cpp,*.hpp - @cmake -Bbuild -H. -DDEBUG_BUILD=OFF; cmake --build build -j$(nproc) + @cmake -Bbuild -S. -DCMAKE_BUILD_TYPE=Release; cmake --build build -j$(nproc) @size build/${PROJECT_NAME} debug: - @astyle --quiet --options=astylerc src/*.cpp,*.hpp - @cmake -Bbuild -H. -DDEBUG_BUILD=ON; cmake --build build -j$(nproc) + @cmake -Bbuild -S. -DCMAKE_BUILD_TYPE=Debug; cmake --build build -j$(nproc) @size build/${PROJECT_NAME} - @echo "Debug build with logging enabled" + +# Separate from the build on purpose: a `make` that rewrites your sources cannot +# run from a read-only checkout and makes the build non-reproducible. +format: + @astyle --quiet --options=astylerc src/*.cpp,*.hpp + @astyle --quiet --options=astylerc "tests/*.cpp,*.hpp" + +check-format: + @{ astyle --dry-run --formatted --options=astylerc src/*.cpp,*.hpp; astyle --dry-run --formatted --options=astylerc "tests/*.cpp,*.hpp"; } | grep Formatted \ + && { echo "run 'make format'"; exit 1; } || echo "formatting OK" install: @bash install.sh @@ -18,4 +25,4 @@ clean: @rm -rf build @echo "All build artifacts removed" -.PHONY: all debug install clean +.PHONY: all debug format check-format install clean diff --git a/README.md b/README.md index 8993abb..6857b68 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,142 @@ ![image](logloader_logo.png) -Downloads PX4 log files (.ulg) and uploads them to a local server and optionally a remote server. +Downloads flight logs from the vehicle over MAVLink FTP and uploads them to a local Flight Review and, optionally, a remote one. Works with PX4 (`.ulg`) and ArduPilot (`.BIN`). -The **config.toml** file is used to configure the program settings. +### What it fetches, and when -### Behavior -Downloading and uploading will only occur while the vehicle is not armed. Downloading and uploading operations are performed in separate threads. An sqlite database per server is used to track log file download/upload status. +logloader does **not** try to mirror the SD card. A companion computer that is powered on for the first time, or one whose database has been reset, would otherwise pull and upload the vehicle's entire history over a link that has better things to do. + +The rule is: + +- **A log that appears while logloader is watching** is downloaded and uploaded. In practice that means the log a flight just produced. +- **On a database that has never seen this vehicle**, only the most recent log is taken. Everything else is left alone. +- **Anything else happens because it was asked for**, from the ARK-OS Logs page or the HTTP API below. + +Two further guards fall out of the same idea. A log is only considered at all once two consecutive listings agree on its size, so the log being written right now is never fetched at a size it will not keep. And if a single listing turns up more than `download.max_auto_queue` new logs, that is a card logloader has not seen rather than a flight that just happened, so it queues only the newest and says so. + +Downloading and uploading are suspended while the vehicle is armed. + +### HTTP API + +A small localhost API on port 3005 (`[api]` in the config) is what the ARK-OS Logs page talks to. + +| | | +| --- | --- | +| `GET /status` | connection, arm state, current transfer | +| `GET /logs` | every known log with its download and upload state | +| `GET /events` | the same as server-sent events, one per change | +| `POST /logs/download` | `{"ids": [1,2]}` or `{"all": true}`; `"upload": false` to fetch without uploading | +| `POST /logs/upload` | `{"ids": [1,2]}` or `{"all": true}`, optional `"targets": ["local"]` | +| `POST /logs/cancel` | `{"ids": [1,2]}` — clears pending requests | +| `DELETE /logs/{id}/file` | removes the downloaded copy to free space | + +A successful upload records the path Flight Review redirected to (`/plot_app?log=`), which is what lets the UI link straight to the plot. + +### Log transport + +Everything runs over **MAVLink FTP** (`FILE_TRANSFER_PROTOCOL`, msg 110): the directory listing that finds the logs and the bulk transfer that downloads them. The classic log protocol (`LOG_REQUEST_LIST` / `LOG_ENTRY` / `LOG_DATA`, msgs 117-120) is not used at all. + +`FILE_TRANSFER_PROTOCOL` carries `target_system` and `target_component`, and both PX4 and ArduPilot address their replies back to the requesting sysid/compid, so a download is unicast between the vehicle and logloader. `LOG_DATA` has no target fields, which leaves a router in between — mavlink-router, for instance — no choice but to copy every chunk to every endpoint it serves, telemetry radio included. A log download over the old protocol saturates links that have no interest in it. + +The autopilot's MAVLink instance must have FTP enabled (`mavlink start -x` on PX4). If it does not, logloader says so and idles. + +### Log discovery + +logloader lists the vehicle's log directory, trying `@MAV_LOG` first — the virtual log directory the MAVLink FTP specification defines, supported by PX4 v1.17 and newer — then the physical `/fs/microsd/log` (PX4), `/APM/LOGS` (ArduPilot) and `/log` (PX4 SITL). Set `download.remote_directory` to skip the probe. + +The listing is what identifies a log: its path below the log root plus its size. PX4's nested `/