diff --git a/CMakeLists.txt b/CMakeLists.txt index afa5ecf..ef8bec3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ set(LWIP_CONTRIB_FREERTOS_DIR "${LWIP_DIR}/contrib/ports/freertos") # link target — only the header-configured packs below do. # https://docs.cososo.co.uk/solid-syslog/getting-started/#path-a--cmake-consumer # Pinned to a commit until there is a release tag to pin to. -set(SOLIDSYSLOG_PLATFORMS "LwipRaw;StdAtomic;FreeRtos" CACHE STRING "" FORCE) +set(SOLIDSYSLOG_PLATFORMS "LwipRaw;StdAtomic;FreeRtos;FatFs" CACHE STRING "" FORCE) include(FetchContent) FetchContent_Declare(SolidSyslog @@ -167,7 +167,7 @@ target_include_directories(baseline PRIVATE # library, or context struct sizes diverge between consumer and library. target_compile_definitions(baseline PRIVATE MBEDTLS_USER_CONFIG_FILE=${MBEDTLS_USER_CONFIG_HEADER}) -target_link_libraries(baseline PRIVATE mbedtls mbedx509 mbedcrypto SolidSyslog SolidSyslog::LwipRaw SolidSyslog::FreeRtos) +target_link_libraries(baseline PRIVATE mbedtls mbedx509 mbedcrypto SolidSyslog SolidSyslog::LwipRaw SolidSyslog::FreeRtos SolidSyslog::FatFs) target_link_options(baseline PRIVATE -mcpu=cortex-m3 -mthumb diff --git a/README.md b/README.md index ed0ff8a..50c0450 100644 --- a/README.md +++ b/README.md @@ -10,42 +10,51 @@ It builds on a baseline that simulates the sort of device you might be adding th measures itself: see [docs/baseline.md](docs/baseline.md) for what the baseline is, how the figures are made, and how to run it. -## This stage — Time quality +## This stage — File store -Add `SolidSyslogTimeQualitySd`, and give `MetaSd` an uptime source alongside its counter. +Spool to a `SolidSyslogBlockStore` over a `SolidSyslogFileBlockDevice` over the library's FatFs +port, replacing the Null store. The service task drains the ring into storage and sends from there, +so a failed send costs a retry rather than the record: the audit trail survives an outage instead of +ending at it. ```c -struct SolidSyslogMetaSdConfig metaConfig = { - .Counter = SolidSyslogStdAtomicCounter_Create(), - .GetSysUpTime = SolidSyslogFreeRtos_GetSysUpTime, /* new */ +#define SYSLOG_STORE_PREFIX "syslog" +#define SYSLOG_STORE_BLOCKS 4U + +struct SolidSyslogBlockStoreConfig storeConfig = { + .BlockDevice = SolidSyslogFileBlockDevice_Create(SolidSyslogFatFsFile_Create(), SYSLOG_STORE_PREFIX, 0U), + .MaxBlocks = SYSLOG_STORE_BLOCKS, + .DiscardPolicy = SOLIDSYSLOG_DISCARD_POLICY_OLDEST, + .SecurityPolicy = SolidSyslogCrc16Policy_Create(), }; -sd[1] = SolidSyslogTimeQualitySd_Create(SyslogTimeQuality); ``` -```text -... BOOT [meta sequenceId="1" sysUpTime="385"][timeQuality tzKnown="1" isSynced="0"] device started -``` +Three decisions come with it: how much to store, which is capacity on the medium rather than RAM; +what happens when it fills — discard oldest, discard newest, or halt; and whether to be warned +before that point, via the capacity-threshold callback. -Time quality states how far the clock can be trusted, which matters when comparing events from -different devices. +This device stores four blocks, one file per block, `syslog00.log` upward on the volume it already +mounts, and discards the oldest when full. -This device reads the host clock once at boot and then free-runs on the FreeRTOS tick, so `isSynced` -is `0` and the callback writes no `syncAccuracy`. `tzKnown` is `1`; the device works in UTC -throughout. +The CRC-16 detects corruption, not tampering. It catches a truncated write or bit-rot; anyone who +can edit a stored record can recompute it. It establishes that a record came back the way it went +in, which is the prerequisite for spooling at all. Making stored records tamper-evident, and then +unreadable, are later stages. -`sysUpTime` accompanies the sequence number. After a reboot the sequence restarts at one, and an -uptime near zero distinguishes that from a counter wrap. +Storing happens on the service task, so a task that calls `SolidSyslog_Log` still knows nothing +about what happens after it returns and its stack does not move. The RAM is pool allocation and +handles rather than buffers — nothing holds a block in memory, so the store costs its handles +rather than its capacity. -The element lands before the store because store-and-forward breaks the assumption that a record -reaches the collector shortly after it was raised. A record can arrive hours later, so the device -states what its clock is worth first. +`SolidSyslog::FatFs` is a header-configured upstream, so it is both named in +`SOLIDSYSLOG_PLATFORMS` and linked. -**When you need it.** If events from this device will be ordered against events from others, or if a -record's timestamp will be relied on after a delay. +**When you need it.** If losing the records raised during an outage is not acceptable, or if they +must survive a reboot. -**Cost above baseline: Flash +7,628 B, RAM +7,692 B.** +**Cost above baseline: Flash +11,584 B, RAM +9,128 B.** @@ -68,6 +77,7 @@ committed as [`run-report.md`](run-report.md), and rewritten by every stage. | Buffered | logging that returns immediately, with the send moved off the logging task | +6,804 | +7,488 | | TCP | records the network retransmits instead of dropping, and a send that fails when the collector is gone | +7,336 | +7,668 | | Time quality | a timestamp the collector knows how far to trust, and an uptime that tells a reboot from a counter wrap | +7,628 | +7,692 | +| File store | records that survive a failed send, spooled to disk with a checksum at rest | +11,584 | +9,128 | *Deltas are bytes above the baseline, which is itself Flash 350,308 B, RAM 111,192 B.* diff --git a/app/syslog/Syslog.c b/app/syslog/Syslog.c index f76416b..620c814 100644 --- a/app/syslog/Syslog.c +++ b/app/syslog/Syslog.c @@ -9,10 +9,14 @@ #include "Syslog.h" +#include "SolidSyslogBlockStore.h" #include "SolidSyslogCircularBuffer.h" #include "SolidSyslogConfig.h" +#include "SolidSyslogCrc16Policy.h" #include "SolidSyslogEndpoint.h" #include "SolidSyslogEndpointHost.h" +#include "SolidSyslogFatFsFile.h" +#include "SolidSyslogFileBlockDevice.h" #include "SolidSyslogFreeRtosMutex.h" #include "SolidSyslogFreeRtosSysUpTime.h" #include "SolidSyslogLwipRawAddress.h" @@ -20,7 +24,6 @@ #include "SolidSyslogLwipRawResolver.h" #include "SolidSyslogLwipRawTcpStream.h" #include "SolidSyslogMetaSd.h" -#include "SolidSyslogNullStore.h" #include "SolidSyslogStdAtomicCounter.h" #include "SolidSyslogStreamSender.h" #include "SolidSyslogTimeQuality.h" @@ -46,6 +49,10 @@ * backlog the store is there to hold. */ #define SYSLOG_BUFFER_RECORDS 8U +/* One "NN.log" per block, on the volume the device already mounts. */ +#define SYSLOG_STORE_PREFIX "syslog" +#define SYSLOG_STORE_BLOCKS 4U + static struct SolidSyslog* s_logger = NULL; static uint8_t s_ring[SOLIDSYSLOG_CIRCULAR_BUFFER_RING_BYTES(SYSLOG_BUFFER_RECORDS)]; @@ -115,12 +122,17 @@ void Syslog_Start(void) s_sd[0] = SolidSyslogMetaSd_Create(&metaConfig); s_sd[1] = SolidSyslogTimeQualitySd_Create(SyslogTimeQuality); + struct SolidSyslogBlockStoreConfig storeConfig = { + .BlockDevice = SolidSyslogFileBlockDevice_Create(SolidSyslogFatFsFile_Create(), SYSLOG_STORE_PREFIX, 0U), + .MaxBlocks = SYSLOG_STORE_BLOCKS, + .DiscardPolicy = SOLIDSYSLOG_DISCARD_POLICY_OLDEST, + .SecurityPolicy = SolidSyslogCrc16Policy_Create(), + }; + struct SolidSyslogConfig config = { .Buffer = SolidSyslogCircularBuffer_Create(SolidSyslogFreeRtosMutex_Create(), s_ring, sizeof(s_ring)), .Sender = sender, - /* No store-and-forward here. The Null object rather than NULL is how - * that is said out loud — NULL is reported as a fault. */ - .Store = SolidSyslogNullStore_Get(), + .Store = SolidSyslogBlockStore_Create(&storeConfig), /* PROCID stays unset — a bare-metal image has no process. */ .Clock = SyslogFields_Clock, .GetHostname = SyslogFields_Hostname, diff --git a/measurements/file-store.csv b/measurements/file-store.csv new file mode 100644 index 0000000..55ec954 --- /dev/null +++ b/measurements/file-store.csv @@ -0,0 +1,13 @@ +# file-store figures (bytes) — captured by scripts/run.sh (CAPTURE=1). +# The device reads measurements/Baseline.csv as its frozen baseline and reports current-minus-Baseline. +flash_text,361260 +flash_data,632 +static_bss,119688 +heap_used,4440 +mbedtls_peak,21332 +mbedtls_free,11436 +lwip_mem_free,7576 +lwip_pbufs_free,14 +stack_log,792 +stack_service,1012 +stack_harness,2848 diff --git a/measurements/stages.tsv b/measurements/stages.tsv index 657708e..4e11c66 100644 --- a/measurements/stages.tsv +++ b/measurements/stages.tsv @@ -19,3 +19,4 @@ sequence-id Sequence numbers every record numbered, so a gap in the sequence is buffered Buffered logging that returns immediately, with the send moved off the logging task tcp TCP records the network retransmits instead of dropping, and a send that fails when the collector is gone time-quality Time quality a timestamp the collector knows how far to trust, and an uptime that tells a reboot from a counter wrap +file-store File store records that survive a failed send, spooled to disk with a checksum at rest diff --git a/run-report.md b/run-report.md index 3b6c660..94fb69a 100644 --- a/run-report.md +++ b/run-report.md @@ -1,4 +1,4 @@ -# solid-syslog-example — run (time-quality) +# solid-syslog-example — run (file-store) ## Device (self-measured) @@ -10,16 +10,16 @@ [device] first record logged: yes [report] --- SolidSyslog cost above baseline (simulated existing application) --- [report] key,current,baseline,used_above_baseline -[report] flash_text,357440,349992,7448 -[report] flash_data,496,316,180 -[report] static_bss,118388,110876,7512 +[report] flash_text,361260,349992,11268 +[report] flash_data,632,316,316 +[report] static_bss,119688,110876,8812 [report] heap_used,4440,4440,0 -[report] mbedtls_peak,21316,21332,-16 -[report] mbedtls_free,11452,11436,16 +[report] mbedtls_peak,21332,21332,0 +[report] mbedtls_free,11436,11436,0 [report] lwip_mem_free,7576,7576,0 [report] lwip_pbufs_free,14,14,0 [report] stack_log,792,120,672 -[report] stack_service,948,52,896 +[report] stack_service,1012,52,960 [report] stack_harness,2848,2840,8 [report] --- end --- [device] ready @@ -29,7 +29,7 @@ ```text text data bss dec hex filename - 357432 504 118388 476324 744a4 /w/build/baseline-cross/baseline.elf + 361252 640 119688 481580 7592c /w/build/baseline-cross/baseline.elf ``` ## Listeners (proved before the device ran) @@ -47,23 +47,23 @@ ## Collector (syslog-ng) received ```text -wire <134>1 2026-08-15T22:00:10.850000Z 10.0.2.15 solid-syslog-example - BOOT [meta sequenceId="1" sysUpTime="385"][timeQuality tzKnown="1" isSynced="0"] device started -parsed PRIORITY=134 TIMESTAMP=2026-08-15T22:00:10+00:00 HOSTNAME=10.0.2.15 APP_NAME=solid-syslog-example PROCID= MSGID=BOOT STRUCTURED_DATA=[meta sequenceId="1" sysUpTime="385"][timeQuality tzKnown="1" isSynced="0"] MSG=device started +wire <134>1 2026-08-16T06:24:46.850000Z 10.0.2.15 solid-syslog-example - BOOT [meta sequenceId="1" sysUpTime="385"][timeQuality tzKnown="1" isSynced="0"] device started +parsed PRIORITY=134 TIMESTAMP=2026-08-16T06:24:46+00:00 HOSTNAME=10.0.2.15 APP_NAME=solid-syslog-example PROCID= MSGID=BOOT STRUCTURED_DATA=[meta sequenceId="1" sysUpTime="385"][timeQuality tzKnown="1" isSynced="0"] MSG=device started ``` -## Self-check (vs measurements/time-quality.csv) +## Self-check (vs measurements/file-store.csv) ```text - OK flash_text: 357440 (expected 357440, Δ0) - OK flash_data: 496 (expected 496, Δ0) - OK static_bss: 118388 (expected 118388, Δ0) + OK flash_text: 361260 (expected 361260, Δ0) + OK flash_data: 632 (expected 632, Δ0) + OK static_bss: 119688 (expected 119688, Δ0) OK heap_used: 4440 (expected 4440, Δ0) - OK mbedtls_peak: 21316 (expected 21288, Δ28) - OK mbedtls_free: 11452 (expected 11480, Δ28) + OK mbedtls_peak: 21332 (expected 21332, Δ0) + OK mbedtls_free: 11436 (expected 11436, Δ0) OK lwip_mem_free: 7576 (expected 7576, Δ0) OK lwip_pbufs_free: 14 (expected 14, Δ0) OK stack_log: 792 (expected 792, Δ0) - OK stack_service: 948 (expected 948, Δ0) + OK stack_service: 1012 (expected 1012, Δ0) OK stack_harness: 2848 (expected 2848, Δ0) ```