diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8e38b63..755d052 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2182,7 +2182,7 @@ The family is derived in the decoder with zero runtime overhead: | 0x30–0x3F | STATE | LIFECYCLE(0x30), FSM_TRANS(0x31) | | 0x40–0x4F | RESOURCE | LIB_INIT(0x40), LIB_DEINIT(0x41) — Phase 2 activates XOBS-3 (queue depth, stack high-water) | | 0x50–0x5F | TIMING | Phase 2 activates XOBS-4 (watchdog checkin, message latency) | -| 0x60–0x6F | FAULT | FAULT(0x60), SECURITY_INCIDENT(0x61), AUTH_EVENT(0x62), FAULT_CLEARED(0x63), LIB_ALLOC_FAIL(0x64), LIB_TIMEOUT(0x65), OSAL_FAULT(0x66 — Phase 2), HAL_FAULT(0x67 — Phase 2) | +| 0x60–0x6F | FAULT | FAULT(0x60), SECURITY_INCIDENT(0x61), AUTH_EVENT(0x62), FAULT_CLEARED(0x63), LIB_ALLOC_FAIL(0x64), LIB_TIMEOUT(0x65), OSAL_FAULT(0x66 — Phase 2), HAL_FAULT(0x67 — Phase 2), NVM_PERSIST_FAIL(0x6B) | | 0x70–0x7F | FUNCTION | TEST_START(0x70), TEST_END(0x71) | | 0x80–0x8F | AI (Phase 3) | RESERVED — Phase 3 AI family. Do not allocate. Taxonomy defined after Phase 1 AI production data. | | 0x90–0xFF | VENDOR | Community and vendor extension range — 7 full 16-slot sub-bands. Never allocated by the framework. | diff --git a/core/include/embediq_cfg.h b/core/include/embediq_cfg.h index 7fb2e52..ecef4ac 100644 --- a/core/include/embediq_cfg.h +++ b/core/include/embediq_cfg.h @@ -23,9 +23,13 @@ * no global mutable state of its own — the getter / setter calls pass * straight through to embediq_nvm_get / _set / _delete. * - * Durability: fb_nvm persists at its own cadence. Callers that must - * guarantee a value is flushed to storage before continuing should call - * embediq_nvm_flush() explicitly (for example, before MSG_SYS_OTA_READY). + * Durability: every mutation persists synchronously and returns whether the + * change reached flash; embediq_nvm_flush() rewrites the current table (use + * it after a failed mutation to retry). If flash cannot be read at boot the + * store starts empty and every setter and delete returns false, leaving the + * stored values unchanged, until embediq_nvm_flush() re-reads it + * successfully; an invalid blob (bad magic or CRC) is replaced by the first + * successful write. * * @author Ritesh Anand * @company embediq.com | ritzylab.com @@ -71,7 +75,10 @@ size_t embediq_cfg_get_str (const char *key, char *buf, /* --------------------------------------------------------------------------- * Typed setters — persist the value under `key`. Return true on success, - * false on NVM error, over-long key, or store full. + * false on NVM error, over-long key, or store full. false also when the + * value could not be persisted; in that case the new value IS in RAM + * (getters return it) and reaches flash with the next successful persist — + * unlike every other false, which leaves the old value. * * Calling context: any thread. A setter writes to flash and may wait for * another writer's persist in flight (see embediq_nvm.h). @@ -84,8 +91,14 @@ bool embediq_cfg_set_bool (const char *key, bool value); bool embediq_cfg_set_str (const char *key, const char *value); /** - * Remove the entry for the given key. Returns true on success or if the - * key was already absent (fb_nvm makes delete idempotent). + * Remove the entry for the given key. Returns true on success. + * Idempotent: deleting an absent key returns true only once the current + * table — which includes the absence of that key — is on flash; false if the + * table could not be written, including when the pending change is another + * key's. + * false also when the removal could not be persisted; in that case the key + * IS gone from RAM and the removal reaches flash with the next successful + * persist. * * Calling context: any thread. May wait for a persist in flight. */ diff --git a/core/include/embediq_nvm.h b/core/include/embediq_nvm.h index 0a26c5d..86854d8 100644 --- a/core/include/embediq_nvm.h +++ b/core/include/embediq_nvm.h @@ -6,7 +6,9 @@ * The ESP32 implementation uses NVS (Non-Volatile Storage). * * All operations are synchronous from the caller's perspective. - * fb_nvm owns internal write buffering and flush scheduling. + * Every mutation persists synchronously and returns whether the change + * reached flash; embediq_nvm_flush() rewrites the current table (use it + * after a failed mutation to retry). * * Concurrency — every function is callable from any thread. fb_nvm uses two * locks with one fixed order (persist lock, then table lock): @@ -19,13 +21,24 @@ * (a later writer persisted a newer table) skips its write, so an older * table never overwrites a newer one. * embediq_nvm_get() takes only the table lock: it never waits for a persist - * in flight. Writes return once their change (or a newer table) is on flash - * and may wait for another writer's persist. + * in flight. Writes return EMBEDIQ_OK once their change (or a newer table) + * is on flash, EMBEDIQ_ERR if the flash write failed, and may wait for + * another writer's persist. + * Loading: fb_nvm's nvm_load() runs only at boot, before any other caller + * exists (and from the host test hook with no concurrent callers). It + * replaces the table with the flash contents; a change made concurrently by + * another thread would be discarded and its persist skipped. Reloading at + * runtime is not supported. + * If flash cannot be read at boot the store starts empty and refuses to + * persist (every mutation returns EMBEDIQ_ERR with an NVM_PERSIST_FAIL event, + * state FLASH_UNREADABLE, and leaves the table unchanged) until + * embediq_nvm_flush() re-reads it successfully. An invalid blob (bad magic or + * CRC) is replaced by the first successful write. * * Constraints (v1): * - Keys are NUL-terminated strings up to EMBEDIQ_NVM_KEY_MAX bytes. * - Values are raw byte arrays up to EMBEDIQ_NVM_VAL_MAX bytes. - * - No transactions. embediq_nvm_flush() is the durability boundary. + * - No transactions. * - embediq_nvm_reset() erases ALL keys — use with care. * * I-01: Compiles standalone with zero OSAL or BSP dependencies. @@ -72,6 +85,9 @@ extern "C" { * @param val Pointer to value bytes. * @param len Number of bytes in val; must be <= EMBEDIQ_NVM_VAL_MAX. * @return EMBEDIQ_OK on success, EMBEDIQ_ERR on invalid args or store error. + * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps + * the change and the next successful persist (any later write, or flush) + * carries it. * * Calling context: any thread. May wait for a persist in flight. */ @@ -92,7 +108,13 @@ embediq_err_t embediq_nvm_get(const char *key, void *val, uint32_t *len); /** * Remove the entry for the given key. - * If the key does not exist, returns EMBEDIQ_OK (idempotent). + * Idempotent: deleting an absent key returns EMBEDIQ_OK only once the + * current table — which includes the absence of that key — is on flash; + * EMBEDIQ_ERR if the table could not be written, including when the pending + * change is another key's. + * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps + * the change and the next successful persist (any later write, or flush) + * carries it. * * Calling context: any thread. May wait for a persist in flight. */ @@ -101,7 +123,14 @@ embediq_err_t embediq_nvm_delete(const char *key); /** * embediq_nvm_provision — write-once provisioning write. * Semantics: if key is absent or empty, write value and return EMBEDIQ_OK. - * if key is already set (non-empty), return EMBEDIQ_OK (idempotent). + * Idempotent: a second call with the key already provisioned does + * not change the value; it returns EMBEDIQ_OK only once the + * current table — which includes that value — is on flash; + * EMBEDIQ_ERR if the table could not be written, including when + * the pending change is another key's. + * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps + * the change and the next successful persist (any later write, or flush) + * carries it. * This is the ONLY authorized write path for CFG_MUT_FACTORY keys at runtime. * IEC 62443 commissioning immutability: operator cannot overwrite after provisioning. * fb_provisioning calls this at UNMANUFACTURED -> MANUFACTURED transition only. @@ -116,9 +145,17 @@ embediq_err_t embediq_nvm_provision(const char *key, const char *value); * (see docs/architecture/lifecycle.md — OTA shutdown contract). * * @return EMBEDIQ_OK if all data is safely persisted, EMBEDIQ_ERR on I/O error. + * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps + * the change and the next successful persist (any later write, or flush) + * carries it. * - * Calling context: any thread. Always writes the current table; may wait for - * a persist in flight. + * If flash was unreadable at boot, flush re-reads it instead of writing: on + * success the flash contents replace the (empty) table and persisting + * resumes; on failure it returns EMBEDIQ_ERR with an NVM_PERSIST_FAIL event. + * flush is the only call that performs this retry. + * + * Calling context: any thread. Otherwise always writes the current table; + * may wait for a persist in flight. */ embediq_err_t embediq_nvm_flush(void); @@ -127,6 +164,9 @@ embediq_err_t embediq_nvm_flush(void); * Typically called during factory reset only. * * @return EMBEDIQ_OK on success, EMBEDIQ_ERR on store error. + * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps + * the change and the next successful persist (any later write, or flush) + * carries it. * * Calling context: any thread. May wait for a persist in flight. */ diff --git a/core/include/embediq_obs.h b/core/include/embediq_obs.h index 2a505de..370ce85 100644 --- a/core/include/embediq_obs.h +++ b/core/include/embediq_obs.h @@ -268,7 +268,10 @@ extern "C" { source_fb_id: fb_nvm endpoint index. state_or_flag: 0=magic_fail 1=crc_fail - 2=major_version_mismatch. + 2=major_version_mismatch + 3=flash_read_fail (the + store refuses to persist + until flush re-reads it). msg_id: 0 (no message context). */ #define EMBEDIQ_OBS_EVT_CFG_MUTABILITY_REJECT 0x69u /**< Fleet push attempted to overwrite a factory or user-class config key. @@ -279,7 +282,23 @@ extern "C" { rejected key (0=factory 2=user). msg_id: low 16 bits of key name CRC32 (key name itself is never logged). */ -/* 0x6A–0x6F reserved */ +/* 0x6A reserved: STREAM_FAULT (ARCHITECTURE.md "Streaming Data Plane") */ +#define EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL 0x6Bu /**< NVM table write to flash failed. + The change stays in RAM; the next + successful persist carries it. + One event per failed write. + source_fb_id: 0. target_fb_id: 0. + state_or_flag: HAL return value + truncated to uint8_t (HAL errors + are -127..-1: 0x81..0xFF), or + EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE. + msg_id: low 16 bits of the table + generation that failed to persist. */ +/** NVM_PERSIST_FAIL state: nothing was written because flash could not be + * read at boot (contents unknown); the change was refused and RAM is + * unchanged. 0x80 is outside the truncated HAL error range. */ +#define EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE 0x80u +/* 0x6C–0x6F reserved */ /* FUNCTION family (0x70–0x7F) --------------------------------------------- */ diff --git a/core/include/ops/embediq_tls.h b/core/include/ops/embediq_tls.h index 401b18d..d9031fc 100644 --- a/core/include/ops/embediq_tls.h +++ b/core/include/ops/embediq_tls.h @@ -13,6 +13,10 @@ * embediq_tls_err_t added; embediq_bus.h dependency removed. * v2 (additive) — EMBEDIQ_TLS_ERR_BUSY (-12): returned by every op on a slot while a * connect_async() is in flight. No field change; EMBEDIQ_TLS_OPS_VERSION stays 2. + * v2 (additive) — connect_async() on a connected slot also returns + * EMBEDIQ_TLS_ERR_BUSY. No field change; EMBEDIQ_TLS_OPS_VERSION stays 2. + * v2 (additive) — configure() stages while connected. No field change; + * EMBEDIQ_TLS_OPS_VERSION stays 2. * * Context safety: INSTANCE-SAFE — each ops table slot represents one * independent TLS connection context. Concurrent use of distinct slots @@ -21,6 +25,14 @@ * slot — configure, a second connect_async, disconnect, send, recv — * returns EMBEDIQ_TLS_ERR_BUSY without blocking. The slot accepts ops again * once the connect has completed (no later than on_complete being invoked). + * The completion callback and user context belong to the attempt that + * started them; a later connect_async() on the same slot cannot redirect or + * drop an earlier completion. A slot that is connected or connecting refuses + * connect_async() with EMBEDIQ_TLS_ERR_BUSY: disconnect first. + * After the peer closes the connection (close_notify or an orderly TCP close) + * recv() returns EMBEDIQ_TLS_ERR_RECV and the session's resources are + * released before recv() returns; a connection reset or other transport + * error leaves the slot connected and is released by disconnect(). * * @author Ritesh Anand * @company embediq.com | ritzylab.com @@ -63,7 +75,7 @@ typedef int32_t embediq_tls_err_t; #define EMBEDIQ_TLS_ERR_NO_RESOURCES ((embediq_tls_err_t) -9) /**< slot index out of range */ #define EMBEDIQ_TLS_ERR_SEND ((embediq_tls_err_t)-10) /**< send() underlying write failed */ #define EMBEDIQ_TLS_ERR_RECV ((embediq_tls_err_t)-11) /**< recv() underlying read failed */ -#define EMBEDIQ_TLS_ERR_BUSY ((embediq_tls_err_t)-12) /**< slot has a connect_async() in flight */ +#define EMBEDIQ_TLS_ERR_BUSY ((embediq_tls_err_t)-12) /**< connect_async() in flight; or connect_async() on a connected slot */ /* --------------------------------------------------------------------------- * Async connect callback @@ -105,10 +117,15 @@ typedef struct { * Must be called before the first connect_async(). Calling connect_async() * without a prior configure() returns EMBEDIQ_TLS_ERR_NOT_CONFIGURED. * - * May be called again while connected — the new credentials are stored - * and applied on the NEXT connect_async() after disconnect(). The current - * session is unaffected. This enables certificate rotation without - * connection downtime. + * May be called while connected: the replacement is validated and + * staged; the current session is unaffected — the active session and the + * configuration it references are untouched; the replacement becomes + * active on the next connect_async(). A rejected replacement (bad PEM, no + * CA) leaves the active configuration and any session intact and returns + * the error; a previously staged replacement is discarded. With no live + * session a validated replacement becomes active at once. This enables + * certificate rotation without connection downtime. + * Memory: the HAL holds at most two configurations per slot. * * ca_cert_pem: NUL-terminated PEM string of the CA certificate used to * verify the server. Must not be NULL. @@ -143,6 +160,8 @@ typedef struct { * user_ctx: passed unchanged to on_complete. May be NULL. * * Returns EMBEDIQ_TLS_OK if the operation was successfully initiated. + * Returns EMBEDIQ_TLS_ERR_BUSY if the slot is connected or connecting + * (disconnect first). * on_complete is NOT called on error return — the caller must check the * return value and handle it directly. */ diff --git a/fbs/drivers/fb_nvm.c b/fbs/drivers/fb_nvm.c index 64e8505..2461d4d 100644 --- a/fbs/drivers/fb_nvm.c +++ b/fbs/drivers/fb_nvm.c @@ -32,6 +32,12 @@ * generation is already on flash (a later writer persisted a newer table) * skips its flash write, so an older table never overwrites a newer one. * + * Durability is a result: every mutation persists synchronously and returns + * whether its change (or a newer table) reached flash. A failed flash write + * returns EMBEDIQ_ERR and emits one EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL; the + * table keeps the change and the next successful persist (any later write, + * or flush) carries it. + * * Boot phase: EMBEDIQ_BOOT_PHASE_INFRASTRUCTURE (2) * * Public API (matches embediq_nvm.h contract exactly): @@ -39,7 +45,7 @@ * embediq_nvm_set(key, val, len) — store value * embediq_nvm_get(key, val, len) — lookup value * embediq_nvm_delete(key) — remove key - * embediq_nvm_flush() — persist all pending writes + * embediq_nvm_flush() — rewrite the current table (retry) * embediq_nvm_reset() — wipe all entries * * Host/test-only API (EMBEDIQ_PLATFORM_HOST): @@ -134,8 +140,14 @@ static EmbedIQ_FB_Handle_t g_nvm_fb = NULL; /* Generation of the table: bumped by every mutation. Under g_mutex. */ static uint32_t g_generation = 0u; -/* Generation of the table last written to flash. Under g_persist_mutex. */ +/* Generation of the table last written to flash. Under g_persist_mutex. + * Invariant: g_persisted_gen <= g_generation (modulo 2^32) — it only ever + * takes a value g_generation has had. */ static uint32_t g_persisted_gen = 0u; +/* Flash could not be read when the table was loaded: its contents are + * unknown, so nothing is written over them until embediq_nvm_flush() re-reads + * them. Written with both locks held; read under either. */ +static bool s_flash_unreadable = false; /** * Composite persist buffer — header + payload in one allocation. @@ -187,29 +199,59 @@ static uint8_t find_free_slot(void) #define NVM_LOAD_OK 0xFFu /* no event */ #define NVM_LOAD_MAGIC_FAIL 0u #define NVM_LOAD_CRC_FAIL 1u +#define NVM_LOAD_READ_FAIL 3u /* 2 is documented for a major version mismatch */ + +/* A persist refused because flash was unreadable at load: one NVM_PERSIST_FAIL + * with state FLASH_UNREADABLE. Caller holds neither lock. */ +static embediq_err_t persist_refused(uint32_t gen) +{ + EMBEDIQ_OBS_EMIT_FAULT(EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL, 0u, 0u, + (uint8_t)EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE, + (uint16_t)(gen & 0xFFFFu)); + return EMBEDIQ_ERR; +} /** * Write the table to flash unless a table at least as new as `gen` is already * there (`force` writes regardless — used by embediq_nvm_flush()). + * Returns EMBEDIQ_OK when generation `gen` (or newer) is on flash after the + * call; EMBEDIQ_ERR when the flash write failed (g_persisted_gen unchanged; + * one NVM_PERSIST_FAIL emitted), when flash was unreadable at load (nothing + * written; one NVM_PERSIST_FAIL, state FLASH_UNREADABLE) or a lock could not + * be taken (the OSAL observes that; no event here). * Calling context: any thread; the caller holds neither lock. Takes the - * persist lock, then the table lock only for the snapshot memcpy. + * persist lock, then the table lock only for the skip test and the snapshot + * memcpy. */ -static void nvm_persist(uint32_t gen, bool force) +static embediq_err_t nvm_persist(uint32_t gen, bool force) { - if (embediq_osal_mutex_lock(g_persist_mutex, UINT32_MAX) != EMBEDIQ_OK) return; - - if (!force && (int32_t)(gen - g_persisted_gen) <= 0) { - embediq_osal_mutex_unlock(g_persist_mutex); /* newer table already on flash */ - return; + if (embediq_osal_mutex_lock(g_persist_mutex, UINT32_MAX) != EMBEDIQ_OK) return EMBEDIQ_ERR; + if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) != EMBEDIQ_OK) { + embediq_osal_mutex_unlock(g_persist_mutex); + return EMBEDIQ_ERR; } - /* Snapshot the live table — the only work done under the table lock. */ - if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) != EMBEDIQ_OK) { + /* Skip iff flash is at least as new as the caller's change, both + * distances measured back from the current generation in uint32 + * arithmetic: (cur - persisted) <= (cur - gen). Both are small unless + * 2^32 mutations happen with no successful persist, so a failing flash + * (persisted falling ever further behind) never wraps into a false skip, + * which a signed difference between gen and persisted would after 2^31. */ + uint32_t cur = g_generation; + if (s_flash_unreadable) { + embediq_osal_mutex_unlock(g_mutex); embediq_osal_mutex_unlock(g_persist_mutex); - return; + return persist_refused(cur); } + if (!force && (uint32_t)(cur - g_persisted_gen) <= (uint32_t)(cur - gen)) { + embediq_osal_mutex_unlock(g_mutex); + embediq_osal_mutex_unlock(g_persist_mutex); /* this change is on flash */ + return EMBEDIQ_OK; + } + + /* Snapshot the live table — the only work done under the table lock. */ memcpy(s_persist_buf.entries, g_nvm, sizeof(g_nvm)); - uint32_t snap_gen = g_generation; + uint32_t snap_gen = cur; embediq_osal_mutex_unlock(g_mutex); /* Build header. */ @@ -238,26 +280,42 @@ static void nvm_persist(uint32_t gen, bool force) /* Single atomic write — HAL atomicity contract (PR-A: hal_flash.h) ensures * either the full blob lands or nothing changes. On failure the generation * stays behind, so the next write retries. */ - if (hal_flash_write(0u, &s_persist_buf, sizeof(s_persist_buf)) == HAL_OK) { + int hal_rc = hal_flash_write(0u, &s_persist_buf, sizeof(s_persist_buf)); + if (hal_rc == HAL_OK) { g_persisted_gen = snap_gen; } embediq_osal_mutex_unlock(g_persist_mutex); + + if (hal_rc != HAL_OK) { + EMBEDIQ_OBS_EMIT_FAULT(EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL, 0u, 0u, + (uint8_t)hal_rc, (uint16_t)(snap_gen & 0xFFFFu)); + return EMBEDIQ_ERR; + } + return EMBEDIQ_OK; } +/* + * Unreadable flash and invalid flash are different facts. A failed read means + * the contents are unknown: RAM starts empty, nothing is persisted over them, + * and only embediq_nvm_flush() re-reads them. A blob with bad magic or CRC is + * known to be invalid: RAM empty is the truth and the first write replaces it. + */ + /** - * Load the table from flash (boot, or a simulated power-on in tests). - * Calling context: any thread; takes the persist lock, then the table lock. - * Observatory events are emitted after both locks are released. + * Replace the table with the flash contents. Caller holds the persist lock; + * takes the table lock. Returns the load outcome (NVM_LOAD_OK when valid) and + * the current generation in *gen. Emits nothing. */ -static void nvm_load(void) +static uint8_t nvm_load_locked(uint32_t *gen) { - if (!g_mutex || !g_persist_mutex) return; /* APIs report EMBEDIQ_ERR */ - if (embediq_osal_mutex_lock(g_persist_mutex, UINT32_MAX) != EMBEDIQ_OK) return; - /* Read the full blob (header + payload) in a single call. */ - uint8_t outcome = NVM_LOAD_OK; - bool valid = hal_flash_read(0u, &s_persist_buf, sizeof(s_persist_buf)) == HAL_OK; + uint8_t outcome = NVM_LOAD_OK; + bool readable = hal_flash_read(0u, &s_persist_buf, sizeof(s_persist_buf)) == HAL_OK; + bool valid = readable; + if (!readable) { + outcome = NVM_LOAD_READ_FAIL; + } /* Verify magic — catches erased flash, pre-PR-D blobs, and corrupt files. */ if (valid && memcmp(s_persist_buf.hdr.magic, k_nvm_blob_magic, @@ -273,20 +331,44 @@ static void nvm_load(void) valid = false; } + *gen = 0u; if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) == EMBEDIQ_OK) { if (valid) { memcpy(g_nvm, s_persist_buf.entries, sizeof(g_nvm)); /* blob valid */ } else { memset(g_nvm, 0, sizeof(g_nvm)); /* start with defaults (empty store) */ } - g_persisted_gen = g_generation; /* the table now matches what boot saw */ + s_flash_unreadable = !readable; + if (readable) { + g_persisted_gen = g_generation; /* the table now matches flash */ + } + *gen = g_generation; embediq_osal_mutex_unlock(g_mutex); } + return outcome; +} + +/** + * Load the table from flash (boot, or a simulated power-on in tests). + * nvm_load() runs only at boot, before any other caller exists (and from the + * host test hook with no concurrent callers). It replaces the table with the + * flash contents; a change made concurrently by another thread would be + * discarded and its persist skipped. Reloading at runtime is not supported. + * Takes the persist lock, then the table lock. Observatory events are emitted + * after both locks are released. + */ +static void nvm_load(void) +{ + if (!g_mutex || !g_persist_mutex) return; /* APIs report EMBEDIQ_ERR */ + if (embediq_osal_mutex_lock(g_persist_mutex, UINT32_MAX) != EMBEDIQ_OK) return; + + uint32_t gen; + uint8_t outcome = nvm_load_locked(&gen); embediq_osal_mutex_unlock(g_persist_mutex); if (outcome != NVM_LOAD_OK) { EMBEDIQ_OBS_EMIT_FAULT(EMBEDIQ_OBS_EVT_CFG_BLOB_INVALID, - 0u, 0u, outcome, 0u); /* 0 = magic_fail, 1 = crc_fail */ + 0u, 0u, outcome, 0u); /* 0 = magic, 1 = crc, 3 = read */ } } @@ -360,6 +442,12 @@ embediq_err_t embediq_nvm_set(const char *key, const void *val, uint32_t len) if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) != EMBEDIQ_OK) return EMBEDIQ_ERR; + if (s_flash_unreadable) { /* contents unknown: change nothing, write nothing */ + uint32_t cur = g_generation; + embediq_osal_mutex_unlock(g_mutex); + return persist_refused(cur); + } + uint8_t slot = find_slot(key); if (slot == NVM_MAX_KEYS) { slot = find_free_slot(); @@ -378,8 +466,7 @@ embediq_err_t embediq_nvm_set(const char *key, const void *val, uint32_t len) uint32_t gen = ++g_generation; embediq_osal_mutex_unlock(g_mutex); - nvm_persist(gen, false); - return EMBEDIQ_OK; + return nvm_persist(gen, false); } /* Write-once provisioning write — the ONLY authorized runtime write path for @@ -408,11 +495,21 @@ embediq_err_t embediq_nvm_provision(const char *key, const char *value) return EMBEDIQ_ERR; } - /* Write-once: a non-empty existing value means already provisioned. */ + if (s_flash_unreadable) { /* contents unknown: change nothing, write nothing */ + uint32_t cur = g_generation; + embediq_osal_mutex_unlock(g_mutex); + return persist_refused(cur); + } + + /* Write-once: a non-empty existing value means already provisioned. + * Idempotent on the value, not on having tried: keep the commissioned + * value and return OK only once the table holding it is on flash — a + * no-op when it is, a real retry when an earlier persist failed. */ uint8_t slot = find_slot(key); if (slot != NVM_MAX_KEYS && g_nvm[slot].val_len > 0u) { + uint32_t cur = g_generation; embediq_osal_mutex_unlock(g_mutex); - return EMBEDIQ_OK; /* idempotent — keep the commissioned identity */ + return nvm_persist(cur, false); } if (slot == NVM_MAX_KEYS) { @@ -433,8 +530,7 @@ embediq_err_t embediq_nvm_provision(const char *key, const char *value) uint32_t gen = ++g_generation; embediq_osal_mutex_unlock(g_mutex); - nvm_persist(gen, false); - return EMBEDIQ_OK; + return nvm_persist(gen, false); } embediq_err_t embediq_nvm_get(const char *key, void *val, uint32_t *len) @@ -468,26 +564,54 @@ embediq_err_t embediq_nvm_delete(const char *key) if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) != EMBEDIQ_OK) return EMBEDIQ_ERR; + if (s_flash_unreadable) { /* contents unknown: change nothing, write nothing */ + uint32_t cur = g_generation; + embediq_osal_mutex_unlock(g_mutex); + return persist_refused(cur); + } + + /* Absent key: idempotent on durability, not on having tried. Return OK + * only once the table without the key is on flash — a no-op when it is, + * a real retry when an earlier delete's persist failed. */ uint8_t slot = find_slot(key); if (slot == NVM_MAX_KEYS) { + uint32_t cur = g_generation; embediq_osal_mutex_unlock(g_mutex); - return EMBEDIQ_OK; /* idempotent per contract */ + return nvm_persist(cur, false); } memset(&g_nvm[slot], 0, sizeof(nvm_entry_t)); uint32_t gen = ++g_generation; embediq_osal_mutex_unlock(g_mutex); - nvm_persist(gen, false); - return EMBEDIQ_OK; + return nvm_persist(gen, false); } embediq_err_t embediq_nvm_flush(void) { if (!g_mutex || !g_persist_mutex) return EMBEDIQ_ERR; - nvm_persist(0u, true); - return EMBEDIQ_OK; + /* Flush makes flash and table agree. If flash was unreadable at load, that + * means re-reading it: the RAM table holds nothing durable (every mutation + * was refused), so the loaded table replaces it. Only flush retries. */ + if (embediq_osal_mutex_lock(g_persist_mutex, UINT32_MAX) != EMBEDIQ_OK) return EMBEDIQ_ERR; + if (s_flash_unreadable) { + uint32_t gen; + uint8_t outcome = nvm_load_locked(&gen); + bool still = s_flash_unreadable; + embediq_osal_mutex_unlock(g_persist_mutex); + if (still) { + return persist_refused(gen); + } + if (outcome != NVM_LOAD_OK) { + EMBEDIQ_OBS_EMIT_FAULT(EMBEDIQ_OBS_EVT_CFG_BLOB_INVALID, + 0u, 0u, outcome, 0u); /* readable, but invalid */ + } + return EMBEDIQ_OK; + } + embediq_osal_mutex_unlock(g_persist_mutex); + + return nvm_persist(0u, true); } embediq_err_t embediq_nvm_reset(void) @@ -495,12 +619,17 @@ embediq_err_t embediq_nvm_reset(void) if (!g_mutex || !g_persist_mutex) return EMBEDIQ_ERR; if (embediq_osal_mutex_lock(g_mutex, UINT32_MAX) != EMBEDIQ_OK) return EMBEDIQ_ERR; + + if (s_flash_unreadable) { /* contents unknown: change nothing, write nothing */ + uint32_t cur = g_generation; + embediq_osal_mutex_unlock(g_mutex); + return persist_refused(cur); + } memset(g_nvm, 0, sizeof(g_nvm)); uint32_t gen = ++g_generation; embediq_osal_mutex_unlock(g_mutex); - nvm_persist(gen, false); - return EMBEDIQ_OK; + return nvm_persist(gen, false); } /* --------------------------------------------------------------------------- diff --git a/fbs/services/fb_provisioning.c b/fbs/services/fb_provisioning.c index 0bc61db..b796c6a 100644 --- a/fbs/services/fb_provisioning.c +++ b/fbs/services/fb_provisioning.c @@ -150,25 +150,23 @@ static void prov_run_startup(void) return; } - /* STEP 3: write mqtt.client_id from the cert CN if not already set. + /* STEP 3: provision mqtt.client_id from the cert CN, every startup. * mqtt.client_id is a CFG_MUT_FACTORY key — embediq_cfg_set_str() would be * rejected by the NVM mutability guard. embediq_nvm_provision() is the * authorized write-once path: it sets the key only while absent/empty and - * is idempotent once provisioned (IEC 62443 commissioning immutability). */ - char existing_id[EMBEDIQ_NVM_VAL_SIZE]; - existing_id[0] = '\0'; - (void)embediq_cfg_get_str("mqtt.client_id", existing_id, sizeof(existing_id), ""); - if (strcmp(existing_id, ctx.device_id) != 0) { - if (embediq_nvm_provision("mqtt.client_id", ctx.device_id) != EMBEDIQ_OK) { - prov_obs_fault(1u); - prov_report_fault(2u); - return; - } - if (embediq_nvm_flush() != EMBEDIQ_OK) { - prov_obs_fault(2u); - prov_report_fault(3u); - return; - } + * keeps a provisioned value (IEC 62443 commissioning immutability). It is + * called even when RAM already holds the value: it returns OK only once + * that value is on flash, so a persist that failed earlier is retried + * here, not mistaken for a durable identity. */ + if (embediq_nvm_provision("mqtt.client_id", ctx.device_id) != EMBEDIQ_OK) { + prov_obs_fault(1u); + prov_report_fault(2u); + return; + } + if (embediq_nvm_flush() != EMBEDIQ_OK) { + prov_obs_fault(2u); + prov_report_fault(3u); + return; } /* STEP 3b: advance to MANUFACTURED if not already there. */ diff --git a/hal/posix/ops/hal_tls_posix.c b/hal/posix/ops/hal_tls_posix.c index e1dea2f..6ac228d 100644 --- a/hal/posix/ops/hal_tls_posix.c +++ b/hal/posix/ops/hal_tls_posix.c @@ -13,13 +13,34 @@ * * Thread safety: one pthread_mutex_t per slot, only ever unlocked by the * thread that locked it. All public functions lock the slot mutex. - * connect_async() sets `connecting` under the mutex, releases it and spawns a - * detached connect thread. While `connecting` is true the connect thread owns - * the slot's mbedTLS and socket state exclusively, without holding the mutex; - * every other op checks `connecting` under the mutex and returns - * EMBEDIQ_TLS_ERR_BUSY. The connect thread publishes its result and clears - * `connecting` under the mutex, unlocks, then calls on_complete (mutex not - * held). + * connect_async() refuses a slot that is connected or connecting; otherwise + * it sets `connecting` under the mutex, fills the slot's argument entry, + * releases the mutex and spawns a detached connect thread. While `connecting` + * is true the connect thread owns the slot's mbedTLS and socket state + * exclusively, without holding the mutex; every other op checks `connecting` + * under the mutex and returns EMBEDIQ_TLS_ERR_BUSY. The connect thread copies + * its argument entry before any blocking work and uses only that copy: the + * entry may be reused by the next attempt once `connecting` is cleared. It + * publishes its result and clears `connecting` under the mutex, unlocks, then + * calls the copy's on_complete with the copy's user_ctx (mutex not held). + * + * Session teardown has one owner, slot_session_free(): it releases the ssl + * context and the socket of a session the connect thread initialised. A peer + * close seen by recv(), a failed attempt and disconnect() each free the + * session; connect_async() frees any session still live before it re-inits, + * so a session is never re-initialised over and never freed twice. + * configure() frees only a session that is no longer connected. + * + * Configuration lifetime: mbedtls_ssl_setup() keeps a pointer to the + * configuration (no copy), so the set a live session references is never + * modified or freed. Each slot holds two configuration sets: the active one + * and a scratch one. configure() builds and validates into the scratch set; + * with no live session it becomes active at once, otherwise it is staged and + * connect_async() promotes it. Promotion happens only in connect_async(), + * after any session has been freed. + * Memory: two configuration sets per slot is this POSIX HAL's choice + * (mbedTLS allocates certificates and keys on the heap); an MCU HAL + * implements the same contract with its own storage. * * Phase 3 note: FreeRTOS implementation uses mbedTLS with OSAL primitives * instead of pthreads. The ops table contract is identical. @@ -35,6 +56,7 @@ #include "hal/hal_defs.h" #include +#include #include #include #include @@ -59,20 +81,36 @@ * Static pool — no malloc (R-02). Size controlled by embediq_config.h. * ------------------------------------------------------------------------- */ +/* One configuration set: everything an ssl context set up from `conf` + * references. Always in the initialised state when not valid. */ typedef struct { - mbedtls_ssl_context ssl; mbedtls_ssl_config conf; mbedtls_x509_crt ca_cert; mbedtls_x509_crt client_cert; mbedtls_pk_context client_key; mbedtls_ctr_drbg_context ctr_drbg; mbedtls_entropy_context entropy; + bool has_client_cert; + bool valid; /* built and validated */ + uint8_t slot_idx; /* owning slot (host counters) */ +} tls_config_t; + +/* Invariant: cfg[active] is referenced by ssl whenever session_live; it is + * never modified or freed while session_live. cfg[1 - active] is scratch: + * built by configure(), promoted by connect_async() when no session is live. + * `staged` means cfg[1 - active].valid holds a validated replacement awaiting + * promotion. All three under the mutex. */ +typedef struct { + mbedtls_ssl_context ssl; + tls_config_t cfg[2]; + uint8_t active; + bool staged; mbedtls_net_context net; pthread_mutex_t mutex; - bool configured; bool connected; - bool has_client_cert; bool connecting; /* connect_async in flight; under mutex */ + bool session_live; /* ssl/net initialised for a session; cleared + only by slot_session_free() */ } tls_slot_t; static tls_slot_t g_slots[EMBEDIQ_TLS_MAX_CONNECTIONS]; @@ -83,6 +121,7 @@ static tls_slot_t g_slots[EMBEDIQ_TLS_MAX_CONNECTIONS]; typedef struct { uint8_t slot_idx; + uint8_t cfg_idx; /* configuration set the attempt uses */ char host[TLS_HOSTNAME_MAX]; uint16_t port; char sni[TLS_HOSTNAME_MAX]; @@ -92,18 +131,152 @@ typedef struct { static connect_args_t g_connect_args[EMBEDIQ_TLS_MAX_CONNECTIONS]; +#ifdef EMBEDIQ_PLATFORM_HOST +/* Test hook: called by connect_finish after the slot is released and before + * on_complete, with the attempt's user_ctx. NULL = no hook. */ +typedef void (*tls_connect_finish_hook_t)(void *user_ctx); +static _Atomic(tls_connect_finish_hook_t) s_connect_finish_hook = NULL; +/* Sessions marked live and not yet released by slot_session_free(). */ +static atomic_int s_live_sessions = 0; +#define LIVE_SESSION_ADD(n) atomic_fetch_add(&s_live_sessions, (n)) +/* Per slot: configuration sets built and not yet freed. */ +static atomic_int s_live_configs[EMBEDIQ_TLS_MAX_CONNECTIONS]; +#define LIVE_CONFIG_ADD(i, n) atomic_fetch_add(&s_live_configs[(i)], (n)) +#else +#define LIVE_SESSION_ADD(n) ((void)0) +#define LIVE_CONFIG_ADD(i, n) ((void)0) +#endif + /* --------------------------------------------------------------------------- * Internal helpers * ------------------------------------------------------------------------- */ -static void slot_free_certs(tls_slot_t *s) +/* Release a live session's ssl context and socket. Caller holds the slot + * mutex. */ +static void slot_session_free(tls_slot_t *s) { mbedtls_ssl_free(&s->ssl); - mbedtls_ssl_config_free(&s->conf); - mbedtls_x509_crt_free(&s->ca_cert); - mbedtls_x509_crt_free(&s->client_cert); - mbedtls_pk_free(&s->client_key); mbedtls_net_free(&s->net); + s->connected = false; + s->session_live = false; + LIVE_SESSION_ADD(-1); +} + +static void config_init(tls_config_t *c) +{ + mbedtls_ssl_config_init(&c->conf); + mbedtls_x509_crt_init(&c->ca_cert); + mbedtls_x509_crt_init(&c->client_cert); + mbedtls_pk_init(&c->client_key); + mbedtls_ctr_drbg_init(&c->ctr_drbg); + mbedtls_entropy_init(&c->entropy); + c->has_client_cert = false; + c->valid = false; +} + +/* The only place a configuration set's members are freed. Leaves the set + * initialised (not valid), ready to build or free again. */ +static void config_free(tls_config_t *c) +{ + if (c->valid) { + LIVE_CONFIG_ADD(c->slot_idx, -1); + } + mbedtls_ssl_config_free(&c->conf); + mbedtls_x509_crt_free(&c->ca_cert); + mbedtls_x509_crt_free(&c->client_cert); + mbedtls_pk_free(&c->client_key); + mbedtls_ctr_drbg_free(&c->ctr_drbg); + mbedtls_entropy_free(&c->entropy); + config_init(c); +} + +/* The build steps of config_build(); stops at the first failure. */ +static embediq_tls_err_t config_build_steps(tls_config_t *c, + const char *ca_cert_pem, + const char *client_cert_pem, + const char *client_key_pem) +{ + int rc; + + /* Seed DRBG from /dev/urandom (POSIX platform) */ + rc = mbedtls_ctr_drbg_seed(&c->ctr_drbg, mbedtls_entropy_func, + &c->entropy, NULL, 0); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); + return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; + } + + /* TLS 1.2 client defaults */ + rc = mbedtls_ssl_config_defaults(&c->conf, + MBEDTLS_SSL_IS_CLIENT, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); + return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; + } + + /* Always verify server certificate */ + mbedtls_ssl_conf_authmode(&c->conf, MBEDTLS_SSL_VERIFY_REQUIRED); + mbedtls_ssl_conf_rng(&c->conf, mbedtls_ctr_drbg_random, &c->ctr_drbg); + + /* Load CA certificate (mandatory) */ + rc = mbedtls_x509_crt_parse(&c->ca_cert, + (const unsigned char *)ca_cert_pem, + strlen(ca_cert_pem) + 1u); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); + return EMBEDIQ_TLS_ERR_CERT_VERIFY; + } + mbedtls_ssl_conf_ca_chain(&c->conf, &c->ca_cert, NULL); + + /* Load client cert + key if provided (mutual TLS) */ + if (client_cert_pem != NULL && client_key_pem != NULL) { + rc = mbedtls_x509_crt_parse(&c->client_cert, + (const unsigned char *)client_cert_pem, + strlen(client_cert_pem) + 1u); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); + return EMBEDIQ_TLS_ERR_CERT_REJECTED; + } + + rc = mbedtls_pk_parse_key(&c->client_key, + (const unsigned char *)client_key_pem, + strlen(client_key_pem) + 1u, + NULL, 0, + mbedtls_ctr_drbg_random, &c->ctr_drbg); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); + return EMBEDIQ_TLS_ERR_CERT_REJECTED; + } + + rc = mbedtls_ssl_conf_own_cert(&c->conf, &c->client_cert, &c->client_key); + if (rc != 0) { + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); + return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; + } + c->has_client_cert = true; + } + return EMBEDIQ_TLS_OK; +} + +/* Build and validate a configuration set from PEM strings into an + * initialised set. On any failure the set is freed: nothing is left + * half-built. */ +static embediq_tls_err_t config_build(tls_config_t *c, + const char *ca_cert_pem, + const char *client_cert_pem, + const char *client_key_pem) +{ + embediq_tls_err_t err = config_build_steps(c, ca_cert_pem, + client_cert_pem, client_key_pem); + if (err != EMBEDIQ_TLS_OK) { + config_free(c); + return err; + } + c->valid = true; + LIVE_CONFIG_ADD(c->slot_idx, 1); + return EMBEDIQ_TLS_OK; } static embediq_tls_err_t map_mbedtls_handshake_err(tls_slot_t *s, int rc) @@ -122,22 +295,35 @@ static embediq_tls_err_t map_mbedtls_handshake_err(tls_slot_t *s, int rc) /* --------------------------------------------------------------------------- * Connect thread — spawned by connect_async() * Owns the slot exclusively while s->connecting is true and does its work - * WITHOUT the slot mutex. At the end it takes the mutex, publishes the result - * and clears `connecting`, unlocks, then calls on_complete (mutex not held). + * WITHOUT the slot mutex, from its own copy of the attempt. At the end it + * takes the mutex, publishes the result and clears `connecting`, unlocks, + * then calls the attempt's on_complete (mutex not held). * ------------------------------------------------------------------------- */ -static void connect_finish(tls_slot_t *s, const connect_args_t *a, embediq_tls_err_t result) +/* attempt is the connect thread's own copy: after the unlock the slot and its + * argument entry belong to the next attempt, so nothing after it reads them. */ +static void connect_finish(tls_slot_t *s, const connect_args_t *attempt, + embediq_tls_err_t result) { pthread_mutex_lock(&s->mutex); + if (result != EMBEDIQ_TLS_OK && s->session_live) { + slot_session_free(s); /* a failed attempt leaves no session behind */ + } s->connected = (result == EMBEDIQ_TLS_OK); s->connecting = false; pthread_mutex_unlock(&s->mutex); - a->on_complete(result, a->user_ctx); +#ifdef EMBEDIQ_PLATFORM_HOST + tls_connect_finish_hook_t hook = atomic_load(&s_connect_finish_hook); + if (hook) hook(attempt->user_ctx); +#endif + attempt->on_complete(result, attempt->user_ctx); } static void *connect_thread(void *arg) { - connect_args_t *a = (connect_args_t *)arg; + /* Copy the attempt first, while `connecting` keeps the entry unchanged. */ + const connect_args_t attempt = *(const connect_args_t *)arg; + const connect_args_t *a = &attempt; tls_slot_t *s = &g_slots[a->slot_idx]; embediq_tls_err_t result; char port_str[TLS_PORT_STR_MAX]; @@ -157,10 +343,11 @@ static void *connect_thread(void *arg) /* Initialise SSL context for this connection */ mbedtls_ssl_init(&s->ssl); - rc = mbedtls_ssl_setup(&s->ssl, &s->conf); + s->session_live = true; /* from here every failure is freed by connect_finish */ + LIVE_SESSION_ADD(1); + rc = mbedtls_ssl_setup(&s->ssl, &s->cfg[a->cfg_idx].conf); if (rc != 0) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); - mbedtls_net_free(&s->net); connect_finish(s, a, EMBEDIQ_TLS_ERR_HANDSHAKE); return NULL; } @@ -169,8 +356,6 @@ static void *connect_thread(void *arg) rc = mbedtls_ssl_set_hostname(&s->ssl, a->sni); if (rc != 0) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); - mbedtls_ssl_free(&s->ssl); - mbedtls_net_free(&s->net); connect_finish(s, a, EMBEDIQ_TLS_ERR_HANDSHAKE); return NULL; } @@ -188,8 +373,6 @@ static void *connect_thread(void *arg) if (rc != 0) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); result = map_mbedtls_handshake_err(s, rc); - mbedtls_ssl_free(&s->ssl); - mbedtls_net_free(&s->net); connect_finish(s, a, result); return NULL; } @@ -208,7 +391,6 @@ static embediq_tls_err_t slot_configure(uint8_t idx, const char *client_key_pem) { tls_slot_t *s = &g_slots[idx]; - int rc; pthread_mutex_lock(&s->mutex); @@ -217,90 +399,34 @@ static embediq_tls_err_t slot_configure(uint8_t idx, return EMBEDIQ_TLS_ERR_BUSY; /* connect thread owns the slot */ } - /* Free any previous configuration */ - slot_free_certs(s); - mbedtls_ctr_drbg_free(&s->ctr_drbg); - mbedtls_entropy_free(&s->entropy); - s->configured = false; - s->connected = false; - s->has_client_cert = false; - - /* Re-initialise mbedTLS contexts */ - mbedtls_ssl_config_init(&s->conf); - mbedtls_x509_crt_init(&s->ca_cert); - mbedtls_x509_crt_init(&s->client_cert); - mbedtls_pk_init(&s->client_key); - mbedtls_ctr_drbg_init(&s->ctr_drbg); - mbedtls_entropy_init(&s->entropy); - mbedtls_net_init(&s->net); - - /* Seed DRBG from /dev/urandom (POSIX platform) */ - rc = mbedtls_ctr_drbg_seed(&s->ctr_drbg, mbedtls_entropy_func, - &s->entropy, NULL, 0); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); - pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; + /* A dead session (peer-closed, not yet reconnected) is released here; a + * live one is untouched — the new credentials apply on the next + * connect_async() after disconnect(). */ + if (s->session_live && !s->connected) { + slot_session_free(s); } - /* TLS 1.2 client defaults */ - rc = mbedtls_ssl_config_defaults(&s->conf, - MBEDTLS_SSL_IS_CLIENT, - MBEDTLS_SSL_TRANSPORT_STREAM, - MBEDTLS_SSL_PRESET_DEFAULT); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); + /* Build into the scratch set, dropping any previously staged one. The + * active set is not touched, so a rejected replacement leaves it, and + * any session using it, intact. */ + tls_config_t *scratch = &s->cfg[1u - s->active]; + config_free(scratch); + s->staged = false; + embediq_tls_err_t rc = config_build(scratch, ca_cert_pem, + client_cert_pem, client_key_pem); + if (rc != EMBEDIQ_TLS_OK) { pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; + return rc; } - /* Always verify server certificate */ - mbedtls_ssl_conf_authmode(&s->conf, MBEDTLS_SSL_VERIFY_REQUIRED); - mbedtls_ssl_conf_rng(&s->conf, mbedtls_ctr_drbg_random, &s->ctr_drbg); - - /* Load CA certificate (mandatory) */ - rc = mbedtls_x509_crt_parse(&s->ca_cert, - (const unsigned char *)ca_cert_pem, - strlen(ca_cert_pem) + 1u); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); - pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_CERT_VERIFY; - } - mbedtls_ssl_conf_ca_chain(&s->conf, &s->ca_cert, NULL); - - /* Load client cert + key if provided (mutual TLS) */ - if (client_cert_pem != NULL && client_key_pem != NULL) { - rc = mbedtls_x509_crt_parse(&s->client_cert, - (const unsigned char *)client_cert_pem, - strlen(client_cert_pem) + 1u); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); - pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_CERT_REJECTED; - } - - rc = mbedtls_pk_parse_key(&s->client_key, - (const unsigned char *)client_key_pem, - strlen(client_key_pem) + 1u, - NULL, 0, - mbedtls_ctr_drbg_random, &s->ctr_drbg); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); - pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_CERT_REJECTED; - } - - rc = mbedtls_ssl_conf_own_cert(&s->conf, &s->client_cert, &s->client_key); - if (rc != 0) { - EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_IO); - pthread_mutex_unlock(&s->mutex); - return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; - } - s->has_client_cert = true; + if (!s->session_live) { + config_free(&s->cfg[s->active]); /* nothing references it */ + s->active = (uint8_t)(1u - s->active); + } else { + s->staged = true; /* promoted by connect_async() */ } - - s->configured = true; + /* The socket belongs to the session, not the configuration: + * connect_async() initialises it and slot_session_free() releases it. */ pthread_mutex_unlock(&s->mutex); return EMBEDIQ_TLS_OK; } @@ -316,11 +442,24 @@ static embediq_tls_err_t slot_connect_async(uint8_t idx, pthread_mutex_lock(&s->mutex); - if (s->connecting) { + /* A live session or an attempt in flight owns the socket and the ssl + * context: disconnect first. */ + if (s->connecting || s->connected) { pthread_mutex_unlock(&s->mutex); return EMBEDIQ_TLS_ERR_BUSY; } - if (!s->configured) { + /* Every path that ends a session frees it; a session still live here + * is freed before the re-init below, never initialised over. */ + if (s->session_live) { + slot_session_free(s); + } + /* No session references the active set now: promote a staged one. */ + if (s->staged) { + config_free(&s->cfg[s->active]); + s->active = (uint8_t)(1u - s->active); + s->staged = false; + } + if (!s->cfg[s->active].valid) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_TLS, HAL_ERR_INVALID); pthread_mutex_unlock(&s->mutex); return EMBEDIQ_TLS_ERR_NOT_CONFIGURED; @@ -331,6 +470,7 @@ static embediq_tls_err_t slot_connect_async(uint8_t idx, s->connecting = true; connect_args_t *a = &g_connect_args[idx]; a->slot_idx = idx; + a->cfg_idx = s->active; /* read after promotion; the worker never re-reads it */ a->port = port; a->on_complete = on_complete; a->user_ctx = user_ctx; @@ -365,11 +505,11 @@ static embediq_tls_err_t slot_disconnect(uint8_t idx) pthread_mutex_unlock(&s->mutex); return EMBEDIQ_TLS_ERR_BUSY; } - if (s->connected) { - mbedtls_ssl_close_notify(&s->ssl); /* best-effort — ignore error */ - mbedtls_ssl_free(&s->ssl); - mbedtls_net_free(&s->net); - s->connected = false; + if (s->session_live) { + if (s->connected) { + mbedtls_ssl_close_notify(&s->ssl); /* best-effort — ignore error */ + } + slot_session_free(s); } pthread_mutex_unlock(&s->mutex); @@ -450,7 +590,7 @@ static int32_t slot_recv(uint8_t idx, uint8_t *buf, uint16_t len, uint32_t timeo return (int32_t)rc; } if (rc == 0 || rc == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) { - s->connected = false; + slot_session_free(s); /* the peer closed: the session is dead */ pthread_mutex_unlock(&s->mutex); return (int32_t)EMBEDIQ_TLS_ERR_RECV; } @@ -523,17 +663,16 @@ static void pool_init(void) for (uint8_t i = 0u; i < (uint8_t)EMBEDIQ_TLS_MAX_CONNECTIONS; i++) { pthread_mutex_init(&g_slots[i].mutex, NULL); mbedtls_ssl_init(&g_slots[i].ssl); - mbedtls_ssl_config_init(&g_slots[i].conf); - mbedtls_x509_crt_init(&g_slots[i].ca_cert); - mbedtls_x509_crt_init(&g_slots[i].client_cert); - mbedtls_pk_init(&g_slots[i].client_key); - mbedtls_ctr_drbg_init(&g_slots[i].ctr_drbg); - mbedtls_entropy_init(&g_slots[i].entropy); + for (uint8_t k = 0u; k < 2u; k++) { + g_slots[i].cfg[k].slot_idx = i; + config_init(&g_slots[i].cfg[k]); + } + g_slots[i].active = 0u; + g_slots[i].staged = false; mbedtls_net_init(&g_slots[i].net); - g_slots[i].configured = false; g_slots[i].connected = false; - g_slots[i].has_client_cert = false; g_slots[i].connecting = false; + g_slots[i].session_live = false; } g_pool_initialised = true; } @@ -561,3 +700,43 @@ const embediq_tls_ops_t *hal_tls_posix_ops(void) { return hal_tls_posix_ops_slot(0u); } + +#ifdef EMBEDIQ_PLATFORM_HOST +/** Install (fn) or remove (NULL) the hook connect_finish calls between + * releasing the slot and invoking on_complete (test use only). */ +void hal_tls__set_connect_finish_hook(void (*fn)(void *user_ctx)) +{ + atomic_store(&s_connect_finish_hook, fn); +} + +/** Slot introspection for configuration-lifetime tests (test use only). */ +typedef struct { + const void *ssl_conf; /* configuration the slot's ssl context references */ + const void *active_conf; /* the slot's active configuration */ + bool active_ca_present; /* the active configuration has a CA chain */ + bool staged; /* a validated replacement awaits the next connect */ + int live_configs; /* configuration sets built and not freed, this slot */ + int live_sessions; /* sessions live, all slots */ +} hal_tls_debug_t; + +void hal_tls__debug(uint8_t idx, hal_tls_debug_t *out) +{ + tls_slot_t *s = &g_slots[idx]; + pthread_mutex_lock(&s->mutex); + const mbedtls_ssl_config *active = &s->cfg[s->active].conf; + const mbedtls_x509_crt *ca = active->MBEDTLS_PRIVATE(ca_chain); + out->ssl_conf = mbedtls_ssl_context_get_config(&s->ssl); + out->active_conf = active; + out->active_ca_present = ca != NULL && ca->version != 0; + out->staged = s->staged; + out->live_configs = atomic_load(&s_live_configs[idx]); + out->live_sessions = atomic_load(&s_live_sessions); + pthread_mutex_unlock(&s->mutex); +} + +/** Number of sessions marked live and not yet released (test use only). */ +int hal_tls__live_sessions(void) +{ + return atomic_load(&s_live_sessions); +} +#endif diff --git a/hal/posix/peripherals/hal_flash_posix.c b/hal/posix/peripherals/hal_flash_posix.c index 65042fc..48a0521 100644 --- a/hal/posix/peripherals/hal_flash_posix.c +++ b/hal/posix/peripherals/hal_flash_posix.c @@ -46,6 +46,15 @@ static char s_path[PATH_BUF]; static char s_tmp_path[PATH_BUF]; +#ifdef EMBEDIQ_PLATFORM_HOST +/* Test-only: hal_flash_write calls attempted, and how many of the next + * calls fail with HAL_ERR_IO without touching the file. Written by the test + * thread between operations; the NVM persist lock serialises the writers. */ +static uint32_t s_write_calls = 0u; +static uint32_t s_fail_writes = 0u; +static uint32_t s_fail_reads = 0u; /* next n hal_flash_read calls fail */ +#endif + /** * Ensure parent directory exists (one level only). * Scans backwards for '/' and mkdir everything up to that point. @@ -124,6 +133,13 @@ int hal_flash_read(uint32_t addr, void *buf, size_t len) { if (!buf) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_FLASH, HAL_ERR_INVALID); return HAL_ERR_INVALID; } resolve_paths(); +#ifdef EMBEDIQ_PLATFORM_HOST + if (s_fail_reads > 0u) { + s_fail_reads--; + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_FLASH, HAL_ERR_IO); + return HAL_ERR_IO; + } +#endif /* Start with zeroed buffer (simulates erased flash). */ memset(buf, 0, len); @@ -144,6 +160,14 @@ int hal_flash_read(uint32_t addr, void *buf, size_t len) int hal_flash_write(uint32_t addr, const void *buf, size_t len) { if (!buf) { EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_FLASH, HAL_ERR_INVALID); return HAL_ERR_INVALID; } +#ifdef EMBEDIQ_PLATFORM_HOST + s_write_calls++; + if (s_fail_writes > 0u) { + s_fail_writes--; + EMBEDIQ_HAL_OBS_EMIT_ERROR(EMBEDIQ_HAL_SRC_FLASH, HAL_ERR_IO); + return HAL_ERR_IO; + } +#endif resolve_paths(); /* Read current file (may be empty). */ @@ -185,3 +209,23 @@ uint32_t hal_flash_page_size(void) { return HAL_FLASH_PAGE_SIZE; } + +#ifdef EMBEDIQ_PLATFORM_HOST +/** The next n hal_flash_write calls fail with HAL_ERR_IO (0 = none; test use only). */ +void hal_flash__fail_next_write(uint32_t n) +{ + s_fail_writes = n; +} + +/** The next n hal_flash_read calls fail with HAL_ERR_IO (0 = none; test use only). */ +void hal_flash__fail_next_read(uint32_t n) +{ + s_fail_reads = n; +} + +/** hal_flash_write calls attempted since the process started (test use only). */ +uint32_t hal_flash__write_calls(void) +{ + return s_write_calls; +} +#endif diff --git a/tests/lib/test_tls_posix.c b/tests/lib/test_tls_posix.c index 517a77d..2ff7a61 100644 --- a/tests/lib/test_tls_posix.c +++ b/tests/lib/test_tls_posix.c @@ -36,6 +36,22 @@ /* Use extern declarations — not HAL headers (boundary_checker rule for tests/) */ extern const embediq_tls_ops_t *hal_tls_posix_ops_slot(uint8_t idx); extern const embediq_tls_ops_t *hal_tls_posix_ops(void); +/* Host-only: called after a connect attempt releases its slot and before its + * on_complete, with that attempt's user_ctx (NULL removes it). */ +extern void hal_tls__set_connect_finish_hook(void (*fn)(void *user_ctx)); +/* Host-only: sessions whose mbedTLS context and socket are initialised and + * not yet released. */ +extern int hal_tls__live_sessions(void); +/* Host-only slot introspection; layout matches hal_tls_posix.c. */ +typedef struct { + const void *ssl_conf; /* configuration the slot's ssl context references */ + const void *active_conf; /* the slot's active configuration */ + bool active_ca_present; /* the active configuration has a CA chain */ + bool staged; /* a validated replacement awaits the next connect */ + int live_configs; /* configuration sets built and not freed, this slot */ + int live_sessions; /* sessions live, all slots */ +} hal_tls_debug_t; +extern void hal_tls__debug(uint8_t idx, hal_tls_debug_t *out); /* Test-local buffer sizes */ #define TLS_CMD_BUF_SIZE 512u /* shell command string for system() */ @@ -486,6 +502,460 @@ static void test_async_connect_mutex_ownership(void) "connect in flight: ops return BUSY, one failing on_complete, slot reusable"); } +/* --------------------------------------------------------------------------- + * test_async_completion_belongs_to_its_attempt + * + * Two attempts on one slot, each failing its TCP connect at once (the port is + * closed). The host hook parks the first attempt's worker after it has + * released the slot and before its on_complete; the second attempt is then + * accepted and completes. Released, the first worker delivers its own + * completion: each callback runs exactly once, with its own context. + * ------------------------------------------------------------------------- */ + +#define ATTEMPT_WAIT_MS 2000u +#define LATE_CALLBACK_WAIT_NS 100000000L /* 100 ms: no duplicate or late call */ + +typedef struct { + pthread_mutex_t mu; + int calls; + void *last_ctx; +} attempt_cb_t; + +static attempt_cb_t g_attempt_a = { PTHREAD_MUTEX_INITIALIZER, 0, NULL }; +static attempt_cb_t g_attempt_b = { PTHREAD_MUTEX_INITIALIZER, 0, NULL }; +static int g_ctx_a; /* distinct addresses: the attempts' contexts */ +static int g_ctx_b; + +static pthread_mutex_t g_park_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_park_cv = PTHREAD_COND_INITIALIZER; +static bool g_park_entered = false; +static bool g_park_release = false; + +static void record_attempt(attempt_cb_t *c, void *user_ctx) +{ + pthread_mutex_lock(&c->mu); + c->calls++; + c->last_ctx = user_ctx; + pthread_mutex_unlock(&c->mu); +} + +static void attempt_a_cb(embediq_tls_err_t result, void *user_ctx) +{ + (void)result; + record_attempt(&g_attempt_a, user_ctx); +} + +static void attempt_b_cb(embediq_tls_err_t result, void *user_ctx) +{ + (void)result; + record_attempt(&g_attempt_b, user_ctx); +} + +/* Parks only the attempt whose context is g_ctx_a. */ +static void park_attempt_a(void *user_ctx) +{ + if (user_ctx != &g_ctx_a) return; + pthread_mutex_lock(&g_park_mu); + g_park_entered = true; + pthread_cond_broadcast(&g_park_cv); + while (!g_park_release) pthread_cond_wait(&g_park_cv, &g_park_mu); + pthread_mutex_unlock(&g_park_mu); +} + +static int attempt_calls(attempt_cb_t *c, void **ctx) +{ + pthread_mutex_lock(&c->mu); + int n = c->calls; + if (ctx) *ctx = c->last_ctx; + pthread_mutex_unlock(&c->mu); + return n; +} + +static bool wait_attempt_calls(attempt_cb_t *c, int n, unsigned timeout_ms) +{ + for (unsigned waited = 0u; waited < timeout_ms; waited++) { + if (attempt_calls(c, NULL) >= n) return true; + struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; + nanosleep(&ts, NULL); + } + return attempt_calls(c, NULL) >= n; +} + +/* A loopback port with no listener: a TCP connect to it fails at once. */ +static uint16_t closed_port(void) +{ + uint16_t port = 0u; + int fd = open_silent_listener(&port); + if (fd < 0) return 0u; + close(fd); + return port; +} + +static void test_async_completion_belongs_to_its_attempt(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + free(ca); + uint16_t port = closed_port(); + + hal_tls__set_connect_finish_hook(park_attempt_a); + embediq_tls_err_t rc_a = ops->connect_async("127.0.0.1", port, "localhost", + attempt_a_cb, &g_ctx_a); + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += (long)(ATTEMPT_WAIT_MS / 1000u); + pthread_mutex_lock(&g_park_mu); + while (!g_park_entered + && pthread_cond_timedwait(&g_park_cv, &g_park_mu, &deadline) == 0) { + } + bool parked = g_park_entered; + pthread_mutex_unlock(&g_park_mu); + + embediq_tls_err_t rc_b = ops->connect_async("127.0.0.1", port, "localhost", + attempt_b_cb, &g_ctx_b); + bool b_done = wait_attempt_calls(&g_attempt_b, 1, ATTEMPT_WAIT_MS); + + pthread_mutex_lock(&g_park_mu); + g_park_release = true; + pthread_cond_broadcast(&g_park_cv); + pthread_mutex_unlock(&g_park_mu); + (void)wait_attempt_calls(&g_attempt_a, 1, ATTEMPT_WAIT_MS); + struct timespec ts_wait = { .tv_sec = 0, .tv_nsec = LATE_CALLBACK_WAIT_NS }; + nanosleep(&ts_wait, NULL); + hal_tls__set_connect_finish_hook(NULL); + alarm(0u); + + void *ctx_a = NULL, *ctx_b = NULL; + int calls_a = attempt_calls(&g_attempt_a, &ctx_a); + int calls_b = attempt_calls(&g_attempt_b, &ctx_b); + printf(" connect_a=%d parked_a=%d connect_b=%d b_completed=%d " + "callback_a=%d ctx_a_own=%d callback_b=%d ctx_b_own=%d\n", + (int)rc_a, (int)parked, (int)rc_b, (int)b_done, calls_a, + (int)(ctx_a == &g_ctx_a), calls_b, (int)(ctx_b == &g_ctx_b)); + ASSERT(configured && port != 0u && rc_a == EMBEDIQ_TLS_OK && parked + && rc_b == EMBEDIQ_TLS_OK && calls_a == 1 && ctx_a == &g_ctx_a + && calls_b == 1 && ctx_b == &g_ctx_b, + "each attempt's completion runs once, with its own context"); +} + +/* --------------------------------------------------------------------------- + * test_connect_async_refused_on_connected_slot + * + * After a successful handshake, a second connect_async on the slot returns + * EMBEDIQ_TLS_ERR_BUSY and the live session still sends and receives. + * ------------------------------------------------------------------------- */ + +static void test_connect_async_refused_on_connected_slot(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + free(ca); + + start_echo_server(TEST_PORT + 4, false); + cb_state_t st; cb_state_init(&st); + (void)ops->connect_async("127.0.0.1", TEST_PORT + 4, "localhost", connect_cb, &st); + bool connected = wait_for_connect(&st, 5000u) == EMBEDIQ_TLS_OK; + + static cb_state_t second; + cb_state_init(&second); + embediq_tls_err_t rc2 = ops->connect_async("127.0.0.1", TEST_PORT + 4, "localhost", + connect_cb, &second); + if (rc2 == EMBEDIQ_TLS_OK) { + (void)wait_for_connect(&second, 5000u); /* let an accepted attempt finish */ + } + + const uint8_t ping[] = "PING"; + embediq_tls_err_t rc_send = ops->send(ping, (uint16_t)sizeof(ping)); + uint8_t buf[TLS_TEST_RECV_BUF]; + int32_t n = ops->recv(buf, (uint16_t)sizeof(buf), 3000u); + bool echoed = n == (int32_t)sizeof(ping) && memcmp(buf, ping, sizeof(ping)) == 0; + + (void)ops->disconnect(); + stop_echo_server(); + alarm(0u); + + printf(" connected=%d second_connect=%d send=%d recv=%d echoed=%d\n", + (int)connected, (int)rc2, (int)rc_send, (int)n, (int)echoed); + ASSERT(configured && connected && rc2 == EMBEDIQ_TLS_ERR_BUSY + && rc_send == EMBEDIQ_TLS_OK && echoed, + "connected slot refuses connect_async; the live session keeps working"); +} + +/* --------------------------------------------------------------------------- + * A session closed by the peer is released before the slot connects again: + * recv reports the close, the slot is disconnected, the next connect_async + * is accepted and works, and after disconnect no session is left live. + * ------------------------------------------------------------------------- */ + +static bool echo_once(const embediq_tls_ops_t *ops) +{ + const uint8_t ping[] = "PING"; + if (ops->send(ping, (uint16_t)sizeof(ping)) != EMBEDIQ_TLS_OK) return false; + uint8_t buf[TLS_TEST_RECV_BUF]; + int32_t n = ops->recv(buf, (uint16_t)sizeof(buf), 3000u); + return n == (int32_t)sizeof(ping) && memcmp(buf, ping, sizeof(ping)) == 0; +} + +static void test_reconnect_after_peer_close_releases_session(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + free(ca); + + start_echo_server(TEST_PORT + 5, false); + cb_state_t first; cb_state_init(&first); + (void)ops->connect_async("127.0.0.1", TEST_PORT + 5, "localhost", connect_cb, &first); + bool connected = wait_for_connect(&first, 5000u) == EMBEDIQ_TLS_OK; + bool echoed_first = connected && echo_once(ops); + + stop_echo_server(); /* the peer closes the connection */ + uint8_t buf[TLS_TEST_RECV_BUF]; + int32_t rc_closed = ops->recv(buf, (uint16_t)sizeof(buf), 3000u); + const uint8_t ping[] = "PING"; + embediq_tls_err_t rc_send_closed = ops->send(ping, (uint16_t)sizeof(ping)); + + start_echo_server(TEST_PORT + 5, false); + cb_state_t second; cb_state_init(&second); + embediq_tls_err_t rc_again = ops->connect_async("127.0.0.1", TEST_PORT + 5, "localhost", + connect_cb, &second); + bool reconnected = rc_again == EMBEDIQ_TLS_OK + && wait_for_connect(&second, 5000u) == EMBEDIQ_TLS_OK; + bool echoed_again = reconnected && echo_once(ops); + + (void)ops->disconnect(); + stop_echo_server(); + alarm(0u); + int live = hal_tls__live_sessions(); + + printf(" connected=%d echoed=%d recv_after_close=%d send_after_close=%d " + "connect_again=%d reconnected=%d echoed_again=%d live_sessions=%d\n", + (int)connected, (int)echoed_first, (int)rc_closed, (int)rc_send_closed, + (int)rc_again, (int)reconnected, (int)echoed_again, live); + ASSERT(configured && connected && echoed_first + && rc_closed == (int32_t)EMBEDIQ_TLS_ERR_RECV + && rc_send_closed == EMBEDIQ_TLS_ERR_SEND + && rc_again == EMBEDIQ_TLS_OK && reconnected && echoed_again && live == 0, + "a peer-closed session is released before the slot reconnects"); +} + +/* --------------------------------------------------------------------------- + * configure() on a connected slot stores credentials for the next connect and + * leaves the live session alone: it still echoes, and disconnect releases it. + * ------------------------------------------------------------------------- */ + +static void test_configure_while_connected_keeps_session(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + + start_echo_server(TEST_PORT + 6, false); + cb_state_t st; cb_state_init(&st); + (void)ops->connect_async("127.0.0.1", TEST_PORT + 6, "localhost", connect_cb, &st); + bool connected = wait_for_connect(&st, 5000u) == EMBEDIQ_TLS_OK; + + embediq_tls_err_t rc_reconf = ops->configure(ca, NULL, NULL); + free(ca); + bool echoed = connected && echo_once(ops); + int live_connected = hal_tls__live_sessions(); + + (void)ops->disconnect(); + stop_echo_server(); + alarm(0u); + int live_after = hal_tls__live_sessions(); + + printf(" connected=%d reconfigure=%d echoed=%d live_while_connected=%d " + "live_after_disconnect=%d\n", (int)connected, (int)rc_reconf, (int)echoed, + live_connected, live_after); + ASSERT(configured && connected && rc_reconf == EMBEDIQ_TLS_OK && echoed + && live_connected == 1 && live_after == 0, + "configure on a connected slot leaves the session working"); +} + +/* --------------------------------------------------------------------------- + * Configuration lifetime: the configuration a live session references is + * never modified or freed by configure(); a validated replacement is staged + * and becomes active on the next connect_async(). + * ------------------------------------------------------------------------- */ + +#define ROTATIONS 20 +#define NOT_A_PEM "not a PEM certificate" + +/* Connect slot 0 to a fresh echo server on port; true once connected. */ +static bool connect_echo(const embediq_tls_ops_t *ops, int port, embediq_tls_err_t *result) +{ + start_echo_server(port, false); + cb_state_t st; cb_state_init(&st); + embediq_tls_err_t rc = ops->connect_async("127.0.0.1", (uint16_t)port, "localhost", + connect_cb, &st); + *result = (rc == EMBEDIQ_TLS_OK) ? wait_for_connect(&st, 5000u) : rc; + return *result == EMBEDIQ_TLS_OK; +} + +static void end_echo(const embediq_tls_ops_t *ops) +{ + (void)ops->disconnect(); + stop_echo_server(); +} + +static void test_reconfigure_while_connected_keeps_active_config_and_traffic(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + + embediq_tls_err_t r1, r2; + bool connected = connect_echo(ops, TEST_PORT + 7, &r1); + hal_tls_debug_t d0, d1, d2; + hal_tls__debug(0u, &d0); + embediq_tls_err_t rc_reconf = ops->configure(ca, NULL, NULL); + hal_tls__debug(0u, &d1); + bool echoed = connected && echo_once(ops); + end_echo(ops); + + bool reconnected = connect_echo(ops, TEST_PORT + 7, &r2) && echo_once(ops); + end_echo(ops); + hal_tls__debug(0u, &d2); + free(ca); + alarm(0u); + + bool same_conf = d1.ssl_conf == d0.ssl_conf && d0.ssl_conf == d0.active_conf; + printf(" connected=%d reconfigure=%d ssl_conf_unchanged=%d active_ca=%d staged=%d " + "live_configs_connected=%d echoed=%d reconnected=%d live_configs_after=%d " + "live_sessions=%d\n", (int)connected, (int)rc_reconf, (int)same_conf, + (int)d1.active_ca_present, (int)d1.staged, d1.live_configs, (int)echoed, + (int)reconnected, d2.live_configs, d2.live_sessions); + ASSERT(configured && connected && rc_reconf == EMBEDIQ_TLS_OK && same_conf + && d1.active_ca_present && d1.staged && d1.live_configs == 2 && echoed + && reconnected && d2.live_configs == 1 && d2.live_sessions == 0, + "reconfigure while connected stages; the active set and traffic are untouched"); +} + +static void test_failed_reconfigure_while_connected_leaves_active_intact(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + free(ca); + + embediq_tls_err_t r1, r2; + bool connected = connect_echo(ops, TEST_PORT + 8, &r1); + hal_tls_debug_t d0, d1; + hal_tls__debug(0u, &d0); + embediq_tls_err_t rc_bad = ops->configure(NOT_A_PEM, NULL, NULL); + hal_tls__debug(0u, &d1); + bool echoed = connected && echo_once(ops); + end_echo(ops); + + bool reconnected = connect_echo(ops, TEST_PORT + 8, &r2) && echo_once(ops); + end_echo(ops); + alarm(0u); + + printf(" connected=%d failed_configure=%d ssl_conf_unchanged=%d active_ca=%d " + "staged=%d echoed=%d reconnect_with_old=%d\n", (int)connected, (int)rc_bad, + (int)(d1.ssl_conf == d0.ssl_conf), (int)d1.active_ca_present, (int)d1.staged, + (int)echoed, (int)reconnected); + ASSERT(configured && connected && rc_bad != EMBEDIQ_TLS_OK + && d1.ssl_conf == d0.ssl_conf && d1.active_ca_present && !d1.staged + && echoed && reconnected, + "a rejected reconfigure leaves the active configuration and session intact"); +} + +static void test_staged_replacement_is_used_by_next_connection(void) +{ + alarm(HANG_GUARD_S); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + char *other = read_pem_file(CERT_DIR "/client.crt"); /* does not sign the server */ + ASSERT(ca != NULL && other != NULL, "CA and client certs must exist"); + if (!ca || !other) { free(ca); free(other); return; } + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool configured = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + + embediq_tls_err_t r1, r_other, r_good; + bool connected = connect_echo(ops, TEST_PORT + 9, &r1); + embediq_tls_err_t rc_other = ops->configure(other, NULL, NULL); + hal_tls_debug_t d1; + hal_tls__debug(0u, &d1); + bool echoed = connected && echo_once(ops); + end_echo(ops); + + (void)connect_echo(ops, TEST_PORT + 9, &r_other); /* uses the replacement */ + end_echo(ops); + + embediq_tls_err_t rc_good = ops->configure(ca, NULL, NULL); /* no session: applied now */ + hal_tls_debug_t d2; + hal_tls__debug(0u, &d2); + bool good = connect_echo(ops, TEST_PORT + 9, &r_good) && echo_once(ops); + end_echo(ops); + free(ca); free(other); + alarm(0u); + + printf(" connected=%d configure_other=%d staged=%d echoed=%d next_connect=%d " + "configure_good=%d staged_after=%d good_connect=%d\n", (int)connected, + (int)rc_other, (int)d1.staged, (int)echoed, (int)r_other, (int)rc_good, + (int)d2.staged, (int)good); + ASSERT(configured && connected && rc_other == EMBEDIQ_TLS_OK && d1.staged && echoed + && r_other == EMBEDIQ_TLS_ERR_CERT_VERIFY && rc_good == EMBEDIQ_TLS_OK + && !d2.staged && good, + "the staged replacement is the configuration of the next connection"); +} + +static void test_config_rotation_releases_everything_once(void) +{ + alarm(HANG_GUARD_S * 2u); + char *ca = read_pem_file(CERT_DIR "/ca.crt"); + ASSERT(ca != NULL, "CA cert must exist"); + if (!ca) return; + const embediq_tls_ops_t *ops = hal_tls_posix_ops_slot(0u); + bool ok = ops->configure(ca, NULL, NULL) == EMBEDIQ_TLS_OK; + + int connects = 0, echoes = 0; + for (int i = 0; i < ROTATIONS; i++) { + embediq_tls_err_t r; + if (connect_echo(ops, TEST_PORT + 10, &r)) connects++; + embediq_tls_err_t rc = (i % 2 == 0) ? ops->configure(ca, NULL, NULL) + : ops->configure(NOT_A_PEM, NULL, NULL); + ok = ok && ((i % 2 == 0) == (rc == EMBEDIQ_TLS_OK)); + if (echo_once(ops)) echoes++; + end_echo(ops); + } + embediq_tls_err_t r_final; + bool final_ok = connect_echo(ops, TEST_PORT + 10, &r_final); + end_echo(ops); + free(ca); + alarm(0u); + + hal_tls_debug_t d; + hal_tls__debug(0u, &d); + printf(" rotations=%d connects=%d echoes=%d configure_results_ok=%d final_connect=%d " + "live_sessions=%d live_configs=%d\n", ROTATIONS, connects, echoes, (int)ok, + (int)r_final, d.live_sessions, d.live_configs); + ASSERT(ok && connects == ROTATIONS && echoes == ROTATIONS && final_ok + && d.live_sessions == 0 && d.live_configs == 1, + "rotations, rejections and reconnects release every set and session once"); +} + /* --------------------------------------------------------------------------- * main * ------------------------------------------------------------------------- */ @@ -510,6 +980,14 @@ int main(void) test_slot_isolation(); #endif test_async_connect_mutex_ownership(); + test_async_completion_belongs_to_its_attempt(); + test_connect_async_refused_on_connected_slot(); + test_reconnect_after_peer_close_releases_session(); + test_configure_while_connected_keeps_session(); + test_reconfigure_while_connected_keeps_active_config_and_traffic(); + test_failed_reconfigure_while_connected_leaves_active_intact(); + test_staged_replacement_is_used_by_next_connection(); + test_config_rotation_releases_everything_once(); printf("\n%d/%d tests passed.\n", g_tests_run - g_tests_failed, g_tests_run); diff --git a/tests/unit/test_fb_nvm.c b/tests/unit/test_fb_nvm.c index 3cc68d0..d1d56a1 100644 --- a/tests/unit/test_fb_nvm.c +++ b/tests/unit/test_fb_nvm.c @@ -11,7 +11,7 @@ * the user's ~/.embediq/ directory. * * Run: ./build/tests/unit/test_fb_nvm - * Expected: "All 25 tests passed. (0 failed)" + * Expected: "All 31 tests passed. (0 failed)" * * @author Ritesh Anand * @company embediq.com | ritzylab.com @@ -22,6 +22,7 @@ #include "embediq_fb.h" #include "embediq_osal.h" #include "embediq_config.h" +#include "embediq_obs.h" #include "hal/hal_flash.h" #include @@ -45,6 +46,15 @@ extern void nvm__init_state(void); extern void nvm__load_state(void); /* triggers nvm_load() — test-only */ extern void nvm__set_persist_delay_ms(uint32_t ms); +/* Host-only flash failure injection (hal/posix/peripherals/hal_flash_posix.c). */ +extern void hal_flash__fail_next_write(uint32_t n); +extern void hal_flash__fail_next_read(uint32_t n); +extern uint32_t hal_flash__write_calls(void); + +/* Observatory host-only ring access (core/src/observatory/obs.c). */ +extern void obs__reset(void); +extern bool obs__ring_read(uint32_t idx, EmbedIQ_Event_t *out); + /* --------------------------------------------------------------------------- * Minimal test harness * ------------------------------------------------------------------------- */ @@ -526,6 +536,257 @@ static void test_concurrent_sets_persist_newest(void) "concurrent sets: flash holds the newest value, before and after flush"); } +/* --------------------------------------------------------------------------- + * Persist failures: returned to the caller, observed, retried until durable + * ------------------------------------------------------------------------- */ + +#define PERSIST_FAIL_SETS 70000u /* well past any 16-bit generation lag */ + +/* Reload the table from flash, as a power cycle does. */ +static void reload_from_flash(void) +{ + nvm__init_state(); + nvm__load_state(); +} + +static bool get_u32(const char *key, uint32_t *out) +{ + uint32_t len = sizeof(*out); + return embediq_nvm_get(key, out, &len) == EMBEDIQ_OK && len == sizeof(*out); +} + +static uint32_t count_persist_fail_events(void) +{ + uint32_t n = 0u; + EmbedIQ_Event_t e; + for (uint32_t i = 0u; obs__ring_read(i, &e); i++) { + if (e.event_type == EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL) n++; + } + return n; +} + +static uint32_t count_events(uint8_t type, uint8_t state) +{ + uint32_t n = 0u; + EmbedIQ_Event_t e; + for (uint32_t i = 0u; obs__ring_read(i, &e); i++) { + if (e.event_type == type && e.state_or_flag == state) n++; + } + return n; +} + +#define BLOB_STATE_READ_FAIL 3u /* CFG_BLOB_INVALID state: flash could not be read */ + +/* test_unreadable_flash_refuses_to_persist_until_flush_reloads + * k persisted. A boot whose flash read fails: RAM empty, one CFG_BLOB_INVALID + * (read failure). delete(k) and set(j) → ERR, each with NVM_PERSIST_FAIL state + * FLASH_UNREADABLE, and nothing is written. flush with the read still failing + * → ERR. Read recovers: flush → OK and RAM holds k, not j; after a reload k is + * present and j absent. + */ +static void test_unreadable_flash_refuses_to_persist_until_flush_reloads(void) +{ + setup(); + uint32_t k = 0x4Bu, j = 0x4Au, got = 0u; + embediq_err_t rc_seed = embediq_nvm_set("persist.kept", &k, sizeof(k)); + obs__reset(); + embediq_obs_set_transport(EMBEDIQ_OBS_TRANSPORT_TCP); /* ring only, no output */ + + nvm__init_state(); + hal_flash__fail_next_read(UINT32_MAX); + nvm__load_state(); + bool empty = !get_u32("persist.kept", &got); + uint32_t read_fails = count_events(EMBEDIQ_OBS_EVT_CFG_BLOB_INVALID, BLOB_STATE_READ_FAIL); + + uint32_t before = hal_flash__write_calls(); + embediq_err_t rc_del = embediq_nvm_delete("persist.kept"); + embediq_err_t rc_set = embediq_nvm_set("persist.unsaved", &j, sizeof(j)); + embediq_err_t rc_flush = embediq_nvm_flush(); + uint32_t writes = hal_flash__write_calls() - before; + uint32_t refused = count_events(EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL, + (uint8_t)EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE); + + hal_flash__fail_next_read(0u); + embediq_err_t rc_reload = embediq_nvm_flush(); + bool k_in_ram = get_u32("persist.kept", &got) && got == k; + bool j_in_ram = get_u32("persist.unsaved", &got); + + reload_from_flash(); + bool k_after = get_u32("persist.kept", &got) && got == k; + bool j_after = get_u32("persist.unsaved", &got); + embediq_obs_set_transport(EMBEDIQ_OBS_TRANSPORT_NULL); + + printf(" seed=%d empty=%d read_fail_events=%u delete=%d set=%d flush_unreadable=%d " + "flash_writes=%u refused_events=%u | flush_recovered=%d k_in_ram=%d j_in_ram=%d " + "k_after_reload=%d j_after_reload=%d\n", + (int)rc_seed, (int)empty, (unsigned)read_fails, (int)rc_del, (int)rc_set, + (int)rc_flush, (unsigned)writes, (unsigned)refused, (int)rc_reload, + (int)k_in_ram, (int)j_in_ram, (int)k_after, (int)j_after); + ASSERT(rc_seed == EMBEDIQ_OK && empty && read_fails == 1u + && rc_del == EMBEDIQ_ERR && rc_set == EMBEDIQ_ERR && rc_flush == EMBEDIQ_ERR + && writes == 0u && refused == 3u + && rc_reload == EMBEDIQ_OK && k_in_ram && !j_in_ram && k_after && !j_after, + "unreadable flash is never overwritten; flush re-reads it and restores the table"); +} + +/* test_set_reports_persist_failure + * A set whose flash write fails returns EMBEDIQ_ERR and emits one + * NVM_PERSIST_FAIL; the value is readable from RAM. The next set (write + * succeeds) returns OK, and a reload shows both values. + */ +static void test_set_reports_persist_failure(void) +{ + setup(); + obs__reset(); + embediq_obs_set_transport(EMBEDIQ_OBS_TRANSPORT_TCP); /* ring only, no output */ + uint32_t a = 0xA1u, b = 0xB2u, got = 0u; + + hal_flash__fail_next_write(1u); + embediq_err_t rc_fail = embediq_nvm_set("persist.fail_a", &a, sizeof(a)); + uint32_t events = count_persist_fail_events(); + bool in_ram = get_u32("persist.fail_a", &got) && got == a; + + embediq_err_t rc_ok = embediq_nvm_set("persist.fail_b", &b, sizeof(b)); + reload_from_flash(); + uint32_t got_a = 0u, got_b = 0u; + bool both = get_u32("persist.fail_a", &got_a) && got_a == a + && get_u32("persist.fail_b", &got_b) && got_b == b; + embediq_obs_set_transport(EMBEDIQ_OBS_TRANSPORT_NULL); + + printf(" failed_set=%d persist_fail_events=%u in_ram=%d next_set=%d " + "both_after_reload=%d\n", (int)rc_fail, (unsigned)events, (int)in_ram, + (int)rc_ok, (int)both); + ASSERT(rc_fail == EMBEDIQ_ERR && events == 1u && in_ram && rc_ok == EMBEDIQ_OK && both, + "failed persist returned and observed; the next persist carries the change"); +} + +/* test_provision_retry_reestablishes_durability + * Seed; flash fails: provision → ERR, flush → ERR. Flash recovers: provision + * again → OK with exactly one flash write; after a reload the identity is + * there. + */ +static void test_provision_retry_reestablishes_durability(void) +{ + setup(); + uint32_t seed = 1u; + (void)embediq_nvm_set("persist.seed", &seed, sizeof(seed)); + + hal_flash__fail_next_write(UINT32_MAX); + embediq_err_t rc_prov = embediq_nvm_provision("factory.identity", "device1"); + embediq_err_t rc_flush = embediq_nvm_flush(); + hal_flash__fail_next_write(0u); + + uint32_t before = hal_flash__write_calls(); + embediq_err_t rc_retry = embediq_nvm_provision("factory.identity", "device1"); + uint32_t retry_writes = hal_flash__write_calls() - before; + + reload_from_flash(); + char id[EMBEDIQ_NVM_VAL_MAX]; + uint32_t len = sizeof(id); + embediq_err_t rc_get = embediq_nvm_get("factory.identity", id, &len); + bool identity = rc_get == EMBEDIQ_OK && len == 7u && memcmp(id, "device1", 7u) == 0; + + printf(" failed_provision=%d failed_flush=%d retry=%d retry_flash_writes=%u " + "identity_after_reload=%d\n", (int)rc_prov, (int)rc_flush, (int)rc_retry, + (unsigned)retry_writes, (int)identity); + ASSERT(rc_prov == EMBEDIQ_ERR && rc_flush == EMBEDIQ_ERR && rc_retry == EMBEDIQ_OK + && retry_writes == 1u && identity, + "provision retried after a failed persist writes once and is durable"); +} + +/* test_delete_retry_reestablishes_durability + * k persisted; flash fails: delete k → ERR, and again → ERR. Flash recovers: + * delete k again → OK with exactly one flash write; after a reload k is + * absent. Deleting a key that never existed, with the absence already on + * flash → OK with no flash write. + */ +static void test_delete_retry_reestablishes_durability(void) +{ + setup(); + embediq_err_t rc_seed = embediq_nvm_set("credential", "secret", 6u); + + hal_flash__fail_next_write(UINT32_MAX); + embediq_err_t rc_del = embediq_nvm_delete("credential"); + embediq_err_t rc_still = embediq_nvm_delete("credential"); + hal_flash__fail_next_write(0u); + + uint32_t before = hal_flash__write_calls(); + embediq_err_t rc_retry = embediq_nvm_delete("credential"); + uint32_t retry_writes = hal_flash__write_calls() - before; + + before = hal_flash__write_calls(); + embediq_err_t rc_never = embediq_nvm_delete("never.stored"); + uint32_t never_writes = hal_flash__write_calls() - before; + + reload_from_flash(); + char out[EMBEDIQ_NVM_VAL_MAX]; + uint32_t len = sizeof(out); + bool absent = embediq_nvm_get("credential", out, &len) != EMBEDIQ_OK; + + printf(" seed=%d failed_delete=%d failed_again=%d retry=%d retry_flash_writes=%u " + "absent_after_reload=%d | never_stored=%d never_flash_writes=%u\n", + (int)rc_seed, (int)rc_del, (int)rc_still, (int)rc_retry, + (unsigned)retry_writes, (int)absent, (int)rc_never, (unsigned)never_writes); + ASSERT(rc_seed == EMBEDIQ_OK && rc_del == EMBEDIQ_ERR && rc_still == EMBEDIQ_ERR + && rc_retry == EMBEDIQ_OK && retry_writes == 1u && absent + && rc_never == EMBEDIQ_OK && never_writes == 0u, + "delete retried after a failed persist writes once; an absence on flash needs no write"); +} + +/* test_flush_reports_failure + * flush returns ERR while the write fails and OK once it succeeds. + */ +static void test_flush_reports_failure(void) +{ + setup(); + hal_flash__fail_next_write(1u); + embediq_err_t rc_fail = embediq_nvm_flush(); + embediq_err_t rc_ok = embediq_nvm_flush(); + printf(" failed_flush=%d recovered_flush=%d\n", (int)rc_fail, (int)rc_ok); + ASSERT(rc_fail == EMBEDIQ_ERR && rc_ok == EMBEDIQ_OK, + "flush returns the flash write result"); +} + +/* test_skip_rule_never_skips_an_unpersisted_change + * (1) 70,000 sets of one key with the flash failing, then one set after it + * recovers: OK, exactly one flash write, the last value after a reload. + * (2) k=1 persisted; k=2 fails; a set of a different key afterwards persists + * the whole table: after a reload k == 2. + */ +static void test_skip_rule_never_skips_an_unpersisted_change(void) +{ + setup(); + hal_flash__fail_next_write(UINT32_MAX); + for (uint32_t i = 0u; i < PERSIST_FAIL_SETS; i++) { + (void)embediq_nvm_set("persist.lag", &i, sizeof(i)); + } + hal_flash__fail_next_write(0u); + uint32_t last = PERSIST_FAIL_SETS; + uint32_t before = hal_flash__write_calls(); + embediq_err_t rc_last = embediq_nvm_set("persist.lag", &last, sizeof(last)); + uint32_t writes = hal_flash__write_calls() - before; + reload_from_flash(); + uint32_t got = 0u; + bool lag_ok = get_u32("persist.lag", &got) && got == last; + + uint32_t k1 = 1u, k2 = 2u, other = 3u; + embediq_err_t rc_k1 = embediq_nvm_set("persist.k", &k1, sizeof(k1)); + hal_flash__fail_next_write(1u); + embediq_err_t rc_k2 = embediq_nvm_set("persist.k", &k2, sizeof(k2)); + embediq_err_t rc_other = embediq_nvm_set("persist.other", &other, sizeof(other)); + reload_from_flash(); + uint32_t k = 0u; + bool carried = get_u32("persist.k", &k) && k == k2; + + printf(" after_%u_failed_sets: set=%d flash_writes=%u reloaded_last=%d | " + "k1=%d k2=%d other=%d k_after_reload=%u\n", (unsigned)PERSIST_FAIL_SETS, + (int)rc_last, (unsigned)writes, (int)lag_ok, (int)rc_k1, (int)rc_k2, + (int)rc_other, (unsigned)k); + ASSERT(rc_last == EMBEDIQ_OK && writes == 1u && lag_ok && rc_k1 == EMBEDIQ_OK + && rc_k2 == EMBEDIQ_ERR && rc_other == EMBEDIQ_OK && carried, + "a change not on flash is never skipped; a later persist carries it"); +} + /* --------------------------------------------------------------------------- * main * ------------------------------------------------------------------------- */ @@ -557,6 +818,14 @@ int main(void) test_nvm_get_does_not_wait_for_persist(); test_concurrent_sets_persist_newest(); + /* Persist failures */ + test_set_reports_persist_failure(); + test_provision_retry_reestablishes_durability(); + test_delete_retry_reestablishes_durability(); + test_unreadable_flash_refuses_to_persist_until_flush_reloads(); + test_flush_reports_failure(); + test_skip_rule_never_skips_an_unpersisted_change(); + printf("\n"); if (g_tests_failed == 0) { printf("All %d tests passed. (0 failed)\n", g_tests_run); diff --git a/tests/unit/test_fb_provisioning.c b/tests/unit/test_fb_provisioning.c index 19ab9ae..40b62b0 100644 --- a/tests/unit/test_fb_provisioning.c +++ b/tests/unit/test_fb_provisioning.c @@ -51,8 +51,13 @@ extern uint16_t fb_provisioning__test_pub_id(int idx); extern uint8_t fb_provisioning__test_pub_status(int idx); extern int fb_provisioning__test_fault_count(void); -/* fb_nvm host-only state reset (declared in fbs/drivers/fb_nvm.c). */ +/* fb_nvm host-only state reset and reload (declared in fbs/drivers/fb_nvm.c). */ extern void nvm__init_state(void); +extern void nvm__load_state(void); + +/* Host-only flash failure injection (hal/posix/peripherals/hal_flash_posix.c). */ +extern void hal_flash__fail_next_write(uint32_t n); +extern uint32_t hal_flash__write_calls(void); /* --------------------------------------------------------------------------- * Mock provisioning ops table — outcomes controlled per test via globals. @@ -293,6 +298,42 @@ static void test_provisioning_state_advances_to_customer_provisioned(void) ASSERT(id[0] != '\0', "mqtt.client_id set during provisioning"); } +/* --------------------------------------------------------------------------- + * TEST 8 — identity provisioning is retried until it is on flash + * The flash fails during the first startup: provisioning the identity fails + * and the FB reports a fault. A second startup with the flash recovered + * provisions it again without a fault, and after a reload (power cycle) the + * identity is there. + * ------------------------------------------------------------------------- */ +static void test_identity_provision_retried_until_durable(void) +{ + setup(); + g_mock_identity_cn = "test-device-002"; + + hal_flash__fail_next_write(UINT32_MAX); + fb_provisioning__run_startup_test(); + int first_faults = fb_provisioning__test_fault_count(); + hal_flash__fail_next_write(0u); + + fb_provisioning__test_reset(); + uint32_t before = hal_flash__write_calls(); + fb_provisioning__run_startup_test(); + int second_faults = fb_provisioning__test_fault_count(); + uint32_t writes = hal_flash__write_calls() - before; + + nvm__init_state(); + nvm__load_state(); + char id[EMBEDIQ_NVM_VAL_SIZE] = {0}; + (void)embediq_cfg_get_str("mqtt.client_id", id, sizeof(id), ""); + bool durable = strcmp(id, "test-device-002") == 0; + + printf(" first_startup_faults=%d second_startup_faults=%d " + "second_startup_flash_writes=%u identity_after_reload=%d\n", + first_faults, second_faults, (unsigned)writes, (int)durable); + ASSERT(first_faults >= 1 && second_faults == 0 && writes >= 1u && durable, + "failed identity persist faults the FB; the next startup makes it durable"); +} + /* --------------------------------------------------------------------------- * Entry point * ------------------------------------------------------------------------- */ @@ -306,6 +347,7 @@ int main(void) test_provisioning_factory_reset_clears_nvm_keys(); test_provisioning_factory_reset_publishes_msg(); test_provisioning_state_advances_to_customer_provisioned(); + test_identity_provision_retried_until_durable(); printf("\n"); if (g_tests_failed == 0) {