tls/nvm: completion belongs to its attempt; persist failures returned and retried

ARCHITECTURE.md

@@ -2182,7 +2182,7 @@ The family is derived in the decoder with zero runtime overhead:
 | 0x30–0x3F | STATE | LIFECYCLE(0x30), FSM_TRANS(0x31) |
 | 0x40–0x4F | RESOURCE | LIB_INIT(0x40), LIB_DEINIT(0x41) — Phase 2 activates XOBS-3 (queue depth, stack high-water) |
 | 0x50–0x5F | TIMING | Phase 2 activates XOBS-4 (watchdog checkin, message latency) |
-| 0x60–0x6F | FAULT | FAULT(0x60), SECURITY_INCIDENT(0x61), AUTH_EVENT(0x62), FAULT_CLEARED(0x63), LIB_ALLOC_FAIL(0x64), LIB_TIMEOUT(0x65), OSAL_FAULT(0x66 — Phase 2), HAL_FAULT(0x67 — Phase 2) |
+| 0x60–0x6F | FAULT | FAULT(0x60), SECURITY_INCIDENT(0x61), AUTH_EVENT(0x62), FAULT_CLEARED(0x63), LIB_ALLOC_FAIL(0x64), LIB_TIMEOUT(0x65), OSAL_FAULT(0x66 — Phase 2), HAL_FAULT(0x67 — Phase 2), NVM_PERSIST_FAIL(0x6B) |
 | 0x70–0x7F | FUNCTION | TEST_START(0x70), TEST_END(0x71) |
 | 0x80–0x8F | AI (Phase 3) | RESERVED — Phase 3 AI family. Do not allocate. Taxonomy defined after Phase 1 AI production data. |
 | 0x90–0xFF | VENDOR | Community and vendor extension range — 7 full 16-slot sub-bands. Never allocated by the framework. |

core/include/embediq_cfg.h

@@ -23,9 +23,13 @@
  * no global mutable state of its own — the getter / setter calls pass
  * straight through to embediq_nvm_get / _set / _delete.
  *
- * Durability: fb_nvm persists at its own cadence. Callers that must
- * guarantee a value is flushed to storage before continuing should call
- * embediq_nvm_flush() explicitly (for example, before MSG_SYS_OTA_READY).
+ * Durability: every mutation persists synchronously and returns whether the
+ * change reached flash; embediq_nvm_flush() rewrites the current table (use
+ * it after a failed mutation to retry). If flash cannot be read at boot the
+ * store starts empty and every setter and delete returns false, leaving the
+ * stored values unchanged, until embediq_nvm_flush() re-reads it
+ * successfully; an invalid blob (bad magic or CRC) is replaced by the first
+ * successful write.
  *
  * @author  Ritesh Anand
  * @company embediq.com | ritzylab.com
@@ -71,7 +75,10 @@ size_t   embediq_cfg_get_str  (const char *key, char *buf,
 
 /* ---------------------------------------------------------------------------
  * Typed setters — persist the value under `key`. Return true on success,
- * false on NVM error, over-long key, or store full.
+ * false on NVM error, over-long key, or store full. false also when the
+ * value could not be persisted; in that case the new value IS in RAM
+ * (getters return it) and reaches flash with the next successful persist —
+ * unlike every other false, which leaves the old value.
  *
  * Calling context: any thread. A setter writes to flash and may wait for
  * another writer's persist in flight (see embediq_nvm.h).
@@ -84,8 +91,14 @@ bool     embediq_cfg_set_bool (const char *key, bool     value);
 bool     embediq_cfg_set_str  (const char *key, const char *value);
 
 /**
- * Remove the entry for the given key. Returns true on success or if the
- * key was already absent (fb_nvm makes delete idempotent).
+ * Remove the entry for the given key. Returns true on success.
+ * Idempotent: deleting an absent key returns true only once the current
+ * table — which includes the absence of that key — is on flash; false if the
+ * table could not be written, including when the pending change is another
+ * key's.
+ * false also when the removal could not be persisted; in that case the key
+ * IS gone from RAM and the removal reaches flash with the next successful
+ * persist.
  *
  * Calling context: any thread. May wait for a persist in flight.
  */

core/include/embediq_nvm.h

@@ -6,7 +6,9 @@
  * The ESP32 implementation uses NVS (Non-Volatile Storage).
  *
  * All operations are synchronous from the caller's perspective.
- * fb_nvm owns internal write buffering and flush scheduling.
+ * Every mutation persists synchronously and returns whether the change
+ * reached flash; embediq_nvm_flush() rewrites the current table (use it
+ * after a failed mutation to retry).
  *
  * Concurrency — every function is callable from any thread. fb_nvm uses two
  * locks with one fixed order (persist lock, then table lock):
@@ -19,13 +21,24 @@
  *     (a later writer persisted a newer table) skips its write, so an older
  *     table never overwrites a newer one.
  * embediq_nvm_get() takes only the table lock: it never waits for a persist
- * in flight. Writes return once their change (or a newer table) is on flash
- * and may wait for another writer's persist.
+ * in flight. Writes return EMBEDIQ_OK once their change (or a newer table)
+ * is on flash, EMBEDIQ_ERR if the flash write failed, and may wait for
+ * another writer's persist.
+ * Loading: fb_nvm's nvm_load() runs only at boot, before any other caller
+ * exists (and from the host test hook with no concurrent callers). It
+ * replaces the table with the flash contents; a change made concurrently by
+ * another thread would be discarded and its persist skipped. Reloading at
+ * runtime is not supported.
+ * If flash cannot be read at boot the store starts empty and refuses to
+ * persist (every mutation returns EMBEDIQ_ERR with an NVM_PERSIST_FAIL event,
+ * state FLASH_UNREADABLE, and leaves the table unchanged) until
+ * embediq_nvm_flush() re-reads it successfully. An invalid blob (bad magic or
+ * CRC) is replaced by the first successful write.
  *
  * Constraints (v1):
  *   - Keys are NUL-terminated strings up to EMBEDIQ_NVM_KEY_MAX bytes.
  *   - Values are raw byte arrays up to EMBEDIQ_NVM_VAL_MAX bytes.
- *   - No transactions. embediq_nvm_flush() is the durability boundary.
+ *   - No transactions.
  *   - embediq_nvm_reset() erases ALL keys — use with care.
  *
  * I-01: Compiles standalone with zero OSAL or BSP dependencies.
@@ -72,6 +85,9 @@ extern "C" {
  * @param val   Pointer to value bytes.
  * @param len   Number of bytes in val; must be <= EMBEDIQ_NVM_VAL_MAX.
  * @return EMBEDIQ_OK on success, EMBEDIQ_ERR on invalid args or store error.
+ * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps
+ * the change and the next successful persist (any later write, or flush)
+ * carries it.
  *
  * Calling context: any thread. May wait for a persist in flight.
  */
@@ -92,7 +108,13 @@ embediq_err_t embediq_nvm_get(const char *key, void *val, uint32_t *len);
 
 /**
  * Remove the entry for the given key.
- * If the key does not exist, returns EMBEDIQ_OK (idempotent).
+ * Idempotent: deleting an absent key returns EMBEDIQ_OK only once the
+ * current table — which includes the absence of that key — is on flash;
+ * EMBEDIQ_ERR if the table could not be written, including when the pending
+ * change is another key's.
+ * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps
+ * the change and the next successful persist (any later write, or flush)
+ * carries it.
  *
  * Calling context: any thread. May wait for a persist in flight.
  */
@@ -101,7 +123,14 @@ embediq_err_t embediq_nvm_delete(const char *key);
 /**
  * embediq_nvm_provision — write-once provisioning write.
  * Semantics: if key is absent or empty, write value and return EMBEDIQ_OK.
- *            if key is already set (non-empty), return EMBEDIQ_OK (idempotent).
+ *            Idempotent: a second call with the key already provisioned does
+ *            not change the value; it returns EMBEDIQ_OK only once the
+ *            current table — which includes that value — is on flash;
+ *            EMBEDIQ_ERR if the table could not be written, including when
+ *            the pending change is another key's.
+ * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps
+ * the change and the next successful persist (any later write, or flush)
+ * carries it.
  * This is the ONLY authorized write path for CFG_MUT_FACTORY keys at runtime.
  * IEC 62443 commissioning immutability: operator cannot overwrite after provisioning.
  * fb_provisioning calls this at UNMANUFACTURED -> MANUFACTURED transition only.
@@ -116,9 +145,17 @@ embediq_err_t embediq_nvm_provision(const char *key, const char *value);
  * (see docs/architecture/lifecycle.md — OTA shutdown contract).
  *
  * @return EMBEDIQ_OK if all data is safely persisted, EMBEDIQ_ERR on I/O error.
+ * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps
+ * the change and the next successful persist (any later write, or flush)
+ * carries it.
  *
- * Calling context: any thread. Always writes the current table; may wait for
- * a persist in flight.
+ * If flash was unreadable at boot, flush re-reads it instead of writing: on
+ * success the flash contents replace the (empty) table and persisting
+ * resumes; on failure it returns EMBEDIQ_ERR with an NVM_PERSIST_FAIL event.
+ * flush is the only call that performs this retry.
+ *
+ * Calling context: any thread. Otherwise always writes the current table;
+ * may wait for a persist in flight.
  */
 embediq_err_t embediq_nvm_flush(void);
 
@@ -127,6 +164,9 @@ embediq_err_t embediq_nvm_flush(void);
  * Typically called during factory reset only.
  *
  * @return EMBEDIQ_OK on success, EMBEDIQ_ERR on store error.
+ * Returns EMBEDIQ_ERR if the flash write failed; the in-memory table keeps
+ * the change and the next successful persist (any later write, or flush)
+ * carries it.
  *
  * Calling context: any thread. May wait for a persist in flight.
  */

core/include/embediq_obs.h

@@ -268,7 +268,10 @@ extern "C" {
                                                           source_fb_id: fb_nvm endpoint index.
                                                           state_or_flag: 0=magic_fail
                                                                          1=crc_fail
-                                                                         2=major_version_mismatch.
+                                                                         2=major_version_mismatch
+                                                                         3=flash_read_fail (the
+                                                                           store refuses to persist
+                                                                           until flush re-reads it).
                                                           msg_id: 0 (no message context). */
 #define EMBEDIQ_OBS_EVT_CFG_MUTABILITY_REJECT 0x69u /**< Fleet push attempted to overwrite
                                                           a factory or user-class config key.
@@ -279,7 +282,23 @@ extern "C" {
                                                             rejected key (0=factory 2=user).
                                                           msg_id: low 16 bits of key name CRC32
                                                             (key name itself is never logged). */
-/* 0x6A–0x6F reserved */
+/* 0x6A reserved: STREAM_FAULT (ARCHITECTURE.md "Streaming Data Plane") */
+#define EMBEDIQ_OBS_EVT_NVM_PERSIST_FAIL      0x6Bu /**< NVM table write to flash failed.
+                                                          The change stays in RAM; the next
+                                                          successful persist carries it.
+                                                          One event per failed write.
+                                                          source_fb_id: 0. target_fb_id: 0.
+                                                          state_or_flag: HAL return value
+                                                            truncated to uint8_t (HAL errors
+                                                            are -127..-1: 0x81..0xFF), or
+                                                            EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE.
+                                                          msg_id: low 16 bits of the table
+                                                            generation that failed to persist. */
+/** NVM_PERSIST_FAIL state: nothing was written because flash could not be
+ *  read at boot (contents unknown); the change was refused and RAM is
+ *  unchanged. 0x80 is outside the truncated HAL error range. */
+#define EMBEDIQ_NVM_PERSIST_FLASH_UNREADABLE  0x80u
+/* 0x6C–0x6F reserved */
 
 /* FUNCTION family (0x70–0x7F) --------------------------------------------- */
 

core/include/ops/embediq_tls.h

@@ -13,6 +13,10 @@
  *         embediq_tls_err_t added; embediq_bus.h dependency removed.
  *   v2 (additive) — EMBEDIQ_TLS_ERR_BUSY (-12): returned by every op on a slot while a
  *         connect_async() is in flight. No field change; EMBEDIQ_TLS_OPS_VERSION stays 2.
+ *   v2 (additive) — connect_async() on a connected slot also returns
+ *         EMBEDIQ_TLS_ERR_BUSY. No field change; EMBEDIQ_TLS_OPS_VERSION stays 2.
+ *   v2 (additive) — configure() stages while connected. No field change;
+ *         EMBEDIQ_TLS_OPS_VERSION stays 2.
  *
  * Context safety: INSTANCE-SAFE — each ops table slot represents one
  * independent TLS connection context. Concurrent use of distinct slots
@@ -21,6 +25,14 @@
  * slot — configure, a second connect_async, disconnect, send, recv —
  * returns EMBEDIQ_TLS_ERR_BUSY without blocking. The slot accepts ops again
  * once the connect has completed (no later than on_complete being invoked).
+ * The completion callback and user context belong to the attempt that
+ * started them; a later connect_async() on the same slot cannot redirect or
+ * drop an earlier completion. A slot that is connected or connecting refuses
+ * connect_async() with EMBEDIQ_TLS_ERR_BUSY: disconnect first.
+ * After the peer closes the connection (close_notify or an orderly TCP close)
+ * recv() returns EMBEDIQ_TLS_ERR_RECV and the session's resources are
+ * released before recv() returns; a connection reset or other transport
+ * error leaves the slot connected and is released by disconnect().
  *
  * @author  Ritesh Anand
  * @company embediq.com | ritzylab.com
@@ -63,7 +75,7 @@ typedef int32_t embediq_tls_err_t;
 #define EMBEDIQ_TLS_ERR_NO_RESOURCES      ((embediq_tls_err_t) -9)  /**< slot index out of range */
 #define EMBEDIQ_TLS_ERR_SEND              ((embediq_tls_err_t)-10)  /**< send() underlying write failed */
 #define EMBEDIQ_TLS_ERR_RECV              ((embediq_tls_err_t)-11)  /**< recv() underlying read failed */
-#define EMBEDIQ_TLS_ERR_BUSY              ((embediq_tls_err_t)-12)  /**< slot has a connect_async() in flight */
+#define EMBEDIQ_TLS_ERR_BUSY              ((embediq_tls_err_t)-12)  /**< connect_async() in flight; or connect_async() on a connected slot */
 
 /* ---------------------------------------------------------------------------
  * Async connect callback
@@ -105,10 +117,15 @@ typedef struct {
      * Must be called before the first connect_async(). Calling connect_async()
      * without a prior configure() returns EMBEDIQ_TLS_ERR_NOT_CONFIGURED.
      *
-     * May be called again while connected — the new credentials are stored
-     * and applied on the NEXT connect_async() after disconnect(). The current
-     * session is unaffected. This enables certificate rotation without
-     * connection downtime.
+     * May be called while connected: the replacement is validated and
+     * staged; the current session is unaffected — the active session and the
+     * configuration it references are untouched; the replacement becomes
+     * active on the next connect_async(). A rejected replacement (bad PEM, no
+     * CA) leaves the active configuration and any session intact and returns
+     * the error; a previously staged replacement is discarded. With no live
+     * session a validated replacement becomes active at once. This enables
+     * certificate rotation without connection downtime.
+     * Memory: the HAL holds at most two configurations per slot.
      *
      * ca_cert_pem:     NUL-terminated PEM string of the CA certificate used to
      *                  verify the server. Must not be NULL.
@@ -143,6 +160,8 @@ typedef struct {
      * user_ctx:    passed unchanged to on_complete. May be NULL.
      *
      * Returns EMBEDIQ_TLS_OK if the operation was successfully initiated.
+     * Returns EMBEDIQ_TLS_ERR_BUSY if the slot is connected or connecting
+     * (disconnect first).
      * on_complete is NOT called on error return — the caller must check the
      * return value and handle it directly.
      */