[vpp] Dedicated binary-API socket and thread for asynchronous VPP events

vslib/vpp/SwitchVpp.cpp

@@ -1303,6 +1303,8 @@ sai_status_t SwitchVpp::create(
 {
     SWSS_LOG_ENTER();
 
+    serviceDeferredOperStatusResync();
+
     if (object_type == SAI_OBJECT_TYPE_DEBUG_COUNTER)
     {
         sai_object_id_t object_id;
@@ -1658,6 +1660,8 @@ sai_status_t SwitchVpp::remove(
 {
     SWSS_LOG_ENTER();
 
+    serviceDeferredOperStatusResync();
+
     if (object_type == SAI_OBJECT_TYPE_DEBUG_COUNTER)
     {
         sai_object_id_t objectId;
@@ -2017,6 +2021,8 @@ sai_status_t SwitchVpp::set(
 {
     SWSS_LOG_ENTER();
 
+    serviceDeferredOperStatusResync();
+
     if (objectType == SAI_OBJECT_TYPE_PORT)
     {
         sai_object_id_t objectId;

vslib/vpp/SwitchVpp.h

@@ -17,6 +17,7 @@
 #include <map>
 #include <unordered_map>
 #include <mutex>
+#include <atomic>
 #include <chrono>
 #include <functional>
 #include <queue>
@@ -1155,6 +1156,17 @@ namespace saivs
 
             void vppProcessEvents ();
 
+            void resyncPortOperStatus();
+
+            // Run a deferred oper-status resync on the command thread if the
+            // event thread has requested one. resyncPortOperStatus() issues VPP
+            // binary-API calls (interface_get_state) which allocate on VPP's
+            // non-thread-safe clib heap; running them on the background event
+            // thread races the command thread's clib allocations and crashes
+            // (os_panic in clib_mem_heap_realloc_aligned). So the event thread
+            // only flags that a resync is due and the command thread performs it.
+            void serviceDeferredOperStatusResync();
+
             void startVppEventsThread();
 
         private: // VPP
@@ -1163,6 +1175,7 @@ namespace saivs
             std::map<std::string, std::string> m_hwif_hostif_map;
             int mapping_init = 0;
             bool m_run_vpp_events_thread = true;
+            std::atomic<bool> m_operResyncDue { false };
             bool VppEventsThreadStarted = false;
             std::shared_ptr<std::thread> m_vpp_thread;
 
@@ -1251,6 +1264,8 @@ namespace saivs
 
             std::map<std::string, std::shared_ptr<HostInterfaceInfo>> m_hostif_info_map;
 
+            std::map<std::string, bool> m_last_oper_up;
+
             CRMTracker m_crmTracker;
 
             bool isIPv4Route(const std::string &serializedObjectId);

vslib/vpp/SwitchVppRif.cpp

@@ -385,6 +385,50 @@ bool SwitchVpp::vpp_get_hwif_name (
     return true;
 }
 
+void SwitchVpp::resyncPortOperStatus()
+{
+    SWSS_LOG_ENTER();
+
+    for (auto& kvp: m_hostif_info_map)
+    {
+        const std::string& tapname = kvp.first;
+        auto& info = kvp.second;
+
+        if (!info)
+        {
+            continue;
+        }
+
+        const char* dev = tapname.c_str();
+        const char* hwif_name = tap_to_hwif_name(dev);
+
+        bool link_up = false;
+
+        if (interface_get_state(hwif_name, &link_up) != 0)
+        {
+            continue;
+        }
+
+        auto prev_it = m_last_oper_up.find(tapname);
+        bool have_prev = (prev_it != m_last_oper_up.end());
+
+        if (have_prev && prev_it->second == link_up)
+        {
+            continue;
+        }
+
+        m_last_oper_up[tapname] = link_up;
+
+        auto state = link_up ? SAI_PORT_OPER_STATUS_UP : SAI_PORT_OPER_STATUS_DOWN;
+
+        send_port_oper_status_notification(info->m_portId, state, false);
+
+        SWSS_LOG_NOTICE("resync oper-status %s(%s) %s",
+                        hwif_name, dev,
+                        link_up ? "UP" : "DOWN");
+    }
+}
+
 void SwitchVpp::vppProcessEvents ()
 {
     SWSS_LOG_ENTER();
@@ -393,10 +437,26 @@ void SwitchVpp::vppProcessEvents ()
     vpp_event_info_t *evp;
     int ret;
 
+    // One-shot level oper-status resync at startup: recover the current link
+    // state of any interface whose state was set before we subscribed to
+    // interface events (edge-only delivery would otherwise miss it).
+    m_operResyncDue.store(true, std::memory_order_relaxed);
+
     while(m_run_vpp_events_thread) {
         nanosleep(&req, NULL);
         ret = vpp_sync_for_events();
         SWSS_LOG_NOTICE("Checking for any VS events status %d", ret);
+        if (ret < 0)
+        {
+            SWSS_LOG_WARN("vpp_sync_for_events failed (%d); event socket reconnect attempted", ret);
+            // The read failure triggers an event-socket reconnect inside
+            // vpp_sync_for_events. VPP only re-delivers *future* edges after
+            // re-subscribe, so schedule a one-shot level resync to recover any
+            // edge missed during the outage (e.g. reboot/config-reload) - the
+            // exact case resyncPortOperStatus was added for. This is event-driven
+            // (not a periodic poll), so it does not exhaust the VPP client clib heap.
+            m_operResyncDue.store(true, std::memory_order_relaxed);
+        }
         while ((evp = vpp_ev_dequeue())) {
             if (evp->type == VPP_INTF_LINK_STATUS) {
                 asyncIntfStateUpdate(evp->data.intf_status.hwif_name,
@@ -414,6 +474,27 @@ void SwitchVpp::vppProcessEvents ()
             }
             vpp_ev_free(evp);
         }
+        // No periodic resync here. resyncPortOperStatus() issues per-interface
+        // SW_INTERFACE_DUMPs; doing that every cycle exhausted VPP's client clib
+        // heap (os_panic in clib_mem_heap_realloc_aligned). Resync is instead
+        // requested only at startup and after a reconnect (above), and executed
+        // on the command thread via serviceDeferredOperStatusResync().
+    }
+}
+
+void SwitchVpp::serviceDeferredOperStatusResync()
+{
+    SWSS_LOG_ENTER();
+
+    // Runs on the command thread (from the create/set/remove entry points).
+    // resyncPortOperStatus() issues VPP binary-API SW_INTERFACE_DUMPs; the event
+    // thread only *requests* a resync (m_operResyncDue) on startup and after an
+    // event-socket reconnect - never on a periodic timer - so this performs at
+    // most a handful of level reads over the switch lifetime, avoiding the VPP
+    // client clib-heap exhaustion that a continuous poll caused.
+    if (m_operResyncDue.exchange(false, std::memory_order_relaxed))
+    {
+        resyncPortOperStatus();
     }
 }