From d15df143080d8b434bb2c043e9c8b6c1d8d8bd1b Mon Sep 17 00:00:00 2001 From: Chenyang Wang Date: Wed, 8 Jul 2026 00:26:59 +0000 Subject: [PATCH 1/7] [vpp] saivpp VXLAN: set L3 tunnel inner source MAC to router MAC When a packet is routed into an L3 VXLAN tunnel, VPP builds the inner Ethernet header using the tunnel interface hardware MAC as the source. The interface default is VPPs auto-generated MAC (02:fe:..), which HW ASICs and the VNET decap tests do not expect. Override the tunnel interface MAC with the router MAC after creating the encap, via a new sw_interface_set_mac_by_index binary API helper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/TunnelManager.cpp | 6 ++++++ vslib/vpp/vppxlate/SaiVppXlate.c | 32 ++++++++++++++++++++++++++++++++ vslib/vpp/vppxlate/SaiVppXlate.h | 1 + 3 files changed, 39 insertions(+) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index d0d5d29786..409507694b 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -299,6 +299,12 @@ TunnelManager::create_vpp_vxlan_encap( } else { ip4_nbr_add_del(NULL, sw_if_index, &req.dst_address.addr.ip4, false, true/*no_fib_entry*/, bvi_mac, 1); } + /* Override the tunnel interface's auto-generated MAC with the router MAC. + * When a packet is routed into this L3 VXLAN tunnel, VPP builds the inner + * Ethernet header using the tunnel interface's hardware MAC as the source. + * Without this, the inner source MAC is VPP's auto MAC (02:fe:..) instead + * of the router MAC that HW ASICs (and the VNET decap test) expect. */ + sw_interface_set_mac_by_index(sw_if_index, bvi_mac); } SWSS_LOG_INFO("successfully created encap for vxlan tunnel %d", sw_if_index); diff --git a/vslib/vpp/vppxlate/SaiVppXlate.c b/vslib/vpp/vppxlate/SaiVppXlate.c index 174d8ac000..62524d00cc 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.c +++ b/vslib/vpp/vppxlate/SaiVppXlate.c @@ -3264,6 +3264,38 @@ int sw_interface_set_mac (const char *hwif_name, uint8_t *mac_address) return ret; } +int sw_interface_set_mac_by_index (uint32_t sw_if_index, uint8_t *mac_address) +{ + vat_main_t *vam = &vat_main; + vl_api_sw_interface_set_mac_address_t *mp; + int ret; + + if (mac_address == NULL) { + SAIVPP_ERROR("mac address can't be NULL"); + return -EINVAL; + } + + VPP_LOCK(); + + __plugin_msg_base = interface_msg_id_base; + + M (SW_INTERFACE_SET_MAC_ADDRESS, mp); + + mp->sw_if_index = htonl(sw_if_index); + memcpy(mp->mac_address, mac_address, sizeof(mp->mac_address)); + + S (mp); + + WR (ret); + + if (ret) { SAIVPP_ERROR("%s failed(%d) sw_if_index %u", __func__, ret, sw_if_index); } + else { SAIVPP_INFO("%s sw_if_index %u", __func__, sw_if_index); } + + VPP_UNLOCK(); + + return ret; +} + int hw_interface_set_mtu (const char *hwif_name, uint32_t mtu) { vat_main_t *vam = &vat_main; diff --git a/vslib/vpp/vppxlate/SaiVppXlate.h b/vslib/vpp/vppxlate/SaiVppXlate.h index 352bfc1534..a027e4b64b 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.h +++ b/vslib/vpp/vppxlate/SaiVppXlate.h @@ -300,6 +300,7 @@ typedef enum { extern int hw_interface_set_mtu(const char *hwif_name, uint32_t mtu); extern int sw_interface_set_mtu(const char *hwif_name, uint32_t mtu); extern int sw_interface_set_mac(const char *hwif_name, uint8_t *mac_address); + extern int sw_interface_set_mac_by_index(uint32_t sw_if_index, uint8_t *mac_address); extern int sw_interface_ip6_enable_disable(const char *hwif_name, bool enable); extern int ip_vrf_add(uint32_t vrf_id, const char *vrf_name, bool is_ipv6); extern int ip_vrf_del(uint32_t vrf_id, const char *vrf_name, bool is_ipv6); From 1b09d816743bb86fd10ca0cf148b30e2b04bdccc Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Tue, 14 Jul 2026 16:15:48 +0000 Subject: [PATCH 2/7] [vpp] saivpp VXLAN: source-independent (secondary-VTEP) VNET tunnel decap Extend the VPP virtual-switch SAI tunnel adaptation (vslib/vpp) to program source-independent VNET tunnel termination, so decap matches on local dst + VNI while ignoring the outer source IP. This supports RIOT / secondary-VTEP topologies where encapsulated traffic may arrive from a VTEP other than the primary tunnel source, which previously failed to decap. vslib/vpp/TunnelManager.{cpp,h}: add the decap-any tunnel-term wiring. Stacked on top of the existing tunnel inner-source-MAC override. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/TunnelManager.cpp | 335 ++++++++++++++++++++++++++++++++++++ vslib/vpp/TunnelManager.h | 25 +++ 2 files changed, 360 insertions(+) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index 409507694b..f5315e15cb 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -660,6 +660,75 @@ TunnelManager::handle_l2_vxlan_tunnel_map_entry( tunnel_map_oid = attr.value.oid; } + // L3 VNET decap (VNI -> Virtual Router): install decap for secondary + // (non-local) VTEPs so the multiple-tunnels "special" VTEP is decappable. + // The primary Loopback0 VTEP is detected and skipped inside the helper. + if (tunnel_map_type == SAI_TUNNEL_MAP_TYPE_VNI_TO_VIRTUAL_ROUTER_ID) { + if (vni == 0 || tunnel_map_oid == SAI_NULL_OBJECT_ID) { + return SAI_STATUS_SUCCESS; + } + + sai_object_id_t term_oid; + sai_deserialize_object_id(serializedObjectId, term_oid); + if (m_vxlan_decap_term_map.find(term_oid) != m_vxlan_decap_term_map.end()) { + return SAI_STATUS_SUCCESS; // already installed + } + + // The map entry VR -> VPP VRF that routes the decapped inner packet. + attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_VIRTUAL_ROUTER_ID_VALUE; + if (map_entry_obj.get_attr(attr) != SAI_STATUS_SUCCESS) { + return SAI_STATUS_SUCCESS; + } + auto ip_vrf = m_switch_db->vpp_get_ip_vrf(attr.value.oid); + if (!ip_vrf) { + SWSS_LOG_ERROR("VXLAN decap map entry %s: no VRF for VNI %u", + serializedObjectId.c_str(), vni); + return SAI_STATUS_SUCCESS; + } + + // Find the VXLAN tunnel(s) referencing this decap mapper; each tunnel's + // ENCAP_SRC_IP is the local VTEP IP to make decappable. + auto mapper_obj = m_switch_db->get_sai_object(SAI_OBJECT_TYPE_TUNNEL_MAP, + sai_serialize_object_id(tunnel_map_oid)); + if (!mapper_obj) { + return SAI_STATUS_SUCCESS; + } + auto tunnels = mapper_obj->get_child_objs(SAI_OBJECT_TYPE_TUNNEL); + if (!tunnels) { + return SAI_STATUS_SUCCESS; + } + + std::vector created; + for (auto& tunnel_pair : *tunnels) { + auto tunnel_obj = tunnel_pair.second; + + attr.id = SAI_TUNNEL_ATTR_TYPE; + if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS || + attr.value.s32 != SAI_TUNNEL_TYPE_VXLAN) continue; + + attr.id = SAI_TUNNEL_ATTR_ENCAP_SRC_IP; + if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS) continue; + sai_ip_address_t vtep_ip = attr.value.ipaddr; + + TunnelVPPData td; + bool skipped = false; + if (create_vxlan_decap_term(vtep_ip, vni, ip_vrf, td, skipped) + != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("VXLAN decap map entry %s: decap create failed VNI %u", + serializedObjectId.c_str(), vni); + continue; + } + if (!skipped) { + created.push_back(td); + } + } + + if (!created.empty()) { + m_vxlan_decap_term_map[term_oid] = created; + } + return SAI_STATUS_SUCCESS; + } + if (tunnel_map_type != SAI_TUNNEL_MAP_TYPE_VNI_TO_VLAN_ID) { return SAI_STATUS_SUCCESS; } @@ -718,6 +787,21 @@ TunnelManager::handle_l2_vxlan_tunnel_map_entry_removal( { SWSS_LOG_ENTER(); + // L3 VNET decap secondary-VTEP teardown, keyed by map-entry OID so it works + // even if the SAI object is already gone from the DB. + { + sai_object_id_t term_oid; + sai_deserialize_object_id(serializedObjectId, term_oid); + auto term_it = m_vxlan_decap_term_map.find(term_oid); + if (term_it != m_vxlan_decap_term_map.end()) { + for (auto& td : term_it->second) { + remove_vxlan_decap_term(td); + } + m_vxlan_decap_term_map.erase(term_it); + return SAI_STATUS_SUCCESS; + } + } + auto entry_obj = m_switch_db->get_sai_object( SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY, serializedObjectId); if (!entry_obj) { @@ -773,6 +857,257 @@ TunnelManager::handle_l2_vxlan_tunnel_map_entry_removal( return SAI_STATUS_SUCCESS; } +sai_status_t +TunnelManager::vxlan_secondary_vtep_local_receive( + _In_ const sai_ip_address_t& vtep_ip, + _In_ bool is_add) +{ + SWSS_LOG_ENTER(); + + bool is_v6 = (vtep_ip.addr_family == SAI_IP_ADDR_FAMILY_IPV6); + + vpp_ip_route_t *route = (vpp_ip_route_t *) + calloc(1, sizeof(vpp_ip_route_t) + sizeof(vpp_ip_nexthop_t)); + if (!route) { + SWSS_LOG_ERROR("Failed to allocate memory for VTEP local-receive route"); + return SAI_STATUS_FAILURE; + } + + route->vrf_id = 0; // underlay / default VRF + route->is_multipath = false; + route->nexthop_cnt = 1; + + if (is_v6) { + route->prefix_len = 128; + route->prefix_addr.sa_family = AF_INET6; + memcpy(route->prefix_addr.addr.ip6.sin6_addr.s6_addr, vtep_ip.addr.ip6, + sizeof(route->prefix_addr.addr.ip6.sin6_addr.s6_addr)); + route->nexthop[0].addr.sa_family = AF_INET6; + } else { + route->prefix_len = 32; + route->prefix_addr.sa_family = AF_INET; + route->prefix_addr.addr.ip4.sin_addr.s_addr = vtep_ip.addr.ip4; + route->nexthop[0].addr.sa_family = AF_INET; + } + route->nexthop[0].type = VPP_NEXTHOP_LOCAL; + route->nexthop[0].sw_if_index = (uint32_t)~0; + + int ret = ip_route_add_del(route, is_add); + free(route); + + if (ret != 0) { + SWSS_LOG_ERROR("Failed to %s VRF0 local-receive for secondary VTEP (ret %d)", + is_add ? "add" : "remove", ret); + return SAI_STATUS_FAILURE; + } + return SAI_STATUS_SUCCESS; +} + +/* + * Fill the decap-only VXLAN tunnel's destination address. + * + * VPP derives a tunnel's address-family from its SOURCE and then REJECTS any + * tunnel whose src/dst families differ (vxlan_add_del_tunnel_clean_input: + * "ip46_address_is_ip4(src) != ip46_address_is_ip4(dst)"). For IPv4 the + * decap-only sentinel dst is the unspecified 0.0.0.0. For IPv6 the unspecified + * :: is misclassified as IPv4 by ip46_address_is_ip4() (its top 12 bytes are + * zero), which would fail the family-match check and reject the tunnel. So for + * IPv6 use a non-zero, non-multicast placeholder derived from src: it is + * guaranteed IPv6-classified and different from src. The dst is never used for + * forwarding - decap matches the source-independent (src, vni) wildcard that + * patch 0014 registers on tunnel add. + */ +static void +vxlan_decap_term_set_dst(_In_ const vpp_ip_addr_t& src, _Out_ vpp_ip_addr_t& dst) +{ + if (src.sa_family == AF_INET6) { + dst.sa_family = AF_INET6; + memcpy(&dst.addr.ip6.sin6_addr, &src.addr.ip6.sin6_addr, + sizeof(struct in6_addr)); + /* Flip a high-order byte so ip46_address_is_ip4(dst) stays false and + * dst != src, while remaining within the (non-multicast) src prefix. */ + dst.addr.ip6.sin6_addr.s6_addr[8] ^= 0x01; + } else { + dst.sa_family = AF_INET; + dst.addr.ip4.sin_addr.s_addr = 0; /* 0.0.0.0 sentinel */ + } +} + +sai_status_t +TunnelManager::create_vxlan_decap_term( + _In_ const sai_ip_address_t& vtep_ip, + _In_ uint32_t vni, + _In_ std::shared_ptr ip_vrf, + _Out_ TunnelVPPData& tunnel_data, + _Out_ bool& is_local_skip) +{ + SWSS_LOG_ENTER(); + + char hw_bvi_ifname[32]; + auto router_mac = get_router_mac(); + auto bvi_mac = router_mac.data(); + vpp_ip_route_t bvi_ip_prefix; + + is_local_skip = false; + + // Primary-VTEP guard: if this VTEP IP already belongs to one of our own + // interfaces (switch Loopback0), decap is already handled by the + // nexthop-driven decap path; do nothing to avoid disturbing it. + refresh_interfaces_list(); + { + vpp_ip_addr_t probe_ip; + sai_ip_address_t vtep_probe = vtep_ip; + sai_ip_address_t_to_vpp_ip_addr_t(vtep_probe, probe_ip); + uint32_t owner_if = 0; + if (vpp_sw_interface_find_by_ip(&probe_ip, (uint32_t)~0, &owner_if) == 0) { + SWSS_LOG_NOTICE("VXLAN decap term: VTEP is a local interface addr " + "(sw_if %u); skipping VNI %u (primary VTEP)", owner_if, vni); + is_local_skip = true; + return SAI_STATUS_SUCCESS; + } + } + + // Allocate a BD + BVI so the decapped inner packet is L3-routed by a BVI + // bound to the decap-mapper VRF (mirrors create_vpp_vxlan_decap). + int bd_id = m_switch_db->dynamic_bd_id_pool.alloc(); + if (bd_id == -1) { + SWSS_LOG_ERROR("Failed to allocate bridge domain ID for vxlan decap term"); + return SAI_STATUS_FAILURE; + } + tunnel_data.bd_id = bd_id; + tunnel_data.ip_vrf = ip_vrf; + tunnel_data.vni = vni; + tunnel_data.src_ip = vtep_ip; + memset(&tunnel_data.dst_ip, 0, sizeof(tunnel_data.dst_ip)); + tunnel_data.dst_ip.addr_family = vtep_ip.addr_family; + + if (create_bvi_interface(bvi_mac, bd_id) != 0) { + SWSS_LOG_ERROR("Failed to create bvi interface for vxlan decap term"); + m_switch_db->dynamic_bd_id_pool.free(bd_id); + return SAI_STATUS_FAILURE; + } + refresh_interfaces_list(); + + snprintf(hw_bvi_ifname, sizeof(hw_bvi_ifname), "bvi%u", bd_id); + interface_set_state(hw_bvi_ifname, true); + set_sw_interface_l2_bridge(hw_bvi_ifname, bd_id, true, VPP_API_PORT_TYPE_BVI); + set_interface_vrf(hw_bvi_ifname, 0, ip_vrf->m_vrf_id, ip_vrf->m_is_ipv6); + + // Synthetic per-BD addresses L3-enable both families on the BVI so a v4 + // VTEP can still route inner v6 and vice-versa (mirrors decap path). + uint16_t offset = (uint16_t)((uint16_t)(bd_id - SwitchVpp::dynamic_bd_id_base) + 2); + + bvi_ip_prefix.prefix_len = 32; + bvi_ip_prefix.prefix_addr.sa_family = AF_INET; + { + struct sockaddr_in *sin = &bvi_ip_prefix.prefix_addr.addr.ip4; + sin->sin_addr.s_addr = htonl(offset); + } + interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true); + + bvi_ip_prefix.prefix_len = 128; + bvi_ip_prefix.prefix_addr.sa_family = AF_INET6; + { + struct sockaddr_in6 *sin6 = &bvi_ip_prefix.prefix_addr.addr.ip6; + memset(&sin6->sin6_addr, 0, sizeof(struct in6_addr)); + sin6->sin6_addr.s6_addr[14] = (uint8_t)((offset >> 8) & 0xFF); + sin6->sin6_addr.s6_addr[15] = (uint8_t)(offset & 0xFF); + } + interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true); + + // Create the decap-capable VXLAN tunnel: src = VTEP IP, dst = unspecified. + // Its add registers the source-independent decap entry (VTEP IP, VNI). + vpp_vxlan_tunnel_t req; + memset(&req, 0, sizeof(req)); + req.dst_port = m_vxlan_port; + req.src_port = m_vxlan_port; + req.instance = ~0; + req.vni = vni; + req.decap_next_index = ~0; + sai_ip_address_t vtep_ip_nc = vtep_ip; + sai_ip_address_t_to_vpp_ip_addr_t(vtep_ip_nc, req.src_address); + vxlan_decap_term_set_dst(req.src_address, req.dst_address); + + if (create_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true) != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("VXLAN decap term: failed to create tunnel for VNI %u", vni); + delete_bvi_interface(hw_bvi_ifname); + m_switch_db->dynamic_bd_id_pool.free(bd_id); + refresh_interfaces_list(); + vpp_bridge_domain_add_del(bd_id, false); + return SAI_STATUS_FAILURE; + } + + // Bridge the tunnel into the BD so decapped frames reach the BVI. + if (set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, bd_id, true, + VPP_API_PORT_TYPE_NORMAL) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to bridge tunnel for VNI %u", vni); + remove_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true); + delete_bvi_interface(hw_bvi_ifname); + m_switch_db->dynamic_bd_id_pool.free(bd_id); + refresh_interfaces_list(); + vpp_bridge_domain_add_del(bd_id, false); + return SAI_STATUS_FAILURE; + } + + // Local-receive the VTEP IP in the underlay (VRF0) so outer VXLAN packets + // to this secondary VTEP are punted to vxlan-input. The primary VTEP is + // already local via Loopback0; a secondary VTEP is not backed by any + // interface address, so add it explicitly. + if (vxlan_secondary_vtep_local_receive(vtep_ip, true) != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("VXLAN decap term: failed to add VRF0 local-receive for VNI %u", vni); + set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, bd_id, false, + VPP_API_PORT_TYPE_NORMAL); + remove_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true); + delete_bvi_interface(hw_bvi_ifname); + m_switch_db->dynamic_bd_id_pool.free(bd_id); + refresh_interfaces_list(); + vpp_bridge_domain_add_del(bd_id, false); + return SAI_STATUS_FAILURE; + } + + SWSS_LOG_NOTICE("VXLAN decap term: installed secondary VTEP decap VNI %u bd %u sw_if %u", + vni, bd_id, tunnel_data.sw_if_index); + return SAI_STATUS_SUCCESS; +} + +sai_status_t +TunnelManager::remove_vxlan_decap_term(_In_ TunnelVPPData& tunnel_data) +{ + SWSS_LOG_ENTER(); + + char hw_bvi_ifname[32]; + + // Remove the VRF0 local-receive for the secondary VTEP IP. + vxlan_secondary_vtep_local_receive(tunnel_data.src_ip, false); + + // Unbridge + delete the decap VXLAN tunnel. + vpp_vxlan_tunnel_t req; + memset(&req, 0, sizeof(req)); + req.dst_port = m_vxlan_port; + req.src_port = m_vxlan_port; + req.instance = ~0; + req.vni = tunnel_data.vni; + req.decap_next_index = ~0; + sai_ip_address_t src_nc = tunnel_data.src_ip; + sai_ip_address_t_to_vpp_ip_addr_t(src_nc, req.src_address); + vxlan_decap_term_set_dst(req.src_address, req.dst_address); + + set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, tunnel_data.bd_id, + false, VPP_API_PORT_TYPE_NORMAL); + remove_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true); + + // Delete the BVI + bridge domain. + snprintf(hw_bvi_ifname, sizeof(hw_bvi_ifname), "bvi%u", tunnel_data.bd_id); + delete_bvi_interface(hw_bvi_ifname); + m_switch_db->dynamic_bd_id_pool.free(tunnel_data.bd_id); + refresh_interfaces_list(); + vpp_bridge_domain_add_del(tunnel_data.bd_id, false); + + SWSS_LOG_NOTICE("VXLAN decap term: removed secondary VTEP decap VNI %u bd %u", + tunnel_data.vni, tunnel_data.bd_id); + return SAI_STATUS_SUCCESS; +} + TunnelManagerIpIp::TunnelManagerIpIp(SwitchVpp *switch_db) : m_switch_db(switch_db) {} uint8_t diff --git a/vslib/vpp/TunnelManager.h b/vslib/vpp/TunnelManager.h index 52b9dad080..a1bcfea941 100644 --- a/vslib/vpp/TunnelManager.h +++ b/vslib/vpp/TunnelManager.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "SwitchVpp.h" #include "vppxlate/SaiVppXlate.h" @@ -187,6 +188,30 @@ namespace saivs // Map from VNI to VPP tunnel data (L2 VXLAN / EVPN) std::unordered_map m_l2_tunnel_map; + // Map from an L3 VNET decap map-entry (VNI -> Virtual Router) OID to the + // VPP decap objects installed for a secondary (non-local) VXLAN VTEP. + std::unordered_map> m_vxlan_decap_term_map; + + // Install decap for one secondary-VTEP VNI: a BD/BVI bound to the mapper + // VRF, a decap-capable VXLAN tunnel (its add registers source-independent + // decap), and a VRF0 local-receive for the VTEP IP. Sets is_local_skip + // and installs nothing when the VTEP IP is already a local interface + // address (the primary Loopback0 VTEP, handled by the nexthop path). + sai_status_t create_vxlan_decap_term( + _In_ const sai_ip_address_t& vtep_ip, + _In_ uint32_t vni, + _In_ std::shared_ptr ip_vrf, + _Out_ TunnelVPPData& tunnel_data, + _Out_ bool& is_local_skip); + + // Tear down one secondary-VTEP decap installed by create_vxlan_decap_term. + sai_status_t remove_vxlan_decap_term(_In_ TunnelVPPData& tunnel_data); + + // Add/remove a VRF0 (underlay) local-receive route for a secondary VTEP + // IP so outer VXLAN packets to it are punted to vxlan-input. + sai_status_t vxlan_secondary_vtep_local_receive( + _In_ const sai_ip_address_t& vtep_ip, _In_ bool is_add); + sai_status_t tunnel_encap_nexthop_action( _In_ const SaiObject* tunnel_nh_obj, _In_ Action action); From 589e8667041f3c13cf2b26dbb452837735b2956f Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Tue, 14 Jul 2026 22:59:15 +0000 Subject: [PATCH 3/7] vslib/vpp: fix VXLAN secondary-VTEP decap setup/teardown Address code-review findings on the L3 VXLAN VNET decap term handling in TunnelManager: - Refcount the VRF0 local-receive route by VTEP IP. Multiple VNIs can share one secondary VTEP, so program the route on the first term and remove it only on the last. Tearing down one VNI no longer removes the shared route and breaks decap for its siblings. - Check every VPP API return in create_vxlan_decap_term (interface state, L2 bridge bind, VRF set, both BVI address adds) and roll back the BD/BVI on failure instead of proceeding half-programmed and returning success. - Set decap_any on both L3 VNET decap tunnel requests so the decap term is source-independent. This covers the secondary-VTEP path (create_vxlan_decap_term) and the primary-VTEP L3 VNET path (tunnel_encap_nexthop_action, VIRTUAL_ROUTER_ID_TO_VNI mappers). L2 EVPN tunnels (create_l2_vxlan_tunnel_for_vni) intentionally leave decap_any unset to keep exact outer-source validation. SaiVppXlate signals the flag to VPP via the high bit of decap_next_index, forcing a valid default next index so the flag stays distinguishable. - Install L3 (VNI_TO_VIRTUAL_ROUTER_ID) decap terms from the tunnel-create path as well, so a TUNNEL created after its TUNNEL_MAP_ENTRY still gets its decap term programmed. A shared install_l3_vxlan_decap_terms helper keeps the map-entry and tunnel-create paths idempotent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/SwitchVpp.cpp | 4 + vslib/vpp/TunnelManager.cpp | 290 +++++++++++++++++++++++-------- vslib/vpp/TunnelManager.h | 34 ++++ vslib/vpp/vppxlate/SaiVppXlate.c | 15 +- vslib/vpp/vppxlate/SaiVppXlate.h | 12 ++ 5 files changed, 286 insertions(+), 69 deletions(-) diff --git a/vslib/vpp/SwitchVpp.cpp b/vslib/vpp/SwitchVpp.cpp index d708c38437..5d364c5b92 100644 --- a/vslib/vpp/SwitchVpp.cpp +++ b/vslib/vpp/SwitchVpp.cpp @@ -1521,6 +1521,10 @@ sai_status_t SwitchVpp::create( sai_status_t status = m_tunnel_mgr.create_l2_vxlan_tunnel(object_id, sw_if_index); SWSS_LOG_INFO("L2 VXLAN tunnel create for %s: status=%d sw_if_index=%u", serializedObjectId.c_str(), status, sw_if_index); + + // Late-tunnel hook: install any L3 secondary-VTEP decap terms whose + // TUNNEL_MAP_ENTRY was created before this tunnel existed (M3). + m_tunnel_mgr.handle_l3_vxlan_tunnel_create(object_id); return status; } diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index f5315e15cb..89967028ad 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -184,6 +184,13 @@ TunnelManager::tunnel_encap_nexthop_action( sai_ip_address_t_to_vpp_ip_addr_t(src_ip, req.src_address); sai_ip_address_t_to_vpp_ip_addr_t(dst_ip, req.dst_address); req.decap_next_index = ~0; + // Primary-VTEP L3 VNET decap is source-independent by design: the + // underlay may deliver VXLAN frames to the local VTEP IP from any + // outer source. This path only handles VIRTUAL_ROUTER_ID_TO_VNI + // (L3 VNET) mappers, so request the source-independent decap term. + // L2 EVPN tunnels (create_l2_vxlan_tunnel_for_vni) intentionally + // leave decap_any unset to keep exact outer-source validation. + req.decap_any = true; attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_VNI_ID_VALUE; CHECK_STATUS_W_MSG(tunnel_encap_mapper_entry->get_attr(attr), @@ -622,6 +629,137 @@ TunnelManager::create_l2_vxlan_tunnel( return SAI_STATUS_SUCCESS; } +sai_status_t +TunnelManager::install_l3_vxlan_decap_terms( + _In_ const std::string& map_entry_serialized_oid, + _In_ uint32_t vni, + _In_ sai_object_id_t tunnel_map_oid, + _In_ const SaiObject* map_entry_obj) +{ + SWSS_LOG_ENTER(); + + if (vni == 0 || tunnel_map_oid == SAI_NULL_OBJECT_ID) { + return SAI_STATUS_SUCCESS; + } + + sai_object_id_t term_oid; + sai_deserialize_object_id(map_entry_serialized_oid, term_oid); + if (m_vxlan_decap_term_map.find(term_oid) != m_vxlan_decap_term_map.end()) { + return SAI_STATUS_SUCCESS; // already installed + } + + // The map entry VR -> VPP VRF that routes the decapped inner packet. + sai_attribute_t attr; + attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_VIRTUAL_ROUTER_ID_VALUE; + if (map_entry_obj->get_attr(attr) != SAI_STATUS_SUCCESS) { + return SAI_STATUS_SUCCESS; + } + auto ip_vrf = m_switch_db->vpp_get_ip_vrf(attr.value.oid); + if (!ip_vrf) { + SWSS_LOG_ERROR("VXLAN decap map entry %s: no VRF for VNI %u", + map_entry_serialized_oid.c_str(), vni); + return SAI_STATUS_SUCCESS; + } + + // Find the VXLAN tunnel(s) referencing this decap mapper; each tunnel's + // ENCAP_SRC_IP is the local VTEP IP to make decappable. + auto mapper_obj = m_switch_db->get_sai_object(SAI_OBJECT_TYPE_TUNNEL_MAP, + sai_serialize_object_id(tunnel_map_oid)); + if (!mapper_obj) { + return SAI_STATUS_SUCCESS; + } + auto tunnels = mapper_obj->get_child_objs(SAI_OBJECT_TYPE_TUNNEL); + if (!tunnels) { + return SAI_STATUS_SUCCESS; + } + + std::vector created; + for (auto& tunnel_pair : *tunnels) { + auto tunnel_obj = tunnel_pair.second; + + attr.id = SAI_TUNNEL_ATTR_TYPE; + if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS || + attr.value.s32 != SAI_TUNNEL_TYPE_VXLAN) continue; + + attr.id = SAI_TUNNEL_ATTR_ENCAP_SRC_IP; + if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS) continue; + sai_ip_address_t vtep_ip = attr.value.ipaddr; + + TunnelVPPData td; + bool skipped = false; + if (create_vxlan_decap_term(vtep_ip, vni, ip_vrf, td, skipped) + != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("VXLAN decap map entry %s: decap create failed VNI %u", + map_entry_serialized_oid.c_str(), vni); + continue; + } + if (!skipped) { + created.push_back(td); + } + } + + if (!created.empty()) { + m_vxlan_decap_term_map[term_oid] = created; + } + return SAI_STATUS_SUCCESS; +} + +sai_status_t +TunnelManager::handle_l3_vxlan_tunnel_create( + _In_ sai_object_id_t tunnel_oid) +{ + SWSS_LOG_ENTER(); + + // Late-tunnel hook (M3): a TUNNEL_MAP_ENTRY of type VNI_TO_VIRTUAL_ROUTER_ID + // may be created before the TUNNEL that references its mapper. In that case + // handle_l2_vxlan_tunnel_map_entry saw no tunnels and installed nothing. Now + // that the tunnel exists, rescan its L3 decap mappers and install any decap + // terms still missing. Idempotent via m_vxlan_decap_term_map. + auto tunnel_obj = m_switch_db->get_sai_object(SAI_OBJECT_TYPE_TUNNEL, + sai_serialize_object_id(tunnel_oid)); + if (!tunnel_obj) { + return SAI_STATUS_SUCCESS; + } + + sai_attribute_t attr; + attr.id = SAI_TUNNEL_ATTR_TYPE; + if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS || + attr.value.s32 != SAI_TUNNEL_TYPE_VXLAN) { + return SAI_STATUS_SUCCESS; + } + + auto decap_mappers = tunnel_obj->get_linked_objects( + SAI_OBJECT_TYPE_TUNNEL_MAP, SAI_TUNNEL_ATTR_DECAP_MAPPERS); + + for (auto mapper : decap_mappers) { + attr.id = SAI_TUNNEL_MAP_ATTR_TYPE; + if (mapper->get_attr(attr) != SAI_STATUS_SUCCESS) continue; + if (attr.value.s32 != SAI_TUNNEL_MAP_TYPE_VNI_TO_VIRTUAL_ROUTER_ID) continue; + + sai_object_id_t mapper_oid; + sai_deserialize_object_id(mapper->get_id(), mapper_oid); + + auto entries = mapper->get_child_objs(SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY); + if (!entries) continue; + + for (auto& entry_pair : *entries) { + auto entry = entry_pair.second; + + uint32_t vni = 0; + attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_VNI_ID_KEY; + if (entry->get_attr(attr) == SAI_STATUS_SUCCESS) { + vni = attr.value.u32; + } + + install_l3_vxlan_decap_terms(entry->get_id(), vni, mapper_oid, + entry.get()); + } + } + + return SAI_STATUS_SUCCESS; +} + + sai_status_t TunnelManager::handle_l2_vxlan_tunnel_map_entry( _In_ const std::string& serializedObjectId, @@ -664,69 +802,8 @@ TunnelManager::handle_l2_vxlan_tunnel_map_entry( // (non-local) VTEPs so the multiple-tunnels "special" VTEP is decappable. // The primary Loopback0 VTEP is detected and skipped inside the helper. if (tunnel_map_type == SAI_TUNNEL_MAP_TYPE_VNI_TO_VIRTUAL_ROUTER_ID) { - if (vni == 0 || tunnel_map_oid == SAI_NULL_OBJECT_ID) { - return SAI_STATUS_SUCCESS; - } - - sai_object_id_t term_oid; - sai_deserialize_object_id(serializedObjectId, term_oid); - if (m_vxlan_decap_term_map.find(term_oid) != m_vxlan_decap_term_map.end()) { - return SAI_STATUS_SUCCESS; // already installed - } - - // The map entry VR -> VPP VRF that routes the decapped inner packet. - attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_VIRTUAL_ROUTER_ID_VALUE; - if (map_entry_obj.get_attr(attr) != SAI_STATUS_SUCCESS) { - return SAI_STATUS_SUCCESS; - } - auto ip_vrf = m_switch_db->vpp_get_ip_vrf(attr.value.oid); - if (!ip_vrf) { - SWSS_LOG_ERROR("VXLAN decap map entry %s: no VRF for VNI %u", - serializedObjectId.c_str(), vni); - return SAI_STATUS_SUCCESS; - } - - // Find the VXLAN tunnel(s) referencing this decap mapper; each tunnel's - // ENCAP_SRC_IP is the local VTEP IP to make decappable. - auto mapper_obj = m_switch_db->get_sai_object(SAI_OBJECT_TYPE_TUNNEL_MAP, - sai_serialize_object_id(tunnel_map_oid)); - if (!mapper_obj) { - return SAI_STATUS_SUCCESS; - } - auto tunnels = mapper_obj->get_child_objs(SAI_OBJECT_TYPE_TUNNEL); - if (!tunnels) { - return SAI_STATUS_SUCCESS; - } - - std::vector created; - for (auto& tunnel_pair : *tunnels) { - auto tunnel_obj = tunnel_pair.second; - - attr.id = SAI_TUNNEL_ATTR_TYPE; - if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS || - attr.value.s32 != SAI_TUNNEL_TYPE_VXLAN) continue; - - attr.id = SAI_TUNNEL_ATTR_ENCAP_SRC_IP; - if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS) continue; - sai_ip_address_t vtep_ip = attr.value.ipaddr; - - TunnelVPPData td; - bool skipped = false; - if (create_vxlan_decap_term(vtep_ip, vni, ip_vrf, td, skipped) - != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("VXLAN decap map entry %s: decap create failed VNI %u", - serializedObjectId.c_str(), vni); - continue; - } - if (!skipped) { - created.push_back(td); - } - } - - if (!created.empty()) { - m_vxlan_decap_term_map[term_oid] = created; - } - return SAI_STATUS_SUCCESS; + return install_l3_vxlan_decap_terms(serializedObjectId, vni, tunnel_map_oid, + &map_entry_obj); } if (tunnel_map_type != SAI_TUNNEL_MAP_TYPE_VNI_TO_VLAN_ID) { @@ -866,6 +943,36 @@ TunnelManager::vxlan_secondary_vtep_local_receive( bool is_v6 = (vtep_ip.addr_family == SAI_IP_ADDR_FAMILY_IPV6); + // Refcount key: address family byte + raw address bytes. Multiple decap + // terms (distinct VNIs) can share one secondary VTEP IP; the VRF0 local- + // receive must be programmed once (first ref) and torn down once (last + // ref), otherwise removing one term breaks decap for its siblings. + std::string key(1, is_v6 ? '6' : '4'); + if (is_v6) { + key.append(reinterpret_cast(vtep_ip.addr.ip6), 16); + } else { + key.append(reinterpret_cast(&vtep_ip.addr.ip4), 4); + } + + if (is_add) { + auto it = m_vtep_local_receive_refcount.find(key); + if (it != m_vtep_local_receive_refcount.end()) { + it->second++; // route already present; just take a reference + return SAI_STATUS_SUCCESS; + } + // first user: fall through and program the route below + } else { + auto it = m_vtep_local_receive_refcount.find(key); + if (it == m_vtep_local_receive_refcount.end()) { + return SAI_STATUS_SUCCESS; // nothing programmed for this VTEP + } + if (--it->second > 0) { + return SAI_STATUS_SUCCESS; // other terms still need the route + } + m_vtep_local_receive_refcount.erase(it); + // last user: fall through and delete the route below + } + vpp_ip_route_t *route = (vpp_ip_route_t *) calloc(1, sizeof(vpp_ip_route_t) + sizeof(vpp_ip_nexthop_t)); if (!route) { @@ -898,8 +1005,19 @@ TunnelManager::vxlan_secondary_vtep_local_receive( if (ret != 0) { SWSS_LOG_ERROR("Failed to %s VRF0 local-receive for secondary VTEP (ret %d)", is_add ? "add" : "remove", ret); + // Adjust the refcount to match the actual programmed state so a retry + // can re-attempt (add failure = no ref held; delete failure = keep it). + if (is_add) { + m_vtep_local_receive_refcount.erase(key); + } else { + m_vtep_local_receive_refcount[key] = 1; + } return SAI_STATUS_FAILURE; } + + if (is_add) { + m_vtep_local_receive_refcount[key] = 1; + } return SAI_STATUS_SUCCESS; } @@ -989,9 +1107,34 @@ TunnelManager::create_vxlan_decap_term( refresh_interfaces_list(); snprintf(hw_bvi_ifname, sizeof(hw_bvi_ifname), "bvi%u", bd_id); - interface_set_state(hw_bvi_ifname, true); - set_sw_interface_l2_bridge(hw_bvi_ifname, bd_id, true, VPP_API_PORT_TYPE_BVI); - set_interface_vrf(hw_bvi_ifname, 0, ip_vrf->m_vrf_id, ip_vrf->m_is_ipv6); + + // Rollback the BD/BVI state allocated so far. Used on any setup failure + // before the decap tunnel is created (mirrors the checked ladder below). + auto rollback_bd_bvi = [&]() { + delete_bvi_interface(hw_bvi_ifname); + m_switch_db->dynamic_bd_id_pool.free(bd_id); + refresh_interfaces_list(); + vpp_bridge_domain_add_del(bd_id, false); + }; + + if (interface_set_state(hw_bvi_ifname, true) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to set BVI %s up for VNI %u", + hw_bvi_ifname, vni); + rollback_bd_bvi(); + return SAI_STATUS_FAILURE; + } + if (set_sw_interface_l2_bridge(hw_bvi_ifname, bd_id, true, VPP_API_PORT_TYPE_BVI) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to bridge BVI %s into bd %u for VNI %u", + hw_bvi_ifname, bd_id, vni); + rollback_bd_bvi(); + return SAI_STATUS_FAILURE; + } + if (set_interface_vrf(hw_bvi_ifname, 0, ip_vrf->m_vrf_id, ip_vrf->m_is_ipv6) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to set VRF %u on BVI %s for VNI %u", + ip_vrf->m_vrf_id, hw_bvi_ifname, vni); + rollback_bd_bvi(); + return SAI_STATUS_FAILURE; + } // Synthetic per-BD addresses L3-enable both families on the BVI so a v4 // VTEP can still route inner v6 and vice-versa (mirrors decap path). @@ -1003,7 +1146,12 @@ TunnelManager::create_vxlan_decap_term( struct sockaddr_in *sin = &bvi_ip_prefix.prefix_addr.addr.ip4; sin->sin_addr.s_addr = htonl(offset); } - interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true); + if (interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to add v4 BVI addr on %s for VNI %u", + hw_bvi_ifname, vni); + rollback_bd_bvi(); + return SAI_STATUS_FAILURE; + } bvi_ip_prefix.prefix_len = 128; bvi_ip_prefix.prefix_addr.sa_family = AF_INET6; @@ -1013,7 +1161,12 @@ TunnelManager::create_vxlan_decap_term( sin6->sin6_addr.s6_addr[14] = (uint8_t)((offset >> 8) & 0xFF); sin6->sin6_addr.s6_addr[15] = (uint8_t)(offset & 0xFF); } - interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true); + if (interface_ip_address_add_del(hw_bvi_ifname, &bvi_ip_prefix, true) != 0) { + SWSS_LOG_ERROR("VXLAN decap term: failed to add v6 BVI addr on %s for VNI %u", + hw_bvi_ifname, vni); + rollback_bd_bvi(); + return SAI_STATUS_FAILURE; + } // Create the decap-capable VXLAN tunnel: src = VTEP IP, dst = unspecified. // Its add registers the source-independent decap entry (VTEP IP, VNI). @@ -1024,6 +1177,7 @@ TunnelManager::create_vxlan_decap_term( req.instance = ~0; req.vni = vni; req.decap_next_index = ~0; + req.decap_any = true; // source-independent decap term (secondary VTEP) sai_ip_address_t vtep_ip_nc = vtep_ip; sai_ip_address_t_to_vpp_ip_addr_t(vtep_ip_nc, req.src_address); vxlan_decap_term_set_dst(req.src_address, req.dst_address); diff --git a/vslib/vpp/TunnelManager.h b/vslib/vpp/TunnelManager.h index a1bcfea941..4b3943b121 100644 --- a/vslib/vpp/TunnelManager.h +++ b/vslib/vpp/TunnelManager.h @@ -2,6 +2,8 @@ #include #include +#include +#include #include "SwitchVpp.h" #include "vppxlate/SaiVppXlate.h" @@ -149,6 +151,22 @@ namespace saivs _In_ sai_object_id_t tunnel_oid, _Out_ uint32_t& sw_if_index); + /** + * @brief Install L3 (VNI_TO_VIRTUAL_ROUTER_ID) secondary-VTEP decap on + * tunnel create (late-tunnel hook). + * + * A TUNNEL_MAP_ENTRY of type VNI_TO_VIRTUAL_ROUTER_ID can be created + * before the TUNNEL that references its mapper, in which case the + * map-entry handler saw no tunnels and installed nothing. Called after + * the tunnel is created to (re)scan the tunnel's L3 decap mappers and + * install any still-missing decap terms. Idempotent. + * + * @param tunnel_oid The tunnel object ID just created. + * @return SAI_STATUS_SUCCESS on success or if not applicable. + */ + sai_status_t handle_l3_vxlan_tunnel_create( + _In_ sai_object_id_t tunnel_oid); + /** * @brief Handle late tunnel map entry for L2 VXLAN. * @@ -192,6 +210,12 @@ namespace saivs // VPP decap objects installed for a secondary (non-local) VXLAN VTEP. std::unordered_map> m_vxlan_decap_term_map; + // Refcount for the VRF0 local-receive route of a secondary VTEP IP, + // keyed by the VTEP address. Multiple decap terms (different VNIs) can + // share one secondary VTEP IP; the underlay local-receive must be added + // on the first term and removed only on the last (see H1 review fix). + std::map m_vtep_local_receive_refcount; + // Install decap for one secondary-VTEP VNI: a BD/BVI bound to the mapper // VRF, a decap-capable VXLAN tunnel (its add registers source-independent // decap), and a VRF0 local-receive for the VTEP IP. Sets is_local_skip @@ -204,6 +228,16 @@ namespace saivs _Out_ TunnelVPPData& tunnel_data, _Out_ bool& is_local_skip); + // Install L3 (VNI_TO_VIRTUAL_ROUTER_ID) secondary-VTEP decap terms for a + // single TUNNEL_MAP_ENTRY. Shared by the map-entry create handler and the + // tunnel-create late hook so a TUNNEL created after its MAP_ENTRY still + // gets its decap term programmed. Idempotent via m_vxlan_decap_term_map. + sai_status_t install_l3_vxlan_decap_terms( + _In_ const std::string& map_entry_serialized_oid, + _In_ uint32_t vni, + _In_ sai_object_id_t tunnel_map_oid, + _In_ const SaiObject* map_entry_obj); + // Tear down one secondary-VTEP decap installed by create_vxlan_decap_term. sai_status_t remove_vxlan_decap_term(_In_ TunnelVPPData& tunnel_data); diff --git a/vslib/vpp/vppxlate/SaiVppXlate.c b/vslib/vpp/vppxlate/SaiVppXlate.c index 62524d00cc..9ba42862a6 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.c +++ b/vslib/vpp/vppxlate/SaiVppXlate.c @@ -3705,7 +3705,20 @@ int vpp_vxlan_tunnel_add_del(vpp_vxlan_tunnel_t *tunnel, bool is_add, u32 *sw_if mp->encap_vrf_id = htonl(tunnel->encap_vrf_id); mp->vni = htonl(tunnel->vni); mp->is_l3 = tunnel->is_l3; - mp->decap_next_index = htonl(tunnel->decap_next_index); + { + /* SONiC VNET decap-any: signal a source-independent decap term by + * setting the high bit of the wire decap_next_index. Force a valid + * default next index if the caller left it unset (~0), so the bit is + * distinguishable and the stripped value stays valid in VPP. */ + u32 dni = tunnel->decap_next_index; + if (tunnel->decap_any) { + if (dni == (u32)~0) { + dni = VPP_VXLAN_DECAP_NEXT_L2_INPUT; + } + dni |= VPP_VXLAN_DECAP_ANY_FLAG; + } + mp->decap_next_index = htonl(dni); + } S (mp); WR (ret); diff --git a/vslib/vpp/vppxlate/SaiVppXlate.h b/vslib/vpp/vppxlate/SaiVppXlate.h index a027e4b64b..0ec149b192 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.h +++ b/vslib/vpp/vppxlate/SaiVppXlate.h @@ -259,6 +259,14 @@ typedef enum { VPP_BOND_API_LB_ALGO_L34_INNER = 6, } vpp_bond_lb_algo; + /* SONiC VNET decap-any: high bit of the wire decap_next_index used to flag + * a source-independent decap term to the VPP vxlan patch. Must match + * VXLAN_DECAP_ANY_FLAG in the VPP 0014 patch (src/plugins/vxlan/vxlan.h). */ +#define VPP_VXLAN_DECAP_ANY_FLAG (1u << 31) + /* Default decap next index (VXLAN_INPUT_NEXT_L2_INPUT) sent alongside the + * flag so the stripped low bits remain a valid next index. */ +#define VPP_VXLAN_DECAP_NEXT_L2_INPUT 1u + typedef struct _vpp_vxlan_tunnel { vpp_ip_addr_t src_address; vpp_ip_addr_t dst_address; @@ -270,6 +278,10 @@ typedef enum { uint32_t encap_vrf_id; uint32_t decap_next_index; bool is_l3; + /* SONiC VNET decap-any: mark this as a secondary-VTEP source-independent + * decap term. Emitted to VPP in the high bit of the wire decap_next_index + * (VPP_VXLAN_DECAP_ANY_FLAG); the VPP patch decodes and strips it. */ + bool decap_any; } vpp_vxlan_tunnel_t; typedef struct _vpp_ipip_tunnel { From a240617d0ce7fb91c0583280be67cf693f127477 Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Thu, 16 Jul 2026 11:13:13 +0000 Subject: [PATCH 4/7] [vpp] saivpp VXLAN: detach tunnel from bridge-domain before BD delete remove_vpp_vxlan_decap deleted the bridge-domain while the VXLAN tunnel interface was still a member of it. The tunnel is only removed later by remove_vpp_vxlan_encap, so at BD-delete time the BD still had a member and vpp_bridge_domain_add_del(is_add=0) failed with -120 (bridge-domain in use). This surfaced as a syncd ERR log during L3 VNET decap teardown and tripped loganalyzer in the vxlan ECMP tests, although it was functionally benign (the BD was reclaimed once the tunnel was later removed). Detach the tunnel interface from the BD (set_sw_interface_l2_bridge_by_index with is_add=false) before freeing the bd_id and deleting the bridge-domain. Validated on a t1-lag-vpp testbed: vxlan/test_vxlan_ecmp.py no longer emits the -120 bridge-domain delete error and leaves zero bridge-domains in use. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/TunnelManager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index 89967028ad..b10131b859 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -450,6 +450,12 @@ TunnelManager::remove_vpp_vxlan_decap( delete_bvi_interface(hw_bvi_ifname); + // Detach the vxlan tunnel interface from the BD before deleting the BD. The + // tunnel itself is deleted later by remove_vpp_vxlan_encap; if it is still a + // member here, vpp_bridge_domain_add_del(is_add=0) fails with -120 (BD in use). + set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, tunnel_data.bd_id, + false, VPP_API_PORT_TYPE_NORMAL); + m_switch_db->dynamic_bd_id_pool.free(tunnel_data.bd_id); refresh_interfaces_list(); //bd is create automatically when the fist interface is add to it but requires manual deletion From 9aa0dcab45ed1829996ef5d1eae9f979e7f39130 Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Mon, 20 Jul 2026 18:43:45 +0000 Subject: [PATCH 5/7] vslib/vpp: add missing SWSS_LOG_ENTER() in vxlan_decap_term_set_dst The sonic-sairedis swsslogentercheck.sh test requires every method to call SWSS_LOG_ENTER() as its first statement. The vxlan_decap_term_set_dst helper in TunnelManager.cpp was missing it, failing "Run sonic sairedis unit tests" (1 of 11 tests) in the Build and BuildAsan legs. Add the macro to match the rest of the file. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/TunnelManager.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index b10131b859..561fb318ac 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -1044,6 +1044,8 @@ TunnelManager::vxlan_secondary_vtep_local_receive( static void vxlan_decap_term_set_dst(_In_ const vpp_ip_addr_t& src, _Out_ vpp_ip_addr_t& dst) { + SWSS_LOG_ENTER(); + if (src.sa_family == AF_INET6) { dst.sa_family = AF_INET6; memcpy(&dst.addr.ip6.sin6_addr, &src.addr.ip6.sin6_addr, From ff8519fe6afbd185659797bb7c2b69176d94b595 Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Fri, 31 Jul 2026 20:40:48 +0000 Subject: [PATCH 6/7] vslib/vpp: address review feedback on VXLAN decap-term Reviewer feedback on sonic-net/sonic-sairedis#1996: - Fix stale VPP patch-number references: the source-independent decap patch is 0017 (0014 is now an unrelated sflow patch after the series was renumbered). Corrects the two pointers in TunnelManager.cpp and SaiVppXlate.h that a future reader follows to find the VPP half of the decap_any contract. - Document the version coupling of the decap_next_index high-bit flag: 0x80000001 against a VPP without patch 0017 is undefined behaviour, so saivpp and the VPP image must be version-locked together. - install_l3_vxlan_decap_terms(): add a comment stating the best-effort /idempotent policy (called from both the map-entry and late-tunnel hooks, so an incomplete intermediate state must return SUCCESS). - vxlan_decap_term_set_dst(): make the IPv6 placeholder correct by construction. Flipping s6_addr[8] alone could zero all top-12 bytes for a src of the form ::0100:0:xxxx:xxxx and misclassify dst as IPv4; also force a fixed non-zero, non-multicast high-order byte. - create_vxlan_decap_term(): reuse the rollback_bd_bvi lambda in the three post-tunnel failure paths instead of open-coding the same steps. Signed-off-by: Aaron Bernardino Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa --- vslib/vpp/TunnelManager.cpp | 39 +++++++++++++++++++------------- vslib/vpp/vppxlate/SaiVppXlate.h | 7 +++++- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index 561fb318ac..3c02a27dbe 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -644,6 +644,15 @@ TunnelManager::install_l3_vxlan_decap_terms( { SWSS_LOG_ENTER(); + // Best-effort / idempotent by design: this is called both when the decap + // map entry is created and from the late-tunnel hook when a tunnel that + // references the mapper appears later, so a legitimately-incomplete + // intermediate state (mapper present but no tunnel yet, or vice-versa) is + // expected and must not fail the caller. Every early return here therefore + // yields SAI_STATUS_SUCCESS; the genuine-error paths (no VRF, per-tunnel + // decap create failure) log SWSS_LOG_ERROR and are skipped rather than + // propagated, because the term is re-attempted on the next hook and a hard + // failure would abort an otherwise valid tunnel-map-entry create. if (vni == 0 || tunnel_map_oid == SAI_NULL_OBJECT_ID) { return SAI_STATUS_SUCCESS; } @@ -1039,7 +1048,7 @@ TunnelManager::vxlan_secondary_vtep_local_receive( * IPv6 use a non-zero, non-multicast placeholder derived from src: it is * guaranteed IPv6-classified and different from src. The dst is never used for * forwarding - decap matches the source-independent (src, vni) wildcard that - * patch 0014 registers on tunnel add. + * patch 0017 registers on tunnel add. */ static void vxlan_decap_term_set_dst(_In_ const vpp_ip_addr_t& src, _Out_ vpp_ip_addr_t& dst) @@ -1050,9 +1059,16 @@ vxlan_decap_term_set_dst(_In_ const vpp_ip_addr_t& src, _Out_ vpp_ip_addr_t& dst dst.sa_family = AF_INET6; memcpy(&dst.addr.ip6.sin6_addr, &src.addr.ip6.sin6_addr, sizeof(struct in6_addr)); - /* Flip a high-order byte so ip46_address_is_ip4(dst) stays false and - * dst != src, while remaining within the (non-multicast) src prefix. */ - dst.addr.ip6.sin6_addr.s6_addr[8] ^= 0x01; + /* dst is a decap-only placeholder, never used for forwarding; it only + * has to pass VPP's family-match check, i.e. ip46_address_is_ip4(dst) + * must stay false (top 12 bytes not all zero) and dst != src. Flipping + * s6_addr[8] gives dst != src, but on its own it is not airtight: for a + * src whose top 12 bytes are zero except s6_addr[8] == 0x01 the flip + * would zero all 12 bytes and misclassify dst as IPv4. Also force a + * fixed non-zero, non-multicast high-order byte so the top 12 bytes can + * never collapse to zero regardless of src (correct by construction). */ + dst.addr.ip6.sin6_addr.s6_addr[8] ^= 0x01; /* guarantee dst != src */ + dst.addr.ip6.sin6_addr.s6_addr[0] = 0x20; /* non-zero, non-multicast */ } else { dst.sa_family = AF_INET; dst.addr.ip4.sin_addr.s_addr = 0; /* 0.0.0.0 sentinel */ @@ -1192,10 +1208,7 @@ TunnelManager::create_vxlan_decap_term( if (create_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true) != SAI_STATUS_SUCCESS) { SWSS_LOG_ERROR("VXLAN decap term: failed to create tunnel for VNI %u", vni); - delete_bvi_interface(hw_bvi_ifname); - m_switch_db->dynamic_bd_id_pool.free(bd_id); - refresh_interfaces_list(); - vpp_bridge_domain_add_del(bd_id, false); + rollback_bd_bvi(); return SAI_STATUS_FAILURE; } @@ -1204,10 +1217,7 @@ TunnelManager::create_vxlan_decap_term( VPP_API_PORT_TYPE_NORMAL) != 0) { SWSS_LOG_ERROR("VXLAN decap term: failed to bridge tunnel for VNI %u", vni); remove_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true); - delete_bvi_interface(hw_bvi_ifname); - m_switch_db->dynamic_bd_id_pool.free(bd_id); - refresh_interfaces_list(); - vpp_bridge_domain_add_del(bd_id, false); + rollback_bd_bvi(); return SAI_STATUS_FAILURE; } @@ -1220,10 +1230,7 @@ TunnelManager::create_vxlan_decap_term( set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, bd_id, false, VPP_API_PORT_TYPE_NORMAL); remove_vpp_vxlan_encap(req, tunnel_data, /*skip_neighbor=*/true); - delete_bvi_interface(hw_bvi_ifname); - m_switch_db->dynamic_bd_id_pool.free(bd_id); - refresh_interfaces_list(); - vpp_bridge_domain_add_del(bd_id, false); + rollback_bd_bvi(); return SAI_STATUS_FAILURE; } diff --git a/vslib/vpp/vppxlate/SaiVppXlate.h b/vslib/vpp/vppxlate/SaiVppXlate.h index 31bacde384..923fb1c24c 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.h +++ b/vslib/vpp/vppxlate/SaiVppXlate.h @@ -261,7 +261,12 @@ typedef enum { /* SONiC VNET decap-any: high bit of the wire decap_next_index used to flag * a source-independent decap term to the VPP vxlan patch. Must match - * VXLAN_DECAP_ANY_FLAG in the VPP 0014 patch (src/plugins/vxlan/vxlan.h). */ + * VXLAN_DECAP_ANY_FLAG in the VPP 0017 patch (src/plugins/vxlan/vxlan.h). + * This encoding requires a VPP built with sonic-platform-vpp patch 0017: + * an older VPP without it reads decap_next_index = 0x80000001 as a raw next + * index (undefined behaviour) instead of failing cleanly, so saivpp and the + * VPP image must be built and version-locked together (see VPP_VERSION in + * sonic-platform-vpp rules/vpp.mk). */ #define VPP_VXLAN_DECAP_ANY_FLAG (1u << 31) /* Default decap next index (VXLAN_INPUT_NEXT_L2_INPUT) sent alongside the * flag so the stripped low bits remain a valid next index. */ From fc3bb73310b4363d0916a19ddd330fade8ca2794 Mon Sep 17 00:00:00 2001 From: Aaron Bernardino Date: Tue, 4 Aug 2026 13:11:27 +0000 Subject: [PATCH 7/7] vslib/vpp: fix per-VTEP VXLAN L3 decap-term installation install_l3_vxlan_decap_terms() guarded idempotency on the tunnel map entry OID as a whole, but the decap terms recorded under an entry are per-tunnel (one per VTEP source IP). A tunnel created after the entry was first processed (e.g. a second VTEP referencing the same mapper) hit the whole-entry early-return and never got its own decap term, so decap for that VTEP silently failed. Make idempotency per-(map entry, VTEP): skip only VTEPs already recorded under the entry (or installed earlier in the same pass) and append newly appearing VTEPs to the entry's vector instead of overwriting it. Reuse the existing saivs::sai_ip_address_equal helper for the VTEP comparison. Also check and log the previously ignored return codes of sw_interface_set_mac_by_index() (router MAC on the encap tunnel) and the unbridge set_sw_interface_l2_bridge_by_index() in remove_vpp_vxlan_decap() so these failures are diagnosable instead of silent. Verified on a t1-lag-vpp testbed: test_vxlan_multiple_tunnels.py 16/16 passed, BGP 24/24, no new cores. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3446b9ac-fa7f-4d26-914b-a4f076c603aa Signed-off-by: Aaron Bernardino --- vslib/vpp/TunnelManager.cpp | 50 +++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/vslib/vpp/TunnelManager.cpp b/vslib/vpp/TunnelManager.cpp index 3c02a27dbe..455e50f8ea 100644 --- a/vslib/vpp/TunnelManager.cpp +++ b/vslib/vpp/TunnelManager.cpp @@ -311,7 +311,11 @@ TunnelManager::create_vpp_vxlan_encap( * Ethernet header using the tunnel interface's hardware MAC as the source. * Without this, the inner source MAC is VPP's auto MAC (02:fe:..) instead * of the router MAC that HW ASICs (and the VNET decap test) expect. */ - sw_interface_set_mac_by_index(sw_if_index, bvi_mac); + if (sw_interface_set_mac_by_index(sw_if_index, bvi_mac) != 0) { + SWSS_LOG_ERROR("Failed to set router MAC on vxlan tunnel sw_if %u; " + "inner source MAC will remain VPP's auto MAC (02:fe:..)", + sw_if_index); + } } SWSS_LOG_INFO("successfully created encap for vxlan tunnel %d", sw_if_index); @@ -453,8 +457,12 @@ TunnelManager::remove_vpp_vxlan_decap( // Detach the vxlan tunnel interface from the BD before deleting the BD. The // tunnel itself is deleted later by remove_vpp_vxlan_encap; if it is still a // member here, vpp_bridge_domain_add_del(is_add=0) fails with -120 (BD in use). - set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, tunnel_data.bd_id, - false, VPP_API_PORT_TYPE_NORMAL); + if (set_sw_interface_l2_bridge_by_index(tunnel_data.sw_if_index, tunnel_data.bd_id, + false, VPP_API_PORT_TYPE_NORMAL) != 0) { + SWSS_LOG_ERROR("VXLAN decap remove: failed to detach tunnel sw_if %u from BD %u; " + "subsequent BD delete may fail with -120 (BD in use)", + tunnel_data.sw_if_index, tunnel_data.bd_id); + } m_switch_db->dynamic_bd_id_pool.free(tunnel_data.bd_id); refresh_interfaces_list(); @@ -659,9 +667,17 @@ TunnelManager::install_l3_vxlan_decap_terms( sai_object_id_t term_oid; sai_deserialize_object_id(map_entry_serialized_oid, term_oid); - if (m_vxlan_decap_term_map.find(term_oid) != m_vxlan_decap_term_map.end()) { - return SAI_STATUS_SUCCESS; // already installed - } + + // Per-(map entry, VTEP) idempotency. The decap terms recorded under this + // map entry are per-tunnel (one per VTEP src IP), so we must NOT early- + // return on the entry as a whole: a tunnel created AFTER this entry was + // first processed (e.g. a second VTEP tunnel referencing the same mapper) + // still needs its own decap term. Only VTEPs already recorded are skipped + // in the loop below; a newly-appearing VTEP is installed and appended to + // the entry's vector. + auto existing_it = m_vxlan_decap_term_map.find(term_oid); + const std::vector* existing = + (existing_it != m_vxlan_decap_term_map.end()) ? &existing_it->second : nullptr; // The map entry VR -> VPP VRF that routes the decapped inner packet. sai_attribute_t attr; @@ -689,6 +705,18 @@ TunnelManager::install_l3_vxlan_decap_terms( } std::vector created; + + // A VTEP already has a decap term if it is recorded under this entry from a + // prior hook (existing) or was installed earlier in this same loop pass. + auto vtep_already_termed = [](const std::vector* vec, + const sai_ip_address_t& vtep) -> bool { + if (!vec) return false; + for (const auto& t : *vec) { + if (sai_ip_address_equal(t.src_ip, vtep)) return true; + } + return false; + }; + for (auto& tunnel_pair : *tunnels) { auto tunnel_obj = tunnel_pair.second; @@ -700,6 +728,13 @@ TunnelManager::install_l3_vxlan_decap_terms( if (tunnel_obj->get_attr(attr) != SAI_STATUS_SUCCESS) continue; sai_ip_address_t vtep_ip = attr.value.ipaddr; + // Skip VTEPs that already have a decap term under this map entry so a + // newly-appearing tunnel still gets one (per-VTEP idempotency). + if (vtep_already_termed(existing, vtep_ip) || + vtep_already_termed(&created, vtep_ip)) { + continue; + } + TunnelVPPData td; bool skipped = false; if (create_vxlan_decap_term(vtep_ip, vni, ip_vrf, td, skipped) @@ -714,7 +749,8 @@ TunnelManager::install_l3_vxlan_decap_terms( } if (!created.empty()) { - m_vxlan_decap_term_map[term_oid] = created; + auto& vec = m_vxlan_decap_term_map[term_oid]; + vec.insert(vec.end(), created.begin(), created.end()); } return SAI_STATUS_SUCCESS; }