diff --git a/src/dhcp_device.cpp b/src/dhcp_device.cpp index 8b1ed4f6c..a34ea8050 100644 --- a/src/dhcp_device.cpp +++ b/src/dhcp_device.cpp @@ -7,8 +7,11 @@ #include #include #include +#include #include #include +#include +#include #include "dhcp_device.h" @@ -69,6 +72,41 @@ static const char *counter_desc[DHCP_COUNTERS_COUNT] = { [DHCP_COUNTERS_SNAPSHOT_V6] = "Snapshot_V6", }; +typedef struct +{ + uint64_t last_rx; + uint64_t last_tx; + uint32_t pending_windows; + uint8_t tx_credit; + bool initialized; +} relay_flow_state_t; + +static std::unordered_map>> relay_flow_states; +static std::mutex relay_flow_state_mutex; + +static void initialize_relay_flow_states(const std::string &ifname, int rx_sock, int tx_sock, + const int *monitored_msgs, size_t monitored_msg_cnt) +{ + const counter_t &rx_counters = sock_mgr_get_sock_info(rx_sock).all_counters.at(ifname); + const counter_t &tx_counters = sock_mgr_get_sock_info(tx_sock).all_counters.at(ifname); + for (size_t i = 0; i < monitored_msg_cnt; i++) { + int msg_type = monitored_msgs[i]; + relay_flow_states[rx_sock][ifname][msg_type] = { + rx_counters.at(msg_type), tx_counters.at(msg_type), 0, 0, true + }; + } +} + +void dhcp_device_reset_health_state(const std::string &ifname) +{ + std::lock_guard lock(relay_flow_state_mutex); + relay_flow_states[rx_sock].erase(ifname); + initialize_relay_flow_states(ifname, rx_sock, tx_sock, + (const int *)monitored_msgs, monitored_msg_sz); +} + +static std::string last_counter_mismatch; + /** * @code check_counter_not_transmitted(ifname, rx_sock, tx_sock, monitored_msgs, monitored_msg_cnt); * @brief Check if there are received DHCP messages that are not transmitted out @@ -78,30 +116,80 @@ static const char *counter_desc[DHCP_COUNTERS_COUNT] = { * @param tx_sock tx socket * @param monitored_msgs array of monitored message types * @param monitored_msg_cnt number of monitored message types - * @return true if there are received messages not transmitted out, false otherwise + * @return DHCP relay health status */ // these helpers use const int * to accept both dhcp_message_type_t and dhcpv6_message_type_t arrays // without duplicating the function for each enum type; safe on GCC/Linux where unscoped enums use int -static bool check_counter_not_transmitted(const std::string &ifname, int rx_sock, int tx_sock, const int *monitored_msgs, size_t monitored_msg_cnt) +static std::unordered_map get_untransmitted_windows(const std::string &ifname, + int rx_sock, int tx_sock, + const int *monitored_msgs, + size_t monitored_msg_cnt) { + std::lock_guard lock(relay_flow_state_mutex); const sock_info_t &rx_sock_info = sock_mgr_get_sock_info(rx_sock); const counter_t &rx_counters = rx_sock_info.all_counters.at(ifname); - const counter_t &rx_counters_snapshot = rx_sock_info.all_counters_snapshot.at(ifname); const sock_info_t &tx_sock_info = sock_mgr_get_sock_info(tx_sock); const counter_t &tx_counters = tx_sock_info.all_counters.at(ifname); - const counter_t &tx_counters_snapshot = tx_sock_info.all_counters_snapshot.at(ifname); - // when there is packet in, no packet out + std::unordered_map result; for (size_t i = 0; i < monitored_msg_cnt; i++) { - if (rx_counters.at(monitored_msgs[i]) > rx_counters_snapshot.at(monitored_msgs[i]) && - tx_counters.at(monitored_msgs[i]) <= tx_counters_snapshot.at(monitored_msgs[i])) { - return true; + int msg_type = monitored_msgs[i]; + uint64_t current_rx = rx_counters.at(msg_type); + uint64_t current_tx = tx_counters.at(msg_type); + relay_flow_state_t &state = relay_flow_states[rx_sock][ifname][msg_type]; + + if (!state.initialized || current_rx < state.last_rx || current_tx < state.last_tx) { + state = {current_rx, current_tx, 0, 0, true}; + result[msg_type] = 0; + continue; } + + uint64_t rx_delta = current_rx - state.last_rx; + uint64_t tx_delta = current_tx - state.last_tx; + bool had_pending = state.pending_windows > 0; + bool previous_tx_credit = state.tx_credit > 0; + bool current_tx_activity = tx_delta > 0; + state.last_rx = current_rx; + state.last_tx = current_tx; + + if (had_pending) { + if (previous_tx_credit || current_tx_activity) { + state.pending_windows = 0; + state.tx_credit = current_tx_activity ? 1 : 0; + } else { + state.pending_windows++; + state.tx_credit = 0; + } + } else if (rx_delta > 0) { + if (previous_tx_credit) { + state.pending_windows = 0; + state.tx_credit = current_tx_activity ? 1 : 0; + } else if (current_tx_activity) { + state.pending_windows = 0; + state.tx_credit = 1; + } else { + state.pending_windows = 1; + state.tx_credit = 0; + } + } else { + state.pending_windows = 0; + state.tx_credit = current_tx_activity ? 1 : 0; + } + result[msg_type] = state.pending_windows; } - return false; + return result; +} + +std::unordered_map dhcp_device_get_untransmitted_windows(const std::string &ifname) +{ + return get_untransmitted_windows(ifname, rx_sock, tx_sock, + (const int *)monitored_msgs, monitored_msg_sz); } +static bool check_counter_increased(const std::string &ifname, int sock, + const int *monitored_msgs, size_t monitored_msg_cnt); + /** * @code dhcp_device_check_positive_health(ifname); * @brief Check that DHCP relayed messages are being transmitted out of this interface/dev @@ -112,8 +200,16 @@ static bool check_counter_not_transmitted(const std::string &ifname, int rx_sock */ static dhcp_mon_status_t dhcp_device_check_positive_health(const std::string &ifname) { - return check_counter_not_transmitted(ifname, rx_sock, tx_sock, (const int *)monitored_msgs, monitored_msg_sz) ? - DHCP_MON_STATUS_UNHEALTHY : DHCP_MON_STATUS_HEALTHY; + bool has_activity = check_counter_increased(ifname, rx_sock, + (const int *)monitored_msgs, monitored_msg_sz) || + check_counter_increased(ifname, tx_sock, + (const int *)monitored_msgs, monitored_msg_sz); + for (const auto &[msg_type, windows] : dhcp_device_get_untransmitted_windows(ifname)) { + if (windows > 0) { + return DHCP_MON_STATUS_UNHEALTHY; + } + } + return has_activity ? DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_INDETERMINATE; } /** @@ -124,10 +220,10 @@ static dhcp_mon_status_t dhcp_device_check_positive_health(const std::string &if * @param ifname interface name * @return DHCP_MON_STATUS_HEALTHY, DHCP_MON_STATUS_UNHEALTHY, or DHCP_MON_STATUS_INDETERMINATE */ -static dhcp_mon_status_t dhcp_device_check_positive_health_v6(const std::string &ifname) +static dhcp_mon_status_t dhcp_device_check_positive_health_v6(const std::string &) { - return check_counter_not_transmitted(ifname, rx_sock_v6, tx_sock_v6, (const int *)monitored_v6_msgs, monitored_v6_msg_sz) ? - DHCP_MON_STATUS_UNHEALTHY : DHCP_MON_STATUS_HEALTHY; + // Client and relay DHCPv6 message types differ across the relay boundary. + return DHCP_MON_STATUS_INDETERMINATE; } /** @@ -217,70 +313,91 @@ static bool check_counters_delta_expected(const std::string &ifname, const std:: uint64_t delta = counters.at(monitored_msgs[i]) - counters_snapshot.at(monitored_msgs[i]); uint64_t other_delta = other_counters.at(monitored_msgs[i]) - other_counters_snapshot.at(monitored_msgs[i]); if (delta * ratio != other_delta) { + const std::string *message_names = sock_info.is_v6 ? db_counter_name_v6 : db_counter_name; + last_counter_mismatch = + std::string(sock_info.is_v6 ? "IPv6 " : "IPv4 ") + + (sock_info.is_rx ? "RX" : "TX") + + " edge parent=" + ifname + + " parent_delta=" + std::to_string(delta) + + " child_aggregate=" + other_ifname + + " child_delta=" + std::to_string(other_delta) + + " expected_ratio=" + std::to_string(ratio) + + " message=" + message_names[monitored_msgs[i]]; return false; } } + return true; } -static dhcp_mon_status_t dhcp_device_check_agg_equal_rx(const std::string &ifname) +static dhcp_mon_status_t check_aggregate_health(const std::string &ifname, int sock, uint8_t ratio, + const int *monitored_msgs, size_t monitored_msg_cnt) { std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, rx_sock, 1, (const int *)monitored_msgs, monitored_msg_sz) ? + if (!check_counter_increased(ifname, sock, monitored_msgs, monitored_msg_cnt) && + !check_counter_increased(agg_ifname, sock, monitored_msgs, monitored_msg_cnt)) { + return DHCP_MON_STATUS_INDETERMINATE; + } + return check_counters_delta_expected(ifname, agg_ifname, sock, ratio, + monitored_msgs, monitored_msg_cnt) ? DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; } +const std::string &dhcp_device_get_last_counter_mismatch() +{ + return last_counter_mismatch; +} + +static dhcp_mon_status_t dhcp_device_check_agg_equal_rx(const std::string &ifname) +{ + return check_aggregate_health(ifname, rx_sock, 1, (const int *)monitored_msgs, monitored_msg_sz); +} + static dhcp_mon_status_t dhcp_device_check_agg_equal_tx(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, tx_sock, 1, (const int *)monitored_msgs, monitored_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, tx_sock, 1, (const int *)monitored_msgs, monitored_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_equal_rx_v6(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, rx_sock_v6, 1, (const int *)monitored_v6_msgs, monitored_v6_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, rx_sock_v6, 1, (const int *)monitored_v6_msgs, monitored_v6_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_equal_tx_v6(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, tx_sock_v6, 1, (const int *)monitored_v6_msgs, monitored_v6_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, tx_sock_v6, 1, (const int *)monitored_v6_msgs, monitored_v6_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_multiple_rx(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, rx_sock, readonly_access(rev_vlan_map, ifname).size() + readonly_access(rev_portchan_map, ifname).size(), - (const int *)monitored_msgs, monitored_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, rx_sock, + readonly_access(rev_vlan_map, ifname).size() + + readonly_access(rev_portchan_map, ifname).size(), + (const int *)monitored_msgs, monitored_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_multiple_tx(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, tx_sock, readonly_access(rev_vlan_map, ifname).size() + readonly_access(rev_portchan_map, ifname).size(), - (const int *)monitored_msgs, monitored_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, tx_sock, + readonly_access(rev_vlan_map, ifname).size() + + readonly_access(rev_portchan_map, ifname).size(), + (const int *)monitored_msgs, monitored_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_multiple_rx_v6(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, rx_sock_v6, readonly_access(rev_vlan_map, ifname).size() + readonly_access(rev_portchan_map, ifname).size(), - (const int *)monitored_v6_msgs, monitored_v6_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, rx_sock_v6, + readonly_access(rev_vlan_map, ifname).size() + + readonly_access(rev_portchan_map, ifname).size(), + (const int *)monitored_v6_msgs, monitored_v6_msg_sz); } static dhcp_mon_status_t dhcp_device_check_agg_multiple_tx_v6(const std::string &ifname) { - std::string agg_ifname = agg_dev_prefix + ifname; - return check_counters_delta_expected(ifname, agg_ifname, tx_sock_v6, readonly_access(rev_vlan_map, ifname).size() + readonly_access(rev_portchan_map, ifname).size(), - (const int *)monitored_v6_msgs, monitored_v6_msg_sz) ? - DHCP_MON_STATUS_HEALTHY : DHCP_MON_STATUS_UNHEALTHY; + return check_aggregate_health(ifname, tx_sock_v6, + readonly_access(rev_vlan_map, ifname).size() + + readonly_access(rev_portchan_map, ifname).size(), + (const int *)monitored_v6_msgs, monitored_v6_msg_sz); } /** @@ -382,7 +499,11 @@ void dhcp_device_print_status_debug(const std::string &ifname, dhcp_counters_typ dhcp_mon_status_t dhcp_device_get_status(const std::string &ifname, dhcp_device_check_t check_type) { - if (sock_mgr_counters_unchanged(ifname, (const int *)monitored_msgs, monitored_msg_sz, (const int *)monitored_v6_msgs, monitored_v6_msg_sz)) { + bool hierarchy_check = check_type >= DHCP_DEVICE_CHECK_AGG_EQUAL_RX; + if (!hierarchy_check && + check_type != DHCP_DEVICE_CHECK_POSITIVE && check_type != DHCP_DEVICE_CHECK_POSITIVE_V6 && + sock_mgr_counters_unchanged(ifname, (const int *)monitored_msgs, monitored_msg_sz, + (const int *)monitored_v6_msgs, monitored_v6_msg_sz)) { return DHCP_MON_STATUS_INDETERMINATE; } diff --git a/src/dhcp_device.h b/src/dhcp_device.h index 5d4b00721..6f73d199f 100644 --- a/src/dhcp_device.h +++ b/src/dhcp_device.h @@ -18,6 +18,7 @@ #include #include #include +#include /** DHCP message types */ typedef enum @@ -242,6 +243,33 @@ void dhcp_device_free(dhcp_device_context_t *context); */ dhcp_mon_status_t dhcp_device_get_status(const std::string &ifname, dhcp_device_check_t check_type); +/** Return details for the most recent interface hierarchy mismatch */ +const std::string &dhcp_device_get_last_counter_mismatch(); + +/** + * @code dhcp_device_get_untransmitted_windows(ifname); + * + * @brief update and return unmatched DHCPv4 relay RX age in health windows per message type. + * Caller must hold the counter-state write lock or otherwise quiesce packet handlers. + * + * @param ifname interface name + * + * @return message type to unmatched-window count + */ +std::unordered_map dhcp_device_get_untransmitted_windows(const std::string &ifname); + +/** + * @code dhcp_device_reset_health_state(ifname); + * + * @brief reset relay health watermarks to the current counters for an interface. + * Caller must hold the counter-state write lock or otherwise quiesce packet handlers. + * + * @param ifname interface name + * + * @return none + */ +void dhcp_device_reset_health_state(const std::string &ifname); + /** * @code dhcp_device_print_status(ifname, type); * diff --git a/src/dhcp_devman.cpp b/src/dhcp_devman.cpp index 9fa951e3c..e803e130f 100644 --- a/src/dhcp_devman.cpp +++ b/src/dhcp_devman.cpp @@ -11,6 +11,8 @@ #include #include +#include + #include "dhcp_devman.h" @@ -205,11 +207,13 @@ bool dhcp_devman_is_tracked_interface(const std::string &ifname) * @param none * @return none */ -static void update_vlan_mapping() +static void update_vlan_mapping(const std::shared_ptr &config_db, + std::unordered_map &vlan_mapping, + std::unordered_map> &reverse_vlan_mapping) { syslog(LOG_INFO, "Updating vlan mapping from VLAN_MEMBER"); auto match_pattern = std::string("VLAN_MEMBER|*"); - auto keys = mConfigDbPtr->keys(match_pattern); + auto keys = config_db->keys(match_pattern); std::string all_ifname; std::string all_skipped_ifname; for (const auto &key : keys) { @@ -221,8 +225,8 @@ static void update_vlan_mapping() all_skipped_ifname += "<" + ifname + ", " + vlan + ">, "; continue; } - vlan_map[ifname] = vlan; - rev_vlan_map[vlan].insert(ifname); + vlan_mapping[ifname] = vlan; + reverse_vlan_mapping[vlan].insert(ifname); all_ifname += "<" + ifname + ", " + vlan + ">, "; } syslog(LOG_INFO, "Added vlan member interface mappings: %s", all_ifname.c_str()); @@ -236,11 +240,14 @@ static void update_vlan_mapping() * @param none * @return none */ -static void update_portchannel_mapping() +static void update_portchannel_mapping(const std::shared_ptr &config_db, + const std::unordered_map &vlan_mapping, + std::unordered_map &portchannel_mapping, + std::unordered_map> &reverse_portchannel_mapping) { syslog(LOG_INFO, "Updating port-channel mapping from PORTCHANNEL_MEMBER"); auto match_pattern = std::string("PORTCHANNEL_MEMBER|*"); - auto keys = mConfigDbPtr->keys(match_pattern); + auto keys = config_db->keys(match_pattern); std::string all_ifname; std::string all_skipped_ifname; for (const auto &key : keys) { @@ -248,18 +255,38 @@ static void update_portchannel_mapping() auto second = key.find_last_of('|'); auto portchannel = key.substr(first + 1, second - first - 1); auto ifname = key.substr(second + 1); - if (intfs.find(portchannel) == intfs.end()) { + bool portchannel_is_context = intfs.find(portchannel) != intfs.end(); + bool portchannel_is_vlan_member = vlan_mapping.find(portchannel) != vlan_mapping.end(); + // Dual-ToR downlink counters require MUX attribution that is unavailable on a nested PortChannel. + if (!portchannel_is_context && (!portchannel_is_vlan_member || dual_tor_mode)) { all_skipped_ifname += "<" + ifname + ", " + portchannel + ">, "; continue; } - portchan_map[ifname] = portchannel; - rev_portchan_map[portchannel].insert(ifname); + portchannel_mapping[ifname] = portchannel; + reverse_portchannel_mapping[portchannel].insert(ifname); all_ifname += "<" + ifname + ", " + portchannel + ">, "; } syslog(LOG_INFO, "Added port-channel member interface mappings: %s", all_ifname.c_str()); syslog(LOG_INFO, "Skipped port-channel member interface mappings: %s", all_skipped_ifname.c_str()); } +void dhcp_devman_refresh_mappings() +{ + std::unordered_map new_vlan_map; + std::unordered_map new_portchan_map; + std::unordered_map> new_rev_vlan_map; + std::unordered_map> new_rev_portchan_map; + auto config_db = std::make_shared("CONFIG_DB", 0); + + update_vlan_mapping(config_db, new_vlan_map, new_rev_vlan_map); + update_portchannel_mapping(config_db, new_vlan_map, new_portchan_map, new_rev_portchan_map); + + vlan_map.swap(new_vlan_map); + portchan_map.swap(new_portchan_map); + rev_vlan_map.swap(new_rev_vlan_map); + rev_portchan_map.swap(new_rev_portchan_map); +} + int dhcp_devman_init() { syslog(LOG_INFO, "Initializing dhcp device manager"); @@ -296,9 +323,13 @@ int dhcp_devman_init() agg_dev_all = "Agg-" + downstream_ifname; agg_dev_prefix = agg_dev_all + "-"; - // vlan and its members, portchannel and its members are initialized regardless of whether they are in cmdline - update_vlan_mapping(); - update_portchannel_mapping(); + // PortChannel members depend on VLAN mappings to recognize a PortChannel under a monitored VLAN. + try { + dhcp_devman_refresh_mappings(); + } catch (const std::exception &e) { + syslog(LOG_ALERT, "Failed to initialize DHCP interface mappings: %s", e.what()); + return -1; + } syslog(LOG_INFO, "Dhcp device manager initialized successfully"); @@ -309,6 +340,8 @@ void dhcp_devman_free() { vlan_map.clear(); portchan_map.clear(); + rev_vlan_map.clear(); + rev_portchan_map.clear(); for (const auto &[ifname, context] : intfs) { dhcp_device_free(context); } @@ -321,15 +354,34 @@ const dhcp_device_context_t *dhcp_devman_get_device_context(const std::string &i if (iter != intfs.end()) { return iter->second; } + const auto port_channel = portchan_map.find(ifname); + if (port_channel != portchan_map.end() && ifname != port_channel->second) { + return dhcp_devman_get_device_context(port_channel->second); + } const auto vlan = vlan_map.find(ifname); if (vlan != vlan_map.end() && ifname != vlan->second) { return dhcp_devman_get_device_context(vlan->second); } + return NULL; +} + +std::string dhcp_devman_get_parent_ifname(const std::string &ifname) +{ const auto port_channel = portchan_map.find(ifname); if (port_channel != portchan_map.end() && ifname != port_channel->second) { - return dhcp_devman_get_device_context(port_channel->second); + return port_channel->second; } - return NULL; + const auto vlan = vlan_map.find(ifname); + if (vlan != vlan_map.end() && ifname != vlan->second) { + return vlan->second; + } + return ""; +} + +std::string dhcp_devman_get_agg_counter_ifname(const std::string &ifname) +{ + const std::string parent_ifname = dhcp_devman_get_parent_ifname(ifname); + return parent_ifname.empty() ? agg_dev_all : agg_dev_prefix + parent_ifname; } void dhcp_devman_print_all_status(dhcp_counters_type_t type) diff --git a/src/dhcp_devman.h b/src/dhcp_devman.h index 73c1cd6f2..e15307b18 100644 --- a/src/dhcp_devman.h +++ b/src/dhcp_devman.h @@ -117,6 +117,15 @@ bool dhcp_devman_is_tracked_interface(const std::string &ifname); */ int dhcp_devman_init(); +/** + * @code dhcp_devman_refresh_mappings(); + * + * @brief rebuild VLAN and PortChannel membership mappings transactionally from CONFIG_DB. + * + * @return none + */ +void dhcp_devman_refresh_mappings(); + /** * @code dhcp_devman_free(); * @@ -138,6 +147,28 @@ void dhcp_devman_free(); */ const dhcp_device_context_t* dhcp_devman_get_device_context(const std::string &ifname); +/** + * @code dhcp_devman_get_parent_ifname(ifname); + * + * @brief find the immediate parent interface of a tracked interface. + * + * @param ifname interface name + * + * @return parent interface name, or an empty string for a root context interface + */ +std::string dhcp_devman_get_parent_ifname(const std::string &ifname); + +/** + * @code dhcp_devman_get_agg_counter_ifname(ifname); + * + * @brief find the aggregate counter for the immediate parent of a tracked interface. + * + * @param ifname interface name + * + * @return aggregate counter name + */ +std::string dhcp_devman_get_agg_counter_ifname(const std::string &ifname); + /** * @code dhcp_devman_print_all_status(type); * diff --git a/src/dhcp_mon.cpp b/src/dhcp_mon.cpp index ef4f6623d..4a645cae7 100644 --- a/src/dhcp_mon.cpp +++ b/src/dhcp_mon.cpp @@ -5,8 +5,15 @@ */ #include +#include +#include #include +#include #include +#include +#include +#include +#include #include #include #include @@ -41,12 +48,20 @@ static constexpr int MINIMAL_CLEAR_COUNTER_TIMEOUT_SEC = 5; static constexpr int CLEAR_COUNTER_DELAY_AFTER_DB_UPDATE_SEC = 1; /** Mutex lock to modify write_counter_to_db for different threads */ static std::mutex db_sync_mutex; +static std::atomic health_reset_pending{false}; /** tag for db_update event */ static const char db_update_tag[] = "DB_UPDATE"; /** Latest timestamp of writing cache counter to COUNTERS_DB */ static std::chrono::steady_clock::time_point last_update_time{}; /** Default time point to check whether a time_point has been initialized or updated yet. */ static const std::chrono::steady_clock::time_point default_time_point{}; +static std::thread::id main_thread_id; +static bool topology_refresh_pending = false; +static bool config_subscribers_failed = false; +static std::shared_ptr vlan_member_subscriber; +static std::shared_ptr portchannel_member_subscriber; +static const char config_event_tag[] = "CONFIG_UPDATE"; +static int dhcp_mon_reconcile_topology(); std::shared_ptr mConfigDbPtr = std::make_shared ("CONFIG_DB", 0); std::shared_ptr mCountersDbPtr = std::make_shared ("COUNTERS_DB", 0); @@ -55,6 +70,62 @@ std::shared_ptr mStateDbMuxTablePtr = std::make_shared mStateDbPtr.get(), "HW_MUX_CABLE_TABLE" ); +static void config_update_callback(evutil_socket_t, short, void *arg) +{ + auto *subscriber = static_cast(arg); + try { + subscriber->readData(); + std::deque entries; + subscriber->pops(entries); + if (!entries.empty()) { + topology_refresh_pending = true; + } + } catch (const std::exception &e) { + syslog(LOG_ALERT, "Failed to read DHCP membership update: %s", e.what()); + config_subscribers_failed = true; + topology_refresh_pending = true; + main_event_mgr->suspend_all_events(config_event_tag); + } +} + +static void clear_config_events() +{ + main_event_mgr->del_all_events(config_event_tag); + vlan_member_subscriber.reset(); + portchannel_member_subscriber.reset(); +} + +static int register_config_events() +{ + clear_config_events(); + try { + vlan_member_subscriber = std::make_shared( + mConfigDbPtr.get(), "VLAN_MEMBER"); + portchannel_member_subscriber = std::make_shared( + mConfigDbPtr.get(), "PORTCHANNEL_MEMBER"); + } catch (const std::exception &e) { + syslog(LOG_ALERT, "Failed to initialize DHCP membership subscribers: %s", e.what()); + return -1; + } + + for (const auto &subscriber : {vlan_member_subscriber, portchannel_member_subscriber}) { + struct event *config_event = event_new(main_event_mgr->get_base(), subscriber->getFd(), + EV_READ | EV_PERSIST, config_update_callback, + subscriber.get()); + if (config_event == NULL || + main_event_mgr->add_event(config_event, NULL, config_event_tag) < 0) { + if (config_event != NULL) { + event_free(config_event); + } + syslog(LOG_ALERT, "Failed to register DHCP membership event"); + clear_config_events(); + return -1; + } + } + config_subscribers_failed = false; + return 0; +} + /** * @code recalculate_agg_counter(all_counters); * @@ -81,7 +152,7 @@ static void recalculate_agg_counter(all_counters_t &all_counters) if (mgmt_ifname == context->intf) { continue; } - counter_t &agg_counter = all_counters.at(get_agg_counter_ifname(ifname, context->intf)); + counter_t &agg_counter = all_counters.at(dhcp_devman_get_agg_counter_ifname(ifname)); for (const auto &[msg_type, count] : counter) { agg_counter[msg_type] += count; } @@ -212,9 +283,14 @@ static void cleanup_stale_db_counters() static void signal_callback(evutil_socket_t fd, short event, void *arg) { syslog(LOG_INFO, "Received signal: %s", strsignal(fd)); - - dhcp_devman_print_all_status(DHCP_COUNTERS_CURRENT); - dhcp_devman_print_all_status(DHCP_COUNTERS_CURRENT_V6); + + { + counter_state_write_lock counter_lock; + if (counter_lock.owns_lock()) { + dhcp_devman_print_all_status(DHCP_COUNTERS_CURRENT); + dhcp_devman_print_all_status(DHCP_COUNTERS_CURRENT_V6); + } + } if ((fd == SIGTERM) || (fd == SIGINT)) { syslog(LOG_INFO, "Received signal to stop dhcpmon"); @@ -223,6 +299,10 @@ static void signal_callback(evutil_socket_t fd, short event, void *arg) if (fd == SIGUSR1) { // we need to sync cache counter from COUNTERS_DB syslog(LOG_INFO, "Received signal to stop writing to DB counter"); + counter_state_write_lock counter_lock; + if (!counter_lock.owns_lock()) { + return; + } std::lock_guard lock(db_sync_mutex); sock_mgr_pause_write_cache_to_db(); syslog(LOG_INFO, "Stopped writing to DB counter"); @@ -260,6 +340,10 @@ static void update_cache_counter_callback(evutil_socket_t fd, short event, void syslog(LOG_INFO, "Start updating %s cache counter from DB counter", sock_info.name); + counter_state_write_lock counter_lock; + if (!counter_lock.owns_lock()) { + return; + } std::lock_guard lock(db_sync_mutex); // can only sync db to cache counter and db updater is paused, otherwise its unexpected @@ -373,6 +457,7 @@ static void update_cache_counter_callback(evutil_socket_t fd, short event, void // for discrepency in interface between cache counter and DB counter, we dont handle it in this function // we leave it to db updater to handle it if (sock_mgr_pause_write_cache_to_db_all_cleared()) { + health_reset_pending = true; syslog(LOG_INFO, "All sockets cleared pause_write_cache_to_db, start write back to DB counter from cache counter"); main_event_mgr->activate_all_events(db_update_tag, EV_TIMEOUT); } @@ -393,6 +478,51 @@ static void timeout_callback(evutil_socket_t fd, short event, void *arg) { syslog_debug(LOG_INFO, "Received timeout signal for DHCP relay health check"); + if (health_reset_pending.exchange(false)) { + counter_state_write_lock counter_lock; + if (!counter_lock.owns_lock()) { + return; + } + reset_dhcp_relay_health_state(agg_dev_all); + } + + bool subscribers_available = true; + if (config_subscribers_failed) { + if (register_config_events() < 0) { + topology_refresh_pending = true; + subscribers_available = false; + } else { + topology_refresh_pending = true; + } + } + + if (topology_refresh_pending && subscribers_available) { + if (sock_mgr_suspend_packet_handler() < 0) { + syslog(LOG_ALERT, "Failed to suspend packet handlers for topology refresh"); + dhcp_mon_stop(); + return; + } + int result = dhcp_mon_reconcile_topology(); + if (result == 0) { + reset_dhcp_relay_health_state(agg_dev_all); + sock_mgr_drain_sock_buffer(); + } + if (sock_mgr_resume_packet_handler() < 0) { + syslog(LOG_ALERT, "Failed to resume packet handlers after topology refresh"); + dhcp_mon_stop(); + return; + } + topology_refresh_pending = result != 0; + if (result == 0) { + syslog(LOG_INFO, "Refreshed DHCP interface membership from CONFIG_DB"); + return; + } + } + + counter_state_write_lock counter_lock; + if (!counter_lock.owns_lock()) { + return; + } dhcp_devman_print_all_status_debug(DHCP_COUNTERS_CURRENT); dhcp_devman_print_all_status_debug(DHCP_COUNTERS_SNAPSHOT); dhcp_devman_print_all_status_debug(DHCP_COUNTERS_CURRENT_V6); @@ -418,22 +548,31 @@ static void db_update_callback(evutil_socket_t fd, short event, void *arg) { syslog_debug(LOG_INFO, "Received db update signal"); syslog_debug(LOG_INFO, "Sync cache counter to DB counter"); - std::lock_guard lock(db_sync_mutex); - // If there is clear counter going on and its been longer than expected - // consider the clear counter operation failed so we don't block db update forever - if (!sock_mgr_pause_write_cache_to_db_all_cleared() && last_update_time != default_time_point) { - auto now = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast(now - last_update_time); - if (elapsed.count() >= clear_counter_timeout) { - syslog(LOG_WARNING, "Clear counter going on for too long, abort clear counter"); - sock_mgr_clear_pause_write_cache_to_db(); - } else { - syslog(LOG_INFO, "Clear counter is ongoing, skip syncing write cache counter to DB counter"); + socket_counters_t counters_by_socket; + std::unique_lock lock; + { + counter_state_write_lock counter_lock; + if (!counter_lock.owns_lock()) { return; } + lock = std::unique_lock(db_sync_mutex); + // If there is clear counter going on and its been longer than expected + // consider the clear counter operation failed so we don't block db update forever + if (!sock_mgr_pause_write_cache_to_db_all_cleared() && last_update_time != default_time_point) { + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - last_update_time); + if (elapsed.count() >= clear_counter_timeout) { + syslog(LOG_WARNING, "Clear counter going on for too long, abort clear counter"); + sock_mgr_clear_pause_write_cache_to_db(); + } else { + syslog(LOG_INFO, "Clear counter is ongoing, skip syncing write cache counter to DB counter"); + return; + } + } + counters_by_socket = sock_mgr_copy_cache_counters(); } last_update_time = std::chrono::steady_clock::now(); - sock_mgr_update_db_counters(); + sock_mgr_update_db_counters(counters_by_socket); cleanup_stale_db_counters(); syslog_debug(LOG_INFO, "Successfully synced cache counter to DB counter"); } @@ -454,50 +593,135 @@ static void free_event_mgr(struct event_mgr *mgr) } /** - * @code initialize_all_intf_counters(); - * @brief Initialize all db counters and cache counters for all tracked interfaces - * @param none - * @return 0 upon success, negative upon failure + * @code reconcile_all_intf_counters(initialize_db); + * @brief Reconcile cache counters for all tracked interfaces + * @param initialize_db initialize missing database counters when true + * @return none */ -static void initialize_all_intf_counters() +static void reconcile_all_intf_counters(bool initialize_db) { - for (const auto &[vlan, intfs] : rev_vlan_map) { - for (const auto &ifname : intfs) { + std::unordered_set valid_ifnames; + auto ensure_interface = [&valid_ifnames, initialize_db](const std::string &ifname) { + valid_ifnames.insert(ifname); + if (initialize_db && !all_counters_initialized(ifname)) { initialize_all_counters(ifname); + } else if (!initialize_db && !sock_mgr_all_cache_counters_initialized(ifname)) { + sock_mgr_init_cache_counters(ifname, DHCP_MESSAGE_TYPE_COUNT, DHCPV6_MESSAGE_TYPE_COUNT); } - initialize_all_counters(vlan); - sock_mgr_init_cache_counters(agg_dev_prefix + vlan, DHCP_MESSAGE_TYPE_COUNT, DHCPV6_MESSAGE_TYPE_COUNT); - } + }; + auto ensure_aggregate = [&valid_ifnames](const std::string &ifname) { + valid_ifnames.insert(ifname); + if (!sock_mgr_all_cache_counters_initialized(ifname)) { + sock_mgr_init_cache_counters(ifname, DHCP_MESSAGE_TYPE_COUNT, DHCPV6_MESSAGE_TYPE_COUNT); + } + }; - for (const auto &[portchan, intfs] : rev_portchan_map) { - for (const auto &ifname : intfs) { - initialize_all_counters(ifname); + for (const auto &[vlan, members] : rev_vlan_map) { + for (const auto &ifname : members) { + ensure_interface(ifname); } - initialize_all_counters(portchan); - sock_mgr_init_cache_counters(agg_dev_prefix + portchan, DHCP_MESSAGE_TYPE_COUNT, DHCPV6_MESSAGE_TYPE_COUNT); + ensure_interface(vlan); + ensure_aggregate(agg_dev_prefix + vlan); } - // Now all vlan and portchannel related interfaces have entries in counters, now do the rest (uplink) - for (const auto &itr : intfs) { - if (!all_counters_initialized(itr.first)) { - initialize_all_counters(itr.first); + for (const auto &[portchan, members] : rev_portchan_map) { + for (const auto &ifname : members) { + ensure_interface(ifname); } + ensure_interface(portchan); + ensure_aggregate(agg_dev_prefix + portchan); + } + + for (const auto &entry : intfs) { + ensure_interface(entry.first); } - // also initialize mgmt and agg device counters if (mgmt_ifname.size() > 0) { - initialize_all_counters(mgmt_ifname); + ensure_interface(mgmt_ifname); } + ensure_aggregate(agg_dev_all); - sock_mgr_init_cache_counters(agg_dev_all, DHCP_MESSAGE_TYPE_COUNT, DHCPV6_MESSAGE_TYPE_COUNT); + sock_mgr_remove_cache_counters_except(valid_ifnames); + for (int sock : {rx_sock, tx_sock, rx_sock_v6, tx_sock_v6}) { + sock_info_t &sock_info = sock_mgr_get_sock_info(sock); + recalculate_agg_counter(sock_info.all_counters); + recalculate_agg_counter(sock_info.all_counters_snapshot); + } - // counter db (the interfaces) might be outdated, clean up stale entries to be in sync with current tracked interfaces - cleanup_stale_db_counters(); + if (initialize_db) { + cleanup_stale_db_counters(); + } +} + +static int dhcp_mon_reconcile_topology() +{ + if (std::this_thread::get_id() != main_thread_id) { + syslog(LOG_ALERT, "Topology reconciliation must run on the main event-loop thread"); + return -1; + } + if (packet_handlers_enabled.load(std::memory_order_acquire)) { + syslog(LOG_ALERT, "Topology reconciliation requires suspended packet handlers"); + return -1; + } + + std::lock_guard lock(db_sync_mutex); + if (!sock_mgr_pause_write_cache_to_db_all_cleared()) { + return 1; + } + + decltype(vlan_map) old_vlan_map; + decltype(portchan_map) old_portchan_map; + decltype(rev_vlan_map) old_rev_vlan_map; + decltype(rev_portchan_map) old_rev_portchan_map; + std::unordered_map> old_counters; + try { + old_vlan_map = vlan_map; + old_portchan_map = portchan_map; + old_rev_vlan_map = rev_vlan_map; + old_rev_portchan_map = rev_portchan_map; + for (int sock : {rx_sock, tx_sock, rx_sock_v6, tx_sock_v6}) { + sock_info_t &sock_info = sock_mgr_get_sock_info(sock); + old_counters[sock] = {sock_info.all_counters, sock_info.all_counters_snapshot}; + } + } catch (const std::exception &e) { + syslog(LOG_ALERT, "Failed to snapshot DHCP topology before reconciliation: %s", e.what()); + return -1; + } + + try { + dhcp_devman_refresh_mappings(); + reconcile_all_intf_counters(false); + mCountersDbPtr = std::make_shared("COUNTERS_DB", 0); + sock_mgr_update_db_counters(); + cleanup_stale_db_counters(); + sock_mgr_update_snapshot(); + } catch (const std::exception &e) { + syslog(LOG_ALERT, "Failed to reconcile DHCP interface membership: %s", e.what()); + vlan_map = std::move(old_vlan_map); + portchan_map = std::move(old_portchan_map); + rev_vlan_map = std::move(old_rev_vlan_map); + rev_portchan_map = std::move(old_rev_portchan_map); + for (auto &[sock, counters] : old_counters) { + sock_info_t &sock_info = sock_mgr_get_sock_info(sock); + sock_info.all_counters = std::move(counters.first); + sock_info.all_counters_snapshot = std::move(counters.second); + } + try { + mCountersDbPtr = std::make_shared("COUNTERS_DB", 0); + sock_mgr_update_db_counters(); + cleanup_stale_db_counters(); + } catch (const std::exception &rollback_error) { + syslog(LOG_ALERT, "Failed to restore COUNTERS_DB after topology rollback: %s", rollback_error.what()); + } + return -1; + } + return 0; } int dhcp_mon_init(size_t snaplen, int window_sec, int max_count, int db_update_interval) { int rv = -1; + main_thread_id = std::this_thread::get_id(); syslog(LOG_INFO, "Initializing dhcp monitor with snaplen %zu, window_sec %d, max_count %d, db_update_interval %d", snaplen, window_sec, max_count, db_update_interval); @@ -519,7 +743,8 @@ int dhcp_mon_init(size_t snaplen, int window_sec, int max_count, int db_update_i // deinitialization of counters is not our responsibility // cache counter will be cleanup by sock_mgr_free and the initialized db we intend to keep - initialize_all_intf_counters(); + reconcile_all_intf_counters(true); + reset_dhcp_relay_health_state(agg_dev_all); syslog(LOG_INFO, "Initialized all counters for tracked interfaces"); window_interval_sec = window_sec; @@ -667,6 +892,10 @@ static int register_main_events() break; } + if (register_config_events() < 0) { + break; + } + rv = 0; syslog(LOG_INFO, "Main events registered successfully"); @@ -693,6 +922,8 @@ static int register_main_events() int dhcp_mon_start() { int rv = -1; + int reconcile_result = -1; + int resume_result = -1; syslog(LOG_INFO, "Starting dhcp monitor in %s", debug_on ? "debug mode" : "normal mode"); @@ -711,6 +942,17 @@ int dhcp_mon_start() goto unregister_cache_counter_updater; } + topology_refresh_pending = true; + if (sock_mgr_suspend_packet_handler() < 0) { + goto unregister_main_events; + } + reconcile_result = dhcp_mon_reconcile_topology(); + resume_result = sock_mgr_resume_packet_handler(); + if (reconcile_result != 0 || resume_result < 0) { + goto unregister_main_events; + } + topology_refresh_pending = false; + sock_mgr_drain_sock_buffer(); // it could fail and we wouldnt know it because its in another thread diff --git a/src/event_mgr.cpp b/src/event_mgr.cpp index 6e1e0e76c..575b3ac15 100644 --- a/src/event_mgr.cpp +++ b/src/event_mgr.cpp @@ -1,4 +1,5 @@ #include +#include #include "event_mgr.h" @@ -69,18 +70,22 @@ int event_mgr::add_event(struct event* event, const struct timeval *timeout, con void event_mgr::del_all_events(const std::string &tag) { int count = 0; - for (const auto &event : this->event_map[tag]) { + const auto tagged_events = this->event_map.find(tag); + if (tagged_events == this->event_map.end()) { + return; + } + auto all_events = this->event_map.find(""); + for (const auto &event : tagged_events->second) { + int fd = event_get_fd(event); + if (!tag.empty() && all_events != this->event_map.end()) { + all_events->second.erase(event); + } event_del(event); event_free(event); count++; - syslog(LOG_INFO, "event_mgr: Deleted event (fd=%d) of tag %s from %s", event_get_fd(event), tag.c_str(), this->name.c_str()); + syslog(LOG_INFO, "event_mgr: Deleted event (fd=%d) of tag %s from %s", fd, tag.c_str(), this->name.c_str()); } - if (tag != "") { - std::unordered_set &tagless_set = this->event_map[""]; - std::unordered_set &tagged_set = this->event_map[tag]; - for (const auto &event : tagged_set) { - tagless_set.erase(event); - } + if (!tag.empty()) { this->event_map.erase(tag); } else { this->event_map.clear(); @@ -88,6 +93,74 @@ void event_mgr::del_all_events(const std::string &tag) syslog(LOG_INFO, "event_mgr: Deleted %d events of tag %s for %s", count, tag.c_str(), this->name.c_str()); } +int event_mgr::suspend_all_events(const std::string &tag) +{ + if (tag.empty()) { + syslog(LOG_ALERT, "event_mgr: Refusing to suspend untagged events for %s", + this->name.c_str()); + return -1; + } + const auto tagged_events = this->event_map.find(tag); + if (tagged_events == this->event_map.end()) { + syslog(LOG_ALERT, "event_mgr: Cannot suspend unknown tag %s for %s", + tag.c_str(), this->name.c_str()); + return -1; + } + for (const auto &event : tagged_events->second) { + if (event_get_fd(event) < 0) { + syslog(LOG_ALERT, "event_mgr: Cannot suspend non-fd event with tag %s for %s", + tag.c_str(), this->name.c_str()); + return -1; + } + } + std::vector deleted_events; + for (const auto &event : tagged_events->second) { + if (event_del(event) < 0) { + bool restore_failed = false; + for (struct event *deleted_event : deleted_events) { + if (event_add(deleted_event, NULL) < 0) { + restore_failed = true; + } + } + syslog(LOG_ALERT, "event_mgr: Failed to suspend event (fd=%d) with tag %s for %s", + event_get_fd(event), tag.c_str(), this->name.c_str()); + return restore_failed ? -2 : -1; + } + deleted_events.push_back(event); + } + return 0; +} + +int event_mgr::resume_all_events(const std::string &tag) +{ + if (tag.empty()) { + syslog(LOG_ALERT, "event_mgr: Refusing to resume untagged events for %s", + this->name.c_str()); + return -1; + } + const auto tagged_events = this->event_map.find(tag); + if (tagged_events == this->event_map.end()) { + syslog(LOG_ALERT, "event_mgr: Cannot resume unknown tag %s for %s", + tag.c_str(), this->name.c_str()); + return -1; + } + for (const auto &event : tagged_events->second) { + if (event_get_fd(event) < 0) { + syslog(LOG_ALERT, "event_mgr: Cannot resume non-fd event with tag %s for %s", + tag.c_str(), this->name.c_str()); + this->suspend_all_events(tag); + return -1; + } + if (event_add(event, NULL) < 0) { + syslog(LOG_ALERT, "event_mgr: Failed to resume event (fd=%d) with tag %s for %s", + event_get_fd(event), tag.c_str(), this->name.c_str()); + this->suspend_all_events(tag); + return -1; + } + } + return 0; +} + /** * @code activate_all_events(tag, res); * @@ -97,7 +170,15 @@ void event_mgr::del_all_events(const std::string &tag) */ void event_mgr::activate_all_events(const std::string &tag, int res) { - for (const auto &event : this->event_map[tag]) { + const auto tagged_events = this->event_map.find(tag); + if (tagged_events == this->event_map.end()) { + if (!tag.empty()) { + syslog(LOG_WARNING, "event_mgr: Cannot activate unknown tag %s for %s", + tag.c_str(), this->name.c_str()); + } + return; + } + for (const auto &event : tagged_events->second) { event_active(event, res, 0); syslog(LOG_INFO, "event_mgr: Activated event (fd=%d) of tag %s from %s", event_get_fd(event), tag.c_str(), this->name.c_str()); } diff --git a/src/event_mgr.h b/src/event_mgr.h index 90ff4a146..cca15edf8 100644 --- a/src/event_mgr.h +++ b/src/event_mgr.h @@ -12,6 +12,8 @@ class event_mgr { int init_base(); int add_event(struct event* event, const struct timeval *timeout, const std::string &tag=""); void del_all_events(const std::string &tag=""); + int suspend_all_events(const std::string &tag); + int resume_all_events(const std::string &tag); void activate_all_events(const std::string &tag="", int res=0); void free(); struct event_base* get_base(); diff --git a/src/health_check.cpp b/src/health_check.cpp index e9949d18c..6cff3e37e 100644 --- a/src/health_check.cpp +++ b/src/health_check.cpp @@ -4,6 +4,8 @@ */ #include +#include +#include #include #include #include @@ -27,10 +29,8 @@ extern std::string agg_dev_prefix; extern std::unordered_map> rev_vlan_map; extern std::unordered_map> rev_portchan_map; -static dhcp_mon_status_t check_agg_health() -{ - return dhcp_device_get_status(agg_dev_all, DHCP_DEVICE_CHECK_POSITIVE); -} +static std::unordered_set reported_disparity_v4; +static std::mutex health_state_mutex; static dhcp_mon_status_t check_mgmt_health() { @@ -48,7 +48,7 @@ static void alert_dhcp_relay_disparity(int duration) static void log_agg_error(int duration) { - syslog(LOG_ALERT, "dhcpmon detected DHCPv4/v6 packets received but none transmitted. Duration: %d (sec) for intf: %s", + syslog(LOG_ALERT, "dhcpmon detected DHCPv4 receive activity without a corresponding transmit. Duration: %d (sec) for intf: %s", duration, agg_dev_all.c_str()); } @@ -59,9 +59,27 @@ static void log_mgmt_error(int duration) duration, mgmt_ifname.c_str()); } -static dhcp_mon_status_t check_agg_health_v6() +static void check_relay_disparity() { - return dhcp_device_get_status(agg_dev_all, DHCP_DEVICE_CHECK_POSITIVE_V6); + auto windows_by_type = dhcp_device_get_untransmitted_windows(agg_dev_all); + uint32_t report_windows = 0; + + for (const auto &[msg_type, windows] : windows_by_type) { + if (windows == 0) { + reported_disparity_v4.erase(msg_type); + continue; + } + if (windows > static_cast(dhcp_unhealthy_max_count) && + reported_disparity_v4.insert(msg_type).second) { + report_windows = std::max(report_windows, windows); + } + } + + if (report_windows > 0) { + int duration = static_cast(report_windows) * window_interval_sec; + alert_dhcp_relay_disparity(duration); + log_agg_error(duration); + } } static dhcp_mon_status_t check_mgmt_health_v6() @@ -90,7 +108,7 @@ static dhcp_mon_status_t check_per_interface_rx_health() static void log_agg_per_interface_rx_error(int duration) { syslog(LOG_ALERT, "sum of rx per interface counter does not equal corresponding vlan/portchan counter." - " Duration: %d (sec)", duration); + " Duration: %d (sec). %s", duration, dhcp_device_get_last_counter_mismatch().c_str()); } static dhcp_mon_status_t check_per_interface_tx_health() @@ -112,7 +130,7 @@ static void log_agg_per_interface_tx_error(int duration) { syslog(LOG_ALERT, "each tx per interface counter does not equal corresponding vlan counter," " or sum of tx per interface counter does not equal corresponding portchan counter." - " Duration: %d (sec)", duration); + " Duration: %d (sec). %s", duration, dhcp_device_get_last_counter_mismatch().c_str()); } static dhcp_mon_status_t check_per_interface_rx_health_v6() @@ -145,49 +163,43 @@ static dhcp_mon_status_t check_per_interface_tx_health_v6() return DHCP_MON_STATUS_HEALTHY; } -/** DHCP monitor state data for aggregate device for mgmt device */ +/** DHCP monitor state for management traffic and interface hierarchy consistency */ static dhcp_mon_state_t state_data[] = { [0] = { - .check_health = check_agg_health, - .alert = alert_dhcp_relay_disparity, - .log = log_agg_error, - .count = 0, - }, - [1] = { .check_health = check_mgmt_health, .log = log_mgmt_error, .count = 0, + .reported = false, }, - [2] = { - .check_health = check_agg_health_v6, - .alert = alert_dhcp_relay_disparity, - .log = log_agg_error, - .count = 0, - }, - [3] = { + [1] = { .check_health = check_mgmt_health_v6, .log = log_mgmt_error, .count = 0, + .reported = false, }, - [4] = { + [2] = { .check_health = check_per_interface_rx_health, .log = log_agg_per_interface_rx_error, .count = 0, + .reported = false, }, - [5] = { + [3] = { .check_health = check_per_interface_tx_health, .log = log_agg_per_interface_tx_error, .count = 0, + .reported = false, }, - [6] = { + [4] = { .check_health = check_per_interface_rx_health_v6, .log = log_agg_per_interface_rx_error, .count = 0, + .reported = false, }, - [7] = { + [5] = { .check_health = check_per_interface_tx_health_v6, .log = log_agg_per_interface_tx_error, .count = 0, + .reported = false, }, }; @@ -195,13 +207,16 @@ static size_t state_data_sz = sizeof(state_data) / sizeof(*state_data); void check_dhcp_relay_health() { + std::lock_guard lock(health_state_mutex); syslog_debug(LOG_INFO, "Checking DHCP relay health"); + check_relay_disparity(); + for (uint8_t i = 0; i < state_data_sz; i++) { dhcp_mon_status_t dhcp_mon_status = state_data[i].check_health(); switch (dhcp_mon_status) { case DHCP_MON_STATUS_UNHEALTHY: - if (++state_data[i].count > dhcp_unhealthy_max_count) { + if (++state_data[i].count > dhcp_unhealthy_max_count && !state_data[i].reported) { int duration = state_data[i].count * window_interval_sec; if (state_data[i].alert) { @@ -210,10 +225,12 @@ void check_dhcp_relay_health() if (state_data[i].log) { state_data[i].log(duration); } + state_data[i].reported = true; } break; case DHCP_MON_STATUS_HEALTHY: state_data[i].count = 0; + state_data[i].reported = false; break; case DHCP_MON_STATUS_INDETERMINATE: if (state_data[i].count) { @@ -227,4 +244,15 @@ void check_dhcp_relay_health() } syslog_debug(LOG_INFO, "Completed DHCP relay health check"); +} + +void reset_dhcp_relay_health_state(const std::string &ifname) +{ + std::lock_guard lock(health_state_mutex); + reported_disparity_v4.clear(); + for (auto &state : state_data) { + state.count = 0; + state.reported = false; + } + dhcp_device_reset_health_state(ifname); } \ No newline at end of file diff --git a/src/health_check.h b/src/health_check.h index 552dd528c..7d4550ccb 100644 --- a/src/health_check.h +++ b/src/health_check.h @@ -8,6 +8,8 @@ #include "dhcp_device.h" +#include + #include /** DHCP device/interface state */ @@ -17,6 +19,7 @@ typedef struct void (*alert)(int duration); /** alert function when check failed */ void (*log)(int duration); /** log function when check passed */ int count; /** count in the number of unhealthy checks */ + bool reported; /** whether the current unhealthy episode was reported */ } dhcp_mon_state_t; extern event_handle_t g_events_handle; @@ -36,4 +39,7 @@ extern int dhcp_unhealthy_max_count; */ void check_dhcp_relay_health(); +/** Reset relay report state globally and flow watermarks for the given interface */ +void reset_dhcp_relay_health_state(const std::string &ifname); + #endif // HEALTH_CHECK_H \ No newline at end of file diff --git a/src/packet_handler.cpp b/src/packet_handler.cpp index 7ec5d07a3..e81dd90db 100644 --- a/src/packet_handler.cpp +++ b/src/packet_handler.cpp @@ -16,6 +16,8 @@ #include "dhcp_check_profile.h" /** to get dhcp/v6 check profile */ #include "util.h" +static constexpr int MAX_PACKETS_PER_CALLBACK = 64; + /** * @code _increase_cache_counter(ifname, sock, type); * @brief helper function to increase cache counter. Simple increase of counter, no complications. In the event of @@ -47,14 +49,12 @@ static void increase_cache_counter(const std::string &ifname, const dhcp_device_ { _increase_cache_counter(ifname, sock, type); - // we seperate mgmt interface from others and do not increase agg counter + // we separate mgmt interface from others and do not increase agg counter if (mgmt_ifname != "" && mgmt_ifname.compare(context->intf) == 0) { return; } - // when ifname belongs to another context ifname, increase the aggregate counter for that context, - // else when ifname is the context, we increase agg counter for all. - _increase_cache_counter(get_agg_counter_ifname(ifname, context->intf), sock, type); + _increase_cache_counter(dhcp_devman_get_agg_counter_ifname(ifname), sock, type); // optionally duplicate to context ifname, it will only be true when this is standby physical interface under a vlan on a dual tor if (dup_to_context) { @@ -859,13 +859,22 @@ void packet_handler_v6(int sock, const std::string &ifname, const dhcp_device_co void callback_common(int fd, short event, void *arg) { + counter_state_read_lock counter_lock; + if (!counter_lock.owns_lock()) { + return; + } ssize_t buffer_sz; struct sockaddr_ll sll; socklen_t slen = sizeof(sll); sock_info_t &sock_info = sock_mgr_get_sock_info(fd); - while ((buffer_sz = recvfrom(fd, sock_info.buffer, sock_info.snaplen, MSG_DONTWAIT, (struct sockaddr *)&sll, &slen)) > 0) - { + for (int packet_count = 0; packet_count < MAX_PACKETS_PER_CALLBACK; packet_count++) { + slen = sizeof(sll); + buffer_sz = recvfrom(fd, sock_info.buffer, sock_info.snaplen, MSG_DONTWAIT, + (struct sockaddr *)&sll, &slen); + if (buffer_sz <= 0) { + break; + } char ifname_buf[IF_NAMESIZE]; if (if_indextoname(sll.sll_ifindex, ifname_buf) == NULL) { syslog_debug(LOG_WARNING, "if_indextoname: invalid input interface index %d %s", sll.sll_ifindex, strerror(errno)); diff --git a/src/sock_mgr.cpp b/src/sock_mgr.cpp index 8d3e48d81..0925b55f3 100644 --- a/src/sock_mgr.cpp +++ b/src/sock_mgr.cpp @@ -11,7 +11,11 @@ #include #include #include +#include +#include +#include #include +#include #include "sock_mgr.h" @@ -33,17 +37,134 @@ static const char dhcp_outbound_filter[] = "outbound and ip and udp and (port 67 static const char dhcpv6_inbound_filter[] = "inbound and ip6 and udp and (port 547 or port 546)"; static const char dhcpv6_outbound_filter[] = "outbound and ip6 and udp and (port 547 or port 546)"; -/** Tags for different events, so we can triiger only one type */ +/** Tags for different events, so we can trigger only one type */ static const char packet_handler_tag[] = "PacketHandler"; static const char cache_counter_updater_tag[] = "CacheCounterUpdater"; +static const char keepalive_tag[] = "Keepalive"; + +static void keepalive_callback(evutil_socket_t, short, void *) +{ +} /* sock fd to sock_info mapping */ std::unordered_map sock_map; +std::shared_mutex packet_handler_quiesce_mutex; +std::atomic packet_handlers_enabled{true}; +std::atomic counter_state_writers_pending{0}; +static std::unique_lock packet_handler_quiesce_lock; +static std::mutex counter_state_wait_mutex; +static std::condition_variable counter_state_wait_cv; + extern std::shared_ptr mCountersDbPtr; extern std::string downstream_ifname; +static void set_packet_handlers_enabled(bool enabled) +{ + { + std::lock_guard wait_lock(counter_state_wait_mutex); + packet_handlers_enabled.store(enabled, std::memory_order_release); + } + counter_state_wait_cv.notify_all(); +} + +counter_state_write_lock::counter_state_write_lock() +{ + { + std::lock_guard wait_lock(counter_state_wait_mutex); + counter_state_writers_pending.fetch_add(1, std::memory_order_acq_rel); + } + try { + lock = std::unique_lock(packet_handler_quiesce_mutex); + } catch (const std::system_error &e) { + bool notify = false; + { + std::lock_guard wait_lock(counter_state_wait_mutex); + notify = counter_state_writers_pending.fetch_sub(1, std::memory_order_acq_rel) == 1; + } + if (notify) { + counter_state_wait_cv.notify_all(); + } + syslog(LOG_ALERT, "Failed to lock DHCP counter state: %s", e.what()); + } +} + +counter_state_write_lock::~counter_state_write_lock() +{ + if (!lock.owns_lock()) { + return; + } + lock.unlock(); + bool notify = false; + { + std::lock_guard wait_lock(counter_state_wait_mutex); + notify = counter_state_writers_pending.fetch_sub(1, std::memory_order_acq_rel) == 1; + } + if (notify) { + counter_state_wait_cv.notify_all(); + } +} + +bool counter_state_write_lock::owns_lock() const +{ + return lock.owns_lock(); +} + +counter_state_read_lock::counter_state_read_lock() +{ + if (packet_handlers_enabled.load(std::memory_order_acquire) && + counter_state_writers_pending.load(std::memory_order_acquire) == 0) { + try { + lock = std::shared_lock(packet_handler_quiesce_mutex, + std::try_to_lock); + } catch (const std::system_error &e) { + syslog(LOG_ALERT, "Failed to lock DHCP counter state for packet handling: %s", e.what()); + return; + } + if (lock.owns_lock() && + packet_handlers_enabled.load(std::memory_order_acquire) && + counter_state_writers_pending.load(std::memory_order_acquire) == 0) { + return; + } + if (lock.owns_lock()) { + lock.unlock(); + } + } + + while (packet_handlers_enabled.load(std::memory_order_acquire)) { + { + std::unique_lock wait_lock(counter_state_wait_mutex); + counter_state_wait_cv.wait(wait_lock, [] { + return !packet_handlers_enabled.load(std::memory_order_acquire) || + counter_state_writers_pending.load(std::memory_order_acquire) == 0; + }); + } + if (!packet_handlers_enabled.load(std::memory_order_acquire)) { + return; + } + try { + lock = std::shared_lock(packet_handler_quiesce_mutex); + } catch (const std::system_error &e) { + syslog(LOG_ALERT, "Failed to lock DHCP counter state for packet handling: %s", e.what()); + return; + } + if (!packet_handlers_enabled.load(std::memory_order_acquire)) { + lock.unlock(); + return; + } + if (counter_state_writers_pending.load(std::memory_order_acquire) == 0) { + return; + } + lock.unlock(); + } +} + +bool counter_state_read_lock::owns_lock() const +{ + return lock.owns_lock(); +} + /** * @code opensocket(); * @@ -385,6 +506,18 @@ int sock_mgr_init_event_mgr() sock_mgr_free_event_mgr(); return -1; } + struct event *keepalive_event = event_new(info.event_mgr_ptr->get_base(), -1, EV_PERSIST, + keepalive_callback, NULL); + struct timeval keepalive_interval = {.tv_sec = 3600, .tv_usec = 0}; + if (keepalive_event == NULL || + info.event_mgr_ptr->add_event(keepalive_event, &keepalive_interval, keepalive_tag) < 0) { + if (keepalive_event != NULL) { + event_free(keepalive_event); + } + syslog(LOG_ALERT, "Failed to initialize event manager keepalive %s", info.name); + sock_mgr_free_event_mgr(); + return -1; + } } return 0; @@ -432,6 +565,90 @@ void sock_mgr_unregister_packet_handler() } } +int sock_mgr_suspend_packet_handler() +{ + if (packet_handler_quiesce_lock.owns_lock()) { + syslog(LOG_ALERT, "Packet handlers are already suspended"); + return -1; + } + std::vector suspended_event_mgrs; + for (const auto &entry : sock_map) { + event_mgr *event_mgr_ptr = entry.second.event_mgr_ptr; + int suspend_result = event_mgr_ptr->suspend_all_events(packet_handler_tag); + if (suspend_result < 0) { + bool rollback_failed = suspend_result < -1; + for (event_mgr *suspended_event_mgr : suspended_event_mgrs) { + if (suspended_event_mgr->resume_all_events(packet_handler_tag) < 0) { + rollback_failed = true; + } + } + if (rollback_failed) { + set_packet_handlers_enabled(false); + for (const auto &rollback_entry : sock_map) { + rollback_entry.second.event_mgr_ptr->suspend_all_events(packet_handler_tag); + } + try { + packet_handler_quiesce_lock = + std::unique_lock(packet_handler_quiesce_mutex); + } catch (const std::system_error &e) { + syslog(LOG_ALERT, "Failed to quiesce packet handlers after suspend rollback failure: %s", + e.what()); + } + } + return -1; + } + suspended_event_mgrs.push_back(event_mgr_ptr); + } + set_packet_handlers_enabled(false); + try { + packet_handler_quiesce_lock = std::unique_lock(packet_handler_quiesce_mutex); + } catch (const std::system_error &e) { + syslog(LOG_ALERT, "Failed to quiesce packet handlers: %s", e.what()); + set_packet_handlers_enabled(true); + int restore_result = 0; + for (const auto &entry : sock_map) { + if (entry.second.event_mgr_ptr->resume_all_events(packet_handler_tag) < 0) { + restore_result = -1; + } + } + if (restore_result < 0) { + set_packet_handlers_enabled(false); + for (const auto &entry : sock_map) { + entry.second.event_mgr_ptr->suspend_all_events(packet_handler_tag); + } + syslog(LOG_ALERT, "Failed to restore packet handlers after quiesce failure"); + } + return -1; + } + return 0; +} + +int sock_mgr_resume_packet_handler() +{ + if (!packet_handler_quiesce_lock.owns_lock()) { + syslog(LOG_ALERT, "Packet handlers are not suspended"); + return -1; + } + set_packet_handlers_enabled(true); + packet_handler_quiesce_lock.unlock(); + + for (const auto &entry : sock_map) { + if (entry.second.event_mgr_ptr->resume_all_events(packet_handler_tag) < 0) { + set_packet_handlers_enabled(false); + for (const auto &suspended_entry : sock_map) { + suspended_entry.second.event_mgr_ptr->suspend_all_events(packet_handler_tag); + } + try { + packet_handler_quiesce_lock = std::unique_lock(packet_handler_quiesce_mutex); + } catch (const std::system_error &e) { + syslog(LOG_ALERT, "Failed to restore packet quiesce lock after resume failure: %s", e.what()); + } + return -1; + } + } + return 0; +} + int sock_mgr_register_cache_counter_updater(event_callback_fn callback) { syslog(LOG_INFO, "Registering cache counter updater for all sockets"); @@ -605,7 +822,8 @@ void sock_mgr_init_cache_counters(const std::string &ifname, uint8_t dhcp_messag bool sock_mgr_all_cache_counters_initialized(const std::string &ifname) { - for (const auto &[sock, info] : sock_map) { + for (const auto &entry : sock_map) { + const auto &info = entry.second; auto itr = info.all_counters.find(ifname); if (itr == info.all_counters.end()) { return false; @@ -614,17 +832,53 @@ bool sock_mgr_all_cache_counters_initialized(const std::string &ifname) return true; } -void sock_mgr_update_db_counters() +void sock_mgr_remove_cache_counters_except(const std::unordered_set &valid_ifnames) { - syslog_debug(LOG_INFO, "Updating all cache counters to DB counters"); + for (auto &entry : sock_map) { + auto &info = entry.second; + for (auto itr = info.all_counters.begin(); itr != info.all_counters.end();) { + if (valid_ifnames.find(itr->first) == valid_ifnames.end()) { + itr = info.all_counters.erase(itr); + } else { + itr++; + } + } + for (auto itr = info.all_counters_snapshot.begin(); itr != info.all_counters_snapshot.end();) { + if (valid_ifnames.find(itr->first) == valid_ifnames.end()) { + itr = info.all_counters_snapshot.erase(itr); + } else { + itr++; + } + } + } +} +socket_counters_t sock_mgr_copy_cache_counters() +{ + socket_counters_t counters_by_socket; for (const auto &[sock, info] : sock_map) { + counters_by_socket.emplace(sock, info.all_counters); + } + return counters_by_socket; +} + +void sock_mgr_update_db_counters(const socket_counters_t &counters_by_socket) +{ + syslog_debug(LOG_INFO, "Updating all cache counters to DB counters"); + + for (const auto &[sock, all_counters] : counters_by_socket) { + const auto sock_info = sock_map.find(sock); + if (sock_info == sock_map.end()) { + syslog(LOG_WARNING, "Skip DB counter snapshot for unknown socket %d", sock); + continue; + } + const sock_info_t &info = sock_info->second; syslog_debug(LOG_INFO, "Start updating socket %d %s DB counter from cache counter", sock, info.name); int msg_type_count = info.is_v6 ? DHCPV6_MESSAGE_TYPE_COUNT : DHCP_MESSAGE_TYPE_COUNT; const std::string *msg_type_name = info.is_v6 ? db_counter_name_v6 : db_counter_name; std::string all_ifname; std::string all_skipped_ifname; - for (const auto &[ifname, counter] : info.all_counters) { + for (const auto &[ifname, counter] : all_counters) { if (is_agg_counter(ifname) == true) { all_skipped_ifname += ifname + ", "; continue; @@ -634,9 +888,14 @@ void sock_mgr_update_db_counters() std::string table_name = construct_counter_db_table_key(ifname, info.is_v6); mCountersDbPtr->hset(table_name, info.is_rx ? "RX" : "TX", value); } - syslog_debug(LOG_INFO, "Processing cache counter entry of %sfor downstream vlan %s", + syslog_debug(LOG_INFO, "Processing cache counter entry of %s for downstream vlan %s", all_ifname.c_str(), downstream_ifname.c_str()); - syslog_debug(LOG_INFO, "Skipped aggregated device counter entry of %sfor downstream vlan %s", + syslog_debug(LOG_INFO, "Skipped aggregated device counter entry of %s for downstream vlan %s", all_skipped_ifname.c_str(), downstream_ifname.c_str()); } +} + +void sock_mgr_update_db_counters() +{ + sock_mgr_update_db_counters(sock_mgr_copy_cache_counters()); } \ No newline at end of file diff --git a/src/sock_mgr.h b/src/sock_mgr.h index 9619a9526..61908c415 100644 --- a/src/sock_mgr.h +++ b/src/sock_mgr.h @@ -9,9 +9,13 @@ #ifndef SOCKET_MANAGER_H_ #define SOCKET_MANAGER_H_ +#include +#include #include +#include #include #include +#include #include #include @@ -19,6 +23,7 @@ typedef std::unordered_map counter_t; typedef std::unordered_map all_counters_t; +typedef std::unordered_map socket_counters_t; /** struct for socket information */ typedef struct { @@ -41,6 +46,34 @@ typedef struct { /** sock file descriptors, serve as the identifier of all related information described in sock_info_t */ extern int rx_sock, tx_sock, rx_sock_v6, tx_sock_v6; +/** Guards in-flight packet callbacks while topology and counters are reconciled */ +extern std::shared_mutex packet_handler_quiesce_mutex; +extern std::atomic packet_handlers_enabled; +extern std::atomic counter_state_writers_pending; + +class counter_state_write_lock +{ + public: + counter_state_write_lock(); + ~counter_state_write_lock(); + bool owns_lock() const; + counter_state_write_lock(const counter_state_write_lock &) = delete; + counter_state_write_lock &operator=(const counter_state_write_lock &) = delete; + + private: + std::unique_lock lock; +}; + +class counter_state_read_lock +{ + public: + counter_state_read_lock(); + bool owns_lock() const; + + private: + std::shared_lock lock; +}; + /** Initialize socket manager with given snaplen */ int sock_mgr_init(uint32_t snaplen); @@ -59,6 +92,12 @@ int sock_mgr_register_packet_handler(); /** Unregister packet handler for socket manager */ void sock_mgr_unregister_packet_handler(); +/** Temporarily suspend registered packet handlers */ +int sock_mgr_suspend_packet_handler(); + +/** Resume registered packet handlers */ +int sock_mgr_resume_packet_handler(); + /** Register cache counter updater callback for socket manager */ int sock_mgr_register_cache_counter_updater(event_callback_fn callback); @@ -104,7 +143,14 @@ void sock_mgr_init_cache_counters(const std::string &ifname, uint8_t dhcp_messag /** Check if cache counters are initialized for given ifname for all sockets */ bool sock_mgr_all_cache_counters_initialized(const std::string &ifname); +/** Remove cache counters that are not present in the valid interface set */ +void sock_mgr_remove_cache_counters_except(const std::unordered_set &valid_ifnames); + /** Update database counters from cache counters for all sockets */ void sock_mgr_update_db_counters(); +void sock_mgr_update_db_counters(const socket_counters_t &counters_by_socket); + +/** Copy cache counters for all sockets */ +socket_counters_t sock_mgr_copy_cache_counters(); #endif /* SOCKET_MANAGER_H_ */ diff --git a/src/util.h b/src/util.h index f4fbd24c3..d5a59763d 100644 --- a/src/util.h +++ b/src/util.h @@ -216,18 +216,6 @@ inline bool is_agg_counter(const std::string &ifname) return ifname.compare(0, agg_dev_prefix.size(), agg_dev_prefix) == 0 || ifname == agg_dev_all; } -/** - * @code get_agg_counter_ifname(ifname, context); - * @brief Get aggregate counter name for given ifname and device context - * @param ifname Interface name - * @param context Pointer to device context - * @return Aggregate counter name - */ -inline std::string get_agg_counter_ifname(const std::string &ifname, const std::string &context_ifname) -{ - return ifname != context_ifname ? agg_dev_prefix + context_ifname : agg_dev_all; -} - /** * @code contains_value(v, value); * @brief Check if a vector contains a specific value