From bcfc80265256528969b51521f8dbcd61a2bef678 Mon Sep 17 00:00:00 2001 From: mssonicbld <79238446+mssonicbld@users.noreply.github.com> Date: Thu, 20 Feb 2025 04:01:43 +0800 Subject: [PATCH 01/14] Add v2r mapping for Fabric Counters in VoQ Chassis (#365) #### Why I did it 1. In order to retrieve Fabric Counters using Streaming telemetry, the COUNTERS_FABRIC_PORT_NAME_MAP table in COUNTERS_DB has to be queried first to get PORT name to OID mapping, then COUNTERS:oid:<> table in COUNTERS_DB has to be queried to get the Fabric Counters. To make the retrieval easy, changes are made in this PR to add v2r mapping for Fabric Counters in Packet Chassis similar to Ethernet interface counters. Reference https://github.com/sonic-net/sonic-telemetry/pull/77 2 The Fabric counters Port name is unique within a given asic namespace and is not unique across a linecard. For example, there will be a port named PORT0 in asic0 and PORT0 in asic1 namespace as well. In order to make this unique across a multi-asic linecard, this PR appends asic namespace in the interface name. For example, PORT0 in asic0 will be called "PORT0-asic0". #### How I did it 1. Add v2r mapping support to read all ports from COUNTERS_FABRIC_PORT_NAME_MAP so that Streaming telemetry can query using Fabric port name and COUNTERS_DB. 2. Modify the Fabric Port name by appending Asic namespace. 3. Add unit-test 4. Modify clean up and set up multi namespace to reset countersFabricPortNameMap so that the port map is re-initialized when changing from single to multi- namespace in unit-test #### How to verify it Verified on a multi-asic linecard: ``` gnmi_get -target_addr localhost:50051 -xpath COUNTERS/PORT0-asic0 -xpath_target COUNTERS_DB -insecure == getRequest: prefix: < target: "COUNTERS_DB" > path: < elem: < name: "COUNTERS" > elem: < name: "PORT0-asic0" > > encoding: JSON_IETF == getResponse: notification: < timestamp: 1737484675266818454 prefix: < target: "COUNTERS_DB" > update: < path: < elem: < name: "COUNTERS" > elem: < name: "PORT0-asic0" > > val: < json_ietf_val: "{\"SAI_PORT_STAT_IF_IN_ERRORS\":\"0\",\"SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS\":\"0\",\"SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES\":\"0\",\"SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES\":\"0\",\"SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS\":\"0\",\"SAI_PORT_STAT_IF_IN_OCTETS\":\"0\",\"SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS\":\"0\",\"SAI_PORT_STAT_IF_OUT_OCTETS\":\"0\"}" > > > ``` Verified on single asic Linecard: ``` gnmi_get -target_addr localhost:50051 -xpath COUNTERS/PORT0 -xpath_target COUNTERS_DB -insecure == getRequest: prefix: < target: "COUNTERS_DB" > path: < elem: < name: "COUNTERS" > elem: < name: "PORT0" > > encoding: JSON_IETF == getResponse: notification: < timestamp: 1737567049704926166 prefix: < target: "COUNTERS_DB" > update: < path: < elem: < name: "COUNTERS" > elem: < name: "PORT0" > > val: < json_ietf_val: "{\"SAI_PORT_STAT_IF_IN_ERRORS\":\"0\",\"SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS\":\"181562945\",\"SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES\":\"0\",\"SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES\":\"0\",\"SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS\":\"0\",\"SAI_PORT_STAT_IF_IN_OCTETS\":\"40499614075\",\"SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS\":\"11152\",\"SAI_PORT_STAT_IF_OUT_OCTETS\":\"2588370\"}" > > > gnmi_get -target_addr localhost:50051 -xpath COUNTERS/PORT* -xpath_target COUNTERS_DB -insecure ``` Verified on Pizza box to ensure no error log is seen. #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 #### Description for the changelog #### Link to config_db schema for YANG module changes #### A picture of a cute animal (not mandatory but encouraged) --- gnmi_server/server_test.go | 99 +++++++++++++++++++ sonic_data_client/db_client.go | 4 + sonic_data_client/virtual_db.go | 105 +++++++++++++++++++++ testdata/COUNTERS:PORT0.txt | 10 ++ testdata/COUNTERS:PORT_wildcard.txt | 22 +++++ testdata/COUNTERS:PORT_wildcardasic0.txt | 22 +++++ testdata/COUNTERS:oid:0x1000000000081.txt | 10 ++ testdata/COUNTERS:oid:0x1000000000082.txt | 11 +++ testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt | 4 + 9 files changed, 287 insertions(+) create mode 100644 testdata/COUNTERS:PORT0.txt create mode 100644 testdata/COUNTERS:PORT_wildcard.txt create mode 100644 testdata/COUNTERS:PORT_wildcardasic0.txt create mode 100644 testdata/COUNTERS:oid:0x1000000000081.txt create mode 100644 testdata/COUNTERS:oid:0x1000000000082.txt create mode 100644 testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 40a90abca..9f15fe1d8 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -574,6 +574,32 @@ func initFullCountersDb(t *testing.T, namespace string) { } mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091f", countersEeth68_4Byte) loadDB(t, rclient, mpi_counter) + + fileName = "../testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt" + countersFabricPortNameMapByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_fab_name_map := loadConfig(t, "COUNTERS_FABRIC_PORT_NAME_MAP", countersFabricPortNameMapByte) + loadDB(t, rclient, mpi_fab_name_map) + + // "PORT0": "oid:0x1000000000081" : Fabric port counter, for COUNTERS/PORT0 vpath test + fileName = "../testdata/COUNTERS:oid:0x1000000000081.txt" + countersPort0_Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_fab_counter_0 := loadConfig(t, "COUNTERS:oid:0x1000000000081", countersPort0_Byte) + loadDB(t, rclient, mpi_fab_counter_0) + + // "PORT1": "oid:0x1000000000082" : Fabric port counter, for COUNTERS/PORT1 vpath test + fileName = "../testdata/COUNTERS:oid:0x1000000000082.txt" + countersPort1_Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_fab_counter_1 := loadConfig(t, "COUNTERS:oid:0x1000000000082", countersPort1_Byte) + loadDB(t, rclient, mpi_fab_counter_1) } func prepareConfigDb(t *testing.T, namespace string) { @@ -640,6 +666,14 @@ func prepareDb(t *testing.T, namespace string) { mpi_qname_map := loadConfig(t, "COUNTERS_QUEUE_NAME_MAP", countersQueueNameMapByte) loadDB(t, rclient, mpi_qname_map) + fileName = "../testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt" + countersFabricPortNameMapByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_fab_name_map := loadConfig(t, "COUNTERS_FABRIC_PORT_NAME_MAP", countersFabricPortNameMapByte) + loadDB(t, rclient, mpi_fab_name_map) + fileName = "../testdata/COUNTERS:Ethernet68.txt" countersEthernet68Byte, err := ioutil.ReadFile(fileName) if err != nil { @@ -694,6 +728,25 @@ func prepareDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091f", countersEeth68_4Byte) loadDB(t, rclient, mpi_counter) + // "PORT0": "oid:0x1000000000081" : Fabric port counters, for COUNTERS/PORT0 vpath test + fileName = "../testdata/COUNTERS:oid:0x1000000000081.txt" + fileName = "../testdata/COUNTERS:oid:0x1000000000081.txt" + countersPort0, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "COUNTERS:oid:0x1000000000081", countersPort0) + loadDB(t, rclient, mpi_counter) + + // "PORT1": "oid:0x1000000000082" : Fabric port counter, for COUNTERS/PORT1 vpath test + fileName = "../testdata/COUNTERS:oid:0x1000000000082.txt" + countersPort1_Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "COUNTERS:oid:0x1000000000082", countersPort1_Byte) + loadDB(t, rclient, mpi_counter) + // Load CONFIG_DB for alias translation prepareConfigDb(t, namespace) @@ -1232,11 +1285,27 @@ func runGnmiTestGet(t *testing.T, namespace string) { t.Fatalf("read file %v err: %v", fileName, err) } + fileName = "../testdata/COUNTERS:PORT0.txt" + countersFabricPort0Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + + fileName = "../testdata/COUNTERS:PORT_wildcard" + namespace + ".txt" + countersFabricPortWildcardByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + stateDBPath := "STATE_DB" ns, _ := sdcfg.GetDbDefaultNamespace() + validFabricPortName := "PORT0" + invalidFabricPortName := "PORT0-" + namespace if namespace != ns { stateDBPath = "STATE_DB" + "/" + namespace + validFabricPortName = "PORT0-" + namespace + invalidFabricPortName = "PORT0" } type testCase struct { @@ -1417,6 +1486,35 @@ func runGnmiTestGet(t *testing.T, namespace string) { valTest: true, wantRetCode: codes.OK, wantRespVal: []byte(`{"test_field": "test_value"}`), + }, { + desc: "get COUNTERS:" + validFabricPortName, + pathTarget: "COUNTERS_DB", + textPbPath: ` + elem: + elem: + `, + wantRetCode: codes.OK, + wantRespVal: countersFabricPort0Byte, + valTest: true, + }, { + desc: "get COUNTERS:PORT*", + pathTarget: "COUNTERS_DB", + textPbPath: ` + elem: + elem: + `, + wantRetCode: codes.OK, + wantRespVal: countersFabricPortWildcardByte, + valTest: true, + }, { + desc: "Invalid fabric port key get" + invalidFabricPortName, + pathTarget: "COUNTERS_DB", + textPbPath: ` + elem: + elem: + `, + wantRetCode: codes.NotFound, + valTest: true, }, { desc: "Invalid DBKey of length 1", pathTarget: stateDBPath, @@ -4650,6 +4748,7 @@ func init() { // Inform gNMI server to use redis tcp localhost connection sdc.UseRedisLocalTcpPort = true + os.Setenv("UNIT_TEST", "1") } func TestMain(m *testing.M) { diff --git a/sonic_data_client/db_client.go b/sonic_data_client/db_client.go index cb94d1b04..deb41114a 100644 --- a/sonic_data_client/db_client.go +++ b/sonic_data_client/db_client.go @@ -609,6 +609,10 @@ func populateDbtablePath(prefix, path *gnmipb.Path, pathG2S *map[*gnmipb.Path][] if err != nil { return err } + err = initCountersFabricPortNameMap() + if err != nil { + log.Errorf("Could not create CountersFabricPortNameMap: %v", err) + } } diff --git a/sonic_data_client/virtual_db.go b/sonic_data_client/virtual_db.go index 2f81a0033..61b4a4c16 100644 --- a/sonic_data_client/virtual_db.go +++ b/sonic_data_client/virtual_db.go @@ -4,6 +4,7 @@ import ( "fmt" log "github.com/golang/glog" "strings" + "os" ) // virtual db is to Handle @@ -44,6 +45,9 @@ var ( // SONiC interface name to their PFC-WD enabled queues, then to oid map countersPfcwdNameMap = make(map[string]map[string]string) + // SONiC interface name to their Fabric port name map, then to oid map + countersFabricPortNameMap = make(map[string]string) + // path2TFuncTbl is used to populate trie tree which is reponsible // for virtual path to real data path translation pathTransFuncTbl = []pathTransFunc{ @@ -59,6 +63,9 @@ var ( }, { // PFC WD stats for one or all Ethernet ports path: []string{"COUNTERS_DB", "COUNTERS", "Ethernet*", "Pfcwd"}, transFunc: v2rTranslate(v2rEthPortPfcwdStats), + }, { // stats for one or all Fabric ports + path: []string{"COUNTERS_DB", "COUNTERS", "PORT*"}, + transFunc: v2rTranslate(v2rFabricPortStats), }, } ) @@ -107,6 +114,7 @@ func initAliasMap() error { } return nil } + func initCountersPfcwdNameMap() error { var err error if len(countersPfcwdNameMap) == 0 { @@ -118,6 +126,20 @@ func initCountersPfcwdNameMap() error { return nil } +func initCountersFabricPortNameMap() error { + var err error + // Reset map for Unit test to ensure that counters db is updated + // after changing from single to multi-asic config + value := os.Getenv("UNIT_TEST") + if len(countersFabricPortNameMap) == 0 || value == "1" { + countersFabricPortNameMap, err = getFabricCountersMap("COUNTERS_FABRIC_PORT_NAME_MAP") + if err != nil { + return err + } + } + return nil +} + // Get the mapping between sonic interface name and oids of their PFC-WD enabled queues in COUNTERS_DB func getPfcwdMap() (map[string]map[string]string, error) { var pfcwdName_map = make(map[string]map[string]string) @@ -284,6 +306,89 @@ func getCountersMap(tableName string) (map[string]string, error) { return counter_map, nil } + +// Get the mapping between objects in counters DB, Ex. port name to oid in "COUNTERS_FABRIC_PORT_NAME_MAP" table. +// Aussuming static port name to oid map in COUNTERS table +func getFabricCountersMap(tableName string) (map[string]string, error) { + counter_map := make(map[string]string) + dbName := "COUNTERS_DB" + redis_client_map, err := GetRedisClientsForDb(dbName) + if err != nil { + return nil, err + } + for namespace, redisDb := range redis_client_map { + fv, err := redisDb.HGetAll(tableName).Result() + if err != nil { + log.V(2).Infof("redis HGetAll failed for COUNTERS_DB in namespace %v, tableName: %s", namespace, tableName) + return nil, err + } + namespaceFv := make(map[string]string) + for k, v := range fv { + // Fabric port names are not unique across asic namespace + // To make them unique, add asic namesapce to the port name + // For example, PORT0 in asic0 will be PORT0-asic0 + var namespace_str = "" + if len(namespace) != 0 { + namespace_str = string('-') + namespace + } + namespaceFv[k + namespace_str] = v + } + addmap(counter_map, namespaceFv) + log.V(6).Infof("tableName: %s in namespace %v, map %v", tableName, namespace, namespaceFv) + } + return counter_map, nil +} + +// Populate real data paths from paths like +// [COUNTER_DB COUNTERS PORT*] or [COUNTER_DB COUNTERS PORT0] +func v2rFabricPortStats(paths []string) ([]tablePath, error) { + var tblPaths []tablePath + if strings.HasSuffix(paths[KeyIdx], "*") { // All Ethernet ports + for port, oid := range countersFabricPortNameMap { + var namespace string + // Extract namespace from port name + // multi-asic Linecard ex: PORT0-asic0 + if strings.Contains(port, "-"){ + namespace = strings.Split(port, "-")[1] + } else { + namespace = "" + } + separator, _ := GetTableKeySeparator(paths[DbIdx], namespace) + tblPath := tablePath{ + dbNamespace: namespace, + dbName: paths[DbIdx], + tableName: paths[TblIdx], + tableKey: oid, + delimitor: separator, + jsonTableKey: port, + } + tblPaths = append(tblPaths, tblPath) + } + } else { //single port + var port, namespace string + port = paths[KeyIdx] + oid, ok := countersFabricPortNameMap[port] + if !ok { + return nil, fmt.Errorf("%v not a valid sonic fabric interface.", port) + } + if strings.Contains(port, "-"){ + namespace = strings.Split(port, "-")[1] + } else { + namespace = "" + } + separator, _ := GetTableKeySeparator(paths[DbIdx], namespace) + tblPaths = []tablePath{{ + dbNamespace: namespace, + dbName: paths[DbIdx], + tableName: paths[TblIdx], + tableKey: oid, + delimitor: separator, + }} + } + log.V(6).Infof("v2rFabricPortStats: %v", tblPaths) + return tblPaths, nil +} + // Populate real data paths from paths like // [COUNTER_DB COUNTERS Ethernet*] or [COUNTER_DB COUNTERS Ethernet68] func v2rEthPortStats(paths []string) ([]tablePath, error) { diff --git a/testdata/COUNTERS:PORT0.txt b/testdata/COUNTERS:PORT0.txt new file mode 100644 index 000000000..8cf8d5962 --- /dev/null +++ b/testdata/COUNTERS:PORT0.txt @@ -0,0 +1,10 @@ +{ + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "6428", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "1007545", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "16807108", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "3747867283" +} diff --git a/testdata/COUNTERS:PORT_wildcard.txt b/testdata/COUNTERS:PORT_wildcard.txt new file mode 100644 index 000000000..f60c99c48 --- /dev/null +++ b/testdata/COUNTERS:PORT_wildcard.txt @@ -0,0 +1,22 @@ +{ + "PORT0": { + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "6428", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "1007545", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "16807108", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "3747867283" + }, + "PORT1": { + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "0", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "0" + } +} diff --git a/testdata/COUNTERS:PORT_wildcardasic0.txt b/testdata/COUNTERS:PORT_wildcardasic0.txt new file mode 100644 index 000000000..c49e1b28b --- /dev/null +++ b/testdata/COUNTERS:PORT_wildcardasic0.txt @@ -0,0 +1,22 @@ +{ + "PORT0-asic0": { + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "6428", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "1007545", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "16807108", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "3747867283" + }, + "PORT1-asic0": { + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "0", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "0" + } +} diff --git a/testdata/COUNTERS:oid:0x1000000000081.txt b/testdata/COUNTERS:oid:0x1000000000081.txt new file mode 100644 index 000000000..8cf8d5962 --- /dev/null +++ b/testdata/COUNTERS:oid:0x1000000000081.txt @@ -0,0 +1,10 @@ +{ + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "6428", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "1007545", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "16807108", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "3747867283" +} diff --git a/testdata/COUNTERS:oid:0x1000000000082.txt b/testdata/COUNTERS:oid:0x1000000000082.txt new file mode 100644 index 000000000..aa6d5fe8f --- /dev/null +++ b/testdata/COUNTERS:oid:0x1000000000082.txt @@ -0,0 +1,11 @@ +{ + "SAI_PORT_STAT_IF_OUT_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_FEC_SYMBOL_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_FEC_NOT_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_IN_FEC_CORRECTABLE_FRAMES": "0", + "SAI_PORT_STAT_IF_OUT_OCTETS": "0", + "SAI_PORT_STAT_IF_IN_FABRIC_DATA_UNITS": "0", + "SAI_PORT_STAT_IF_IN_ERRORS": "0", + "SAI_PORT_STAT_IF_IN_OCTETS": "0" +} + diff --git a/testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt b/testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt new file mode 100644 index 000000000..530772147 --- /dev/null +++ b/testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt @@ -0,0 +1,4 @@ +{ + "PORT0": "oid:0x1000000000081", + "PORT1": "oid:0x1000000000082" +} From 369059641e7af8cab156177731cf3cd1d7f35cdd Mon Sep 17 00:00:00 2001 From: mssonicbld <79238446+mssonicbld@users.noreply.github.com> Date: Thu, 20 Feb 2025 19:02:24 +0800 Subject: [PATCH 02/14] Enable Pfcwd Queries (#347) #### Why I did it Pfcwd queries currently use PFC_WD_TABLE which is not the correct table name. Use PFC_WD instead. Since Pfcwd are not crucial to other COUNTERS paths, if map creation of Pfcwd fails, we will fail gracefully and return empty json for Pfcwd instead of impacting all COUNTER_DB queries. #### How I did it Change table name #### How to verify it UT and manual testing #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 #### Description for the changelog #### Link to config_db schema for YANG module changes #### A picture of a cute animal (not mandatory but encouraged) --- gnmi_server/server_test.go | 100 +++++++++++++++++++++++++++ sonic_data_client/db_client.go | 2 +- sonic_data_client/virtual_db.go | 24 +++++-- testdata/CONFIG_PFCWD_PORTS.txt | 116 +++++++++++++++++--------------- 4 files changed, 180 insertions(+), 62 deletions(-) diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 9f15fe1d8..26e5a834a 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -242,6 +242,106 @@ func createKeepAliveServer(t *testing.T, port int64) *Server { return s } +func TestPFCWDErrors(t *testing.T) { + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + mock := gomonkey.ApplyFunc(sdc.GetPfcwdMap, func() (map[string]map[string]string, error) { + return nil, fmt.Errorf("Mock error") + }) + defer mock.Reset() + + fileName := "../testdata/COUNTERS:Ethernet_wildcard_alias.txt" + countersEthernetWildcardByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var countersEthernetWildcardJson interface{} + json.Unmarshal(countersEthernetWildcardByte, &countersEthernetWildcardJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query COUNTERS/Ethernet*", + poll: 1, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"COUNTERS", "Ethernet*"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Update{Path: []string{"COUNTERS", "Ethernet*"}, TS: time.Unix(0, 200), Val: countersEthernetWildcardJson}, + client.Update{Path: []string{"COUNTERS", "Ethernet*"}, TS: time.Unix(0, 200), Val: countersEthernetWildcardJson}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + prepareDb(t, ns) + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + + // runTestGet requests a path from the server by Get grpc call, and compares if // the return code and response value are expected. func runTestGet(t *testing.T, ctx context.Context, gClient pb.GNMIClient, pathTarget string, diff --git a/sonic_data_client/db_client.go b/sonic_data_client/db_client.go index deb41114a..5768bf7b1 100644 --- a/sonic_data_client/db_client.go +++ b/sonic_data_client/db_client.go @@ -607,7 +607,7 @@ func populateDbtablePath(prefix, path *gnmipb.Path, pathG2S *map[*gnmipb.Path][] } err = initCountersPfcwdNameMap() if err != nil { - return err + log.Errorf("Could not create CountersPfcwdNameMap: %v", err) } err = initCountersFabricPortNameMap() if err != nil { diff --git a/sonic_data_client/virtual_db.go b/sonic_data_client/virtual_db.go index 61b4a4c16..7f52c7a51 100644 --- a/sonic_data_client/virtual_db.go +++ b/sonic_data_client/virtual_db.go @@ -118,7 +118,7 @@ func initAliasMap() error { func initCountersPfcwdNameMap() error { var err error if len(countersPfcwdNameMap) == 0 { - countersPfcwdNameMap, err = getPfcwdMap() + countersPfcwdNameMap, err = GetPfcwdMap() if err != nil { return err } @@ -141,10 +141,11 @@ func initCountersFabricPortNameMap() error { } // Get the mapping between sonic interface name and oids of their PFC-WD enabled queues in COUNTERS_DB -func getPfcwdMap() (map[string]map[string]string, error) { +func GetPfcwdMap() (map[string]map[string]string, error) { var pfcwdName_map = make(map[string]map[string]string) dbName := "CONFIG_DB" + pfcwdTableName := "PFC_WD" redis_client_map, err := GetRedisClientsForDb(dbName) if err != nil { return nil, err @@ -156,8 +157,8 @@ func getPfcwdMap() (map[string]map[string]string, error) { log.V(1).Infof("Can not connect to %v in namsespace %v, err: %v", dbName, namespace, err) return nil, err } - - keyName := fmt.Sprintf("PFC_WD_TABLE%v*", separator) + + keyName := fmt.Sprintf("%s%v*", pfcwdTableName, separator) resp, err := redisDb.Keys(keyName).Result() if err != nil { log.V(1).Infof("redis get keys failed for %v in namsepace %v, key = %v, err: %v", dbName, namespace, keyName, err) @@ -171,7 +172,10 @@ func getPfcwdMap() (map[string]map[string]string, error) { } for _, key := range resp { - name := key[13:] + if strings.Contains(key, "GLOBAL") || strings.Contains(key, "global") { // ignore PFC_WD|global / PFC_WD|GLOBAL + continue + } + name := key[len(keyName) - 1:] pfcwdName_map[name] = make(map[string]string) } @@ -186,7 +190,15 @@ func getPfcwdMap() (map[string]map[string]string, error) { log.V(1).Infof("PFC WD not enabled on device") return nil, nil } - qos_key := resp[0] + + var qos_key string + for _, key := range resp { + if strings.Contains(key, "GLOBAL") || strings.Contains(key, "global") { // ignore PORT_QOS_MAP|global / PORT_QOS_MAP|GLOBAL + continue + } + qos_key = key + break + } fieldName := "pfc_enable" priorities, err := redisDb.HGet(qos_key, fieldName).Result() diff --git a/testdata/CONFIG_PFCWD_PORTS.txt b/testdata/CONFIG_PFCWD_PORTS.txt index acb894a0a..fe320dd35 100644 --- a/testdata/CONFIG_PFCWD_PORTS.txt +++ b/testdata/CONFIG_PFCWD_PORTS.txt @@ -3,169 +3,175 @@ "3": "3", "4": "4" }, - "PFC_WD_TABLE|Ethernet0": { + "PFC_WD|GLOBAL": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet1": { + "PFC_WD|Ethernet0": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet10": { + "PFC_WD|Ethernet1": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet11": { + "PFC_WD|Ethernet10": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet12": { + "PFC_WD|Ethernet11": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet13": { + "PFC_WD|Ethernet12": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet14": { + "PFC_WD|Ethernet13": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet15": { + "PFC_WD|Ethernet14": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet16": { + "PFC_WD|Ethernet15": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet17": { + "PFC_WD|Ethernet16": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet18": { + "PFC_WD|Ethernet17": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet19": { + "PFC_WD|Ethernet18": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet2": { + "PFC_WD|Ethernet19": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet20": { + "PFC_WD|Ethernet2": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet21": { + "PFC_WD|Ethernet20": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet22": { + "PFC_WD|Ethernet21": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet23": { + "PFC_WD|Ethernet22": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet24": { + "PFC_WD|Ethernet23": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet25": { + "PFC_WD|Ethernet24": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet26": { + "PFC_WD|Ethernet25": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet27": { + "PFC_WD|Ethernet26": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet28": { + "PFC_WD|Ethernet27": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet29": { + "PFC_WD|Ethernet28": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet3": { + "PFC_WD|Ethernet29": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet30": { + "PFC_WD|Ethernet3": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet31": { + "PFC_WD|Ethernet30": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet32": { + "PFC_WD|Ethernet31": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet33": { + "PFC_WD|Ethernet32": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet34": { + "PFC_WD|Ethernet33": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet35": { + "PFC_WD|Ethernet34": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet36": { + "PFC_WD|Ethernet35": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet37": { + "PFC_WD|Ethernet36": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet38": { + "PFC_WD|Ethernet37": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet39": { + "PFC_WD|Ethernet38": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet4": { + "PFC_WD|Ethernet39": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet40": { + "PFC_WD|Ethernet4": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet41": { + "PFC_WD|Ethernet40": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet42": { + "PFC_WD|Ethernet41": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet43": { + "PFC_WD|Ethernet42": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet44": { + "PFC_WD|Ethernet43": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet45": { + "PFC_WD|Ethernet44": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet46": { + "PFC_WD|Ethernet45": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet47": { + "PFC_WD|Ethernet46": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet48": { + "PFC_WD|Ethernet47": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet5": { + "PFC_WD|Ethernet48": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet52": { + "PFC_WD|Ethernet5": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet56": { + "PFC_WD|Ethernet52": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet6": { + "PFC_WD|Ethernet56": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet60": { + "PFC_WD|Ethernet6": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet64": { + "PFC_WD|Ethernet60": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet68": { + "PFC_WD|Ethernet64": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet7": { + "PFC_WD|Ethernet68": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet8": { + "PFC_WD|Ethernet7": { "action": "drop" }, - "PFC_WD_TABLE|Ethernet9": { + "PFC_WD|Ethernet8": { + "action": "drop" + }, + "PFC_WD|Ethernet9": { "action": "drop" }, "PORT_QOS_MAP|Ethernet0,Ethernet1,Ethernet2,Ethernet3,Ethernet4,Ethernet5,Ethernet6,Ethernet7,Ethernet8,Ethernet9,Ethernet10,Ethernet11,Ethernet12,Ethernet13,Ethernet14,Ethernet15,Ethernet16,Ethernet17,Ethernet18,Ethernet19,Ethernet20,Ethernet21,Ethernet22,Ethernet23,Ethernet24,Ethernet25,Ethernet26,Ethernet27,Ethernet28,Ethernet29,Ethernet30,Ethernet31,Ethernet32,Ethernet33,Ethernet34,Ethernet35,Ethernet36,Ethernet37,Ethernet38,Ethernet39,Ethernet40,Ethernet41,Ethernet42,Ethernet43,Ethernet44,Ethernet45,Ethernet46,Ethernet47,Ethernet48,Ethernet52,Ethernet56,Ethernet60,Ethernet64,Ethernet68": { "pfc_enable": "3,4" + }, + "PORT_QOS_MAP|global": { + "dummy": "value" } -} \ No newline at end of file +} From e9a85a66fa67775e0833a4c5f0e7fde40a601120 Mon Sep 17 00:00:00 2001 From: Zain Budhwani Date: Fri, 4 Apr 2025 01:15:03 +0000 Subject: [PATCH 03/14] Revert delete field --- proto/sonic_internal.pb.go | 8 -------- proto/sonic_internal.proto | 3 --- sonic_data_client/db_client.go | 28 ---------------------------- 3 files changed, 39 deletions(-) diff --git a/proto/sonic_internal.pb.go b/proto/sonic_internal.pb.go index d281e1fab..239d72865 100644 --- a/proto/sonic_internal.pb.go +++ b/proto/sonic_internal.pb.go @@ -99,7 +99,6 @@ type Value struct { Fatal string `protobuf:"bytes,6,opt,name=fatal,proto3" json:"fatal,omitempty"` // Notification to be used in place of 1-4 if present Notification *gnmi.Notification `protobuf:"bytes,7,opt,name=notification,proto3" json:"notification,omitempty"` - Delete []*gnmi.Path `protobuf:"bytes,8,opt,name=delete,proto3" json:"delete,omitempty"` } func (x *Value) Reset() { @@ -183,13 +182,6 @@ func (x *Value) GetNotification() *gnmi.Notification { return nil } -func (x *Value) GetDelete() []*gnmi.Path { - if x != nil { - return x.Delete - } - return []*gnmi.Path{} -} - var File_sonic_internal_proto protoreflect.FileDescriptor var file_sonic_internal_proto_rawDesc = []byte{ diff --git a/proto/sonic_internal.proto b/proto/sonic_internal.proto index b37d20550..489cbb6df 100644 --- a/proto/sonic_internal.proto +++ b/proto/sonic_internal.proto @@ -36,7 +36,4 @@ message Value { // Notification to be used in place of 1-4 if present gnmi.Notification notification = 7; - - // Delete to be used to indicate that node was deleted - repeated gnmi.Path delete = 8; } diff --git a/sonic_data_client/db_client.go b/sonic_data_client/db_client.go index 5768bf7b1..fee1b0e54 100644 --- a/sonic_data_client/db_client.go +++ b/sonic_data_client/db_client.go @@ -389,25 +389,6 @@ func ValToResp(val Value) (*gnmipb.SubscribeResponse, error) { Response: &gnmipb.SubscribeResponse_Update{Update: n}}, nil } - // In case of path deletion - if deleted := val.GetDelete(); deleted != nil { - return &gnmipb.SubscribeResponse{ - Response: &gnmipb.SubscribeResponse_Update{ - Update: &gnmipb.Notification{ - Timestamp: val.GetTimestamp(), - Prefix: val.GetPrefix(), - Delete: deleted, - Update: []*gnmipb.Update{ - { - Path: val.GetPath(), - Val: val.GetVal(), - }, - }, - }, - }, - }, nil - } - return &gnmipb.SubscribeResponse{ Response: &gnmipb.SubscribeResponse_Update{ Update: &gnmipb.Notification{ @@ -1151,7 +1132,6 @@ func dbSingleTableKeySubscribe(c *DbClient, rsd redisSubData, updateChannel chan } key := subscr.Channel[prefixLen:] newMsi[key] = fp - newMsi["delete"] = "null_value" } } else if subscr.Payload == "hset" { //op := "SET" @@ -1223,11 +1203,6 @@ func dbTableKeySubscribe(c *DbClient, gnmiPath *gnmipb.Path, interval time.Durat // Helper to send hash data over the stream sendMsiData := func(msiData map[string]interface{}) error { - sendDeleteField := false - if _, isDelete := msiData["delete"]; isDelete { - sendDeleteField = true - } - delete(msiData, "delete") val, err := Msi2TypedValue(msiData) if err != nil { return err @@ -1240,9 +1215,6 @@ func dbTableKeySubscribe(c *DbClient, gnmiPath *gnmipb.Path, interval time.Durat Timestamp: time.Now().UnixNano(), Val: val, } - if sendDeleteField { - (*spbv).Delete = []*gnmipb.Path{gnmiPath} - } if err = c.q.Put(Value{spbv}); err != nil { return fmt.Errorf("Queue error: %v", err) } From 3b020765e9f507eb0abd5554f4aa4e273f5f8989 Mon Sep 17 00:00:00 2001 From: Zain Budhwani Date: Fri, 4 Apr 2025 18:15:50 +0000 Subject: [PATCH 04/14] Remove delete fields --- sonic_data_client/mixed_db_client.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sonic_data_client/mixed_db_client.go b/sonic_data_client/mixed_db_client.go index 4fe1bb22b..113a8cfd5 100644 --- a/sonic_data_client/mixed_db_client.go +++ b/sonic_data_client/mixed_db_client.go @@ -1900,11 +1900,6 @@ func (c *MixedDbClient) dbTableKeySubscribe(gnmiPath *gnmipb.Path, interval time // Helper to send hash data over the stream sendMsiData := func(msiData map[string]interface{}) error { - sendDeleteField := false - if _, isDelete := msiData["delete"]; isDelete { - sendDeleteField = true - } - delete(msiData, "delete") val, err := c.msi2TypedValue(msiData) if err != nil { return err @@ -1917,9 +1912,6 @@ func (c *MixedDbClient) dbTableKeySubscribe(gnmiPath *gnmipb.Path, interval time Timestamp: time.Now().UnixNano(), Val: val, } - if sendDeleteField { - (*spbv).Delete = []*gnmipb.Path{gnmiPath} - } if err = c.q.Put(Value{spbv}); err != nil { return fmt.Errorf("Queue error: %v", err) } From 6c04f758025296fe98ab570f3836eae5a8eb5944 Mon Sep 17 00:00:00 2001 From: mssonicbld <79238446+mssonicbld@users.noreply.github.com> Date: Wed, 28 May 2025 08:35:55 +0800 Subject: [PATCH 05/14] Update agent-pool to use ubuntu 22.04 (#401) Update agent-pool to 22.04 since 20.04 is no longer available. #### Why I did it Ubuntu 20.04 agent is no longer available. #### How I did it Update agent-pool to use ubuntu 22.04 #### How to verify it See azure-pipeline runs. #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 #### Description for the changelog #### Link to config_db schema for YANG module changes #### A picture of a cute animal (not mandatory but encouraged) --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fa7747cfa..6c2c3d084 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,7 +47,7 @@ stages: timeoutInMinutes: 60 pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-22.04 variables: DIFF_COVER_CHECK_THRESHOLD: 80 From 3c5a7d6a135d7294dfe098aa1f51f86879d5655f Mon Sep 17 00:00:00 2001 From: Zain Budhwani <99770260+zbud-msft@users.noreply.github.com> Date: Fri, 13 Jun 2025 15:57:02 -0700 Subject: [PATCH 06/14] [202411] Manual backport of Fix poll mode for querying non existing table and key for APPL_DB (#419) Why I did it Backport of #375 MS: 32418202 SONiC telemetry does not behave according to gnmi specifications As per gnmi specification regarding missing data, "There is no requirement that the path specified in the message must exist within the current data tree on the server. While the path within the subscription SHOULD be a valid path within the set of schema modules that the target supports, subscribing to any syntactically valid path within such modules MUST be allowed. In the case that a particular path does not (yet) exist, the target MUST NOT close the RPC and instead should continue to monitor for the existence of the path, and transmit telemetry updates should it exist in the future." As per gnmi specification regarding deleted keys/paths, "Where a node within the subscribed paths has been removed, the delete field of the Notification message MUST have the path of the node that has been removed appended to it." How I did it If there is missing data, we will not return any failures, but instead we will send sync responses only until data is on the device for polling APPL_DB If data is deleted, we will send a delete notification to tell client that path they are querying is deleted and after send sync responses for polling APPL_DB. How to verify it UT and reverified on 20241110.17 --- gnmi_server/client_subscribe.go | 11 +- gnmi_server/poll_mode_test.go | 1683 +++++++++++++++++ proto/sonic_internal.pb.go | 8 + proto/sonic_internal.proto | 3 + sonic_data_client/db_client.go | 239 ++- sonic_data_client/events_client.go | 3 + sonic_data_client/mixed_db_client.go | 4 + sonic_data_client/non_db_client.go | 5 + sonic_data_client/transl_data_client.go | 4 + testdata/LLDP_ENTRY_TABLE_UPDATE.txt | 6 + testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt | 4 + 11 files changed, 1953 insertions(+), 17 deletions(-) create mode 100644 gnmi_server/poll_mode_test.go create mode 100644 testdata/LLDP_ENTRY_TABLE_UPDATE.txt create mode 100644 testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt diff --git a/gnmi_server/client_subscribe.go b/gnmi_server/client_subscribe.go index 2207bd30a..c4b2b5083 100644 --- a/gnmi_server/client_subscribe.go +++ b/gnmi_server/client_subscribe.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net" + "strings" "sync" "github.com/Workiva/go-datastructures/queue" @@ -59,8 +60,8 @@ func (c *Client) setConnectionManager(threshold int) { return } connectionManager = &ConnectionManager { - connections: make(map[string]struct{}), - threshold: threshold, + connections: make(map[string]struct{}), + threshold: threshold, } connectionManager.PrepareRedis() } @@ -195,7 +196,11 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) { c.polled = make(chan struct{}, 1) c.polled <- struct{}{} c.w.Add(1) - go dc.PollRun(c.q, c.polled, &c.w, c.subscribe) + if target == "APPL_DB" || strings.HasPrefix(target, "APPL_DB/") { + go dc.AppDBPollRun(c.q, c.polled, &c.w, c.subscribe) + } else { + go dc.PollRun(c.q, c.polled, &c.w, c.subscribe) + } case gnmipb.SubscriptionList_ONCE: c.once = make(chan struct{}, 1) c.once <- struct{}{} diff --git a/gnmi_server/poll_mode_test.go b/gnmi_server/poll_mode_test.go new file mode 100644 index 000000000..291560778 --- /dev/null +++ b/gnmi_server/poll_mode_test.go @@ -0,0 +1,1683 @@ +package gnmi + +// server_test covers gNMI get, subscribe (stream and poll) test +// Prerequisite: redis-server should be running. +import ( + "crypto/tls" + "encoding/json" + "github.com/kylelemons/godebug/pretty" + "github.com/openconfig/gnmi/client" + sdcfg "github.com/sonic-net/sonic-gnmi/sonic_db_config" + "golang.org/x/net/context" + "io/ioutil" + "reflect" + "sync" + "testing" + "time" +) + +func TestPollMissingTableThenTableKey(t *testing.T) { + // Test that 1) missing table 2)table + key should just send sync responses and rpc connection should be alive + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE", + poll: 3, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + { + desc: "query ROUTE_TABLE:0.0.0.0/0", + poll: 3, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollMissingTableAndTableKey(t *testing.T) { + // Test that missing table and table + key should just send sync responses and rpc connection should be alive + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE:0.0.0.0/0", + poll: 3, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } + +} + +func TestPollMissingTableThenAdded(t *testing.T) { + // Test that missing table should just send sync responses and rpc connection should be alive + // When we add data for Table, we should receive update notifications + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, add data + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + // Sleep just one second to allow redis data to be entered + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollMissingKeyThenAdded(t *testing.T) { + // Test that missing table+key should just send sync responses and rpc connection should be alive + // When we add data for table+key, we should receive update notifications + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, add data + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + // Sleep just one second to allow redis data to be entered + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollMissingTableAndKeyThenAdded(t *testing.T) { + // Test that we get not updates from missing table and table key queried but still get sync responses + // After adding back, we will get both updates + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, add data + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + // Sleep just one second to allow redis data to be entered + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // check that the connected and sync messages are identical + for i := 0; i < 4; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{4, 5}, {7, 8}, {10, 11}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } + +} + +func TestPollPresentTableMissingTableKey(t *testing.T) { + // Test that we receive update notification for table query and no data for missing key + // After 2 polls, we will add back the missing key data to get both data in our notifications + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, add data + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + // Sleep just one second to allow redis data to be entered + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // check that the connected and sync messages and table update notifications are identical + for i := 0; i < 7; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{7, 8}, {10, 11}, {13, 14}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollPresentTableKeyMissingTable(t *testing.T) { + // Test that we receive update notification for table key query and no data for missing table + // After 2 polls, we will add back the missing table data to get both data in our notifications + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + if nn, ok := n.(client.Update); ok { + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + } else { + gotNoti = append(gotNoti, n) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, add data + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + // Sleep just one second to allow redis data to be entered + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // check that the connected and sync messages and table key update notifications are identical + for i := 0; i < 7; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{7, 8}, {10, 11}, {13, 14}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableDeleted(t *testing.T) { + // Test that we received update notifications for existing table data, then delete table, we should receive 1 delete notification + // After delete notification, we should only see sync responses + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Delete{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200)}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, del data + rclient.FlushDB() + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableFieldDeleted(t *testing.T) { + // Test that we received update notifications for existing table field data, then delete table field, we should receive 1 delete notification + // After delete notification, we should only see sync responses + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_LOC_CHASSIS + lldp_loc_sys_name", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_LOC_CHASSIS", "lldp_loc_sys_name"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_LOC_CHASSIS", "lldp_loc_sys_name"}, TS: time.Unix(0, 200), Val: "dummy"}, + client.Sync{}, + client.Update{Path: []string{"LLDP_LOC_CHASSIS", "lldp_loc_sys_name"}, TS: time.Unix(0, 200), Val: "dummy"}, + client.Sync{}, + client.Update{Path: []string{"LLDP_LOC_CHASSIS", "lldp_loc_sys_name"}, TS: time.Unix(0, 200), Val: "dummy"}, + client.Sync{}, + client.Delete{Path: []string{"LLDP_LOC_CHASSIS", "lldp_loc_sys_name"}, TS: time.Unix(0, 200)}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_LOC_CHASSIS", "lldp_loc_sys_name", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, del data + rclient.FlushDB() + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableKeyDeleted(t *testing.T) { + // Test that we received update notifications for existing table key data, then delete table key, we should receive 1 delete notification + // After delete notification, we should only see sync responses + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Delete{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200)}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, del data + rclient.FlushDB() + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 { + t.Errorf("Expected non zero length of notifications") + } + + if diff := pretty.Compare(tt.wantNoti, gotNoti); diff != "" { + t.Log("\n Want: \n", tt.wantNoti) + t.Log("\n Got : \n", gotNoti) + t.Errorf("unexpected updates:\n%s", diff) + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableAndTableKeyBothDeleted(t *testing.T) { + // Test that we received update notifications for existing data, then delete both table and table key, we should receive 2 delete notifications + // After delete notifications, we should only see sync responses + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Delete{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200)}, + client.Delete{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200)}, + client.Sync{}, + client.Sync{}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, delete data + rclient.FlushDB() + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{1, 2}, {4, 5}, {7, 8}, {10, 11}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + // check that the sync messages are identical + for i := 12; i < 15; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableAndTableKeyTableDeleted(t *testing.T) { + // Test that we receive update notifications for existing data, and then when we delete table we should receive delete notification + // After delete notification, we should see sync responses and continued update for existing data + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Delete{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200)}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, delete data + rclient.Del("LLDP_ENTRY_TABLE:eth0") + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{1, 2}, {4, 5}, {7, 8}, {10, 11}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + // check that the sync messages are identical + for i := 13; i < 17; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} + +func TestPollTableAndTableKeyTableKeyDeleted(t *testing.T) { + // Test that we receive update notifications for existing data, and then when we delete table key we should receive delete notification + // After delete notification, we should see sync responses and continued update for existing data + s := createServer(t, 8081) + go runServer(t, s) + defer s.ForceStop() + + fileName := "../testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt" + routeTableDefaultRouteUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var routeTableDefaultRouteUpdateJson interface{} + json.Unmarshal(routeTableDefaultRouteUpdateByte, &routeTableDefaultRouteUpdateJson) + + fileName = "../testdata/LLDP_ENTRY_TABLE_UPDATE.txt" + lldpEntryTableUpdateByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var lldpEntryTableUpdateJson interface{} + json.Unmarshal(lldpEntryTableUpdateByte, &lldpEntryTableUpdateJson) + + tests := []struct { + desc string + q client.Query + wantNoti []client.Notification + poll int + }{ + { + desc: "query LLDP_ENTRY_TABLE + ROUTE_TABLE/0.0.0.0/0", + poll: 5, + q: client.Query{ + Target: "APPL_DB", + Type: client.Poll, + Queries: []client.Path{{"LLDP_ENTRY_TABLE"}, {"ROUTE_TABLE", "0.0.0.0/0"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Update{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200), Val: routeTableDefaultRouteUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Delete{Path: []string{"ROUTE_TABLE", "0.0.0.0/0"}, TS: time.Unix(0, 200)}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + client.Update{Path: []string{"LLDP_ENTRY_TABLE"}, TS: time.Unix(0, 200), Val: lldpEntryTableUpdateJson}, + client.Sync{}, + }, + }, + } + + ns, _ := sdcfg.GetDbDefaultNamespace() + rclient := getRedisClientN(t, 0, ns) + defer rclient.Close() + rclient.FlushDB() + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_port_id", "dummy") + rclient.HSet("LLDP_ENTRY_TABLE:eth0", "lldp_rem_sys_name", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "ifname", "dummy") + rclient.HSet("ROUTE_TABLE:0.0.0.0/0", "nexthop", "dummy") + + var mutexGotNoti sync.Mutex + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + q := tt.q + q.Addrs = []string{"127.0.0.1:8081"} + c := client.New() + var gotNoti []client.Notification + + q.NotificationHandler = func(n client.Notification) error { + mutexGotNoti.Lock() + switch nn := n.(type) { + case client.Connected, client.Sync: + gotNoti = append(gotNoti, nn) + case client.Delete: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + case client.Update: + nn.TS = time.Unix(0, 200) + gotNoti = append(gotNoti, nn) + default: + t.Errorf("Unexpected Client Notification: %v", nn) + } + mutexGotNoti.Unlock() + return nil + } + + wg := new(sync.WaitGroup) + wg.Add(1) + + go func() { + defer wg.Done() + if err := c.Subscribe(context.Background(), q); err != nil { + t.Errorf("c.Subscribe(): got error %v, expected nil", err) + } + }() + + wg.Wait() + + for i := 0; i < tt.poll; i++ { + if i == 2 { // After first 2 polls, delete data + rclient.Del("ROUTE_TABLE:0.0.0.0/0") + // Sleep just one second to allow redis data to be deleted + time.Sleep(time.Millisecond * 1000) + } + err := c.Poll() + if err != nil { + t.Errorf("c.Poll(): got error %v, expected nil", err) + } + } + + mutexGotNoti.Lock() + + if len(gotNoti) == 0 || len(gotNoti) != len(tt.wantNoti) { + t.Errorf("Expected non zero length of notifications or equal notifications") + } + + // Check that both notifications are coming at every poll interval + for _, pair := range [][2]int{{1, 2}, {4, 5}, {7, 8}, {10, 11}} { // these indexes are our update notifications + i, j := pair[0], pair[1] + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) || !reflect.DeepEqual(gotNoti[j], tt.wantNoti[j]) { + if !reflect.DeepEqual(gotNoti[j], tt.wantNoti[i]) && !reflect.DeepEqual(gotNoti[i], tt.wantNoti[j]) { + t.Fatalf("mismatch at indices %d/%d:\n got (%#v, %#v)\n want (%#v, %#v)", i, j, gotNoti[i], gotNoti[j], tt.wantNoti[i], tt.wantNoti[j]) + } + } + } + + // check that the sync messages are identical + for i := 13; i < 17; i++ { + if !reflect.DeepEqual(gotNoti[i], tt.wantNoti[i]) { + t.Fatalf("notification %d mismatch:\n got %#v\n want %#v", i, gotNoti[i], tt.wantNoti[i]) + } + } + + mutexGotNoti.Unlock() + + c.Close() + }) + } +} diff --git a/proto/sonic_internal.pb.go b/proto/sonic_internal.pb.go index 239d72865..d281e1fab 100644 --- a/proto/sonic_internal.pb.go +++ b/proto/sonic_internal.pb.go @@ -99,6 +99,7 @@ type Value struct { Fatal string `protobuf:"bytes,6,opt,name=fatal,proto3" json:"fatal,omitempty"` // Notification to be used in place of 1-4 if present Notification *gnmi.Notification `protobuf:"bytes,7,opt,name=notification,proto3" json:"notification,omitempty"` + Delete []*gnmi.Path `protobuf:"bytes,8,opt,name=delete,proto3" json:"delete,omitempty"` } func (x *Value) Reset() { @@ -182,6 +183,13 @@ func (x *Value) GetNotification() *gnmi.Notification { return nil } +func (x *Value) GetDelete() []*gnmi.Path { + if x != nil { + return x.Delete + } + return []*gnmi.Path{} +} + var File_sonic_internal_proto protoreflect.FileDescriptor var file_sonic_internal_proto_rawDesc = []byte{ diff --git a/proto/sonic_internal.proto b/proto/sonic_internal.proto index 489cbb6df..b37d20550 100644 --- a/proto/sonic_internal.proto +++ b/proto/sonic_internal.proto @@ -36,4 +36,7 @@ message Value { // Notification to be used in place of 1-4 if present gnmi.Notification notification = 7; + + // Delete to be used to indicate that node was deleted + repeated gnmi.Path delete = 8; } diff --git a/sonic_data_client/db_client.go b/sonic_data_client/db_client.go index fee1b0e54..2b56392ec 100644 --- a/sonic_data_client/db_client.go +++ b/sonic_data_client/db_client.go @@ -40,6 +40,8 @@ type Client interface { // The service will stop upon detection of poll channel closing. // It should run as a go routine PollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) + // Poll to service AppDB only + AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) OnceRun(q *queue.PriorityQueue, once chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) // Get return data from the data source in format of *spb.Value Get(w *sync.WaitGroup) ([]*spb.Value, error) @@ -295,6 +297,69 @@ func streamSampleSubscription(c *DbClient, sub *gnmipb.Subscription, updateOnly } } +func (c *DbClient) AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { + c.w = w + defer c.w.Done() + c.q = q + c.channel = poll + + prevUpdates := make(map[string]bool) + + for { + _, more := <-c.channel + if !more { + log.V(1).Infof("%v poll channel closed, exiting pollDb routine", c) + return + } + t1 := time.Now() + + for gnmiPath, tblPaths := range c.pathG2S { + pathKey := fmt.Sprintf("%v", gnmiPath) + val, err, updateReceived := AppDBTableData2TypedValue(tblPaths, nil) + if !updateReceived { // No updates sent for missing data + if prevUpdate, exists := prevUpdates[pathKey]; exists && prevUpdate == true { + log.V(6).Infof("Delete received for message for %v", gnmiPath) + spbv := &spb.Value{ + Timestamp: time.Now().UnixNano(), + Prefix: c.prefix, + SyncResponse: false, + Delete: []*gnmipb.Path{gnmiPath}, + Val: &gnmipb.TypedValue{ + Value: &gnmipb.TypedValue_StringVal{ + StringVal: "", + }, + }, + } + c.q.Put(Value{spbv}) + log.V(6).Infof("Added spbv #%v", spbv) + prevUpdates[pathKey] = false + } + } else if err != nil { + log.V(2).Infof("Unable to create gnmi TypedValue due to err: %v", err) + return + } else { + spbv := &spb.Value{ + Prefix: c.prefix, + Path: gnmiPath, + Timestamp: time.Now().UnixNano(), + SyncResponse: false, + Val: val, + } + c.q.Put(Value{spbv}) + prevUpdates[pathKey] = true + log.V(6).Infof("Added spbv #%v", spbv) + } + } + spbv := &spb.Value{ + Timestamp: time.Now().UnixNano(), + SyncResponse: true, + } + c.q.Put(Value{spbv}) + log.V(6).Infof("Added spbv #%v", spbv) + log.V(4).Infof("Sync done, poll time taken: %v ms", int64(time.Since(t1)/time.Millisecond)) + } +} + func (c *DbClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { c.w = w defer c.w.Done() @@ -335,6 +400,7 @@ func (c *DbClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.W log.V(4).Infof("Sync done, poll time taken: %v ms", int64(time.Since(t1)/time.Millisecond)) } } + func (c *DbClient) OnceRun(q *queue.PriorityQueue, once chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { return } @@ -389,6 +455,18 @@ func ValToResp(val Value) (*gnmipb.SubscribeResponse, error) { Response: &gnmipb.SubscribeResponse_Update{Update: n}}, nil } + if deleted := val.GetDelete(); deleted != nil { + return &gnmipb.SubscribeResponse{ + Response: &gnmipb.SubscribeResponse_Update{ + Update: &gnmipb.Notification{ + Timestamp: val.GetTimestamp(), + Prefix: val.GetPrefix(), + Delete: deleted, + }, + }, + }, nil + } + return &gnmipb.SubscribeResponse{ Response: &gnmipb.SubscribeResponse_Update{ Update: &gnmipb.Notification{ @@ -664,14 +742,34 @@ func populateDbtablePath(prefix, path *gnmipb.Path, pathG2S *map[*gnmipb.Path][] log.V(6).Infof("Result of keys operation for %v %v, got %v", target, dbPath, res) tblPath.tableKey = "" case 3: // Third element could be table key; or field name in which case table name itself is the key too - n, err := redisDb.Exists(tblPath.tableName + tblPath.delimitor + mappedKey).Result() - if err != nil { - return fmt.Errorf("redis Exists op failed for %v", dbPath) - } - if n == 1 { - tblPath.tableKey = mappedKey + if targetDbName == "APPL_DB" { + keyExists, err := redisDb.Exists(tblPath.tableName + tblPath.delimitor + mappedKey).Result() + if err != nil { + return fmt.Errorf("redis Exists op failed for %v", dbPath) + } + if keyExists == 1 { // Existing Table:Key + tblPath.tableKey = mappedKey + } else { + fieldExists, err := redisDb.HExists(tblPath.tableName, mappedKey).Result() + if err != nil { + return fmt.Errorf("redis HExists op failed for %v", dbPath) + } + if fieldExists { // Existing field in Table + tblPath.field = mappedKey + } else { // Non existing table key + tblPath.tableKey = mappedKey + } + } } else { - tblPath.field = mappedKey + n, err := redisDb.Exists(tblPath.tableName + tblPath.delimitor + mappedKey).Result() + if err != nil { + return fmt.Errorf("redis Exists op failed for %v", dbPath) + } + if n == 1 { + tblPath.tableKey = mappedKey + } else { + tblPath.field = mappedKey + } } case 4: // Fourth element could part of the table key or field name tblPath.tableKey = mappedKey + tblPath.delimitor + stringSlice[3] @@ -693,13 +791,15 @@ func populateDbtablePath(prefix, path *gnmipb.Path, pathG2S *map[*gnmipb.Path][] return fmt.Errorf("Invalid db table Path %v", dbPath) } - var key string - if tblPath.tableKey != "" { - key = tblPath.tableName + tblPath.delimitor + tblPath.tableKey - n, _ := redisDb.Exists(key).Result() - if n != 1 { - log.V(2).Infof("No valid entry found on %v with key %v", dbPath, key) - return fmt.Errorf("No valid entry found on %v with key %v", dbPath, key) + if targetDbName != "APPL_DB" { + var key string + if tblPath.tableKey != "" { + key = tblPath.tableName + tblPath.delimitor + tblPath.tableKey + n, _ := redisDb.Exists(key).Result() + if n != 1 { + log.V(2).Infof("No valid entry found on %v with key %v", dbPath, key) + return fmt.Errorf("No valid entry found on %v with key %v", dbPath, key) + } } } @@ -830,6 +930,64 @@ func TableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string] return nil } +func AppDBTableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string]interface{}) error { + redisDb := Target2RedisDb[tblPath.dbNamespace][tblPath.dbName] + + var pattern string + var dbkeys []string + var err error + var fv map[string]string + + //Only table name provided + if tblPath.tableKey == "" { + // tables in COUNTERS_DB other than COUNTERS table doesn't have keys + pattern = tblPath.tableName + tblPath.delimitor + "*" + dbkeys, err = redisDb.Keys(pattern).Result() + if err != nil { + log.V(2).Infof("redis Keys failed for %v, pattern %s", tblPath, pattern) + return fmt.Errorf("redis Keys failed for %v, pattern %s %v", tblPath, pattern, err) + } + } else { + // both table name and key provided + dbkeys = []string{tblPath.tableName + tblPath.delimitor + tblPath.tableKey} + } + + log.V(4).Infof("dbkeys to be pulled from redis %v", dbkeys) + + for idx, dbkey := range dbkeys { + fv, err = redisDb.HGetAll(dbkey).Result() + if err != nil { + log.V(2).Infof("redis HGetAll failed for %v, dbkey %s", tblPath, dbkey) + return err + } + log.V(4).Infof("Data pulled for dbkey %s: %v", dbkey, fv) + + if len(fv) == 0 { // Skip update for non data path + log.V(6).Infof("Missing data for dbkey %s, will check next key", dbkey) + continue + } + + if (tblPath.tableKey != "" && !useKey) || tblPath.tableName == dbkey { + err = makeJSON_redis(msi, nil, op, fv) + } else { + var key string + // Split dbkey string into two parts and second part is key in table + keys := strings.SplitN(dbkey, tblPath.delimitor, 2) + if len(keys) < 2 { + return fmt.Errorf("dbkey: %s, failed split from delimitor %v", dbkey, tblPath.delimitor) + } + key = keys[1] + err = makeJSON_redis(msi, &key, op, fv) + } + if err != nil { + log.V(2).Infof("makeJSON err %s for fv %v", err, fv) + return err + } + log.V(6).Infof("Added idex %v fv %v ", idx, fv) + } + return nil +} + func Msi2TypedValue(msi map[string]interface{}) (*gnmipb.TypedValue, error) { log.V(4).Infof("State of map after adding redis data %v", msi) jv, err := emitJSON(&msi) @@ -886,6 +1044,59 @@ func tableData2TypedValue(tblPaths []tablePath, op *string) (*gnmipb.TypedValue, return Msi2TypedValue(msi) } +// Returns typed value, error, or bool. Bool specifies that there is data available for the tablePath that was queried. +func AppDBTableData2TypedValue(tblPaths []tablePath, op *string) (*gnmipb.TypedValue, error, bool) { + var useKey bool + var updateReceived bool + msi := make(map[string]interface{}) + for _, tblPath := range tblPaths { + redisDb := Target2RedisDb[tblPath.dbNamespace][tblPath.dbName] + + if tblPath.jsonField == "" { // Not asked to include field in json value, which means not wildcard query + // table path includes table, key and field + if tblPath.field != "" { + if len(tblPaths) != 1 { + log.V(2).Infof("WARNING: more than one path exists for field granularity query: %v", tblPaths) + } + var key string + if tblPath.tableKey != "" { + key = tblPath.tableName + tblPath.delimitor + tblPath.tableKey + } else { + key = tblPath.tableName + } + + val, err := redisDb.HGet(key, tblPath.field).Result() + if err != nil { + log.V(2).Infof("redis HGet failed for %v, data does not exist", tblPath) + continue + } + log.V(4).Infof("Data pulled for key %s and field %s: %s", key, tblPath.field, val) + // TODO: support multiple table paths + return &gnmipb.TypedValue{ + Value: &gnmipb.TypedValue_StringVal{ + StringVal: val, + }}, nil, true + } + } + err := AppDBTableData2Msi(&tblPath, useKey, nil, &msi) + + if err != nil { + return nil, err, true + } + + if !updateReceived { + if len(msi) > 0 { // Update occurred + updateReceived = true + } + } + } + if !updateReceived { + return nil, nil, false + } + val, err := Msi2TypedValue(msi) + return val, err, true +} + func enqueueFatalMsg(c *DbClient, msg string) { putFatalMsg(c.q, msg) } diff --git a/sonic_data_client/events_client.go b/sonic_data_client/events_client.go index e8fa26487..cf9e30039 100644 --- a/sonic_data_client/events_client.go +++ b/sonic_data_client/events_client.go @@ -455,6 +455,9 @@ func (evtc *EventClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, wg return } +func (evtc *EventClient) AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, wg *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { + return +} func (evtc *EventClient) Close() error { return nil diff --git a/sonic_data_client/mixed_db_client.go b/sonic_data_client/mixed_db_client.go index 113a8cfd5..e1720a181 100644 --- a/sonic_data_client/mixed_db_client.go +++ b/sonic_data_client/mixed_db_client.go @@ -1584,6 +1584,10 @@ func (c *MixedDbClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, w *s } } +func (c *MixedDbClient) AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { + return +} + func (c *MixedDbClient) StreamRun(q *queue.PriorityQueue, stop chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { c.w = w defer c.w.Done() diff --git a/sonic_data_client/non_db_client.go b/sonic_data_client/non_db_client.go index a1c4a6c5c..ec730e953 100644 --- a/sonic_data_client/non_db_client.go +++ b/sonic_data_client/non_db_client.go @@ -553,6 +553,11 @@ func (c *NonDbClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, w *syn log.V(4).Infof("Sync done, poll time taken: %v ms", int64(time.Since(t1)/time.Millisecond)) } } + +func (c *NonDbClient) AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { + return +} + func (c *NonDbClient) OnceRun(q *queue.PriorityQueue, once chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { return } diff --git a/sonic_data_client/transl_data_client.go b/sonic_data_client/transl_data_client.go index 21e639253..da545ab87 100644 --- a/sonic_data_client/transl_data_client.go +++ b/sonic_data_client/transl_data_client.go @@ -415,6 +415,10 @@ func (c *TranslClient) PollRun(q *queue.PriorityQueue, poll chan struct{}, w *sy } } +func (c *TranslClient) AppDBPollRun(q *queue.PriorityQueue, poll chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { + return +} + func (c *TranslClient) OnceRun(q *queue.PriorityQueue, once chan struct{}, w *sync.WaitGroup, subscribe *gnmipb.SubscriptionList) { c.w = w defer c.w.Done() diff --git a/testdata/LLDP_ENTRY_TABLE_UPDATE.txt b/testdata/LLDP_ENTRY_TABLE_UPDATE.txt new file mode 100644 index 000000000..ceae6879b --- /dev/null +++ b/testdata/LLDP_ENTRY_TABLE_UPDATE.txt @@ -0,0 +1,6 @@ +{ + "eth0": { + "lldp_rem_port_id": "dummy", + "lldp_rem_sys_name": "dummy" + } +} diff --git a/testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt b/testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt new file mode 100644 index 000000000..19092360a --- /dev/null +++ b/testdata/ROUTE_TABLE_DEFAULT_ROUTE_UPDATE.txt @@ -0,0 +1,4 @@ +{ + "ifname": "dummy", + "nexthop": "dummy" +} From 9a1e06ff8ba4f05d614edc1a6ecf0646eb0552cf Mon Sep 17 00:00:00 2001 From: mramezani95 Date: Tue, 24 Jun 2025 09:47:57 -0700 Subject: [PATCH 07/14] [202411] Added queue periodic watermark stats to the existing virtual path `COUNTERS/Ethernet*/Queues` (#422) * Added periodic watermarks for egress queues under the existing virtual path COUNTERS/Ethernet*/Queues. Signed-off-by: Mahdi Ramezani * Formatting changes. Signed-off-by: Mahdi Ramezani * Ran 'go mod tidy'. Signed-off-by: Mahdi Ramezani * Revert "Ran 'go mod tidy'." This reverts commit a4ac0b7d947a4ee0301171cb8dfe9ceda27bc753. * Removed database name from update paths in tests. Signed-off-by: Mahdi Ramezani --------- Signed-off-by: Mahdi Ramezani --- doc/grpc_telemetry.md | 52 ++++- gnmi_server/server_test.go | 210 ++++++++++++++---- sonic_data_client/virtual_db.go | 88 +++++--- testdata/COUNTERS:Ethernet68:Queues.txt | 8 +- testdata/COUNTERS:Ethernet68:Queues_alias.txt | 8 +- testdata/COUNTERS:Ethernet7:Queues.txt | 28 +++ testdata/COUNTERS:Ethernet7:Queues_alias.txt | 28 +++ .../COUNTERS:Ethernet_wildcard_Queues.txt | 16 +- ...OUNTERS:Ethernet_wildcard_Queues_alias.txt | 16 +- testdata/COUNTERS:oid:0x150000000001d0.txt | 7 + testdata/COUNTERS:oid:0x1500000000091c.txt | 3 +- ...RIODIC_WATERMARKS:oid:0x1500000000091c.txt | 3 + 12 files changed, 380 insertions(+), 87 deletions(-) create mode 100644 testdata/COUNTERS:Ethernet7:Queues.txt create mode 100644 testdata/COUNTERS:Ethernet7:Queues_alias.txt create mode 100644 testdata/COUNTERS:oid:0x150000000001d0.txt create mode 100644 testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt diff --git a/doc/grpc_telemetry.md b/doc/grpc_telemetry.md index 718b94f84..dedbb291b 100644 --- a/doc/grpc_telemetry.md +++ b/doc/grpc_telemetry.md @@ -473,7 +473,7 @@ notification: < ``` ``` -jipan@sonicvm1:~/work/go/src/github.com/jipanyang/gnxi/gnmi_get$ go run gnmi_get.go -xpath_target COUNTERS_DB -xpath COUNTERS/Ethernet68/Queues -target_addr 30.57.185.38:8080 -alsologtostderr -insecure true +gnmi_get -target_addr localhost:50051 -xpath "COUNTERS/Ethernet68/Queues" -xpath_target COUNTERS_DB -insecure == getRequest: prefix: < target: "COUNTERS_DB" @@ -493,7 +493,7 @@ encoding: JSON_IETF == getResponse: notification: < - timestamp: 1520493393388263053 + timestamp: 1748556404127308274 prefix: < target: "COUNTERS_DB" > @@ -510,7 +510,53 @@ notification: < > > val: < - json_ietf_val: "{\"Ethernet68:0\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:1\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:10\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:11\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:12\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:13\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:14\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:15\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:16\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:17\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:18\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:19\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:2\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:3\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:4\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:5\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:6\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:7\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:8\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"},\"Ethernet68:9\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\"}}" + json_ietf_val: "{\"Ethernet68:0\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:0:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:1\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:10\":{},\"Ethernet68:10:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:11\":{},\"Ethernet68:11:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:12\":{},\"Ethernet68:12:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:13\":{},\"Ethernet68:13:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:14\":{},\"Ethernet68:14:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:15\":{},\"Ethernet68:15:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:1:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:2\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:2:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:3\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:3:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:4\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:4:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:5\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:5:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:6\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:6:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:7\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:7:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:8\":{},\"Ethernet68:8:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"Ethernet68:9\":{},\"Ethernet68:9:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"}}" + > + > +> +``` + +``` +gnmi_get -target_addr localhost:50051 -xpath "COUNTERS/Ethernet*/Queues" -xpath_target COUNTERS_DB -insecure +== getRequest: +prefix: < + target: "COUNTERS_DB" +> +path: < + elem: < + name: "COUNTERS" + > + elem: < + name: "Ethernet*" + > + elem: < + name: "Queues" + > +> +encoding: JSON_IETF + +== getResponse: +notification: < + timestamp: 1748556657384573964 + prefix: < + target: "COUNTERS_DB" + > + update: < + path: < + elem: < + name: "COUNTERS" + > + elem: < + name: "Ethernet*" + > + elem: < + name: "Queues" + > + > + val: < + json_ietf_val: "{\"etp10:0\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:0:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:1\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:10\":{},\"etp10:10:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:11\":{},\"etp10:11:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:12\":{},\"etp10:12:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:13\":{},\"etp10:13:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:14\":{},\"etp10:14:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:15\":{},\"etp10:15:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:1:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:2\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:2:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:3\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:3:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:4\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:4:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:5\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:5:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:6\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:6:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:7\":{\"SAI_QUEUE_STAT_BYTES\":\"656106\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"527\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:7:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:8\":{},\"etp10:8:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp10:9\":{},\"etp10:9:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:0\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:0:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:1\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:10\":{},\"etp11a:10:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:11\":{},\"etp11a:11:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:12\":{},\"etp11a:12:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:13\":{},\"etp11a:13:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:14\":{},\"etp11a:14:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:15\":{},\"etp11a:15:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:1:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:2\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:2:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:3\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:3:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:4\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:4:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:5\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:5:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:6\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:6:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:7\":{\"SAI_QUEUE_STAT_BYTES\":\"50750\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"190\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:7:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:8\":{},\"etp11a:8:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp11a:9\":{},\"etp11a:9:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"}, + ..., + \"etp9:0\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:0:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:1\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:10\":{},\"etp9:10:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:11\":{},\"etp9:11:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:12\":{},\"etp9:12:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:13\":{},\"etp9:13:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:14\":{},\"etp9:14:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:15\":{},\"etp9:15:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:1:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:2\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:2:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:3\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:3:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:4\":{\"PFC_WD_ACTION\":\"drop\",\"PFC_WD_DETECTION_TIME\":\"400000\",\"PFC_WD_DETECTION_TIME_LEFT\":\"400000\",\"PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED\":\"0\",\"PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED\":\"0\",\"PFC_WD_RESTORATION_TIME\":\"400000\",\"PFC_WD_STATUS\":\"operational\",\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS_last\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:4:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:5\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:5:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:6\":{\"SAI_QUEUE_STAT_BYTES\":\"0\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"0\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:6:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:7\":{\"SAI_QUEUE_STAT_BYTES\":\"652260\",\"SAI_QUEUE_STAT_DROPPED_PACKETS\":\"0\",\"SAI_QUEUE_STAT_PACKETS\":\"535\",\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:7:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:8\":{},\"etp9:8:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"},\"etp9:9\":{},\"etp9:9:periodic\":{\"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES\":\"0\"}}" > > > diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 26e5a834a..df770e632 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -657,6 +657,15 @@ func initFullCountersDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091c", countersEeth68_1Byte) loadDB(t, rclient, mpi_counter) + // "Ethernet68:1": "oid:0x1500000000091c" : periodic queue watermark, for COUNTERS/Ethernet68/Queue vpath test + fileName = "../testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt" + periodicWMEth68_1Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "PERIODIC_WATERMARKS:oid:0x1500000000091c", periodicWMEth68_1Byte) + loadDB(t, rclient, mpi_counter) + // "Ethernet68:3": "oid:0x1500000000091e" : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test fileName = "../testdata/COUNTERS:oid:0x1500000000091e.txt" countersEeth68_3Byte, err := ioutil.ReadFile(fileName) @@ -801,7 +810,7 @@ func prepareDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000092a", counters92aByte) loadDB(t, rclient, mpi_counter) - // "Ethernet68:1": "oid:0x1500000000091c" : queue counter, for COUNTERS/Ethernet68/Queue vpath test + // "Ethernet68:1": "oid:0x1500000000091c" : queue counter, for COUNTERS/Ethernet68/Queues vpath test fileName = "../testdata/COUNTERS:oid:0x1500000000091c.txt" countersEeth68_1Byte, err := ioutil.ReadFile(fileName) if err != nil { @@ -810,6 +819,24 @@ func prepareDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091c", countersEeth68_1Byte) loadDB(t, rclient, mpi_counter) + // "Ethernet68:1": "oid:0x1500000000091c" : periodic queue watermark, for COUNTERS/Ethernet68/Queues vpath test + fileName = "../testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt" + periodicWMEth68_1Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "PERIODIC_WATERMARKS:oid:0x1500000000091c", periodicWMEth68_1Byte) + loadDB(t, rclient, mpi_counter) + + // "Ethernet7:5": "oid:0x150000000001d0" : queue counter, for COUNTERS/Ethernet7/Queues vpath test + fileName = "../testdata/COUNTERS:oid:0x150000000001d0.txt" + countersEeth7_5Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "COUNTERS:oid:0x150000000001d0", countersEeth7_5Byte) + loadDB(t, rclient, mpi_counter) + // "Ethernet68:3": "oid:0x1500000000091e" : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test fileName = "../testdata/COUNTERS:oid:0x1500000000091e.txt" countersEeth68_3Byte, err := ioutil.ReadFile(fileName) @@ -1023,6 +1050,18 @@ func createCountersTableDeleteUpdate(tableKey string, fieldName string) tablePat } } +// createPeriodicWatermarksTableSetUpdate creates a HSET request on the PERIODIC_WATERMARKS table. +func createPeriodicWatermarksTableSetUpdate(tableKey string, fieldName string, fieldValue string) tablePathValue { + return tablePathValue{ + dbName: "COUNTERS_DB", + tableName: "PERIODIC_WATERMARKS", + tableKey: tableKey, + delimitor: ":", + field: fieldName, + value: fieldValue, + } +} + // createIntervalTickerUpdate creates a request for triggering the interval clock. func createIntervalTickerUpdate() tablePathValue { return tablePathValue{ @@ -1856,6 +1895,64 @@ type tablePathValue struct { op string } +func createQueueCountersJsonObjects(ethNum int, queueNum int, updatedCounters map[string]interface{}) ( + queueCountersJson interface{}, + queueCountersJsonAfterUpdate map[string]interface{}, + queueAliasCountersJson interface{}, + queueAliasCountersJsonAfterUpdate map[string]interface{}, + err error) { + + ethName := fmt.Sprintf("Ethernet%d", ethNum) + queueName := fmt.Sprintf("%s:%d", ethName, queueNum) + queueFile := fmt.Sprintf("../testdata/COUNTERS:%s:Queues.txt", ethName) + queueCountersByte, err := ioutil.ReadFile(queueFile) + if err != nil { + err = fmt.Errorf("read file %v err: %v", queueFile, err) + return + } + json.Unmarshal(queueCountersByte, &queueCountersJson) + + queueCountersJsonAfterUpdate = make(map[string]interface{}) + json.Unmarshal(queueCountersByte, &queueCountersJsonAfterUpdate) + queueCountersJsonAfterUpdate[queueName] = updatedCounters + + // Alias translation + queueAliasName := fmt.Sprintf("%s/1:%d", ethName, queueNum) + queueAliasFile := fmt.Sprintf("../testdata/COUNTERS:%s:Queues_alias.txt", ethName) + queueAliasCountersByte, err := ioutil.ReadFile(queueAliasFile) + if err != nil { + err = fmt.Errorf("read file %v err: %v", queueAliasFile, err) + return + } + json.Unmarshal(queueAliasCountersByte, &queueAliasCountersJson) + + queueAliasCountersJsonAfterUpdate = make(map[string]interface{}) + json.Unmarshal(queueAliasCountersByte, &queueAliasCountersJsonAfterUpdate) + queueAliasCountersJsonAfterUpdate[queueAliasName] = updatedCounters + return +} + +func createQueueCountersJsonObjectsWithPeriodicWatermark( + ethNum int, queueNum int, updatedCounters map[string]interface{}, + updatedPeriodicWatermarks map[string]interface{}, +) ( + queueCountersJson interface{}, + queueCountersJsonAfterUpdate map[string]interface{}, + queueAliasCountersJson interface{}, + queueAliasCountersJsonAfterUpdate map[string]interface{}, + err error) { + + queueCountersJson, queueCountersJsonAfterUpdate, queueAliasCountersJson, queueAliasCountersJsonAfterUpdate, err = + createQueueCountersJsonObjects(ethNum, queueNum, updatedCounters) + + queuePeriodicKeyName := fmt.Sprintf("Ethernet%d:%d:periodic", ethNum, queueNum) + queueCountersJsonAfterUpdate[queuePeriodicKeyName] = updatedPeriodicWatermarks + + queueAliasPeriodicKeyName := fmt.Sprintf("Ethernet%d/1:%d:periodic", ethNum, queueNum) + queueAliasCountersJsonAfterUpdate[queueAliasPeriodicKeyName] = updatedPeriodicWatermarks + return +} + // runTestSubscribe subscribe DB path in stream mode or poll mode. // The return code and response value are compared with expected code and value. func runTestSubscribe(t *testing.T, namespace string) { @@ -1982,36 +2079,35 @@ func runTestSubscribe(t *testing.T, namespace string) { var countersEthernetWildQueuesJson interface{} json.Unmarshal(countersEthernetWildQueuesByte, &countersEthernetWildQueuesJson) - fileName = "../testdata/COUNTERS:Ethernet68:Queues.txt" - countersEthernet68QueuesByte, err := ioutil.ReadFile(fileName) + eth68_1 := map[string]interface{}{ + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "4", // Changed from 0 to 4 + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "128", // Changed from 64 to 128 + } + eth68_1_periodic := map[string]interface{}{ + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "32", // Changed from 192 to 32 + } + countersEthernet68QueuesJson, countersEthernet68QueuesJsonUpdate, countersEthernet68QueuesAliasJson, + countersEthernet68QueuesAliasJsonUpdate, + err := createQueueCountersJsonObjectsWithPeriodicWatermark(68, 1, eth68_1, eth68_1_periodic) if err != nil { - t.Fatalf("read file %v err: %v", fileName, err) + t.Fatalf("%v", err) } - var countersEthernet68QueuesJson interface{} - json.Unmarshal(countersEthernet68QueuesByte, &countersEthernet68QueuesJson) - countersEthernet68QueuesJsonUpdate := make(map[string]interface{}) - json.Unmarshal(countersEthernet68QueuesByte, &countersEthernet68QueuesJsonUpdate) - eth68_1 := map[string]interface{}{ - "SAI_QUEUE_STAT_BYTES": "0", - "SAI_QUEUE_STAT_DROPPED_BYTES": "0", - "SAI_QUEUE_STAT_DROPPED_PACKETS": "4", - "SAI_QUEUE_STAT_PACKETS": "0", + eth7_5 := map[string]interface{}{ + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "10", // Changed from 0 to 10 + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "198", // Changed from 100 to 198 } - countersEthernet68QueuesJsonUpdate["Ethernet68:1"] = eth68_1 - - // Alias translation for query Ethernet68/1:Queues - fileName = "../testdata/COUNTERS:Ethernet68:Queues_alias.txt" - countersEthernet68QueuesAliasByte, err := ioutil.ReadFile(fileName) + countersEthernet7QueuesJson, countersEthernet7QueuesJsonUpdate, countersEthernet7QueuesAliasJson, + countersEthernet7QueuesAliasJsonUpdate, err := createQueueCountersJsonObjects(7, 5, eth7_5) if err != nil { - t.Fatalf("read file %v err: %v", fileName, err) + t.Fatalf("%v", err) } - var countersEthernet68QueuesAliasJson interface{} - json.Unmarshal(countersEthernet68QueuesAliasByte, &countersEthernet68QueuesAliasJson) - - countersEthernet68QueuesAliasJsonUpdate := make(map[string]interface{}) - json.Unmarshal(countersEthernet68QueuesAliasByte, &countersEthernet68QueuesAliasJsonUpdate) - countersEthernet68QueuesAliasJsonUpdate["Ethernet68/1:1"] = eth68_1 type TestExec struct { desc string @@ -2590,14 +2686,9 @@ func runTestSubscribe(t *testing.T, namespace string) { TLS: &tls.Config{InsecureSkipVerify: true}, }, updates: []tablePathValue{ - { - dbName: "COUNTERS_DB", - tableName: "COUNTERS", - tableKey: "oid:0x1500000000091c", // "Ethernet68:1": "oid:0x1500000000091c", - delimitor: ":", - field: "SAI_QUEUE_STAT_DROPPED_PACKETS", - value: "4", // being changed to 0 from 4 - }, + createCountersTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_DROPPED_PACKETS", "4"), // being changed from 0 to 4 + createCountersTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "128"), // being changed from 64 to 128 + createPeriodicWatermarksTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "32"), // being changed from 192 to 32 }, wantNoti: []client.Notification{ client.Connected{}, @@ -2621,14 +2712,9 @@ func runTestSubscribe(t *testing.T, namespace string) { TLS: &tls.Config{InsecureSkipVerify: true}, }, updates: []tablePathValue{ - { - dbName: "COUNTERS_DB", - tableName: "COUNTERS", - tableKey: "oid:0x1500000000091c", // "Ethernet68:1": "oid:0x1500000000091c", - delimitor: ":", - field: "SAI_QUEUE_STAT_DROPPED_PACKETS", - value: "4", // being changed to 0 from 4 - }, + createCountersTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_DROPPED_PACKETS", "4"), // being changed from 0 to 4 + createCountersTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "128"), // being changed from 64 to 128 + createPeriodicWatermarksTableSetUpdate("oid:0x1500000000091c", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "32"), // being changed from 192 to 32 }, wantNoti: []client.Notification{ client.Connected{}, @@ -2642,6 +2728,48 @@ func runTestSubscribe(t *testing.T, namespace string) { client.Sync{}, }, }, + { + desc: "poll query for COUNTERS/Ethernet7/Queues with field value change (no PERIODIC_WATERMARKS table)", + poll: 1, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"COUNTERS", "Ethernet7", "Queues"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + createCountersTableSetUpdate("oid:0x150000000001d0", "SAI_QUEUE_STAT_DROPPED_PACKETS", "10"), // being changed from 0 to 10 + createCountersTableSetUpdate("oid:0x150000000001d0", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "198"), // being changed from 100 to 198 + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"COUNTERS", "Ethernet7", "Queues"}, TS: time.Unix(0, 200), Val: countersEthernet7QueuesJson}, + client.Sync{}, + client.Update{Path: []string{"COUNTERS", "Ethernet7", "Queues"}, TS: time.Unix(0, 200), Val: countersEthernet7QueuesJsonUpdate}, + client.Sync{}, + }, + }, + { + desc: "(use vendor alias) poll query for COUNTERS/Ethernet7/Queues with field value change (no PERIODIC_WATERMARKS table)", + poll: 1, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"COUNTERS", "Ethernet7/1", "Queues"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + createCountersTableSetUpdate("oid:0x150000000001d0", "SAI_QUEUE_STAT_DROPPED_PACKETS", "10"), // being changed from 0 to 10 + createCountersTableSetUpdate("oid:0x150000000001d0", "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", "198"), // being changed from 100 to 198 + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"COUNTERS", "Ethernet7/1", "Queues"}, TS: time.Unix(0, 200), Val: countersEthernet7QueuesAliasJson}, + client.Sync{}, + client.Update{Path: []string{"COUNTERS", "Ethernet7/1", "Queues"}, TS: time.Unix(0, 200), Val: countersEthernet7QueuesAliasJsonUpdate}, + client.Sync{}, + }, + }, { desc: "use invalid sample interval", q: createCountersDbQuerySampleMode(t, 10*time.Millisecond, false, "COUNTERS", "Ethernet1"), diff --git a/sonic_data_client/virtual_db.go b/sonic_data_client/virtual_db.go index 7f52c7a51..32f34138d 100644 --- a/sonic_data_client/virtual_db.go +++ b/sonic_data_client/virtual_db.go @@ -2,9 +2,10 @@ package client import ( "fmt" - log "github.com/golang/glog" - "strings" "os" + "strings" + + log "github.com/golang/glog" ) // virtual db is to Handle @@ -157,7 +158,7 @@ func GetPfcwdMap() (map[string]map[string]string, error) { log.V(1).Infof("Can not connect to %v in namsespace %v, err: %v", dbName, namespace, err) return nil, err } - + keyName := fmt.Sprintf("%s%v*", pfcwdTableName, separator) resp, err := redisDb.Keys(keyName).Result() if err != nil { @@ -175,7 +176,7 @@ func GetPfcwdMap() (map[string]map[string]string, error) { if strings.Contains(key, "GLOBAL") || strings.Contains(key, "global") { // ignore PFC_WD|global / PFC_WD|GLOBAL continue } - name := key[len(keyName) - 1:] + name := key[len(keyName)-1:] pfcwdName_map[name] = make(map[string]string) } @@ -318,7 +319,6 @@ func getCountersMap(tableName string) (map[string]string, error) { return counter_map, nil } - // Get the mapping between objects in counters DB, Ex. port name to oid in "COUNTERS_FABRIC_PORT_NAME_MAP" table. // Aussuming static port name to oid map in COUNTERS table func getFabricCountersMap(tableName string) (map[string]string, error) { @@ -326,7 +326,7 @@ func getFabricCountersMap(tableName string) (map[string]string, error) { dbName := "COUNTERS_DB" redis_client_map, err := GetRedisClientsForDb(dbName) if err != nil { - return nil, err + return nil, err } for namespace, redisDb := range redis_client_map { fv, err := redisDb.HGetAll(tableName).Result() @@ -343,7 +343,7 @@ func getFabricCountersMap(tableName string) (map[string]string, error) { if len(namespace) != 0 { namespace_str = string('-') + namespace } - namespaceFv[k + namespace_str] = v + namespaceFv[k+namespace_str] = v } addmap(counter_map, namespaceFv) log.V(6).Infof("tableName: %s in namespace %v, map %v", tableName, namespace, namespaceFv) @@ -358,10 +358,10 @@ func v2rFabricPortStats(paths []string) ([]tablePath, error) { if strings.HasSuffix(paths[KeyIdx], "*") { // All Ethernet ports for port, oid := range countersFabricPortNameMap { var namespace string - // Extract namespace from port name + // Extract namespace from port name // multi-asic Linecard ex: PORT0-asic0 - if strings.Contains(port, "-"){ - namespace = strings.Split(port, "-")[1] + if strings.Contains(port, "-") { + namespace = strings.Split(port, "-")[1] } else { namespace = "" } @@ -383,7 +383,7 @@ func v2rFabricPortStats(paths []string) ([]tablePath, error) { if !ok { return nil, fmt.Errorf("%v not a valid sonic fabric interface.", port) } - if strings.Contains(port, "-"){ + if strings.Contains(port, "-") { namespace = strings.Split(port, "-")[1] } else { namespace = "" @@ -459,9 +459,13 @@ func v2rEthPortStats(paths []string) ([]tablePath, error) { // Supported cases: // <1> port name having suffix of "*" with specific field; -// Ex. [COUNTER_DB COUNTERS Ethernet* SAI_PORT_STAT_PFC_0_RX_PKTS] +// +// Ex. [COUNTER_DB COUNTERS Ethernet* SAI_PORT_STAT_PFC_0_RX_PKTS] +// // <2> exact port name with specific field. -// Ex. [COUNTER_DB COUNTERS Ethernet68 SAI_PORT_STAT_PFC_0_RX_PKTS] +// +// Ex. [COUNTER_DB COUNTERS Ethernet68 SAI_PORT_STAT_PFC_0_RX_PKTS] +// // case of "*" field could be covered in v2rEthPortStats() func v2rEthPortFieldStats(paths []string) ([]tablePath, error) { var tblPaths []tablePath @@ -591,14 +595,29 @@ func v2rEthPortPfcwdStats(paths []string) ([]tablePath, error) { return tblPaths, nil } +func buildTablePath(namespace, dbName, tableName, tableKey, separator, field, jsonTableName, jsonTableKey, jsonField string) tablePath { + return tablePath{ + dbNamespace: namespace, + dbName: dbName, + tableName: tableName, + tableKey: tableKey, + delimitor: separator, + field: field, + jsonTableName: jsonTableName, + jsonTableKey: jsonTableKey, + jsonField: jsonField, + } +} + // Populate real data paths from paths like -// [COUNTER_DB COUNTERS Ethernet* Queues] or [COUNTER_DB COUNTERS Ethernet68 Queues] +// [COUNTERS_DB COUNTERS Ethernet* Queues] or [COUNTERS_DB COUNTERS Ethernet68 Queues] func v2rEthPortQueStats(paths []string) ([]tablePath, error) { + // paths[DbIdx] = "COUNTERS_DB" separator, _ := GetTableKeySeparator(paths[DbIdx], "") + field := "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES" var tblPaths []tablePath if strings.HasSuffix(paths[KeyIdx], "*") { // queues on all Ethernet ports for que, oid := range countersQueueNameMap { - // que is in format of "Internal_Ethernet:12" names := strings.Split(que, separator) var oname string if alias, ok := name2aliasMap[names[0]]; ok { @@ -612,14 +631,17 @@ func v2rEthPortQueStats(paths []string) ([]tablePath, error) { return nil, fmt.Errorf("%v does not have namespace associated", names[0]) } que = strings.Join([]string{oname, names[1]}, separator) - tblPath := tablePath{ - dbNamespace: namespace, - dbName: paths[DbIdx], - tableName: paths[TblIdx], - tableKey: oid, - delimitor: separator, - jsonTableKey: que, - } + // que is in format of "Internal_Ethernet:12" + tblPath := buildTablePath(namespace, paths[DbIdx], paths[TblIdx], oid, separator, "", "", que, "") + tblPaths = append(tblPaths, tblPath) + /* + * Adding the field "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES" from the PERIODIC_WATERMARKS table + * to the virtual path. + */ + periodic_que := strings.Join([]string{que, "periodic"}, separator) + // periodic_que is in format of "Internal_Ethernet:12:periodic" + tblPath = buildTablePath(namespace, paths[DbIdx], "PERIODIC_WATERMARKS", oid, separator, + field, paths[TblIdx], periodic_que, field) tblPaths = append(tblPaths, tblPath) } } else { //queues on single port @@ -633,20 +655,22 @@ func v2rEthPortQueStats(paths []string) ([]tablePath, error) { return nil, fmt.Errorf("%v does not have namespace associated", name) } for que, oid := range countersQueueNameMap { - //que is in format of "Ethernet64:12" names := strings.Split(que, separator) if name != names[0] { continue } que = strings.Join([]string{alias, names[1]}, separator) - tblPath := tablePath{ - dbNamespace: namespace, - dbName: paths[DbIdx], - tableName: paths[TblIdx], - tableKey: oid, - delimitor: separator, - jsonTableKey: que, - } + //que is in format of "Ethernet64:12" + tblPath := buildTablePath(namespace, paths[DbIdx], paths[TblIdx], oid, separator, "", "", que, "") + tblPaths = append(tblPaths, tblPath) + /* + * Adding the field "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES" from the PERIODIC_WATERMARKS table + * to the virtual path. + */ + periodic_que := strings.Join([]string{que, "periodic"}, separator) + //periodic_que is in format of "Ethernet64:12:periodic" + tblPath = buildTablePath(namespace, paths[DbIdx], "PERIODIC_WATERMARKS", oid, separator, + field, paths[TblIdx], periodic_que, field) tblPaths = append(tblPaths, tblPath) } } diff --git a/testdata/COUNTERS:Ethernet68:Queues.txt b/testdata/COUNTERS:Ethernet68:Queues.txt index 23bda1894..d225dcfc3 100644 --- a/testdata/COUNTERS:Ethernet68:Queues.txt +++ b/testdata/COUNTERS:Ethernet68:Queues.txt @@ -4,7 +4,8 @@ "SAI_QUEUE_STAT_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", - "SAI_QUEUE_STAT_PACKETS": "0" + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "64" }, "Ethernet68:10": {}, "Ethernet68:11": {}, @@ -55,5 +56,8 @@ "Ethernet68:6": {}, "Ethernet68:7": {}, "Ethernet68:8": {}, - "Ethernet68:9": {} + "Ethernet68:9": {}, + "Ethernet68:1:periodic": { + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "192" + } } \ No newline at end of file diff --git a/testdata/COUNTERS:Ethernet68:Queues_alias.txt b/testdata/COUNTERS:Ethernet68:Queues_alias.txt index 279f07992..75123ac0a 100644 --- a/testdata/COUNTERS:Ethernet68:Queues_alias.txt +++ b/testdata/COUNTERS:Ethernet68:Queues_alias.txt @@ -4,7 +4,8 @@ "SAI_QUEUE_STAT_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", - "SAI_QUEUE_STAT_PACKETS": "0" + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "64" }, "Ethernet68/1:10": {}, "Ethernet68/1:11": {}, @@ -55,5 +56,8 @@ "Ethernet68/1:6": {}, "Ethernet68/1:7": {}, "Ethernet68/1:8": {}, - "Ethernet68/1:9": {} + "Ethernet68/1:9": {}, + "Ethernet68/1:1:periodic": { + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "192" + } } \ No newline at end of file diff --git a/testdata/COUNTERS:Ethernet7:Queues.txt b/testdata/COUNTERS:Ethernet7:Queues.txt new file mode 100644 index 000000000..f9fa60e3b --- /dev/null +++ b/testdata/COUNTERS:Ethernet7:Queues.txt @@ -0,0 +1,28 @@ +{ + "Ethernet7:0": {}, + "Ethernet7:1": {}, + "Ethernet7:10": {}, + "Ethernet7:11": {}, + "Ethernet7:12": {}, + "Ethernet7:13": {}, + "Ethernet7:14": {}, + "Ethernet7:15": {}, + "Ethernet7:16": {}, + "Ethernet7:17": {}, + "Ethernet7:18": {}, + "Ethernet7:19": {}, + "Ethernet7:2": {}, + "Ethernet7:3": {}, + "Ethernet7:4": {}, + "Ethernet7:5": { + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "100" + }, + "Ethernet7:6": {}, + "Ethernet7:7": {}, + "Ethernet7:8": {}, + "Ethernet7:9": {} +} diff --git a/testdata/COUNTERS:Ethernet7:Queues_alias.txt b/testdata/COUNTERS:Ethernet7:Queues_alias.txt new file mode 100644 index 000000000..f108727a3 --- /dev/null +++ b/testdata/COUNTERS:Ethernet7:Queues_alias.txt @@ -0,0 +1,28 @@ +{ + "Ethernet7/1:0": {}, + "Ethernet7/1:1": {}, + "Ethernet7/1:10": {}, + "Ethernet7/1:11": {}, + "Ethernet7/1:12": {}, + "Ethernet7/1:13": {}, + "Ethernet7/1:14": {}, + "Ethernet7/1:15": {}, + "Ethernet7/1:16": {}, + "Ethernet7/1:17": {}, + "Ethernet7/1:18": {}, + "Ethernet7/1:19": {}, + "Ethernet7/1:2": {}, + "Ethernet7/1:3": {}, + "Ethernet7/1:4": {}, + "Ethernet7/1:5": { + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "100" + }, + "Ethernet7/1:6": {}, + "Ethernet7/1:7": {}, + "Ethernet7/1:8": {}, + "Ethernet7/1:9": {} +} diff --git a/testdata/COUNTERS:Ethernet_wildcard_Queues.txt b/testdata/COUNTERS:Ethernet_wildcard_Queues.txt index acd5d1564..41a370550 100644 --- a/testdata/COUNTERS:Ethernet_wildcard_Queues.txt +++ b/testdata/COUNTERS:Ethernet_wildcard_Queues.txt @@ -984,7 +984,8 @@ "SAI_QUEUE_STAT_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", - "SAI_QUEUE_STAT_PACKETS": "0" + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "64" }, "Ethernet68:10": {}, "Ethernet68:11": {}, @@ -1039,7 +1040,13 @@ "Ethernet7:2": {}, "Ethernet7:3": {}, "Ethernet7:4": {}, - "Ethernet7:5": {}, + "Ethernet7:5": { + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "100" + }, "Ethernet7:6": {}, "Ethernet7:7": {}, "Ethernet7:8": {}, @@ -1083,6 +1090,9 @@ "Ethernet9:6": {}, "Ethernet9:7": {}, "Ethernet9:8": {}, - "Ethernet9:9": {} + "Ethernet9:9": {}, + "Ethernet68:1:periodic": { + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "192" + } } diff --git a/testdata/COUNTERS:Ethernet_wildcard_Queues_alias.txt b/testdata/COUNTERS:Ethernet_wildcard_Queues_alias.txt index 42f6428fd..19b438f49 100644 --- a/testdata/COUNTERS:Ethernet_wildcard_Queues_alias.txt +++ b/testdata/COUNTERS:Ethernet_wildcard_Queues_alias.txt @@ -1004,7 +1004,8 @@ "SAI_QUEUE_STAT_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", - "SAI_QUEUE_STAT_PACKETS": "0" + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "64" }, "Ethernet68/1:10": {}, "Ethernet68/1:11": {}, @@ -1071,7 +1072,13 @@ "Ethernet7/1:2": {}, "Ethernet7/1:3": {}, "Ethernet7/1:4": {}, - "Ethernet7/1:5": {}, + "Ethernet7/1:5": { + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "100" + }, "Ethernet7/1:6": {}, "Ethernet7/1:7": {}, "Ethernet7/1:8": {}, @@ -1115,5 +1122,8 @@ "Ethernet9/1:6": {}, "Ethernet9/1:7": {}, "Ethernet9/1:8": {}, - "Ethernet9/1:9": {} + "Ethernet9/1:9": {}, + "Ethernet68/1:1:periodic": { + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "192" + } } \ No newline at end of file diff --git a/testdata/COUNTERS:oid:0x150000000001d0.txt b/testdata/COUNTERS:oid:0x150000000001d0.txt new file mode 100644 index 000000000..e6759c62b --- /dev/null +++ b/testdata/COUNTERS:oid:0x150000000001d0.txt @@ -0,0 +1,7 @@ +{ + "SAI_QUEUE_STAT_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_BYTES": "0", + "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "100" +} diff --git a/testdata/COUNTERS:oid:0x1500000000091c.txt b/testdata/COUNTERS:oid:0x1500000000091c.txt index d8b944a0b..f17811d13 100644 --- a/testdata/COUNTERS:oid:0x1500000000091c.txt +++ b/testdata/COUNTERS:oid:0x1500000000091c.txt @@ -2,5 +2,6 @@ "SAI_QUEUE_STAT_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_BYTES": "0", "SAI_QUEUE_STAT_DROPPED_PACKETS": "0", - "SAI_QUEUE_STAT_PACKETS": "0" + "SAI_QUEUE_STAT_PACKETS": "0", + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "64" } diff --git a/testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt b/testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt new file mode 100644 index 000000000..c5805d9e8 --- /dev/null +++ b/testdata/PERIODIC_WATERMARKS:oid:0x1500000000091c.txt @@ -0,0 +1,3 @@ +{ + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES": "192" +} From 1410e721dd4f776140be3f7ac495ea5da9d39b4c Mon Sep 17 00:00:00 2001 From: mramezani95 Date: Thu, 26 Jun 2025 21:09:08 -0700 Subject: [PATCH 08/14] [202411] Adding virtual paths for periodic priority group watermarks (#431) PR for master: #370 Why I did it Periodic Priority Group (PG) watermarks can be found in the PERIODIC_WATERMARKS table in COUNTERS DB. The entries in this table can be accessed by OIDs (i.e., PERIODIC_WATERMARKS:), and the mapping between PG names (e.g., Ethernet16:5) and OIDs (e.g., oid:0x1a0000000000d6) is provided in the COUNTERS_PG_NAME_MAP table. So we need to add this mapping and virtual paths for periodic PG watermarks in order to be able to access the periodic PG watermarks directly (i.e., by specifying the port name instead of OID) in sonic-gnmi. How I did it Created the mapping between PG names and OIDs by reading the COUNTERS_PG_NAME_MAP table. Created the virtual path PERIODIC_WATERMARKS/Ethernet*/PriorityGroups and mapped it to the relevant entries in the PERIODIC_WATERMARKS table in COUNTERS DB. Added a few tests under TestGnmiSubscribe to verify that periodic PG watermark values are updated correctly. How to verify it I used gnmi_get to verify that the virtual path PERIODIC_WATERMARKS/Ethernet16/PriorityGroups retrieves all periodic PG watermarks for Ethernet16 and that the virtual path PERIODIC_WATERMARKS/Ethernet*/PriorityGroups retrieves all periodic PG watermarks for all ports. The results are added to grpc_telemetry.md. Note: The DUT on which the gnmi-get command was run had only one priority group per port. Description for the changelog Added the virtual path PERIODIC_WATERMARKS/Ethernet*/PriorityGroups so that periodic PG watermarks can be queried directly by specifying the port name in sonic-gnmi. --- doc/grpc_telemetry.md | 89 ++++ gnmi_server/server_test.go | 161 ++++++- sonic_data_client/db_client.go | 4 + sonic_data_client/virtual_db.go | 96 +++- testdata/COUNTERS_PG_NAME_MAP.txt | 434 +++++++++++++++++ ...C_WATERMARKS:Ethernet16:PriorityGroups.txt | 13 + ...RMARKS:Ethernet16:PriorityGroups_alias.txt | 13 + ...Ethernet_wildcard:PriorityGroups_alias.txt | 437 ++++++++++++++++++ ...RIODIC_WATERMARKS:oid:0x1a0000000002d2.txt | 4 + 9 files changed, 1247 insertions(+), 4 deletions(-) create mode 100644 testdata/COUNTERS_PG_NAME_MAP.txt create mode 100644 testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups.txt create mode 100644 testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups_alias.txt create mode 100644 testdata/PERIODIC_WATERMARKS:Ethernet_wildcard:PriorityGroups_alias.txt create mode 100644 testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt diff --git a/doc/grpc_telemetry.md b/doc/grpc_telemetry.md index dedbb291b..e2cd16e89 100644 --- a/doc/grpc_telemetry.md +++ b/doc/grpc_telemetry.md @@ -386,6 +386,8 @@ Some of the SONiC database tables contain aggregated data. Ex. COUNTERS in COUNT |COUNTERS_DB | "COUNTERS/Ethernet``/``"| One counter on one Ethernet port |COUNTERS_DB | "COUNTERS/Ethernet*/Queues"| Queues stats on all Ethernet ports |COUNTERS_DB | "COUNTERS/Ethernet``/Queues"| Queue stats on one Ethernet ports +|COUNTERS_DB | "PERIODIC_WATERMARKS/Ethernet*/PriorityGroups"| Periodic watermarks for priority groups on all Ethernet ports +|COUNTERS_DB | "PERIODIC_WATERMARKS/Ethernet<``port number``>/PriorityGroups"| Periodic watermarks for priority groups on one Ethernet port Virtual path supports Get, Subscribe Poll and stream operations. @@ -562,6 +564,93 @@ notification: < > ``` +``` +gnmi_get -target_addr 127.0.0.1:50051 -xpath "PERIODIC_WATERMARKS/Ethernet16/PriorityGroups" -xpath_target COUNTERS_DB -insecure +== getRequest: +prefix: < + target: "COUNTERS_DB" +> +path: < + elem: < + name: "PERIODIC_WATERMARKS" + > + elem: < + name: "Ethernet16" + > + elem: < + name: "PriorityGroups" + > +> +encoding: JSON_IETF + +== getResponse: +notification: < + timestamp: 1749598624296471376 + prefix: < + target: "COUNTERS_DB" + > + update: < + path: < + elem: < + name: "PERIODIC_WATERMARKS" + > + elem: < + name: "Ethernet16" + > + elem: < + name: "PriorityGroups" + > + > + val: < + json_ietf_val: "{\"Ethernet16:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"}}" + > + > +> +``` + +``` +gnmi_get -target_addr 127.0.0.1:50051 -xpath "PERIODIC_WATERMARKS/Ethernet*/PriorityGroups" -xpath_target COUNTERS_DB -insecure +== getRequest: +prefix: < + target: "COUNTERS_DB" +> +path: < + elem: < + name: "PERIODIC_WATERMARKS" + > + elem: < + name: "Ethernet*" + > + elem: < + name: "PriorityGroups" + > +> +encoding: JSON_IETF + +== getResponse: +notification: < + timestamp: 1749598585172470836 + prefix: < + target: "COUNTERS_DB" + > + update: < + path: < + elem: < + name: "PERIODIC_WATERMARKS" + > + elem: < + name: "Ethernet*" + > + elem: < + name: "PriorityGroups" + > + > + val: < + json_ietf_val: "{\"etp10a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp11a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp12a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp13a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp13b:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"0\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp15a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp16a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp19a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp1a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},...,\"etp61a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp62a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp63a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp64a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp6a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp7a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp8a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"},\"etp9a:0\":{\"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES\":\"192\",\"SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES\":\"0\"}}" + > + > +> +``` # Authentication To be implemented, may support integration with SONiC TACACS. User will be authenticated on per RPC basis. diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index df770e632..1cfbb31df 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -58,8 +58,8 @@ import ( "github.com/jipanyang/gnxi/utils/xpath" cacheclient "github.com/openconfig/gnmi/client" gnmipb "github.com/openconfig/gnmi/proto/gnmi" - gnoi_system_pb "github.com/openconfig/gnoi/system" gnoi_file_pb "github.com/openconfig/gnoi/file" + gnoi_system_pb "github.com/openconfig/gnoi/system" "github.com/sonic-net/sonic-gnmi/common_utils" "github.com/sonic-net/sonic-gnmi/swsscommon" ) @@ -621,6 +621,14 @@ func initFullCountersDb(t *testing.T, namespace string) { mpi_qname_map := loadConfig(t, "COUNTERS_QUEUE_NAME_MAP", countersQueueNameMapByte) loadDB(t, rclient, mpi_qname_map) + fileName = "../testdata/COUNTERS_PG_NAME_MAP.txt" + countersPGNameMapByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_pgname_map := loadConfig(t, "COUNTERS_PG_NAME_MAP", countersPGNameMapByte) + loadDB(t, rclient, mpi_pgname_map) + fileName = "../testdata/COUNTERS:Ethernet68.txt" countersEthernet68Byte, err := ioutil.ReadFile(fileName) if err != nil { @@ -666,6 +674,25 @@ func initFullCountersDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "PERIODIC_WATERMARKS:oid:0x1500000000091c", periodicWMEth68_1Byte) loadDB(t, rclient, mpi_counter) + // "Ethernet7:5": "oid:0x150000000001d0" : queue counter, for COUNTERS/Ethernet7/Queues vpath test + fileName = "../testdata/COUNTERS:oid:0x150000000001d0.txt" + countersEth7_5Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "COUNTERS:oid:0x150000000001d0", countersEth7_5Byte) + loadDB(t, rclient, mpi_counter) + + // "Ethernet16:5": "oid:0x1a0000000002d2" + // PG periodic watermark, for PERIODIC_WATERMARKS/Ethernet16/PriorityGroups vpath test + fileName = "../testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt" + pgWatermarksEth16_5Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "PERIODIC_WATERMARKS:oid:0x1a0000000002d2", pgWatermarksEth16_5Byte) + loadDB(t, rclient, mpi_counter) + // "Ethernet68:3": "oid:0x1500000000091e" : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test fileName = "../testdata/COUNTERS:oid:0x1500000000091e.txt" countersEeth68_3Byte, err := ioutil.ReadFile(fileName) @@ -775,6 +802,14 @@ func prepareDb(t *testing.T, namespace string) { mpi_qname_map := loadConfig(t, "COUNTERS_QUEUE_NAME_MAP", countersQueueNameMapByte) loadDB(t, rclient, mpi_qname_map) + fileName = "../testdata/COUNTERS_PG_NAME_MAP.txt" + countersPGNameMapByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_pgname_map := loadConfig(t, "COUNTERS_PG_NAME_MAP", countersPGNameMapByte) + loadDB(t, rclient, mpi_pgname_map) + fileName = "../testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt" countersFabricPortNameMapByte, err := ioutil.ReadFile(fileName) if err != nil { @@ -830,11 +865,21 @@ func prepareDb(t *testing.T, namespace string) { // "Ethernet7:5": "oid:0x150000000001d0" : queue counter, for COUNTERS/Ethernet7/Queues vpath test fileName = "../testdata/COUNTERS:oid:0x150000000001d0.txt" - countersEeth7_5Byte, err := ioutil.ReadFile(fileName) + countersEth7_5Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + mpi_counter = loadConfig(t, "COUNTERS:oid:0x150000000001d0", countersEth7_5Byte) + loadDB(t, rclient, mpi_counter) + + // "Ethernet16:5": "oid:0x1a0000000002d2" + // PG periodic watermark, for PERIODIC_WATERMARKS/Ethernet16/PriorityGroups vpath test + fileName = "../testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt" + pgWatermarksEth16_5Byte, err := ioutil.ReadFile(fileName) if err != nil { t.Fatalf("read file %v err: %v", fileName, err) } - mpi_counter = loadConfig(t, "COUNTERS:oid:0x150000000001d0", countersEeth7_5Byte) + mpi_counter = loadConfig(t, "PERIODIC_WATERMARKS:oid:0x1a0000000002d2", pgWatermarksEth16_5Byte) loadDB(t, rclient, mpi_counter) // "Ethernet68:3": "oid:0x1500000000091e" : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test @@ -2071,6 +2116,8 @@ func runTestSubscribe(t *testing.T, namespace string) { tmp6.(map[string]interface{})["Ethernet68/1:3"].(map[string]interface{})["PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED"] = "1" countersEthernetWildPfcwdUpdate := tmp6 + /////////////////////////////////////////////////////////////////////////////////////////////// + fileName = "../testdata/COUNTERS:Ethernet_wildcard_Queues_alias.txt" countersEthernetWildQueuesByte, err := ioutil.ReadFile(fileName) if err != nil { @@ -2109,6 +2156,47 @@ func runTestSubscribe(t *testing.T, namespace string) { t.Fatalf("%v", err) } + /////////////////////////////////////////////////////////////////////////////////////////////// + + fileName = "../testdata/PERIODIC_WATERMARKS:Ethernet_wildcard:PriorityGroups_alias.txt" + pgWatermarksEthernetWildByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var pgWatermarksEthernetWildJson interface{} + json.Unmarshal(pgWatermarksEthernetWildByte, &pgWatermarksEthernetWildJson) + + fileName = "../testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups.txt" + pgWatermarksEthernet16Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var pgWatermarksEthernet16Json interface{} + json.Unmarshal(pgWatermarksEthernet16Byte, &pgWatermarksEthernet16Json) + + pgWatermarksEthernet16JsonUpdate := make(map[string]interface{}) + json.Unmarshal(pgWatermarksEthernet16Byte, &pgWatermarksEthernet16JsonUpdate) + eth16_5 := map[string]interface{}{ + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES": "200", // Changed from 0 to 200 + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES": "10", // Changed from 0 to 10 + } + pgWatermarksEthernet16JsonUpdate["Ethernet16:5"] = eth16_5 + + // Alias translation for query PERIODIC_WATERMARKS:Ethernet16/1:PriorityGroups + fileName = "../testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups_alias.txt" + pgWatermarksEthernet16AliasByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var pgWatermarksEthernet16AliasJson interface{} + json.Unmarshal(pgWatermarksEthernet16AliasByte, &pgWatermarksEthernet16AliasJson) + + pgWatermarksEthernet16AliasJsonUpdate := make(map[string]interface{}) + json.Unmarshal(pgWatermarksEthernet16AliasByte, &pgWatermarksEthernet16AliasJsonUpdate) + pgWatermarksEthernet16AliasJsonUpdate["Ethernet16/1:5"] = eth16_5 + + /////////////////////////////////////////////////////////////////////////////////////////////// + type TestExec struct { desc string q client.Query @@ -2770,6 +2858,73 @@ func runTestSubscribe(t *testing.T, namespace string) { client.Sync{}, }, }, + { + desc: "poll query for PERIODIC_WATERMARKS/Ethernet*/PriorityGroups", + poll: 1, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"PERIODIC_WATERMARKS", "Ethernet*", "PriorityGroups"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet*", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernetWildJson}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet*", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernetWildJson}, + client.Sync{}, + }, + }, + { + desc: "poll query for PERIODIC_WATERMARKS/Ethernet16/PriorityGroups with field value change", + poll: 3, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"PERIODIC_WATERMARKS", "Ethernet16", "PriorityGroups"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + createPeriodicWatermarksTableSetUpdate("oid:0x1a0000000002d2", "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES", "200"), // being changed from 0 to 200 + createPeriodicWatermarksTableSetUpdate("oid:0x1a0000000002d2", "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES", "10"), // being changed from 0 to 10 + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16Json}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16JsonUpdate}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16JsonUpdate}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16JsonUpdate}, + client.Sync{}, + }, + }, + { + desc: "(use vendor alias) poll query for PERIODIC_WATERMARKS/Ethernet16/PriorityGroups with field value change", + poll: 3, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"PERIODIC_WATERMARKS", "Ethernet16/1", "PriorityGroups"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + createPeriodicWatermarksTableSetUpdate("oid:0x1a0000000002d2", "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES", "200"), // being changed from 0 to 200 + createPeriodicWatermarksTableSetUpdate("oid:0x1a0000000002d2", "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES", "10"), // being changed from 0 to 10 + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16/1", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16AliasJson}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16/1", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16AliasJsonUpdate}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16/1", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16AliasJsonUpdate}, + client.Sync{}, + client.Update{Path: []string{"PERIODIC_WATERMARKS", "Ethernet16/1", "PriorityGroups"}, TS: time.Unix(0, 200), Val: pgWatermarksEthernet16AliasJsonUpdate}, + client.Sync{}, + }, + }, { desc: "use invalid sample interval", q: createCountersDbQuerySampleMode(t, 10*time.Millisecond, false, "COUNTERS", "Ethernet1"), diff --git a/sonic_data_client/db_client.go b/sonic_data_client/db_client.go index 2b56392ec..65e08339e 100644 --- a/sonic_data_client/db_client.go +++ b/sonic_data_client/db_client.go @@ -660,6 +660,10 @@ func populateDbtablePath(prefix, path *gnmipb.Path, pathG2S *map[*gnmipb.Path][] if err != nil { return err } + err = initCountersPGNameMap() + if err != nil { + return err + } err = initAliasMap() if err != nil { return err diff --git a/sonic_data_client/virtual_db.go b/sonic_data_client/virtual_db.go index 32f34138d..589c6aa23 100644 --- a/sonic_data_client/virtual_db.go +++ b/sonic_data_client/virtual_db.go @@ -36,6 +36,9 @@ var ( // Queue name to oid map in COUNTERS table of COUNTERS_DB countersQueueNameMap = make(map[string]string) + // SONiC interface name to priority group indices and then to oid (from COUNTERS table of COUNTERS_DB) + countersPGNameMap = make(map[string]map[string]string) + // Alias translation: from vendor port name to sonic interface name alias2nameMap = make(map[string]string) // Alias translation: from sonic interface name to vendor port name @@ -67,7 +70,10 @@ var ( }, { // stats for one or all Fabric ports path: []string{"COUNTERS_DB", "COUNTERS", "PORT*"}, transFunc: v2rTranslate(v2rFabricPortStats), - }, + }, { // Periodic PG watermarks for one or all Ethernet ports + path: []string{"COUNTERS_DB", "PERIODIC_WATERMARKS", "Ethernet*", "PriorityGroups"}, + transFunc: v2rTranslate(v2rEthPortPGPeriodicWMs), + }, } ) @@ -94,6 +100,27 @@ func initCountersQueueNameMap() error { return nil } +func initCountersPGNameMap() error { + if len(countersPGNameMap) == 0 { + pgOidMap, err := getCountersMap("COUNTERS_PG_NAME_MAP") + if err != nil { + return err + } + for pg, oid := range pgOidMap { + // pg is in format of "Ethernet64:7" + pg_parts := strings.Split(pg, ":") + if len(pg_parts) != 2 { + return fmt.Errorf("invalid pg name %v", pg) + } + if _, ok := countersPGNameMap[pg_parts[0]]; !ok { + countersPGNameMap[pg_parts[0]] = make(map[string]string) + } + countersPGNameMap[pg_parts[0]][pg_parts[1]] = oid + } + } + return nil +} + func initCountersPortNameMap() error { var err error if len(countersPortNameMap) == 0 { @@ -678,6 +705,73 @@ func v2rEthPortQueStats(paths []string) ([]tablePath, error) { return tblPaths, nil } +func getVendorPortName(port string) string { + // Get vendor port name from name2aliasMap + if alias, ok := name2aliasMap[port]; ok { + return alias + } + log.V(2).Infof(" %v does not have a vendor alias", port) + return port +} + +func getSonicPortName(port string) string { + // Get sonic port name from alias2nameMap + if sonic_name, ok := alias2nameMap[port]; ok { + return sonic_name + } + return port +} + +func getPortNamespace(port string) (string, error) { + // Get namespace from port2namespaceMap + namespace, ok := port2namespaceMap[port] + if !ok { + return "", fmt.Errorf("%v does not have an associated namespace", port) + } + return namespace, nil +} + +// Populate real data paths from paths like +// [COUNTERS_DB PERIODIC_WATERMARKS Ethernet* PriorityGroups] or +// [COUNTERS_DB PERIODIC_WATERMARKS Ethernet64 PriorityGroups] +func v2rEthPortPGPeriodicWMs(paths []string) ([]tablePath, error) { + // paths[DbIdx] = "COUNTERS_DB" + separator, _ := GetTableKeySeparator(paths[DbIdx], "") + var tblPaths []tablePath + if strings.HasSuffix(paths[KeyIdx], "*") { // priority groups on all Ethernet ports + for port, pg_index_to_oid := range countersPGNameMap { + vendor_port := getVendorPortName(port) + namespace, err := getPortNamespace(port) + if err != nil { + return nil, err + } + for pg_index, oid := range pg_index_to_oid { + pg := strings.Join([]string{vendor_port, pg_index}, separator) + // pg is in format of "Internal_Ethernet:7" + tblPath := buildTablePath(namespace, paths[DbIdx], paths[TblIdx], oid, separator, "", "", pg, "") + tblPaths = append(tblPaths, tblPath) + } + } + } else { // priority groups on a single port + port := getSonicPortName(paths[KeyIdx]) + namespace, err := getPortNamespace(port) + if err != nil { + return nil, err + } + if _, ok := countersPGNameMap[port]; !ok { + return nil, fmt.Errorf("No priority groups associated with %v", port) + } + for pg_index, oid := range countersPGNameMap[port] { + pg := strings.Join([]string{paths[KeyIdx], pg_index}, separator) + // pg is in format of "Ethernet64:7" or "Ethernet64/1:7" + tblPath := buildTablePath(namespace, paths[DbIdx], paths[TblIdx], oid, separator, "", "", pg, "") + tblPaths = append(tblPaths, tblPath) + } + } + log.V(6).Infof("v2rEthPortPGPeriodicWMs: %v", tblPaths) + return tblPaths, nil +} + func lookupV2R(paths []string) ([]tablePath, error) { n, ok := v2rTrie.Find(paths) if ok { diff --git a/testdata/COUNTERS_PG_NAME_MAP.txt b/testdata/COUNTERS_PG_NAME_MAP.txt new file mode 100644 index 000000000..ba30f6cb2 --- /dev/null +++ b/testdata/COUNTERS_PG_NAME_MAP.txt @@ -0,0 +1,434 @@ +{ + "Ethernet0:0": "oid:0x1a00000000002d", + "Ethernet0:1": "oid:0x1a00000000002e", + "Ethernet0:2": "oid:0x1a00000000002f", + "Ethernet0:3": "oid:0x1a000000000030", + "Ethernet0:4": "oid:0x1a000000000031", + "Ethernet0:5": "oid:0x1a000000000032", + "Ethernet0:6": "oid:0x1a000000000033", + "Ethernet0:7": "oid:0x1a000000000034", + "Ethernet1:0": "oid:0x1a000000000057", + "Ethernet1:1": "oid:0x1a000000000058", + "Ethernet1:2": "oid:0x1a000000000059", + "Ethernet1:3": "oid:0x1a00000000005a", + "Ethernet1:4": "oid:0x1a00000000005b", + "Ethernet1:5": "oid:0x1a00000000005c", + "Ethernet1:6": "oid:0x1a00000000005d", + "Ethernet1:7": "oid:0x1a00000000005e", + "Ethernet10:0": "oid:0x1a0000000001d1", + "Ethernet10:1": "oid:0x1a0000000001d2", + "Ethernet10:2": "oid:0x1a0000000001d3", + "Ethernet10:3": "oid:0x1a0000000001d4", + "Ethernet10:4": "oid:0x1a0000000001d5", + "Ethernet10:5": "oid:0x1a0000000001d6", + "Ethernet10:6": "oid:0x1a0000000001d7", + "Ethernet10:7": "oid:0x1a0000000001d8", + "Ethernet11:0": "oid:0x1a0000000001fb", + "Ethernet11:1": "oid:0x1a0000000001fc", + "Ethernet11:2": "oid:0x1a0000000001fd", + "Ethernet11:3": "oid:0x1a0000000001fe", + "Ethernet11:4": "oid:0x1a0000000001ff", + "Ethernet11:5": "oid:0x1a000000000200", + "Ethernet11:6": "oid:0x1a000000000201", + "Ethernet11:7": "oid:0x1a000000000202", + "Ethernet12:0": "oid:0x1a000000000225", + "Ethernet12:1": "oid:0x1a000000000226", + "Ethernet12:2": "oid:0x1a000000000227", + "Ethernet12:3": "oid:0x1a000000000228", + "Ethernet12:4": "oid:0x1a000000000229", + "Ethernet12:5": "oid:0x1a00000000022a", + "Ethernet12:6": "oid:0x1a00000000022b", + "Ethernet12:7": "oid:0x1a00000000022c", + "Ethernet13:0": "oid:0x1a00000000024f", + "Ethernet13:1": "oid:0x1a000000000250", + "Ethernet13:2": "oid:0x1a000000000251", + "Ethernet13:3": "oid:0x1a000000000252", + "Ethernet13:4": "oid:0x1a000000000253", + "Ethernet13:5": "oid:0x1a000000000254", + "Ethernet13:6": "oid:0x1a000000000255", + "Ethernet13:7": "oid:0x1a000000000256", + "Ethernet14:0": "oid:0x1a000000000279", + "Ethernet14:1": "oid:0x1a00000000027a", + "Ethernet14:2": "oid:0x1a00000000027b", + "Ethernet14:3": "oid:0x1a00000000027c", + "Ethernet14:4": "oid:0x1a00000000027d", + "Ethernet14:5": "oid:0x1a00000000027e", + "Ethernet14:6": "oid:0x1a00000000027f", + "Ethernet14:7": "oid:0x1a000000000280", + "Ethernet15:0": "oid:0x1a0000000002a3", + "Ethernet15:1": "oid:0x1a0000000002a4", + "Ethernet15:2": "oid:0x1a0000000002a5", + "Ethernet15:3": "oid:0x1a0000000002a6", + "Ethernet15:4": "oid:0x1a0000000002a7", + "Ethernet15:5": "oid:0x1a0000000002a8", + "Ethernet15:6": "oid:0x1a0000000002a9", + "Ethernet15:7": "oid:0x1a0000000002aa", + "Ethernet16:0": "oid:0x1a0000000002cd", + "Ethernet16:1": "oid:0x1a0000000002ce", + "Ethernet16:2": "oid:0x1a0000000002cf", + "Ethernet16:3": "oid:0x1a0000000002d0", + "Ethernet16:4": "oid:0x1a0000000002d1", + "Ethernet16:5": "oid:0x1a0000000002d2", + "Ethernet16:6": "oid:0x1a0000000002d3", + "Ethernet16:7": "oid:0x1a0000000002d4", + "Ethernet17:0": "oid:0x1a0000000002f7", + "Ethernet17:1": "oid:0x1a0000000002f8", + "Ethernet17:2": "oid:0x1a0000000002f9", + "Ethernet17:3": "oid:0x1a0000000002fa", + "Ethernet17:4": "oid:0x1a0000000002fb", + "Ethernet17:5": "oid:0x1a0000000002fc", + "Ethernet17:6": "oid:0x1a0000000002fd", + "Ethernet17:7": "oid:0x1a0000000002fe", + "Ethernet18:0": "oid:0x1a000000000321", + "Ethernet18:1": "oid:0x1a000000000322", + "Ethernet18:2": "oid:0x1a000000000323", + "Ethernet18:3": "oid:0x1a000000000324", + "Ethernet18:4": "oid:0x1a000000000325", + "Ethernet18:5": "oid:0x1a000000000326", + "Ethernet18:6": "oid:0x1a000000000327", + "Ethernet18:7": "oid:0x1a000000000328", + "Ethernet19:0": "oid:0x1a00000000034b", + "Ethernet19:1": "oid:0x1a00000000034c", + "Ethernet19:2": "oid:0x1a00000000034d", + "Ethernet19:3": "oid:0x1a00000000034e", + "Ethernet19:4": "oid:0x1a00000000034f", + "Ethernet19:5": "oid:0x1a000000000350", + "Ethernet19:6": "oid:0x1a000000000351", + "Ethernet19:7": "oid:0x1a000000000352", + "Ethernet2:0": "oid:0x1a000000000081", + "Ethernet2:1": "oid:0x1a000000000082", + "Ethernet2:2": "oid:0x1a000000000083", + "Ethernet2:3": "oid:0x1a000000000084", + "Ethernet2:4": "oid:0x1a000000000085", + "Ethernet2:5": "oid:0x1a000000000086", + "Ethernet2:6": "oid:0x1a000000000087", + "Ethernet2:7": "oid:0x1a000000000088", + "Ethernet20:0": "oid:0x1a000000000375", + "Ethernet20:1": "oid:0x1a000000000376", + "Ethernet20:2": "oid:0x1a000000000377", + "Ethernet20:3": "oid:0x1a000000000378", + "Ethernet20:4": "oid:0x1a000000000379", + "Ethernet20:5": "oid:0x1a00000000037a", + "Ethernet20:6": "oid:0x1a00000000037b", + "Ethernet20:7": "oid:0x1a00000000037c", + "Ethernet21:0": "oid:0x1a00000000039f", + "Ethernet21:1": "oid:0x1a0000000003a0", + "Ethernet21:2": "oid:0x1a0000000003a1", + "Ethernet21:3": "oid:0x1a0000000003a2", + "Ethernet21:4": "oid:0x1a0000000003a3", + "Ethernet21:5": "oid:0x1a0000000003a4", + "Ethernet21:6": "oid:0x1a0000000003a5", + "Ethernet21:7": "oid:0x1a0000000003a6", + "Ethernet22:0": "oid:0x1a0000000003c9", + "Ethernet22:1": "oid:0x1a0000000003ca", + "Ethernet22:2": "oid:0x1a0000000003cb", + "Ethernet22:3": "oid:0x1a0000000003cc", + "Ethernet22:4": "oid:0x1a0000000003cd", + "Ethernet22:5": "oid:0x1a0000000003ce", + "Ethernet22:6": "oid:0x1a0000000003cf", + "Ethernet22:7": "oid:0x1a0000000003d0", + "Ethernet23:0": "oid:0x1a0000000003f3", + "Ethernet23:1": "oid:0x1a0000000003f4", + "Ethernet23:2": "oid:0x1a0000000003f5", + "Ethernet23:3": "oid:0x1a0000000003f6", + "Ethernet23:4": "oid:0x1a0000000003f7", + "Ethernet23:5": "oid:0x1a0000000003f8", + "Ethernet23:6": "oid:0x1a0000000003f9", + "Ethernet23:7": "oid:0x1a0000000003fa", + "Ethernet24:0": "oid:0x1a00000000041d", + "Ethernet24:1": "oid:0x1a00000000041e", + "Ethernet24:2": "oid:0x1a00000000041f", + "Ethernet24:3": "oid:0x1a000000000420", + "Ethernet24:4": "oid:0x1a000000000421", + "Ethernet24:5": "oid:0x1a000000000422", + "Ethernet24:6": "oid:0x1a000000000423", + "Ethernet24:7": "oid:0x1a000000000424", + "Ethernet25:0": "oid:0x1a000000000447", + "Ethernet25:1": "oid:0x1a000000000448", + "Ethernet25:2": "oid:0x1a000000000449", + "Ethernet25:3": "oid:0x1a00000000044a", + "Ethernet25:4": "oid:0x1a00000000044b", + "Ethernet25:5": "oid:0x1a00000000044c", + "Ethernet25:6": "oid:0x1a00000000044d", + "Ethernet25:7": "oid:0x1a00000000044e", + "Ethernet26:0": "oid:0x1a000000000471", + "Ethernet26:1": "oid:0x1a000000000472", + "Ethernet26:2": "oid:0x1a000000000473", + "Ethernet26:3": "oid:0x1a000000000474", + "Ethernet26:4": "oid:0x1a000000000475", + "Ethernet26:5": "oid:0x1a000000000476", + "Ethernet26:6": "oid:0x1a000000000477", + "Ethernet26:7": "oid:0x1a000000000478", + "Ethernet27:0": "oid:0x1a00000000049b", + "Ethernet27:1": "oid:0x1a00000000049c", + "Ethernet27:2": "oid:0x1a00000000049d", + "Ethernet27:3": "oid:0x1a00000000049e", + "Ethernet27:4": "oid:0x1a00000000049f", + "Ethernet27:5": "oid:0x1a0000000004a0", + "Ethernet27:6": "oid:0x1a0000000004a1", + "Ethernet27:7": "oid:0x1a0000000004a2", + "Ethernet28:0": "oid:0x1a0000000004c5", + "Ethernet28:1": "oid:0x1a0000000004c6", + "Ethernet28:2": "oid:0x1a0000000004c7", + "Ethernet28:3": "oid:0x1a0000000004c8", + "Ethernet28:4": "oid:0x1a0000000004c9", + "Ethernet28:5": "oid:0x1a0000000004ca", + "Ethernet28:6": "oid:0x1a0000000004cb", + "Ethernet28:7": "oid:0x1a0000000004cc", + "Ethernet29:0": "oid:0x1a0000000004ef", + "Ethernet29:1": "oid:0x1a0000000004f0", + "Ethernet29:2": "oid:0x1a0000000004f1", + "Ethernet29:3": "oid:0x1a0000000004f2", + "Ethernet29:4": "oid:0x1a0000000004f3", + "Ethernet29:5": "oid:0x1a0000000004f4", + "Ethernet29:6": "oid:0x1a0000000004f5", + "Ethernet29:7": "oid:0x1a0000000004f6", + "Ethernet3:0": "oid:0x1a0000000000ab", + "Ethernet3:1": "oid:0x1a0000000000ac", + "Ethernet3:2": "oid:0x1a0000000000ad", + "Ethernet3:3": "oid:0x1a0000000000ae", + "Ethernet3:4": "oid:0x1a0000000000af", + "Ethernet3:5": "oid:0x1a0000000000b0", + "Ethernet3:6": "oid:0x1a0000000000b1", + "Ethernet3:7": "oid:0x1a0000000000b2", + "Ethernet30:0": "oid:0x1a000000000519", + "Ethernet30:1": "oid:0x1a00000000051a", + "Ethernet30:2": "oid:0x1a00000000051b", + "Ethernet30:3": "oid:0x1a00000000051c", + "Ethernet30:4": "oid:0x1a00000000051d", + "Ethernet30:5": "oid:0x1a00000000051e", + "Ethernet30:6": "oid:0x1a00000000051f", + "Ethernet30:7": "oid:0x1a000000000520", + "Ethernet31:0": "oid:0x1a000000000543", + "Ethernet31:1": "oid:0x1a000000000544", + "Ethernet31:2": "oid:0x1a000000000545", + "Ethernet31:3": "oid:0x1a000000000546", + "Ethernet31:4": "oid:0x1a000000000547", + "Ethernet31:5": "oid:0x1a000000000548", + "Ethernet31:6": "oid:0x1a000000000549", + "Ethernet31:7": "oid:0x1a00000000054a", + "Ethernet32:0": "oid:0x1a00000000056d", + "Ethernet32:1": "oid:0x1a00000000056e", + "Ethernet32:2": "oid:0x1a00000000056f", + "Ethernet32:3": "oid:0x1a000000000570", + "Ethernet32:4": "oid:0x1a000000000571", + "Ethernet32:5": "oid:0x1a000000000572", + "Ethernet32:6": "oid:0x1a000000000573", + "Ethernet32:7": "oid:0x1a000000000574", + "Ethernet33:0": "oid:0x1a000000000597", + "Ethernet33:1": "oid:0x1a000000000598", + "Ethernet33:2": "oid:0x1a000000000599", + "Ethernet33:3": "oid:0x1a00000000059a", + "Ethernet33:4": "oid:0x1a00000000059b", + "Ethernet33:5": "oid:0x1a00000000059c", + "Ethernet33:6": "oid:0x1a00000000059d", + "Ethernet33:7": "oid:0x1a00000000059e", + "Ethernet34:0": "oid:0x1a0000000005c1", + "Ethernet34:1": "oid:0x1a0000000005c2", + "Ethernet34:2": "oid:0x1a0000000005c3", + "Ethernet34:3": "oid:0x1a0000000005c4", + "Ethernet34:4": "oid:0x1a0000000005c5", + "Ethernet34:5": "oid:0x1a0000000005c6", + "Ethernet34:6": "oid:0x1a0000000005c7", + "Ethernet34:7": "oid:0x1a0000000005c8", + "Ethernet35:0": "oid:0x1a0000000005eb", + "Ethernet35:1": "oid:0x1a0000000005ec", + "Ethernet35:2": "oid:0x1a0000000005ed", + "Ethernet35:3": "oid:0x1a0000000005ee", + "Ethernet35:4": "oid:0x1a0000000005ef", + "Ethernet35:5": "oid:0x1a0000000005f0", + "Ethernet35:6": "oid:0x1a0000000005f1", + "Ethernet35:7": "oid:0x1a0000000005f2", + "Ethernet36:0": "oid:0x1a000000000615", + "Ethernet36:1": "oid:0x1a000000000616", + "Ethernet36:2": "oid:0x1a000000000617", + "Ethernet36:3": "oid:0x1a000000000618", + "Ethernet36:4": "oid:0x1a000000000619", + "Ethernet36:5": "oid:0x1a00000000061a", + "Ethernet36:6": "oid:0x1a00000000061b", + "Ethernet36:7": "oid:0x1a00000000061c", + "Ethernet37:0": "oid:0x1a00000000063f", + "Ethernet37:1": "oid:0x1a000000000640", + "Ethernet37:2": "oid:0x1a000000000641", + "Ethernet37:3": "oid:0x1a000000000642", + "Ethernet37:4": "oid:0x1a000000000643", + "Ethernet37:5": "oid:0x1a000000000644", + "Ethernet37:6": "oid:0x1a000000000645", + "Ethernet37:7": "oid:0x1a000000000646", + "Ethernet38:0": "oid:0x1a000000000669", + "Ethernet38:1": "oid:0x1a00000000066a", + "Ethernet38:2": "oid:0x1a00000000066b", + "Ethernet38:3": "oid:0x1a00000000066c", + "Ethernet38:4": "oid:0x1a00000000066d", + "Ethernet38:5": "oid:0x1a00000000066e", + "Ethernet38:6": "oid:0x1a00000000066f", + "Ethernet38:7": "oid:0x1a000000000670", + "Ethernet39:0": "oid:0x1a000000000693", + "Ethernet39:1": "oid:0x1a000000000694", + "Ethernet39:2": "oid:0x1a000000000695", + "Ethernet39:3": "oid:0x1a000000000696", + "Ethernet39:4": "oid:0x1a000000000697", + "Ethernet39:5": "oid:0x1a000000000698", + "Ethernet39:6": "oid:0x1a000000000699", + "Ethernet39:7": "oid:0x1a00000000069a", + "Ethernet4:0": "oid:0x1a0000000000d5", + "Ethernet4:1": "oid:0x1a0000000000d6", + "Ethernet4:2": "oid:0x1a0000000000d7", + "Ethernet4:3": "oid:0x1a0000000000d8", + "Ethernet4:4": "oid:0x1a0000000000d9", + "Ethernet4:5": "oid:0x1a0000000000da", + "Ethernet4:6": "oid:0x1a0000000000db", + "Ethernet4:7": "oid:0x1a0000000000dc", + "Ethernet40:0": "oid:0x1a0000000006bd", + "Ethernet40:1": "oid:0x1a0000000006be", + "Ethernet40:2": "oid:0x1a0000000006bf", + "Ethernet40:3": "oid:0x1a0000000006c0", + "Ethernet40:4": "oid:0x1a0000000006c1", + "Ethernet40:5": "oid:0x1a0000000006c2", + "Ethernet40:6": "oid:0x1a0000000006c3", + "Ethernet40:7": "oid:0x1a0000000006c4", + "Ethernet41:0": "oid:0x1a0000000006e7", + "Ethernet41:1": "oid:0x1a0000000006e8", + "Ethernet41:2": "oid:0x1a0000000006e9", + "Ethernet41:3": "oid:0x1a0000000006ea", + "Ethernet41:4": "oid:0x1a0000000006eb", + "Ethernet41:5": "oid:0x1a0000000006ec", + "Ethernet41:6": "oid:0x1a0000000006ed", + "Ethernet41:7": "oid:0x1a0000000006ee", + "Ethernet42:0": "oid:0x1a000000000711", + "Ethernet42:1": "oid:0x1a000000000712", + "Ethernet42:2": "oid:0x1a000000000713", + "Ethernet42:3": "oid:0x1a000000000714", + "Ethernet42:4": "oid:0x1a000000000715", + "Ethernet42:5": "oid:0x1a000000000716", + "Ethernet42:6": "oid:0x1a000000000717", + "Ethernet42:7": "oid:0x1a000000000718", + "Ethernet43:0": "oid:0x1a00000000073b", + "Ethernet43:1": "oid:0x1a00000000073c", + "Ethernet43:2": "oid:0x1a00000000073d", + "Ethernet43:3": "oid:0x1a00000000073e", + "Ethernet43:4": "oid:0x1a00000000073f", + "Ethernet43:5": "oid:0x1a000000000740", + "Ethernet43:6": "oid:0x1a000000000741", + "Ethernet43:7": "oid:0x1a000000000742", + "Ethernet44:0": "oid:0x1a000000000765", + "Ethernet44:1": "oid:0x1a000000000766", + "Ethernet44:2": "oid:0x1a000000000767", + "Ethernet44:3": "oid:0x1a000000000768", + "Ethernet44:4": "oid:0x1a000000000769", + "Ethernet44:5": "oid:0x1a00000000076a", + "Ethernet44:6": "oid:0x1a00000000076b", + "Ethernet44:7": "oid:0x1a00000000076c", + "Ethernet45:0": "oid:0x1a00000000078f", + "Ethernet45:1": "oid:0x1a000000000790", + "Ethernet45:2": "oid:0x1a000000000791", + "Ethernet45:3": "oid:0x1a000000000792", + "Ethernet45:4": "oid:0x1a000000000793", + "Ethernet45:5": "oid:0x1a000000000794", + "Ethernet45:6": "oid:0x1a000000000795", + "Ethernet45:7": "oid:0x1a000000000796", + "Ethernet46:0": "oid:0x1a0000000007b9", + "Ethernet46:1": "oid:0x1a0000000007ba", + "Ethernet46:2": "oid:0x1a0000000007bb", + "Ethernet46:3": "oid:0x1a0000000007bc", + "Ethernet46:4": "oid:0x1a0000000007bd", + "Ethernet46:5": "oid:0x1a0000000007be", + "Ethernet46:6": "oid:0x1a0000000007bf", + "Ethernet46:7": "oid:0x1a0000000007c0", + "Ethernet47:0": "oid:0x1a0000000007e3", + "Ethernet47:1": "oid:0x1a0000000007e4", + "Ethernet47:2": "oid:0x1a0000000007e5", + "Ethernet47:3": "oid:0x1a0000000007e6", + "Ethernet47:4": "oid:0x1a0000000007e7", + "Ethernet47:5": "oid:0x1a0000000007e8", + "Ethernet47:6": "oid:0x1a0000000007e9", + "Ethernet47:7": "oid:0x1a0000000007ea", + "Ethernet48:0": "oid:0x1a00000000080d", + "Ethernet48:1": "oid:0x1a00000000080e", + "Ethernet48:2": "oid:0x1a00000000080f", + "Ethernet48:3": "oid:0x1a000000000810", + "Ethernet48:4": "oid:0x1a000000000811", + "Ethernet48:5": "oid:0x1a000000000812", + "Ethernet48:6": "oid:0x1a000000000813", + "Ethernet48:7": "oid:0x1a000000000814", + "Ethernet5:0": "oid:0x1a0000000000ff", + "Ethernet5:1": "oid:0x1a000000000100", + "Ethernet5:2": "oid:0x1a000000000101", + "Ethernet5:3": "oid:0x1a000000000102", + "Ethernet5:4": "oid:0x1a000000000103", + "Ethernet5:5": "oid:0x1a000000000104", + "Ethernet5:6": "oid:0x1a000000000105", + "Ethernet5:7": "oid:0x1a000000000106", + "Ethernet52:0": "oid:0x1a000000000837", + "Ethernet52:1": "oid:0x1a000000000838", + "Ethernet52:2": "oid:0x1a000000000839", + "Ethernet52:3": "oid:0x1a00000000083a", + "Ethernet52:4": "oid:0x1a00000000083b", + "Ethernet52:5": "oid:0x1a00000000083c", + "Ethernet52:6": "oid:0x1a00000000083d", + "Ethernet52:7": "oid:0x1a00000000083e", + "Ethernet56:0": "oid:0x1a000000000861", + "Ethernet56:1": "oid:0x1a000000000862", + "Ethernet56:2": "oid:0x1a000000000863", + "Ethernet56:3": "oid:0x1a000000000864", + "Ethernet56:4": "oid:0x1a000000000865", + "Ethernet56:5": "oid:0x1a000000000866", + "Ethernet56:6": "oid:0x1a000000000867", + "Ethernet56:7": "oid:0x1a000000000868", + "Ethernet6:0": "oid:0x1a000000000129", + "Ethernet6:1": "oid:0x1a00000000012a", + "Ethernet6:2": "oid:0x1a00000000012b", + "Ethernet6:3": "oid:0x1a00000000012c", + "Ethernet6:4": "oid:0x1a00000000012d", + "Ethernet6:5": "oid:0x1a00000000012e", + "Ethernet6:6": "oid:0x1a00000000012f", + "Ethernet6:7": "oid:0x1a000000000130", + "Ethernet60:0": "oid:0x1a00000000088b", + "Ethernet60:1": "oid:0x1a00000000088c", + "Ethernet60:2": "oid:0x1a00000000088d", + "Ethernet60:3": "oid:0x1a00000000088e", + "Ethernet60:4": "oid:0x1a00000000088f", + "Ethernet60:5": "oid:0x1a000000000890", + "Ethernet60:6": "oid:0x1a000000000891", + "Ethernet60:7": "oid:0x1a000000000892", + "Ethernet64:0": "oid:0x1a0000000008b5", + "Ethernet64:1": "oid:0x1a0000000008b6", + "Ethernet64:2": "oid:0x1a0000000008b7", + "Ethernet64:3": "oid:0x1a0000000008b8", + "Ethernet64:4": "oid:0x1a0000000008b9", + "Ethernet64:5": "oid:0x1a0000000008ba", + "Ethernet64:6": "oid:0x1a0000000008bb", + "Ethernet64:7": "oid:0x1a0000000008bc", + "Ethernet68:0": "oid:0x1a0000000008df", + "Ethernet68:1": "oid:0x1a0000000008e0", + "Ethernet68:2": "oid:0x1a0000000008e1", + "Ethernet68:3": "oid:0x1a0000000008e2", + "Ethernet68:4": "oid:0x1a0000000008e3", + "Ethernet68:5": "oid:0x1a0000000008e4", + "Ethernet68:6": "oid:0x1a0000000008e5", + "Ethernet68:7": "oid:0x1a0000000008e6", + "Ethernet7:0": "oid:0x1a000000000153", + "Ethernet7:1": "oid:0x1a000000000154", + "Ethernet7:2": "oid:0x1a000000000155", + "Ethernet7:3": "oid:0x1a000000000156", + "Ethernet7:4": "oid:0x1a000000000157", + "Ethernet7:5": "oid:0x1a000000000158", + "Ethernet7:6": "oid:0x1a000000000159", + "Ethernet7:7": "oid:0x1a00000000015a", + "Ethernet8:0": "oid:0x1a00000000017d", + "Ethernet8:1": "oid:0x1a00000000017e", + "Ethernet8:2": "oid:0x1a00000000017f", + "Ethernet8:3": "oid:0x1a000000000180", + "Ethernet8:4": "oid:0x1a000000000181", + "Ethernet8:5": "oid:0x1a000000000182", + "Ethernet8:6": "oid:0x1a000000000183", + "Ethernet8:7": "oid:0x1a000000000184", + "Ethernet9:0": "oid:0x1a0000000001a7", + "Ethernet9:1": "oid:0x1a0000000001a8", + "Ethernet9:2": "oid:0x1a0000000001a9", + "Ethernet9:3": "oid:0x1a0000000001aa", + "Ethernet9:4": "oid:0x1a0000000001ab", + "Ethernet9:5": "oid:0x1a0000000001ac", + "Ethernet9:6": "oid:0x1a0000000001ad", + "Ethernet9:7": "oid:0x1a0000000001ae" +} diff --git a/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups.txt b/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups.txt new file mode 100644 index 000000000..ba2bec5c6 --- /dev/null +++ b/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups.txt @@ -0,0 +1,13 @@ +{ + "Ethernet16:0": {}, + "Ethernet16:1": {}, + "Ethernet16:2": {}, + "Ethernet16:3": {}, + "Ethernet16:4": {}, + "Ethernet16:5": { + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES": "0", + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES": "0" + }, + "Ethernet16:6": {}, + "Ethernet16:7": {} +} diff --git a/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups_alias.txt b/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups_alias.txt new file mode 100644 index 000000000..2b9ede019 --- /dev/null +++ b/testdata/PERIODIC_WATERMARKS:Ethernet16:PriorityGroups_alias.txt @@ -0,0 +1,13 @@ +{ + "Ethernet16/1:0": {}, + "Ethernet16/1:1": {}, + "Ethernet16/1:2": {}, + "Ethernet16/1:3": {}, + "Ethernet16/1:4": {}, + "Ethernet16/1:5": { + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES": "0", + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES": "0" + }, + "Ethernet16/1:6": {}, + "Ethernet16/1:7": {} +} diff --git a/testdata/PERIODIC_WATERMARKS:Ethernet_wildcard:PriorityGroups_alias.txt b/testdata/PERIODIC_WATERMARKS:Ethernet_wildcard:PriorityGroups_alias.txt new file mode 100644 index 000000000..d0a39e228 --- /dev/null +++ b/testdata/PERIODIC_WATERMARKS:Ethernet_wildcard:PriorityGroups_alias.txt @@ -0,0 +1,437 @@ +{ + "Ethernet0/1:0": {}, + "Ethernet0/1:1": {}, + "Ethernet0/1:2": {}, + "Ethernet0/1:3": {}, + "Ethernet0/1:4": {}, + "Ethernet0/1:5": {}, + "Ethernet0/1:6": {}, + "Ethernet0/1:7": {}, + "Ethernet1/1:0": {}, + "Ethernet1/1:1": {}, + "Ethernet1/1:2": {}, + "Ethernet1/1:3": {}, + "Ethernet1/1:4": {}, + "Ethernet1/1:5": {}, + "Ethernet1/1:6": {}, + "Ethernet1/1:7": {}, + "Ethernet10/1:0": {}, + "Ethernet10/1:1": {}, + "Ethernet10/1:2": {}, + "Ethernet10/1:3": {}, + "Ethernet10/1:4": {}, + "Ethernet10/1:5": {}, + "Ethernet10/1:6": {}, + "Ethernet10/1:7": {}, + "Ethernet11/1:0": {}, + "Ethernet11/1:1": {}, + "Ethernet11/1:2": {}, + "Ethernet11/1:3": {}, + "Ethernet11/1:4": {}, + "Ethernet11/1:5": {}, + "Ethernet11/1:6": {}, + "Ethernet11/1:7": {}, + "Ethernet12/1:0": {}, + "Ethernet12/1:1": {}, + "Ethernet12/1:2": {}, + "Ethernet12/1:3": {}, + "Ethernet12/1:4": {}, + "Ethernet12/1:5": {}, + "Ethernet12/1:6": {}, + "Ethernet12/1:7": {}, + "Ethernet13/1:0": {}, + "Ethernet13/1:1": {}, + "Ethernet13/1:2": {}, + "Ethernet13/1:3": {}, + "Ethernet13/1:4": {}, + "Ethernet13/1:5": {}, + "Ethernet13/1:6": {}, + "Ethernet13/1:7": {}, + "Ethernet14/1:0": {}, + "Ethernet14/1:1": {}, + "Ethernet14/1:2": {}, + "Ethernet14/1:3": {}, + "Ethernet14/1:4": {}, + "Ethernet14/1:5": {}, + "Ethernet14/1:6": {}, + "Ethernet14/1:7": {}, + "Ethernet15/1:0": {}, + "Ethernet15/1:1": {}, + "Ethernet15/1:2": {}, + "Ethernet15/1:3": {}, + "Ethernet15/1:4": {}, + "Ethernet15/1:5": {}, + "Ethernet15/1:6": {}, + "Ethernet15/1:7": {}, + "Ethernet16/1:0": {}, + "Ethernet16/1:1": {}, + "Ethernet16/1:2": {}, + "Ethernet16/1:3": {}, + "Ethernet16/1:4": {}, + "Ethernet16/1:5": { + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES": "0", + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES": "0" + }, + "Ethernet16/1:6": {}, + "Ethernet16/1:7": {}, + "Ethernet17/1:0": {}, + "Ethernet17/1:1": {}, + "Ethernet17/1:2": {}, + "Ethernet17/1:3": {}, + "Ethernet17/1:4": {}, + "Ethernet17/1:5": {}, + "Ethernet17/1:6": {}, + "Ethernet17/1:7": {}, + "Ethernet18/1:0": {}, + "Ethernet18/1:1": {}, + "Ethernet18/1:2": {}, + "Ethernet18/1:3": {}, + "Ethernet18/1:4": {}, + "Ethernet18/1:5": {}, + "Ethernet18/1:6": {}, + "Ethernet18/1:7": {}, + "Ethernet19/1:0": {}, + "Ethernet19/1:1": {}, + "Ethernet19/1:2": {}, + "Ethernet19/1:3": {}, + "Ethernet19/1:4": {}, + "Ethernet19/1:5": {}, + "Ethernet19/1:6": {}, + "Ethernet19/1:7": {}, + "Ethernet2/1:0": {}, + "Ethernet2/1:1": {}, + "Ethernet2/1:2": {}, + "Ethernet2/1:3": {}, + "Ethernet2/1:4": {}, + "Ethernet2/1:5": {}, + "Ethernet2/1:6": {}, + "Ethernet2/1:7": {}, + "Ethernet20/1:0": {}, + "Ethernet20/1:1": {}, + "Ethernet20/1:2": {}, + "Ethernet20/1:3": {}, + "Ethernet20/1:4": {}, + "Ethernet20/1:5": {}, + "Ethernet20/1:6": {}, + "Ethernet20/1:7": {}, + "Ethernet21/1:0": {}, + "Ethernet21/1:1": {}, + "Ethernet21/1:2": {}, + "Ethernet21/1:3": {}, + "Ethernet21/1:4": {}, + "Ethernet21/1:5": {}, + "Ethernet21/1:6": {}, + "Ethernet21/1:7": {}, + "Ethernet22/1:0": {}, + "Ethernet22/1:1": {}, + "Ethernet22/1:2": {}, + "Ethernet22/1:3": {}, + "Ethernet22/1:4": {}, + "Ethernet22/1:5": {}, + "Ethernet22/1:6": {}, + "Ethernet22/1:7": {}, + "Ethernet23/1:0": {}, + "Ethernet23/1:1": {}, + "Ethernet23/1:2": {}, + "Ethernet23/1:3": {}, + "Ethernet23/1:4": {}, + "Ethernet23/1:5": {}, + "Ethernet23/1:6": {}, + "Ethernet23/1:7": {}, + "Ethernet24/1:0": {}, + "Ethernet24/1:1": {}, + "Ethernet24/1:2": {}, + "Ethernet24/1:3": {}, + "Ethernet24/1:4": {}, + "Ethernet24/1:5": {}, + "Ethernet24/1:6": {}, + "Ethernet24/1:7": {}, + "Ethernet25/1:0": {}, + "Ethernet25/1:1": {}, + "Ethernet25/1:2": {}, + "Ethernet25/1:3": {}, + "Ethernet25/1:4": {}, + "Ethernet25/1:5": {}, + "Ethernet25/1:6": {}, + "Ethernet25/1:7": {}, + "Ethernet26/1:0": {}, + "Ethernet26/1:1": {}, + "Ethernet26/1:2": {}, + "Ethernet26/1:3": {}, + "Ethernet26/1:4": {}, + "Ethernet26/1:5": {}, + "Ethernet26/1:6": {}, + "Ethernet26/1:7": {}, + "Ethernet27/1:0": {}, + "Ethernet27/1:1": {}, + "Ethernet27/1:2": {}, + "Ethernet27/1:3": {}, + "Ethernet27/1:4": {}, + "Ethernet27/1:5": {}, + "Ethernet27/1:6": {}, + "Ethernet27/1:7": {}, + "Ethernet28/1:0": {}, + "Ethernet28/1:1": {}, + "Ethernet28/1:2": {}, + "Ethernet28/1:3": {}, + "Ethernet28/1:4": {}, + "Ethernet28/1:5": {}, + "Ethernet28/1:6": {}, + "Ethernet28/1:7": {}, + "Ethernet29/1:0": {}, + "Ethernet29/1:1": {}, + "Ethernet29/1:2": {}, + "Ethernet29/1:3": {}, + "Ethernet29/1:4": {}, + "Ethernet29/1:5": {}, + "Ethernet29/1:6": {}, + "Ethernet29/1:7": {}, + "Ethernet3/1:0": {}, + "Ethernet3/1:1": {}, + "Ethernet3/1:2": {}, + "Ethernet3/1:3": {}, + "Ethernet3/1:4": {}, + "Ethernet3/1:5": {}, + "Ethernet3/1:6": {}, + "Ethernet3/1:7": {}, + "Ethernet30/1:0": {}, + "Ethernet30/1:1": {}, + "Ethernet30/1:2": {}, + "Ethernet30/1:3": {}, + "Ethernet30/1:4": {}, + "Ethernet30/1:5": {}, + "Ethernet30/1:6": {}, + "Ethernet30/1:7": {}, + "Ethernet31/1:0": {}, + "Ethernet31/1:1": {}, + "Ethernet31/1:2": {}, + "Ethernet31/1:3": {}, + "Ethernet31/1:4": {}, + "Ethernet31/1:5": {}, + "Ethernet31/1:6": {}, + "Ethernet31/1:7": {}, + "Ethernet32/1:0": {}, + "Ethernet32/1:1": {}, + "Ethernet32/1:2": {}, + "Ethernet32/1:3": {}, + "Ethernet32/1:4": {}, + "Ethernet32/1:5": {}, + "Ethernet32/1:6": {}, + "Ethernet32/1:7": {}, + "Ethernet33/1:0": {}, + "Ethernet33/1:1": {}, + "Ethernet33/1:2": {}, + "Ethernet33/1:3": {}, + "Ethernet33/1:4": {}, + "Ethernet33/1:5": {}, + "Ethernet33/1:6": {}, + "Ethernet33/1:7": {}, + "Ethernet34/1:0": {}, + "Ethernet34/1:1": {}, + "Ethernet34/1:2": {}, + "Ethernet34/1:3": {}, + "Ethernet34/1:4": {}, + "Ethernet34/1:5": {}, + "Ethernet34/1:6": {}, + "Ethernet34/1:7": {}, + "Ethernet35/1:0": {}, + "Ethernet35/1:1": {}, + "Ethernet35/1:2": {}, + "Ethernet35/1:3": {}, + "Ethernet35/1:4": {}, + "Ethernet35/1:5": {}, + "Ethernet35/1:6": {}, + "Ethernet35/1:7": {}, + "Ethernet36/1:0": {}, + "Ethernet36/1:1": {}, + "Ethernet36/1:2": {}, + "Ethernet36/1:3": {}, + "Ethernet36/1:4": {}, + "Ethernet36/1:5": {}, + "Ethernet36/1:6": {}, + "Ethernet36/1:7": {}, + "Ethernet37/1:0": {}, + "Ethernet37/1:1": {}, + "Ethernet37/1:2": {}, + "Ethernet37/1:3": {}, + "Ethernet37/1:4": {}, + "Ethernet37/1:5": {}, + "Ethernet37/1:6": {}, + "Ethernet37/1:7": {}, + "Ethernet38/1:0": {}, + "Ethernet38/1:1": {}, + "Ethernet38/1:2": {}, + "Ethernet38/1:3": {}, + "Ethernet38/1:4": {}, + "Ethernet38/1:5": {}, + "Ethernet38/1:6": {}, + "Ethernet38/1:7": {}, + "Ethernet39/1:0": {}, + "Ethernet39/1:1": {}, + "Ethernet39/1:2": {}, + "Ethernet39/1:3": {}, + "Ethernet39/1:4": {}, + "Ethernet39/1:5": {}, + "Ethernet39/1:6": {}, + "Ethernet39/1:7": {}, + "Ethernet4/1:0": {}, + "Ethernet4/1:1": {}, + "Ethernet4/1:2": {}, + "Ethernet4/1:3": {}, + "Ethernet4/1:4": {}, + "Ethernet4/1:5": {}, + "Ethernet4/1:6": {}, + "Ethernet4/1:7": {}, + "Ethernet40/1:0": {}, + "Ethernet40/1:1": {}, + "Ethernet40/1:2": {}, + "Ethernet40/1:3": {}, + "Ethernet40/1:4": {}, + "Ethernet40/1:5": {}, + "Ethernet40/1:6": {}, + "Ethernet40/1:7": {}, + "Ethernet41/1:0": {}, + "Ethernet41/1:1": {}, + "Ethernet41/1:2": {}, + "Ethernet41/1:3": {}, + "Ethernet41/1:4": {}, + "Ethernet41/1:5": {}, + "Ethernet41/1:6": {}, + "Ethernet41/1:7": {}, + "Ethernet42/1:0": {}, + "Ethernet42/1:1": {}, + "Ethernet42/1:2": {}, + "Ethernet42/1:3": {}, + "Ethernet42/1:4": {}, + "Ethernet42/1:5": {}, + "Ethernet42/1:6": {}, + "Ethernet42/1:7": {}, + "Ethernet43/1:0": {}, + "Ethernet43/1:1": {}, + "Ethernet43/1:2": {}, + "Ethernet43/1:3": {}, + "Ethernet43/1:4": {}, + "Ethernet43/1:5": {}, + "Ethernet43/1:6": {}, + "Ethernet43/1:7": {}, + "Ethernet44/1:0": {}, + "Ethernet44/1:1": {}, + "Ethernet44/1:2": {}, + "Ethernet44/1:3": {}, + "Ethernet44/1:4": {}, + "Ethernet44/1:5": {}, + "Ethernet44/1:6": {}, + "Ethernet44/1:7": {}, + "Ethernet45/1:0": {}, + "Ethernet45/1:1": {}, + "Ethernet45/1:2": {}, + "Ethernet45/1:3": {}, + "Ethernet45/1:4": {}, + "Ethernet45/1:5": {}, + "Ethernet45/1:6": {}, + "Ethernet45/1:7": {}, + "Ethernet46/1:0": {}, + "Ethernet46/1:1": {}, + "Ethernet46/1:2": {}, + "Ethernet46/1:3": {}, + "Ethernet46/1:4": {}, + "Ethernet46/1:5": {}, + "Ethernet46/1:6": {}, + "Ethernet46/1:7": {}, + "Ethernet47/1:0": {}, + "Ethernet47/1:1": {}, + "Ethernet47/1:2": {}, + "Ethernet47/1:3": {}, + "Ethernet47/1:4": {}, + "Ethernet47/1:5": {}, + "Ethernet47/1:6": {}, + "Ethernet47/1:7": {}, + "Ethernet48/1:0": {}, + "Ethernet48/1:1": {}, + "Ethernet48/1:2": {}, + "Ethernet48/1:3": {}, + "Ethernet48/1:4": {}, + "Ethernet48/1:5": {}, + "Ethernet48/1:6": {}, + "Ethernet48/1:7": {}, + "Ethernet5/1:0": {}, + "Ethernet5/1:1": {}, + "Ethernet5/1:2": {}, + "Ethernet5/1:3": {}, + "Ethernet5/1:4": {}, + "Ethernet5/1:5": {}, + "Ethernet5/1:6": {}, + "Ethernet5/1:7": {}, + "Ethernet52/1:0": {}, + "Ethernet52/1:1": {}, + "Ethernet52/1:2": {}, + "Ethernet52/1:3": {}, + "Ethernet52/1:4": {}, + "Ethernet52/1:5": {}, + "Ethernet52/1:6": {}, + "Ethernet52/1:7": {}, + "Ethernet56/1:0": {}, + "Ethernet56/1:1": {}, + "Ethernet56/1:2": {}, + "Ethernet56/1:3": {}, + "Ethernet56/1:4": {}, + "Ethernet56/1:5": {}, + "Ethernet56/1:6": {}, + "Ethernet56/1:7": {}, + "Ethernet6/1:0": {}, + "Ethernet6/1:1": {}, + "Ethernet6/1:2": {}, + "Ethernet6/1:3": {}, + "Ethernet6/1:4": {}, + "Ethernet6/1:5": {}, + "Ethernet6/1:6": {}, + "Ethernet6/1:7": {}, + "Ethernet60/1:0": {}, + "Ethernet60/1:1": {}, + "Ethernet60/1:2": {}, + "Ethernet60/1:3": {}, + "Ethernet60/1:4": {}, + "Ethernet60/1:5": {}, + "Ethernet60/1:6": {}, + "Ethernet60/1:7": {}, + "Ethernet64/1:0": {}, + "Ethernet64/1:1": {}, + "Ethernet64/1:2": {}, + "Ethernet64/1:3": {}, + "Ethernet64/1:4": {}, + "Ethernet64/1:5": {}, + "Ethernet64/1:6": {}, + "Ethernet64/1:7": {}, + "Ethernet68/1:0": {}, + "Ethernet68/1:1": {}, + "Ethernet68/1:2": {}, + "Ethernet68/1:3": {}, + "Ethernet68/1:4": {}, + "Ethernet68/1:5": {}, + "Ethernet68/1:6": {}, + "Ethernet68/1:7": {}, + "Ethernet7/1:0": {}, + "Ethernet7/1:1": {}, + "Ethernet7/1:2": {}, + "Ethernet7/1:3": {}, + "Ethernet7/1:4": {}, + "Ethernet7/1:5": {}, + "Ethernet7/1:6": {}, + "Ethernet7/1:7": {}, + "Ethernet8/1:0": {}, + "Ethernet8/1:1": {}, + "Ethernet8/1:2": {}, + "Ethernet8/1:3": {}, + "Ethernet8/1:4": {}, + "Ethernet8/1:5": {}, + "Ethernet8/1:6": {}, + "Ethernet8/1:7": {}, + "Ethernet9/1:0": {}, + "Ethernet9/1:1": {}, + "Ethernet9/1:2": {}, + "Ethernet9/1:3": {}, + "Ethernet9/1:4": {}, + "Ethernet9/1:5": {}, + "Ethernet9/1:6": {}, + "Ethernet9/1:7": {} +} \ No newline at end of file diff --git a/testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt b/testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt new file mode 100644 index 000000000..88a7704a4 --- /dev/null +++ b/testdata/PERIODIC_WATERMARKS:oid:0x1a0000000002d2.txt @@ -0,0 +1,4 @@ +{ + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES": "0", + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES": "0" +} From 04bd9cd1362a7f56abc6acbe371fb72434e0b5f4 Mon Sep 17 00:00:00 2001 From: mssonicbld <79238446+mssonicbld@users.noreply.github.com> Date: Sat, 28 Jun 2025 04:12:40 +0800 Subject: [PATCH 09/14] Fix CodeQL failure due to missing sonic-mgmt-common. (#432) #### Why I did it CodeQL has been failing at head due to missing ../sonic-mgmt-common #### How I did it Add checking out sonic-mgmt-common. #### How to verify it Run at my forked master: https://github.com/hdwhdw/sonic-gnmi/actions/runs/12385901251 The run on this PR pipeline uses the workflow file on its master, hence failing is expected. #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 #### Description for the changelog #### Link to config_db schema for YANG module changes #### A picture of a cute animal (not mandatory but encouraged) --- .github/workflows/codeql-analysis.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2997ffeaf..470a2e51d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,6 +13,10 @@ on: - '202[0-9][0-9][0-9]' workflow_dispatch: +env: + BUILD_BRANCH: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref_name }} + + jobs: analyze: name: Analyze @@ -31,6 +35,20 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + # Checkout sonic-mgmt-common repository which is used by sonic-gnmi + - name: Checkout sonic-mgmt-common repository + uses: actions/checkout@v3 + with: + repository: sonic-net/sonic-mgmt-common + path: sonic-mgmt-common + ref: refs/heads/${{ env.BUILD_BRANCH }} + + # Update go.mod to use local sonic-mgmt-common. + # This is the same hack used in the CI pipeline. See lgtm.yml. + # We should find a better way to do this. + - name: Update go.mod for sonic-mgmt-common + run: sed -i 's@replace github.com/Azure/sonic-mgmt-common => ../sonic-mgmt-common@replace github.com/Azure/sonic-mgmt-common => ./sonic-mgmt-common@g' go.mod + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2.1.29 From c3a4726ca7eff52c81f85c34762013ad49e87bcf Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Fri, 18 Apr 2025 13:43:18 +0800 Subject: [PATCH 10/14] Add cert multiple roles support (#366) * Update clientCertAuth.go * Update server_test.go * Add strings package import in clientCertAuth.go * Fix role key lookup in clientCertAuth * Add backward compatibility for single role schema * Add tests for client certificate authentication --- gnmi_server/clientCertAuth.go | 7 ++- gnmi_server/server_test.go | 94 +++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/gnmi_server/clientCertAuth.go b/gnmi_server/clientCertAuth.go index 48fecf3ca..4e770f226 100644 --- a/gnmi_server/clientCertAuth.go +++ b/gnmi_server/clientCertAuth.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "io" "net/http" + "strings" "time" "github.com/sonic-net/sonic-gnmi/common_utils" "github.com/sonic-net/sonic-gnmi/swsscommon" @@ -260,7 +261,11 @@ func PopulateAuthStructByCommonName(certCommonName string, auth *common_utils.Au var fieldValuePairs = configDbConnector.Get_entry(serviceConfigTableName, certCommonName) if fieldValuePairs.Size() > 0 { - if fieldValuePairs.Has_key("role") { + if fieldValuePairs.Has_key("role@") { + var role = fieldValuePairs.Get("role@") + auth.Roles = strings.Split(role, ",") + } else if fieldValuePairs.Has_key("role") { + // Backward compatibility for single role DB schema var role = fieldValuePairs.Get("role") auth.Roles = []string{role} } diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 1cfbb31df..69f4414e8 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -5036,6 +5036,100 @@ func TestClientCertAuthenAndAuthor(t *testing.T) { swsscommon.DeleteDBConnector(configDb) } +func TestClientCertAuthenAndAuthorMultiRole(t *testing.T) { + if !swsscommon.SonicDBConfigIsInit() { + swsscommon.SonicDBConfigInitialize() + } + + var configDb = swsscommon.NewDBConnector("CONFIG_DB", uint(0), true) + var gnmiTable = swsscommon.NewTable(configDb, "GNMI_CLIENT_CERT") + configDb.Flushdb() + + // initialize err variable + err := status.Error(codes.Unauthenticated, "") + + // when config table is empty, will authorize with PopulateAuthStruct + mockpopulate := gomonkey.ApplyFunc(PopulateAuthStruct, func(username string, auth *common_utils.AuthInfo, r []string) error { + return nil + }) + defer mockpopulate.Reset() + + // check auth with nil cert name + ctx, cancel := CreateAuthorizationCtx() + ctx, err = ClientCertAuthenAndAuthor(ctx, "", false) + if err != nil { + t.Errorf("CommonNameMatch with empty config table should success: %v", err) + } + + cancel() + + // check get 1 cert name + ctx, cancel = CreateAuthorizationCtx() + configDb.Flushdb() + gnmiTable.Hset("certname1", "role@", "readwrite") + ctx, err = ClientCertAuthenAndAuthor(ctx, "GNMI_CLIENT_CERT", false) + if err != nil { + t.Errorf("CommonNameMatch with correct cert name should success: %v", err) + } + + cancel() + + // check get multiple cert names + ctx, cancel = CreateAuthorizationCtx() + configDb.Flushdb() + gnmiTable.Hset("certname1", "role@", "readwrite") + gnmiTable.Hset("certname2", "role@", "readonly") + ctx, err = ClientCertAuthenAndAuthor(ctx, "GNMI_CLIENT_CERT", false) + if err != nil { + t.Errorf("CommonNameMatch with correct cert name should success: %v", err) + } + + cancel() + + // check a invalid cert cname + ctx, cancel = CreateAuthorizationCtx() + configDb.Flushdb() + gnmiTable.Hset("certname2", "role@", "readonly") + ctx, err = ClientCertAuthenAndAuthor(ctx, "GNMI_CLIENT_CERT", false) + if err == nil { + t.Errorf("CommonNameMatch with invalid cert name should fail: %v", err) + } + + cancel() + + swsscommon.DeleteTable(gnmiTable) + swsscommon.DeleteDBConnector(configDb) +} + +func TestAuthenticate(t *testing.T) { + if !swsscommon.SonicDBConfigIsInit() { + swsscommon.SonicDBConfigInitialize() + } + + var tableName = "GNMI_CLIENT_CERT" + var configDb = swsscommon.NewDBConnector("CONFIG_DB", uint(0), true) + var gnmiTable = swsscommon.NewTable(configDb, tableName) + defer swsscommon.DeleteTable(gnmiTable) + defer swsscommon.DeleteDBConnector(configDb) + configDb.Flushdb() + + // initialize err variable + err := status.Error(codes.Unauthenticated, "") + + // check a invalid role + cfg := &Config{ConfigTableName: tableName, UserAuth: AuthTypes{"password": false, "cert": true, "jwt": false}} + ctx, cancel := CreateAuthorizationCtx() + configDb.Flushdb() + gnmiTable.Hset("certname1", "role@", "readonly") + // Call authenticate to verify the user's role. This should fail if the role is "readonly". + _, err = authenticate(cfg, ctx, true) + if err == nil { + t.Errorf("authenticate with readonly role should fail: %v", err) + } + + cancel() +} + type MockServerStream struct { grpc.ServerStream } From 0370b80a6040c9d9a5eee205863118444e8cf959 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:38:42 +0800 Subject: [PATCH 11/14] Update server_test.go --- gnmi_server/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 69f4414e8..f8f01dcb4 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -5122,7 +5122,7 @@ func TestAuthenticate(t *testing.T) { configDb.Flushdb() gnmiTable.Hset("certname1", "role@", "readonly") // Call authenticate to verify the user's role. This should fail if the role is "readonly". - _, err = authenticate(cfg, ctx, true) + _, err = authenticate(cfg, ctx) if err == nil { t.Errorf("authenticate with readonly role should fail: %v", err) } From b81e9f0ba5662ebd4d9ce75c6aed3f053bf56c58 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:46:58 +0800 Subject: [PATCH 12/14] Update server_test.go --- gnmi_server/server_test.go | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index f8f01dcb4..7a16d30ca 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -5101,35 +5101,6 @@ func TestClientCertAuthenAndAuthorMultiRole(t *testing.T) { swsscommon.DeleteDBConnector(configDb) } -func TestAuthenticate(t *testing.T) { - if !swsscommon.SonicDBConfigIsInit() { - swsscommon.SonicDBConfigInitialize() - } - - var tableName = "GNMI_CLIENT_CERT" - var configDb = swsscommon.NewDBConnector("CONFIG_DB", uint(0), true) - var gnmiTable = swsscommon.NewTable(configDb, tableName) - defer swsscommon.DeleteTable(gnmiTable) - defer swsscommon.DeleteDBConnector(configDb) - configDb.Flushdb() - - // initialize err variable - err := status.Error(codes.Unauthenticated, "") - - // check a invalid role - cfg := &Config{ConfigTableName: tableName, UserAuth: AuthTypes{"password": false, "cert": true, "jwt": false}} - ctx, cancel := CreateAuthorizationCtx() - configDb.Flushdb() - gnmiTable.Hset("certname1", "role@", "readonly") - // Call authenticate to verify the user's role. This should fail if the role is "readonly". - _, err = authenticate(cfg, ctx) - if err == nil { - t.Errorf("authenticate with readonly role should fail: %v", err) - } - - cancel() -} - type MockServerStream struct { grpc.ServerStream } From d04415890d8b300286824bc6be22ffdd0a607af0 Mon Sep 17 00:00:00 2001 From: Zain Budhwani <99770260+zbud-msft@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:30:50 -0800 Subject: [PATCH 13/14] Merge pull request #417 from FengPan-Frank/temptest (#535) Manual cherrypick for adding support for RATES tables in Counters DB so that PRE_FEC/POST_FEC_BER via ST --- gnmi_server/server_test.go | 209 ++++++++++++++++++ sonic_data_client/virtual_db.go | 8 +- testdata/COUNTERS:Ethernet_wildcard_alias.txt | 3 +- testdata/COUNTERS_PORT_ALIAS_MAP.txt | 3 + testdata/COUNTERS_PORT_NAME_MAP.txt | 3 +- testdata/RATES:Ethernet_wildcard_alias.txt | 68 ++++++ testdata/RATES:oid:Ethernet89.txt | 12 + testdata/db_dump.json | 14 +- 8 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 testdata/RATES:Ethernet_wildcard_alias.txt create mode 100644 testdata/RATES:oid:Ethernet89.txt diff --git a/gnmi_server/server_test.go b/gnmi_server/server_test.go index 7a16d30ca..ba55194e5 100644 --- a/gnmi_server/server_test.go +++ b/gnmi_server/server_test.go @@ -711,6 +711,16 @@ func initFullCountersDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091f", countersEeth68_4Byte) loadDB(t, rclient, mpi_counter) + // "oid:0x1000000000056" : port counter, for RATES/PORT vpath test + fileName = "../testdata/RATES:oid:Ethernet89.txt" + ratesPort0_Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + // "Ethernet89": "oid:0x1000000000003", + mpi_counter = loadConfig(t, "RATES:oid:0x1000000000056", ratesPort0_Byte) + loadDB(t, rclient, mpi_counter) + fileName = "../testdata/COUNTERS_FABRIC_PORT_NAME_MAP.txt" countersFabricPortNameMapByte, err := ioutil.ReadFile(fileName) if err != nil { @@ -900,6 +910,15 @@ func prepareDb(t *testing.T, namespace string) { mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091f", countersEeth68_4Byte) loadDB(t, rclient, mpi_counter) + fileName = "../testdata/RATES:oid:Ethernet89.txt" + ratesPort0, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + // "Ethernet89": "oid:0x1000000000056", + mpi_counter = loadConfig(t, "RATES:oid:0x1000000000056", ratesPort0) + loadDB(t, rclient, mpi_counter) + // "PORT0": "oid:0x1000000000081" : Fabric port counters, for COUNTERS/PORT0 vpath test fileName = "../testdata/COUNTERS:oid:0x1000000000081.txt" fileName = "../testdata/COUNTERS:oid:0x1000000000081.txt" @@ -1055,6 +1074,18 @@ func createCountersDbQueryOnChangeMode(t *testing.T, paths ...string) client.Que false) } +// createRatesTableSetUpdate creates a HSET request on the RATES table. +func createRatesTableSetUpdate(tableKey string, fieldName string, fieldValue string) tablePathValue { + return tablePathValue{ + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: tableKey, + delimitor: ":", + field: fieldName, + value: fieldValue, + } +} + // createCountersDbQuerySampleMode creates a query with SAMPLE mode. func createCountersDbQuerySampleMode(t *testing.T, interval time.Duration, updateOnly bool, paths ...string) client.Query { return createQueryOrFail(t, @@ -2027,6 +2058,22 @@ func runTestSubscribe(t *testing.T, namespace string) { countersEthernet68JsonUpdate := tmp2.(map[string]interface{}) countersEthernet68JsonUpdate["test_field"] = "test_value" + // for table key subscription + fileName = "../testdata/RATES:oid:Ethernet89.txt" + ratesEthernet89Byte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + + var tmp89 interface{} + json.Unmarshal(ratesEthernet89Byte, &tmp89) + ratesEthernet89JsonUpdate := tmp89.(map[string]interface{}) + ratesEthernet89JsonUpdate["FEC_PRE_BER"] = "0" + var tmp891 interface{} + json.Unmarshal(ratesEthernet89Byte, &tmp891) + ratesEthernet89JsonPfcUpdate := tmp891.(map[string]interface{}) + ratesEthernet89JsonPfcUpdate["FEC_PRE_BER"] = "4" + var tmp3 interface{} json.Unmarshal(countersEthernet68Byte, &tmp3) countersEthernet68JsonPfcUpdate := tmp3.(map[string]interface{}) @@ -2079,6 +2126,21 @@ func runTestSubscribe(t *testing.T, namespace string) { // Will have "test_field" : "test_value" in Ethernet68, countersEtherneWildcardJsonUpdate := map[string]interface{}{"Ethernet68/1": countersEthernet68JsonUpdate} + fileName = "../testdata/RATES:Ethernet_wildcard_alias.txt" + ratesEthernetWildcardByte, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatalf("read file %v err: %v", fileName, err) + } + var ratesEthernetWildcardJson interface{} + json.Unmarshal(ratesEthernetWildcardByte, &ratesEthernetWildcardJson) + var ratesFieldUpdate2 map[string]interface{} + json.Unmarshal(ratesEthernetWildcardByte, &ratesFieldUpdate2) + ratesFieldUpdate2["Ethernet89/1"] = ratesEthernet89JsonUpdate + + var ratesFieldUpdate map[string]interface{} + json.Unmarshal(ratesEthernetWildcardByte, &ratesFieldUpdate) + ratesFieldUpdate["Ethernet89/1"] = ratesEthernet89JsonPfcUpdate + // all counters on all ports with change on one field of one port var countersFieldUpdate map[string]interface{} json.Unmarshal(countersEthernetWildcardByte, &countersFieldUpdate) @@ -2340,6 +2402,153 @@ func runTestSubscribe(t *testing.T, namespace string) { client.Update{Path: []string{"COUNTERS", "Ethernet68/1", "SAI_PORT_STAT_PFC_7_RX_PKTS"}, TS: time.Unix(0, 200), Val: "3"}, }, }, + { + desc: "poll query for RATES/Ethernet89/FEC_POST_BER with field value change", + poll: 3, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"RATES", "Ethernet89", "FEC_POST_BER"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + { + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", + delimitor: ":", + field: "FEC_POST_BER", + value: "4", // being changed to 4 from 0 + }, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_POST_BER"}, TS: time.Unix(0, 200), Val: "0"}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_POST_BER"}, TS: time.Unix(0, 200), Val: "4"}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_POST_BER"}, TS: time.Unix(0, 200), Val: "4"}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_POST_BER"}, TS: time.Unix(0, 200), Val: "4"}, + client.Sync{}, + }, + }, + { + desc: "stream query for RATES/Ethernet89/FEC_PRE_BER with update of field value", + q: createCountersDbQueryOnChangeMode(t, "RATES", "Ethernet89", "FEC_PRE_BER"), + updates: []tablePathValue{ + { + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", // "Ethernet89": "oid:0x1000000000056", + delimitor: ":", + field: "FEC_PRE_BER", + value: "1", + }, + { + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", // "Ethernet89": "oid:0x1000000000056", + delimitor: ":", + field: "FEC_POST_BER", + value: "1", + }, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_PRE_BER"}, TS: time.Unix(0, 200), Val: "0"}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_PRE_BER"}, TS: time.Unix(0, 200), Val: "1"}, + }, + }, + { + desc: "sample stream query for RATES/Ethernet89/FEC_PRE_BER with 2 updates", + q: createCountersDbQuerySampleMode(t, 0, false, "RATES", "Ethernet89", "FEC_PRE_BER"), + generateIntervals: true, + updates: []tablePathValue{ + createRatesTableSetUpdate("oid:0x1000000000056", "FEC_PRE_BER", "3"), // be changed to 3 from 0 + createIntervalTickerUpdate(), // no value change but imitate interval ticker + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_PRE_BER"}, TS: time.Unix(0, 200), Val: "0"}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_PRE_BER"}, TS: time.Unix(0, 200), Val: "3"}, + client.Update{Path: []string{"RATES", "Ethernet89", "FEC_PRE_BER"}, TS: time.Unix(0, 200), Val: "3"}, + }, + }, + { + desc: "poll query for table key Ethernet* with Ethernet89/FEC_PRE_BER field value change", + poll: 3, + q: client.Query{ + Target: "COUNTERS_DB", + Type: client.Poll, + Queries: []client.Path{{"RATES", "Ethernet*"}}, + TLS: &tls.Config{InsecureSkipVerify: true}, + }, + updates: []tablePathValue{ + { + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", // "Ethernet89": "oid:0x1000000000056", + delimitor: ":", + field: "FEC_PRE_BER", + value: "4", // being changed to 4 from 2 + }, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesEthernetWildcardJson}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesFieldUpdate}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesFieldUpdate}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesFieldUpdate}, + client.Sync{}, + }, + }, + { + desc: "stream query for key Ethernet* with new test_field field on Ethernet89", + q: createCountersDbQueryOnChangeMode(t, "RATES", "Ethernet*"), + updates: []tablePathValue{ + { + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", // "Ethernet89": "oid:0x1000000000056", + delimitor: ":", + field: "FEC_PRE_BER", + value: "5", + }, + { //Same value set should not trigger multiple updates + dbName: "COUNTERS_DB", + tableName: "RATES", + tableKey: "oid:0x1000000000056", // "Ethernet68": "oid:0x1000000000039", + delimitor: ":", + field: "FEC_PRE_BER", + value: "5", + }, + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesEthernetWildcardJson}, + client.Sync{}, + }, + }, + { + desc: "sample stream query for table key Ethernet* with new test_field field on Ethernet89", + q: createCountersDbQuerySampleMode(t, 0, false, "RATES", "Ethernet*"), + generateIntervals: true, + updates: []tablePathValue{ + createRatesTableSetUpdate("oid:0x1000000000056", "FEC_PRE_BER", "0"), + }, + wantNoti: []client.Notification{ + client.Connected{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: ratesEthernetWildcardJson}, + client.Sync{}, + client.Update{Path: []string{"RATES", "Ethernet*"}, TS: time.Unix(0, 200), Val: mergeStrMaps(ratesEthernetWildcardJson, ratesFieldUpdate2)}, + }, + }, { desc: "stream query for COUNTERS/Ethernet68/Pfcwd with update of field value", q: createCountersDbQueryOnChangeMode(t, "COUNTERS", "Ethernet68", "Pfcwd"), diff --git a/sonic_data_client/virtual_db.go b/sonic_data_client/virtual_db.go index 589c6aa23..f480038a1 100644 --- a/sonic_data_client/virtual_db.go +++ b/sonic_data_client/virtual_db.go @@ -73,7 +73,13 @@ var ( }, { // Periodic PG watermarks for one or all Ethernet ports path: []string{"COUNTERS_DB", "PERIODIC_WATERMARKS", "Ethernet*", "PriorityGroups"}, transFunc: v2rTranslate(v2rEthPortPGPeriodicWMs), - }, + }, { // COUNTER_DB RATES Ethernet* + path: []string{"COUNTERS_DB", "RATES", "Ethernet*"}, + transFunc: v2rTranslate(v2rEthPortStats), + }, { // COUNTER_DB RATES Ethernet* FEC_PRE_BER + path: []string{"COUNTERS_DB", "RATES", "Ethernet*", "*"}, + transFunc: v2rTranslate(v2rEthPortFieldStats), + }, } ) diff --git a/testdata/COUNTERS:Ethernet_wildcard_alias.txt b/testdata/COUNTERS:Ethernet_wildcard_alias.txt index 227d8f8cd..e0b85fbb8 100644 --- a/testdata/COUNTERS:Ethernet_wildcard_alias.txt +++ b/testdata/COUNTERS:Ethernet_wildcard_alias.txt @@ -266,5 +266,6 @@ }, "Ethernet7/1": {}, "Ethernet8/1": {}, - "Ethernet9/1": {} + "Ethernet9/1": {}, + "Ethernet89/1": {} } \ No newline at end of file diff --git a/testdata/COUNTERS_PORT_ALIAS_MAP.txt b/testdata/COUNTERS_PORT_ALIAS_MAP.txt index 20180feee..c7de20687 100644 --- a/testdata/COUNTERS_PORT_ALIAS_MAP.txt +++ b/testdata/COUNTERS_PORT_ALIAS_MAP.txt @@ -160,5 +160,8 @@ }, "PORT|Ethernet9": { "alias": "Ethernet9/1" + }, + "PORT|Ethernet89": { + "alias": "Ethernet89/1" } } \ No newline at end of file diff --git a/testdata/COUNTERS_PORT_NAME_MAP.txt b/testdata/COUNTERS_PORT_NAME_MAP.txt index a4be83924..39f0be9c3 100644 --- a/testdata/COUNTERS_PORT_NAME_MAP.txt +++ b/testdata/COUNTERS_PORT_NAME_MAP.txt @@ -52,6 +52,7 @@ "Ethernet68": "oid:0x1000000000039", "Ethernet7": "oid:0x1000000000009", "Ethernet8": "oid:0x100000000000a", - "Ethernet9": "oid:0x100000000000b" + "Ethernet9": "oid:0x100000000000b", + "Ethernet89": "oid:0x1000000000056" } diff --git a/testdata/RATES:Ethernet_wildcard_alias.txt b/testdata/RATES:Ethernet_wildcard_alias.txt new file mode 100644 index 000000000..3a76d65a7 --- /dev/null +++ b/testdata/RATES:Ethernet_wildcard_alias.txt @@ -0,0 +1,68 @@ +{ + "Ethernet0/1": {}, + "Ethernet1/1": {}, + "Ethernet10/1": {}, + "Ethernet11/1": {}, + "Ethernet12/1": {}, + "Ethernet13/1": {}, + "Ethernet14/1": {}, + "Ethernet15/1": {}, + "Ethernet16/1": {}, + "Ethernet17/1": {}, + "Ethernet18/1": {}, + "Ethernet19/1": {}, + "Ethernet2/1": {}, + "Ethernet20/1": {}, + "Ethernet21/1": {}, + "Ethernet22/1": {}, + "Ethernet23/1": {}, + "Ethernet24/1": {}, + "Ethernet25/1": {}, + "Ethernet26/1": {}, + "Ethernet27/1": {}, + "Ethernet28/1": {}, + "Ethernet29/1": {}, + "Ethernet3/1": {}, + "Ethernet30/1": {}, + "Ethernet31/1": {}, + "Ethernet32/1": {}, + "Ethernet33/1": {}, + "Ethernet34/1": {}, + "Ethernet35/1": {}, + "Ethernet36/1": {}, + "Ethernet37/1": {}, + "Ethernet38/1": {}, + "Ethernet39/1": {}, + "Ethernet4/1": {}, + "Ethernet40/1": {}, + "Ethernet41/1": {}, + "Ethernet42/1": {}, + "Ethernet43/1": {}, + "Ethernet44/1": {}, + "Ethernet45/1": {}, + "Ethernet46/1": {}, + "Ethernet47/1": {}, + "Ethernet48/1": {}, + "Ethernet5/1": {}, + "Ethernet52/1": {}, + "Ethernet56/1": {}, + "Ethernet6/1": {}, + "Ethernet60/1": {}, + "Ethernet64/1": {}, + "Ethernet68/1": {}, + "Ethernet7/1": {}, + "Ethernet8/1": {}, + "Ethernet9/1": {}, + "Ethernet89/1": { + "SAI_PORT_STAT_IF_IN_UCAST_PKTS_last": "0", + "SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS_last": "15952", + "SAI_PORT_STAT_IF_OUT_UCAST_PKTS_last": "0", + "SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS_last": "15993", + "SAI_PORT_STAT_IF_IN_OCTETS_last": "4379392", + "SAI_PORT_STAT_IF_OUT_OCTETS_last": "4384675", + "SAI_PORT_STAT_IF_FEC_CORRECTED_BITS_last": "63971", + "SAI_PORT_STAT_IF_FEC_NOT_CORRECTABLE_FARMES_last": "0", + "FEC_PRE_BER": "0", + "FEC_POST_BER": "0" + } +} \ No newline at end of file diff --git a/testdata/RATES:oid:Ethernet89.txt b/testdata/RATES:oid:Ethernet89.txt new file mode 100644 index 000000000..ec5ce4cfd --- /dev/null +++ b/testdata/RATES:oid:Ethernet89.txt @@ -0,0 +1,12 @@ +{ + "SAI_PORT_STAT_IF_IN_UCAST_PKTS_last": "0", + "SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS_last": "15952", + "SAI_PORT_STAT_IF_OUT_UCAST_PKTS_last": "0", + "SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS_last": "15993", + "SAI_PORT_STAT_IF_IN_OCTETS_last": "4379392", + "SAI_PORT_STAT_IF_OUT_OCTETS_last": "4384675", + "SAI_PORT_STAT_IF_FEC_CORRECTED_BITS_last": "63971", + "SAI_PORT_STAT_IF_FEC_NOT_CORRECTABLE_FARMES_last": "0", + "FEC_PRE_BER": "0", + "FEC_POST_BER": "0" +} \ No newline at end of file diff --git a/testdata/db_dump.json b/testdata/db_dump.json index f0c26f354..ac42424c2 100644 --- a/testdata/db_dump.json +++ b/testdata/db_dump.json @@ -1593,12 +1593,14 @@ "ASIC_STATE:SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:oid:0x1a00000000010b":{"NULL":"NULL"}, "ASIC_STATE:SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:oid:0x1a0000000003de":{"NULL":"NULL"}},{ "USER_WATERMARKS:oid:0x1a000000000221":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, -"COUNTERS:oid:0x1500000000014b":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, -"COUNTERS:oid:0x15000000000229":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, -"COUNTERS:oid:0x150000000000f4":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, -"PERIODIC_WATERMARKS:oid:0x1a0000000003b1":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, -"USER_WATERMARKS:oid:0x150000000003cd":{"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, -"PERSISTENT_WATERMARKS:oid:0x1a0000000001af":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, +"COUNTERS:oid:0x1500000000014b":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, +"COUNTERS:oid:0x15000000000229":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, +"COUNTERS:oid:0x150000000000f4":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, +"COUNTERS:oid:0x1000000000056":{"SAI_QUEUE_STAT_PACKETS":"0","SAI_QUEUE_STAT_BYTES":"0","SAI_QUEUE_STAT_DROPPED_PACKETS":"0","SAI_QUEUE_STAT_DROPPED_BYTES":"0","SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, +"RATES:oid:0x1000000000056":{"FEC_PRE_BER":"0","FEC_POST_BER":"0"}, +"PERIODIC_WATERMARKS:oid:0x1a0000000003b1":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, +"USER_WATERMARKS:oid:0x150000000003cd":{"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, +"PERSISTENT_WATERMARKS:oid:0x1a0000000001af":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, "PERIODIC_WATERMARKS:oid:0x1a0000000004f7":{"SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES":"0","SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES":"0"}, "PERSISTENT_WATERMARKS:oid:0x1500000000055c":{"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, "USER_WATERMARKS:oid:0x1500000000028d":{"SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES":"0"}, From e302e6cdfe375b2ae92d434d30e6ca0f5495344d Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Thu, 6 Aug 2026 22:46:59 +0000 Subject: [PATCH 14/14] fix: upgrade vulnerable Go dependencies to fix CRITICAL/HIGH CVEs - google.golang.org/grpc v1.64.1 -> v1.82.1 (CVE-2026-33186, GHSA-hrxh-6v49-42gf) - golang.org/x/crypto v0.24.0 -> v0.52.0 (CVE-2024-45337, CVE-2025-22869, CVE-2026-39828-39832) - golang.org/x/net v0.26.0 -> v0.55.0 (CVE-2023-39325, CVE-2024-45338, CVE-2026-25681) - golang.org/x/text v0.16.0 -> v0.39.0 (CVE-2026-56852) - antchfx/jsonquery v1.1.4 -> v1.3.7 (to fix antchfx/xpath CVE-2026-32287) - antchfx/xmlquery v1.3.1 -> v1.5.1 (to fix antchfx/xpath CVE-2026-32287) - antchfx/xpath v1.1.10 -> v1.3.6 (CVE-2026-32287) Signed-off-by: qiluo --- go.mod | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 1607686a1..f3d642664 100644 --- a/go.mod +++ b/go.mod @@ -22,18 +22,18 @@ require ( github.com/openconfig/gnmi v0.0.0-20200617225440-d2b4e6a45802 github.com/openconfig/gnoi v0.0.0-20211029052138-349b3dcd04ec github.com/openconfig/ygot v0.7.1 - golang.org/x/crypto v0.24.0 - golang.org/x/net v0.26.0 - google.golang.org/grpc v1.64.1 + golang.org/x/crypto v0.52.0 + golang.org/x/net v0.55.0 + google.golang.org/grpc v1.82.1 google.golang.org/grpc/security/advancedtls v1.0.0 google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v2 v2.2.8 ) require ( - github.com/antchfx/jsonquery v1.1.4 // indirect - github.com/antchfx/xmlquery v1.3.1 // indirect - github.com/antchfx/xpath v1.1.10 // indirect + github.com/antchfx/jsonquery v1.3.7 // indirect + github.com/antchfx/xmlquery v1.5.1 // indirect + github.com/antchfx/xpath v1.3.6 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cenkalti/backoff/v4 v4.0.0 // indirect github.com/go-redis/redis/v7 v7.0.0-beta.3.0.20190824101152-d19aba07b476 // indirect @@ -48,12 +48,12 @@ require ( go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.39.0 // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a // indirect ) replace ( github.com/Azure/sonic-mgmt-common => ../sonic-mgmt-common - golang.org/x/crypto => golang.org/x/crypto v0.24.0 + golang.org/x/crypto => golang.org/x/crypto v0.52.0 )