From e59d0319721b4da6a884eea0e6c1deac019b1cd0 Mon Sep 17 00:00:00 2001 From: Brad House Date: Fri, 7 Mar 2025 11:33:03 -0500 Subject: [PATCH 1/6] loadData: do not mutate user-provided data The user-provided configdb should not be mutated as the user may expect it to be unmodified. It is fairly unlikely to need to modify the input data, and it doesn't make sense to force the caller to deep copy the data before passing it in just in case it gets modified. --- src/sonic-yang-mgmt/sonic_yang_ext.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/sonic-yang-mgmt/sonic_yang_ext.py b/src/sonic-yang-mgmt/sonic_yang_ext.py index bb816326598..1e96c4294d3 100644 --- a/src/sonic-yang-mgmt/sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/sonic_yang_ext.py @@ -7,6 +7,7 @@ from json import dump, dumps, loads from xmltodict import parse from glob import glob +import copy Type_1_list_maps_model = [ 'DSCP_TO_TC_MAP_LIST', @@ -221,15 +222,21 @@ def _getModuleTLCcontainer(self, table): """ Crop config as per yang models, This Function crops from config only those TABLEs, for which yang models is - provided. The Tables without YANG models are stored in - self.tablesWithOutYangModels. + provided. If there are tables to modify it will perform a deepcopy of the + original structure in case anyone is holding a reference. + The Tables without YANG models are stored in self.tablesWithOutYangModels. """ def _cropConfigDB(self, croppedFile=None): - + isCopy = False tables = list(self.jIn.keys()) for table in tables: if table not in self.confDbYangMap: - # store in tablesWithOutYang + # Make sure we duplicate if we're modifying so if a caller + # has a reference we don't clobber it. + if not isCopy: + isCopy = True + self.jIn = copy.deepcopy(self.jIn) + # store in tablesWithOutYang and purge self.tablesWithOutYang[table] = self.jIn[table] del self.jIn[table] @@ -1153,7 +1160,7 @@ def _findXpathList(self, xpath, list, keys): """ load_data: load Config DB, crop, xlate and create data tree from it. (Public) - input: data + input: configdbJson - will NOT be modified debug Flag returns: True - success False - failed """ @@ -1168,7 +1175,8 @@ def loadData(self, configdbJson, debug=False): # reset xlate and tablesWithOutYang self.xlateJson = dict() self.tablesWithOutYang = dict() - # self.jIn will be cropped + # self.jIn will be cropped if needed, however it will duplicate the object + # so the original is not modified self._cropConfigDB() # xlated result will be in self.xlateJson self._xlateConfigDB(xlateFile=xlateFile) From d7b006b04ed9a41492511d7b266e8a094db167db Mon Sep 17 00:00:00 2001 From: "Brad House (nexthop)" Date: Tue, 11 Mar 2025 12:15:39 -0400 Subject: [PATCH 2/6] test yang models: they need to closer mimic sonic models sonic requires top-level container name be the same as the module name, since things depend on this within sonic it doesn't make sense to have tests that don't honor this restriction. NOTE: future patches will expect this consistency in tests. --- .../libyang-python-tests/config_data.json | 10 +- .../config_data_merge.json | 4 +- .../sample-yang-models/test-acl.yang | 8 +- .../sample-yang-models/test-interface.yang | 4 +- .../sample-yang-models/test-port.yang | 2 +- .../sample-yang-models/test-portchannel.yang | 4 +- .../sample-yang-models/test-vlan.yang | 4 +- .../test-yang-structure.yang | 2 +- .../libyang-python-tests/test_SonicYang.json | 146 +++++++++--------- 9 files changed, 92 insertions(+), 92 deletions(-) diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data.json b/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data.json index 154ef21b494..da7837b020d 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data.json +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data.json @@ -1,5 +1,5 @@ { - "test-vlan:vlan": { + "test-vlan:test-vlan": { "test-vlan:VLAN_INTERFACE": { "VLAN_INTERFACE_LIST": [{ "vlanid": 111, @@ -101,7 +101,7 @@ ] } }, - "test-port:port": { + "test-port:test-port": { "test-port:PORT": { "PORT_LIST": [{ "port_name": "Ethernet0", @@ -187,7 +187,7 @@ } }, - "test-acl:acl": { + "test-acl:test-acl": { "test-acl:ACL_RULE": { "ACL_RULE_LIST": [{ "ACL_TABLE_NAME": "PACL-V4", @@ -240,7 +240,7 @@ } }, - "test-interface:interface": { + "test-interface:test-interface": { "test-interface:INTERFACE": { "INTERFACE_LIST": [{ "interface": "Ethernet8", @@ -258,7 +258,7 @@ } }, - "test-yang-structure:test-yang-container": { + "test-yang-structure:test-yang-structure": { "test-yang-structure:YANG_STRUCT_TEST": { "YANG_LIST_TEST_LIST": [{ "name" : "Vlan1001", diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data_merge.json b/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data_merge.json index 73838a157c4..e58d37b2bd0 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data_merge.json +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/config_data_merge.json @@ -1,5 +1,5 @@ { - "test-vlan:vlan": { + "test-vlan:test-vlan": { "test-vlan:VLAN_INTERFACE": { "VLAN_INTERFACE_LIST": [{ "vlanid": 111, @@ -58,7 +58,7 @@ }] } }, - "test-port:port": { + "test-port:test-port": { "test-port:PORT": { "PORT_LIST": [{ "port_name": "Ethernet0", diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-acl.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-acl.yang index 0d7d93a1429..67ab472ad95 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-acl.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-acl.yang @@ -32,7 +32,7 @@ module test-acl { description "First Revision"; } - container acl { + container test-acl { container ACL_RULE { @@ -44,7 +44,7 @@ module test-acl { leaf ACL_TABLE_NAME { type leafref { - path "/acl:acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME"; + path "/acl:test-acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME"; } } @@ -251,10 +251,10 @@ module test-acl { /* union of leafref is allowed in YANG 1.1 */ type union { type leafref { - path /port:port/port:PORT/port:PORT_LIST/port:port_name; + path /port:test-port/port:PORT/port:PORT_LIST/port:port_name; } type leafref { - path /lag:portchannel/lag:PORTCHANNEL/lag:PORTCHANNEL_LIST/lag:portchannel_name; + path /lag:test-portchannel/lag:PORTCHANNEL/lag:PORTCHANNEL_LIST/lag:portchannel_name; } } } diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-interface.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-interface.yang index 2f453ac6e27..b10f6939517 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-interface.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-interface.yang @@ -25,7 +25,7 @@ module test-interface { description "First Revision"; } - container interface { + container test-interface { container INTERFACE { description "INTERFACE part of config_db.json"; @@ -36,7 +36,7 @@ module test-interface { leaf interface { type leafref { - path /port:port/port:PORT/port:PORT_LIST/port:port_name; + path /port:test-port/port:PORT/port:PORT_LIST/port:port_name; } } diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-port.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-port.yang index 5e2bf68f791..bf550c45d4a 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-port.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-port.yang @@ -20,7 +20,7 @@ module test-port{ description "First Revision"; } - container port{ + container test-port { container PORT { description "PORT part of config_db.json"; diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-portchannel.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-portchannel.yang index 2081383aeda..29cf0fcc9f8 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-portchannel.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-portchannel.yang @@ -25,7 +25,7 @@ module test-portchannel { description "First Revision"; } - container portchannel { + container test-portchannel { container PORTCHANNEL { description "PORTCHANNEL part of config_db.json"; @@ -44,7 +44,7 @@ module test-portchannel { leaf-list members { /* leaf-list members are unique by default */ type leafref { - path /port:port/port:PORT/port:PORT_LIST/port:port_name; + path /port:test-port/port:PORT/port:PORT_LIST/port:port_name; } } diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-vlan.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-vlan.yang index 2ca80607b41..4f647f55c6c 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-vlan.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-vlan.yang @@ -25,7 +25,7 @@ module test-vlan { description "First Revision"; } - container vlan { + container test-vlan { container VLAN_INTERFACE { description "VLAN_INTERFACE part of config_db.json"; @@ -126,7 +126,7 @@ module test-vlan { /* key elements are mandatory by default */ mandatory true; type leafref { - path /port:port/port:PORT/port:PORT_LIST/port:port_name; + path /port:test-port/port:PORT/port:PORT_LIST/port:port_name; } } diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-yang-structure.yang b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-yang-structure.yang index 1f57ae337a4..1c0a9e31043 100755 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-yang-structure.yang +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/sample-yang-models/test-yang-structure.yang @@ -22,7 +22,7 @@ module test-yang-structure { description "First Revision"; } - container test-yang-container { + container test-yang-structure { container YANG_STRUCT_TEST { diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json index 2376c357b75..9b53f5e145c 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json @@ -4,144 +4,144 @@ "data_nodes" : [ { "valid" : "True", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/alias" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/alias" }, { "valid" : "False", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet20']/alias" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet20']/alias" }, { "valid" : "True", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE" }, { "valid" : "False", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST" }, { "valid" : "True", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']" } ], "data_type" : [ { "data_type" : "LY_TYPE_STRING", - "xpath" : "/test-port:port/test-port:PORT/test-port:PORT_LIST/test-port:port_name" + "xpath" : "/test-port:test-port/test-port:PORT/test-port:PORT_LIST/test-port:port_name" }, { "data_type" : "LY_TYPE_LEAFREF", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" } ], "delete_nodes" : [ { "valid" : "False", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet10']/speed" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet10']/speed" }, { "valid" : "True", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/mtu" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/mtu" }, { "valid" : "False", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet20']/mtu" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet20']/mtu" } ], "dependencies" : [ { "dependencies" : [ - "/test-acl:acl/ACL_TABLE/ACL_TABLE_LIST[ACL_TABLE_NAME='PACL-V6']/ports[.='Ethernet8']", - "/test-interface:interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='10.1.1.64/26']/interface", - "/test-interface:interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='2000:f500:40:a749::/126']/interface" + "/test-acl:test-acl/ACL_TABLE/ACL_TABLE_LIST[ACL_TABLE_NAME='PACL-V6']/ports[.='Ethernet8']", + "/test-interface:test-interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='10.1.1.64/26']/interface", + "/test-interface:test-interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='2000:f500:40:a749::/126']/interface" ], - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet8']/port_name" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet8']/port_name" }, { "dependencies" : [], - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet8']/alias" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet8']/alias" } ], "leafref_path" : [ { "leafref_path" : "../../../VLAN/VLAN_LIST/vlanid", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" }, { - "leafref_path" : "/test-port:port/test-port:PORT/test-port:PORT_LIST/test-port:port_name", - "xpath" : "/test-interface:interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface" + "leafref_path" : "/test-port:test-port/test-port:PORT/test-port:PORT_LIST/test-port:port_name", + "xpath" : "/test-interface:test-interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface" }, { - "leafref_path" : "/test-port:port/test-port:PORT/test-port:PORT_LIST/test-port:port_name", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" + "leafref_path" : "/test-port:test-port/test-port:PORT/test-port:PORT_LIST/test-port:port_name", + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" }, { "leafref_path" : "../../../VLAN/VLAN_LIST/vlanid", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:vlanid" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:vlanid" } ], "leafref_type" : [ { "data_type" : "LY_TYPE_UINT16", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/vlanid" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/vlanid" }, { "data_type" : "LY_TYPE_STRING", - "xpath" : "/test-interface:interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='2000:f500:40:a749::/126']/interface" + "xpath" : "/test-interface:test-interface/INTERFACE/INTERFACE_LIST[interface='Ethernet8'][ip-prefix='2000:f500:40:a749::/126']/interface" }, { "data_type" : "LY_TYPE_STRING", - "xpath" : "/test-vlan:vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='111'][port='Ethernet0']/port" + "xpath" : "/test-vlan:test-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='111'][port='Ethernet0']/port" }, { "data_type" : "LY_TYPE_UINT16", - "xpath" : "/test-vlan:vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='111'][port='Ethernet0']/vlanid" + "xpath" : "/test-vlan:test-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='111'][port='Ethernet0']/vlanid" } ], "leafref_type_schema" : [ { "data_type" : "LY_TYPE_UINT16", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:vlanid" }, { "data_type" : "LY_TYPE_STRING", - "xpath" : "/test-interface:interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface" + "xpath" : "/test-interface:test-interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface" }, { "data_type" : "LY_TYPE_STRING", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" }, { "data_type" : "LY_TYPE_UINT16", - "xpath" : "/test-vlan:vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:vlanid" + "xpath" : "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:vlanid" } ], "members" : [ { "members" : [ - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet0']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet1']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet2']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet3']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet4']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet5']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet6']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet7']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet8']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet10']", - "/test-port:port/PORT/PORT_LIST[port_name='Ethernet12']" + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet0']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet1']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet2']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet3']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet4']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet5']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet6']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet7']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet8']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet10']", + "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet12']" ], - "xpath" : "/test-port:port/PORT/PORT_LIST" + "xpath" : "/test-port:test-port/PORT/PORT_LIST" } ], "merged_nodes" : [ { "value" : "25000", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet10']/speed" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet10']/speed" }, { "value" : "IPv6", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='200'][ip-prefix='2000:f500:45:6708::/64']/family" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='200'][ip-prefix='2000:f500:45:6708::/64']/family" } ], "modules" : [ @@ -177,51 +177,51 @@ "new_nodes" : [ { "value" : "Ethernet10_alias", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet12']/alias" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet12']/alias" }, { "value" : "5000", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet12']/speed" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet12']/speed" }, { "value" : "rule_20", - "xpath" : "/test-acl:acl/ACL_RULE/ACL_RULE_LIST[ACL_TABLE_NAME='PACL-test'][RULE_NAME='rule_20']/RULE_NAME" + "xpath" : "/test-acl:test-acl/ACL_RULE/ACL_RULE_LIST[ACL_TABLE_NAME='PACL-test'][RULE_NAME='rule_20']/RULE_NAME" } ], "node_values" : [ { "value" : "25000", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/speed" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/speed" }, { "value" : "IPv6", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/family" + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/family" } ], "parents" : [ { - "parent" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/family" + "parent" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='2000:f500:45:6709::/64']/family" }, { - "parent" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/scope" + "parent" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/scope" }, { - "parent" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/vlanid" + "parent" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/vlanid" }, { - "parent" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/ip-prefix" + "parent" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/ip-prefix" }, { - "parent" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/family" + "parent" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/family" }, { - "parent" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/speed" + "parent" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']", + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/speed" } ], "prefix" : [ @@ -257,36 +257,36 @@ "schema_dependencies" : [ { "schema_dependencies" : [ - "/test-acl:acl/test-acl:ACL_TABLE/test-acl:ACL_TABLE_LIST/test-acl:ports", - "/test-portchannel:portchannel/test-portchannel:PORTCHANNEL/test-portchannel:PORTCHANNEL_LIST/test-portchannel:members", - "/test-interface:interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface", - "/test-vlan:vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" + "/test-acl:test-acl/test-acl:ACL_TABLE/test-acl:ACL_TABLE_LIST/test-acl:ports", + "/test-portchannel:test-portchannel/test-portchannel:PORTCHANNEL/test-portchannel:PORTCHANNEL_LIST/test-portchannel:members", + "/test-interface:test-interface/test-interface:INTERFACE/test-interface:INTERFACE_LIST/test-interface:interface", + "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:port" ], - "xpath" : "/test-port:port/test-port:PORT/test-port:PORT_LIST/test-port:port_name" + "xpath" : "/test-port:test-port/test-port:PORT/test-port:PORT_LIST/test-port:port_name" } ], "schema_nodes" : [ { - "value" : "/test-vlan:vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:family", - "xpath" : "/test-vlan:vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/family" + "value" : "/test-vlan:test-vlan/test-vlan:VLAN_INTERFACE/test-vlan:VLAN_INTERFACE_LIST/test-vlan:family", + "xpath" : "/test-vlan:test-vlan/VLAN_INTERFACE/VLAN_INTERFACE_LIST[vlanid='111'][ip-prefix='10.1.1.64/26']/family" }, { - "value" : "/test-port:port/test-port:PORT/test-port:PORT_LIST/test-port:speed", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/speed" + "value" : "/test-port:test-port/test-port:PORT/test-port:PORT_LIST/test-port:speed", + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/speed" } ], "set_nodes" : [ { "value" : "10000", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet10']/speed" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet10']/speed" }, { "value" : "1500", - "xpath" : "/test-port:port/PORT/PORT_LIST[port_name='Ethernet9']/mtu" + "xpath" : "/test-port:test-port/PORT/PORT_LIST[port_name='Ethernet9']/mtu" }, { "value" : "server_vlan111", - "xpath" : "/test-vlan:vlan/VLAN/VLAN_LIST[vlanid='111']/description" + "xpath" : "/test-vlan:test-vlan/VLAN/VLAN_LIST[vlanid='111']/description" } ], "yang_dir" : "./tests/libyang-python-tests/sample-yang-models/" From f3209b0bee9e08e50a2edcf194834d1f9ce51f01 Mon Sep 17 00:00:00 2001 From: Brad House Date: Sat, 15 Mar 2025 09:55:23 -0400 Subject: [PATCH 3/6] libyang: schema leaf nodes need to expose must clauses It appears there was an oversight in upstream libyang v1 that must clauses weren't exposed via SWIG on leaf nodes, but were on all other relevant node types. This patch corrects this oversight. --- src/libyang/patch/libyang-leaf-must.patch | 29 +++++++++++++++++++++++ src/libyang/patch/series | 1 + 2 files changed, 30 insertions(+) create mode 100644 src/libyang/patch/libyang-leaf-must.patch diff --git a/src/libyang/patch/libyang-leaf-must.patch b/src/libyang/patch/libyang-leaf-must.patch new file mode 100644 index 00000000000..7e1c01c49e4 --- /dev/null +++ b/src/libyang/patch/libyang-leaf-must.patch @@ -0,0 +1,29 @@ +diff --git a/swig/cpp/src/Tree_Schema.cpp b/swig/cpp/src/Tree_Schema.cpp +index 3587320..8da206a 100644 +--- a/swig/cpp/src/Tree_Schema.cpp ++++ b/swig/cpp/src/Tree_Schema.cpp +@@ -344,6 +344,7 @@ S_Schema_Node Schema_Node_Choice::dflt() { + Schema_Node_Leaf::~Schema_Node_Leaf() {}; + S_Set Schema_Node_Leaf::backlinks() LY_NEW_CASTED(lys_node_leaf, node, backlinks, Set); + S_When Schema_Node_Leaf::when() LY_NEW_CASTED(lys_node_leaf, node, when, When); ++std::vector Schema_Node_Leaf::must() LY_NEW_LIST_CASTED(lys_node_leaf, node, must, must_size, Restr); + S_Type Schema_Node_Leaf::type() {return std::make_shared(&((struct lys_node_leaf *)node)->type, deleter);} + S_Schema_Node_List Schema_Node_Leaf::is_key() { + uint8_t pos; +diff --git a/swig/cpp/src/Tree_Schema.hpp b/swig/cpp/src/Tree_Schema.hpp +index d506891..f8ecc50 100644 +--- a/swig/cpp/src/Tree_Schema.hpp ++++ b/swig/cpp/src/Tree_Schema.hpp +@@ -683,8 +683,12 @@ public: + ~Schema_Node_Leaf(); + /** get backlinks variable from [lys_node_leaf](@ref lys_node_leaf)*/ + S_Set backlinks(); ++ /** get must_size variable from [lys_node_leaf](@ref lys_node_leaf)*/ ++ uint8_t must_size() {return ((struct lys_node_leaf *)node)->must_size;}; + /** get when variable from [lys_node_leaf](@ref lys_node_leaf)*/ + S_When when(); ++ /** get must variable from [lys_node_leaf](@ref lys_node_leaf)*/ ++ std::vector must(); + /** get type variable from [lys_node_leaf](@ref lys_node_leaf)*/ + S_Type type(); + /** get units variable from [lys_node_leaf](@ref lys_node_leaf)*/ diff --git a/src/libyang/patch/series b/src/libyang/patch/series index 9796e28649c..07a11507ec8 100644 --- a/src/libyang/patch/series +++ b/src/libyang/patch/series @@ -3,3 +3,4 @@ libyang_mgmt_framework.patch swig.patch large_file_support_arm32.patch debian-packaging-files.patch +libyang-leaf-must.patch From 0d217f85ec7c35fd333d8e962197ad57115f9030 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 19 Mar 2025 17:40:26 -0400 Subject: [PATCH 4/6] implement configdb<>xpath conversion Import modified xpath to configdb path and configdb path to xpath helpers from sonic-utilites/generic_config_updater as they don't belong there. Modify the code to be more readable, add the ability to extract schema paths instead of data xpaths. Don't require a configdb to be passed in unless we need to know a list index. This also caches configdb->xpath conversion which is used in recursive searches as it may be an expensive operation. --- src/sonic-yang-mgmt/setup.py | 3 +- src/sonic-yang-mgmt/sonic_yang.py | 5 +- src/sonic-yang-mgmt/sonic_yang_ext.py | 3 +- src/sonic-yang-mgmt/sonic_yang_path.py | 572 ++++++++++++++++++ .../libyang-python-tests/test_SonicYang.json | 18 + .../libyang-python-tests/test_sonic_yang.py | 50 ++ 6 files changed, 648 insertions(+), 3 deletions(-) create mode 100644 src/sonic-yang-mgmt/sonic_yang_path.py diff --git a/src/sonic-yang-mgmt/setup.py b/src/sonic-yang-mgmt/setup.py index 87617d93289..989abc124cf 100644 --- a/src/sonic-yang-mgmt/setup.py +++ b/src/sonic-yang-mgmt/setup.py @@ -30,6 +30,7 @@ install_requires = [ 'xmltodict==0.12.0', 'ijson==3.2.3', + 'jsonpointer>=1.9', 'jsondiff>=1.2.0', 'tabulate==0.9.0' ], @@ -46,7 +47,7 @@ include_package_data=True, keywords='sonic-yang-mgmt', name='sonic-yang-mgmt', - py_modules=['sonic_yang', 'sonic_yang_ext'], + py_modules=['sonic_yang', 'sonic_yang_ext', 'sonic_yang_path'], packages=find_packages(), version='1.0', zip_safe=False, diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index 9af9217ad34..9c47e245a11 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -4,13 +4,14 @@ from json import dump from glob import glob from sonic_yang_ext import SonicYangExtMixin, SonicYangException +from sonic_yang_path import SonicYangPathMixin """ Yang schema and data tree python APIs based on libyang python Here, sonic_yang_ext_mixin extends funtionality of sonic_yang, i.e. it is mixin not parent class. """ -class SonicYang(SonicYangExtMixin): +class SonicYang(SonicYangExtMixin, SonicYangPathMixin): def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_options=0): self.yang_dir = yang_dir @@ -44,6 +45,8 @@ def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_opt # below dict will store preProcessed yang objects, which may be needed by # all yang modules, such as grouping. self.preProcessedYang = dict() + # Lazy caching for configdb to xpath + self.configPathCache = dict() # element path for CONFIG DB. An example for this list could be: # ['PORT', 'Ethernet0', 'speed'] self.elementPath = [] diff --git a/src/sonic-yang-mgmt/sonic_yang_ext.py b/src/sonic-yang-mgmt/sonic_yang_ext.py index 1e96c4294d3..6214e6885bd 100644 --- a/src/sonic-yang-mgmt/sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/sonic_yang_ext.py @@ -8,6 +8,7 @@ from xmltodict import parse from glob import glob import copy +from sonic_yang_path import SonicYangPathMixin Type_1_list_maps_model = [ 'DSCP_TO_TC_MAP_LIST', @@ -43,7 +44,7 @@ class SonicYangException(Exception): pass # class sonic_yang methods, use mixin to extend sonic_yang -class SonicYangExtMixin: +class SonicYangExtMixin(SonicYangPathMixin): """ load all YANG models, create JSON of yang models. (Public function) diff --git a/src/sonic-yang-mgmt/sonic_yang_path.py b/src/sonic-yang-mgmt/sonic_yang_path.py new file mode 100644 index 00000000000..a132d3b395c --- /dev/null +++ b/src/sonic-yang-mgmt/sonic_yang_path.py @@ -0,0 +1,572 @@ +# This script is used as extension of sonic_yang class. It has methods of +# class sonic_yang. A separate file is used to avoid a single large file. + +from __future__ import print_function +from json import dump, dumps, loads +import sonic_yang_ext +import re +from jsonpointer import JsonPointer + +# class sonic_yang methods related to path handling, use mixin to extend sonic_yang +class SonicYangPathMixin: + """ + All xpath operations in this class are only relevent to ConfigDb and the conversion to YANG xpath. + It is not meant to support all the xpath functionalities, just the ones relevent to ConfigDb/YANG. + """ + @staticmethod + def configdb_path_split(configdb_path: str): + if configdb_path is None or configdb_path == "" or configdb_path == "/": + return [] + return JsonPointer(configdb_path).parts + + @staticmethod + def configdb_path_join(configdb_tokens: list[str]): + return JsonPointer.from_parts(configdb_tokens).path + + @staticmethod + def xpath_join(xpath_tokens: list[str], schema_xpath: bool) -> str: + if not schema_xpath: + return "/" + "/".join(xpath_tokens) + + # Schema XPath in libyang v1 wants each token prefixed with the module name. + # The first token should have this, use that to prefix the rest. + module_name = xpath_tokens[0].split(":")[0] + + return "/" + ("/" + module_name + ":").join(xpath_tokens) + + @staticmethod + def xpath_split(xpath: str) -> list[str]: + """ + Splits the given xpath into tokens by '/'. + + Example: + xpath: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8']/tagging_mode + tokens: sonic-vlan:sonic-vlan, VLAN_MEMBER, VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8'], tagging_mode + """ + if xpath == "": + raise ValueError("xpath cannot be empty") + + if xpath == "/": + return [] + + idx = 0 + tokens = [] + while idx < len(xpath): + end = SonicYangPathMixin.__get_xpath_token_end(idx+1, xpath) + token = xpath[idx+1:end] + tokens.append(token) + idx = end + + return tokens + + def configdb_path_to_xpath(self, configdb_path: str, schema_xpath: bool=False, configdb: dict=None) -> str: + """ + Converts the given ConfigDB path to a Yang data module xpath. + Parameters: + - configdb_path: The JSON path in the form taken by Config DB, + e.g. /VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode + - schema_xpath: Whether or not to output the xpath in schema form or data form. Schema form will not use + the data in the path, only table/list names. Defaults to false, so will emit data xpaths. + - configdb: If provided, and schema_xpath is false, will also emit the xpath token for a specific leaf-list + entry based on the value within the configdb itself. This is provided in the parsed configdb format, such + as returned from json.loads(). + + Example: + 1. configdb_path: /VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode + schema_xpath: False + returns: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8']/tagging_mode + 2. configdb_path: /VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode + schema_xpath: True + returns: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST/tagging_mode + """ + + if configdb_path is None or len(configdb_path) == 0 or configdb_path == "/": + return "/" + + # Fetch from cache if available + key = (configdb_path, schema_xpath) + result = self.configPathCache.get(key) + if result is not None: + return result + + # Not available, go through conversion + tokens = self.configdb_path_split(configdb_path) + if len(tokens) == 0: + return None + + xpath_tokens = [] + table = tokens[0] + + cmap = self.confDbYangMap[table] + + # getting the top level element : + xpath_tokens.append(cmap['module']+":"+cmap['topLevelContainer']) + + xpath_tokens.extend(self.__get_xpath_tokens_from_container(cmap['container'], tokens, 0, schema_xpath, configdb)) + + xpath = self.xpath_join(xpath_tokens, schema_xpath) + + # Save to cache + self.configPathCache[key] = xpath + + return xpath + + + def xpath_to_configdb_path(self, xpath: str, configdb: dict = None) -> str: + """ + Converts the given XPATH to ConfigDB Path. + If the xpath references a list value and the configdb is provided, the + generated path will reference the index of the list value + Example: + xpath: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8']/tagging_mode + path: /VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode + """ + tokens = self.xpath_split(xpath) + if len(tokens) == 0: + return "" + + if len(tokens) == 1: + raise ValueError("xpath cannot be just the module-name, there is no mapping to path") + + table = tokens[1] + cmap = self.confDbYangMap[table] + + configdb_path_tokens = self.__get_configdb_path_tokens_from_container(cmap['container'], tokens, 1, configdb) + return self.configdb_path_join(configdb_path_tokens) + + + def __get_xpath_tokens_from_container(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict) -> list[str]: + token = configdb_path_tokens[token_index] + xpath_tokens = [token] + + if len(configdb_path_tokens)-1 == token_index: + return xpath_tokens + + # check if the configdb token is referring to a list + list_model = self.__get_list_model(model, configdb_path_tokens, token_index) + if list_model: + new_xpath_tokens = self.__get_xpath_tokens_from_list(list_model, configdb_path_tokens, token_index+1, schema_xpath, configdb) + xpath_tokens.extend(new_xpath_tokens) + return xpath_tokens + + # check if it is targetting a child container + child_container_model = self.__get_model(model.get('container'), configdb_path_tokens[token_index+1]) + if child_container_model: + new_xpath_tokens = self.__get_xpath_tokens_from_container(child_container_model, configdb_path_tokens, token_index+1, schema_xpath, configdb) + xpath_tokens.extend(new_xpath_tokens) + return xpath_tokens + + leaf_token = self.__get_xpath_token_from_leaf(model, configdb_path_tokens, token_index+1, schema_xpath, configdb) + xpath_tokens.append(leaf_token) + + return xpath_tokens + + + # Locate a model matching the given name. If the model provided is a dict, + # it simply ensures the name matches and returns self. If the model + # provided is a list it scans the list for a matching name and returns that + # model. + def __get_model(self, model, name: str) -> dict: + if isinstance(model, dict) and model['@name'] == name: + return model + if isinstance(model, list): + for submodel in model: + if submodel['@name'] == name: + return submodel + + return None + + + # A configdb list specifies the container name, plus the keys separated by |. We are + # scanning the model for a list with a matching *number* of keys and returning the + # reference to the model with the definition. It is not valid to have 2 lists in + # the same container with the same number of keys since we have no way to match. + def __get_list_model(self, model: dict, configdb_path_tokens: list[str], token_index: int) -> dict: + parent_container_name = configdb_path_tokens[token_index] + clist = model.get('list') + # Container contains a single list, just return it + # TODO: check if matching also by name is necessary + if isinstance(clist, dict): + return clist + + if isinstance(clist, list): + configdb_values_str = configdb_path_tokens[token_index+1] + # Format: "value1|value2|value|..." + configdb_values = configdb_values_str.split("|") + for list_model in clist: + yang_keys_str = list_model['key']['@value'] + # Format: "key1 key2 key3 ..." + yang_keys = yang_keys_str.split() + # if same number of values and keys, this is the intended list-model + # TODO: Match also on types and not only the length of the keys/values + if len(yang_keys) == len(configdb_values): + return list_model + raise ValueError(f"Container {parent_container_name} has multiple lists, " + f"but none of them match the config_db value {configdb_values_str}") + + + def __get_xpath_tokens_from_list(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict): + item_token="" + + if schema_xpath: + item_token = model['@name'] + else: + keyDict = self.__parse_configdb_key_to_dict(model['key']['@value'], configdb_path_tokens[token_index]) + keyTokens = [f"[{key}='{keyDict[key]}']" for key in keyDict] + item_token = f"{model['@name']}{''.join(keyTokens)}" + + xpath_tokens = [item_token] + + # If we're pointing to the top level list item, and not a child leaf + # then we can just return. + if len(configdb_path_tokens)-1 == token_index: + return xpath_tokens + + type_1_list_model = self.__type1_get_model(model) + if type_1_list_model: + token = self.__type1_get_xpath_token(type_1_list_model, configdb_path_tokens, token_index+1, schema_xpath) + xpath_tokens.append(token) + return xpath_tokens + + leaf_token = self.__get_xpath_token_from_leaf(model, configdb_path_tokens, token_index+1, schema_xpath, configdb) + xpath_tokens.append(leaf_token) + return xpath_tokens + + + # Parse configdb key like Vlan1000|Ethernet8 (such as a key might be under /VLAN_MEMBER/) + # into its key/value dictionary form: { "name": "VLAN1000", "port": "Ethernet8" } + def __parse_configdb_key_to_dict(self, listKeys: str, configDbKey: str) -> dict: + xpath_list_keys = listKeys.split() + configdb_values = configDbKey.split("|") + # match lens + if len(xpath_list_keys) != len(configdb_values): + raise ValueError("Value not found for {} in {}".format(listKeys, configDbKey)) + # create the keyDict + rv = dict() + for i in range(len(xpath_list_keys)): + rv[xpath_list_keys[i]] = configdb_values[i].strip() + return rv + + + # Type1 lists are lists contained within another list. They always have exactly 1 key, and due to + # this they are special cased with a static lookup table. Check to see if the + # specified model is a type1 list and if so, return the model. + def __type1_get_model(self, model: dict) -> dict: + list_name = model['@name'] + if list_name not in sonic_yang_ext.Type_1_list_maps_model: + return None + + # Type 1 list is expected to have a single inner list model. + # No need to check if it is a dictionary of list models. + return model.get('list') + + + # Type1 lists are lists contained within another list. They always have exactly 1 key, and due to + # this they are special cased with a static lookup table. This is just a helper to do a quick + # transformation from configdb to the xpath key. + def __type1_get_xpath_token(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool) -> str: + if schema_xpath: + return model['@name'] + return f"{model['@name']}[{model['key']['@value']}='{configdb_path_tokens[token_index]}']" + + + # This function outputs the xpath token for leaf, choice, and leaf-list entries. + def __get_xpath_token_from_leaf(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict) -> str: + token = configdb_path_tokens[token_index] + + # checking all leaves + leaf_model = self.__get_model(model.get('leaf'), token) + if leaf_model: + return token + + # checking choice + choices = model.get('choice') + if choices: + for choice in choices: + cases = choice['case'] + for case in cases: + leaf_model = self.__get_model(case.get('leaf'), token) + if leaf_model: + return token + + # checking leaf-list (i.e. arrays of string, number or bool) + leaf_list_model = self.__get_model(model.get('leaf-list'), token) + if leaf_list_model: + # If there are no more tokens, just return the current token. + if len(configdb_path_tokens)-1 == token_index: + return token + + value = self.__get_configdb_value(configdb_path_tokens, configdb) + if value is None or schema_xpath: + return token + + # Reference an explicit leaf list value + return f"{token}[.='{value}']" + + raise ValueError(f"Path token not found.\n model: {model}\n token_index: {token_index}\n " + \ + f"path_tokens: {configdb_path_tokens}\n config: {configdb}") + + + def __get_configdb_value(self, configdb_path_tokens: list[str], configdb: dict) -> str: + if configdb is None: + return None + + ptr = configdb + for i in range(len(configdb_path_tokens)): + if isinstance(ptr, dict): + ptr = ptr[configdb_path_tokens[i]] + elif isinstance(ptr, list): + ptr = ptr[int(configdb_path_tokens[i])] + else: + return None + return ptr + + @staticmethod + def __get_xpath_token_end(start: int, xpath: str) -> int: + idx = start + while idx < len(xpath): + if xpath[idx] == "/": + break + elif xpath[idx] == "[": + idx = SonicYangPathMixin.__get_xpath_predicate_end(idx, xpath) + idx = idx+1 + + return idx + + @staticmethod + def __get_xpath_predicate_end(start: int, xpath: str) -> int: + idx = start + while idx < len(xpath): + if xpath[idx] == "]": + break + elif xpath[idx] == "'" or xpath[idx] == '"': + idx = SonicYangPathMixin.__get_xpath_quote_str_end(xpath[idx], idx, xpath) + + idx = idx+1 + + return idx + + @staticmethod + def __get_xpath_quote_str_end(ch: str, start: int, xpath: str) -> int: + idx = start+1 # skip first single quote + while idx < len(xpath): + if xpath[idx] == ch: + break + # libyang implements XPATH 1.0 which does not escape single or double quotes + # libyang src: https://netopeer.liberouter.org/doc/libyang/master/html/howtoxpath.html + # XPATH 1.0 src: https://www.w3.org/TR/1999/REC-xpath-19991116/#NT-Literal + idx = idx+1 + + return idx + + + def __get_configdb_path_tokens_from_container(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict) -> list[str]: + token = xpath_tokens[token_index] + configdb_path_tokens = [token] + + if len(xpath_tokens)-1 == token_index: + return configdb_path_tokens + + if configdb is not None: + configdb = configdb[token] + + # check child list + list_name = xpath_tokens[token_index+1].split("[")[0] + list_model = self.__get_model(model.get('list'), list_name) + if list_model: + new_path_tokens = self.__get_configdb_path_tokens_from_list(list_model, xpath_tokens, token_index+1, configdb) + configdb_path_tokens.extend(new_path_tokens) + return configdb_path_tokens + + container_name = xpath_tokens[token_index+1] + container_model = self.__get_model(model.get('container'), container_name) + if container_model: + new_path_tokens = self.__get_configdb_path_tokens_from_container(container_model, xpath_tokens, token_index+1, configdb) + configdb_path_tokens.extend(new_path_tokens) + return configdb_path_tokens + + new_path_tokens = self.__get_configdb_path_tokens_from_leaf(model, xpath_tokens, token_index+1, configdb) + configdb_path_tokens.extend(new_path_tokens) + + return configdb_path_tokens + + + def __xpath_keys_to_dict(self, token: str) -> dict: + # Token passed in is something like: + # VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8'] + # Strip off the Table name, and return a dictionary of key/value pairs. + + # See if we have keys + idx = token.find("[") + if idx == -1: + return dict() + + # Strip off table name + token = token[idx:] + + # Use regex to extract our keys and values + key_value_pattern = "\[([^=]+)='([^']*)'\]" + matches = re.findall(key_value_pattern, token) + kv = dict() + for item in matches: + kv[item[0]] = item[1] + + return kv + + def __get_configdb_path_tokens_from_list(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict): + token = xpath_tokens[token_index] + key_dict = self.__xpath_keys_to_dict(token) + + # If no keys specified return empty tokens, as we are already inside the correct table. + # Also note that the list name in SonicYang has no correspondence in ConfigDb and is ignored. + # Example where VLAN_MEMBER_LIST has no specific key/value: + # xpath: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST + # path: /VLAN_MEMBER + if not(key_dict): + return [] + + listKeys = model['key']['@value'] + key_list = listKeys.split() + + if len(key_list) != len(key_dict): + raise ValueError(f"Keys in configDb not matching keys in SonicYang. ConfigDb keys: {key_dict.keys()}. SonicYang keys: {key_list}") + + values = [key_dict[k] for k in key_list] + configdb_path_token = '|'.join(values) + configdb_path_tokens = [ configdb_path_token ] + + # Set pointer to pass for recursion + if configdb is not None: + # use .get() here as if configdb doesn't have the key it could return failure, but it can actually still + # generate a mostly relevant path. + configdb = configdb.get(configdb_path_token) + + # At end, just return + if len(xpath_tokens)-1 == token_index: + return configdb_path_tokens + + next_token = xpath_tokens[token_index+1] + # if the target node is a key, then it does not have a correspondene to path. + # Just return the current 'key1|key2|..' token as it already refers to the keys + # Example where the target node is 'name' which is a key in VLAN_MEMBER_LIST: + # xpath: /sonic-vlan:sonic-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[name='Vlan1000'][port='Ethernet8']/name + # path: /VLAN_MEMBER/Vlan1000|Ethernet8 + if next_token in key_dict: + return configdb_path_tokens + + type_1_list_model = self.__type1_get_model(model) + if type_1_list_model: + new_path_tokens = self.__get_configdb_path_tokens_from_type_1_list(type_1_list_model, xpath_tokens, token_index+1, configdb) + configdb_path_tokens.extend(new_path_tokens) + return configdb_path_tokens + + new_path_tokens = self.__get_configdb_path_tokens_from_leaf(model, xpath_tokens, token_index+1, configdb) + configdb_path_tokens.extend(new_path_tokens) + return configdb_path_tokens + + + def __get_configdb_path_tokens_from_leaf(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict) -> list[str]: + token = xpath_tokens[token_index] + + # checking all leaves + leaf_model = self.__get_model(model.get('leaf'), token) + if leaf_model: + return [token] + + # checking choices + choices = model.get('choice') + if choices: + for choice in choices: + cases = choice['case'] + for case in cases: + leaf_model = self.__get_model(case.get('leaf'), token) + if leaf_model: + return [token] + + # checking leaf-list + leaf_list_tokens = token.split("[", 1) # split once on the first '[', a regex is used later to fetch keys/values + leaf_list_name = leaf_list_tokens[0] + leaf_list_model = self.__get_model(model.get('leaf-list'), leaf_list_name) + if leaf_list_model: + # if whole-list is to be returned, such as if there is no key, or if configdb is not provided, + # Just return the list-name without checking the list items + # Example: + # xpath: /sonic-vlan:sonic-vlan/VLAN/VLAN_LIST[name='Vlan1000']/dhcp_servers + # path: /VLAN/Vlan1000/dhcp_servers + if configdb is None or len(leaf_list_tokens) == 1: + return [leaf_list_name] + leaf_list_pattern = "^[^\[]+(?:\[\.='([^']*)'\])?$" + leaf_list_regex = re.compile(leaf_list_pattern) + match = leaf_list_regex.match(token) + # leaf_list_name = match.group(1) + leaf_list_value = match.group(1) + list_config = configdb[leaf_list_name] + # Workaround for those fields who is defined as leaf-list in YANG model but have string value in config DB + # No need to lookup the item index in ConfigDb since the list is represented as a string, return path to string immediately + # Example: + # xpath: /sonic-buffer-port-egress-profile-list:sonic-buffer-port-egress-profile-list/BUFFER_PORT_EGRESS_PROFILE_LIST/BUFFER_PORT_EGRESS_PROFILE_LIST_LIST[port='Ethernet9']/profile_list[.='egress_lossy_profile'] + # path: /BUFFER_PORT_EGRESS_PROFILE_LIST/Ethernet9/profile_list + if isinstance(list_config, str): + return [leaf_list_name] + + if not isinstance(list_config, list): + raise ValueError(f"list_config is expected to be of type list or string. Found {type(list_config)}.\n " + \ + f"model: {model}\n token_index: {token_index}\n " + \ + f"xpath_tokens: {xpath_tokens}\n config: {configdb}") + + list_idx = list_config.index(leaf_list_value) + return [leaf_list_name, list_idx] + + raise ValueError(f"Xpath token not found.\n model: {model}\n token_index: {token_index}\n " + \ + f"xpath_tokens: {xpath_tokens}\n config: {configdb}") + + + def __get_configdb_path_tokens_from_type_1_list(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict): + type_1_inner_list_name = model['@name'] + + token = xpath_tokens[token_index] + list_tokens = token.split("[", 1) # split once on the first '[', first element will be the inner list name + inner_list_name = list_tokens[0] + + if type_1_inner_list_name != inner_list_name: + raise ValueError(f"Type 1 inner list name '{type_1_inner_list_name}' does match xpath inner list name '{inner_list_name}'.") + + key_dict = self.__xpath_keys_to_dict(token) + + # If no keys specified return empty tokens, as we are already inside the correct table. + # Also note that the type 1 inner list name in SonicYang has no correspondence in ConfigDb and is ignored. + # Example where DOT1P_TO_TC_MAP_LIST has no specific key/value: + # xpath: /sonic-dot1p-tc-map:sonic-dot1p-tc-map/DOT1P_TO_TC_MAP/DOT1P_TO_TC_MAP_LIST[name='Dot1p_to_tc_map1']/DOT1P_TO_TC_MAP + # path: /DOT1P_TO_TC_MAP/Dot1p_to_tc_map1 + if not(key_dict): + return [] + + if len(key_dict) > 1: + raise ValueError(f"Type 1 inner list should have only 1 key in xpath, {len(key_dict)} specified. Key dictionary: {key_dict}") + + keyName = next(iter(key_dict.keys())) + value = key_dict[keyName] + + path_tokens = [value] + + # If this is the last xpath token, return the path tokens we have built so far, no need for futher checks + # Example: + # xpath: /sonic-dot1p-tc-map:sonic-dot1p-tc-map/DOT1P_TO_TC_MAP/DOT1P_TO_TC_MAP_LIST[name='Dot1p_to_tc_map1']/DOT1P_TO_TC_MAP[dot1p='2'] + # path: /DOT1P_TO_TC_MAP/Dot1p_to_tc_map1/2 + if token_index+1 >= len(xpath_tokens): + return path_tokens + + # Checking if the next_token is actually a child leaf of the inner type 1 list, for which case + # just ignore the token, and return the already created ConfigDb path pointing to the whole object + # Example where the leaf specified is the key: + # xpath: /sonic-dot1p-tc-map:sonic-dot1p-tc-map/DOT1P_TO_TC_MAP/DOT1P_TO_TC_MAP_LIST[name='Dot1p_to_tc_map1']/DOT1P_TO_TC_MAP[dot1p='2']/dot1p + # path: /DOT1P_TO_TC_MAP/Dot1p_to_tc_map1/2 + # Example where the leaf specified is not the key: + # xpath: /sonic-dot1p-tc-map:sonic-dot1p-tc-map/DOT1P_TO_TC_MAP/DOT1P_TO_TC_MAP_LIST[name='Dot1p_to_tc_map1']/DOT1P_TO_TC_MAP[dot1p='2']/tc + # path: /DOT1P_TO_TC_MAP/Dot1p_to_tc_map1/2 + next_token = xpath_tokens[token_index+1] + leaf_model = self.__get_model(model.get('leaf'), next_token) + if leaf_model: + return path_tokens + + raise ValueError(f"Type 1 inner list '{type_1_inner_list_name}' does not have a child leaf named '{next_token}'") diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json index 9b53f5e145c..998893b7ff4 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_SonicYang.json @@ -289,5 +289,23 @@ "xpath" : "/test-vlan:test-vlan/VLAN/VLAN_LIST[vlanid='111']/description" } ], + "configdb_path_to_xpath": [ + { + "configdb_path": "/VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode", + "schema_xpath": false, + "xpath": "/test-vlan:test-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='Vlan1000'][port='Ethernet8']/tagging_mode" + }, + { + "configdb_path": "/VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode", + "schema_xpath": true, + "xpath": "/test-vlan:test-vlan/test-vlan:VLAN_MEMBER/test-vlan:VLAN_MEMBER_LIST/test-vlan:tagging_mode" + } + ], + "xpath_to_configdb_path": [ + { + "xpath": "/test-vlan:test-vlan/VLAN_MEMBER/VLAN_MEMBER_LIST[vlanid='Vlan1000'][port='Ethernet8']/tagging_mode", + "configdb_path": "/VLAN_MEMBER/Vlan1000|Ethernet8/tagging_mode" + } + ], "yang_dir" : "./tests/libyang-python-tests/sample-yang-models/" } diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index 86b27ef174e..3517ada4b62 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -261,6 +261,56 @@ def test_get_leafref_type_schema(self, yang_s, data): data_type = yang_s._get_leafref_type_schema(xpath) assert expected_type == data_type + def test_configdb_path_to_xpath(self, yang_s, data): + yang_s.loadYangModel() + for node in data['configdb_path_to_xpath']: + configdb_path = str(node['configdb_path']) + schema_xpath = bool(node['schema_xpath']) + expected = node['xpath'] + received = yang_s.configdb_path_to_xpath(configdb_path, schema_xpath=schema_xpath) + assert received == expected + + def test_xpath_to_configdb_path(self, yang_s, data): + yang_s.loadYangModel() + for node in data['xpath_to_configdb_path']: + xpath = str(node['xpath']) + expected = node['configdb_path'] + received = yang_s.xpath_to_configdb_path(xpath) + assert received == expected + + def test_configdb_path_split(self, yang_s, data): + def check(path, tokens): + expected=tokens + actual=yang_s.configdb_path_split(path) + assert expected == actual + + check("", []) + check("/", []) + check("/token", ["token"]) + check("/more/than/one/token", ["more", "than", "one", "token"]) + check("/has/numbers/0/and/symbols/^", ["has", "numbers", "0", "and", "symbols", "^"]) + check("/~0/this/is/telda", ["~", "this", "is", "telda"]) + check("/~1/this/is/forward-slash", ["/", "this", "is", "forward-slash"]) + check("/\\\\/no-escaping", ["\\\\", "no-escaping"]) + check("////empty/tokens/are/ok", ["", "", "", "empty", "tokens", "are", "ok"]) + + def configdb_path_join(self, yang_s, data): + def check(tokens, path): + expected=path + actual=yang_s.configdb_path_join(tokens) + assert expected == actual + + check([], "/",) + check([""], "/",) + check(["token"], "/token") + check(["more", "than", "one", "token"], "/more/than/one/token") + check(["has", "numbers", "0", "and", "symbols", "^"], "/has/numbers/0/and/symbols/^") + check(["~", "this", "is", "telda"], "/~0/this/is/telda") + check(["/", "this", "is", "forward-slash"], "/~1/this/is/forward-slash") + check(["\\\\", "no-escaping"], "/\\\\/no-escaping") + check(["", "", "", "empty", "tokens", "are", "ok"], "////empty/tokens/are/ok") + check(["~token", "telda-not-followed-by-0-or-1"], "/~0token/telda-not-followed-by-0-or-1") + """ This is helper function to load YANG models for tests cases, which works on Real SONiC Yang models. Mainly tests for translation and reverse From 09eff1eb08671c3b179f99553eaaaf0dc454162b Mon Sep 17 00:00:00 2001 From: "Brad House (nexthop)" Date: Tue, 11 Mar 2025 14:01:26 -0400 Subject: [PATCH 5/6] find_schema_dependencies() and find_schema_must_count() implementations Using schema metadata for generic config updater can be used to determine change ordering so these need to be implemented and exposed as public. Due the the repeated nature of these calls, they implement caching for performance. --- src/sonic-yang-mgmt/sonic_yang.py | 140 ++++++++++++++++-- .../libyang-python-tests/test_sonic_yang.py | 2 +- 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index 9c47e245a11..0cb8bb6414d 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -18,7 +18,6 @@ def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_opt self.ctx = None self.module = None self.root = None - # logging vars self.SYSLOG_IDENTIFIER = "sonic_yang" self.DEBUG = debug @@ -45,6 +44,10 @@ def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_opt # below dict will store preProcessed yang objects, which may be needed by # all yang modules, such as grouping. self.preProcessedYang = dict() + # Lazy caching for backlinks lookups + self.backlinkCache = dict() + # Lazy caching for must counts + self.mustCache = dict() # Lazy caching for configdb to xpath self.configPathCache = dict() # element path for CONFIG DB. An example for this list could be: @@ -495,13 +498,89 @@ def _find_data_nodes(self, data_xpath): list.append(data_set.path()) return list + """ + find_schema_must_count(): find the number of must clauses for the schema path + input: schema_xpath of the schema node + match_ancestors whether or not to treat the specified path as + an ancestor rather than a full path. If set to + true, will add recursively. + returns: - count of must statements encountered + - Exception if schema node not found + """ + def find_schema_must_count(self, schema_xpath, match_ancestors: bool=False): + # See if we have this cached + key = ( schema_xpath, match_ancestors ) + result = self.mustCache.get(key) + if result is not None: + return result + + try: + schema_node = self._find_schema_node(schema_xpath) + except Exception as e: + self.sysLog(msg="Cound not find the schema node from xpath: " + str(schema_xpath), debug=syslog.LOG_ERR, doPrint=True) + self.fail(e) + return 0 + + # If not doing recursion, just return the result. This will internally + # cache the child so no need to update the cache ourselves + if not match_ancestors: + return self.__find_schema_must_count_only(schema_node) + + count = 0 + # Recurse first + for elem in schema_node.tree_dfs(): + count += self.__find_schema_must_count_only(elem) + + # Pull self + count += self.__find_schema_must_count_only(schema_node) + + # Save in cache + self.mustCache[key] = count + + return count + + def __find_schema_must_count_only(self, schema_node): + # Check non-recursive cache + key = ( schema_node.path(), False ) + result = self.mustCache.get(key) + if result is not None: + return result + + count = 0 + if schema_node.nodetype() == ly.LYS_CONTAINER: + schema_leaf = ly.Schema_Node_Container(schema_node) + if schema_leaf.must() is not None: + count += 1 + elif schema_node.nodetype() == ly.LYS_LEAF: + schema_leaf = ly.Schema_Node_Leaf(schema_node) + count += schema_leaf.must_size() + elif schema_node.nodetype() == ly.LYS_LEAFLIST: + schema_leaf = ly.Schema_Node_Leaflist(schema_node) + count += schema_leaf.must_size() + elif schema_node.nodetype() == ly.LYS_LIST: + schema_leaf = ly.Schema_Node_List(schema_node) + count += schema_leaf.must_size() + + # Cache result + self.mustCache[key] = count + return count + """ find_schema_dependencies(): find the schema dependencies from schema xpath - input: schema_xpath of the schema node + input: schema_xpath of the schema node + match_ancestors whether or not to treat the specified path as + an ancestor rather than a full path. If set to + true, will add recursively. returns: - list of xpath of the dependencies - Exception if schema node not found """ - def _find_schema_dependencies(self, schema_xpath): + def find_schema_dependencies(self, schema_xpath, match_ancestors: bool=False): + # See if we have this cached + key = ( schema_xpath, match_ancestors ) + result = self.backlinkCache.get(key) + if result is not None: + return result + ref_list = [] try: schema_node = self._find_schema_node(schema_xpath) @@ -510,12 +589,46 @@ def _find_schema_dependencies(self, schema_xpath): self.fail(e) return ref_list - schema_node = ly.Schema_Node_Leaf(schema_node) - backlinks = schema_node.backlinks() - if backlinks.number() > 0: - for link in backlinks.schema(): - self.sysLog(msg="backlink schema: {}".format(link.path()), doPrint=True) - ref_list.append(link.path()) + # If not doing recursion, just return the result. This will internally + # cache the child so no need to update the cache ourselves + if not match_ancestors: + return self.__find_schema_dependencies_only(schema_node) + + # Recurse first + for elem in schema_node.tree_dfs(): + ref_list.extend(self.__find_schema_dependencies_only(elem)) + + # Pull self + ref_list.extend(self.__find_schema_dependencies_only(schema_node)) + + # Save in cache + self.backlinkCache[key] = ref_list + + return ref_list + + def __find_schema_dependencies_only(self, schema_node): + # Check non-recursive cache + key = ( schema_node.path(), False ) + result = self.backlinkCache.get(key) + if result is not None: + return result + + # New lookup + ref_list = [] + schema_leaf = None + if schema_node.nodetype() == ly.LYS_LEAF: + schema_leaf = ly.Schema_Node_Leaf(schema_node) + elif schema_node.nodetype() == ly.LYS_LEAFLIST: + schema_leaf = ly.Schema_Node_Leaflist(schema_node) + + if schema_leaf is not None: + backlinks = schema_leaf.backlinks() + if backlinks is not None and backlinks.number() > 0: + for link in backlinks.schema(): + ref_list.append(link.path()) + + # Cache result + self.backlinkCache[key] = ref_list return ref_list """ @@ -536,11 +649,10 @@ def find_data_dependencies(self, data_xpath): try: value = str(self._find_data_node_value(data_xpath)) - schema_node = ly.Schema_Node_Leaf(data_node.schema()) - backlinks = schema_node.backlinks() - if backlinks is not None and backlinks.number() > 0: - for link in backlinks.schema(): - node_set = node.find_path(link.path()) + backlinks = self.find_schema_dependencies(data_node.schema().path(), False) + if backlinks is not None and len(backlinks) > 0: + for link in backlinks: + node_set = node.find_path(link) for data_set in node_set.data(): data_set.schema() casted = data_set.subtype() diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index 3517ada4b62..9d773680ce8 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -211,7 +211,7 @@ def test_find_schema_dependencies(self, yang_s, data): for node in data['schema_dependencies']: xpath = str(node['xpath']) list = node['schema_dependencies'] - depend = yang_s._find_schema_dependencies(xpath) + depend = yang_s.find_schema_dependencies(xpath) assert set(depend) == set(list) #test merge data tree From a095f03baa45af7adaa526b3751a67efbf667858 Mon Sep 17 00:00:00 2001 From: "Brad House (nexthop)" Date: Thu, 17 Apr 2025 10:30:58 -0400 Subject: [PATCH 6/6] fix python 3.7 since azure vs is still building buster --- src/sonic-yang-mgmt/sonic_yang_path.py | 27 +++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/sonic-yang-mgmt/sonic_yang_path.py b/src/sonic-yang-mgmt/sonic_yang_path.py index a132d3b395c..5cf672326d6 100644 --- a/src/sonic-yang-mgmt/sonic_yang_path.py +++ b/src/sonic-yang-mgmt/sonic_yang_path.py @@ -6,6 +6,7 @@ import sonic_yang_ext import re from jsonpointer import JsonPointer +from typing import List # class sonic_yang methods related to path handling, use mixin to extend sonic_yang class SonicYangPathMixin: @@ -20,11 +21,11 @@ def configdb_path_split(configdb_path: str): return JsonPointer(configdb_path).parts @staticmethod - def configdb_path_join(configdb_tokens: list[str]): + def configdb_path_join(configdb_tokens: List[str]): return JsonPointer.from_parts(configdb_tokens).path @staticmethod - def xpath_join(xpath_tokens: list[str], schema_xpath: bool) -> str: + def xpath_join(xpath_tokens: List[str], schema_xpath: bool) -> str: if not schema_xpath: return "/" + "/".join(xpath_tokens) @@ -35,7 +36,7 @@ def xpath_join(xpath_tokens: list[str], schema_xpath: bool) -> str: return "/" + ("/" + module_name + ":").join(xpath_tokens) @staticmethod - def xpath_split(xpath: str) -> list[str]: + def xpath_split(xpath: str) -> List[str]: """ Splits the given xpath into tokens by '/'. @@ -135,7 +136,7 @@ def xpath_to_configdb_path(self, xpath: str, configdb: dict = None) -> str: return self.configdb_path_join(configdb_path_tokens) - def __get_xpath_tokens_from_container(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict) -> list[str]: + def __get_xpath_tokens_from_container(self, model: dict, configdb_path_tokens: List[str], token_index: int, schema_xpath: bool, configdb: dict) -> List[str]: token = configdb_path_tokens[token_index] xpath_tokens = [token] @@ -181,7 +182,7 @@ def __get_model(self, model, name: str) -> dict: # scanning the model for a list with a matching *number* of keys and returning the # reference to the model with the definition. It is not valid to have 2 lists in # the same container with the same number of keys since we have no way to match. - def __get_list_model(self, model: dict, configdb_path_tokens: list[str], token_index: int) -> dict: + def __get_list_model(self, model: dict, configdb_path_tokens: List[str], token_index: int) -> dict: parent_container_name = configdb_path_tokens[token_index] clist = model.get('list') # Container contains a single list, just return it @@ -205,7 +206,7 @@ def __get_list_model(self, model: dict, configdb_path_tokens: list[str], token_i f"but none of them match the config_db value {configdb_values_str}") - def __get_xpath_tokens_from_list(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict): + def __get_xpath_tokens_from_list(self, model: dict, configdb_path_tokens: List[str], token_index: int, schema_xpath: bool, configdb: dict): item_token="" if schema_xpath: @@ -264,14 +265,14 @@ def __type1_get_model(self, model: dict) -> dict: # Type1 lists are lists contained within another list. They always have exactly 1 key, and due to # this they are special cased with a static lookup table. This is just a helper to do a quick # transformation from configdb to the xpath key. - def __type1_get_xpath_token(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool) -> str: + def __type1_get_xpath_token(self, model: dict, configdb_path_tokens: List[str], token_index: int, schema_xpath: bool) -> str: if schema_xpath: return model['@name'] return f"{model['@name']}[{model['key']['@value']}='{configdb_path_tokens[token_index]}']" # This function outputs the xpath token for leaf, choice, and leaf-list entries. - def __get_xpath_token_from_leaf(self, model: dict, configdb_path_tokens: list[str], token_index: int, schema_xpath: bool, configdb: dict) -> str: + def __get_xpath_token_from_leaf(self, model: dict, configdb_path_tokens: List[str], token_index: int, schema_xpath: bool, configdb: dict) -> str: token = configdb_path_tokens[token_index] # checking all leaves @@ -307,7 +308,7 @@ def __get_xpath_token_from_leaf(self, model: dict, configdb_path_tokens: list[st f"path_tokens: {configdb_path_tokens}\n config: {configdb}") - def __get_configdb_value(self, configdb_path_tokens: list[str], configdb: dict) -> str: + def __get_configdb_value(self, configdb_path_tokens: List[str], configdb: dict) -> str: if configdb is None: return None @@ -360,7 +361,7 @@ def __get_xpath_quote_str_end(ch: str, start: int, xpath: str) -> int: return idx - def __get_configdb_path_tokens_from_container(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict) -> list[str]: + def __get_configdb_path_tokens_from_container(self, model: dict, xpath_tokens: List[str], token_index: int, configdb: dict) -> List[str]: token = xpath_tokens[token_index] configdb_path_tokens = [token] @@ -413,7 +414,7 @@ def __xpath_keys_to_dict(self, token: str) -> dict: return kv - def __get_configdb_path_tokens_from_list(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict): + def __get_configdb_path_tokens_from_list(self, model: dict, xpath_tokens: List[str], token_index: int, configdb: dict): token = xpath_tokens[token_index] key_dict = self.__xpath_keys_to_dict(token) @@ -465,7 +466,7 @@ def __get_configdb_path_tokens_from_list(self, model: dict, xpath_tokens: list[s return configdb_path_tokens - def __get_configdb_path_tokens_from_leaf(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict) -> list[str]: + def __get_configdb_path_tokens_from_leaf(self, model: dict, xpath_tokens: List[str], token_index: int, configdb: dict) -> List[str]: token = xpath_tokens[token_index] # checking all leaves @@ -521,7 +522,7 @@ def __get_configdb_path_tokens_from_leaf(self, model: dict, xpath_tokens: list[s f"xpath_tokens: {xpath_tokens}\n config: {configdb}") - def __get_configdb_path_tokens_from_type_1_list(self, model: dict, xpath_tokens: list[str], token_index: int, configdb: dict): + def __get_configdb_path_tokens_from_type_1_list(self, model: dict, xpath_tokens: List[str], token_index: int, configdb: dict): type_1_inner_list_name = model['@name'] token = xpath_tokens[token_index]