Remove direct dependency on libyang

sonic-utilities should never call directly into libyang.  All
functionality it needs should be provided in sonic-yang-mgmt.  This
resolves issues when libyang changes API/ABI such as happened
between libyang1 and libyang2, and again between libyang2 and libyang3.

Depends on sonic-net/sonic-buildimage#24414

Author: bhouse-nexthop

README.md

@@ -34,6 +34,8 @@ Currently, this list of dependencies is as follows:
 - libyang_1.0.73_amd64.deb
 - libyang-cpp_1.0.73_amd64.deb
 - python3-yang_1.0.73_amd64.deb
+- libyang3_3.*_amd64.deb
+- python3-libyang_3.*_amd64.deb
 - redis_dump_load-1.1-py3-none-any.whl
 - sonic_py_common-1.0-py3-none-any.whl
 - sonic_config_engine-1.0-py3-none-any.whl

azure-pipelines.yml

@@ -85,6 +85,8 @@ stages:
         sudo dpkg -i libyang_1.0.73_amd64.deb
         sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
         sudo dpkg -i python3-yang_1.0.73_amd64.deb
+        sudo dpkg -i libyang3_3.*_amd64.deb
+        sudo dpkg -i python3-libyang_3.*_amd64.deb
       workingDirectory: $(Pipeline.Workspace)/target/debs/bookworm/
       displayName: 'Install Debian dependencies'
 

config/config_mgmt.py

@@ -8,7 +8,6 @@
 import shutil
 import syslog
 import tempfile
-import yang as ly
 from json import load
 from sys import flags
 from time import sleep as tsleep
@@ -35,8 +34,7 @@ class ConfigMgmt():
     to verify config for the commands which are capable of change in config DB.
     '''
 
-    def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True,
-                 sonicYangOptions=0, configdb=None):
+    def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, configdb=None):
         '''
         Initialise the class, --read the config, --load in data tree.
 
@@ -55,7 +53,6 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True,
             self.configdbJsonOut = None
             self.source = source
             self.allowTablesWithoutYang = allowTablesWithoutYang
-            self.sonicYangOptions = sonicYangOptions
             self.configdb = configdb
 
             # logging vars
@@ -71,7 +68,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True,
         return
 
     def __init_sonic_yang(self):
-        self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG, sonic_yang_options=self.sonicYangOptions)
+        self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
         # load yang models
         self.sy.loadYangModel()
         # load jIn from config DB or from config DB json file.
@@ -291,8 +288,7 @@ def get_module_name(yang_module_str):
 
         # Instantiate new context since parse_module_mem() loads the module into context.
         sy = sonic_yang.SonicYang(YANG_DIR)
-        module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
-        return module.name()
+        return sy.load_module_str_name(yang_module_str);
 
 
 # End of Class ConfigMgmt

generic_config_updater/gu_common.py

@@ -5,7 +5,6 @@
 import sonic_yang
 import sonic_yang_ext
 import subprocess
-import yang as ly
 import copy
 import re
 import os
@@ -539,10 +538,8 @@ def find_ref_paths(self, paths, config, reload_config: bool = True):
         # Iterate across all paths fetching references
         for path in paths:
             xpath = self.convert_path_to_xpath(path, config, sy)
-
-            leaf_xpaths = self._get_inner_leaf_xpaths(xpath, sy)
-            for xpath in leaf_xpaths:
-                ref_xpaths.extend(sy.find_data_dependencies(xpath))
+            # NOTE: This will recursively find dependencies for all decendents
+            ref_xpaths.extend(sy.find_data_dependencies(xpath))
 
         # For each xpath, convert to configdb path
         for ref_xpath in ref_xpaths:
@@ -554,22 +551,6 @@ def find_ref_paths(self, paths, config, reload_config: bool = True):
         ref_paths.sort()
         return ref_paths
 
-    def _get_inner_leaf_xpaths(self, xpath, sy):
-        if xpath == "/": # Point to Root element which contains all xpaths
-            nodes = sy.root.tree_for()
-        else: # Otherwise get all nodes that match xpath
-            nodes = sy.root.find_path(xpath).data()
-
-        for node in nodes:
-            for inner_node in node.tree_dfs():
-                # TODO: leaflist also can be used as the 'path' argument in 'leafref' so add support to leaflist
-                if self._is_leaf_node(inner_node):
-                    yield inner_node.path()
-
-    def _is_leaf_node(self, node):
-        schema = node.schema()
-        return ly.LYS_LEAF == schema.nodetype()
-
     def convert_path_to_xpath(self, path, config=None, sy=None):
         """
         Converts the given JsonPatch path (i.e. JsonPointer) to XPATH.

sonic_package_manager/manager.py

@@ -5,7 +5,6 @@
 import os
 import pkgutil
 import tempfile
-import yang as ly
 from inspect import signature
 from typing import Any, Iterable, List, Callable, Dict, Optional
 
@@ -1299,7 +1298,7 @@ def get_manager() -> 'PackageManager':
         docker_api = DockerApi(docker.from_env(), ProgressManager())
         registry_resolver = RegistryResolver()
         metadata_resolver = MetadataResolver(docker_api, registry_resolver)
-        cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON, sonicYangOptions=ly.LY_CTX_DISABLE_SEARCHDIR_CWD)
+        cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON)
         cli_generator = CliGenerator(log)
         feature_registry = FeatureRegistry(SonicDB)
         service_creator = ServiceCreator(feature_registry,