Merge pull request #4398 from jestabro/commitd

T7121: Set up communication vyconfd to vyos-commitd

Author: dmbaturin

.gitignore

@@ -151,6 +151,9 @@ data/reftree.cache
 # autogenerated vyos-configd JSON definition
 data/configd-include.json
 
+# autogenerated vyos-commitd protobuf files
+python/vyos/proto/*pb2.py
+
 # We do not use pip
 Pipfile
 Pipfile.lock

Makefile

@@ -27,7 +27,7 @@ libvyosconfig:
 		rm -rf /tmp/libvyosconfig && \
 			git clone https://github.com/vyos/libvyosconfig.git /tmp/libvyosconfig || exit 1
 		cd /tmp/libvyosconfig && \
-			git checkout 677d1e2bf8109b9fd4da60e20376f992b747e384 || exit 1
+			git checkout 5f15d8095efd11756a867e552a3f8fe6c77e57cc || exit 1
 		eval $$(opam env --root=/opt/opam --set-root) && ./build.sh
 	fi
 

debian/control

@@ -15,6 +15,8 @@ Build-Depends:
 # For generating command definitions
   python3-lxml,
   python3-xmltodict,
+# For generating serialization functions
+  protobuf-compiler,
 # For running tests
   python3-coverage,
   python3-hurry.filesize,
@@ -70,13 +72,15 @@ Depends:
   python3-netifaces,
   python3-paramiko,
   python3-passlib,
+  python3-protobuf,
   python3-pyroute2,
   python3-psutil,
   python3-pyhumps,
   python3-pystache,
   python3-pyudev,
   python3-six,
   python3-tabulate,
+  python3-tomli,
   python3-voluptuous,
   python3-xmltodict,
   python3-zmq,

python/setup.py

@@ -1,5 +1,11 @@
 import os
+import sys
+import subprocess
 from setuptools import setup
+from setuptools.command.build_py import build_py
+
+sys.path.append('./vyos')
+from defaults import directories
 
 def packages(directory):
     return [
@@ -8,6 +14,35 @@ def packages(directory):
         if os.path.isfile(os.path.join(_[0], '__init__.py'))
     ]
 
+
+class GenerateProto(build_py):
+    ver = os.environ.get('OCAML_VERSION')
+    if ver:
+        proto_path = f'/opt/opam/{ver}/share/vyconf'
+    else:
+        proto_path = directories['proto_path']
+
+    def run(self):
+        # find all .proto files in vyconf proto_path
+        proto_files = []
+        for _, _, files in os.walk(self.proto_path):
+            for file in files:
+                if file.endswith('.proto'):
+                    proto_files.append(file)
+
+        # compile each .proto file to Python
+        for proto_file in proto_files:
+            subprocess.check_call(
+                [
+                    'protoc',
+                    '--python_out=vyos/proto',
+                    f'--proto_path={self.proto_path}/',
+                    proto_file,
+                ]
+            )
+
+        build_py.run(self)
+
 setup(
     name = "vyos",
     version = "1.3.0",
@@ -29,4 +64,7 @@ def packages(directory):
             "config-mgmt = vyos.config_mgmt:run",
         ],
     },
+    cmdclass={
+        'build_py': GenerateProto,
+    },
 )

python/vyos/config_mgmt.py

@@ -287,7 +287,7 @@ def revert_soft(self) -> Tuple[str, int]:
 
         # commits under commit-confirm are not added to revision list unless
         # confirmed, hence a soft revert is to revision 0
-        revert_ct = self._get_config_tree_revision(0)
+        revert_ct = self.get_config_tree_revision(0)
 
         message = '[commit-confirm] Reverting to previous config now'
         os.system('wall -n ' + message)
@@ -351,7 +351,7 @@ def rollback_soft(self, rev: int):
             )
             return msg, 1
 
-        rollback_ct = self._get_config_tree_revision(rev)
+        rollback_ct = self.get_config_tree_revision(rev)
         try:
             load(rollback_ct, switch='explicit')
             print('Rollback diff has been applied.')
@@ -382,15 +382,15 @@ def compare(
         if rev1 is not None:
             if not self._check_revision_number(rev1):
                 return f'Invalid revision number {rev1}', 1
-            ct1 = self._get_config_tree_revision(rev1)
+            ct1 = self.get_config_tree_revision(rev1)
             ct2 = self.working_config
             msg = f'No changes between working and revision {rev1} configurations.\n'
         if rev2 is not None:
             if not self._check_revision_number(rev2):
                 return f'Invalid revision number {rev2}', 1
             # compare older to newer
             ct2 = ct1
-            ct1 = self._get_config_tree_revision(rev2)
+            ct1 = self.get_config_tree_revision(rev2)
             msg = f'No changes between revisions {rev2} and {rev1} configurations.\n'
 
         out = ''
@@ -575,7 +575,7 @@ def _get_file_revision(self, rev: int):
             r = f.read().decode()
         return r
 
-    def _get_config_tree_revision(self, rev: int):
+    def get_config_tree_revision(self, rev: int):
         c = self._get_file_revision(rev)
         return ConfigTree(c)
 

python/vyos/configsource.py

@@ -319,3 +319,13 @@ def __init__(self, running_config_text=None, session_config_text=None):
             self._session_config = ConfigTree(session_config_text) if session_config_text else None
         except ValueError:
             raise ConfigSourceError(f"Init error in {type(self)}")
+
+class ConfigSourceCache(ConfigSource):
+    def __init__(self, running_config_cache=None, session_config_cache=None):
+        super().__init__()
+
+        try:
+            self._running_config = ConfigTree(internal=running_config_cache) if running_config_cache else None
+            self._session_config = ConfigTree(internal=session_config_cache) if session_config_cache else None
+        except ValueError:
+            raise ConfigSourceError(f"Init error in {type(self)}")

python/vyos/configtree.py

@@ -66,9 +66,14 @@ class ConfigTreeError(Exception):
 
 
 class ConfigTree(object):
-    def __init__(self, config_string=None, address=None, libpath=LIBPATH):
-        if config_string is None and address is None:
-            raise TypeError("ConfigTree() requires one of 'config_string' or 'address'")
+    def __init__(
+        self, config_string=None, address=None, internal=None, libpath=LIBPATH
+    ):
+        if config_string is None and address is None and internal is None:
+            raise TypeError(
+                "ConfigTree() requires one of 'config_string', 'address', or 'internal'"
+            )
+
         self.__config = None
         self.__lib = cdll.LoadLibrary(libpath)
 
@@ -89,6 +94,13 @@ def __init__(self, config_string=None, address=None, libpath=LIBPATH):
         self.__to_commands.argtypes = [c_void_p, c_char_p]
         self.__to_commands.restype = c_char_p
 
+        self.__read_internal = self.__lib.read_internal
+        self.__read_internal.argtypes = [c_char_p]
+        self.__read_internal.restype = c_void_p
+
+        self.__write_internal = self.__lib.write_internal
+        self.__write_internal.argtypes = [c_void_p, c_char_p]
+
         self.__to_json = self.__lib.to_json
         self.__to_json.argtypes = [c_void_p]
         self.__to_json.restype = c_char_p
@@ -168,7 +180,21 @@ def __init__(self, config_string=None, address=None, libpath=LIBPATH):
         self.__destroy = self.__lib.destroy
         self.__destroy.argtypes = [c_void_p]
 
-        if address is None:
+        self.__equal = self.__lib.equal
+        self.__equal.argtypes = [c_void_p, c_void_p]
+        self.__equal.restype = c_bool
+
+        if address is not None:
+            self.__config = address
+            self.__version = ''
+        elif internal is not None:
+            config = self.__read_internal(internal.encode())
+            if config is None:
+                msg = self.__get_error().decode()
+                raise ValueError('Failed to read internal rep: {0}'.format(msg))
+            else:
+                self.__config = config
+        elif config_string is not None:
             config_section, version_section = extract_version(config_string)
             config_section = escape_backslash(config_section)
             config = self.__from_string(config_section.encode())
@@ -179,8 +205,9 @@ def __init__(self, config_string=None, address=None, libpath=LIBPATH):
                 self.__config = config
                 self.__version = version_section
         else:
-            self.__config = address
-            self.__version = ''
+            raise TypeError(
+                "ConfigTree() requires one of 'config_string', 'address', or 'internal'"
+            )
 
         self.__migration = os.environ.get('VYOS_MIGRATION')
         if self.__migration:
@@ -190,6 +217,11 @@ def __del__(self):
         if self.__config is not None:
             self.__destroy(self.__config)
 
+    def __eq__(self, other):
+        if isinstance(other, ConfigTree):
+            return self.__equal(self._get_config(), other._get_config())
+        return False
+
     def __str__(self):
         return self.to_string()
 
@@ -199,6 +231,9 @@ def _get_config(self):
     def get_version_string(self):
         return self.__version
 
+    def write_cache(self, file_name):
+        self.__write_internal(self._get_config(), file_name)
+
     def to_string(self, ordered_values=False, no_version=False):
         config_string = self.__to_string(self.__config, ordered_values).decode()
         config_string = unescape_backslash(config_string)
@@ -488,6 +523,35 @@ def mask_inclusive(left, right, libpath=LIBPATH):
     return tree
 
 
+def show_commit_data(active_tree, proposed_tree, libpath=LIBPATH):
+    if not (
+        isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree)
+    ):
+        raise TypeError('Arguments must be instances of ConfigTree')
+
+    __lib = cdll.LoadLibrary(libpath)
+    __show_commit_data = __lib.show_commit_data
+    __show_commit_data.argtypes = [c_void_p, c_void_p]
+    __show_commit_data.restype = c_char_p
+
+    res = __show_commit_data(active_tree._get_config(), proposed_tree._get_config())
+
+    return res.decode()
+
+
+def test_commit(active_tree, proposed_tree, libpath=LIBPATH):
+    if not (
+        isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree)
+    ):
+        raise TypeError('Arguments must be instances of ConfigTree')
+
+    __lib = cdll.LoadLibrary(libpath)
+    __test_commit = __lib.test_commit
+    __test_commit.argtypes = [c_void_p, c_void_p]
+
+    __test_commit(active_tree._get_config(), proposed_tree._get_config())
+
+
 def reference_tree_to_json(from_dir, to_file, internal_cache='', libpath=LIBPATH):
     try:
         __lib = cdll.LoadLibrary(libpath)

python/vyos/defaults.py

@@ -38,7 +38,8 @@
   'vyos_configdir' : '/opt/vyatta/config',
   'completion_dir' : f'{base_dir}/completion',
   'ca_certificates' : '/usr/local/share/ca-certificates/vyos',
-  'ppp_nexthop_dir' : '/run/ppp_nexthop'
+  'ppp_nexthop_dir' : '/run/ppp_nexthop',
+  'proto_path' : '/usr/share/vyos/vyconf'
 }
 
 systemd_services = {
@@ -69,3 +70,5 @@
 
 rt_global_vrf = rt_symbolic_names['main']
 rt_global_table = rt_symbolic_names['main']
+
+vyconfd_conf = '/etc/vyos/vyconfd.conf'

python/vyos/proto/__init__.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2025 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This script is used to show the commit data of the configuration
+
+import sys
+from pathlib import Path
+from argparse import ArgumentParser
+
+from vyos.config_mgmt import ConfigMgmt
+from vyos.configtree import ConfigTree
+from vyos.configtree import show_commit_data
+
+cm = ConfigMgmt()
+
+parser = ArgumentParser(
+    description='Show commit priority queue; no options compares the last two commits'
+)
+parser.add_argument('--active-config', help='Path to the active configuration file')
+parser.add_argument('--proposed-config', help='Path to the proposed configuration file')
+args = parser.parse_args()
+
+active_arg = args.active_config
+proposed_arg = args.proposed_config
+
+if active_arg and not proposed_arg:
+    print('--proposed-config is required when --active-config is specified')
+    sys.exit(1)
+
+if not active_arg and not proposed_arg:
+    active = cm.get_config_tree_revision(1)
+    proposed = cm.get_config_tree_revision(0)
+else:
+    if active_arg:
+        active = ConfigTree(Path(active_arg).read_text())
+    else:
+        active = cm.get_config_tree_revision(0)
+
+    proposed = ConfigTree(Path(proposed_arg).read_text())
+
+ret = show_commit_data(active, proposed)
+print(ret)

src/helpers/show_commit_data.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2025 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This script is used to test execution of the commit algorithm by vyos-commitd
+
+from pathlib import Path
+from argparse import ArgumentParser
+from datetime import datetime
+
+from vyos.configtree import ConfigTree
+from vyos.configtree import test_commit
+
+
+parser = ArgumentParser(
+    description='Execute commit priority queue'
+)
+parser.add_argument(
+    '--active-config', help='Path to the active configuration file', required=True
+)
+parser.add_argument(
+    '--proposed-config', help='Path to the proposed configuration file', required=True
+)
+args = parser.parse_args()
+
+active_arg = args.active_config
+proposed_arg = args.proposed_config
+
+active = ConfigTree(Path(active_arg).read_text())
+proposed = ConfigTree(Path(proposed_arg).read_text())
+
+
+time_begin_commit = datetime.now()
+test_commit(active, proposed)
+time_end_commit = datetime.now()
+print(f'commit time: {time_end_commit - time_begin_commit}')

src/helpers/test_commit.py

@@ -0,0 +1,27 @@
+[Unit]
+Description=VyOS commit daemon
+
+# Without this option, lots of default dependencies are added,
+# among them network.target, which creates a dependency cycle
+DefaultDependencies=no
+
+# Seemingly sensible way to say "as early as the system is ready"
+# All vyos-configd needs is read/write mounted root
+After=systemd-remount-fs.service
+Before=vyos-router.service
+
+[Service]
+ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/services/vyos-commitd
+Type=idle
+
+SyslogIdentifier=vyos-commitd
+SyslogFacility=daemon
+
+Restart=on-failure
+
+# Does't work in Jessie but leave it here
+User=root
+Group=vyattacfg
+
+[Install]
+WantedBy=vyos.target

src/services/vyos-commitd

@@ -31,11 +31,11 @@ def setUp(self):
     def test_unit(self):
         diff = vyos.configtree.DiffTree(self.config_left, self.config_null)
         sub = diff.sub
-        self.assertEqual(sub.to_string(), self.config_left.to_string())
+        self.assertEqual(sub, self.config_left)
 
         diff = vyos.configtree.DiffTree(self.config_null, self.config_left)
         add = diff.add
-        self.assertEqual(add.to_string(), self.config_left.to_string())
+        self.assertEqual(add, self.config_left)
 
     def test_symmetry(self):
         lr_diff = vyos.configtree.DiffTree(self.config_left,
@@ -45,10 +45,10 @@ def test_symmetry(self):
 
         sub = lr_diff.sub
         add = rl_diff.add
-        self.assertEqual(sub.to_string(), add.to_string())
+        self.assertEqual(sub, add)
         add = lr_diff.add
         sub = rl_diff.sub
-        self.assertEqual(add.to_string(), sub.to_string())
+        self.assertEqual(add, sub)
 
     def test_identity(self):
         lr_diff = vyos.configtree.DiffTree(self.config_left,
@@ -61,6 +61,9 @@ def test_identity(self):
         r_union = vyos.configtree.union(add, inter)
         l_union = vyos.configtree.union(sub, inter)
 
+        # here we must compare string representations instead of using
+        # dunder equal, as we assert equivalence of the values list, which
+        # is optionally ordered at render
         self.assertEqual(r_union.to_string(),
                          self.config_right.to_string(ordered_values=True))
         self.assertEqual(l_union.to_string(),