Reapply "config loading: pass discovery rulesets explicitly"

This reverts commit ddc78be.

Change-Id: I4d3aececbfdaf029fae5b04dd415b6b85d1c2bab

bin/check_mk

@@ -45,6 +45,10 @@ from cmk.utils.log import console
 
 import cmk.base.utils
 from cmk.base import config, profiling
+from cmk.base.api.agent_based.register import (
+    extract_known_discovery_rulesets,
+    get_previously_loaded_plugins,
+)
 from cmk.base.modes import modes
 
 from cmk import trace
@@ -144,7 +148,7 @@ try:
     # certain operation modes that does not need them and should not be harmed
     # by a broken configuration
     if mode_name not in modes.non_config_options():
-        config.load()
+        config.load(extract_known_discovery_rulesets(get_previously_loaded_plugins()))
 
     done, exit_status = False, 0
     if mode_name is not None and mode_args is not None:

bin/cmk-validate-config

@@ -9,6 +9,10 @@ from cmk.utils import paths
 from cmk.utils.redis import disable_redis
 
 from cmk.base import config
+from cmk.base.api.agent_based.register import (
+    extract_known_discovery_rulesets,
+    get_previously_loaded_plugins,
+)
 
 from cmk.gui import main_modules
 from cmk.gui.utils.script_helpers import gui_context
@@ -24,9 +28,10 @@ def main() -> int:
                 local_checks_dir=paths.local_checks_dir,
                 checks_dir=paths.checks_dir,
             )
+            plugins = get_previously_loaded_plugins()
             # Watch out: always load the plugins before loading the config.
             # The validation step will not be executed otherwise.
-            config.load()
+            config.load(discovery_rulesets=extract_known_discovery_rulesets(plugins))
 
             result = validate_mk_files()
 

cmk/base/api/agent_based/register/__init__.py

@@ -5,11 +5,9 @@
 
 from ._config import (
     AgentBasedPlugins,
-    get_discovery_ruleset,
-    get_host_label_ruleset,
+    extract_known_discovery_rulesets,
+    get_previously_collected_discovery_rules,
     get_previously_loaded_plugins,
-    is_stored_ruleset,
-    iter_all_discovery_rulesets,
     set_discovery_ruleset,
 )
 from ._discover import load_all_plugins, load_selected_plugins
@@ -18,13 +16,11 @@
 
 __all__ = [
     "AgentBasedPlugins",
+    "extract_known_discovery_rulesets",
     "get_check_plugin",
-    "get_discovery_ruleset",
-    "get_host_label_ruleset",
     "get_previously_loaded_plugins",
+    "get_previously_collected_discovery_rules",
     "filter_relevant_raw_sections",
-    "is_stored_ruleset",
-    "iter_all_discovery_rulesets",
     "load_all_plugins",
     "load_selected_plugins",
     "sections_needing_redetection",

cmk/base/api/agent_based/register/_config.py

@@ -3,7 +3,7 @@
 # This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
 # conditions defined in the file COPYING, which is part of this source code package.
 
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Collection, Mapping, Sequence
 from dataclasses import dataclass
 
 from cmk.utils.rulesets import RuleSetName
@@ -56,12 +56,24 @@ def get_previously_loaded_plugins() -> AgentBasedPlugins:
     )
 
 
-def add_check_plugin(check_plugin: CheckPlugin) -> None:
-    registered_check_plugins[check_plugin.name] = check_plugin
+def extract_known_discovery_rulesets(plugins: AgentBasedPlugins) -> Collection[RuleSetName]:
+    return {
+        r
+        for r in (
+            *(p.discovery_ruleset_name for p in plugins.check_plugins.values()),
+            *(p.host_label_ruleset_name for p in plugins.agent_sections.values()),
+            *(p.host_label_ruleset_name for p in plugins.snmp_sections.values()),
+        )
+        if r is not None
+    }
 
 
-def add_discovery_ruleset(ruleset_name: RuleSetName) -> None:
-    stored_rulesets.setdefault(ruleset_name, [])
+def get_previously_collected_discovery_rules() -> Mapping[RuleSetName, Sequence[RuleSpec]]:
+    return stored_rulesets
+
+
+def add_check_plugin(check_plugin: CheckPlugin) -> None:
+    registered_check_plugins[check_plugin.name] = check_plugin
 
 
 def add_inventory_plugin(inventory_plugin: InventoryPlugin) -> None:
@@ -75,16 +87,6 @@ def add_section_plugin(section_plugin: SectionPlugin) -> None:
         registered_snmp_sections[section_plugin.name] = section_plugin
 
 
-def get_discovery_ruleset(ruleset_name: RuleSetName) -> Sequence[RuleSpec]:
-    """Returns all rulesets of a given name"""
-    return stored_rulesets.get(ruleset_name, [])
-
-
-def get_host_label_ruleset(ruleset_name: RuleSetName) -> Sequence[RuleSpec]:
-    """Returns all rulesets of a given name"""
-    return stored_rulesets.get(ruleset_name, [])
-
-
 def get_inventory_plugin(plugin_name: InventoryPluginName) -> InventoryPlugin | None:
     """Returns the registered inventory plug-in"""
     return registered_inventory_plugins.get(plugin_name)
@@ -102,14 +104,6 @@ def is_registered_section_plugin(section_name: SectionName) -> bool:
     return section_name in registered_snmp_sections or section_name in registered_agent_sections
 
 
-def is_stored_ruleset(ruleset_name: RuleSetName) -> bool:
-    return ruleset_name in stored_rulesets
-
-
-def iter_all_discovery_rulesets() -> Iterable[RuleSetName]:
-    return stored_rulesets.keys()
-
-
 def set_discovery_ruleset(
     ruleset_name: RuleSetName,
     rules: Sequence[RuleSpec],

cmk/base/api/agent_based/register/_discover.py

@@ -31,7 +31,6 @@
 
 from ._config import (
     add_check_plugin,
-    add_discovery_ruleset,
     add_inventory_plugin,
     add_section_plugin,
     get_inventory_plugin,
@@ -129,8 +128,6 @@ def register_agent_section(
         raise ValueError(f"duplicate section definition: {section_plugin.name}")
 
     add_section_plugin(section_plugin)
-    if section_plugin.host_label_ruleset_name is not None:
-        add_discovery_ruleset(section_plugin.host_label_ruleset_name)
 
 
 def register_snmp_section(
@@ -150,8 +147,6 @@ def register_snmp_section(
         raise ValueError(f"duplicate section definition: {section_plugin.name}")
 
     add_section_plugin(section_plugin)
-    if section_plugin.host_label_ruleset_name is not None:
-        add_discovery_ruleset(section_plugin.host_label_ruleset_name)
 
 
 def register_check_plugin(check: CheckPlugin, location: PluginLocation) -> None:
@@ -185,8 +180,6 @@ def register_check_plugin(check: CheckPlugin, location: PluginLocation) -> None:
         raise ValueError(f"duplicate check plug-in definition: {plugin.name}")
 
     add_check_plugin(plugin)
-    if plugin.discovery_ruleset_name is not None:
-        add_discovery_ruleset(plugin.discovery_ruleset_name)
 
 
 def register_inventory_plugin(inventory: InventoryPlugin, location: PluginLocation) -> None:

cmk/base/automation_helper/_app.py

@@ -27,6 +27,10 @@
 from cmk.automations.results import ABCAutomationResult
 
 from cmk.base import config
+from cmk.base.api.agent_based.register import (
+    extract_known_discovery_rulesets,
+    get_previously_loaded_plugins,
+)
 from cmk.base.automations import AutomationError
 
 from ._cache import Cache, CacheError
@@ -41,7 +45,9 @@ class HealthCheckResponse(BaseModel, frozen=True):
 
 def reload_automation_config() -> None:
     cache_manager.clear()
-    config.load(validate_hosts=False)
+    plugins = get_previously_loaded_plugins()
+    discovery_rulesets = extract_known_discovery_rulesets(plugins)
+    config.load(discovery_rulesets, validate_hosts=False)
 
 
 def clear_caches_before_each_call() -> None:
@@ -93,6 +99,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
             local_checks_dir=paths.local_checks_dir, checks_dir=paths.checks_dir
         )
         tty.reinit()
+
         reload_config()
 
         reloader_task = asyncio.create_task(

cmk/base/automations/__init__.py

@@ -22,6 +22,10 @@
 from cmk.automations.results import ABCAutomationResult
 
 from cmk.base import config, profiling
+from cmk.base.api.agent_based.register import (
+    extract_known_discovery_rulesets,
+    get_previously_loaded_plugins,
+)
 
 from cmk import trace
 
@@ -120,8 +124,11 @@ def _execute(
                     )
 
             if not called_from_automation_helper and automation.needs_config:
+                discovery_rulesets = extract_known_discovery_rulesets(
+                    get_previously_loaded_plugins()
+                )
                 with tracer.span("load_config"):
-                    config.load(validate_hosts=False)
+                    config.load(discovery_rulesets, validate_hosts=False)
 
             with tracer.span(f"execute_automation[{cmd}]"):
                 result = automation.execute(args, called_from_automation_helper)

cmk/base/automations/check_mk.py

@@ -287,6 +287,7 @@ def execute(
 
         config_cache = config.get_config_cache()
         plugins = agent_based_register.get_previously_loaded_plugins()
+        discovery_rules = agent_based_register.get_previously_collected_discovery_rules()
         ruleset_matcher = config_cache.ruleset_matcher
         autochecks_config = config.AutochecksConfigurer(config_cache, plugins.check_plugins)
 
@@ -354,9 +355,12 @@ def section_error_handling(
                 host_label_plugins=HostLabelPluginMapper(
                     ruleset_matcher=ruleset_matcher,
                     sections={**plugins.agent_sections, **plugins.snmp_sections},
+                    discovery_rules=discovery_rules,
                 ),
                 plugins=DiscoveryPluginMapper(
-                    ruleset_matcher=ruleset_matcher, check_plugins=plugins.check_plugins
+                    ruleset_matcher=ruleset_matcher,
+                    check_plugins=plugins.check_plugins,
+                    discovery_rules=discovery_rules,
                 ),
                 autochecks_config=autochecks_config,
                 settings=settings,
@@ -644,6 +648,7 @@ def _execute_discovery(
     config_cache = config.get_config_cache()
     hosts_config = config.make_hosts_config()
     plugins = agent_based_register.get_previously_loaded_plugins()
+    discovery_rules = agent_based_register.get_previously_collected_discovery_rules()
     ruleset_matcher = config_cache.ruleset_matcher
     autochecks_config = config.AutochecksConfigurer(config_cache, plugins.check_plugins)
     parser = CMKParser(
@@ -691,13 +696,16 @@ def _execute_discovery(
             host_label_plugins=HostLabelPluginMapper(
                 ruleset_matcher=ruleset_matcher,
                 sections={**plugins.agent_sections, **plugins.snmp_sections},
+                discovery_rules=discovery_rules,
             ),
             check_plugins=check_plugins,
             compute_check_parameters=_make_compute_check_parameters_of_autocheck(
                 ruleset_matcher, plugins.check_plugins
             ),
             discovery_plugins=DiscoveryPluginMapper(
-                ruleset_matcher=ruleset_matcher, check_plugins=plugins.check_plugins
+                ruleset_matcher=ruleset_matcher,
+                check_plugins=plugins.check_plugins,
+                discovery_rules=discovery_rules,
             ),
             autochecks_config=autochecks_config,
             enforced_services=config_cache.enforced_services_table(
@@ -747,10 +755,11 @@ def _execute_autodiscovery(
     if not (autodiscovery_queue := AutoQueue(autodiscovery_dir)):
         return {}, False
 
+    ab_plugins = agent_based_register.get_previously_loaded_plugins()
+    discovery_rules = agent_based_register.get_previously_collected_discovery_rules()
     if not called_from_automation_helper:
-        config.load()
+        config.load(discovery_rules)
     config_cache = config.get_config_cache()
-    ab_plugins = agent_based_register.get_previously_loaded_plugins()
     autochecks_config = config.AutochecksConfigurer(config_cache, ab_plugins.check_plugins)
     ip_address_of = config.ConfiguredIPLookup(
         config_cache,
@@ -787,9 +796,12 @@ def _execute_autodiscovery(
     host_label_plugins = HostLabelPluginMapper(
         ruleset_matcher=ruleset_matcher,
         sections={**ab_plugins.agent_sections, **ab_plugins.snmp_sections},
+        discovery_rules=discovery_rules,
     )
     plugins = DiscoveryPluginMapper(
-        ruleset_matcher=ruleset_matcher, check_plugins=ab_plugins.check_plugins
+        ruleset_matcher=ruleset_matcher,
+        check_plugins=ab_plugins.check_plugins,
+        discovery_rules=discovery_rules,
     )
     on_error = OnError.IGNORE
 
@@ -1454,7 +1466,7 @@ def execute(
         servicedesc = args[1]
         config_cache = config.get_config_cache()
         config_cache.ruleset_matcher.ruleset_optimizer.set_all_processed_hosts({host_name})
-        plugins = get_previously_loaded_plugins()
+        plugins = agent_based_register.get_previously_loaded_plugins()
         return (
             AnalyseServiceResult(
                 service_info=found.service_info,
@@ -1971,7 +1983,7 @@ def execute(
         # that could be too much for the command line
         variable_names = ast.literal_eval(sys.stdin.read())
 
-        config.load(with_conf_d=False)
+        config.load(discovery_rulesets=(), with_conf_d=False)
 
         missing_variables = [v for v in variable_names if not hasattr(config, v)]
 
@@ -1980,7 +1992,11 @@ def execute(
                 local_checks_dir=local_checks_dir,
                 checks_dir=checks_dir,
             )
-            config.load(with_conf_d=False)
+            plugins = agent_based_register.get_previously_loaded_plugins()
+            config.load(
+                discovery_rulesets=agent_based_register.extract_known_discovery_rulesets(plugins),
+                with_conf_d=False,
+            )
 
         result = {}
         for varname in variable_names:

cmk/base/checkers.py

@@ -500,10 +500,12 @@ def __init__(
         *,
         ruleset_matcher: RulesetMatcher,
         sections: Mapping[SectionName, AgentSectionPluginAPI | SNMPSectionPluginAPI],
+        discovery_rules: Mapping[RuleSetName, Sequence[RuleSpec]],
     ) -> None:
         super().__init__()
         self.ruleset_matcher: Final = ruleset_matcher
         self._sections = sections
+        self._discovery_rules: Final = discovery_rules
 
     def __getitem__(self, __key: SectionName) -> HostLabelPlugin:
         plugin = self._sections.get(__key)
@@ -516,7 +518,7 @@ def __getitem__(self, __key: SectionName) -> HostLabelPlugin:
                     default_parameters=plugin.host_label_default_parameters,
                     ruleset_name=plugin.host_label_ruleset_name,
                     ruleset_type=plugin.host_label_ruleset_type,
-                    rules_getter_function=agent_based_register.get_host_label_ruleset,
+                    rules_getter_function=lambda n: self._discovery_rules.get(n, []),
                 ),
             )
             if plugin is not None
@@ -1038,10 +1040,12 @@ def __init__(
         *,
         ruleset_matcher: RulesetMatcher,
         check_plugins: Mapping[CheckPluginName, CheckPluginAPI],
+        discovery_rules: Mapping[RuleSetName, Sequence[RuleSpec]],
     ) -> None:
         super().__init__()
         self.ruleset_matcher: Final = ruleset_matcher
         self._check_plugins: Final = check_plugins
+        self._discovery_rules: Final = discovery_rules
 
     def __getitem__(self, __key: CheckPluginName) -> DiscoveryPlugin:
         # `get_check_plugin()` is not an error.  Both check plug-ins and
@@ -1074,7 +1078,7 @@ def __discovery_function(
                 default_parameters=plugin.discovery_default_parameters,
                 ruleset_name=plugin.discovery_ruleset_name,
                 ruleset_type=plugin.discovery_ruleset_type,
-                rules_getter_function=agent_based_register.get_discovery_ruleset,
+                rules_getter_function=lambda n: self._discovery_rules.get(n, []),
             ),
         )