feat(config): introduce profile-based configuration system

Author: yaugenst-flex

CHANGELOG.md

@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `sort_spec` in `ModeSpec` allows for fine-grained filtering and sorting of modes. This also deprecates `filter_pol`. The equivalent usage for example to `filter_pol="te"` is `sort_spec=ModeSortSpec(filter_key="TE_polarization", filter_reference=0.5)`. `ModeSpec.track_freq` has also been deprecated and moved to `ModeSortSpec.track_freq`.
 - Added `custom_source_time` parameter to `ComponentModeler` classes (`ModalComponentModeler` and `TerminalComponentModeler`), allowing specification of custom source time dependence.
 - Validation for `run_only` field in component modelers to catch duplicate or invalid matrix indices early with clear error messages.
+- Introduced a profile-based configuration manager with TOML persistence and runtime overrides exposed via `tidy3d.config`.
 
 ### Changed
 - Improved performance of antenna metrics calculation by utilizing cached wave amplitude calculations instead of recomputing wave amplitudes for each port excitation in the `TerminalComponentModelerData`.

poetry.lock

@@ -37,6 +37,7 @@ pydantic = "^2.0"
 PyYAML = "*"
 dask = "*"
 toml = "*"
+tomlkit = "^0.13.2"
 autograd = ">=1.7.0"
 scipy = "*"
 ### NOT CORE

pyproject.toml

@@ -0,0 +1,57 @@
+"""Shared fixtures for the configuration test suite."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tidy3d.config.__init__ import get_manager, reload_config
+
+_ENV_VARS_TO_CLEAR = {
+    "TIDY3D_PROFILE",
+    "TIDY3D_CONFIG_PROFILE",
+    "TIDY3D_ENV",
+    "SIMCLOUD_APIKEY",
+    "TIDY3D_AUTH__APIKEY",
+    "TIDY3D_WEB__APIKEY",
+    "TIDY3D_BASE_DIR",
+}
+
+
+@pytest.fixture(autouse=True)
+def clean_env(monkeypatch):
+    """Ensure configuration-related env vars do not leak between tests."""
+
+    original: dict[str, str | None] = {var: os.environ.get(var) for var in _ENV_VARS_TO_CLEAR}
+    for var in _ENV_VARS_TO_CLEAR:
+        monkeypatch.delenv(var, raising=False)
+    try:
+        yield
+    finally:
+        for var, value in original.items():
+            if value is None:
+                monkeypatch.delenv(var, raising=False)
+            else:
+                monkeypatch.setenv(var, value)
+
+
+@pytest.fixture
+def mock_config_dir(tmp_path, monkeypatch):
+    """Point the config system at a temporary directory."""
+
+    base_dir = tmp_path / "config_home"
+    monkeypatch.setenv("TIDY3D_BASE_DIR", str(base_dir))
+    return base_dir / ".tidy3d"
+
+
+@pytest.fixture
+def config_manager(mock_config_dir):
+    """Return a freshly initialized configuration manager."""
+
+    from tidy3d.config import config as config_wrapper
+
+    reload_config(profile="default")
+    config_wrapper.switch_profile("default")
+    manager = get_manager()
+    return manager

tests/config/conftest.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import importlib
+
+from tidy3d.config.__init__ import get_manager, reload_config
+
+
+def test_legacy_logging_level(config_manager):
+    cfg = reload_config(profile=config_manager.profile)
+    cfg.logging_level = "DEBUG"
+    manager = get_manager()
+    assert manager.get_section("logging").level == "DEBUG"
+
+
+def test_env_switch(config_manager):
+    config_module = importlib.import_module("tidy3d.config.__init__")
+    config_module.Env.dev.active()
+    assert get_manager().profile == "dev"
+    config_module.Env.set_current(config_module.Env.prod)
+    assert get_manager().profile == "prod"
+
+
+def test_legacy_wrapper_str(config_manager):
+    from tidy3d.config import config
+
+    text = str(config)
+    assert "Config (profile='default')" in text
+    assert "├── microwave" in text
+    assert "'api_endpoint': 'https://tidy3d-api.simulation.cloud'" in text

tests/config/test_legacy.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from click.testing import CliRunner
+from pydantic import Field
+
+from tidy3d.config import get_manager, reload_config
+from tidy3d.config import registry as config_registry
+from tidy3d.config.sections import ConfigSection
+from tidy3d.web.cli.app import tidy3d_cli
+
+
+def _config_path(config_dir: Path) -> Path:
+    return config_dir / "config.toml"
+
+
+def test_loads_legacy_flat_config(mock_config_dir):
+    legacy_path = mock_config_dir / "config"
+    legacy_path.parent.mkdir(parents=True, exist_ok=True)
+    legacy_path.write_text('apikey = "legacy-key"\n', encoding="utf-8")
+
+    reload_config(profile="default")
+    manager = get_manager()
+    web = manager.get_section("web")
+    assert web.apikey is not None
+    assert web.apikey.get_secret_value() == "legacy-key"
+
+
+def test_save_includes_descriptions(config_manager, mock_config_dir):
+    manager = config_manager
+    manager.save(include_defaults=True)
+
+    content = _config_path(mock_config_dir).read_text(encoding="utf-8")
+    assert "# Web/HTTP configuration." in content
+
+
+def test_preserves_user_comments(config_manager, mock_config_dir):
+    manager = config_manager
+    manager.save(include_defaults=True)
+
+    config_path = _config_path(mock_config_dir)
+    text = config_path.read_text(encoding="utf-8")
+    text = text.replace(
+        "Web/HTTP configuration.",
+        "user-modified comment",
+    )
+    config_path.write_text(text, encoding="utf-8")
+
+    reload_config(profile="default")
+    manager = get_manager()
+    manager.save(include_defaults=True)
+
+    updated = config_path.read_text(encoding="utf-8")
+    assert "user-modified comment" in updated
+    assert "Web/HTTP configuration." not in updated
+
+
+def test_profile_preserves_comments(config_manager, mock_config_dir):
+    @config_registry.register_plugin("profile_comment")
+    class ProfileComment(ConfigSection):
+        """Profile comment plugin."""
+
+        knob: int = Field(
+            1,
+            description="Profile knob description.",
+            json_schema_extra={"persist": True},
+        )
+
+    try:
+        manager = config_manager
+        manager.switch_profile("custom")
+        manager.update_section("plugins.profile_comment", knob=5)
+        manager.save()
+
+        profile_path = mock_config_dir / "profiles" / "custom.toml"
+        text = profile_path.read_text(encoding="utf-8")
+        assert "Profile knob description." in text
+        text = text.replace("Profile knob description.", "user comment")
+        profile_path.write_text(text, encoding="utf-8")
+
+        manager.update_section("plugins.profile_comment", knob=7)
+        manager.save()
+
+        updated = profile_path.read_text(encoding="utf-8")
+        assert "user comment" in updated
+        assert "Profile knob description." not in updated
+    finally:
+        config_registry._SECTIONS.pop("plugins.profile_comment", None)
+        reload_config(profile="default")
+
+
+def test_cli_reset_config(mock_config_dir):
+    @config_registry.register_plugin("cli_comment")
+    class CLIPlugin(ConfigSection):
+        """CLI plugin configuration."""
+
+        knob: int = Field(
+            3,
+            description="CLI knob description.",
+            json_schema_extra={"persist": True},
+        )
+
+    try:
+        reload_config(profile="default")
+        manager = get_manager()
+        manager.update_section("web", apikey="secret")
+        manager.save(include_defaults=True)
+        manager.switch_profile("custom")
+        manager.update_section("plugins.cli_comment", knob=42)
+        manager.save()
+
+        profiles_dir = mock_config_dir / "profiles"
+        assert profiles_dir.exists()
+
+        runner = CliRunner()
+        result = runner.invoke(tidy3d_cli, ["config", "reset", "--yes"])
+        assert result.exit_code == 0, result.output
+
+        config_text = _config_path(mock_config_dir).read_text(encoding="utf-8")
+        assert "Web/HTTP configuration." in config_text
+        assert "[web]" in config_text
+        assert "secret" not in config_text
+        assert not profiles_dir.exists()
+    finally:
+        config_registry._SECTIONS.pop("plugins.cli_comment", None)
+        reload_config(profile="default")
+
+
+def test_plugin_descriptions(mock_config_dir):
+    @config_registry.register_plugin("comment_test")
+    class CommentPlugin(ConfigSection):
+        """Comment plugin configuration."""
+
+        knob: int = Field(
+            3,
+            description="Plugin knob description.",
+            json_schema_extra={"persist": True},
+        )
+
+    try:
+        reload_config(profile="default")
+        manager = get_manager()
+        manager.save(include_defaults=True)
+        content = _config_path(mock_config_dir).read_text(encoding="utf-8")
+        assert "Comment plugin configuration." in content
+        assert "Plugin knob description." in content
+    finally:
+        config_registry._SECTIONS.pop("plugins.comment_test", None)
+        reload_config(profile="default")

tests/config/test_loader.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from tidy3d.config import Env, get_manager, reload_config
+
+
+def test_default_web_settings(config_manager):
+    web = config_manager.get_section("web")
+    assert str(web.api_endpoint) == "https://tidy3d-api.simulation.cloud"
+    assert str(web.website_endpoint) == "https://tidy3d.simulation.cloud"
+    assert web.ssl_verify is True
+
+
+def test_update_section_runtime_overlay(config_manager):
+    config_manager.update_section("logging", level="DEBUG", suppression=False)
+    logging_section = config_manager.get_section("logging")
+    assert logging_section.level == "DEBUG"
+    assert logging_section.suppression is False
+
+
+def test_runtime_isolated_per_profile(config_manager):
+    config_manager.update_section("web", timeout=45)
+    config_manager.switch_profile("customer")
+    assert config_manager.get_section("web").timeout == 120
+    config_manager.switch_profile("default")
+    assert config_manager.get_section("web").timeout == 45
+
+
+def test_environment_variable_precedence(monkeypatch, config_manager):
+    monkeypatch.setenv("TIDY3D_LOGGING__LEVEL", "WARNING")
+    config_manager.switch_profile(config_manager.profile)
+    config_manager.update_section("logging", level="DEBUG")
+    logging_section = config_manager.get_section("logging")
+    # env var should still take precedence
+    assert logging_section.level == "WARNING"
+
+
+@pytest.mark.parametrize("profile", ["dev", "uat"])
+def test_builtin_profiles(profile, config_manager):
+    config_manager.switch_profile(profile)
+    web = config_manager.get_section("web")
+    assert web.s3_region is not None
+
+
+def test_uppercase_profile_normalization(monkeypatch):
+    monkeypatch.setenv("TIDY3D_ENV", "DEV")
+    try:
+        reload_config()
+        manager = get_manager()
+        assert manager.profile == "dev"
+        web = manager.get_section("web")
+        assert str(web.api_endpoint) == "https://tidy3d-api.dev-simulation.cloud"
+        assert Env.current.name == "dev"
+    finally:
+        reload_config(profile="default")
+
+
+def test_adjoint_defaults(config_manager):
+    adjoint = config_manager.get_section("adjoint")
+    assert adjoint.min_wvl_fraction == pytest.approx(5e-2)
+    assert adjoint.points_per_wavelength == 10
+    assert adjoint.monitor_interval_poly == (1, 1, 1)
+    assert adjoint.quadrature_sample_fraction == pytest.approx(0.4)
+    assert adjoint.gauss_quadrature_order == 7
+    assert adjoint.edge_clip_tolerance == pytest.approx(1e-9)
+    assert adjoint.minimum_spacing_fraction == pytest.approx(1e-2)
+    assert adjoint.gradient_precision == "single"
+    assert adjoint.max_traced_structures == 500
+    assert adjoint.max_adjoint_per_fwd == 10
+
+
+def test_adjoint_update_section(config_manager):
+    config_manager.update_section(
+        "adjoint",
+        min_wvl_fraction=0.08,
+        points_per_wavelength=12,
+        solver_freq_chunk_size=3,
+        gradient_precision="double",
+        minimum_spacing_fraction=0.02,
+        gauss_quadrature_order=5,
+        edge_clip_tolerance=2e-9,
+        max_traced_structures=600,
+        max_adjoint_per_fwd=7,
+    )
+    adjoint = config_manager.get_section("adjoint")
+    assert adjoint.min_wvl_fraction == pytest.approx(0.08)
+    assert adjoint.points_per_wavelength == 12
+    assert adjoint.solver_freq_chunk_size == 3
+    assert adjoint.gauss_quadrature_order == 5
+    assert adjoint.edge_clip_tolerance == pytest.approx(2e-9)
+    assert adjoint.minimum_spacing_fraction == pytest.approx(0.02)
+    assert adjoint.gradient_precision == "double"
+    assert adjoint.max_traced_structures == 600
+    assert adjoint.max_adjoint_per_fwd == 7
+
+    assert adjoint.gradient_dtype_float is np.float64
+    assert adjoint.gradient_dtype_complex is np.complex128
+
+
+def test_config_str_formatting(config_manager):
+    text = str(config_manager)
+    assert "Config (profile='default')" in text
+    assert "├── adjoint" in text
+    assert "├── logging" in text
+    assert "└── web" in text
+    assert "'api_endpoint': 'https://tidy3d-api.simulation.cloud'" in text
+    assert "'s3_region': 'us-gov-west-1'" in text
+
+
+def test_section_accessor_str_formatting(config_manager):
+    text = str(config_manager.adjoint)
+    assert "adjoint" in text
+    assert "'gradient_precision': 'single'" in text
+    assert "'monitor_interval_poly': [" in text
+
+
+def test_as_dict_includes_defaults(config_manager):
+    data = config_manager.as_dict()
+    assert "logging" in data
+    assert data["logging"]["level"] == "WARNING"
+    assert "adjoint" in data
+    assert data["adjoint"]["local_adjoint_dir"] == "adjoint_data"
+    assert "simulation" in data