diff --git a/core/cli/main.py b/core/cli/main.py
index 204e790f2..7299ed30e 100644
--- a/core/cli/main.py
+++ b/core/cli/main.py
@@ -88,6 +88,7 @@ async def handler(*args, **kwargs):
                 log.warning(f"API check for {llm_config.provider.value} failed.")
             else:
                 log.info(f"API check for {llm_config.provider.value} succeeded.")
+                checked_llms.add(llm_config.provider)
         except APIError as err:
             await ui.send_message(
                 f"API check for {llm_config.provider.value} failed with: {err}",