From 5ab5248afacae75685698548856bdcf26893367e Mon Sep 17 00:00:00 2001 From: Yuandong Zhang <107053257+Ydz0616@users.noreply.github.com> Date: Mon, 27 Apr 2026 21:10:17 -0700 Subject: [PATCH 1/2] Decouple RFOpenAIAPIModelConfig and RFGeminiAPIModelConfig from vLLM availability Previously, all three evals model config classes (RFvLLMModelConfig, RFOpenAIAPIModelConfig, RFGeminiAPIModelConfig) were defined inside a single `if _VLLM_AVAILABLE and _EVALS_MODULES_AVAILABLE` guard. As a result, on platforms where vLLM cannot be installed (e.g., macOS Apple Silicon, CPU-only Datahub pods), all three classes would silently become `None`, even though RFOpenAIAPIModelConfig and RFGeminiAPIModelConfig only call remote APIs and have no actual vLLM dependency. Split the guards: RFvLLMModelConfig stays gated on _VLLM_AVAILABLE (since it genuinely needs SamplingParams), while the API configs only require _EVALS_MODULES_AVAILABLE (LangChainRagSpec, PromptManager, OpenAI/Gemini inference engines). Verified locally on macOS Apple Silicon (vllm not installed): RFvLLMModelConfig: None # correct RFOpenAIAPIModelConfig: # now usable RFGeminiAPIModelConfig: # now usable --- rapidfireai/automl/model_config.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/rapidfireai/automl/model_config.py b/rapidfireai/automl/model_config.py index 94c74be1..f05cd3d7 100644 --- a/rapidfireai/automl/model_config.py +++ b/rapidfireai/automl/model_config.py @@ -293,7 +293,8 @@ def __setattr__(self, name, value): RFPromptManager = None -# Conditionally define evals model config classes only if dependencies are available +# RFvLLMModelConfig requires vLLM (a self-hosted local inference engine). +# vLLM is unavailable on macOS Apple Silicon and other CPU-only platforms. if ( _VLLM_AVAILABLE and _EVALS_MODULES_AVAILABLE @@ -357,6 +358,17 @@ def sampling_params_to_dict(self) -> dict[str, Any]: # This works across different vLLM versions return dict(vars(self.sampling_params)) +else: + RFvLLMModelConfig = None + + +# RFOpenAIAPIModelConfig and RFGeminiAPIModelConfig only require the evals modules +# (LangChainRagSpec, PromptManager, OpenAIInferenceEngine / GoogleGeminiInferenceEngine). +# They do NOT require vLLM because OpenAI and Gemini are remote APIs and never run a +# local inference engine. Gating them behind _VLLM_AVAILABLE makes them unusable on +# CPU-only platforms (macOS Apple Silicon, Datahub CPU pods) for no reason. +if _EVALS_MODULES_AVAILABLE and InferenceEngine is not None: + class RFOpenAIAPIModelConfig(ModelConfig): """OpenAI API model configuration for evals mode.""" @@ -514,7 +526,6 @@ def sampling_params_to_dict(self) -> dict[str, Any]: return {k: v for k, v in self.model_config.items() if k not in _non_sampling_keys} else: - # Define placeholder classes if dependencies are not available - RFvLLMModelConfig = None + # Evals modules unavailable: API configs cannot be defined. RFOpenAIAPIModelConfig = None RFGeminiAPIModelConfig = None From b6e9d8baa62225e7c0136ebfddce416c80f95454 Mon Sep 17 00:00:00 2001 From: Yuandong Zhang <107053257+Ydz0616@users.noreply.github.com> Date: Thu, 30 Apr 2026 20:41:57 -0700 Subject: [PATCH 2/2] Guard isinstance() calls against None API config classes The previous fix decoupled RFOpenAIAPIModelConfig and RFGeminiAPIModelConfig from vLLM availability, but RFvLLMModelConfig is still None on platforms where vLLM isn't installable (macOS Apple Silicon, etc.). Several existing isinstance() call sites use these classes directly, which raises TypeError: 'isinstance() arg 2 must be a type, a tuple of types, or a union' when the class is None. Guard each call site by checking 'cls is not None' before isinstance(), or by filtering None out of class tuples. Sites fixed: - evals/utils/serialize.py:96 (extract_pipeline_config_json) - evals/scheduling/interactive_control.py:373 (Clone-Modify pipeline_type inference) - evals/scheduling/interactive_control.py:579 (model name extraction) Verified end-to-end on macOS Apple Silicon: experiment.run_evals() now proceeds past pipeline registration without TypeError. --- .../evals/scheduling/interactive_control.py | 15 +++++++++++---- rapidfireai/evals/utils/serialize.py | 4 +++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/rapidfireai/evals/scheduling/interactive_control.py b/rapidfireai/evals/scheduling/interactive_control.py index ea6e4eb4..9ffc7cc5 100644 --- a/rapidfireai/evals/scheduling/interactive_control.py +++ b/rapidfireai/evals/scheduling/interactive_control.py @@ -370,11 +370,14 @@ def _handle_clone( pipeline_type = edited_json.get("pipeline_type") if not pipeline_type: # If not specified in JSON, infer from parent - if isinstance(parent_model_config, RFvLLMModelConfig): + # Each isinstance() guarded because the class is None on platforms + # where the corresponding inference engine isn't available + # (e.g. RFvLLMModelConfig is None on macOS Apple Silicon). + if RFvLLMModelConfig is not None and isinstance(parent_model_config, RFvLLMModelConfig): pipeline_type = "vllm" - elif isinstance(parent_model_config, RFOpenAIAPIModelConfig): + elif RFOpenAIAPIModelConfig is not None and isinstance(parent_model_config, RFOpenAIAPIModelConfig): pipeline_type = "openai" - elif isinstance(parent_model_config, RFGeminiAPIModelConfig): + elif RFGeminiAPIModelConfig is not None and isinstance(parent_model_config, RFGeminiAPIModelConfig): pipeline_type = "gemini" else: raise ValueError("Cannot determine pipeline type from parent") @@ -576,7 +579,11 @@ def _handle_clone( pipeline = pipeline_config["pipeline"] # Extract model name - if isinstance(model_config, (RFOpenAIAPIModelConfig, RFGeminiAPIModelConfig, RFvLLMModelConfig)): + # Filter out classes that are None on this platform (vLLM unavailable on macOS, etc.) + _model_cfg_classes = tuple( + c for c in (RFOpenAIAPIModelConfig, RFGeminiAPIModelConfig, RFvLLMModelConfig) if c is not None + ) + if _model_cfg_classes and isinstance(model_config, _model_cfg_classes): model_name = model_config.model_config.get("model", "Unknown") elif hasattr(pipeline, "model_config") and pipeline.model_config is not None: model_name = pipeline.model_config.get("model", "Unknown") diff --git a/rapidfireai/evals/utils/serialize.py b/rapidfireai/evals/utils/serialize.py index 27d379d3..0afc2361 100644 --- a/rapidfireai/evals/utils/serialize.py +++ b/rapidfireai/evals/utils/serialize.py @@ -93,7 +93,9 @@ def extract_rag_params(rag_spec): return rag_config if rag_config else None - if isinstance(pipeline, RFvLLMModelConfig): + # RFvLLMModelConfig is None on platforms where vLLM cannot be installed + # (e.g., macOS Apple Silicon). Guard isinstance() to avoid TypeError. + if RFvLLMModelConfig is not None and isinstance(pipeline, RFvLLMModelConfig): json_config["pipeline_type"] = "vllm" # Extract model_config (dict)