fix/feat: P0 bug fixes, type annotations, docstrings, and more tests

.github/workflows/ci.yml

@@ -46,4 +46,11 @@ jobs:
         run: uv sync --dev
 
       - name: Run unit tests
-        run: uv run pytest tests/ -v --tb=short
+        run: uv run pytest tests/ -v --tb=short --cov=igenbench --cov-report=term-missing --cov-report=xml
+
+      - name: Upload coverage report
+        if: matrix.python-version == '3.12'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.xml

AGENTS.md

@@ -146,5 +146,5 @@ outputs/
 - **Never** add API keys to source files. Use environment variables (see `.env.example`).
 - `VISItem.from_dict(path)` takes a **file path**, not a dict.
 - `EvalEntry.judgments` is a list — multiple (gen_model, eval_model) pairs can coexist on the same question (multi-model comparison).
-- The `Question` dataclass in `vis_item.py` is **deprecated**; use `EvalEntry` instead.
+- The `Question` dataclass in `vis_item.py` is **deprecated** and will be removed in a future version. Migrate to `EvalEntry`: replace `q` → `question`, `q_ground` → `ground`, `q_type` → `question_type`, and use `VISItem.evaluation: list[EvalEntry]` instead of a separate JSONL questions file.
 - When editing CLI commands, register them via `igenbench/cli/main.py` imports (side-effect import pattern).

igenbench/cli/batch_cli.py

@@ -27,7 +27,7 @@ def cmd_batch_gen(
         "--resume/--no-resume",
         help="Resume from existing state, skipping already generated images",
     ),
-):
+) -> None:
     """Batch generate infographic images for all VISItem JSON files in a directory."""
     json_files = sorted(Path(data_dir).glob("*.json"))
 
@@ -79,7 +79,7 @@ def cmd_batch_eval(
         "--resume/--no-resume",
         help="Resume from existing state, skipping already evaluated questions",
     ),
-):
+) -> None:
     """Batch evaluate generated images for all VISItem JSON files in a directory."""
     json_files = sorted(Path(data_dir).glob("*.json"))
 

igenbench/cli/eval_cli.py

@@ -33,7 +33,7 @@ def cmd_run_evaluation(
         "--resume",
         help="Resume from output directory to skip already processed items",
     ),
-):
+) -> None:
     """Run evaluation on a generated image using pre-generated questions.
 
     Evaluates the generated image on all questions in the VISItem.

igenbench/cli/gen_cli.py

@@ -25,7 +25,7 @@ def cmd_gen(
         "--resume",
         help="Resume from output directory to skip already generated images",
     ),
-):
+) -> None:
     """Generate image from text prompt using text-to-image model.
 
     Reads the VISItem and generates an image based on the t2i_prompt field.

igenbench/cli/main.py

@@ -11,7 +11,7 @@
 import igenbench.cli.score_cli  # noqa: E402, F401
 
 
-def main():
+def main() -> None:
     """Entry point for the IGenBench CLI."""
     app()
 

igenbench/cli/score_cli.py

@@ -32,7 +32,7 @@ def cmd_score(
         "--by-type/--no-by-type",
         help="Show accuracy breakdown by question type",
     ),
-):
+) -> None:
     """Aggregate and display evaluation accuracy scores from the output directory."""
     output_path = Path(output_dir)
 

igenbench/engine/eval_engine.py

@@ -49,11 +49,18 @@ def judge_entry(
             model=eval_model, image_path=image_path, prompt=question_judgment_prompt
         )
 
+        if isinstance(response, dict):
+            analysis = response.get("analysis", "")
+            answer = response.get("answer", "")
+        else:
+            analysis = str(response) if response is not None else ""
+            answer = ""
+
         judgment = Judgment(
             eval_model=eval_model,
             gen_model=gen_model,
-            analysis=response.get("analysis", ""),
-            answer=response.get("answer", ""),
+            analysis=analysis,
+            answer=answer,
         )
 
         return judgment

igenbench/engine/gen_engine.py

@@ -1,3 +1,5 @@
+from PIL.Image import Image as PILImage
+
 from igenbench.engine.base_engine import BaseEngine
 from igenbench.utils.llm.client import LLMClient
 from igenbench.vis_item import VISItem
@@ -9,7 +11,7 @@ class GenEngine(BaseEngine):
     def __init__(self, llm_client: LLMClient, model: str):
         super().__init__(llm_client, model)
 
-    def text2image(self, item: VISItem):
+    def text2image(self, item: VISItem) -> PILImage:
         """Generate image from text prompt.
 
         Args:

igenbench/utils/io.py

@@ -100,8 +100,22 @@ def get_model_name_from_image_path(image_path: Path | str) -> str:
     return stem.split("_")[-1]
 
 
-def split_senmantic_and_data_in_t2i_prompt(t2i_prompt: str) -> tuple[str, str]:
-    """
-    Split the semantic and data parts of the T2I prompt.
+def split_semantic_and_data_in_t2i_prompt(t2i_prompt: str) -> tuple[str, str]:
+    """Split a T2I prompt into semantic description and data sections.
+
+    Expects the prompt to contain the separator "The given data is:".
+    Returns a (semantic_part, data_part) tuple.
+
+    Raises:
+        ValueError: If the separator is not found in the prompt.
     """
-    return t2i_prompt.split("The given data is:")
+    if not t2i_prompt:
+        raise ValueError("Prompt cannot be empty or None.")
+    separator = "The given data is:"
+    if separator not in t2i_prompt:
+        raise ValueError(
+            f"Prompt does not contain expected separator '{separator}'. "
+            "Cannot split semantic and data sections."
+        )
+    parts = t2i_prompt.split(separator, maxsplit=1)
+    return parts[0], parts[1]

igenbench/utils/llm/caller_registry.py

@@ -1,13 +1,24 @@
-from typing import Dict, Type, TYPE_CHECKING
+from typing import Callable, Dict, Type, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .llm_caller import LLMCaller
 
 CALLER_REGISTRY: Dict[str, Type["LLMCaller"]] = {}
 
 
-def register_caller(name: str):
-    def wrapper(cls: "LLMCaller") -> "LLMCaller":
+def register_caller(name: str) -> Callable[[Type["LLMCaller"]], Type["LLMCaller"]]:
+    """Class decorator that registers an LLMCaller implementation under *name*.
+
+    Usage::
+
+        @register_caller("google")
+        class GoogleCaller(LLMCaller): ...
+
+    Raises:
+        ValueError: If *name* is already registered.
+    """
+
+    def wrapper(cls: Type["LLMCaller"]) -> Type["LLMCaller"]:
         if name in CALLER_REGISTRY:
             raise ValueError(f"Caller {name} already registered")
         CALLER_REGISTRY[name] = cls

igenbench/utils/llm/client.py

@@ -21,17 +21,38 @@ def _get_caller(self, provider: str) -> LLMCaller:
     def call_text_generation(
         self, model: str, prompt: str, **kwargs: Any
     ) -> Union[dict, str]:
+        """Generate text and parse any fenced code block / JSON in the response.
+
+        Returns:
+            Parsed dict if the response contains valid JSON, otherwise a plain string.
+            Returns an empty string if the provider returns None or an empty response.
+        """
         text_response = self._caller.generate_text(model, prompt, **kwargs)
+        if not text_response:
+            return ""
         return extract_from_markdown(text_response)
 
     def call_image_understanding(
         self, model: str, prompt: str, image_path: str, **kwargs: Any
     ) -> Union[dict, str]:
+        """Analyse an image and parse any fenced code block / JSON in the response.
+
+        Returns:
+            Parsed dict if the response contains valid JSON, otherwise a plain string.
+            Returns an empty string if the provider returns None or an empty response.
+        """
         text_response = self._caller.understand_image(
             model, prompt, image_path, **kwargs
         )
+        if not text_response:
+            return ""
         return extract_from_markdown(text_response)
 
     def call_image_generation(self, model: str, prompt: str, **kwargs: Any) -> PILImage:
+        """Generate an image and return it as a PIL Image.
+
+        Returns:
+            PIL Image object of the generated image.
+        """
         pil_image = self._caller.generate_image(model, prompt, **kwargs)
         return pil_image

igenbench/utils/llm/llm_caller.py

@@ -41,17 +41,54 @@ def base64_to_PILImage(base64_image_url: str) -> PILImage:
 
 
 class LLMCaller:
+    """Abstract base class for LLM provider integrations.
+
+    Subclasses must implement all three methods and register themselves with
+    ``@register_caller("<provider_name>")``.
+    """
+
     def generate_text(self, model: str, prompt: str, **kwargs: Any) -> str:
+        """Generate plain text from a text prompt.
+
+        Args:
+            model: Provider-specific model identifier.
+            prompt: Text prompt.
+            **kwargs: Additional provider-specific parameters.
+
+        Returns:
+            Generated text string.
+        """
         raise NotImplementedError
 
     def understand_image(
         self, model: str, prompt: str, image_path: str, **kwargs: Any
     ) -> str:
+        """Analyse an image and return a text response.
+
+        Args:
+            model: Provider-specific model identifier.
+            prompt: Instruction / question about the image.
+            image_path: Absolute or relative path to the image file.
+            **kwargs: Additional provider-specific parameters.
+
+        Returns:
+            Model's text response.
+        """
         raise NotImplementedError
 
     def generate_image(
         self, model: str, prompt: str, **kwargs: Any
     ) -> GoogleImage | PILImage:
+        """Generate an image from a text prompt.
+
+        Args:
+            model: Provider-specific model identifier.
+            prompt: Text description of the desired image.
+            **kwargs: Additional provider-specific parameters.
+
+        Returns:
+            Generated image as a Google ``Image`` or PIL ``Image`` object.
+        """
         raise NotImplementedError
 
 

igenbench/vis_item.py

@@ -156,7 +156,7 @@ class VISItem:
     chart_type: Optional[str] = None
 
     # generation result: {model_name: image_path}
-    generation: Optional[dict] = field(default_factory=dict)
+    generation: Dict[str, Any] = field(default_factory=dict)
 
     # evaluation result: List[EvalEntry] with source field
     # Each entry has: source ("prompt" or "seed"), ground, question, question_type, judgments

prompts/gen_text2image.py

@@ -1,4 +1,4 @@
-from nanochart.vis_item import VISItem
+from igenbench.vis_item import VISItem
 
 
 def get_prompt_text2image(item: VISItem) -> str:

pyproject.toml

@@ -1,10 +1,33 @@
 [project]
 name = "igenbench"
 version = "0.1.0"
-description = "IGenBench - Benchmark for Infographic Generation"
+description = "IGenBench - Benchmark for evaluating the reliability of text-to-infographic generation"
 readme = "README.md"
 license = {text = "MIT"}
 requires-python = ">=3.10"
+authors = [
+    {name = "MisterBrookT", email = "yinghaotang2001@gmail.com"},
+]
+keywords = [
+    "benchmark",
+    "infographic",
+    "text-to-image",
+    "evaluation",
+    "multimodal",
+    "LLM",
+    "generative-ai",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
 dependencies = [
     "google-genai>=1.49.0",
     "pillow>=12.0.0",
@@ -13,6 +36,11 @@ dependencies = [
     "requests>=2.31.0",
 ]
 
+[project.urls]
+Homepage = "https://github.com/MisterBrookT/IGenBench"
+Repository = "https://github.com/MisterBrookT/IGenBench"
+"Bug Tracker" = "https://github.com/MisterBrookT/IGenBench/issues"
+
 [project.optional-dependencies]
 replicate = ["replicate>=0.34.0"]
 
@@ -37,6 +65,7 @@ dev = [
     "commitizen>=4.10.0",
     "pre-commit>=4.5.0",
     "pytest>=9.0.1",
+    "pytest-cov>=6.0.0",
     "ruff>=0.14.6",
 ]
 

tests/test_caller_registry.py

@@ -0,0 +1,56 @@
+"""Unit tests for the @register_caller decorator and CALLER_REGISTRY."""
+
+import pytest
+
+from igenbench.utils.llm.caller_registry import CALLER_REGISTRY, register_caller
+from igenbench.utils.llm.llm_caller import LLMCaller
+
+
+def test_register_caller_adds_to_registry():
+    name = "_test_provider_add"
+    try:
+
+        @register_caller(name)
+        class _DummyCaller(LLMCaller):
+            pass
+
+        assert name in CALLER_REGISTRY
+        assert CALLER_REGISTRY[name] is _DummyCaller
+    finally:
+        CALLER_REGISTRY.pop(name, None)
+
+
+def test_register_caller_duplicate_raises():
+    name = "_test_provider_dup"
+    try:
+
+        @register_caller(name)
+        class _First(LLMCaller):
+            pass
+
+        with pytest.raises(ValueError, match="already registered"):
+
+            @register_caller(name)
+            class _Second(LLMCaller):
+                pass
+
+    finally:
+        CALLER_REGISTRY.pop(name, None)
+
+
+def test_register_caller_returns_class_unchanged():
+    name = "_test_provider_ret"
+    try:
+
+        @register_caller(name)
+        class _MyProvider(LLMCaller):
+            pass
+
+        assert _MyProvider.__name__ == "_MyProvider"
+    finally:
+        CALLER_REGISTRY.pop(name, None)
+
+
+def test_built_in_providers_registered():
+    for provider in ("google", "openrouter", "replicate"):
+        assert provider in CALLER_REGISTRY, f"{provider} not found in registry"