Add LLM-as-a-judge to eval suite and bump NDD version

eval/llm_judge/cc_extract.py

@@ -0,0 +1,173 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Create Common Crawl rows for the generic text-extraction LLM-judge example.
+
+Example:
+    python eval/llm_judge/cc_extract.py \
+        --download-dir data/cc_warcs --output-path data/cc_extractions
+"""
+
+from __future__ import annotations
+
+import argparse
+from typing import Any
+
+from nemo_curator.backends.ray_data import RayDataExecutor
+from nemo_curator.core.client import RayClient
+from nemo_curator.pipeline import Pipeline
+from nemo_curator.stages.text.download import DocumentDownloadExtractStage, DocumentExtractor
+from nemo_curator.stages.text.download.common_crawl.download import CommonCrawlWARCDownloader
+from nemo_curator.stages.text.download.common_crawl.url_generation import MainCommonCrawlUrlGenerator
+from nemo_curator.stages.text.download.common_crawl.warc_iterator import CommonCrawlWarcIterator
+from nemo_curator.stages.text.download.html_extractors import JusTextExtractor, TrafilaturaExtractor
+from nemo_curator.stages.text.download.html_extractors.utils import get_stop_list_dict
+from nemo_curator.stages.text.download.utils import decode_html, lang_detect
+from nemo_curator.stages.text.io.writer import JsonlWriter
+
+
+OUTPUT_FIELDS = [
+    "url",
+    "warc_id",
+    "source_id",
+    "language",
+    "raw_text",
+    "justext_text",
+    "trafilatura_text",
+]
+
+
+def _extract_text(
+    extractor: JusTextExtractor | TrafilaturaExtractor,
+    html: str,
+    stop_words: frozenset[str],
+    language: str,
+) -> str | None:
+    """Run a Curator HTML extractor and normalize its paragraph output."""
+    paragraphs = extractor.extract_text(html, stop_words, language)
+    return "\n\n".join(paragraphs) if paragraphs else None
+
+
+class JusTextTrafilaturaExtractor(DocumentExtractor):
+    """
+    Preserve decoded HTML and run jusText plus Trafilatura on each WARC record.
+
+    ``raw_text`` is decoded raw HTML, not plain visible-page text;
+    it intentionally gives the LLM judge the source that the two extractors processed.
+    """
+
+    def __init__(self) -> None:
+        self.justext = JusTextExtractor()
+        self.trafilatura = TrafilaturaExtractor()
+        self.stop_lists = get_stop_list_dict()
+
+    def extract(self, record: dict[str, Any]) -> dict[str, Any] | None:
+        html = decode_html(record.get("content", b""))
+        if html is None:
+            return None
+
+        language: str | None = None
+        justext_text: str | None = None
+        trafilatura_text: str | None = None
+        try:
+            language = lang_detect(html)
+            stop_words = self.stop_lists.get(language)
+            if stop_words is not None:
+                justext_text = _extract_text(self.justext, html, stop_words, language)
+                trafilatura_text = _extract_text(self.trafilatura, html, stop_words, language)
+        except Exception:  # noqa: BLE001
+            # Keep the raw HTML row even when language detection or one extractor fails.
+            pass
+
+        return {
+            "url": record["url"],
+            "warc_id": record["warc_id"],
+            "source_id": record["source_id"],
+            "language": language,
+            "raw_text": html,
+            "justext_text": justext_text,
+            "trafilatura_text": trafilatura_text,
+        }
+
+    def input_columns(self) -> list[str]:
+        return ["url", "warc_id", "source_id", "content"]
+
+    def output_columns(self) -> list[str]:
+        return OUTPUT_FIELDS
+
+
+def build_pipeline(args: argparse.Namespace) -> Pipeline:
+    """Build Common Crawl download -> dual extraction -> JSONL writer."""
+    url_generator = MainCommonCrawlUrlGenerator(
+        start_snapshot_str=args.start_snapshot,
+        end_snapshot_str=args.end_snapshot,
+        limit=args.url_limit,
+    )
+
+    cc_extract = DocumentDownloadExtractStage(
+        url_generator=url_generator,
+        downloader=CommonCrawlWARCDownloader(
+            download_dir=args.download_dir,
+            use_aws_to_download=args.use_aws_to_download,
+            verbose=args.verbose,
+        ),
+        iterator=CommonCrawlWarcIterator(),
+        extractor=JusTextTrafilaturaExtractor(),
+        url_limit=args.url_limit,
+        record_limit=args.record_limit,
+        add_filename_column=False,
+        # jusText uses lxml and benefits from Curator's worker recycling.
+        extractor_max_calls_per_worker=args.extractor_max_calls_per_worker,
+    )
+    return Pipeline(
+        name="common_crawl_extraction_comparison",
+        description="Download Common Crawl WARC files and compare jusText with Trafilatura extraction.",
+        stages=[cc_extract, JsonlWriter(path=args.output_path, fields=OUTPUT_FIELDS)],
+    )
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "--start-snapshot",
+        default="2026-30",
+        help="CC-MAIN snapshot in YYYY-WW format (default: 2026-30).",
+    )
+    parser.add_argument(
+        "--end-snapshot",
+        default="2026-30",
+        help="CC-MAIN snapshot in YYYY-WW format (default: 2026-30).",
+    )
+    parser.add_argument("--download-dir", required=True, help="Local directory for downloaded WARC files.")
+    parser.add_argument("--output-path", required=True, help="Directory for JSONL output partitions.")
+    parser.add_argument("--url-limit", type=int, default=1, help="Maximum WARC files to download (default: 1).")
+    parser.add_argument("--record-limit", type=int, default=100, help="Maximum records per WARC file (default: 100).")
+    parser.add_argument("--extractor-max-calls-per-worker", type=int, default=2)
+    parser.add_argument("--use-aws-to-download", action="store_true", help="Use s5cmd against Common Crawl S3.")
+    parser.add_argument("--verbose", action="store_true", help="Show Common Crawl downloader output.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = _parse_args()
+    client = RayClient()
+    client.start()
+    try:
+        build_pipeline(args).run(executor=RayDataExecutor())
+    finally:
+        client.stop()
+
+
+if __name__ == "__main__":
+    main()

eval/llm_judge/examples/text_extraction_disagreement_prompt.jinja

@@ -0,0 +1,16 @@
+Compare the two text-extraction candidates below. Focus on meaningful document
+content, not formatting or harmless whitespace differences. Identify whether
+either candidate loses, invents, or substantially changes content relative to
+the other and to the raw source.
+
+<raw_text>
+{{ (raw_text or "")[:12000] }}
+</raw_text>
+
+<justext_text>
+{{ (justext_text or "")[:8000] }}
+</justext_text>
+
+<trafilatura_text>
+{{ (trafilatura_text or "")[:8000] }}
+</trafilatura_text>

eval/llm_judge/examples/text_extraction_judge.yaml

@@ -0,0 +1,94 @@
+# User-owned evaluation semantics: replace prompts, rubrics, models, and execution mode for another task.
+models:
+  - alias: judge
+    model: /raid/syurick/hf_cache/Qwen3.8-27B
+    # Keep the Hugging Face identifier as the API name while serving local weights.
+    served_model_name: Qwen/Qwen3.8-27B
+    # Passed directly to Curator's local Dynamo + vLLM InferenceServer.
+    # This is the 4-GPU baseline used by the document-translation pipelines.
+    dynamo_model:
+      num_replicas: 4
+      mode: aggregated
+      engine_kwargs:
+        tensor_parallel_size: 1
+        max_model_len: 32768
+        max_num_seqs: 256
+        gpu_memory_utilization: 0.8
+        enforce_eager: true
+    inference_parameters:
+      temperature: 0.0
+      # Structured judging needs the final JSON, not Qwen's private deliberation.
+      extra_body:
+        chat_template_kwargs:
+          enable_thinking: false
+      max_tokens: 512
+      max_parallel_requests: 32
+
+dynamo_server:
+  subprocess_env:
+    DYN_SYSTEM_PORT: "0"
+
+execution:
+  # single_stage: all judge groups run as one NDD dependency graph.
+  # multi_stage: each group is a separate NDD stage in the same Curator pipeline.
+  mode: single_stage
+  stages:
+    - name: extraction_quality
+      judges:
+        - name: qwen3_8_27b_text_extraction_judgment
+          model_alias: judge
+          system_prompt_path: text_extraction_system.jinja
+          prompt_path: text_extraction_prompt.jinja
+          with_trace: last_message
+          extract_reasoning_content: false
+          scores:
+            - name: best_extraction
+              description: Select the candidate that is most useful as clean document text.
+              options:
+                justext: jusText best preserves the meaningful source content with minimal boilerplate.
+                trafilatura: Trafilatura best preserves the meaningful source content with minimal boilerplate.
+                raw: The raw text is the most useful representation.
+                none: None of the supplied candidates is a useful clean-text extraction.
+            - name: content_fidelity
+              description: Assess whether the best extraction preserves the source's meaningful content.
+              options:
+                1: Important content is missing or materially distorted.
+                2: Significant omissions or distortions are present.
+                3: Most meaningful content is preserved, with noticeable issues.
+                4: Meaningful content is well preserved, with minor issues.
+                5: Meaningful content is comprehensively and faithfully preserved.
+            - name: boilerplate_removal
+              description: Assess how effectively the best extraction removes navigation, ads, repeated chrome, and other non-content text.
+              options:
+                1: Boilerplate dominates the extraction.
+                2: Significant boilerplate remains.
+                3: Some noticeable boilerplate remains.
+                4: Only minor boilerplate remains.
+                5: Boilerplate is effectively removed.
+    - name: semantic_disagreement
+      # In multi_stage this mapping may set a stage-specific Curator runtime_env.
+      # runtime_env:
+      #   env_vars:
+      #     SOME_STAGE_SETTING: value
+      judges:
+        - name: qwen3_8_27b_text_extraction_disagreement
+          model_alias: judge
+          system_prompt_path: text_extraction_system.jinja
+          prompt_path: text_extraction_disagreement_prompt.jinja
+          scores:
+            - name: semantic_disagreement
+              description: Assess whether the two extraction candidates differ materially in meaningful content.
+              options:
+                1: The candidates preserve essentially the same meaningful content.
+                2: The candidates have small meaningful-content differences.
+                3: The candidates have noticeable meaningful-content differences.
+                4: The candidates have major meaningful-content differences.
+                5: The candidates disagree so substantially that manual review is needed.
+
+# Optional: filters are placed automatically after the NDD stage that produces
+# their `judge` column (or after the one combined NDD stage in single_stage).
+# filters:
+#   - judge: qwen3_8_27b_text_extraction_judgment
+#     score: content_fidelity
+#     operator: gte
+#     value: 4

eval/llm_judge/examples/text_extraction_prompt.jinja

@@ -0,0 +1,15 @@
+Compare two web-text extraction candidates against their raw source text.
+
+Choose the candidate that best preserves meaningful article or document content while removing navigation, advertising, repeated page chrome, cookie notices, and other boilerplate. Treat the raw text as evidence, not automatically as the desired output.
+
+<raw_text>
+{{ (raw_text or "")[:12000] }}
+</raw_text>
+
+<justext_text>
+{{ (justext_text or "")[:8000] }}
+</justext_text>
+
+<trafilatura_text>
+{{ (trafilatura_text or "")[:8000] }}
+</trafilatura_text>

eval/llm_judge/examples/text_extraction_system.jinja

@@ -0,0 +1 @@
+You are a careful evaluator of web-text extraction quality. Use only the supplied source and candidate texts. Do not infer source content that is not present.