feat: add sayf-eval results-record converter

every_eval_ever/cli.py

@@ -466,6 +466,63 @@ def _cmd_convert_alpaca_eval(args: argparse.Namespace) -> int:
     return 0
 
 
+def _cmd_convert_sayf_eval(args: argparse.Namespace) -> int:
+    from every_eval_ever.converters.sayf_eval.adapter import SayfEvalAdapter
+
+    adapter = SayfEvalAdapter()
+    metadata = _common_metadata(args)
+    # The shared arg loop defaults --eval_library_name to the source id
+    # ('sayf_eval'); prefer the canonical package name unless overridden.
+    if args.eval_library_name == 'sayf_eval':
+        metadata['eval_library_name'] = 'sayf-eval'
+    # Route each task into its own namespaced collection (data/<prefix><task>/...)
+    # so the datastore has one collection per benchmark, matching EEE's
+    # per-collection Community-Evals tooling; upstream dataset names stay in each
+    # log's source_data.
+    collection_prefix = getattr(args, 'collection_prefix', None) or 'sayf-eval-'
+
+    log_path = Path(args.log_path)
+    input_result: SourceConversionResult[Any] | None = None
+    if log_path.is_file():
+        logs = adapter.transform_from_file(log_path, metadata)
+    elif log_path.is_dir():
+        input_result = adapter.transform_from_directory_result(
+            log_path, metadata
+        )
+        logs = input_result.records
+    else:
+        raise FileNotFoundError(f'Path is not a file or directory: {log_path}')
+
+    if not logs and input_result is None:
+        raise ValueError(
+            f'sayf-eval conversion produced no logs from {log_path}'
+        )
+
+    output_dir = Path(args.output_dir)
+    # Aggregate-only: sayf-eval per-sample item text is dual-use and never
+    # published, so no instance-level samples and no staging directory. Each log
+    # goes into its own per-task collection (the task is the evaluation_id stem).
+    paths = []
+    for log, eval_uuid in zip(logs, [str(uuid.uuid4()) for _ in logs]):
+        task = log.evaluation_id.split('/', 1)[0]
+        collection = f'{collection_prefix}{task.replace("_", "-")}'
+        paths.extend(
+            publish_evaluation_logs(
+                [log], output_dir, [eval_uuid], collection_override=collection
+            )
+        )
+    for path in paths:
+        print(path)
+
+    _save_partial_conversion_report(
+        input_result, output_dir, 'sayf_eval_inputs'
+    )
+    if input_result is not None:
+        input_result.raise_if_incomplete()
+    print(f'Converted {len(paths)} evaluation log(s).')
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog='every_eval_ever',
@@ -535,7 +592,7 @@ def build_parser() -> argparse.ArgumentParser:
         dest='source', required=True
     )
 
-    for source in ['lm_eval', 'inspect', 'helm', 'alpaca_eval']:
+    for source in ['lm_eval', 'inspect', 'helm', 'alpaca_eval', 'sayf_eval']:
         source_parser = convert_subparsers.add_parser(
             source,
             help=f'Convert {source} logs',
@@ -631,6 +688,18 @@ def build_parser() -> argparse.ArgumentParser:
                     'evaluation details.'
                 ),
             )
+        if source == 'sayf_eval':
+            source_parser.add_argument(
+                '--collection_prefix',
+                '--collection-prefix',
+                default='sayf-eval-',
+                help=(
+                    'Prefix for the per-task datastore collection '
+                    '(data/<prefix><task>/...): one collection per benchmark. '
+                    'Upstream dataset names are kept in each log source_data. '
+                    'Default: sayf-eval-.'
+                ),
+            )
 
     return parser
 
@@ -667,6 +736,8 @@ def main(argv: list[str] | None = None) -> int:
             return _cmd_convert_helm(args)
         if args.source == 'alpaca_eval':
             return _cmd_convert_alpaca_eval(args)
+        if args.source == 'sayf_eval':
+            return _cmd_convert_sayf_eval(args)
 
     parser.print_help()
     return 1

every_eval_ever/converters/README.md

@@ -274,3 +274,54 @@ options:
   --version {v1,v2}            Which leaderboard to convert. Omit to convert both (default).
   --output_dir OUTPUT_DIR      Base output directory (default: data).
 ```
+
+## sayf-eval
+
+[sayf-eval](https://pypi.org/project/sayf-eval/) (`pip install sayf-eval`) is a
+model-agnostic cybersecurity LLM-evaluation framework
+([source](https://github.com/qcri/sayf-eval)). Each run writes a canonical
+*results record* (`<output_dir>/results/<model>/results_<ts>.json`) that embeds
+the full pipeline configuration (decoding params, `<think>` handling,
+denominator policy, judge model) and per-task dataset provenance alongside the
+scores. The converter maps that record onto the unified schema, producing **one
+aggregate log per task** (accuracy, plus CVSS MAD for VSP and micro-F1 for ATE),
+with the judge recorded under `metric_config.llm_scoring`.
+
+You produce a record by running sayf-eval; this converter then reads the record
+as JSON (it does **not** import sayf-eval). `--log_path` accepts a single
+`results_*.json` record or a run output directory (searched recursively for
+`results_*.json`).
+
+Each task is routed into its own namespaced collection (`data/sayf-eval-<task>/...`,
+one collection per benchmark), while the upstream dataset name is preserved in each
+log's `source_data`.
+
+```bash
+uv run every_eval_ever convert sayf_eval \
+  --log_path outputs/gpt4o \
+  --source_organization_name QCRI --evaluator_relationship third_party \
+  --collection-prefix sayf-eval- --output_dir data
+```
+
+> **Aggregate-only (by design).** sayf-eval is a cybersecurity benchmark whose
+> per-sample item text is dual-use and kept private, so this converter emits only
+> aggregate score files — never instance-level `_samples.jsonl`. The results
+> record contains no prompt/gold/response text, so its output is safe to publish.
+
+```
+usage: every_eval_ever convert sayf_eval [-h] --log_path LOG_PATH
+                                         [--output_dir OUTPUT_DIR]
+                                         [--source_organization_name ...]
+                                         [--evaluator_relationship {first_party,third_party,collaborative,other}]
+                                         [--source_organization_url ...]
+                                         [--source_organization_logo_url ...]
+                                         [--collection_prefix COLLECTION_PREFIX]
+                                         [--eval_library_name ...]
+                                         [--eval_library_version ...]
+
+options:
+  --collection_prefix PREFIX   Prefix for the per-task datastore collection
+                               (data/<prefix><task>/...): one collection per
+                               benchmark. Upstream dataset names are kept in each
+                               log's source_data. Default: sayf-eval-.
+```

every_eval_ever/converters/sayf_eval/__init__.py

@@ -0,0 +1,7 @@
+"""sayf-eval adapter for every_eval_ever.
+
+Converts sayf-eval's canonical *results record* (a pipeline-config-embedded
+scores artifact) into the Every Eval Ever schema. Aggregate-only by design:
+sayf-eval is a cybersecurity benchmark whose per-sample item text is dual-use and
+kept private, so this converter never emits instance-level ``_samples.jsonl``.
+"""

every_eval_ever/converters/sayf_eval/__main__.py

@@ -0,0 +1,75 @@
+"""CLI for converting sayf-eval results records to every_eval_ever format."""
+
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Convert sayf-eval results records to every_eval_ever format'
+    )
+    parser.add_argument(
+        '--log_path',
+        type=str,
+        required=True,
+        help='Path to a sayf-eval results record JSON, or a run output directory '
+        '(searched recursively for results_*.json).',
+    )
+    parser.add_argument(
+        '--output_dir',
+        type=str,
+        default='data',
+        help='Output directory for converted files',
+    )
+    parser.add_argument(
+        '--source_organization_name',
+        type=str,
+        default='unknown',
+        help='Name of the organization that ran the evaluation',
+    )
+    parser.add_argument(
+        '--evaluator_relationship',
+        type=str,
+        default='third_party',
+        choices=['first_party', 'third_party', 'collaborative', 'other'],
+        help='Relationship of the evaluator to the model',
+    )
+    parser.add_argument(
+        '--source_organization_url',
+        type=str,
+        default=None,
+        help='URL of the source organization',
+    )
+    parser.add_argument(
+        '--source_organization_logo_url',
+        type=str,
+        default=None,
+        help='Logo of the source organization',
+    )
+    parser.add_argument(
+        '--collection_prefix',
+        type=str,
+        default='sayf-eval-',
+        help='Prefix for the per-task datastore collection '
+        '(data/<prefix><task>/...). Upstream dataset names are kept in source_data.',
+    )
+    parser.add_argument(
+        '--eval_library_name',
+        type=str,
+        default='sayf-eval',
+        help='Name of the evaluation library (recorded in eval_library.name)',
+    )
+    parser.add_argument(
+        '--eval_library_version',
+        type=str,
+        default='unknown',
+        help='Fallback eval library version (the record embeds its own version).',
+    )
+
+    args = parser.parse_args()
+    from every_eval_ever.cli import _cmd_convert_sayf_eval
+
+    return _cmd_convert_sayf_eval(args)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())