[Adapter] AlpacaEval 1.0 and 2.0, consolidated with the merged adapter and resolved through the eval-card-registry

every_eval_ever/cli.py

@@ -369,15 +369,51 @@ def _cmd_convert_helm(args: argparse.Namespace) -> int:
     return 0
 
 
+#: What ``convert alpaca_eval`` writes to when no ``--output_dir`` is given. A
+#: marker, not the directory itself: it is resolved per run (see
+#: :func:`_cmd_convert_alpaca_eval`), and building the parser must not create
+#: anything on disk.
+SMOKE_OUTPUT_DIR = str(
+    Path(tempfile.gettempdir()) / 'alpaca-eval-smoke' / 'data'
+)
+
+
 def _cmd_convert_alpaca_eval(args: argparse.Namespace) -> int:
+    import json
+
     from every_eval_ever.converters.alpaca_eval.adapter import (
         LEADERBOARDS,
         AlpacaEvalAdapter,
     )
-
-    adapter = AlpacaEvalAdapter()
+    from every_eval_ever.converters.alpaca_eval.upstream import UpstreamSnapshot
+    from every_eval_ever.helpers.eval_card_registry import Registry, gaps
+
+    snapshot = None
+    if args.input_json:
+        with open(args.input_json, encoding='utf-8') as handle:
+            snapshot = UpstreamSnapshot.from_payload(json.load(handle))
+        print(f'Replaying upstream snapshot {args.input_json} (ref {snapshot.ref})')
+    registry = Registry(
+        enabled=not args.no_registry_resolve, live=args.registry_live
+    )
+    print(f'eval-card-registry: {registry.status()}')
+    if registry.enabled and gaps():
+        # Surfaced every run: a missing canonical is a registry-side follow-up,
+        # and it silently shapes the ids in the output until someone files it.
+        print('  no canonical entry for: ' + ', '.join(gaps()))
+    adapter = AlpacaEvalAdapter(
+        ref=args.ref, snapshot=snapshot, registry=registry
+    )
     versions = [args.version] if args.version else list(LEADERBOARDS.keys())
-    output_dir = Path(args.output_dir)
+    if args.output_dir == SMOKE_OUTPUT_DIR:
+        # Records are named with a fresh UUID per run, so a fixed throwaway
+        # directory accumulates earlier runs' output and a reader cannot tell
+        # which files this run produced. One directory per run instead of
+        # deleting: a smoke run is worth looking at.
+        output_dir = Path(tempfile.mkdtemp(prefix='alpaca-eval-smoke-')) / 'data'
+        print(f'No --output_dir given; writing throwaway output to {output_dir}')
+    else:
+        output_dir = Path(args.output_dir)
 
     logs_to_publish = []
     eval_uuids = []
@@ -448,6 +484,22 @@ def _cmd_convert_alpaca_eval(args: argparse.Namespace) -> int:
             logs_to_publish.append(log)
             eval_uuids.append(str(uuid.uuid4()))
 
+    if registry.live:
+        # The line printed before conversion cannot carry these: no lookup has
+        # happened yet. `live_error` is sticky, so it reports the run, not a call.
+        print(
+            f'\neval-card-registry live lookups: {registry.live_queries} '
+            f'queries, {registry.live_hits} resolved'
+            + (f', error: {registry.live_error}' if registry.live_error else '')
+        )
+
+    if args.save_raw_json:
+        raw_path = Path(args.save_raw_json)
+        raw_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(raw_path, 'w', encoding='utf-8') as handle:
+            json.dump(adapter.snapshot.to_payload(), handle, indent=2)
+        print(f'Upstream snapshot: {raw_path}')
+
     paths = publish_evaluation_logs(logs_to_publish, output_dir, eval_uuids)
     for path in paths:
         print(f'  {path}')
@@ -592,6 +644,16 @@ def build_parser() -> argparse.ArgumentParser:
         )
 
         if source == 'alpaca_eval':
+            from every_eval_ever.converters.alpaca_eval.upstream import (
+                DEFAULT_UPSTREAM_REF,
+            )
+
+            # This source fetches from the network rather than from a local log,
+            # so a plain `convert alpaca_eval` would otherwise write a data/
+            # tree into whatever directory it was run from. Default to a temp
+            # path so a smoke run is throwaway; publishing is opt-in via
+            # --output_dir.
+            source_parser.set_defaults(output_dir=SMOKE_OUTPUT_DIR)
             source_parser.add_argument(
                 '--version',
                 choices=['v1', 'v2'],
@@ -601,6 +663,56 @@ def build_parser() -> argparse.ArgumentParser:
                     'or v2 (AlpacaEval 2.0). Omit to convert both (default).'
                 ),
             )
+            source_parser.add_argument(
+                '--ref',
+                default=DEFAULT_UPSTREAM_REF,
+                help=(
+                    'Upstream tatsu-lab/alpaca_eval git ref to convert from. '
+                    'Pinning a commit (the default) keeps evaluation_id stable '
+                    'across reruns; pass a branch to pick up new submissions.'
+                ),
+            )
+            source_parser.add_argument(
+                '--save_raw_json',
+                '--save-raw-json',
+                default=None,
+                help=(
+                    'Write the fetched upstream artefacts (leaderboard CSVs, '
+                    'judge configs and prompts, per-model configs) to this JSON '
+                    'file so the conversion can be replayed offline.'
+                ),
+            )
+            source_parser.add_argument(
+                '--input_json',
+                '--input-json',
+                default=None,
+                help=(
+                    'Convert from a --save_raw_json snapshot instead of '
+                    'fetching from GitHub. Nothing is fetched unless '
+                    '--registry_live is also given.'
+                ),
+            )
+            source_parser.add_argument(
+                '--no_registry_resolve',
+                '--no-registry-resolve',
+                action='store_true',
+                help=(
+                    'Do not resolve organization, metric and benchmark ids '
+                    'against the eval-card-registry. Records then carry the '
+                    'source-derived spellings, marked registry_disabled.'
+                ),
+            )
+            source_parser.add_argument(
+                '--registry_live',
+                '--registry-live',
+                action='store_true',
+                help=(
+                    'Additionally query the live registry for values the '
+                    'vendored snapshot cannot place. Uses mode=exact, which '
+                    'resolves without creating draft canonicals. Never fatal: '
+                    'a failure falls back to the snapshot.'
+                ),
+            )
 
         if source == 'lm_eval':
             source_parser.add_argument(

every_eval_ever/converters/README.md

@@ -219,58 +219,118 @@ options:
 
 ## AlpacaEval
 
-The AlpacaEval converter fetches the public leaderboard CSV directly from GitHub
-and converts all model entries into the unified schema. No local log files are required.
-Semantically this is a source adapter rather than a local-log converter; it
-remains under `converters` to preserve the existing
-`every_eval_ever convert alpaca_eval` API.
-
-Both AlpacaEval 1.0 (GPT-4 judge, `text_davinci_003` baseline) and
-AlpacaEval 2.0 (weighted LC win rate, `gpt4_turbo` baseline) are supported.
-
-Metrics converted per model:
-
-| Metric | Description |
-|---|---|
-| Win Rate | Fraction of outputs preferred over the baseline (raw) |
-| Length-Controlled Win Rate | Win rate debiased for response length (v2 only) |
-| Discrete Win Rate | Binary win rate — no partial credit for ties |
-| Average Response Length | Mean token count of model responses |
-
-
-### Usage
-
-Convert both leaderboards (default):
-
 ```bash
-uv run every_eval_ever convert alpaca_eval --output_dir data
+uv run every_eval_ever convert alpaca_eval --output_dir data          # both
+uv run every_eval_ever convert alpaca_eval --version v2 --output_dir data
 ```
 
-Convert only AlpacaEval 2.0:
+Fetches the leaderboards from the upstream `tatsu-lab/alpaca_eval` repository
+and converts every model entry; no local log files. Semantically a source
+adapter rather than a local-log converter, kept under `converters` to preserve
+the existing `every_eval_ever convert alpaca_eval` API. Both AlpacaEval 1.0
+(`alpaca_eval_gpt4` judge, `text_davinci_003` baseline) and 2.0
+(`weighted_alpaca_eval_gpt4_turbo` judge, `gpt4_turbo` baseline) are supported.
+
+One record per leaderboard entry, with one result per published column:
+
+| Metric | Unit | Scope |
+|---|---|---|
+| `win_rate` | percent | Share of the 805 instructions where the judge preferred this model over the baseline |
+| `length_controlled_win_rate` | percent | Win rate debiased for output length (Dubois et al., 2024) |
+| `discrete_win_rate` | percent | Binary win rate, no partial credit for ties |
+| `avg_length` | characters | Mean output length in **characters** (`output.str.len().mean()`), for length-bias context, not a quality score |
+
+Scores are published on the scale the registry declares for the metric — `[0,
+100]` for `win-rate`, which is what the CSV holds — with `score_scale_divisor`
+recording any conversion. Standard errors are the upstream `preferences.sem()`,
+so they are recorded as `analytic`, not bootstrap.
+
+Beyond the score tables the converter reads the judge configuration and its
+verbatim prompt template, the upstream package version, and one config per entry
+(`models_configs/<slug>/configs.yaml`). Those configs are what `identity.py`
+resolves each model's repo id, developer, availability and generation settings
+from, recording which rung of its ladder fired as `identity_source`; an entry it
+cannot resolve is reported as a failure rather than published under a guessed
+name. `deployment_type` normally comes from the config's `fn_completions`, but
+28 configs record none, and for those the `completions_kwargs` decide it — see
+`local_generate_evidence`, which names the deciding kwarg on the record as
+`deployment_evidence`.
+
+### Canonical ids
+
+Organization, metric, benchmark and harness ids come from the
+[eval-card-registry](https://evaleval-entity-registry.hf.space) through
+`helpers/eval_card_registry.py`, which every consumer in this repo shares, and
+which reads a vendored snapshot of the registry's read-only list endpoints
+(`helpers/data/eval_card_registry.json`). `model_info.developer` is the canonical
+organization id while `model_info.id` keeps the HuggingFace namespace the weights
+are published under; the registry records both (`meta` and `meta-llama`), so
+these are two identities for one organization rather than drift.
+
+A value the registry has no canonical for keeps a namespaced `alpaca_eval.*` id
+and says so through `*_registry_strategy`, which also records how much of the
+spelling had to be discarded to reach a canonical id. The CLI prints the current
+gap list on every run. Refresh the snapshot with:
 
 ```bash
-uv run every_eval_ever convert alpaca_eval --version v2 --output_dir data
+uv run python -m every_eval_ever.tools.refresh_eval_card_registry
+uv run python -m every_eval_ever.tools.refresh_eval_card_registry --check
 ```
 
-Convert only AlpacaEval 1.0:
+### Repo ids
+
+The per-model configs are hand-written and the leaderboards are from 2023-2024,
+so two things make a repo id in them differ from the id HuggingFace serves today.
+Both are corrected after the identity ladder runs, and both change only *how a
+repo is spelled*, never *which* repo a record points at:
+
+- **Casing.** `01-ai/Yi-34b-Chat` in a `completions_kwargs` is the same repo as
+  `01-ai/Yi-34B-Chat` in a `link`, because repo ids are case-insensitive. A link
+  is a URL that resolved for whoever wrote it, so a link's spelling wins.
+- **Renames.** `WizardLM` became `WizardLMTeam`, `THUDM` became `zai-org`,
+  `cognitivecomputations` became `dphn`, and Meta dropped the `Meta-` prefix from
+  the Llama 3.1 repos. The old id still answers via a redirect, so nothing looks
+  broken, but the datastore already holds records under the current id from other
+  sources and a stale id silently fails to join with them.
+  `data/hf_canonical_ids.json` maps the referenced id to the current one, and
+  `model_id_as_referenced` keeps the spelling the source used.
+
+A rename applies only to the rungs in `HF_GROUNDED_SOURCES`, and is not
+extrapolated to repos HuggingFace would not confirm: `WizardLM/WizardLM-13B-V1.1`
+and `WizardLMTeam/WizardLM-13B-V1.1` both answer `401`, so guessing the new
+namespace would publish an id that resolves nowhere. `developer` does not follow
+the new namespace either, since a redirect cannot distinguish an organization
+renaming itself from a repo transferred to someone else.
+
+Refresh the map with (GET-only, unauthenticated — of 135 published repo ids the
+last sweep confirmed 124 and left 11 as the source spells them, since `401`
+conflates *gated* with *nonexistent*):
 
 ```bash
-uv run every_eval_ever convert alpaca_eval --version v1 --output_dir data
+uv run python -m every_eval_ever.converters.alpaca_eval.refresh_hf_canonical_ids
+uv run python -m every_eval_ever.converters.alpaca_eval.refresh_hf_canonical_ids --check
 ```
 
-Full argument list:
+### Reruns
 
-```
-usage: every_eval_ever convert alpaca_eval [-h] [--log_path LOG_PATH]
-                                           [--output_dir OUTPUT_DIR]
-                                           [--version {v1,v2}]
-                                           [--source_organization_name ...]
-                                           [--evaluator_relationship ...]
-                                           [--source_organization_url ...]
-                                           [--eval_library_name ...]
-                                           [--eval_library_version ...]
+Two runs over the same upstream ref produce the same `evaluation_id`s, output
+directories and record contents, with two repo-wide exceptions: the file name is
+a fresh `uuid4()` per write (`helpers/io.py` requires a v4 UUID, so a
+content-derived name would have to be a UUIDv5 the validator rejects), and
+`retrieved_timestamp` is when the run fetched, one value per leaderboard. So
+compare two runs on record contents minus `retrieved_timestamp`, not
+byte-for-byte on the tree.
 
-options:
-  --version {v1,v2}            Which leaderboard to convert. Omit to convert both (default).
-  --output_dir OUTPUT_DIR      Base output directory (default: data).
-```
+### Options
+
+Beyond the shared converter arguments:
+
+| Option | |
+|---|---|
+| `--version {v1,v2}` | Which leaderboard. Omit to convert both. |
+| `--output_dir PATH` | Defaults to a temporary path, so a smoke run does not write a `data/` tree into the current directory. |
+| `--ref REF` | Upstream git ref. Pinning a commit (the default) keeps `evaluation_id` stable across reruns; pass a branch to pick up new submissions. |
+| `--save_raw_json PATH` | Write the fetched upstream artefacts to PATH. |
+| `--input_json PATH` | Convert from such a snapshot, with no network access. |
+| `--no_registry_resolve` | Skip registry resolution; records carry source-derived spellings, marked `registry_disabled`. |
+| `--registry_live` | Also query the live registry for values the snapshot cannot place, in side-effect-free `mode=exact`. |

every_eval_ever/converters/alpaca_eval/__main__.py

@@ -1,42 +1,19 @@
-"""CLI for converting AlpacaEval leaderboard data to every_eval_ever format."""
+"""``python -m every_eval_ever.converters.alpaca_eval`` — the shared CLI.
 
-import argparse
+Every option comes from ``every_eval_ever.cli``'s ``convert alpaca_eval``
+parser, so this entry point cannot fall behind it.
+"""
 
-from .adapter import LEADERBOARDS
+import sys
+from typing import List, Optional
 
 
-def main():
-    parser = argparse.ArgumentParser(
-        description=(
-            'Fetch AlpacaEval leaderboard data from GitHub and convert it '
-            'to Every Eval Ever schema JSON files.'
-        )
-    )
-    parser.add_argument(
-        '--version',
-        choices=list(LEADERBOARDS.keys()),
-        default=None,
-        help=(
-            'Which leaderboard to convert. '
-            'Omit to convert all versions (default).'
-        ),
-    )
-    parser.add_argument(
-        '--output_dir',
-        default='data',
-        help='Base output directory (default: data).',
-    )
-    args = parser.parse_args()
-    args.source_organization_name = 'unknown'
-    args.evaluator_relationship = 'third_party'
-    args.source_organization_url = None
-    args.source_organization_logo_url = None
-    args.eval_library_name = 'alpaca_eval'
-    args.eval_library_version = 'unknown'
+def main(argv: Optional[List[str]] = None) -> int:
+    from every_eval_ever.cli import main as cli_main
 
-    from every_eval_ever.cli import _cmd_convert_alpaca_eval
-
-    return _cmd_convert_alpaca_eval(args)
+    if argv is None:
+        argv = sys.argv[1:]
+    return cli_main(['convert', 'alpaca_eval', *argv])
 
 
 if __name__ == '__main__':