-
Notifications
You must be signed in to change notification settings - Fork 65
Expand file tree
/
Copy pathtask.py
More file actions
103 lines (83 loc) · 4.12 KB
/
Copy pathtask.py
File metadata and controls
103 lines (83 loc) · 4.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Task: one task row — an env name, a task id, bound args, and metadata.
``foo(x, y)`` (an ``@env.template`` factory call) returns one of these. ``env``
is the environment's *name*: the join key between the data plane (rows) and
whatever placement can bring that environment up. Running a task never needs
a live env — the prompt and grading arrive over the wire from the substrate
the placement brought up — so the row holds the reference explicitly instead
of wrapping it in an :class:`~hud.environment.Environment` object.
The model *is* the row: field names are the wire keys, so plain pydantic
(``Task.model_validate(entry)`` / ``task.model_dump()``) is the whole codec —
there is no bespoke serialization layer.
Placement is ``runtime: Provider | HostedRuntime | None`` (see :mod:`.runtime`).
Execution lives entirely in :mod:`.rollout` and scheduling in
:mod:`.taskset` — :meth:`Task.run` is the single-task form of
``Taskset.run``, so the row is always an argument to the engine, never a
participant in it. Platform sync lives in :mod:`hud.eval.sync`.
"""
from __future__ import annotations
import hashlib
import json
from typing import TYPE_CHECKING, Any
from pydantic import BaseModel, Field
from .runtime import RuntimeConfig
if TYPE_CHECKING:
from hud.agents.base import Agent
from .job import Job
from .runtime import HostedRuntime, Provider
class Task(BaseModel):
"""One concrete task: an env name plus data (id, args, metadata).
Pure data — holds no execution state, so one ``Task`` can drive many
concurrent rollouts. ``run`` it for a graded :class:`~hud.eval.job.Job`;
placement comes from ``runtime=`` (a provider), else the HUD runtime
tunnel by ``env`` name.
"""
env: str = Field(min_length=1)
id: str = Field(min_length=1)
args: dict[str, Any] = Field(default_factory=dict)
slug: str | None = None
validation: list[dict[str, Any]] | None = None
agent_config: dict[str, Any] | None = None
#: Arbitrary metadata fields surfaced as filterable columns / leaderboard
#: facets on the platform (e.g. ``{"difficulty": "easy", "suite": "coding"}``).
columns: dict[str, Any] | None = None
#: Optional row-level runtime construction input. Runtime adapters apply the
#: supported subset into their native launch shape or reject it.
runtime_config: RuntimeConfig | None = None
def default_slug(self) -> str:
"""A stable slug from the task id, disambiguated by an args hash when present."""
if not self.args:
return self.id
digest = hashlib.sha1( # noqa: S324 - non-crypto, stable disambiguator
json.dumps(self.args, sort_keys=True, default=str).encode("utf-8"),
).hexdigest()[:8]
return f"{self.id}-{digest}"
# ─── execution ────────────────────────────────────────────────────
async def run(
self,
agent: Agent,
*,
runtime: Provider | HostedRuntime | None = None,
group: int | None = None,
max_concurrent: int | None = None,
job: Job | None = None,
rollout_timeout: float | None = None,
) -> Job:
"""Run this task with ``agent``: the single-task form of ``Taskset.run``.
Identical scheduling semantics — one HUD job as the receipt (or an
open ``job`` from :meth:`Job.start` to accumulate into), ``group``
repeats sharing a group_id, ``max_concurrent`` capping parallelism —
over a taskset of one. ``runtime`` is the placement; left unset it
falls back to the HUD runtime tunnel by ``env`` name. For a local
run, pass one explicitly (``runtime=LocalRuntime("env.py")``).
"""
from .taskset import Taskset # circular: taskset -> sync -> task
taskset = Taskset(self.default_slug(), [self])
return await taskset.run(
agent,
runtime=runtime,
group=group,
max_concurrent=max_concurrent,
job=job,
rollout_timeout=rollout_timeout,
)
__all__ = ["RuntimeConfig", "Task"]