From acfab4dbe0ecfe9468a673a26990947928b9495a Mon Sep 17 00:00:00 2001 From: nhungbi Date: Thu, 8 Oct 2026 14:31:30 -0400 Subject: [PATCH 1/3] feat(redteam): public as_target_session --- AGENTS.md | 3 +- .../experimental/redteam/__init__.py | 2 + .../redteam/strategies/__init__.py | 2 + .../redteam/strategies/target_session.py | 55 +++++++++++++++++++ .../experimental/redteam/task.py | 36 +----------- .../redteam/test_target_session.py | 53 ++++++++++++++++++ 6 files changed, 116 insertions(+), 35 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6755997f..b989c61d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,7 +115,8 @@ strands-evals/ │ │ │ ├── base.py # AttackStrategy ABC + AttackRunResult │ │ │ ├── _common.py # Shared helpers │ │ │ ├── target_session.py # TargetSession Protocol + StrandsAgentSession, -│ │ │ │ # StrandsMultiAgentSession, TargetCheckpoint, ToolUseEntry +│ │ │ │ # StrandsMultiAgentSession, TargetCheckpoint, ToolUseEntry, +│ │ │ │ # as_target_session │ │ │ ├── bad_likert_judge/ # BadLikertJudgeStrategy │ │ │ ├── crescendo/ # CrescendoStrategy + crescendo_v0 prompt │ │ │ ├── goat/ # GoatStrategy + goat_v0 prompt diff --git a/src/strands_evals/experimental/redteam/__init__.py b/src/strands_evals/experimental/redteam/__init__.py index 06e62753..526277e8 100644 --- a/src/strands_evals/experimental/redteam/__init__.py +++ b/src/strands_evals/experimental/redteam/__init__.py @@ -16,6 +16,7 @@ StrandsMultiAgentSession, TargetCheckpoint, TargetSession, + as_target_session, ) from .types import RISK_CATEGORIES, AttackGoal, RedTeamConfig @@ -43,4 +44,5 @@ "TargetCheckpoint", "TargetSession", "TargetSpec", + "as_target_session", ] diff --git a/src/strands_evals/experimental/redteam/strategies/__init__.py b/src/strands_evals/experimental/redteam/strategies/__init__.py index 9ec2d3ff..8355d5d1 100644 --- a/src/strands_evals/experimental/redteam/strategies/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/__init__.py @@ -12,6 +12,7 @@ TargetCheckpoint, TargetSession, ToolUseEntry, + as_target_session, ) # Ready-made strategy instances users can pass to RedTeamExperiment(attack_strategies=[...]). @@ -37,4 +38,5 @@ "TargetCheckpoint", "TargetSession", "ToolUseEntry", + "as_target_session", ] diff --git a/src/strands_evals/experimental/redteam/strategies/target_session.py b/src/strands_evals/experimental/redteam/strategies/target_session.py index b63558f0..da0cc278 100644 --- a/src/strands_evals/experimental/redteam/strategies/target_session.py +++ b/src/strands_evals/experimental/redteam/strategies/target_session.py @@ -341,6 +341,60 @@ def _multi_agent_result_text(result: Any) -> str: return str(result) +def as_target_session(target: Agent | MultiAgentBase | TargetSession) -> TargetSession: + """Wrap a target in the `TargetSession` a strategy's `run_attack` expects. + + Use this in a custom task to turn a freshly built target into a session: + + session = as_target_session(build_agent()) + result = strategy.run_attack(case, session, max_turns=MAX_ALLOWED_TURNS) + + Args: + target: A `strands.Agent` (wrapped in `StrandsAgentSession`), a `MultiAgentBase` such as a `Graph` or + `Swarm` (wrapped in `StrandsMultiAgentSession`), or a ready `TargetSession` (returned as is). + + Returns: + A `TargetSession` driving `target`. + + Raises: + TypeError: If `target` is none of the above. A custom `TargetSession` must expose + `invoke`/`reset`/`snapshot`/`restore` and a `trace: list`. + """ + return _build_session(target) + + +def _build_session( + agent: Agent | MultiAgentBase | TargetSession, + *, + baseline: Any = None, +) -> TargetSession: + """Wrap an `Agent` / `MultiAgentBase`, or pass a `TargetSession` through. + + Args: + agent: The target to wrap, or a ready `TargetSession`. + baseline: Clean snapshot the wrapped session resets to between cases. Ignored for a passed-in + `TargetSession`. Typed `Any` because the two session types use different opaque baseline shapes. + + Raises: + TypeError: If `agent` is not an `Agent`, `MultiAgentBase`, or a structural `TargetSession` (must expose + `invoke`/`reset`/`snapshot`/`restore` and a `trace: list`). + """ + if isinstance(agent, Agent): + return StrandsAgentSession(agent, baseline=baseline) + if isinstance(agent, MultiAgentBase): + return StrandsMultiAgentSession(agent, baseline=baseline) + # Structural check: TargetSession is a Protocol. The `trace: list` check is + # load-bearing because the task runner dereferences `.trace` directly. + has_methods = all(callable(getattr(agent, method, None)) for method in ("invoke", "reset", "snapshot", "restore")) + if has_methods and isinstance(getattr(agent, "trace", None), list): + return agent + raise TypeError( + f"agent must be a strands.Agent, strands.multiagent.MultiAgentBase, or a TargetSession, " + f"got {type(agent).__name__!r}; wrap a custom target in a TargetSession so the strategy " + "can snapshot/restore its state." + ) + + __all__ = [ "MALFORMED_TOOL_NAME", "StrandsAgentSession", @@ -348,4 +402,5 @@ def _multi_agent_result_text(result: Any) -> str: "TargetCheckpoint", "TargetSession", "ToolUseEntry", + "as_target_session", ] diff --git a/src/strands_evals/experimental/redteam/task.py b/src/strands_evals/experimental/redteam/task.py index 4f3238c4..4cef4fbf 100644 --- a/src/strands_evals/experimental/redteam/task.py +++ b/src/strands_evals/experimental/redteam/task.py @@ -12,7 +12,7 @@ from .case import RedTeamCase from .strategies import AttackStrategy -from .strategies.target_session import StrandsAgentSession, StrandsMultiAgentSession, TargetSession +from .strategies.target_session import StrandsMultiAgentSession, TargetSession, _build_session, as_target_session logger = logging.getLogger(__name__) @@ -124,7 +124,7 @@ def task_fn(case: RedTeamCase) -> dict: # No baseline: each case starts from a freshly built target, and `session.reset()` only # needs to clear the per-case trace. - session = _build_session(make_target(), baseline=None) + session = as_target_session(make_target()) session.reset() # CPython dict assignment for a single distinct key is atomic, and case names are unique @@ -180,38 +180,6 @@ def _resolve_target_source( ) -def _build_session( - agent: Agent | MultiAgentBase | TargetSession, - *, - baseline: Any = None, -) -> TargetSession: - """Wrap an `Agent` / `MultiAgentBase`, or pass a `TargetSession` through. - - Args: - agent: The target to wrap, or a ready `TargetSession`. - baseline: Clean snapshot the wrapped session resets to between cases. Ignored for a passed-in - `TargetSession`. Typed `Any` because the two session types use different opaque baseline shapes. - - Raises: - TypeError: If `agent` is not an `Agent`, `MultiAgentBase`, or a structural `TargetSession` (must expose - `invoke`/`reset`/`snapshot`/`restore` and a `trace: list`). - """ - if isinstance(agent, Agent): - return StrandsAgentSession(agent, baseline=baseline) - if isinstance(agent, MultiAgentBase): - return StrandsMultiAgentSession(agent, baseline=baseline) - # Structural check: TargetSession is a Protocol. The `trace: list` check is - # load-bearing because the task runner dereferences `.trace` directly. - has_methods = all(callable(getattr(agent, method, None)) for method in ("invoke", "reset", "snapshot", "restore")) - if has_methods and isinstance(getattr(agent, "trace", None), list): - return agent - raise TypeError( - f"agent must be a strands.Agent, strands.multiagent.MultiAgentBase, or a TargetSession, " - f"got {type(agent).__name__!r}; wrap a custom target in a TargetSession so the strategy " - "can snapshot/restore its state." - ) - - def _resolve_case_strategy(case: RedTeamCase, by_label: dict[str, AttackStrategy]) -> AttackStrategy: """Look up the strategy for `case` from its `metadata["strategy"]` label.""" metadata = case.metadata or {} diff --git a/tests/strands_evals/experimental/redteam/test_target_session.py b/tests/strands_evals/experimental/redteam/test_target_session.py index cc38af0b..df87e033 100644 --- a/tests/strands_evals/experimental/redteam/test_target_session.py +++ b/tests/strands_evals/experimental/redteam/test_target_session.py @@ -4,13 +4,17 @@ import pytest from strands import Agent +from strands.multiagent import Swarm +from strands_evals.experimental import redteam from strands_evals.experimental.redteam.strategies.target_session import ( MALFORMED_TOOL_NAME, StrandsAgentSession, + StrandsMultiAgentSession, TargetCheckpoint, _single_shot_attempts, _tool_uses_in, + as_target_session, ) @@ -293,3 +297,52 @@ def test_skipped_attempt_does_not_restore(self): begin_attempt() # attempt 3: restores to entry seen.append(session.state) # 0 again assert seen == [0, 0] + + +# --------------------------------------------------------------------------- +# as_target_session — public wrapper for custom tasks +# --------------------------------------------------------------------------- + + +class _GuardedSession(StrandsAgentSession): + """A user subclass of a built-in session; must come back as is, not re-wrapped.""" + + +class _NoTraceSession: + """Has the four TargetSession methods but no `trace` list.""" + + def invoke(self, message): + return "" + + def reset(self): + pass + + def snapshot(self): + return None + + def restore(self, checkpoint): + pass + + +class TestAsTargetSession: + def test_wraps_agent(self): + assert isinstance(as_target_session(Agent(model=None, callback_handler=None)), StrandsAgentSession) + + def test_wraps_multi_agent(self): + swarm = Swarm([Agent(model=None, callback_handler=None)]) + assert isinstance(as_target_session(swarm), StrandsMultiAgentSession) + + def test_passes_target_session_through(self): + session = _GuardedSession(Agent(model=None, callback_handler=None)) + assert as_target_session(session) is session + + def test_rejects_session_without_trace_list(self): + with pytest.raises(TypeError, match="TargetSession"): + as_target_session(_NoTraceSession()) + + def test_rejects_bare_callable(self): + with pytest.raises(TypeError, match="TargetSession"): + as_target_session(lambda message: "reply") + + def test_exported_from_package_root(self): + assert redteam.as_target_session is as_target_session From 7515243fc9881285076b6d7e3e8dbc94c0158b90 Mon Sep 17 00:00:00 2001 From: nhungbi Date: Thu, 8 Oct 2026 15:37:14 -0400 Subject: [PATCH 2/3] feat(redteam): default run_attack max_turns to MAX_ALLOWED_TURNS --- AGENTS.md | 4 +- .../experimental/redteam/README.md | 2 +- .../experimental/redteam/__init__.py | 2 + .../redteam/strategies/__init__.py | 3 +- .../strategies/bad_likert_judge/__init__.py | 3 +- .../experimental/redteam/strategies/base.py | 24 +++++++++-- .../redteam/strategies/crescendo/__init__.py | 4 +- .../redteam/strategies/goat/__init__.py | 4 +- .../redteam/strategies/pair/__init__.py | 4 +- .../strategies/prompt_strategy/__init__.py | 4 +- .../strategies/sequentialbreak/__init__.py | 4 +- .../redteam/strategies/target_session.py | 2 +- .../experimental/redteam/task.py | 8 ++-- .../experimental/redteam/test_strategies.py | 42 ++++++++++++++++++- 14 files changed, 84 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b989c61d..39cc9923 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,13 +106,13 @@ strands-evals/ │ │ ├── README.md # Module quick-start + walkthrough │ │ ├── case.py # RedTeamCase + RedTeamConfig │ │ ├── experiment.py # RedTeamExperiment (case × strategy cross-product) -│ │ ├── task.py # _build_attacker_task; MAX_ALLOWED_TURNS = 50 (hard cap) +│ │ ├── task.py # _build_attacker_task │ │ ├── utils.py # _put_model_field (used by to_dict) │ │ ├── report.py # RedTeamReport / AttackResult / GroupedSummary │ │ ├── evaluators/ # AttackSuccessEvaluator │ │ ├── generators/ # AdversarialCaseGenerator + TargetSpec │ │ ├── strategies/ # AttackStrategy base + per-strategy subpackages -│ │ │ ├── base.py # AttackStrategy ABC + AttackRunResult +│ │ │ ├── base.py # AttackStrategy ABC + AttackRunResult; MAX_ALLOWED_TURNS = 50 (hard cap) │ │ │ ├── _common.py # Shared helpers │ │ │ ├── target_session.py # TargetSession Protocol + StrandsAgentSession, │ │ │ │ # StrandsMultiAgentSession, TargetCheckpoint, ToolUseEntry, diff --git a/src/strands_evals/experimental/redteam/README.md b/src/strands_evals/experimental/redteam/README.md index 5d4811d5..37607310 100644 --- a/src/strands_evals/experimental/redteam/README.md +++ b/src/strands_evals/experimental/redteam/README.md @@ -84,7 +84,7 @@ contract and talk to the target only through `target_session.invoke(...)`. | `BadLikertJudgeStrategy` | Casts the target as a harmfulness-rating judge, elicits a top-score example | no | [Unit 42](https://unit42.paloaltonetworks.com/multi-turn-technique-jailbreaks-llms/) | | `SequentialBreakStrategy` | Hides the harmful request among benign siblings in one narrative scaffold | no | [arXiv:2411.06426](https://arxiv.org/abs/2411.06426) | -Each strategy accepts a `max_turns=` kwarg (its own per-attack ceiling); the task runner additionally caps every strategy at a hard `MAX_ALLOWED_TURNS = 50` (`task.py`), so a strategy configured higher will be silently clamped. Pass `label="..."` to compare two instances of the same strategy in one experiment (e.g. `CrescendoStrategy(max_turns=5, label="cresc_short")`); duplicate labels raise at construction. +Each strategy accepts a `max_turns=` kwarg (its own per-attack ceiling); `run_attack` additionally caps every strategy at a hard `MAX_ALLOWED_TURNS = 50` (`strategies/base.py`, exported from `strands_evals.experimental.redteam`), so a strategy configured higher will be silently clamped. A custom task can omit `run_attack`'s `max_turns`, which defaults to that ceiling; a larger value is clamped to it. Pass `label="..."` to compare two instances of the same strategy in one experiment (e.g. `CrescendoStrategy(max_turns=5, label="cresc_short")`); duplicate labels raise at construction. `PromptStrategy` (a no-attacker-LLM, system-prompt-template strategy) and the `BUILTIN_STRATEGIES` registry are the extension points for adding new template-driven strategies without subclassing — see `strategies/prompt_strategy/`. The full set of exported symbols (experiment, cases, evaluator, target sessions) is the `__all__` of `strands_evals.experimental.redteam`. diff --git a/src/strands_evals/experimental/redteam/__init__.py b/src/strands_evals/experimental/redteam/__init__.py index 526277e8..e774c03d 100644 --- a/src/strands_evals/experimental/redteam/__init__.py +++ b/src/strands_evals/experimental/redteam/__init__.py @@ -4,6 +4,7 @@ from .generators import AdversarialCaseGenerator, TargetSpec from .report import AttackResult, GroupedSummary, RedTeamReport from .strategies import ( + MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy, BadLikertJudgeStrategy, @@ -21,6 +22,7 @@ from .types import RISK_CATEGORIES, AttackGoal, RedTeamConfig __all__ = [ + "MAX_ALLOWED_TURNS", "RISK_CATEGORIES", "AdversarialCaseGenerator", "AttackGoal", diff --git a/src/strands_evals/experimental/redteam/strategies/__init__.py b/src/strands_evals/experimental/redteam/strategies/__init__.py index 8355d5d1..ec6e97a8 100644 --- a/src/strands_evals/experimental/redteam/strategies/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/__init__.py @@ -1,5 +1,5 @@ from .bad_likert_judge import BadLikertJudgeStrategy -from .base import AttackRunResult, AttackStrategy +from .base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from .crescendo import CrescendoStrategy from .goat import GoatStrategy from .pair import PairStrategy @@ -25,6 +25,7 @@ __all__ = [ "BUILTIN_STRATEGIES", + "MAX_ALLOWED_TURNS", "AttackRunResult", "AttackStrategy", "BadLikertJudgeStrategy", diff --git a/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py b/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py index 5f8f6082..1013128e 100644 --- a/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py @@ -108,11 +108,12 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal + max_turns = self._turn_cap(max_turns) # The payload appears at turn 2; a smaller budget would send the role-set turn and never elicit, # reporting succeeded=False indistinguishably from "defended". diff --git a/src/strands_evals/experimental/redteam/strategies/base.py b/src/strands_evals/experimental/redteam/strategies/base.py index 667e57eb..d617f362 100644 --- a/src/strands_evals/experimental/redteam/strategies/base.py +++ b/src/strands_evals/experimental/redteam/strategies/base.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from strands.models.model import Model @@ -12,6 +12,10 @@ from ..case import RedTeamCase from .target_session import TargetSession +# Hard ceiling on turns per attack. `run_attack` defaults `max_turns` to it and clamps any larger value; a +# strategy's own ctor `max_turns` is the per-attack budget and wins when smaller. +MAX_ALLOWED_TURNS = 50 + @dataclass class AttackRunResult: @@ -43,6 +47,8 @@ class AttackStrategy(ABC): `snapshot`/`restore` if they backtrack). """ + MAX_ALLOWED_TURNS: ClassVar[int] = MAX_ALLOWED_TURNS + def __init__(self, *, label: str | None = None) -> None: """Initialize the strategy. @@ -69,7 +75,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -86,13 +92,23 @@ def run_attack( case: The red team case carrying the attack goal. target_session: Session for invoking the target, snapshotting/restoring its state, and reading its tool-use `trace`. - max_turns: Experiment-level ceiling. A strategy with its own `max_turns` should run - `min(self._max_turns, max_turns)`. + max_turns: Experiment-level ceiling; defaults to, and is clamped to, `MAX_ALLOWED_TURNS`. + Implementations resolve it with `self._turn_cap(max_turns, own=...)`. model: Model for any strategy-internal LLM calls; ctor model takes precedence. **kwargs: Reserved for forward compatibility. """ ... + def _turn_cap(self, max_turns: int | None, own: int | None = None) -> int: + """Resolve the turn budget for one attack. + + Args: + max_turns: The `run_attack` ceiling; `None` means `MAX_ALLOWED_TURNS`, larger values are clamped to it. + own: The strategy's own ctor budget, if any; wins when smaller. + """ + cap = self.MAX_ALLOWED_TURNS if max_turns is None else min(max_turns, self.MAX_ALLOWED_TURNS) + return cap if own is None else min(own, cap) + def reset(self) -> None: # noqa: B027 """Per-case reset hook; no-op by default. Override only if `self` holds mutable state.""" diff --git a/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py b/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py index aae6a26f..d13908a7 100644 --- a/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py @@ -158,7 +158,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -168,7 +168,7 @@ def run_attack( # Judge factories: each scoring/refusal call gets a fresh judge with no carryover. make_judge = partial(self._build_judge, resolved_model) make_refusal_judge = partial(self._build_refusal_judge, resolved_model) - turn_cap = min(self._max_turns, max_turns) + turn_cap = self._turn_cap(max_turns, own=self._max_turns) conversation: list[dict[str, Any]] = [] pruned_branches: list[dict[str, Any]] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/goat/__init__.py b/src/strands_evals/experimental/redteam/strategies/goat/__init__.py index bb228a7e..ac7b6ae5 100644 --- a/src/strands_evals/experimental/redteam/strategies/goat/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/goat/__init__.py @@ -145,7 +145,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -153,7 +153,7 @@ def run_attack( resolved_model = self._model or model attacker = self._build_attacker(goal, resolved_model) make_judge = partial(self._build_judge, resolved_model) - turn_cap = min(self._max_turns, max_turns) + turn_cap = self._turn_cap(max_turns, own=self._max_turns) conversation: list[dict[str, Any]] = [] attacks_used: list[str] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/pair/__init__.py b/src/strands_evals/experimental/redteam/strategies/pair/__init__.py index 275abbff..e0c39906 100644 --- a/src/strands_evals/experimental/redteam/strategies/pair/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/pair/__init__.py @@ -144,7 +144,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -152,7 +152,7 @@ def run_attack( resolved_model = self._model or model attacker = self._build_attacker(goal, resolved_model) make_judge = partial(self._build_judge, resolved_model) - turn_cap = min(self._max_turns, max_turns) + turn_cap = self._turn_cap(max_turns, own=self._max_turns) conversation: list[dict[str, Any]] = [] last_response = "" diff --git a/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py b/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py index 8a8288c9..e470c1ff 100644 --- a/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py @@ -42,12 +42,12 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal - turn_cap = min(self._max_turns, max_turns) + turn_cap = self._turn_cap(max_turns, own=self._max_turns) actor_profile = ActorProfile( traits=case.config.traits, context=goal.context, diff --git a/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py b/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py index 9271aa1e..0690f461 100644 --- a/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py @@ -130,13 +130,13 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int, + max_turns: int | None = None, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal make_judge = partial(self._build_judge, self._model or model) - turn_cap = min(self._max_turns, max_turns) + turn_cap = self._turn_cap(max_turns, own=self._max_turns) variants = self._variants[:turn_cap] conversation: list[dict[str, Any]] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/target_session.py b/src/strands_evals/experimental/redteam/strategies/target_session.py index da0cc278..54a48b4e 100644 --- a/src/strands_evals/experimental/redteam/strategies/target_session.py +++ b/src/strands_evals/experimental/redteam/strategies/target_session.py @@ -347,7 +347,7 @@ def as_target_session(target: Agent | MultiAgentBase | TargetSession) -> TargetS Use this in a custom task to turn a freshly built target into a session: session = as_target_session(build_agent()) - result = strategy.run_attack(case, session, max_turns=MAX_ALLOWED_TURNS) + result = strategy.run_attack(case, session) # max_turns defaults to MAX_ALLOWED_TURNS Args: target: A `strands.Agent` (wrapped in `StrandsAgentSession`), a `MultiAgentBase` such as a `Graph` or diff --git a/src/strands_evals/experimental/redteam/task.py b/src/strands_evals/experimental/redteam/task.py index 4cef4fbf..8eaab117 100644 --- a/src/strands_evals/experimental/redteam/task.py +++ b/src/strands_evals/experimental/redteam/task.py @@ -12,12 +12,11 @@ from .case import RedTeamCase from .strategies import AttackStrategy +from .strategies.base import MAX_ALLOWED_TURNS as MAX_ALLOWED_TURNS # re-exported for the pre-move import path from .strategies.target_session import StrandsMultiAgentSession, TargetSession, _build_session, as_target_session logger = logging.getLogger(__name__) -MAX_ALLOWED_TURNS = 50 - def _build_attacker_task( agent: Agent | MultiAgentBase | TargetSession | None, @@ -31,7 +30,7 @@ def _build_attacker_task( """Build a `task(case) -> {"output": conversation, "trajectory": tool_uses}` callable. Looks up each case's strategy by `metadata["strategy"]` and delegates the multi-turn loop to - `strategy.run_attack`, injecting a `TargetSession`. `MAX_ALLOWED_TURNS` is the hard ceiling. Run metadata + `strategy.run_attack`, injecting a `TargetSession` and the strategy's `MAX_ALLOWED_TURNS` ceiling. Run metadata is recorded into `run_meta` keyed by case name. Args: @@ -147,7 +146,8 @@ def _run_attack( Errors propagate: the base `Experiment` retries throttling, records any other failure as an error reason (which the report classifies as errored), and skips caching the failed case. """ - result = strategy.run_attack(case, session, max_turns=MAX_ALLOWED_TURNS, model=model) + # Passed explicitly: custom strategies written against the old contract declare `max_turns` without a default. + result = strategy.run_attack(case, session, max_turns=strategy.MAX_ALLOWED_TURNS, model=model) if run_meta is not None and case.name is not None: run_meta[case.name] = {**result.metadata, "pruned_branches": result.pruned_branches} return { diff --git a/tests/strands_evals/experimental/redteam/test_strategies.py b/tests/strands_evals/experimental/redteam/test_strategies.py index 96172f22..cd105c40 100644 --- a/tests/strands_evals/experimental/redteam/test_strategies.py +++ b/tests/strands_evals/experimental/redteam/test_strategies.py @@ -2,9 +2,11 @@ from unittest.mock import MagicMock, patch +from strands_evals.experimental import redteam +from strands_evals.experimental.redteam import strategies, task from strands_evals.experimental.redteam.case import RedTeamCase -from strands_evals.experimental.redteam.strategies import BUILTIN_STRATEGIES, PromptStrategy -from strands_evals.experimental.redteam.strategies.base import AttackRunResult +from strands_evals.experimental.redteam.strategies import BUILTIN_STRATEGIES, PromptStrategy, base +from strands_evals.experimental.redteam.strategies.base import AttackRunResult, AttackStrategy from strands_evals.experimental.redteam.strategies.target_session import TargetCheckpoint from strands_evals.experimental.redteam.types import AttackGoal, RedTeamConfig @@ -99,6 +101,42 @@ def test_prompt_strategy_ctor_max_turns_caps_below_ceiling(mock_simulator_cls): assert mock_simulator_cls.call_args.kwargs["max_turns"] == 3 +def test_max_allowed_turns_importable_from_every_location(): + """The constant moved to strategies/base.py; the old task.py import and the package roots still work.""" + assert redteam.MAX_ALLOWED_TURNS is strategies.MAX_ALLOWED_TURNS is task.MAX_ALLOWED_TURNS is base.MAX_ALLOWED_TURNS + assert base.MAX_ALLOWED_TURNS == AttackStrategy.MAX_ALLOWED_TURNS == 50 + + +@patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") +def test_run_attack_max_turns_defaults_to_ceiling(mock_simulator_cls): + """A custom task can omit max_turns; the strategy runs under MAX_ALLOWED_TURNS.""" + mock_simulator_cls.return_value.has_next.return_value = False + + strategy = PromptStrategy("gradual_escalation", "p {max_turns}", max_turns=100) + strategy.run_attack(_case(), _FakeSession(lambda _m: "r")) + + assert mock_simulator_cls.call_args.kwargs["max_turns"] == base.MAX_ALLOWED_TURNS + + +@patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") +def test_run_attack_max_turns_above_ceiling_is_clamped(mock_simulator_cls): + mock_simulator_cls.return_value.has_next.return_value = False + + strategy = PromptStrategy("gradual_escalation", "p {max_turns}", max_turns=100) + strategy.run_attack(_case(), _FakeSession(lambda _m: "r"), max_turns=100) + + assert mock_simulator_cls.call_args.kwargs["max_turns"] == base.MAX_ALLOWED_TURNS + + +def test_turn_cap_resolution(): + strategy = BUILTIN_STRATEGIES["gradual_escalation"] + assert strategy._turn_cap(None) == 50 + assert strategy._turn_cap(100) == 50 + assert strategy._turn_cap(7) == 7 + assert strategy._turn_cap(None, own=3) == 3 + assert strategy._turn_cap(7, own=10) == 7 + + @patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") def test_prompt_strategy_run_attack_handles_target_exception(mock_simulator_cls): mock_sim = MagicMock() From 20d153df0f7724ffc240e066f0a7dc33f9051237 Mon Sep 17 00:00:00 2001 From: nhungbi Date: Fri, 9 Oct 2026 13:43:11 -0400 Subject: [PATCH 3/3] fix(redteam): default run_attack max_turns to the module cap and address review --- SKILL.md | 4 +- .../experimental/redteam/README.md | 10 ++-- .../strategies/bad_likert_judge/__init__.py | 5 +- .../experimental/redteam/strategies/base.py | 25 +++------ .../redteam/strategies/crescendo/__init__.py | 6 +-- .../redteam/strategies/goat/__init__.py | 6 +-- .../redteam/strategies/pair/__init__.py | 6 +-- .../strategies/prompt_strategy/__init__.py | 6 +-- .../strategies/sequentialbreak/__init__.py | 6 +-- .../redteam/strategies/target_session.py | 27 +++++----- .../experimental/redteam/task.py | 4 +- .../experimental/redteam/test_strategies.py | 53 +++++++++++++------ .../redteam/test_target_session.py | 2 +- 13 files changed, 87 insertions(+), 73 deletions(-) diff --git a/SKILL.md b/SKILL.md index 16b66c8b..ab004325 100644 --- a/SKILL.md +++ b/SKILL.md @@ -418,7 +418,7 @@ Built-in strategies: | `BadLikertJudgeStrategy(...)` | Likert-scale judge-prompt attack | | `SequentialBreakStrategy(...)` | Narrative-scaffold attack (PR #254) | -Targets and sessions in `redteam.strategies`: `StrandsAgentSession`, `StrandsMultiAgentSession`, `TargetCheckpoint`, `TargetSession` (Protocol). +Targets and sessions in `redteam.strategies`: `StrandsAgentSession`, `StrandsMultiAgentSession`, `TargetCheckpoint`, `TargetSession` (Protocol), and `as_target_session(target)`, which wraps an `Agent` / `MultiAgentBase` in the right session for a custom task. Cases are typed `RedTeamCase` carrying a `RedTeamConfig(attack_goal=AttackGoal(risk_category=..., actor_goal=..., severity=..., success_criteria=...), traits={...})`. `RISK_CATEGORIES` is the canonical category list for case generation. @@ -426,7 +426,7 @@ Cases are typed `RedTeamCase` carrying a `RedTeamConfig(attack_goal=AttackGoal(r `RedTeamReport` adds case-centric grouping: one `AttackResult` per case, plus `GroupedSummary` aggregations exposed via `report.by_risk_category()` and `report.by_strategy()`. Severity is recorded on each `AttackResult` (no `by_severity()` aggregator). `trajectory` holds raw tool I/O — sanitize before sharing if tools return sensitive data. -**Hard turn cap:** `task.py` enforces `MAX_ALLOWED_TURNS = 50` regardless of a strategy's own `max_turns`. A `CrescendoStrategy(max_turns=100)` will still stop at 50 inside `RedTeamExperiment`. Lower turn budgets honor the strategy setting. +**Hard turn cap:** the built-in task passes `MAX_ALLOWED_TURNS = 50` (`strategies/base.py`) regardless of a strategy's own `max_turns`, and `run_attack`'s `max_turns` defaults to it. A `CrescendoStrategy(max_turns=100)` will still stop at 50 inside `RedTeamExperiment`. Lower turn budgets honor the strategy setting. Stability: `experimental.redteam` APIs may change in a minor release. Breaking changes (renames, removed args, changed defaults) go through a deprecation cycle with a `DeprecationWarning` for at least one minor version. diff --git a/src/strands_evals/experimental/redteam/README.md b/src/strands_evals/experimental/redteam/README.md index 37607310..d01e6d1c 100644 --- a/src/strands_evals/experimental/redteam/README.md +++ b/src/strands_evals/experimental/redteam/README.md @@ -73,8 +73,10 @@ report = experiment.run_evaluations() # sync; equivalent to run_evaluations_asy ## Attack strategies -All strategies share the `run_attack(case, target_session, *, max_turns, model)` -contract and talk to the target only through `target_session.invoke(...)`. +All strategies share the `run_attack(case, target_session, *, max_turns=MAX_ALLOWED_TURNS, model=None)` +contract and talk to the target only through `target_session.invoke(...)`. To call `run_attack` from your own +task, wrap a freshly built target with `as_target_session(target)`: an `Agent` becomes a `StrandsAgentSession`, a +`Graph` / `Swarm` becomes a `StrandsMultiAgentSession`, and a `TargetSession` is passed through. | Strategy | Mechanism | Attacker LLM? | Paper | |----------|-----------|---------------|-------| @@ -84,7 +86,7 @@ contract and talk to the target only through `target_session.invoke(...)`. | `BadLikertJudgeStrategy` | Casts the target as a harmfulness-rating judge, elicits a top-score example | no | [Unit 42](https://unit42.paloaltonetworks.com/multi-turn-technique-jailbreaks-llms/) | | `SequentialBreakStrategy` | Hides the harmful request among benign siblings in one narrative scaffold | no | [arXiv:2411.06426](https://arxiv.org/abs/2411.06426) | -Each strategy accepts a `max_turns=` kwarg (its own per-attack ceiling); `run_attack` additionally caps every strategy at a hard `MAX_ALLOWED_TURNS = 50` (`strategies/base.py`, exported from `strands_evals.experimental.redteam`), so a strategy configured higher will be silently clamped. A custom task can omit `run_attack`'s `max_turns`, which defaults to that ceiling; a larger value is clamped to it. Pass `label="..."` to compare two instances of the same strategy in one experiment (e.g. `CrescendoStrategy(max_turns=5, label="cresc_short")`); duplicate labels raise at construction. +Each strategy accepts a `max_turns=` kwarg (its own per-attack ceiling); the built-in task additionally caps every strategy at a hard `MAX_ALLOWED_TURNS = 50` (`strategies/base.py`, exported from `strands_evals.experimental.redteam`), so a strategy configured higher will be silently clamped. A custom task can omit `run_attack`'s `max_turns`, which defaults to that ceiling. A custom `AttackStrategy` subclass should declare the same default (`max_turns: int = MAX_ALLOWED_TURNS`); one that declares `max_turns: int` without it still works with `RedTeamExperiment`, which always passes the value, but a custom task must pass `max_turns=` explicitly. Pass `label="..."` to compare two instances of the same strategy in one experiment (e.g. `CrescendoStrategy(max_turns=5, label="cresc_short")`); duplicate labels raise at construction. `PromptStrategy` (a no-attacker-LLM, system-prompt-template strategy) and the `BUILTIN_STRATEGIES` registry are the extension points for adding new template-driven strategies without subclassing — see `strategies/prompt_strategy/`. The full set of exported symbols (experiment, cases, evaluator, target sessions) is the `__all__` of `strands_evals.experimental.redteam`. @@ -226,7 +228,7 @@ redteam/ ├── experiment.py # RedTeamExperiment ├── case.py # RedTeamCase ├── report.py # RedTeamReport, AttackResult, GroupedSummary -├── task.py # wraps Agent / MultiAgentBase into a TargetSession per case +├── task.py # built-in task: runs each case's strategy against its TargetSession ├── types/ # AttackGoal, RedTeamConfig, RISK_CATEGORIES ├── generators/ # AdversarialCaseGenerator ├── evaluators/ # AttackSuccessEvaluator + judge prompt templates diff --git a/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py b/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py index 1013128e..3d5a20f0 100644 --- a/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/bad_likert_judge/__init__.py @@ -17,7 +17,7 @@ from strands.models.model import Model from ...utils import _put_model_field -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from . import bad_likert_judge_v0 as blj_v0 if TYPE_CHECKING: @@ -108,12 +108,11 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal - max_turns = self._turn_cap(max_turns) # The payload appears at turn 2; a smaller budget would send the role-set turn and never elicit, # reporting succeeded=False indistinguishably from "defended". diff --git a/src/strands_evals/experimental/redteam/strategies/base.py b/src/strands_evals/experimental/redteam/strategies/base.py index d617f362..21d5ac68 100644 --- a/src/strands_evals/experimental/redteam/strategies/base.py +++ b/src/strands_evals/experimental/redteam/strategies/base.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any from strands.models.model import Model @@ -12,8 +12,8 @@ from ..case import RedTeamCase from .target_session import TargetSession -# Hard ceiling on turns per attack. `run_attack` defaults `max_turns` to it and clamps any larger value; a -# strategy's own ctor `max_turns` is the per-attack budget and wins when smaller. +# Default `max_turns` for `run_attack`, and the hard ceiling the built-in task always passes. A strategy's own +# ctor `max_turns` is its per-attack budget and wins when smaller. MAX_ALLOWED_TURNS = 50 @@ -47,8 +47,6 @@ class AttackStrategy(ABC): `snapshot`/`restore` if they backtrack). """ - MAX_ALLOWED_TURNS: ClassVar[int] = MAX_ALLOWED_TURNS - def __init__(self, *, label: str | None = None) -> None: """Initialize the strategy. @@ -75,7 +73,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -92,23 +90,14 @@ def run_attack( case: The red team case carrying the attack goal. target_session: Session for invoking the target, snapshotting/restoring its state, and reading its tool-use `trace`. - max_turns: Experiment-level ceiling; defaults to, and is clamped to, `MAX_ALLOWED_TURNS`. - Implementations resolve it with `self._turn_cap(max_turns, own=...)`. + max_turns: Experiment-level ceiling; defaults to `MAX_ALLOWED_TURNS`. Overrides should keep that + default so custom tasks can omit it. A strategy with its own `max_turns` should run + `min(self._max_turns, max_turns)`. model: Model for any strategy-internal LLM calls; ctor model takes precedence. **kwargs: Reserved for forward compatibility. """ ... - def _turn_cap(self, max_turns: int | None, own: int | None = None) -> int: - """Resolve the turn budget for one attack. - - Args: - max_turns: The `run_attack` ceiling; `None` means `MAX_ALLOWED_TURNS`, larger values are clamped to it. - own: The strategy's own ctor budget, if any; wins when smaller. - """ - cap = self.MAX_ALLOWED_TURNS if max_turns is None else min(max_turns, self.MAX_ALLOWED_TURNS) - return cap if own is None else min(own, cap) - def reset(self) -> None: # noqa: B027 """Per-case reset hook; no-op by default. Override only if `self` holds mutable state.""" diff --git a/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py b/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py index f46ef78d..c18bd30b 100644 --- a/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/crescendo/__init__.py @@ -17,7 +17,7 @@ from strands.models.model import Model from ...utils import _put_model_field -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from . import crescendo_v0 if TYPE_CHECKING: @@ -161,7 +161,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -171,7 +171,7 @@ def run_attack( # Judge factories: each scoring/refusal call gets a fresh judge with no carryover. make_judge = partial(self._build_judge, resolved_model) make_refusal_judge = partial(self._build_refusal_judge, resolved_model) - turn_cap = self._turn_cap(max_turns, own=self._max_turns) + turn_cap = min(self._max_turns, max_turns) conversation: list[dict[str, Any]] = [] pruned_branches: list[dict[str, Any]] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/goat/__init__.py b/src/strands_evals/experimental/redteam/strategies/goat/__init__.py index a398e0ee..c5532a22 100644 --- a/src/strands_evals/experimental/redteam/strategies/goat/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/goat/__init__.py @@ -17,7 +17,7 @@ from strands.models.model import Model from ...utils import _put_model_field -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from . import goat_v0 if TYPE_CHECKING: @@ -145,7 +145,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -153,7 +153,7 @@ def run_attack( resolved_model = self._model or model attacker = self._build_attacker(goal, resolved_model) make_judge = partial(self._build_judge, resolved_model) - turn_cap = self._turn_cap(max_turns, own=self._max_turns) + turn_cap = min(self._max_turns, max_turns) conversation: list[dict[str, Any]] = [] attacks_used: list[str] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/pair/__init__.py b/src/strands_evals/experimental/redteam/strategies/pair/__init__.py index c9091127..e74f58fe 100644 --- a/src/strands_evals/experimental/redteam/strategies/pair/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/pair/__init__.py @@ -17,7 +17,7 @@ from strands.models.model import Model from ...utils import _put_model_field -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from ..target_session import _single_shot_attempts from . import pair_v0 @@ -145,7 +145,7 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: @@ -153,7 +153,7 @@ def run_attack( resolved_model = self._model or model attacker = self._build_attacker(goal, resolved_model) make_judge = partial(self._build_judge, resolved_model) - turn_cap = self._turn_cap(max_turns, own=self._max_turns) + turn_cap = min(self._max_turns, max_turns) conversation: list[dict[str, Any]] = [] last_response = "" diff --git a/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py b/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py index e470c1ff..3bd744e6 100644 --- a/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/prompt_strategy/__init__.py @@ -9,7 +9,7 @@ from .....simulation.actor_simulator import ActorSimulator from .....types.simulation import ActorProfile -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy if TYPE_CHECKING: from ...case import RedTeamCase @@ -42,12 +42,12 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal - turn_cap = self._turn_cap(max_turns, own=self._max_turns) + turn_cap = min(self._max_turns, max_turns) actor_profile = ActorProfile( traits=case.config.traits, context=goal.context, diff --git a/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py b/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py index 0690f461..9a9f91ee 100644 --- a/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py +++ b/src/strands_evals/experimental/redteam/strategies/sequentialbreak/__init__.py @@ -22,7 +22,7 @@ from strands.models.model import Model from ...utils import _put_model_field -from ..base import AttackRunResult, AttackStrategy +from ..base import MAX_ALLOWED_TURNS, AttackRunResult, AttackStrategy from ..target_session import _single_shot_attempts from . import sequentialbreak_v0 @@ -130,13 +130,13 @@ def run_attack( case: RedTeamCase, target_session: TargetSession, *, - max_turns: int | None = None, + max_turns: int = MAX_ALLOWED_TURNS, model: Model | str | None = None, **kwargs: Any, ) -> AttackRunResult: goal = case.config.attack_goal make_judge = partial(self._build_judge, self._model or model) - turn_cap = self._turn_cap(max_turns, own=self._max_turns) + turn_cap = min(self._max_turns, max_turns) variants = self._variants[:turn_cap] conversation: list[dict[str, Any]] = [] diff --git a/src/strands_evals/experimental/redteam/strategies/target_session.py b/src/strands_evals/experimental/redteam/strategies/target_session.py index 54a48b4e..1f8988a6 100644 --- a/src/strands_evals/experimental/redteam/strategies/target_session.py +++ b/src/strands_evals/experimental/redteam/strategies/target_session.py @@ -349,6 +349,9 @@ def as_target_session(target: Agent | MultiAgentBase | TargetSession) -> TargetS session = as_target_session(build_agent()) result = strategy.run_attack(case, session) # max_turns defaults to MAX_ALLOWED_TURNS + Build a fresh target for every case. The session's `reset()` clears only the conversation, so a target + shared across cases carries other state (such as `agent.state`) from one case into the next. + Args: target: A `strands.Agent` (wrapped in `StrandsAgentSession`), a `MultiAgentBase` such as a `Graph` or `Swarm` (wrapped in `StrandsMultiAgentSession`), or a ready `TargetSession` (returned as is). @@ -364,33 +367,33 @@ def as_target_session(target: Agent | MultiAgentBase | TargetSession) -> TargetS def _build_session( - agent: Agent | MultiAgentBase | TargetSession, + target: Agent | MultiAgentBase | TargetSession, *, baseline: Any = None, ) -> TargetSession: """Wrap an `Agent` / `MultiAgentBase`, or pass a `TargetSession` through. Args: - agent: The target to wrap, or a ready `TargetSession`. + target: The target to wrap, or a ready `TargetSession`. baseline: Clean snapshot the wrapped session resets to between cases. Ignored for a passed-in `TargetSession`. Typed `Any` because the two session types use different opaque baseline shapes. Raises: - TypeError: If `agent` is not an `Agent`, `MultiAgentBase`, or a structural `TargetSession` (must expose + TypeError: If `target` is not an `Agent`, `MultiAgentBase`, or a structural `TargetSession` (must expose `invoke`/`reset`/`snapshot`/`restore` and a `trace: list`). """ - if isinstance(agent, Agent): - return StrandsAgentSession(agent, baseline=baseline) - if isinstance(agent, MultiAgentBase): - return StrandsMultiAgentSession(agent, baseline=baseline) + if isinstance(target, Agent): + return StrandsAgentSession(target, baseline=baseline) + if isinstance(target, MultiAgentBase): + return StrandsMultiAgentSession(target, baseline=baseline) # Structural check: TargetSession is a Protocol. The `trace: list` check is # load-bearing because the task runner dereferences `.trace` directly. - has_methods = all(callable(getattr(agent, method, None)) for method in ("invoke", "reset", "snapshot", "restore")) - if has_methods and isinstance(getattr(agent, "trace", None), list): - return agent + has_methods = all(callable(getattr(target, method, None)) for method in ("invoke", "reset", "snapshot", "restore")) + if has_methods and isinstance(getattr(target, "trace", None), list): + return target raise TypeError( - f"agent must be a strands.Agent, strands.multiagent.MultiAgentBase, or a TargetSession, " - f"got {type(agent).__name__!r}; wrap a custom target in a TargetSession so the strategy " + f"target must be a strands.Agent, strands.multiagent.MultiAgentBase, or a TargetSession, " + f"got {type(target).__name__!r}; wrap a custom target in a TargetSession so the strategy " "can snapshot/restore its state." ) diff --git a/src/strands_evals/experimental/redteam/task.py b/src/strands_evals/experimental/redteam/task.py index 8eaab117..319ba4fc 100644 --- a/src/strands_evals/experimental/redteam/task.py +++ b/src/strands_evals/experimental/redteam/task.py @@ -30,7 +30,7 @@ def _build_attacker_task( """Build a `task(case) -> {"output": conversation, "trajectory": tool_uses}` callable. Looks up each case's strategy by `metadata["strategy"]` and delegates the multi-turn loop to - `strategy.run_attack`, injecting a `TargetSession` and the strategy's `MAX_ALLOWED_TURNS` ceiling. Run metadata + `strategy.run_attack`, injecting a `TargetSession`. `MAX_ALLOWED_TURNS` is the hard ceiling. Run metadata is recorded into `run_meta` keyed by case name. Args: @@ -147,7 +147,7 @@ def _run_attack( error reason (which the report classifies as errored), and skips caching the failed case. """ # Passed explicitly: custom strategies written against the old contract declare `max_turns` without a default. - result = strategy.run_attack(case, session, max_turns=strategy.MAX_ALLOWED_TURNS, model=model) + result = strategy.run_attack(case, session, max_turns=MAX_ALLOWED_TURNS, model=model) if run_meta is not None and case.name is not None: run_meta[case.name] = {**result.metadata, "pruned_branches": result.pruned_branches} return { diff --git a/tests/strands_evals/experimental/redteam/test_strategies.py b/tests/strands_evals/experimental/redteam/test_strategies.py index cd105c40..7a82cfed 100644 --- a/tests/strands_evals/experimental/redteam/test_strategies.py +++ b/tests/strands_evals/experimental/redteam/test_strategies.py @@ -1,11 +1,23 @@ """Tests for AttackStrategy implementations.""" +import inspect from unittest.mock import MagicMock, patch +import pytest + from strands_evals.experimental import redteam from strands_evals.experimental.redteam import strategies, task from strands_evals.experimental.redteam.case import RedTeamCase -from strands_evals.experimental.redteam.strategies import BUILTIN_STRATEGIES, PromptStrategy, base +from strands_evals.experimental.redteam.strategies import ( + BUILTIN_STRATEGIES, + BadLikertJudgeStrategy, + CrescendoStrategy, + GoatStrategy, + PairStrategy, + PromptStrategy, + SequentialBreakStrategy, + base, +) from strands_evals.experimental.redteam.strategies.base import AttackRunResult, AttackStrategy from strands_evals.experimental.redteam.strategies.target_session import TargetCheckpoint from strands_evals.experimental.redteam.types import AttackGoal, RedTeamConfig @@ -104,12 +116,29 @@ def test_prompt_strategy_ctor_max_turns_caps_below_ceiling(mock_simulator_cls): def test_max_allowed_turns_importable_from_every_location(): """The constant moved to strategies/base.py; the old task.py import and the package roots still work.""" assert redteam.MAX_ALLOWED_TURNS is strategies.MAX_ALLOWED_TURNS is task.MAX_ALLOWED_TURNS is base.MAX_ALLOWED_TURNS - assert base.MAX_ALLOWED_TURNS == AttackStrategy.MAX_ALLOWED_TURNS == 50 + assert base.MAX_ALLOWED_TURNS == 50 + + +@pytest.mark.parametrize( + "strategy_cls", + [ + AttackStrategy, + BadLikertJudgeStrategy, + CrescendoStrategy, + GoatStrategy, + PairStrategy, + PromptStrategy, + SequentialBreakStrategy, + ], +) +def test_run_attack_max_turns_defaults_to_ceiling(strategy_cls): + """A custom task can omit max_turns on every strategy.""" + default = inspect.signature(strategy_cls.run_attack).parameters["max_turns"].default + assert default == base.MAX_ALLOWED_TURNS @patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") -def test_run_attack_max_turns_defaults_to_ceiling(mock_simulator_cls): - """A custom task can omit max_turns; the strategy runs under MAX_ALLOWED_TURNS.""" +def test_run_attack_without_max_turns_runs_under_ceiling(mock_simulator_cls): mock_simulator_cls.return_value.has_next.return_value = False strategy = PromptStrategy("gradual_escalation", "p {max_turns}", max_turns=100) @@ -119,22 +148,14 @@ def test_run_attack_max_turns_defaults_to_ceiling(mock_simulator_cls): @patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") -def test_run_attack_max_turns_above_ceiling_is_clamped(mock_simulator_cls): +def test_run_attack_explicit_max_turns_above_ceiling_is_honored(mock_simulator_cls): + """Only the built-in task enforces MAX_ALLOWED_TURNS; a direct caller's explicit value is not clamped.""" mock_simulator_cls.return_value.has_next.return_value = False strategy = PromptStrategy("gradual_escalation", "p {max_turns}", max_turns=100) - strategy.run_attack(_case(), _FakeSession(lambda _m: "r"), max_turns=100) - - assert mock_simulator_cls.call_args.kwargs["max_turns"] == base.MAX_ALLOWED_TURNS + strategy.run_attack(_case(), _FakeSession(lambda _m: "r"), max_turns=80) - -def test_turn_cap_resolution(): - strategy = BUILTIN_STRATEGIES["gradual_escalation"] - assert strategy._turn_cap(None) == 50 - assert strategy._turn_cap(100) == 50 - assert strategy._turn_cap(7) == 7 - assert strategy._turn_cap(None, own=3) == 3 - assert strategy._turn_cap(7, own=10) == 7 + assert mock_simulator_cls.call_args.kwargs["max_turns"] == 80 @patch("strands_evals.experimental.redteam.strategies.prompt_strategy.ActorSimulator") diff --git a/tests/strands_evals/experimental/redteam/test_target_session.py b/tests/strands_evals/experimental/redteam/test_target_session.py index df87e033..7144ae19 100644 --- a/tests/strands_evals/experimental/redteam/test_target_session.py +++ b/tests/strands_evals/experimental/redteam/test_target_session.py @@ -341,7 +341,7 @@ def test_rejects_session_without_trace_list(self): as_target_session(_NoTraceSession()) def test_rejects_bare_callable(self): - with pytest.raises(TypeError, match="TargetSession"): + with pytest.raises(TypeError, match="^target must be .*TargetSession"): as_target_session(lambda message: "reply") def test_exported_from_package_root(self):