Version to 1.15.3. Various bug fixes in agents and evals. Expanded command agent tool. Update cyclopts.

Author: monoxgas

docs/sdk/agent.mdx

@@ -91,10 +91,20 @@ The maximum number of steps (generation + tool calls).
 ### model
 
 ```python
-model: str | None = Config(default=None)
+model: str | Generator | None = Config(
+    default=None, expose_as=str | None
+)
 ```
 
-Inference model (rigging generator identifier).
+Inference model (rigging generator or identifier).
+
+### model\_name
+
+```python
+model_name: str | None
+```
+
+The model name if specified as a string, otherwise None.
 
 ### name
 
@@ -160,6 +170,34 @@ tools: Annotated[
 
 Tools the agent can use.
 
+### clone
+
+```python
+clone() -> te.Self
+```
+
+Clone the agent.
+
+**Returns:**
+
+* `Self`
+  –A new Agent instance with the same attributes as this one.
+
+<Accordion title="Source code in dreadnode/agent/agent.py" icon="code">
+```python
+def clone(self) -> te.Self:
+    """
+    Clone the agent.
+
+    Returns:
+        A new Agent instance with the same attributes as this one.
+    """
+    return self.model_copy(deep=True)
+```
+
+
+</Accordion>
+
 ### get\_prompt
 
 ```python
@@ -203,6 +241,102 @@ def reset(self) -> Thread:
 ```
 
 
+</Accordion>
+
+### with\_
+
+```python
+with_(
+    *,
+    name: str | None = None,
+    description: str | None = None,
+    tags: list[str] | None = None,
+    label: str | None = None,
+    model: str | Generator | None = None,
+    instructions: str | None = None,
+    max_steps: int | None = None,
+    caching: CacheMode | None = None,
+    tools: list[AnyTool | Toolset] | None = None,
+    tool_mode: ToolMode | None = None,
+    hooks: list[Hook] | None = None,
+    stop_conditions: list[StopCondition] | None = None,
+    scorers: ScorersLike[AgentResult] | None = None,
+    assert_scores: list[str] | Literal[True] | None = None,
+    append: bool = False,
+) -> te.Self
+```
+
+Clone the agent and modify its attributes.
+
+**Returns:**
+
+* `Self`
+  –A new Agent instance with the modified attributes.
+
+<Accordion title="Source code in dreadnode/agent/agent.py" icon="code">
+```python
+def with_(
+    self,
+    *,
+    name: str | None = None,
+    description: str | None = None,
+    tags: list[str] | None = None,
+    label: str | None = None,
+    model: str | rg.Generator | None = None,
+    instructions: str | None = None,
+    max_steps: int | None = None,
+    caching: rg.caching.CacheMode | None = None,
+    tools: list[AnyTool | Toolset] | None = None,
+    tool_mode: ToolMode | None = None,
+    hooks: list[Hook] | None = None,
+    stop_conditions: list[StopCondition] | None = None,
+    scorers: ScorersLike[AgentResult] | None = None,
+    assert_scores: list[str] | t.Literal[True] | None = None,
+    append: bool = False,
+) -> te.Self:
+    """
+    Clone the agent and modify its attributes.
+
+    Returns:
+        A new Agent instance with the modified attributes.
+    """
+    new = self.clone()
+
+    new.name = name or new.name
+    new.description = description or new.description
+    new.label = label or new.label
+    new.model = model or new.model
+    new.instructions = instructions or new.instructions
+    new.max_steps = max_steps or new.max_steps
+    new.caching = caching or new.caching
+    new.tool_mode = tool_mode or new.tool_mode
+
+    if append:
+        new.tags = [*new.tags, *(tags or [])]
+        new.tools = [*new.tools, *(tools or [])]
+        new.hooks = [*new.hooks, *(hooks or [])]
+        new.stop_conditions = [*new.stop_conditions, *(stop_conditions or [])]
+        new.scorers = [*new.scorers, *(scorers or [])]
+        if isinstance(assert_scores, bool):
+            new.assert_scores = assert_scores
+        elif isinstance(new.assert_scores, list):
+            new.assert_scores = [*new.assert_scores, *(assert_scores or [])]
+        else:
+            new.assert_scores = assert_scores or new.assert_scores
+    else:
+        new.tags = tags if tags is not None else new.tags
+        new.tools = tools if tools is not None else new.tools
+        new.hooks = hooks if hooks is not None else new.hooks
+        new.stop_conditions = (
+            stop_conditions if stop_conditions is not None else new.stop_conditions
+        )
+        new.scorers = scorers if scorers is not None else new.scorers
+        new.assert_scores = assert_scores if assert_scores is not None else new.assert_scores
+
+    return new
+```
+
+
 </Accordion>
 
 AgentWarning

docs/sdk/agent_tools.mdx

@@ -330,39 +330,44 @@ command(
     timeout: int = 120,
     cwd: str | None = None,
     env: dict[str, str] | None = None,
+    input: str | None = None,
 ) -> str
 ```
 
 Execute a shell command.
 
-Use this tool to run system utilities and command-line programs (e.g., `ls`, `cat`, `grep`). It is designed for straightforward, single-shot operations and returns the combined output and error streams.
-
 **Best Practices**
 
-* Argument Format: The command and its arguments *must* be provided as a list of strings (e.g., `["ls", "-la", "/tmp"]`), not as a single string.
-* No Shell Syntax: Does not use a shell. Features like pipes (`|`), redirection (`>`), and variable expansion (`$VAR`) are not supported.
-* Error on Failure: The tool will raise a `RuntimeError` if the command returns a non-zero exit code.
+* Argument Format: Command and arguments must be a list of strings.
+* No Shell Syntax: Does not use a shell (no pipes, redirection, var expansion, etc.).
+* Error on Failure: Raises RuntimeError for non-zero exit codes.
+* Use input Parameter: Send data to the command's standard input to avoid hanging.
 
 **Parameters:**
 
 * **`cmd`**
   (`list[str]`)
-  –The command to execute, provided as a list of strings.
+  –The command to execute as a list of strings.
 * **`timeout`**
   (`int`, default:
   `120`
   )
-  –Maximum time in seconds to allow for command execution.
+  –Maximum execution time in seconds.
 * **`cwd`**
   (`str | None`, default:
   `None`
   )
-  –The working directory in which to execute the command.
+  –The working directory for the command.
 * **`env`**
   (`dict[str, str] | None`, default:
   `None`
   )
-  –Optional environment variables to set for the command.
+  –Environment variables for the command.
+* **`input`**
+  (`str | None`, default:
+  `None`
+  )
+  –Optional string to send to the command's standard input.
 
 <Accordion title="Source code in dreadnode/agent/tools/execute.py" icon="code">
 ```python
@@ -373,52 +378,85 @@ async def command(
     timeout: int = 120,
     cwd: str | None = None,
     env: dict[str, str] | None = None,
+    input: str | None = None,
 ) -> str:
     """
     Execute a shell command.
 
-    Use this tool to run system utilities and command-line programs (e.g., `ls`, `cat`, `grep`). \
-    It is designed for straightforward, single-shot operations and returns the combined output and error streams.
-
     ## Best Practices
-    - Argument Format: The command and its arguments *must* be provided as a \
-    list of strings (e.g., `["ls", "-la", "/tmp"]`), not as a single string.
-    - No Shell Syntax: Does not use a shell. Features like pipes (`|`), \
-    redirection (`>`), and variable expansion (`$VAR`) are not supported.
-    - Error on Failure: The tool will raise a `RuntimeError` if the command returns a non-zero exit code.
+    - Argument Format: Command and arguments must be a list of strings.
+    - No Shell Syntax: Does not use a shell (no pipes, redirection, var expansion, etc.).
+    - Error on Failure: Raises RuntimeError for non-zero exit codes.
+    - Use input Parameter: Send data to the command's standard input to avoid hanging.
 
     Args:
-        cmd: The command to execute, provided as a list of strings.
-        timeout: Maximum time in seconds to allow for command execution.
-        cwd: The working directory in which to execute the command.
-        env: Optional environment variables to set for the command.
+        cmd: The command to execute as a list of strings.
+        timeout: Maximum execution time in seconds.
+        cwd: The working directory for the command.
+        env: Environment variables for the command.
+        input: Optional string to send to the command's standard input.
     """
+    command_str = " ".join(cmd)
+    logger.debug(f"Executing '{command_str}'")
+
+    process_env = os.environ.copy()
+    if env:
+        process_env.update(env)
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+        stdin=asyncio.subprocess.PIPE if input is not None else None,
+        env=process_env,
+        cwd=cwd,
+    )
+
+    output = ""
+
+    async def read_stdout() -> None:
+        nonlocal output
+
+        if not proc.stdout:
+            return
+
+        while True:
+            chunk = await proc.stdout.read(1024)
+            if not chunk:
+                break
+            output += chunk.decode(errors="replace")
+
+    async def write_and_close_stdin() -> None:
+        if proc.stdin and input:
+            proc.stdin.write(input.encode())
+            await proc.stdin.drain()
+            proc.stdin.close()
+
     try:
-        command_str = " ".join(cmd)
-        logger.debug(f"Executing '{command_str}'")
-        proc = await asyncio.create_subprocess_exec(
-            *cmd,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            env=env,
-            cwd=cwd,
+        await asyncio.wait_for(
+            asyncio.gather(read_stdout(), write_and_close_stdin()), timeout=timeout
         )
-        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
-        output = stdout.decode() + stderr.decode()
+        await proc.wait()
+
     except asyncio.TimeoutError as e:
-        logger.warning(f"Command '{command_str}' timed out after {timeout} seconds.")
+        error_message = f"Command '{command_str}' timed out after {timeout} seconds."
+        if output:
+            error_message += f"\n\nPartial Output:\n{output}"
+        logger.warning(error_message)
+
         with contextlib.suppress(OSError):
             proc.kill()
-        raise TimeoutError(f"Command timed out after {timeout} seconds") from e
-    except Exception as e:
-        logger.error(f"Error executing '{command_str}': {e}")
-        raise
+            await proc.wait()
+
+        raise TimeoutError(error_message) from e
 
     if proc.returncode != 0:
-        logger.error(f"Command '{command_str}' failed with return code {proc.returncode}: {output}")
-        raise RuntimeError(f"Command failed ({proc.returncode}): {output}")
+        logger.error(
+            f"Command '{command_str}' failed with return code {proc.returncode}:\n{output}"
+        )
+        raise RuntimeError(f"Command failed ({proc.returncode}):\n{output}")
 
-    logger.debug(f"Command '{command_str}':\n{output}")
+    logger.debug(f"Command '{command_str}' completed:\n{output}")
     return output
 ```
 

docs/sdk/eval.mdx

@@ -122,11 +122,11 @@ The name of the evaluation.
 
 ```python
 parameters: dict[str, list[Any]] | None = Config(
-    default=None
+    default=None, expose_as=str | None
 )
 ```
 
-A dictionary defining a parameter space to run experiments against.
+A dictionary (or JSON string) defining a parameter space to run experiments against.
 A scenario will be created for every combination of the parameters defined here.
 Key names should align with arguments on the task assigned with a `Config` context.
 
@@ -157,7 +157,7 @@ A list of tags to associate during tracing.
 ### task
 
 ```python
-task: Task[[In], Out] | str
+task: Task[..., Out] | str
 ```
 
 The task to evaluate. Can be a Task object or a string representing qualified task name.

docs/sdk/main.mdx

@@ -2582,7 +2582,7 @@ def task_and_run(
                     _tracer=_tracer,
                 )
             )
-            self.log_inputs(**(inputs or {}))
+            self.log_inputs(**(inputs or {}), to="run")
 
         task_span = stack.enter_context(
             self.task_span(name, label=label, tags=tags, _tracer=_tracer)