fix(validate): error when verifier S3 solutions path doesn't match task key

README.md

@@ -8,88 +8,278 @@ The Fleet Python SDK provides programmatic access to Fleet's environment infrast
 
 ## Installation
 
-Install the Fleet SDK using pip:
-
 ```bash
 pip install fleet-python
 ```
 
-### Alpha/Pre-release Versions
+## API Key Setup
 
-To install the latest alpha or pre-release version:
+Get your API key from the [Fleet Dashboard](https://fleetai.com/dashboard/api-keys), then set it as an environment variable:
 
 ```bash
-pip install --pre fleet-python
+export FLEET_API_KEY="sk_your_key_here"
 ```
 
-To install a specific alpha version:
+## Tasks
 
-```bash
-pip install fleet-python==0.2.64-alpha1
+A **Task** represents a unit of work for an agent to complete within an environment. Each task combines:
+
+- **Prompt**: Instructions describing what needs to be done
+- **Environment**: The environment configuration (`env_key`, `data_key`, `env_variables`)
+- **Verifier**: Code that validates task completion and returns a score (0.0 to 1.0)
+
+Tasks provide all the configuration needed to spin up an environment and verify an agent's work.
+
+### Task Properties
+
+| Property        | Description                                           |
+| --------------- | ----------------------------------------------------- |
+| `key`           | Unique task identifier                                |
+| `prompt`        | Instructions for the agent                            |
+| `env_key`       | Environment identifier (e.g., `"hubspot:v1.2"`)       |
+| `data_key`      | Data configuration identifier                         |
+| `env_variables` | Environment variables for the task                    |
+| `metadata`      | Additional info (e.g., `avg_steps` for task difficulty) |
+
+## Quick Start
+
+```python
+import fleet
+
+async def main():
+    # Load a task
+    tasks = await fleet.load_tasks_async(
+        keys=["task_abcdef"]
+    )
+    task = tasks[0]
+
+    # Create an environment from the task
+    env = await fleet.env.make_async(
+        env_key=task.env_key,
+        data_key=task.data_key,
+        env_variables=task.env_variables,
+        ttl_seconds=7200,
+        run_id="run-123",
+    )
+
+    # Access the environment URL
+    print(env.urls.app[0])
+
+    # ... interact with the environment ...
+
+    # Verify task completion
+    result = await task.verify_detailed_async(env.instance_id)
+    print(result)
+
+    # Clean up
+    await env.close()
 ```
 
-## API Key Setup
+## Loading Tasks
 
-Fleet requires an API key for authentication. You can obtain one from the [Fleet Platform](https://fleetai.com/dashboard/api-keys).
+### By Task Keys
 
-Set your API key as an environment variable:
+```python
+tasks = await fleet.load_tasks_async(
+    keys=["task_abcdef"]
+)
+```
 
-```bash
-export FLEET_API_KEY="sk_your_key_here"
+### By Project Key
+
+```python
+tasks = await fleet.load_tasks_async(project_key="my-project")
 ```
 
-## Basic Usage
+## Creating Environments
 
 ```python
-import fleet
-import datetime
+env = await fleet.env.make_async(
+    env_key=task.env_key,
+    data_key=task.data_key,
+    env_variables=task.env_variables,
+    ttl_seconds=7200,
+    run_id="run-123",
+)
+```
 
-# Create environment by key
-env = fleet.env.make("fira")
+### With Heartbeats (Optional)
 
-# Reset environment with seed and options
-env.reset(
-    seed=42,
-    timestamp=int(datetime.datetime.now().timestamp())
+Optionally enable heartbeats to keep environments alive during long-running operations:
+
+```python
+env = await fleet.env.make_async(
+    env_key=task.env_key,
+    data_key=task.data_key,
+    env_variables=task.env_variables,
+    ttl_seconds=10800,
+    heartbeat_interval=30,  # seconds
 )
+```
+
+Send heartbeats to keep the environment alive:
+
+```python
+# Via the environment object
+await env.heartbeat()
 
-# Access environment state ('current' is the resource id for a sqlite database)
-sql = env.state("sqlite://current")
-sql.exec("UPDATE customers SET status = 'active' WHERE id = 123")
+# Or via instance ID
+await fleet.env.heartbeat_async(instance_id)
+```
+
+Heartbeats are optional. If `heartbeat_interval` is not set, the instance lifetime is controlled solely by `ttl_seconds`. If heartbeats are enabled and missed 3 consecutive times, the instance will be terminated. Heartbeats take precedence over the TTL.
+
+## Instance Management
+
+### List Instances
+
+```python
+# List all instances for a run
+instances = await fleet.env.list_instances_async(run_id="run-123")
+
+# List all instances for your profile
+instances = await fleet.env.list_instances_async(profile_id="self")
+```
+
+### Close Instances
 
-# Clean up
-env.close()
+```python
+# Close all instances for a run
+await fleet.env.close_all_async(run_id="run-123")
+
+# Close all instances for your profile
+await fleet.env.close_all_async(profile_id="self")
+
+# Close a specific instance by ID
+await fleet.env.close_async("bc8954c2")
 ```
 
-## Environment Management
+`"self"` is an alias for the profile associated with your `FLEET_API_KEY`.
 
-### Creating Instances
+## Account Information
+
+View your current account details including team info, instance limits, and profile ID:
 
 ```python
-# Create environment instance with explicit version
-env = fleet.env.make("fira:v1.2.5")
+account = await fleet.env.account_async()
+```
 
-# Create environment instance with default (latest) version
-env = fleet.env.make("fira")
+Returns:
 
+```json
+{
+  "team_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
+  "team_name": "My Team",
+  "instance_limit": 32000,
+  "instance_count": 924,
+  "profile_id": "11111111-2222-3333-4444-555555555555",
+  "profile_name": "Jane Doe"
+}
 ```
 
-### Connecting to Existing Instances
+## Run Tracking
+
+Track active and past runs:
 
 ```python
-# Connect to a running instance
-env = fleet.env.get("env_instance_id")
+# List active runs
+runs = await fleet.env.list_runs_async()
+
+# List all runs (active and inactive)
+runs = await fleet.env.list_runs_async(status="all")
+
+# Filter by profile
+runs = await fleet.env.list_runs_async(profile_id="self")
+```
 
-# List all running instances
-instances = fleet.env.list_instances()
-for instance in instances:
-    print(f"Instance: {instance.instance_id}")
-    print(f"Type: {instance.environment_type}")
-    print(f"Status: {instance.status}")
+Returns:
+
+```json
+[
+  {
+    "run_id": "run-123",
+    "running_count": 0,
+    "total_count": 4,
+    "first_created_at": "2025-10-24T09:48:47.152387",
+    "last_created_at": "2025-10-24T09:55:19.284294",
+    "profile_id": "11111111-2222-3333-4444-555555555555"
+  }
+]
+```
+
+## Task Verification
+
+Verify task completion and get detailed results:
+
+```python
+result = await task.verify_detailed_async(env.instance_id)
+print(result)
+```
+
+Returns:
+
+```json
+{
+  "key": "task_abcdef",
+  "version": 4,
+  "success": true,
+  "result": 1.0,
+  "error": null,
+  "execution_time_ms": 2291,
+  "stdout": ""
+}
+```
+
+On failure, `stdout` contains detailed verification errors:
+
+```json
+{
+  "key": "task_abcdef",
+  "version": 4,
+  "success": true,
+  "result": 0,
+  "error": null,
+  "execution_time_ms": 2291,
+  "stdout": "Verification errors: [\"Expected field to be 'value', got None\", \"Form not marked as complete\"]"
+}
+```
+
+## Complete Example
+
+```python
+import fleet
+import asyncio
 
-# Filter instances by status (running, pending, stopped, error)
-running_instances = fleet.env.list_instances(status_filter="running")
+async def main():
+    # Load tasks from a project
+    tasks = await fleet.load_tasks_async(project_key="my-project")
+
+    for task in tasks:
+        # Create environment
+        env = await fleet.env.make_async(
+            env_key=task.env_key,
+            data_key=task.data_key,
+            env_variables=task.env_variables,
+            ttl_seconds=7200,
+            run_id="my-evaluation-run",
+        )
+
+        try:
+            # Access the environment URL
+            print(env.urls.app[0])
+
+            # ... run your agent ...
+
+            # Verify task completion
+            result = await task.verify_detailed_async(env.instance_id)
+            print(f"Task {task.key}: score={result['result']}")
+
+        finally:
+            await env.close()
+
+    # Clean up all instances from this run
+    await fleet.env.close_all_async(run_id="my-evaluation-run")
 
-# List available environment types
-available_envs = fleet.env.list_envs()
+if __name__ == "__main__":
+    asyncio.run(main())
 ```

examples/example_task.py

@@ -112,7 +112,7 @@ async def main():
     print(f"Task definition:")
     print(f"  Key: {task.key}")
     print(f"  Prompt: {task.prompt}")
-    print(f"  Environment: {task.env_id}")
+    print(f"  Environment: {task.env_key}")
     print(
         f"  Verifier: {task.verifier.key if hasattr(task.verifier, 'key') else 'create_bug_issue'}"
     )

examples/export_tasks.py

@@ -38,6 +38,12 @@ def main():
         help="Output JSON filename (defaults to {team_id}.json)",
         default=None,
     )
+    parser.add_argument(
+        "--skip-missing-verifiers",
+        "-s",
+        action="store_true",
+        help="Skip tasks missing verifier_func instead of erroring",
+    )
 
     args = parser.parse_args()
 
@@ -91,16 +97,32 @@ def main():
             missing_verifier.append(task.key)
 
     if missing_verifier:
-        print(f"\n✗ Error: {len(missing_verifier)} task(s) missing verifier_func:")
-        for key in missing_verifier[:10]:  # Show first 10
-            print(f"  - {key}")
-        if len(missing_verifier) > 10:
-            print(f"  ... and {len(missing_verifier) - 10} more")
-        raise ValueError(
-            "All tasks must have a verifier_func. Cannot export tasks without verifiers."
-        )
-
-    print("✓ All tasks have verifier_func")
+        if args.skip_missing_verifiers:
+            print(
+                f"\n⚠ Skipping {len(missing_verifier)} task(s) missing verifier_func:"
+            )
+            for key in missing_verifier[:10]:  # Show first 10
+                print(f"  - {key}")
+            if len(missing_verifier) > 10:
+                print(f"  ... and {len(missing_verifier) - 10} more")
+            # Filter out tasks without verifiers
+            tasks = [task for task in tasks if task.verifier_func]
+            print(f"\n{len(tasks)} task(s) remaining after filtering")
+            if not tasks:
+                raise ValueError(
+                    "No tasks remaining after filtering. Nothing to export."
+                )
+        else:
+            print(f"\n✗ Error: {len(missing_verifier)} task(s) missing verifier_func:")
+            for key in missing_verifier[:10]:  # Show first 10
+                print(f"  - {key}")
+            if len(missing_verifier) > 10:
+                print(f"  ... and {len(missing_verifier) - 10} more")
+            raise ValueError(
+                "All tasks must have a verifier_func. Cannot export tasks without verifiers."
+            )
+    else:
+        print("✓ All tasks have verifier_func")
 
     # Determine output filename
     output_file = args.output or f"{account.team_id}.json"