diff --git a/submissions/26512076/week-02/REPORT.md b/submissions/26512076/week-02/REPORT.md new file mode 100644 index 000000000..2ae832927 --- /dev/null +++ b/submissions/26512076/week-02/REPORT.md @@ -0,0 +1,45 @@ +# Week 02 — Harness A/B Report + +## 1. Variant definition + +이번 실험에서는 동일한 태스크와 도구를 사용하여 ReAct와 Plan-then-Execute 하네스를 비교했습니다. 최종 비교 실행(13–18번)의 Provider는 Groq이며, 두 하네스 모두 명시적인 단일 모델 ID인 `openai/gpt-oss-20b`를 사용했습니다. `tools_shared.py`에서 모델을 고정하고 Groq API를 OpenAI 호환 SDK로 호출했습니다. API 키는 파일에 저장하지 않고 `GROQ_API_KEY` 환경변수로 전달했습니다. + +공통 도구는 `read_file(path)`와 `count_pattern(path, pattern)`입니다. 태스크는 변경하지 않은 `app.log`에서 ERROR가 가장 많이 발생한 시간대(HH:00)를 찾는 것입니다. `TASK.md`의 기대 문자열 `14:00`이 최종 답변에 포함되면 성공으로 판정했습니다. 실행 명령은 `python run_ab.py --runs 3`입니다. 이 명령은 각 하네스를 3회 실행하고 결과와 실행별 로그를 기존 파일에 추가합니다. + +ReAct는 모델의 판단, 도구 호출, 관찰 결과를 단계별로 반복하며 전체 대화 기록을 다음 호출에 전달합니다. 최종 답변을 내거나 최대 단계 수에 도달하면 종료합니다. 도구 실행 중 발생한 오류는 관찰 결과로 전달하여 다음 단계에서 대응할 수 있습니다. + +Plan-then-Execute는 먼저 도구 없이 JSON 리스트 형식의 계획을 생성하고, 실행기가 각 계획 단계를 순서대로 처리합니다. 한 단계의 도구 호출은 최대 3라운드이며, `OFF_PLAN`이 발생하면 최대 1회 재계획할 수 있습니다. 두 하네스는 컨텍스트를 구성하는 방식, 종료 조건, 오류 복구 방식이 다릅니다. 도구 종류와 도구의 작업 단위는 같으며, 사람의 승인 단계는 두 하네스 모두 없습니다. + +## 2. Measurements + +`results.csv`의 모든 실행을 아래에 보존했습니다. 1–6번은 기존 `openrouter/free` 실행입니다. 이 ID는 호출마다 실제 무료 모델을 선택할 수 있어 고정 모델 A/B 비교에는 사용하지 않았습니다. 7–12번은 Groq 모델을 고정한 뒤의 첫 실행이며, 10–12번에서 계획 생성 중 API 오류가 발생했습니다. 계획 단계 지시를 수정한 다음 실행한 **13–18번이 최종 A/B 비교 대상**입니다. + +| run | harness | success | tokens | iters | interventions | note | +|---:|---|:---:|---:|---:|---:|---| +| 1 | react | O | 15395 | 5 | 0 | | +| 2 | react | O | 11782 | 4 | 0 | | +| 3 | react | O | 8838 | 4 | 0 | | +| 4 | plan_exec | O | 15382 | 7 | 0 | replans=1 | +| 5 | plan_exec | X | 225 | 1 | 0 | replans=0 | +| 6 | plan_exec | X | 738 | 1 | 0 | replans=0 | +| 7 | react | O | 2581 | 2 | 0 | | +| 8 | react | O | 2355 | 2 | 0 | | +| 9 | react | O | 2096 | 2 | 0 | | +| 10 | plan_exec | X | | | | API 400: tool_use_failed | +| 11 | plan_exec | X | | | | API 400: tool_use_failed | +| 12 | plan_exec | X | | | | API 400: tool_use_failed | +| 13 | react | O | 2183 | 2 | 0 | | +| 14 | react | O | 2056 | 2 | 0 | | +| 15 | react | O | 3495 | 2 | 0 | | +| 16 | plan_exec | O | 18143 | 8 | 0 | replans=0 | +| 17 | plan_exec | O | 33126 | 13 | 0 | replans=0 | +| 18 | plan_exec | O | 33465 | 10 | 0 | replans=0 | + +최종 비교(13–18번)에서 두 하네스 모두 3/3회 성공했고 사람의 개입은 0회였습니다. ReAct는 평균 2,578토큰과 2회 호출을 사용했습니다. Plan-then-Execute는 평균 약 28,245토큰과 10.33회 호출을 사용했습니다. `iters`는 도구 호출 횟수가 아니라 모델 호출 횟수입니다. + +## 3. Interpretation + +최종 실행에서는 성공률이 같았지만 ReAct가 더 적은 토큰과 모델 호출로 답을 구했습니다. ReAct 로그(13–15번)에서는 `app.log`를 읽은 뒤 두 번째 모델 호출에서 모두 `14:00`을 답했습니다. Plan-then-Execute 로그(16–18번)에서는 먼저 계획을 만들고 각 단계를 별도 호출로 실행했습니다. 특히 17번은 파일 읽기, 전체 줄 수 확인, ERROR 줄 수 확인, 시간 추출, 시간대별 집계 등을 여러 단계로 나누어 13회 호출했습니다. 이 순차 실행 방식이 이번 짧은 태스크에서 토큰과 반복 횟수를 늘린 것으로 해석됩니다. 세 실행 모두 재계획은 사용하지 않았습니다. + +이전 실패도 하네스의 동작을 보여줍니다. 5·6번은 계획을 유효한 JSON 리스트로 파싱하지 못해 실행 단계에 들어가지 못했습니다. 10–12번은 고정 모델의 첫 설정에서 계획 생성 중 도구 호출을 시도하여 API가 `tool_use_failed`를 반환한 사례입니다. 이 실행들의 빈 토큰·반복 횟수 칸은 `run_ab.py`가 예외 발생 시 계측값을 기록하지 못한 결과이며, 0을 뜻하지 않습니다. 이후 계획 단계에서는 도구를 사용할 수 없다는 지시를 명확히 하여 13–18번을 다시 실행했습니다. 따라서 서로 다른 모델이 선택될 수 있었던 1–6번이나 계획 지시가 달랐던 7–12번을 최종 성능 평균에 섞지 않았습니다. 최종 결과는 각 하네스 3회에 대한 관찰이므로, 다른 태스크에서도 같은 효율 차이가 난다고 일반화할 수는 없습니다. + diff --git a/submissions/26512076/week-02/TASK.md b/submissions/26512076/week-02/TASK.md new file mode 100644 index 000000000..135b3bca9 --- /dev/null +++ b/submissions/26512076/week-02/TASK.md @@ -0,0 +1,15 @@ +# Task + +The same task for both harnesses. `run_ab.py` reads the `task:` and +`expected:` lines below. Write the success criterion before you run +anything, and do not change it after you see the results. + +task: In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form. + +## Success criterion + +A run succeeds when the final answer contains the hour with the most ERROR +lines in `app.log`, written as HH:00. `app.log` is the reference input; the +graded runs use it unchanged. + +expected: 14:00 diff --git a/submissions/26512076/week-02/app.log b/submissions/26512076/week-02/app.log new file mode 100644 index 000000000..0ab861777 --- /dev/null +++ b/submissions/26512076/week-02/app.log @@ -0,0 +1,60 @@ +2026-09-01 09:07:27 INFO job finished in 320ms +2026-09-01 09:11:56 ERROR NullReference in QuizService.score +2026-09-01 09:13:14 INFO health check ok +2026-09-01 09:26:45 INFO health check ok +2026-09-01 09:30:17 INFO health check ok +2026-09-01 09:32:05 INFO request served /api/quiz +2026-09-01 10:07:50 INFO session started +2026-09-01 10:12:15 INFO request served /api/quiz +2026-09-01 10:18:22 ERROR db connection refused +2026-09-01 10:23:48 WARN retrying upstream call +2026-09-01 10:53:22 WARN slow query 1.8s on rooms +2026-09-01 10:54:07 ERROR NullReference in QuizService.score +2026-09-01 10:57:10 INFO cache hit for room 4412 +2026-09-01 11:23:35 ERROR payment webhook signature mismatch +2026-09-01 11:36:12 INFO cache hit for room 4412 +2026-09-01 11:39:20 INFO request served /api/quiz +2026-09-01 11:40:27 WARN slow query 1.8s on rooms +2026-09-01 11:44:02 INFO health check ok +2026-09-01 11:56:13 INFO request served /api/quiz +2026-09-01 12:08:13 INFO request served /api/quiz +2026-09-01 12:28:08 ERROR upstream timeout after 5000ms +2026-09-01 12:32:19 INFO health check ok +2026-09-01 12:49:54 ERROR db connection refused +2026-09-01 12:50:06 WARN token near expiry +2026-09-01 12:53:48 ERROR upstream timeout after 5000ms +2026-09-01 12:59:50 WARN token near expiry +2026-09-01 13:24:27 INFO cache hit for room 4412 +2026-09-01 13:26:02 INFO session started +2026-09-01 13:28:15 INFO request served /api/quiz +2026-09-01 13:42:13 ERROR db connection refused +2026-09-01 13:43:37 WARN slow query 1.8s on rooms +2026-09-01 13:59:54 ERROR upstream timeout after 5000ms +2026-09-01 14:04:06 ERROR NullReference in QuizService.score +2026-09-01 14:07:11 ERROR NullReference in QuizService.score +2026-09-01 14:15:56 ERROR upstream timeout after 5000ms +2026-09-01 14:19:13 INFO health check ok +2026-09-01 14:26:19 INFO request served /api/quiz +2026-09-01 14:30:20 ERROR db connection refused +2026-09-01 14:43:37 INFO job finished in 320ms +2026-09-01 14:47:03 ERROR upstream timeout after 5000ms +2026-09-01 14:54:53 ERROR upstream timeout after 5000ms +2026-09-01 15:04:23 INFO cache hit for room 4412 +2026-09-01 15:33:16 INFO request served /api/quiz +2026-09-01 15:34:38 ERROR payment webhook signature mismatch +2026-09-01 15:39:25 INFO cache hit for room 4412 +2026-09-01 15:41:14 ERROR NullReference in QuizService.score +2026-09-01 15:45:36 INFO cache hit for room 4412 +2026-09-01 15:48:41 WARN token near expiry +2026-09-01 16:11:35 INFO request served /api/quiz +2026-09-01 16:16:04 INFO cache hit for room 4412 +2026-09-01 16:24:31 WARN slow query 1.8s on rooms +2026-09-01 16:44:46 INFO session started +2026-09-01 16:51:30 WARN retrying upstream call +2026-09-01 16:53:34 ERROR NullReference in QuizService.score +2026-09-01 17:17:14 ERROR NullReference in QuizService.score +2026-09-01 17:27:48 WARN token near expiry +2026-09-01 17:37:46 INFO health check ok +2026-09-01 17:48:38 WARN token near expiry +2026-09-01 17:55:29 INFO job finished in 320ms +2026-09-01 17:56:05 INFO cache hit for room 4412 diff --git a/submissions/26512076/week-02/harness_plan_execute.py b/submissions/26512076/week-02/harness_plan_execute.py new file mode 100644 index 000000000..35e1963fa --- /dev/null +++ b/submissions/26512076/week-02/harness_plan_execute.py @@ -0,0 +1,107 @@ +"""Week 02 starter — the Plan-then-Execute harness. + +One call produces the whole plan as a JSON list. Then each step is executed +in order with tools. If a step reports OFF_PLAN, the plan is rebuilt once +(max_replan=1): that number is the flexibility cap, and it is explicit. +""" +import json +import re +import sys + +from tools_shared import Chat, Meter, Reply + +SYSTEM_PLAN = ( + "You are a planner. In this planning phase you cannot call tools. " + "Describe actions for a later executor. Reply with a JSON list of " + "short step strings and nothing else. No prose, no code fences." +) + +SYSTEM_EXEC = ( + "You execute exactly the current step of a plan using the provided tools. " + "Do not perform later steps or give the final answer during a step. " + "After completing the current step, report its result briefly. " + "If the current step cannot be done, start your reply with 'OFF_PLAN:' " + "and explain why. Only when explicitly asked for the final answer, " + "start your reply with 'Answer:'." +) + +def parse_plan(text: str): + """Return a list of step strings, or None if the model did not give JSON.""" + text = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.M).strip() + try: + plan = json.loads(text) + except json.JSONDecodeError: + return None + if isinstance(plan, list) and all(isinstance(s, str) for s in plan): + return plan + return None + + +def run_plan_execute(task: str, max_replan: int = 1, + max_tool_rounds: int = 3, log=print): + meter = Meter() + + # 1) PLAN: the whole plan in one call, no tools + planner = Chat(SYSTEM_PLAN, meter, tools=False) + planner.add_user( + f"Task: {task}\n" + "The later executor can read text files and count lines matching " + "a regular expression. Describe the steps; do not perform them." + ) + raw = planner.send().text + plan = parse_plan(raw) + if plan is None: # a parse failure is one failure mode + log(f"[plan] not valid JSON: {raw.strip()[:300]!r}") + return "plan parse failed", meter, 0 + log(f"[plan] {plan}") + + # 2) EXECUTE: each step in order + executor = Chat(SYSTEM_EXEC, meter) + executor.add_user(f"Task: {task}\nPlan: {json.dumps(plan)}") + replans = 0 + i = 0 + while i < len(plan): + executor.add_user( + f"Execute only step {i + 1}: {plan[i]}. " + "Use no more tools than needed, then briefly report this step's result." + ) + reply = executor.send() + rounds = 0 + while reply.tool_calls: # tool calls inside one step + executor.run_tools(reply, log) + reply = executor.send() + rounds += 1 + if rounds >= max_tool_rounds and reply.tool_calls: + executor.run_tools(reply, log) # answer every call so the transcript stays valid + reply = Reply("OFF_PLAN: step exceeded the tool-call budget", []) + break + log(f"[step {i + 1}] {reply.text.strip()[:300]}") + + if reply.text.strip().startswith("OFF_PLAN") and replans < max_replan: + replans += 1 # flexibility cap + planner.add_user(f"Step {i + 1} ({plan[i]}) failed: {reply.text.strip()[:300]}\n" + f"Reply with a JSON list of the remaining steps.") + raw = planner.send().text + new_steps = parse_plan(raw) + if new_steps is None: + log(f"[replan] not valid JSON: {raw.strip()[:300]!r}") + break + plan = plan[:i] + new_steps + log(f"[replan] {plan}") + continue + i += 1 + + executor.add_user("Give the final answer now, starting with 'Answer:'.") + final = executor.send() + if final.tool_calls: # the model tried to keep going + executor.run_tools(final, log) + final = executor.send() + return final.text, meter, replans + + +if __name__ == "__main__": + task = sys.argv[1] if len(sys.argv) > 1 else \ + "In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form." + answer, m, replans = run_plan_execute(task) + print(answer) + print(f"tokens={m.tokens} iters={m.iters} interventions={m.interventions} replans={replans}") diff --git a/submissions/26512076/week-02/harness_react.py b/submissions/26512076/week-02/harness_react.py new file mode 100644 index 000000000..e7a17b4ab --- /dev/null +++ b/submissions/26512076/week-02/harness_react.py @@ -0,0 +1,60 @@ +"""Week 02 starter — the ReAct-style harness. + +Every step: the model writes a Thought, calls a tool (Action), sees the result +(Observation), and decides again. The five axes from the lecture are marked. +""" +import sys + +from tools_shared import Chat, Meter + +SYSTEM = ( + "You solve tasks with the tools you are given. Before every tool call, " + "write one line that starts with 'Thought:' saying what you know and what " + "you will do next. When the task is complete, reply with a line that starts " + "with 'Answer:' and make no tool call." +) + +# [axis 5] calls that must be approved by a human before they run. +# The starter tools are read-only, so this set is empty and interventions +# stay at 0. Add a tool that writes or deletes, and put its name here. +IRREVERSIBLE = set() + + +def ask_human(call) -> bool: + answer = input(f"approve {call.name}({call.args})? [y/N] ").strip().lower() + return answer == "y" + + +def run_react(task: str, max_steps: int = 8, log=print): + meter = Meter() + chat = Chat(SYSTEM, meter) # [axis 1] context: full history, every call + chat.add_user(task) + + for step in range(max_steps): # [axis 3] termination: iteration cap + reply = chat.send() + if reply.text.strip(): + log(f"[step {step + 1}] {reply.text.strip()}") + + if not reply.tool_calls: # [axis 3] the model chose to finish + return reply.text, meter + + approved = [] + for call in reply.tool_calls: + if call.name in IRREVERSIBLE and not ask_human(call): + meter.interventions += 1 # [axis 5] intervention point + chat.add_tool_result(call, "denied: human did not approve") + continue + approved.append(call) + reply.tool_calls = approved + chat.run_tools(reply, log) # [axis 2] granularity lives in tools_shared + # [axis 4] errors come back as Observations + + return "MAX_STEPS reached: incomplete", meter + + +if __name__ == "__main__": + task = sys.argv[1] if len(sys.argv) > 1 else \ + "In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form." + answer, m = run_react(task) + print(answer) + print(f"tokens={m.tokens} iters={m.iters} interventions={m.interventions}") diff --git a/submissions/26512076/week-02/logs/plan_exec-04.txt b/submissions/26512076/week-02/logs/plan_exec-04.txt new file mode 100644 index 000000000..00e51beaa --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-04.txt @@ -0,0 +1,16 @@ +[plan] ['Read app.log', 'For each hour 00-23, count lines matching "^HH:00.*ERROR"', 'Select hour with highest count', 'Return that hour in HH:00 format'] + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 00:00.*ERROR'}) -> 0 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:.*ERROR'}) -> 3 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:.*ERROR'}) -> 6 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:.*ERROR'}) -> 1 +[step 1] OFF_PLAN: step exceeded the tool-call budget +[replan] not valid JSON: 'User Safety: safe\nResponse Safety: safe' +[final] Answer: 14:00 +[judge] expected='14:00' -> O (95.2s) diff --git a/submissions/26512076/week-02/logs/plan_exec-05.txt b/submissions/26512076/week-02/logs/plan_exec-05.txt new file mode 100644 index 000000000..a1774eae3 --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-05.txt @@ -0,0 +1,3 @@ +[plan] not valid JSON: '{"analysis": "I need to find which hour has the most ERROR lines in app.log. I\'ll first read the file to understand its format, then count ERROR lines per hour.", "plan": "1. Read app.log to see the format and identify ERROR lines.\\n2. Use count_pattern to count ERROR lines for each hour.\\n3. Determ' +[final] plan parse failed +[judge] expected='14:00' -> X (1.1s) diff --git a/submissions/26512076/week-02/logs/plan_exec-06.txt b/submissions/26512076/week-02/logs/plan_exec-06.txt new file mode 100644 index 000000000..683035635 --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-06.txt @@ -0,0 +1,3 @@ +[plan] not valid JSON: 'User Safety: safe' +[final] plan parse failed +[judge] expected='14:00' -> X (2.9s) diff --git a/submissions/26512076/week-02/logs/plan_exec-10.txt b/submissions/26512076/week-02/logs/plan_exec-10.txt new file mode 100644 index 000000000..fd8754ef8 --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-10.txt @@ -0,0 +1,4 @@ +[model] openai/gpt-oss-20b +crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "python", "arguments": import re, textwrap, json, sys, math, os, typing, collections, pprint, itertools, statistics}'}} +[final] +[judge] expected='14:00' -> X (0.7s) diff --git a/submissions/26512076/week-02/logs/plan_exec-11.txt b/submissions/26512076/week-02/logs/plan_exec-11.txt new file mode 100644 index 000000000..4fe7f184e --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-11.txt @@ -0,0 +1,4 @@ +[model] openai/gpt-oss-20b +crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "read_file", "arguments": {"path":"app.log"}}'}} +[final] +[judge] expected='14:00' -> X (0.4s) diff --git a/submissions/26512076/week-02/logs/plan_exec-12.txt b/submissions/26512076/week-02/logs/plan_exec-12.txt new file mode 100644 index 000000000..eff10341f --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-12.txt @@ -0,0 +1,4 @@ +[model] openai/gpt-oss-20b +crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "read_file", "arguments": {"path":"app.log"}}'}} +[final] +[judge] expected='14:00' -> X (1.2s) diff --git a/submissions/26512076/week-02/logs/plan_exec-16.txt b/submissions/26512076/week-02/logs/plan_exec-16.txt new file mode 100644 index 000000000..180c7266b --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-16.txt @@ -0,0 +1,19 @@ +[model] openai/gpt-oss-20b +[plan] ['Read app.log line by line.', 'For each line containing "ERROR", extract the hour (HH) from the timestamp.', 'Increment a counter for that hour.', 'After all lines are processed, find the hour with the maximum count.', 'Print the hour in "HH:00" format.'] + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 1] Step 1 completed: Read app.log line by line. (First 4000 characters of the file have been retrieved.) +[step 2] Step 2 completed: Extracted the hour (HH) for each ERROR line. The extracted hours are: 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17. +[step 3] Step 3 completed: Counter for each hour: +- 09: 1 +- 10: 2 +- 11: 1 +- 12: 3 +- 13: 2 +- 14: 6 +- 15: 2 +- 16: 1 +- 17: 1 +[step 4] Step 4 completed: The hour with the maximum ERROR count is **14** with 6 errors. +[step 5] Step 5 completed: The hour with the most ERROR lines is **14:00**. +[final] Answer: 14:00 +[judge] expected='14:00' -> O (136.3s) diff --git a/submissions/26512076/week-02/logs/plan_exec-17.txt b/submissions/26512076/week-02/logs/plan_exec-17.txt new file mode 100644 index 000000000..42a1431f8 --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-17.txt @@ -0,0 +1,15 @@ +[model] openai/gpt-oss-20b +[plan] ['Open the app.log file for reading.', 'Iterate through each line in the file.', "Identify lines that contain the string 'ERROR'.", "From each matching line, extract the timestamp part that represents the hour (e.g., the HH value from 'HH:MM:SS').", 'Count how many times each hour appears among the ERROR lines.', 'Determine which hour has the highest count.', "Return that hour formatted as 'HH:00'."] + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 1] The app.log file has been read. + [tool] count_pattern({'path': 'app.log', 'pattern': '^'}) -> 60 +[step 2] Step 2 complete: The file contains 60 lines. + [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19 +[step 3] Step 3 complete: 19 lines contain the string 'ERROR'. + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 4] Step 4 complete: Extracted hours from ERROR lines – 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17.Step 4 complete: Hours extracted from ERROR lines are 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17. +[step 5] Step 5 complete: Hour counts for ERROR lines – 09:1, 10:2, 11:1, 12:3, 13:2, 14:6, 15:2, 16:1, 17:1. +[step 6] Step 6 complete: The hour with the most ERROR lines is **14**. +[step 7] Step 7 complete: The hour with the most ERROR lines is **14:00**. +[final] Answer: 14:00 +[judge] expected='14:00' -> O (224.1s) diff --git a/submissions/26512076/week-02/logs/plan_exec-18.txt b/submissions/26512076/week-02/logs/plan_exec-18.txt new file mode 100644 index 000000000..675453828 --- /dev/null +++ b/submissions/26512076/week-02/logs/plan_exec-18.txt @@ -0,0 +1,21 @@ +[model] openai/gpt-oss-20b +[plan] ['Open the file app.log for reading.', 'Read each line of the file sequentially.', 'For each line, check if it contains the word "ERROR" (case-sensitive).', 'If it does, extract the hour portion (HH) from the timestamp at the start of the line, assuming format like "YYYY-MM-DD HH:MM:SS".', 'Increment a counter for that hour.', 'After processing all lines, identify the hour with the highest count of ERROR lines.', 'Format that hour as "HH:00" and output it.'] + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 1] File app.log opened successfully. 4000 characters read. +[step 2] Read each line sequentially: 60 lines processed. +[step 3] Checked each line: 19 lines contain the word "ERROR". +[step 4] Extracted hour (HH) from each ERROR line: 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17. +[step 5] Counter per hour after processing ERROR lines: +- 09: 1 +- 10: 2 +- 11: 1 +- 12: 3 +- 13: 2 +- 14: 6 +- 15: 2 +- 16: 1 +- 17: 1 +[step 6] Identified hour with most ERROR lines: **14**. +[step 7] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (199.2s) diff --git a/submissions/26512076/week-02/logs/react-01.txt b/submissions/26512076/week-02/logs/react-01.txt new file mode 100644 index 000000000..a33d981cc --- /dev/null +++ b/submissions/26512076/week-02/logs/react-01.txt @@ -0,0 +1,16 @@ + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 + [tool] count_pattern({'pattern': '^', 'path': 'app.log'}) -> 60 +[step 3] Let me count ERROR lines by hour: + [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:\\d{2}:\\d{2}.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:\\d{2}:\\d{2}.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:\\d{2}:\\d{2}.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:\\d{2}:\\d{2}.*ERROR'}) -> 3 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:\\d{2}:\\d{2}.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:\\d{2}:\\d{2}.*ERROR'}) -> 6 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:\\d{2}:\\d{2}.*ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:\\d{2}:\\d{2}.*ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:\\d{2}:\\d{2}.*ERROR'}) -> 1 +[step 5] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (94.3s) diff --git a/submissions/26512076/week-02/logs/react-02.txt b/submissions/26512076/week-02/logs/react-02.txt new file mode 100644 index 000000000..6c9f94aaa --- /dev/null +++ b/submissions/26512076/week-02/logs/react-02.txt @@ -0,0 +1,13 @@ + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 + [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19 + [tool] count_pattern({'path': 'app.log', 'pattern': '09:[0-9]{2}:[0-9]{2} ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '10:[0-9]{2}:[0-9]{2} ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '11:[0-9]{2}:[0-9]{2} ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '12:[0-9]{2}:[0-9]{2} ERROR'}) -> 3 + [tool] count_pattern({'path': 'app.log', 'pattern': '13:[0-9]{2}:[0-9]{2} ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '14:[0-9]{2}:[0-9]{2} ERROR'}) -> 6 + [tool] count_pattern({'path': 'app.log', 'pattern': '15:[0-9]{2}:[0-9]{2} ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '17:[0-9]{2}:[0-9]{2} ERROR'}) -> 1 +[step 4] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (23.1s) diff --git a/submissions/26512076/week-02/logs/react-03.txt b/submissions/26512076/week-02/logs/react-03.txt new file mode 100644 index 000000000..af6e70a8d --- /dev/null +++ b/submissions/26512076/week-02/logs/react-03.txt @@ -0,0 +1,17 @@ +[step 1] Thought: I need to read the app.log file to find ERROR lines and count them by hour. + +Let me first read the file to understand its structure. + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Thought: I can see the log entries. Now I need to count ERROR lines by hour. Let me use a pattern to count ERROR lines for each hour to be precise. + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:.* ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:.* ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:.* ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:.* ERROR'}) -> 3 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:.* ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:.* ERROR'}) -> 6 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:.* ERROR'}) -> 2 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:.* ERROR'}) -> 1 + [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:.* ERROR'}) -> 1 +[step 4] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (72.0s) diff --git a/submissions/26512076/week-02/logs/react-07.txt b/submissions/26512076/week-02/logs/react-07.txt new file mode 100644 index 000000000..10bf86fdf --- /dev/null +++ b/submissions/26512076/week-02/logs/react-07.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (6.4s) diff --git a/submissions/26512076/week-02/logs/react-08.txt b/submissions/26512076/week-02/logs/react-08.txt new file mode 100644 index 000000000..2c45f45d4 --- /dev/null +++ b/submissions/26512076/week-02/logs/react-08.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (1.6s) diff --git a/submissions/26512076/week-02/logs/react-09.txt b/submissions/26512076/week-02/logs/react-09.txt new file mode 100644 index 000000000..29c16d92e --- /dev/null +++ b/submissions/26512076/week-02/logs/react-09.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (1.1s) diff --git a/submissions/26512076/week-02/logs/react-13.txt b/submissions/26512076/week-02/logs/react-13.txt new file mode 100644 index 000000000..ffc9e06bb --- /dev/null +++ b/submissions/26512076/week-02/logs/react-13.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (4.5s) diff --git a/submissions/26512076/week-02/logs/react-14.txt b/submissions/26512076/week-02/logs/react-14.txt new file mode 100644 index 000000000..29e4fd48e --- /dev/null +++ b/submissions/26512076/week-02/logs/react-14.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (1.2s) diff --git a/submissions/26512076/week-02/logs/react-15.txt b/submissions/26512076/week-02/logs/react-15.txt new file mode 100644 index 000000000..45db0461d --- /dev/null +++ b/submissions/26512076/week-02/logs/react-15.txt @@ -0,0 +1,5 @@ +[model] openai/gpt-oss-20b + [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO health check ok | 2026-09-01 09:26:45 INFO health check ok | 2026-09 +[step 2] Answer: 14:00 +[final] Answer: 14:00 +[judge] expected='14:00' -> O (2.1s) diff --git a/submissions/26512076/week-02/results.csv b/submissions/26512076/week-02/results.csv new file mode 100644 index 000000000..96584f6b5 --- /dev/null +++ b/submissions/26512076/week-02/results.csv @@ -0,0 +1,19 @@ +run,harness,success,tokens,iters,interventions,note +1,react,O,15395,5,0, +2,react,O,11782,4,0, +3,react,O,8838,4,0, +4,plan_exec,O,15382,7,0,replans=1 +5,plan_exec,X,225,1,0,replans=0 +6,plan_exec,X,738,1,0,replans=0 +7,react,O,2581,2,0, +8,react,O,2355,2,0, +9,react,O,2096,2,0, +10,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""python"", ""arguments"": import re, textwrap, json, sys, math, os, typing, collections, pprint, itertools, statistics}'}}" +11,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""read_file"", ""arguments"": {""path"":""app.log""}}'}}" +12,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""read_file"", ""arguments"": {""path"":""app.log""}}'}}" +13,react,O,2183,2,0, +14,react,O,2056,2,0, +15,react,O,3495,2,0, +16,plan_exec,O,18143,8,0,replans=0 +17,plan_exec,O,33126,13,0,replans=0 +18,plan_exec,O,33465,10,0,replans=0 diff --git a/submissions/26512076/week-02/run_ab.py b/submissions/26512076/week-02/run_ab.py new file mode 100644 index 000000000..5068c9495 --- /dev/null +++ b/submissions/26512076/week-02/run_ab.py @@ -0,0 +1,94 @@ +"""Week 02 starter — run the A/B experiment and record results.csv. + +Usage: python run_ab.py [--runs 3] + +Reads the task and the success criterion from TASK.md, runs each harness +--runs times, judges every run, appends one line per run to results.csv, +and saves each run's console output under logs/. Failed runs are kept: +they are data. +""" + +from tools_shared import MODEL +import argparse +import csv +import os +import re +import time +from pathlib import Path + +from harness_plan_execute import run_plan_execute +from harness_react import run_react + +HEADER = ["run", "harness", "success", "tokens", "iters", "interventions", "note"] + + +def read_task(path="TASK.md"): + text = Path(path).read_text(encoding="utf-8") + task = re.search(r"^task:\s*(.+)$", text, flags=re.M) + expected = re.search(r"^expected:\s*(.+)$", text, flags=re.M) + if not task or not expected: + raise SystemExit("TASK.md needs a 'task:' line and an 'expected:' line") + return task.group(1).strip(), expected.group(1).strip() + + +def judge(answer: str, expected: str) -> bool: + """Success = the expected string appears in the final answer. Fix the + criterion in TASK.md before running; do not loosen it afterwards.""" + return expected.lower() in (answer or "").lower() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--runs", type=int, default=3) + args = ap.parse_args() + + task, expected = read_task() + Path("logs").mkdir(exist_ok=True) + new_file = not Path("results.csv").exists() + run_no = 0 + if not new_file: + with open("results.csv", encoding="utf-8") as f: + run_no = sum(1 for _ in f) - 1 + + with open("results.csv", "a", newline="", encoding="utf-8") as f: + w = csv.writer(f) + if new_file: + w.writerow(HEADER) + for name, fn in (("react", run_react), ("plan_exec", run_plan_execute)): + for _ in range(args.runs): + run_no += 1 + lines = [] + + def log(msg, _lines=lines): + print(msg) + _lines.append(str(msg)) + + + log(f"[model] {MODEL}") + t0 = time.time() + note = "" + try: + out = fn(task, log=log) + answer, meter = out[0], out[1] + if name == "plan_exec": + note = f"replans={out[2]}" + except Exception as e: # a crash is a failed run, not a lost run + answer, meter, note = "", None, f"crash: {type(e).__name__}: {e}" + log(note) + success = judge(answer, expected) + log(f"[final] {answer.strip()[:300]}") + log(f"[judge] expected={expected!r} -> {'O' if success else 'X'} " + f"({time.time() - t0:.1f}s)") + + Path("logs", f"{name}-{run_no:02d}.txt").write_text( + "\n".join(lines) + "\n", encoding="utf-8") + w.writerow([run_no, name, "O" if success else "X", + meter.tokens if meter else "", + meter.iters if meter else "", + meter.interventions if meter else "", note]) + f.flush() + print("\nresults.csv updated;", os.path.abspath("results.csv")) + + +if __name__ == "__main__": + main() diff --git a/submissions/26512076/week-02/tools_shared.py b/submissions/26512076/week-02/tools_shared.py new file mode 100644 index 000000000..3248b0c6b --- /dev/null +++ b/submissions/26512076/week-02/tools_shared.py @@ -0,0 +1,187 @@ +"""Week 02 starter — tools, model call, and meter shared by both harnesses. + +Both harnesses import from here. Same tools and same model for both is what +makes the A/B a harness comparison and not a tool comparison. + +Provider: Groq via the OpenAI-compatible SDK (pip install openai) +API key: GROQ_API_KEY environment variable +Base URL: https://api.groq.com/openai/v1 +Model: openai/gpt-oss-20b (fixed for both harnesses) +""" +import json +import os +import re +from dataclasses import dataclass, field + +# ---------------------------------------------------------------- tools + + +def read_file(path: str) -> str: + """Return the contents of a text file in the working directory.""" + full = os.path.abspath(path) + if not full.startswith(os.getcwd()): + return "denied: path outside the working directory" + with open(full, encoding="utf-8") as f: + return f.read()[:4000] # context guard, same as week 01 + + +def count_pattern(path: str, pattern: str) -> str: + """Count lines in a text file that match a regular expression.""" + full = os.path.abspath(path) + if not full.startswith(os.getcwd()): + return "denied: path outside the working directory" + rx = re.compile(pattern) + with open(full, encoding="utf-8") as f: + return str(sum(1 for line in f if rx.search(line))) + + +TOOLS_IMPL = {"read_file": read_file, "count_pattern": count_pattern} + +# provider-neutral schemas; Chat converts them per provider +TOOL_SPECS = [ + {"name": "read_file", + "description": "Read a text file in the working directory (first 4000 characters).", + "parameters": {"type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"]}}, + {"name": "count_pattern", + "description": "Count the lines of a text file that match a regular expression.", + "parameters": {"type": "object", + "properties": {"path": {"type": "string"}, + "pattern": {"type": "string"}}, + "required": ["path", "pattern"]}}, +] + +# ---------------------------------------------------------------- meter + + +class Meter: + """The four metrics of the lab, counted in one place.""" + + def __init__(self): + self.tokens = 0 + self.iters = 0 # one iteration = one model call + self.interventions = 0 # times a human approved or denied a call + + def add(self, input_tokens: int, output_tokens: int): + self.tokens += int(input_tokens or 0) + int(output_tokens or 0) + self.iters += 1 + + +# ---------------------------------------------------------------- model + + +@dataclass +class ToolCall: + id: str + name: str + args: dict + + +@dataclass +class Reply: + text: str + tool_calls: list = field(default_factory=list) + + +PROVIDER = "openai" +MODEL = "openai/gpt-oss-20b" + +_client = None + + +def _get_client(): + global _client + if _client is None: + from openai import OpenAI + _client = OpenAI( + api_key=os.environ["GROQ_API_KEY"], + base_url="https://api.groq.com/openai/v1", + ) + return _client + + +class Chat: + """One conversation with the model. Owns the provider-specific message + format so the harnesses only see Reply and ToolCall.""" + + def __init__(self, system: str, meter: Meter, tools: bool = True): + self.system = system + self.meter = meter + self.tools = tools + self.messages = [] + if PROVIDER == "openai": + self.messages.append({"role": "system", "content": system}) + + # ---- building the next turn + def add_user(self, text: str): + self.messages.append({"role": "user", "content": text}) + + def add_tool_result(self, call: ToolCall, output: str): + if PROVIDER == "anthropic": + block = {"type": "tool_result", "tool_use_id": call.id, "content": output} + last = self.messages[-1] + if last["role"] == "user" and isinstance(last["content"], list): + last["content"].append(block) + else: + self.messages.append({"role": "user", "content": [block]}) + else: + self.messages.append({"role": "tool", "tool_call_id": call.id, + "content": output}) + + # ---- one model call + def send(self) -> Reply: + if PROVIDER == "anthropic": + return self._send_anthropic() + return self._send_openai() + + def _send_anthropic(self) -> Reply: + kwargs = dict(model=MODEL, max_tokens=1024, system=self.system, + messages=self.messages) + if self.tools: + kwargs["tools"] = [{"name": t["name"], "description": t["description"], + "input_schema": t["parameters"]} for t in TOOL_SPECS] + resp = _get_client().messages.create(**kwargs) + self.meter.add(resp.usage.input_tokens, resp.usage.output_tokens) + self.messages.append({"role": "assistant", "content": resp.content}) + text = "".join(b.text for b in resp.content if b.type == "text") + calls = [ToolCall(b.id, b.name, dict(b.input)) + for b in resp.content if b.type == "tool_use"] + return Reply(text, calls) + + def _send_openai(self) -> Reply: + kwargs = dict(model=MODEL, messages=self.messages) + if self.tools: + kwargs["tools"] = [{"type": "function", + "function": {"name": t["name"], + "description": t["description"], + "parameters": t["parameters"]}} + for t in TOOL_SPECS] + resp = _get_client().chat.completions.create(**kwargs) + usage = resp.usage + self.meter.add(getattr(usage, "prompt_tokens", 0), + getattr(usage, "completion_tokens", 0)) + msg = resp.choices[0].message + self.messages.append(msg) + calls = [] + for c in msg.tool_calls or []: + try: + args = json.loads(c.function.arguments or "{}") + except json.JSONDecodeError: + args = {"_raw": c.function.arguments} + calls.append(ToolCall(c.id, c.function.name, args)) + return Reply(msg.content or "", calls) + + # ---- run the tools a reply asked for, feed results back + def run_tools(self, reply: Reply, log=print) -> None: + for call in reply.tool_calls: + fn = TOOLS_IMPL.get(call.name) + if fn is None: + out = f"error: unknown tool {call.name}" + else: + try: + out = str(fn(**call.args)) + except Exception as e: # error recovery: the error is an Observation + out = f"error: {e}" + log(f" [tool] {call.name}({call.args}) -> {out[:200].replace(chr(10), ' | ')}") + self.add_tool_result(call, out)