From 8de17c1a0c7c29e63633e4a34efbac76ecaa26f2 Mon Sep 17 00:00:00 2001
From: jaeook26 <jgc9268@gmail.com>
Date: Tue, 8 Sep 2026 21:23:29 +0900
Subject: [PATCH 1/3] Add week 02 task definition

---
 submissions/26512076/week-02/TASK.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 submissions/26512076/week-02/TASK.md

diff --git a/submissions/26512076/week-02/TASK.md b/submissions/26512076/week-02/TASK.md
new file mode 100644
index 000000000..135b3bca9
--- /dev/null
+++ b/submissions/26512076/week-02/TASK.md
@@ -0,0 +1,15 @@
+# Task
+
+The same task for both harnesses. `run_ab.py` reads the `task:` and
+`expected:` lines below. Write the success criterion before you run
+anything, and do not change it after you see the results.
+
+task: In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form.
+
+## Success criterion
+
+A run succeeds when the final answer contains the hour with the most ERROR
+lines in `app.log`, written as HH:00. `app.log` is the reference input; the
+graded runs use it unchanged.
+
+expected: 14:00

From db06d9e92378f4b8d04cc7ded8420f9f1d93d70d Mon Sep 17 00:00:00 2001
From: jaeook26 <jgc9268@gmail.com>
Date: Tue, 8 Sep 2026 21:38:31 +0900
Subject: [PATCH 2/3] Complete week 02 harness experiment

---
 submissions/26512076/week-02/REPORT.md        |  60 ++++++
 submissions/26512076/week-02/app.log          |  60 ++++++
 .../26512076/week-02/harness_plan_execute.py  |  98 +++++++++
 submissions/26512076/week-02/harness_react.py |  60 ++++++
 .../26512076/week-02/logs/plan_exec-04.txt    |  16 ++
 .../26512076/week-02/logs/plan_exec-05.txt    |   3 +
 .../26512076/week-02/logs/plan_exec-06.txt    |   3 +
 .../26512076/week-02/logs/react-01.txt        |  16 ++
 .../26512076/week-02/logs/react-02.txt        |  13 ++
 .../26512076/week-02/logs/react-03.txt        |  17 ++
 submissions/26512076/week-02/results.csv      |   7 +
 submissions/26512076/week-02/run_ab.py        |  90 ++++++++
 submissions/26512076/week-02/tools_shared.py  | 192 ++++++++++++++++++
 13 files changed, 635 insertions(+)
 create mode 100644 submissions/26512076/week-02/REPORT.md
 create mode 100644 submissions/26512076/week-02/app.log
 create mode 100644 submissions/26512076/week-02/harness_plan_execute.py
 create mode 100644 submissions/26512076/week-02/harness_react.py
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-04.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-05.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-06.txt
 create mode 100644 submissions/26512076/week-02/logs/react-01.txt
 create mode 100644 submissions/26512076/week-02/logs/react-02.txt
 create mode 100644 submissions/26512076/week-02/logs/react-03.txt
 create mode 100644 submissions/26512076/week-02/results.csv
 create mode 100644 submissions/26512076/week-02/run_ab.py
 create mode 100644 submissions/26512076/week-02/tools_shared.py

diff --git a/submissions/26512076/week-02/REPORT.md b/submissions/26512076/week-02/REPORT.md
new file mode 100644
index 000000000..ff4e23c66
--- /dev/null
+++ b/submissions/26512076/week-02/REPORT.md
@@ -0,0 +1,60 @@
+\# Week 02 — Harness A/B Report
+
+
+
+\## 1. Variant definition
+
+
+
+이번 실험에서는 모델, 태스크, 도구를 동일하게 유지하고 하네스만 변경했다. Provider는 OpenRouter, 모델은 `openrouter/free`를 사용했다. 공통 도구는 `read\_file(path)`와 `count\_pattern(path, pattern)`이다. 태스크는 `app.log`에서 ERROR가 가장 많이 발생한 시간대를 찾는 것이며, 정답 문자열이 최종 답변에 포함되면 성공으로 판정했다.
+
+
+
+ReAct 하네스는 매 단계에서 Thought → Action → Observation을 반복하며 전체 대화 기록을 다음 호출에 전달한다. 모델이 답변을 완료하거나 최대 8단계에 도달하면 종료한다. 도구 오류도 Observation으로 돌려주므로 다음 단계에서 행동을 수정할 수 있다.
+
+
+
+Plan-then-Execute 하네스는 먼저 전체 계획을 JSON 리스트로 생성한 다음 각 단계를 순서대로 실행한다. 한 단계당 도구 호출은 최대 3라운드이며, 계획에서 벗어났을 때 재계획은 최대 1회 허용된다. 따라서 두 하네스는 주로 컨텍스트 관리, 종료 조건, 오류 복구 축에서 다르다. 도구 granularity는 동일하며, 읽기 전용 도구만 사용했으므로 사람의 개입 지점도 동일하게 0회였다.
+
+
+
+실행 명령은 다음과 같다.
+
+
+
+`python run\_ab.py --runs 3`
+
+
+
+\## 2. Measurements
+
+
+
+| run | harness | success | tokens | iters | interventions | note |
+
+|---:|---|:---:|---:|---:|---:|---|
+
+| 1 | react | O | 15395 | 5 | 0 | |
+
+| 2 | react | O | 11782 | 4 | 0 | |
+
+| 3 | react | O | 8838 | 4 | 0 | |
+
+| 4 | plan\_exec | O | 15382 | 7 | 0 | replans=1 |
+
+| 5 | plan\_exec | X | 225 | 1 | 0 | replans=0 |
+
+| 6 | plan\_exec | X | 738 | 1 | 0 | replans=0 |
+
+
+
+ReAct의 성공률은 3/3이며 평균 토큰은 12,005개, 평균 반복 횟수는 4.33회였다. Plan-then-Execute의 성공률은 1/3이며 평균 토큰은 약 5,448개, 평균 반복 횟수는 3회였다.
+
+
+
+\## 3. Interpretation
+
+
+
+이 태스크에서는 ReAct가 성공률에서 우세했다. ReAct는 세 번 모두 성공했으며, 매 단계의 Observation을 다음 판단에 반영하는 오류 복구 방식이 안정성에 기여했다. 반면 Plan-then-Execute는 한 번만 성공했고 두 번은 계획 생성 호출 1회 후 종료되었다. 5번과 6번 실행은 반복 횟수가 1이고 재계획도 0회이므로, 실행 단계에 들어가기 전 계획을 유효한 JSON으로 파싱하지 못한 실패임을 로그에서 확인할 수 있다. Plan 방식의 평균 토큰이 더 적은 것은 단순히 효율성이 높아서가 아니라 두 실행이 초기에 실패했기 때문이다. 유일하게 성공한 Plan 실행은 재계획 1회를 사용했고, ReAct의 각 성공 실행보다 많은 7회 호출과 15,382토큰을 소비했다. 따라서 이번 실험에서는 ReAct의 단계별 컨텍스트 갱신과 유연한 오류 복구가 성공률을 높였고, Plan-then-Execute의 엄격한 계획 형식과 제한된 재계획 조건은 초기 계획 오류에 취약하게 작용했다.
+
diff --git a/submissions/26512076/week-02/app.log b/submissions/26512076/week-02/app.log
new file mode 100644
index 000000000..0ab861777
--- /dev/null
+++ b/submissions/26512076/week-02/app.log
@@ -0,0 +1,60 @@
+2026-09-01 09:07:27 INFO  job finished in 320ms
+2026-09-01 09:11:56 ERROR NullReference in QuizService.score
+2026-09-01 09:13:14 INFO  health check ok
+2026-09-01 09:26:45 INFO  health check ok
+2026-09-01 09:30:17 INFO  health check ok
+2026-09-01 09:32:05 INFO  request served /api/quiz
+2026-09-01 10:07:50 INFO  session started
+2026-09-01 10:12:15 INFO  request served /api/quiz
+2026-09-01 10:18:22 ERROR db connection refused
+2026-09-01 10:23:48 WARN  retrying upstream call
+2026-09-01 10:53:22 WARN  slow query 1.8s on rooms
+2026-09-01 10:54:07 ERROR NullReference in QuizService.score
+2026-09-01 10:57:10 INFO  cache hit for room 4412
+2026-09-01 11:23:35 ERROR payment webhook signature mismatch
+2026-09-01 11:36:12 INFO  cache hit for room 4412
+2026-09-01 11:39:20 INFO  request served /api/quiz
+2026-09-01 11:40:27 WARN  slow query 1.8s on rooms
+2026-09-01 11:44:02 INFO  health check ok
+2026-09-01 11:56:13 INFO  request served /api/quiz
+2026-09-01 12:08:13 INFO  request served /api/quiz
+2026-09-01 12:28:08 ERROR upstream timeout after 5000ms
+2026-09-01 12:32:19 INFO  health check ok
+2026-09-01 12:49:54 ERROR db connection refused
+2026-09-01 12:50:06 WARN  token near expiry
+2026-09-01 12:53:48 ERROR upstream timeout after 5000ms
+2026-09-01 12:59:50 WARN  token near expiry
+2026-09-01 13:24:27 INFO  cache hit for room 4412
+2026-09-01 13:26:02 INFO  session started
+2026-09-01 13:28:15 INFO  request served /api/quiz
+2026-09-01 13:42:13 ERROR db connection refused
+2026-09-01 13:43:37 WARN  slow query 1.8s on rooms
+2026-09-01 13:59:54 ERROR upstream timeout after 5000ms
+2026-09-01 14:04:06 ERROR NullReference in QuizService.score
+2026-09-01 14:07:11 ERROR NullReference in QuizService.score
+2026-09-01 14:15:56 ERROR upstream timeout after 5000ms
+2026-09-01 14:19:13 INFO  health check ok
+2026-09-01 14:26:19 INFO  request served /api/quiz
+2026-09-01 14:30:20 ERROR db connection refused
+2026-09-01 14:43:37 INFO  job finished in 320ms
+2026-09-01 14:47:03 ERROR upstream timeout after 5000ms
+2026-09-01 14:54:53 ERROR upstream timeout after 5000ms
+2026-09-01 15:04:23 INFO  cache hit for room 4412
+2026-09-01 15:33:16 INFO  request served /api/quiz
+2026-09-01 15:34:38 ERROR payment webhook signature mismatch
+2026-09-01 15:39:25 INFO  cache hit for room 4412
+2026-09-01 15:41:14 ERROR NullReference in QuizService.score
+2026-09-01 15:45:36 INFO  cache hit for room 4412
+2026-09-01 15:48:41 WARN  token near expiry
+2026-09-01 16:11:35 INFO  request served /api/quiz
+2026-09-01 16:16:04 INFO  cache hit for room 4412
+2026-09-01 16:24:31 WARN  slow query 1.8s on rooms
+2026-09-01 16:44:46 INFO  session started
+2026-09-01 16:51:30 WARN  retrying upstream call
+2026-09-01 16:53:34 ERROR NullReference in QuizService.score
+2026-09-01 17:17:14 ERROR NullReference in QuizService.score
+2026-09-01 17:27:48 WARN  token near expiry
+2026-09-01 17:37:46 INFO  health check ok
+2026-09-01 17:48:38 WARN  token near expiry
+2026-09-01 17:55:29 INFO  job finished in 320ms
+2026-09-01 17:56:05 INFO  cache hit for room 4412
diff --git a/submissions/26512076/week-02/harness_plan_execute.py b/submissions/26512076/week-02/harness_plan_execute.py
new file mode 100644
index 000000000..07705a55d
--- /dev/null
+++ b/submissions/26512076/week-02/harness_plan_execute.py
@@ -0,0 +1,98 @@
+"""Week 02 starter — the Plan-then-Execute harness.
+
+One call produces the whole plan as a JSON list. Then each step is executed
+in order with tools. If a step reports OFF_PLAN, the plan is rebuilt once
+(max_replan=1): that number is the flexibility cap, and it is explicit.
+"""
+import json
+import re
+import sys
+
+from tools_shared import Chat, Meter, Reply
+
+SYSTEM_PLAN = (
+    "You are a planner. Reply with a JSON list of short strings, one per step, "
+    "and nothing else. No prose, no code fences."
+)
+SYSTEM_EXEC = (
+    "You execute one step of a plan at a time with the tools you are given. "
+    "If the step cannot be done as planned, reply with a line that starts with "
+    "'OFF_PLAN:' and explain why. When asked for the final answer, reply with a "
+    "line that starts with 'Answer:'."
+)
+
+
+def parse_plan(text: str):
+    """Return a list of step strings, or None if the model did not give JSON."""
+    text = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.M).strip()
+    try:
+        plan = json.loads(text)
+    except json.JSONDecodeError:
+        return None
+    if isinstance(plan, list) and all(isinstance(s, str) for s in plan):
+        return plan
+    return None
+
+
+def run_plan_execute(task: str, max_replan: int = 1,
+                     max_tool_rounds: int = 3, log=print):
+    meter = Meter()
+
+    # 1) PLAN: the whole plan in one call, no tools
+    planner = Chat(SYSTEM_PLAN, meter, tools=False)
+    planner.add_user(f"Task: {task}\nAvailable tools: read_file(path), "
+                     f"count_pattern(path, pattern).")
+    raw = planner.send().text
+    plan = parse_plan(raw)
+    if plan is None:                              # a parse failure is one failure mode
+        log(f"[plan] not valid JSON: {raw.strip()[:300]!r}")
+        return "plan parse failed", meter, 0
+    log(f"[plan] {plan}")
+
+    # 2) EXECUTE: each step in order
+    executor = Chat(SYSTEM_EXEC, meter)
+    executor.add_user(f"Task: {task}\nPlan: {json.dumps(plan)}")
+    replans = 0
+    i = 0
+    while i < len(plan):
+        executor.add_user(f"Execute step {i + 1}: {plan[i]}")
+        reply = executor.send()
+        rounds = 0
+        while reply.tool_calls:                   # tool calls inside one step
+            executor.run_tools(reply, log)
+            reply = executor.send()
+            rounds += 1
+            if rounds >= max_tool_rounds and reply.tool_calls:
+                executor.run_tools(reply, log)    # answer every call so the transcript stays valid
+                reply = Reply("OFF_PLAN: step exceeded the tool-call budget", [])
+                break
+        log(f"[step {i + 1}] {reply.text.strip()[:300]}")
+
+        if reply.text.strip().startswith("OFF_PLAN") and replans < max_replan:
+            replans += 1                          # flexibility cap
+            planner.add_user(f"Step {i + 1} ({plan[i]}) failed: {reply.text.strip()[:300]}\n"
+                             f"Reply with a JSON list of the remaining steps.")
+            raw = planner.send().text
+            new_steps = parse_plan(raw)
+            if new_steps is None:
+                log(f"[replan] not valid JSON: {raw.strip()[:300]!r}")
+                break
+            plan = plan[:i] + new_steps
+            log(f"[replan] {plan}")
+            continue
+        i += 1
+
+    executor.add_user("Give the final answer now, starting with 'Answer:'.")
+    final = executor.send()
+    if final.tool_calls:                          # the model tried to keep going
+        executor.run_tools(final, log)
+        final = executor.send()
+    return final.text, meter, replans
+
+
+if __name__ == "__main__":
+    task = sys.argv[1] if len(sys.argv) > 1 else \
+        "In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form."
+    answer, m, replans = run_plan_execute(task)
+    print(answer)
+    print(f"tokens={m.tokens} iters={m.iters} interventions={m.interventions} replans={replans}")
diff --git a/submissions/26512076/week-02/harness_react.py b/submissions/26512076/week-02/harness_react.py
new file mode 100644
index 000000000..e7a17b4ab
--- /dev/null
+++ b/submissions/26512076/week-02/harness_react.py
@@ -0,0 +1,60 @@
+"""Week 02 starter — the ReAct-style harness.
+
+Every step: the model writes a Thought, calls a tool (Action), sees the result
+(Observation), and decides again. The five axes from the lecture are marked.
+"""
+import sys
+
+from tools_shared import Chat, Meter
+
+SYSTEM = (
+    "You solve tasks with the tools you are given. Before every tool call, "
+    "write one line that starts with 'Thought:' saying what you know and what "
+    "you will do next. When the task is complete, reply with a line that starts "
+    "with 'Answer:' and make no tool call."
+)
+
+# [axis 5] calls that must be approved by a human before they run.
+# The starter tools are read-only, so this set is empty and interventions
+# stay at 0. Add a tool that writes or deletes, and put its name here.
+IRREVERSIBLE = set()
+
+
+def ask_human(call) -> bool:
+    answer = input(f"approve {call.name}({call.args})? [y/N] ").strip().lower()
+    return answer == "y"
+
+
+def run_react(task: str, max_steps: int = 8, log=print):
+    meter = Meter()
+    chat = Chat(SYSTEM, meter)                    # [axis 1] context: full history, every call
+    chat.add_user(task)
+
+    for step in range(max_steps):                 # [axis 3] termination: iteration cap
+        reply = chat.send()
+        if reply.text.strip():
+            log(f"[step {step + 1}] {reply.text.strip()}")
+
+        if not reply.tool_calls:                  # [axis 3] the model chose to finish
+            return reply.text, meter
+
+        approved = []
+        for call in reply.tool_calls:
+            if call.name in IRREVERSIBLE and not ask_human(call):
+                meter.interventions += 1          # [axis 5] intervention point
+                chat.add_tool_result(call, "denied: human did not approve")
+                continue
+            approved.append(call)
+        reply.tool_calls = approved
+        chat.run_tools(reply, log)                # [axis 2] granularity lives in tools_shared
+                                                  # [axis 4] errors come back as Observations
+
+    return "MAX_STEPS reached: incomplete", meter
+
+
+if __name__ == "__main__":
+    task = sys.argv[1] if len(sys.argv) > 1 else \
+        "In app.log, which hour (HH:00) has the most ERROR lines? Answer with the hour in HH:00 form."
+    answer, m = run_react(task)
+    print(answer)
+    print(f"tokens={m.tokens} iters={m.iters} interventions={m.interventions}")
diff --git a/submissions/26512076/week-02/logs/plan_exec-04.txt b/submissions/26512076/week-02/logs/plan_exec-04.txt
new file mode 100644
index 000000000..00e51beaa
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-04.txt
@@ -0,0 +1,16 @@
+[plan] ['Read app.log', 'For each hour 00-23, count lines matching "^HH:00.*ERROR"', 'Select hour with highest count', 'Return that hour in HH:00 format']
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 00:00.*ERROR'}) -> 0
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:.*ERROR'}) -> 3
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:.*ERROR'}) -> 6
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:.*ERROR'}) -> 1
+[step 1] OFF_PLAN: step exceeded the tool-call budget
+[replan] not valid JSON: 'User Safety: safe\nResponse Safety: safe'
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (95.2s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-05.txt b/submissions/26512076/week-02/logs/plan_exec-05.txt
new file mode 100644
index 000000000..a1774eae3
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-05.txt
@@ -0,0 +1,3 @@
+[plan] not valid JSON: '{"analysis": "I need to find which hour has the most ERROR lines in app.log. I\'ll first read the file to understand its format, then count ERROR lines per hour.", "plan": "1. Read app.log to see the format and identify ERROR lines.\\n2. Use count_pattern to count ERROR lines for each hour.\\n3. Determ'
+[final] plan parse failed
+[judge] expected='14:00' -> X (1.1s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-06.txt b/submissions/26512076/week-02/logs/plan_exec-06.txt
new file mode 100644
index 000000000..683035635
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-06.txt
@@ -0,0 +1,3 @@
+[plan] not valid JSON: 'User Safety: safe'
+[final] plan parse failed
+[judge] expected='14:00' -> X (2.9s)
diff --git a/submissions/26512076/week-02/logs/react-01.txt b/submissions/26512076/week-02/logs/react-01.txt
new file mode 100644
index 000000000..a33d981cc
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-01.txt
@@ -0,0 +1,16 @@
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+  [tool] count_pattern({'pattern': '^', 'path': 'app.log'}) -> 60
+[step 3] Let me count ERROR lines by hour:
+  [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:\\d{2}:\\d{2}.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:\\d{2}:\\d{2}.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:\\d{2}:\\d{2}.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:\\d{2}:\\d{2}.*ERROR'}) -> 3
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:\\d{2}:\\d{2}.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:\\d{2}:\\d{2}.*ERROR'}) -> 6
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:\\d{2}:\\d{2}.*ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:\\d{2}:\\d{2}.*ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:\\d{2}:\\d{2}.*ERROR'}) -> 1
+[step 5] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (94.3s)
diff --git a/submissions/26512076/week-02/logs/react-02.txt b/submissions/26512076/week-02/logs/react-02.txt
new file mode 100644
index 000000000..6c9f94aaa
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-02.txt
@@ -0,0 +1,13 @@
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+  [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19
+  [tool] count_pattern({'path': 'app.log', 'pattern': '09:[0-9]{2}:[0-9]{2} ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '10:[0-9]{2}:[0-9]{2} ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '11:[0-9]{2}:[0-9]{2} ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '12:[0-9]{2}:[0-9]{2} ERROR'}) -> 3
+  [tool] count_pattern({'path': 'app.log', 'pattern': '13:[0-9]{2}:[0-9]{2} ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '14:[0-9]{2}:[0-9]{2} ERROR'}) -> 6
+  [tool] count_pattern({'path': 'app.log', 'pattern': '15:[0-9]{2}:[0-9]{2} ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '17:[0-9]{2}:[0-9]{2} ERROR'}) -> 1
+[step 4] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (23.1s)
diff --git a/submissions/26512076/week-02/logs/react-03.txt b/submissions/26512076/week-02/logs/react-03.txt
new file mode 100644
index 000000000..af6e70a8d
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-03.txt
@@ -0,0 +1,17 @@
+[step 1] Thought: I need to read the app.log file to find ERROR lines and count them by hour.
+
+Let me first read the file to understand its structure.
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Thought: I can see the log entries. Now I need to count ERROR lines by hour. Let me use a pattern to count ERROR lines for each hour to be precise.
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 09:.* ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 10:.* ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 11:.* ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 12:.* ERROR'}) -> 3
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 13:.* ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 14:.* ERROR'}) -> 6
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 15:.* ERROR'}) -> 2
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 16:.* ERROR'}) -> 1
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^2026-09-01 17:.* ERROR'}) -> 1
+[step 4] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (72.0s)
diff --git a/submissions/26512076/week-02/results.csv b/submissions/26512076/week-02/results.csv
new file mode 100644
index 000000000..182a075f7
--- /dev/null
+++ b/submissions/26512076/week-02/results.csv
@@ -0,0 +1,7 @@
+run,harness,success,tokens,iters,interventions,note
+1,react,O,15395,5,0,
+2,react,O,11782,4,0,
+3,react,O,8838,4,0,
+4,plan_exec,O,15382,7,0,replans=1
+5,plan_exec,X,225,1,0,replans=0
+6,plan_exec,X,738,1,0,replans=0
diff --git a/submissions/26512076/week-02/run_ab.py b/submissions/26512076/week-02/run_ab.py
new file mode 100644
index 000000000..bb43c0efa
--- /dev/null
+++ b/submissions/26512076/week-02/run_ab.py
@@ -0,0 +1,90 @@
+"""Week 02 starter — run the A/B experiment and record results.csv.
+
+Usage: python run_ab.py [--runs 3]
+
+Reads the task and the success criterion from TASK.md, runs each harness
+--runs times, judges every run, appends one line per run to results.csv,
+and saves each run's console output under logs/. Failed runs are kept:
+they are data.
+"""
+import argparse
+import csv
+import os
+import re
+import time
+from pathlib import Path
+
+from harness_plan_execute import run_plan_execute
+from harness_react import run_react
+
+HEADER = ["run", "harness", "success", "tokens", "iters", "interventions", "note"]
+
+
+def read_task(path="TASK.md"):
+    text = Path(path).read_text(encoding="utf-8")
+    task = re.search(r"^task:\s*(.+)$", text, flags=re.M)
+    expected = re.search(r"^expected:\s*(.+)$", text, flags=re.M)
+    if not task or not expected:
+        raise SystemExit("TASK.md needs a 'task:' line and an 'expected:' line")
+    return task.group(1).strip(), expected.group(1).strip()
+
+
+def judge(answer: str, expected: str) -> bool:
+    """Success = the expected string appears in the final answer. Fix the
+    criterion in TASK.md before running; do not loosen it afterwards."""
+    return expected.lower() in (answer or "").lower()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--runs", type=int, default=3)
+    args = ap.parse_args()
+
+    task, expected = read_task()
+    Path("logs").mkdir(exist_ok=True)
+    new_file = not Path("results.csv").exists()
+    run_no = 0
+    if not new_file:
+        with open("results.csv", encoding="utf-8") as f:
+            run_no = sum(1 for _ in f) - 1
+
+    with open("results.csv", "a", newline="", encoding="utf-8") as f:
+        w = csv.writer(f)
+        if new_file:
+            w.writerow(HEADER)
+        for name, fn in (("react", run_react), ("plan_exec", run_plan_execute)):
+            for _ in range(args.runs):
+                run_no += 1
+                lines = []
+
+                def log(msg, _lines=lines):
+                    print(msg)
+                    _lines.append(str(msg))
+
+                t0 = time.time()
+                note = ""
+                try:
+                    out = fn(task, log=log)
+                    answer, meter = out[0], out[1]
+                    if name == "plan_exec":
+                        note = f"replans={out[2]}"
+                except Exception as e:            # a crash is a failed run, not a lost run
+                    answer, meter, note = "", None, f"crash: {type(e).__name__}: {e}"
+                    log(note)
+                success = judge(answer, expected)
+                log(f"[final] {answer.strip()[:300]}")
+                log(f"[judge] expected={expected!r} -> {'O' if success else 'X'} "
+                    f"({time.time() - t0:.1f}s)")
+
+                Path("logs", f"{name}-{run_no:02d}.txt").write_text(
+                    "\n".join(lines) + "\n", encoding="utf-8")
+                w.writerow([run_no, name, "O" if success else "X",
+                            meter.tokens if meter else "",
+                            meter.iters if meter else "",
+                            meter.interventions if meter else "", note])
+                f.flush()
+    print("\nresults.csv updated;", os.path.abspath("results.csv"))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/submissions/26512076/week-02/tools_shared.py b/submissions/26512076/week-02/tools_shared.py
new file mode 100644
index 000000000..a3fccd624
--- /dev/null
+++ b/submissions/26512076/week-02/tools_shared.py
@@ -0,0 +1,192 @@
+"""Week 02 starter — tools, model call, and meter shared by both harnesses.
+
+Both harnesses import from here. Same tools and same model for both is what
+makes the A/B a harness comparison and not a tool comparison.
+
+Provider is picked from the environment:
+  ANTHROPIC_API_KEY set          -> Anthropic SDK (pip install anthropic)
+  otherwise                      -> OpenAI-compatible (pip install openai)
+                                    OPENAI_API_KEY, optional OPENAI_BASE_URL
+                                    (https://openrouter.ai/api/v1 for OpenRouter)
+  AGENT_MODEL                    optional model override for either provider
+"""
+import json
+import os
+import re
+from dataclasses import dataclass, field
+
+# ---------------------------------------------------------------- tools
+
+
+def read_file(path: str) -> str:
+    """Return the contents of a text file in the working directory."""
+    full = os.path.abspath(path)
+    if not full.startswith(os.getcwd()):
+        return "denied: path outside the working directory"
+    with open(full, encoding="utf-8") as f:
+        return f.read()[:4000]          # context guard, same as week 01
+
+
+def count_pattern(path: str, pattern: str) -> str:
+    """Count lines in a text file that match a regular expression."""
+    full = os.path.abspath(path)
+    if not full.startswith(os.getcwd()):
+        return "denied: path outside the working directory"
+    rx = re.compile(pattern)
+    with open(full, encoding="utf-8") as f:
+        return str(sum(1 for line in f if rx.search(line)))
+
+
+TOOLS_IMPL = {"read_file": read_file, "count_pattern": count_pattern}
+
+# provider-neutral schemas; Chat converts them per provider
+TOOL_SPECS = [
+    {"name": "read_file",
+     "description": "Read a text file in the working directory (first 4000 characters).",
+     "parameters": {"type": "object",
+                    "properties": {"path": {"type": "string"}},
+                    "required": ["path"]}},
+    {"name": "count_pattern",
+     "description": "Count the lines of a text file that match a regular expression.",
+     "parameters": {"type": "object",
+                    "properties": {"path": {"type": "string"},
+                                   "pattern": {"type": "string"}},
+                    "required": ["path", "pattern"]}},
+]
+
+# ---------------------------------------------------------------- meter
+
+
+class Meter:
+    """The four metrics of the lab, counted in one place."""
+
+    def __init__(self):
+        self.tokens = 0
+        self.iters = 0            # one iteration = one model call
+        self.interventions = 0    # times a human approved or denied a call
+
+    def add(self, input_tokens: int, output_tokens: int):
+        self.tokens += int(input_tokens or 0) + int(output_tokens or 0)
+        self.iters += 1
+
+
+# ---------------------------------------------------------------- model
+
+
+@dataclass
+class ToolCall:
+    id: str
+    name: str
+    args: dict
+
+
+@dataclass
+class Reply:
+    text: str
+    tool_calls: list = field(default_factory=list)
+
+
+PROVIDER = "anthropic" if os.environ.get("ANTHROPIC_API_KEY") else "openai"
+MODEL = os.environ.get(
+    "AGENT_MODEL",
+    "claude-sonnet-4-5" if PROVIDER == "anthropic" else "gpt-4o-mini")
+
+_client = None
+
+
+def _get_client():
+    global _client
+    if _client is None:
+        if PROVIDER == "anthropic":
+            import anthropic
+            _client = anthropic.Anthropic()
+        else:
+            from openai import OpenAI
+            _client = OpenAI()
+    return _client
+
+
+class Chat:
+    """One conversation with the model. Owns the provider-specific message
+    format so the harnesses only see Reply and ToolCall."""
+
+    def __init__(self, system: str, meter: Meter, tools: bool = True):
+        self.system = system
+        self.meter = meter
+        self.tools = tools
+        self.messages = []
+        if PROVIDER == "openai":
+            self.messages.append({"role": "system", "content": system})
+
+    # ---- building the next turn
+    def add_user(self, text: str):
+        self.messages.append({"role": "user", "content": text})
+
+    def add_tool_result(self, call: ToolCall, output: str):
+        if PROVIDER == "anthropic":
+            block = {"type": "tool_result", "tool_use_id": call.id, "content": output}
+            last = self.messages[-1]
+            if last["role"] == "user" and isinstance(last["content"], list):
+                last["content"].append(block)
+            else:
+                self.messages.append({"role": "user", "content": [block]})
+        else:
+            self.messages.append({"role": "tool", "tool_call_id": call.id,
+                                  "content": output})
+
+    # ---- one model call
+    def send(self) -> Reply:
+        if PROVIDER == "anthropic":
+            return self._send_anthropic()
+        return self._send_openai()
+
+    def _send_anthropic(self) -> Reply:
+        kwargs = dict(model=MODEL, max_tokens=1024, system=self.system,
+                      messages=self.messages)
+        if self.tools:
+            kwargs["tools"] = [{"name": t["name"], "description": t["description"],
+                                "input_schema": t["parameters"]} for t in TOOL_SPECS]
+        resp = _get_client().messages.create(**kwargs)
+        self.meter.add(resp.usage.input_tokens, resp.usage.output_tokens)
+        self.messages.append({"role": "assistant", "content": resp.content})
+        text = "".join(b.text for b in resp.content if b.type == "text")
+        calls = [ToolCall(b.id, b.name, dict(b.input))
+                 for b in resp.content if b.type == "tool_use"]
+        return Reply(text, calls)
+
+    def _send_openai(self) -> Reply:
+        kwargs = dict(model=MODEL, messages=self.messages)
+        if self.tools:
+            kwargs["tools"] = [{"type": "function",
+                                "function": {"name": t["name"],
+                                             "description": t["description"],
+                                             "parameters": t["parameters"]}}
+                               for t in TOOL_SPECS]
+        resp = _get_client().chat.completions.create(**kwargs)
+        usage = resp.usage
+        self.meter.add(getattr(usage, "prompt_tokens", 0),
+                       getattr(usage, "completion_tokens", 0))
+        msg = resp.choices[0].message
+        self.messages.append(msg)
+        calls = []
+        for c in msg.tool_calls or []:
+            try:
+                args = json.loads(c.function.arguments or "{}")
+            except json.JSONDecodeError:
+                args = {"_raw": c.function.arguments}
+            calls.append(ToolCall(c.id, c.function.name, args))
+        return Reply(msg.content or "", calls)
+
+    # ---- run the tools a reply asked for, feed results back
+    def run_tools(self, reply: Reply, log=print) -> None:
+        for call in reply.tool_calls:
+            fn = TOOLS_IMPL.get(call.name)
+            if fn is None:
+                out = f"error: unknown tool {call.name}"
+            else:
+                try:
+                    out = str(fn(**call.args))
+                except Exception as e:           # error recovery: the error is an Observation
+                    out = f"error: {e}"
+            log(f"  [tool] {call.name}({call.args}) -> {out[:200].replace(chr(10), ' | ')}")
+            self.add_tool_result(call, out)

From 5df42133366101a070145656bfea3fd050f3a7db Mon Sep 17 00:00:00 2001
From: jaeook26 <jgc9268@gmail.com>
Date: Tue, 29 Sep 2026 12:14:46 +0900
Subject: [PATCH 3/3] Fix Week 02 fixed-model A/B experiment'

---
 submissions/26512076/week-02/REPORT.md        | 77 ++++++++-----------
 .../26512076/week-02/harness_plan_execute.py  | 29 ++++---
 .../26512076/week-02/logs/plan_exec-10.txt    |  4 +
 .../26512076/week-02/logs/plan_exec-11.txt    |  4 +
 .../26512076/week-02/logs/plan_exec-12.txt    |  4 +
 .../26512076/week-02/logs/plan_exec-16.txt    | 19 +++++
 .../26512076/week-02/logs/plan_exec-17.txt    | 15 ++++
 .../26512076/week-02/logs/plan_exec-18.txt    | 21 +++++
 .../26512076/week-02/logs/react-07.txt        |  5 ++
 .../26512076/week-02/logs/react-08.txt        |  5 ++
 .../26512076/week-02/logs/react-09.txt        |  5 ++
 .../26512076/week-02/logs/react-13.txt        |  5 ++
 .../26512076/week-02/logs/react-14.txt        |  5 ++
 .../26512076/week-02/logs/react-15.txt        |  5 ++
 submissions/26512076/week-02/results.csv      | 12 +++
 submissions/26512076/week-02/run_ab.py        |  6 +-
 submissions/26512076/week-02/tools_shared.py  | 27 +++----
 17 files changed, 175 insertions(+), 73 deletions(-)
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-10.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-11.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-12.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-16.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-17.txt
 create mode 100644 submissions/26512076/week-02/logs/plan_exec-18.txt
 create mode 100644 submissions/26512076/week-02/logs/react-07.txt
 create mode 100644 submissions/26512076/week-02/logs/react-08.txt
 create mode 100644 submissions/26512076/week-02/logs/react-09.txt
 create mode 100644 submissions/26512076/week-02/logs/react-13.txt
 create mode 100644 submissions/26512076/week-02/logs/react-14.txt
 create mode 100644 submissions/26512076/week-02/logs/react-15.txt

diff --git a/submissions/26512076/week-02/REPORT.md b/submissions/26512076/week-02/REPORT.md
index ff4e23c66..2ae832927 100644
--- a/submissions/26512076/week-02/REPORT.md
+++ b/submissions/26512076/week-02/REPORT.md
@@ -1,60 +1,45 @@
-\# Week 02 — Harness A/B Report
+# Week 02 — Harness A/B Report
 
+## 1. Variant definition
 
+이번 실험에서는 동일한 태스크와 도구를 사용하여 ReAct와 Plan-then-Execute 하네스를 비교했습니다. 최종 비교 실행(13–18번)의 Provider는 Groq이며, 두 하네스 모두 명시적인 단일 모델 ID인 `openai/gpt-oss-20b`를 사용했습니다. `tools_shared.py`에서 모델을 고정하고 Groq API를 OpenAI 호환 SDK로 호출했습니다. API 키는 파일에 저장하지 않고 `GROQ_API_KEY` 환경변수로 전달했습니다.
 
-\## 1. Variant definition
+공통 도구는 `read_file(path)`와 `count_pattern(path, pattern)`입니다. 태스크는 변경하지 않은 `app.log`에서 ERROR가 가장 많이 발생한 시간대(HH:00)를 찾는 것입니다. `TASK.md`의 기대 문자열 `14:00`이 최종 답변에 포함되면 성공으로 판정했습니다. 실행 명령은 `python run_ab.py --runs 3`입니다. 이 명령은 각 하네스를 3회 실행하고 결과와 실행별 로그를 기존 파일에 추가합니다.
 
+ReAct는 모델의 판단, 도구 호출, 관찰 결과를 단계별로 반복하며 전체 대화 기록을 다음 호출에 전달합니다. 최종 답변을 내거나 최대 단계 수에 도달하면 종료합니다. 도구 실행 중 발생한 오류는 관찰 결과로 전달하여 다음 단계에서 대응할 수 있습니다.
 
+Plan-then-Execute는 먼저 도구 없이 JSON 리스트 형식의 계획을 생성하고, 실행기가 각 계획 단계를 순서대로 처리합니다. 한 단계의 도구 호출은 최대 3라운드이며, `OFF_PLAN`이 발생하면 최대 1회 재계획할 수 있습니다. 두 하네스는 컨텍스트를 구성하는 방식, 종료 조건, 오류 복구 방식이 다릅니다. 도구 종류와 도구의 작업 단위는 같으며, 사람의 승인 단계는 두 하네스 모두 없습니다.
 
-이번 실험에서는 모델, 태스크, 도구를 동일하게 유지하고 하네스만 변경했다. Provider는 OpenRouter, 모델은 `openrouter/free`를 사용했다. 공통 도구는 `read\_file(path)`와 `count\_pattern(path, pattern)`이다. 태스크는 `app.log`에서 ERROR가 가장 많이 발생한 시간대를 찾는 것이며, 정답 문자열이 최종 답변에 포함되면 성공으로 판정했다.
-
-
-
-ReAct 하네스는 매 단계에서 Thought → Action → Observation을 반복하며 전체 대화 기록을 다음 호출에 전달한다. 모델이 답변을 완료하거나 최대 8단계에 도달하면 종료한다. 도구 오류도 Observation으로 돌려주므로 다음 단계에서 행동을 수정할 수 있다.
-
-
-
-Plan-then-Execute 하네스는 먼저 전체 계획을 JSON 리스트로 생성한 다음 각 단계를 순서대로 실행한다. 한 단계당 도구 호출은 최대 3라운드이며, 계획에서 벗어났을 때 재계획은 최대 1회 허용된다. 따라서 두 하네스는 주로 컨텍스트 관리, 종료 조건, 오류 복구 축에서 다르다. 도구 granularity는 동일하며, 읽기 전용 도구만 사용했으므로 사람의 개입 지점도 동일하게 0회였다.
-
-
-
-실행 명령은 다음과 같다.
-
-
-
-`python run\_ab.py --runs 3`
-
-
-
-\## 2. Measurements
-
+## 2. Measurements
 
+`results.csv`의 모든 실행을 아래에 보존했습니다. 1–6번은 기존 `openrouter/free` 실행입니다. 이 ID는 호출마다 실제 무료 모델을 선택할 수 있어 고정 모델 A/B 비교에는 사용하지 않았습니다. 7–12번은 Groq 모델을 고정한 뒤의 첫 실행이며, 10–12번에서 계획 생성 중 API 오류가 발생했습니다. 계획 단계 지시를 수정한 다음 실행한 **13–18번이 최종 A/B 비교 대상**입니다.
 
 | run | harness | success | tokens | iters | interventions | note |
-
 |---:|---|:---:|---:|---:|---:|---|
-
 | 1 | react | O | 15395 | 5 | 0 | |
-
 | 2 | react | O | 11782 | 4 | 0 | |
-
 | 3 | react | O | 8838 | 4 | 0 | |
-
-| 4 | plan\_exec | O | 15382 | 7 | 0 | replans=1 |
-
-| 5 | plan\_exec | X | 225 | 1 | 0 | replans=0 |
-
-| 6 | plan\_exec | X | 738 | 1 | 0 | replans=0 |
-
-
-
-ReAct의 성공률은 3/3이며 평균 토큰은 12,005개, 평균 반복 횟수는 4.33회였다. Plan-then-Execute의 성공률은 1/3이며 평균 토큰은 약 5,448개, 평균 반복 횟수는 3회였다.
-
-
-
-\## 3. Interpretation
-
-
-
-이 태스크에서는 ReAct가 성공률에서 우세했다. ReAct는 세 번 모두 성공했으며, 매 단계의 Observation을 다음 판단에 반영하는 오류 복구 방식이 안정성에 기여했다. 반면 Plan-then-Execute는 한 번만 성공했고 두 번은 계획 생성 호출 1회 후 종료되었다. 5번과 6번 실행은 반복 횟수가 1이고 재계획도 0회이므로, 실행 단계에 들어가기 전 계획을 유효한 JSON으로 파싱하지 못한 실패임을 로그에서 확인할 수 있다. Plan 방식의 평균 토큰이 더 적은 것은 단순히 효율성이 높아서가 아니라 두 실행이 초기에 실패했기 때문이다. 유일하게 성공한 Plan 실행은 재계획 1회를 사용했고, ReAct의 각 성공 실행보다 많은 7회 호출과 15,382토큰을 소비했다. 따라서 이번 실험에서는 ReAct의 단계별 컨텍스트 갱신과 유연한 오류 복구가 성공률을 높였고, Plan-then-Execute의 엄격한 계획 형식과 제한된 재계획 조건은 초기 계획 오류에 취약하게 작용했다.
+| 4 | plan_exec | O | 15382 | 7 | 0 | replans=1 |
+| 5 | plan_exec | X | 225 | 1 | 0 | replans=0 |
+| 6 | plan_exec | X | 738 | 1 | 0 | replans=0 |
+| 7 | react | O | 2581 | 2 | 0 | |
+| 8 | react | O | 2355 | 2 | 0 | |
+| 9 | react | O | 2096 | 2 | 0 | |
+| 10 | plan_exec | X | | | | API 400: tool_use_failed |
+| 11 | plan_exec | X | | | | API 400: tool_use_failed |
+| 12 | plan_exec | X | | | | API 400: tool_use_failed |
+| 13 | react | O | 2183 | 2 | 0 | |
+| 14 | react | O | 2056 | 2 | 0 | |
+| 15 | react | O | 3495 | 2 | 0 | |
+| 16 | plan_exec | O | 18143 | 8 | 0 | replans=0 |
+| 17 | plan_exec | O | 33126 | 13 | 0 | replans=0 |
+| 18 | plan_exec | O | 33465 | 10 | 0 | replans=0 |
+
+최종 비교(13–18번)에서 두 하네스 모두 3/3회 성공했고 사람의 개입은 0회였습니다. ReAct는 평균 2,578토큰과 2회 호출을 사용했습니다. Plan-then-Execute는 평균 약 28,245토큰과 10.33회 호출을 사용했습니다. `iters`는 도구 호출 횟수가 아니라 모델 호출 횟수입니다.
+
+## 3. Interpretation
+
+최종 실행에서는 성공률이 같았지만 ReAct가 더 적은 토큰과 모델 호출로 답을 구했습니다. ReAct 로그(13–15번)에서는 `app.log`를 읽은 뒤 두 번째 모델 호출에서 모두 `14:00`을 답했습니다. Plan-then-Execute 로그(16–18번)에서는 먼저 계획을 만들고 각 단계를 별도 호출로 실행했습니다. 특히 17번은 파일 읽기, 전체 줄 수 확인, ERROR 줄 수 확인, 시간 추출, 시간대별 집계 등을 여러 단계로 나누어 13회 호출했습니다. 이 순차 실행 방식이 이번 짧은 태스크에서 토큰과 반복 횟수를 늘린 것으로 해석됩니다. 세 실행 모두 재계획은 사용하지 않았습니다.
+
+이전 실패도 하네스의 동작을 보여줍니다. 5·6번은 계획을 유효한 JSON 리스트로 파싱하지 못해 실행 단계에 들어가지 못했습니다. 10–12번은 고정 모델의 첫 설정에서 계획 생성 중 도구 호출을 시도하여 API가 `tool_use_failed`를 반환한 사례입니다. 이 실행들의 빈 토큰·반복 횟수 칸은 `run_ab.py`가 예외 발생 시 계측값을 기록하지 못한 결과이며, 0을 뜻하지 않습니다. 이후 계획 단계에서는 도구를 사용할 수 없다는 지시를 명확히 하여 13–18번을 다시 실행했습니다. 따라서 서로 다른 모델이 선택될 수 있었던 1–6번이나 계획 지시가 달랐던 7–12번을 최종 성능 평균에 섞지 않았습니다. 최종 결과는 각 하네스 3회에 대한 관찰이므로, 다른 태스크에서도 같은 효율 차이가 난다고 일반화할 수는 없습니다.
 
diff --git a/submissions/26512076/week-02/harness_plan_execute.py b/submissions/26512076/week-02/harness_plan_execute.py
index 07705a55d..35e1963fa 100644
--- a/submissions/26512076/week-02/harness_plan_execute.py
+++ b/submissions/26512076/week-02/harness_plan_execute.py
@@ -11,17 +11,20 @@
 from tools_shared import Chat, Meter, Reply
 
 SYSTEM_PLAN = (
-    "You are a planner. Reply with a JSON list of short strings, one per step, "
-    "and nothing else. No prose, no code fences."
+    "You are a planner. In this planning phase you cannot call tools. "
+    "Describe actions for a later executor. Reply with a JSON list of "
+    "short step strings and nothing else. No prose, no code fences."
 )
+
 SYSTEM_EXEC = (
-    "You execute one step of a plan at a time with the tools you are given. "
-    "If the step cannot be done as planned, reply with a line that starts with "
-    "'OFF_PLAN:' and explain why. When asked for the final answer, reply with a "
-    "line that starts with 'Answer:'."
+    "You execute exactly the current step of a plan using the provided tools. "
+    "Do not perform later steps or give the final answer during a step. "
+    "After completing the current step, report its result briefly. "
+    "If the current step cannot be done, start your reply with 'OFF_PLAN:' "
+    "and explain why. Only when explicitly asked for the final answer, "
+    "start your reply with 'Answer:'."
 )
 
-
 def parse_plan(text: str):
     """Return a list of step strings, or None if the model did not give JSON."""
     text = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.M).strip()
@@ -40,8 +43,11 @@ def run_plan_execute(task: str, max_replan: int = 1,
 
     # 1) PLAN: the whole plan in one call, no tools
     planner = Chat(SYSTEM_PLAN, meter, tools=False)
-    planner.add_user(f"Task: {task}\nAvailable tools: read_file(path), "
-                     f"count_pattern(path, pattern).")
+    planner.add_user(
+       f"Task: {task}\n"
+       "The later executor can read text files and count lines matching "
+       "a regular expression. Describe the steps; do not perform them."
+    )
     raw = planner.send().text
     plan = parse_plan(raw)
     if plan is None:                              # a parse failure is one failure mode
@@ -55,7 +61,10 @@ def run_plan_execute(task: str, max_replan: int = 1,
     replans = 0
     i = 0
     while i < len(plan):
-        executor.add_user(f"Execute step {i + 1}: {plan[i]}")
+        executor.add_user(
+            f"Execute only step {i + 1}: {plan[i]}. "
+            "Use no more tools than needed, then briefly report this step's result."
+        )
         reply = executor.send()
         rounds = 0
         while reply.tool_calls:                   # tool calls inside one step
diff --git a/submissions/26512076/week-02/logs/plan_exec-10.txt b/submissions/26512076/week-02/logs/plan_exec-10.txt
new file mode 100644
index 000000000..fd8754ef8
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-10.txt
@@ -0,0 +1,4 @@
+[model] openai/gpt-oss-20b
+crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "python", "arguments": import re, textwrap, json, sys, math, os, typing, collections, pprint, itertools, statistics}'}}
+[final] 
+[judge] expected='14:00' -> X (0.7s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-11.txt b/submissions/26512076/week-02/logs/plan_exec-11.txt
new file mode 100644
index 000000000..4fe7f184e
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-11.txt
@@ -0,0 +1,4 @@
+[model] openai/gpt-oss-20b
+crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "read_file", "arguments": {"path":"app.log"}}'}}
+[final] 
+[judge] expected='14:00' -> X (0.4s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-12.txt b/submissions/26512076/week-02/logs/plan_exec-12.txt
new file mode 100644
index 000000000..eff10341f
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-12.txt
@@ -0,0 +1,4 @@
+[model] openai/gpt-oss-20b
+crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{"name": "read_file", "arguments": {"path":"app.log"}}'}}
+[final] 
+[judge] expected='14:00' -> X (1.2s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-16.txt b/submissions/26512076/week-02/logs/plan_exec-16.txt
new file mode 100644
index 000000000..180c7266b
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-16.txt
@@ -0,0 +1,19 @@
+[model] openai/gpt-oss-20b
+[plan] ['Read app.log line by line.', 'For each line containing "ERROR", extract the hour (HH) from the timestamp.', 'Increment a counter for that hour.', 'After all lines are processed, find the hour with the maximum count.', 'Print the hour in "HH:00" format.']
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 1] Step 1 completed: Read app.log line by line. (First 4000 characters of the file have been retrieved.)
+[step 2] Step 2 completed: Extracted the hour (HH) for each ERROR line. The extracted hours are: 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17.
+[step 3] Step 3 completed: Counter for each hour:
+- 09: 1
+- 10: 2
+- 11: 1
+- 12: 3
+- 13: 2
+- 14: 6
+- 15: 2
+- 16: 1
+- 17: 1
+[step 4] Step 4 completed: The hour with the maximum ERROR count is **14** with 6 errors.
+[step 5] Step 5 completed: The hour with the most ERROR lines is **14:00**.
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (136.3s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-17.txt b/submissions/26512076/week-02/logs/plan_exec-17.txt
new file mode 100644
index 000000000..42a1431f8
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-17.txt
@@ -0,0 +1,15 @@
+[model] openai/gpt-oss-20b
+[plan] ['Open the app.log file for reading.', 'Iterate through each line in the file.', "Identify lines that contain the string 'ERROR'.", "From each matching line, extract the timestamp part that represents the hour (e.g., the HH value from 'HH:MM:SS').", 'Count how many times each hour appears among the ERROR lines.', 'Determine which hour has the highest count.', "Return that hour formatted as 'HH:00'."]
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 1] The app.log file has been read.
+  [tool] count_pattern({'path': 'app.log', 'pattern': '^'}) -> 60
+[step 2] Step 2 complete: The file contains 60 lines.
+  [tool] count_pattern({'path': 'app.log', 'pattern': 'ERROR'}) -> 19
+[step 3] Step 3 complete: 19 lines contain the string 'ERROR'.
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 4] Step 4 complete: Extracted hours from ERROR lines – 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17.Step 4 complete: Hours extracted from ERROR lines are 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17.
+[step 5] Step 5 complete: Hour counts for ERROR lines – 09:1, 10:2, 11:1, 12:3, 13:2, 14:6, 15:2, 16:1, 17:1.
+[step 6] Step 6 complete: The hour with the most ERROR lines is **14**.
+[step 7] Step 7 complete: The hour with the most ERROR lines is **14:00**.
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (224.1s)
diff --git a/submissions/26512076/week-02/logs/plan_exec-18.txt b/submissions/26512076/week-02/logs/plan_exec-18.txt
new file mode 100644
index 000000000..675453828
--- /dev/null
+++ b/submissions/26512076/week-02/logs/plan_exec-18.txt
@@ -0,0 +1,21 @@
+[model] openai/gpt-oss-20b
+[plan] ['Open the file app.log for reading.', 'Read each line of the file sequentially.', 'For each line, check if it contains the word "ERROR" (case-sensitive).', 'If it does, extract the hour portion (HH) from the timestamp at the start of the line, assuming format like "YYYY-MM-DD HH:MM:SS".', 'Increment a counter for that hour.', 'After processing all lines, identify the hour with the highest count of ERROR lines.', 'Format that hour as "HH:00" and output it.']
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 1] File app.log opened successfully. 4000 characters read.
+[step 2] Read each line sequentially: 60 lines processed.
+[step 3] Checked each line: 19 lines contain the word "ERROR".
+[step 4] Extracted hour (HH) from each ERROR line: 09, 10, 10, 11, 12, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 16, 17.
+[step 5] Counter per hour after processing ERROR lines:
+- 09: 1
+- 10: 2
+- 11: 1
+- 12: 3
+- 13: 2
+- 14: 6
+- 15: 2
+- 16: 1
+- 17: 1
+[step 6] Identified hour with most ERROR lines: **14**.
+[step 7] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (199.2s)
diff --git a/submissions/26512076/week-02/logs/react-07.txt b/submissions/26512076/week-02/logs/react-07.txt
new file mode 100644
index 000000000..10bf86fdf
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-07.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (6.4s)
diff --git a/submissions/26512076/week-02/logs/react-08.txt b/submissions/26512076/week-02/logs/react-08.txt
new file mode 100644
index 000000000..2c45f45d4
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-08.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (1.6s)
diff --git a/submissions/26512076/week-02/logs/react-09.txt b/submissions/26512076/week-02/logs/react-09.txt
new file mode 100644
index 000000000..29c16d92e
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-09.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (1.1s)
diff --git a/submissions/26512076/week-02/logs/react-13.txt b/submissions/26512076/week-02/logs/react-13.txt
new file mode 100644
index 000000000..ffc9e06bb
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-13.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (4.5s)
diff --git a/submissions/26512076/week-02/logs/react-14.txt b/submissions/26512076/week-02/logs/react-14.txt
new file mode 100644
index 000000000..29e4fd48e
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-14.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (1.2s)
diff --git a/submissions/26512076/week-02/logs/react-15.txt b/submissions/26512076/week-02/logs/react-15.txt
new file mode 100644
index 000000000..45db0461d
--- /dev/null
+++ b/submissions/26512076/week-02/logs/react-15.txt
@@ -0,0 +1,5 @@
+[model] openai/gpt-oss-20b
+  [tool] read_file({'path': 'app.log'}) -> 2026-09-01 09:07:27 INFO  job finished in 320ms | 2026-09-01 09:11:56 ERROR NullReference in QuizService.score | 2026-09-01 09:13:14 INFO  health check ok | 2026-09-01 09:26:45 INFO  health check ok | 2026-09
+[step 2] Answer: 14:00
+[final] Answer: 14:00
+[judge] expected='14:00' -> O (2.1s)
diff --git a/submissions/26512076/week-02/results.csv b/submissions/26512076/week-02/results.csv
index 182a075f7..96584f6b5 100644
--- a/submissions/26512076/week-02/results.csv
+++ b/submissions/26512076/week-02/results.csv
@@ -5,3 +5,15 @@ run,harness,success,tokens,iters,interventions,note
 4,plan_exec,O,15382,7,0,replans=1
 5,plan_exec,X,225,1,0,replans=0
 6,plan_exec,X,738,1,0,replans=0
+7,react,O,2581,2,0,
+8,react,O,2355,2,0,
+9,react,O,2096,2,0,
+10,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""python"", ""arguments"": import re, textwrap, json, sys, math, os, typing, collections, pprint, itertools, statistics}'}}"
+11,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""read_file"", ""arguments"": {""path"":""app.log""}}'}}"
+12,plan_exec,X,,,,"crash: BadRequestError: Error code: 400 - {'error': {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{""name"": ""read_file"", ""arguments"": {""path"":""app.log""}}'}}"
+13,react,O,2183,2,0,
+14,react,O,2056,2,0,
+15,react,O,3495,2,0,
+16,plan_exec,O,18143,8,0,replans=0
+17,plan_exec,O,33126,13,0,replans=0
+18,plan_exec,O,33465,10,0,replans=0
diff --git a/submissions/26512076/week-02/run_ab.py b/submissions/26512076/week-02/run_ab.py
index bb43c0efa..5068c9495 100644
--- a/submissions/26512076/week-02/run_ab.py
+++ b/submissions/26512076/week-02/run_ab.py
@@ -7,6 +7,8 @@
 and saves each run's console output under logs/. Failed runs are kept:
 they are data.
 """
+
+from tools_shared import MODEL
 import argparse
 import csv
 import os
@@ -60,7 +62,9 @@ def main():
                 def log(msg, _lines=lines):
                     print(msg)
                     _lines.append(str(msg))
-
+                      
+                
+                log(f"[model] {MODEL}")
                 t0 = time.time()
                 note = ""
                 try:
diff --git a/submissions/26512076/week-02/tools_shared.py b/submissions/26512076/week-02/tools_shared.py
index a3fccd624..3248b0c6b 100644
--- a/submissions/26512076/week-02/tools_shared.py
+++ b/submissions/26512076/week-02/tools_shared.py
@@ -3,12 +3,10 @@
 Both harnesses import from here. Same tools and same model for both is what
 makes the A/B a harness comparison and not a tool comparison.
 
-Provider is picked from the environment:
-  ANTHROPIC_API_KEY set          -> Anthropic SDK (pip install anthropic)
-  otherwise                      -> OpenAI-compatible (pip install openai)
-                                    OPENAI_API_KEY, optional OPENAI_BASE_URL
-                                    (https://openrouter.ai/api/v1 for OpenRouter)
-  AGENT_MODEL                    optional model override for either provider
+Provider: Groq via the OpenAI-compatible SDK (pip install openai)
+API key: GROQ_API_KEY environment variable
+Base URL: https://api.groq.com/openai/v1
+Model: openai/gpt-oss-20b (fixed for both harnesses)
 """
 import json
 import os
@@ -86,10 +84,8 @@ class Reply:
     tool_calls: list = field(default_factory=list)
 
 
-PROVIDER = "anthropic" if os.environ.get("ANTHROPIC_API_KEY") else "openai"
-MODEL = os.environ.get(
-    "AGENT_MODEL",
-    "claude-sonnet-4-5" if PROVIDER == "anthropic" else "gpt-4o-mini")
+PROVIDER = "openai"
+MODEL = "openai/gpt-oss-20b"
 
 _client = None
 
@@ -97,12 +93,11 @@ class Reply:
 def _get_client():
     global _client
     if _client is None:
-        if PROVIDER == "anthropic":
-            import anthropic
-            _client = anthropic.Anthropic()
-        else:
-            from openai import OpenAI
-            _client = OpenAI()
+        from openai import OpenAI
+        _client = OpenAI(
+            api_key=os.environ["GROQ_API_KEY"],
+            base_url="https://api.groq.com/openai/v1",
+        )
     return _client