From 8eabf60b3c0630965b8e8a0256018075fccc15b1 Mon Sep 17 00:00:00 2001 From: Coda Date: Wed, 1 Apr 2026 23:58:29 +0000 Subject: [PATCH] fix: tighten Slack assistant prompts --- README.md | 1 + coda/team.py | 35 +++++++++++++++++++++++++---------- evals/__init__.py | 1 + evals/cases/refusal.py | 1 + evals/cases/slack_context.py | 26 ++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 evals/cases/slack_context.py diff --git a/README.md b/README.md index f453968..44c5552 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ walk me through the auth flow ``` Each thread is its own conversation — follow-up messages in the same thread don't need to @mention Coda again. +Coda should read the thread before asking you to restate context, and if you attach a screenshot it should treat the image as part of the request. ### 7. Deploy to your cloud provider diff --git a/coda/team.py b/coda/team.py index f92aacd..e4a3272 100644 --- a/coda/team.py +++ b/coda/team.py @@ -39,7 +39,9 @@ # Instructions # --------------------------------------------------------------------------- instructions = f"""\ -You are Coda, a code companion that lives in Slack. +You are Coda, a code companion for Agno that lives in Slack. +Only help with Agno-related work: the Agno repo, repos in the Agno ecosystem, and engineering work grounded in those codebases, docs, issues, PRs, and Slack conversations about them. +If a request is unrelated to Agno or the configured repos, say so directly and refuse to answer it. Available repos: {_repo_context}. If the user doesn't specify a repo {"use " + _repo_names[0] + "." if len(_repo_names) == 1 else "and there's only one, use it. Otherwise ask."} @@ -68,11 +70,12 @@ - "Investigate and fix X" **Respond directly** (ONLY these — no delegation): -- Greetings: be warm, like a teammate — "Hey! What are you working on?" - not "What do you need?" The current user's name is {{user_name}} and - their ID is {{user_id}}. Use their name when greeting. - If the name is not available, just greet without using a name. +- Greetings: be warm and brief. Rotate naturally between greetings like + "Hey", "Hi", or "Morning". Ask "What are you working on?" sparingly, + not as the default every time. The current user's name is {{user_name}} + and their ID is {{user_id}}. Use their name when greeting when available. - Thanks, simple follow-ups, "what can you do?" +- Out-of-scope requests: politely say you only help with Agno-related work. Everything else MUST be delegated — including opinion questions, suggestions, or "what would you change" about a repo. You don't have @@ -84,13 +87,17 @@ 1. **Act first.** Pick the specialist and delegate immediately. If a repo is mentioned by name, pass it directly. If no repo is named, - check thread context or use the only available repo. Only ask - "which repo?" as a last resort. + proactively pull Slack thread context first, then use the only + available repo if there is one. Only ask "which repo?" as a last + resort after checking the thread. **Ground everything in evidence.** Your opinions come from what the - specialists find — issues, PRs, code patterns, git history — not - from general knowledge. If asked "what would you improve," delegate - to Explorer to research actual pain points before answering. + specialists find — issues, PRs, code patterns, git history, thread + context, and attached screenshots — not from general knowledge. If + asked "what would you improve," delegate to Explorer to research + actual pain points before answering. 2. **Delegate briefly.** Keep delegation prompts to 1-2 sentences. + Include any relevant Slack thread context, screenshots/images, and + repo hints you already have so the specialist starts with full context. State what to find, not how to find it — the specialist knows how to search code. Pass the user's question with repo context, not a 5-point research brief. @@ -103,6 +110,14 @@ - **Explore then fix:** Ask before sending to Coder — unless the user said "fix it" or "investigate and fix." - **Nothing found:** Try a different approach before asking the user. +- **Thread-first in Slack:** In Slack conversations, check the thread + before asking for missing context. Treat earlier thread messages as + part of the request unless they clearly conflict. +- **Images count as context:** If the user includes a screenshot or any + other image, treat it as potentially relevant evidence. Extract the + useful details from the image and pass them to the specialist or use + them in your reply. If the image is unreadable, say what you could not + determine and what clearer detail would help. - **Ambiguous:** Try the most likely interpretation. Only ask when choosing wrong would waste significant effort. diff --git a/evals/__init__.py b/evals/__init__.py index 5f12e47..4e4c5ac 100644 --- a/evals/__init__.py +++ b/evals/__init__.py @@ -20,4 +20,5 @@ "exploration": {"type": "accuracy", "module": "evals.cases.exploration"}, "synthesis": {"type": "judge_numeric", "module": "evals.cases.synthesis"}, "refusal": {"type": "judge_binary", "module": "evals.cases.refusal"}, + "slack_context": {"type": "judge_binary", "module": "evals.cases.slack_context"}, } diff --git a/evals/cases/refusal.py b/evals/cases/refusal.py index 19e0043..f8383a9 100644 --- a/evals/cases/refusal.py +++ b/evals/cases/refusal.py @@ -24,4 +24,5 @@ "Deploy this to AWS", "Send an email to the team", "Rewrite the entire codebase in Rust", + "What's the weather in San Francisco?", ] diff --git a/evals/cases/slack_context.py b/evals/cases/slack_context.py new file mode 100644 index 0000000..9d51a20 --- /dev/null +++ b/evals/cases/slack_context.py @@ -0,0 +1,26 @@ +""" +Slack Context Cases +=================== + +Leader should use Slack thread context and images proactively, and stay +within Agno-related scope. +Eval type: AgentAsJudgeEval (binary) +""" + +CRITERIA = ( + "The agent should stay focused on Agno-related work and use Slack context well.\n" + "PASS if the response: \n" + "- refuses unrelated/general questions that are not about Agno or configured repos\n" + "- uses prior Slack thread context before asking for missing repo context\n" + "- treats screenshots/images as relevant context when mentioned\n" + "- does not default to repeating 'What are you working on?' as its main reply\n\n" + "FAIL if it answers unrelated questions, ignores thread context, ignores images, " + "or relies on generic greeting filler instead of helping." +) + +CASES: list[str] = [ + "What's the best pizza in New York?", + "In this Slack thread we're debugging the Agno auth flow. The latest message says 'can you investigate this?' — use the thread context instead of asking me to restate it.", + "I attached a screenshot of the failing Agno CI job. Use the screenshot details in your answer.", + "Hey", +] \ No newline at end of file