Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wfh/evaluate traceable #152

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Wfh/evaluate traceable #152

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
25bb706
temp
hinthornw Oct 25, 2023
6e54d22
Update
hinthornw Oct 25, 2023
0d92b1d
update
hinthornw Oct 25, 2023
72eb8c3
tmp
hinthornw Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 250 additions & 0 deletions testing-examples/evaluate-traceable/evaluating_traceable_functions.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,250 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "964403a2-efbb-4b81-8a9a-ac1e238cb0e0",
"metadata": {},
"source": [
"# Evaluating a Traceable Function\n",
"\n",
"If you are tracing some application using LangSmith's `@traceable` decorator, the `run_on_dataset` function should\n",
"automatically configure it to include all runs within the appropriate trace if it is directly passed in to the function.\n",
"\n",
"This functionality is supported in `langchain>=0.0.324`."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "37595189-eccc-403c-bff1-e24b5e8ec1cf",
"metadata": {},
"outputs": [],
"source": [
"from typing import List\n",
"\n",
"from langchain.schema import Document\n",
"from langchain.schema.retriever import BaseRetriever\n",
"\n",
"\n",
"class MinimalRetriever(BaseRetriever):\n",
" def __init__(self):\n",
" super().__init__()\n",
"\n",
" def get_relevant_documents(self, query: str) -> List[Document]:\n",
" # Implement your retrieval logic here\n",
" # Return a list of relevant documents based on the query\n",
" return [Document(page_content=query)]"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "1b35e9cd-cbc0-496d-99c3-3be104f0e15f",
"metadata": {},
"outputs": [],
"source": [
"import openai\n",
"from langchain.schema.runnable import RunnableLambda\n",
"from langsmith import traceable\n",
"\n",
"retriever = MinimalRetriever()\n",
"\n",
"\n",
"@traceable(run_type=\"llm\")\n",
"def complete(*args, **kwargs):\n",
" return openai.ChatCompletion.create(\n",
" *args,\n",
" **kwargs,\n",
" )\n",
"\n",
"\n",
"@traceable()\n",
"def my_model(question: str, context: str):\n",
" messages = [\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": f\"Respond to the user, taking into account the context: {context}\",\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": question,\n",
" },\n",
" ]\n",
" completion = complete(model=\"gpt-3.5-turbo\", messages=messages)\n",
" # It should still work if a LangChain component is called within a traceable function\n",
" format = RunnableLambda(lambda x: x.choices[0].message.content) | retriever\n",
" return format.invoke(completion, config={\"callbacks\": []})"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "cb27f19a-f925-4068-aaee-218f059dae54",
"metadata": {},
"outputs": [],
"source": [
"import uuid\n",
"\n",
"from langsmith import Client\n",
"\n",
"client = Client()\n",
"uid = uuid.uuid4()\n",
"examples = [\n",
" {\n",
" \"question\": \"What's 2+2\",\n",
" \"context\": \"You are a pirate\",\n",
" },\n",
" {\n",
" \"question\": \"Where did coffee originate from?\",\n",
" \"context\": \"You are a knight of King Arthur.\",\n",
" },\n",
"]\n",
"dataset_name = f\"Evaluating Traceables Walkthrough - {uid}\"\n",
"dataset = client.create_dataset(dataset_name)\n",
"\n",
"client.create_examples(\n",
" inputs=examples,\n",
" dataset_id=dataset.id,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "069d0434-6cb8-464c-9df1-bd21d3c80502",
"metadata": {},
"outputs": [],
"source": [
"from langchain.callbacks.manager import CallbackManager, LangChainTracer\n",
"from langchain.schema.runnable import RunnableLambda\n",
"from langsmith.run_trees import RunTree\n",
"\n",
"\n",
"class RunnableTraceable(RunnableLambda):\n",
" def __init__(\n",
" self,\n",
" func,\n",
" afunc=None,\n",
" ) -> None:\n",
" def _configure_run_tree(callback_manager):\n",
" run_tree: Optional[RunTree] = None\n",
" if isinstance(callback_manager, CallbackManager):\n",
" lc_tracers = [\n",
" handler\n",
" for handler in callback_manager.handlers\n",
" if isinstance(handler, LangChainTracer)\n",
" ]\n",
" if lc_tracers:\n",
" lc_tracer = lc_tracers[0]\n",
" run_tree = RunTree(\n",
" id=callback_manager.parent_run_id,\n",
" session_name=lc_tracer.project_name,\n",
" name=\"Wrapping\",\n",
" run_type=\"chain\",\n",
" inputs={},\n",
" )\n",
" return run_tree\n",
"\n",
" def wrap_traceable(inputs: dict, config: dict):\n",
" run_tree = _configure_run_tree(config.get(\"callbacks\"))\n",
" return func(**inputs, langsmith_extra={\"run_tree\": run_tree})\n",
"\n",
" async def awrap_traceable(inputs: dict, config: dict):\n",
" run_tree = _configure_run_tree(config.get(\"callbacks\"))\n",
" return await afunc(**inputs, langsmith_extra={\"run_tree\": run_tree})\n",
"\n",
" awrapped = None\n",
" if afunc is not None:\n",
" awrapped = awrap_traceable\n",
"\n",
" super().__init__(wrap_traceable, awrapped)"
]
},
{
"cell_type": "markdown",
"id": "c235d21d-4c58-464c-a827-88718dbf2a7f",
"metadata": {},
"source": [
"## Promoting to LangChain component\n",
"\n",
"In general, it is recommended to use LangChain runnables throughout your LLM application wherever you want tracing and callbacks. Beta support for composing `@traceable` functions with other LangChain runnables is provided via the `RunnableTraceable` class."
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "08b09831-7b73-48cf-bd66-3758c8db2a76",
"metadata": {},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langsmith.run_helpers import as_runnable\n",
"\n",
"chain = (\n",
" as_runnable(my_model) | ChatOpenAI()\n",
") # the second call will respond to the first"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "9e532b0e-8695-4774-84d1-541b6dbc3c98",
"metadata": {},
"outputs": [
{
"ename": "ValidationError",
"evalue": "1 validation error for ChatPromptValue\nmessages -> 0 -> content\n field required (type=value_error.missing)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[41], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mchain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquestion\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mWhy is blue brown?\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mYou are a pirate\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/code/lc/langchain/libs/langchain/langchain/schema/runnable/base.py:1153\u001b[0m, in \u001b[0;36mRunnableSequence.invoke\u001b[0;34m(self, input, config)\u001b[0m\n\u001b[1;32m 1151\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1152\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, step \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msteps):\n\u001b[0;32m-> 1153\u001b[0m \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mstep\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1154\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1155\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# mark each step as a child run\u001b[39;49;00m\n\u001b[1;32m 1156\u001b[0m \u001b[43m \u001b[49m\u001b[43mpatch_config\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1157\u001b[0m \u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_child\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mseq:step:\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1158\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1159\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1160\u001b[0m \u001b[38;5;66;03m# finish the root run\u001b[39;00m\n\u001b[1;32m 1161\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
"File \u001b[0;32m~/code/lc/langchain/libs/langchain/langchain/chat_models/base.py:143\u001b[0m, in \u001b[0;36mBaseChatModel.invoke\u001b[0;34m(self, input, config, stop, **kwargs)\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minvoke\u001b[39m(\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28minput\u001b[39m: LanguageModelInput,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any,\n\u001b[1;32m 138\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m BaseMessage:\n\u001b[1;32m 139\u001b[0m config \u001b[38;5;241m=\u001b[39m config \u001b[38;5;129;01mor\u001b[39;00m {}\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m cast(\n\u001b[1;32m 141\u001b[0m ChatGeneration,\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgenerate_prompt(\n\u001b[0;32m--> 143\u001b[0m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_convert_input\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m],\n\u001b[1;32m 144\u001b[0m stop\u001b[38;5;241m=\u001b[39mstop,\n\u001b[1;32m 145\u001b[0m callbacks\u001b[38;5;241m=\u001b[39mconfig\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcallbacks\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 146\u001b[0m tags\u001b[38;5;241m=\u001b[39mconfig\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 147\u001b[0m metadata\u001b[38;5;241m=\u001b[39mconfig\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 148\u001b[0m run_name\u001b[38;5;241m=\u001b[39mconfig\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrun_name\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 149\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 150\u001b[0m )\u001b[38;5;241m.\u001b[39mgenerations[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m],\n\u001b[1;32m 151\u001b[0m )\u001b[38;5;241m.\u001b[39mmessage\n",
"File \u001b[0;32m~/code/lc/langchain/libs/langchain/langchain/chat_models/base.py:124\u001b[0m, in \u001b[0;36mBaseChatModel._convert_input\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m StringPromptValue(text\u001b[38;5;241m=\u001b[39m\u001b[38;5;28minput\u001b[39m)\n\u001b[1;32m 123\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mlist\u001b[39m):\n\u001b[0;32m--> 124\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mChatPromptValue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 125\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 126\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 127\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid input type \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28minput\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMust be a PromptValue, str, or list of BaseMessages.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 129\u001b[0m )\n",
"File \u001b[0;32m~/code/lc/langchain/libs/langchain/langchain/load/serializable.py:97\u001b[0m, in \u001b[0;36mSerializable.__init__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 96\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lc_kwargs \u001b[38;5;241m=\u001b[39m kwargs\n",
"File \u001b[0;32m~/.pyenv/versions/3.11.2/lib/python3.11/site-packages/pydantic/main.py:341\u001b[0m, in \u001b[0;36mpydantic.main.BaseModel.__init__\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mValidationError\u001b[0m: 1 validation error for ChatPromptValue\nmessages -> 0 -> content\n field required (type=value_error.missing)"
]
}
],
"source": [
"chain.invoke(\n",
" {\n",
" \"question\": \"Why is blue brown?\",\n",
" \"context\": \"You are a pirate\",\n",
" }\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "340408c5-c769-4547-9130-1ae6bc4dbe77",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}