From d4f82ba465d47a951f83d9355a37d1e575a2e736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?El=C3=ADas=20Snorrason?= Date: Tue, 25 Mar 2025 10:16:31 -0700 Subject: [PATCH 1/3] Update response_helpfulness Eval criteria to include 'N.A.' as an unhelpful response --- src/cleanlab_tlm/utils/rag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cleanlab_tlm/utils/rag.py b/src/cleanlab_tlm/utils/rag.py index bfe09f3..ebc08d9 100644 --- a/src/cleanlab_tlm/utils/rag.py +++ b/src/cleanlab_tlm/utils/rag.py @@ -437,7 +437,7 @@ def __repr__(self) -> str: A Response is not helpful if it: - Is not useful, incomplete, or unclear - Abstains or refuses to answer the question -- Contains statements which are similar to 'I don't know', 'Sorry', or 'No information available' +- Contains statements which are similar to 'I don't know', 'Sorry', 'N.A.', or 'No information available' - Leaves part of the original User Query unresolved""", "query_identifier": "User Query", "context_identifier": None, From a06ebb4ece9a77b9d599e552f1e135a1bcf09838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?El=C3=ADas=20Snorrason?= Date: Tue, 25 Mar 2025 10:18:13 -0700 Subject: [PATCH 2/3] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 486faca..f708968 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Update `response_helpfulness` criteria + ## [1.0.13] - 2025-03-20 ### Added From 67140d26163057c5e0917c88fb91dc2b9ed617c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?El=C3=ADas=20Snorrason?= Date: Tue, 25 Mar 2025 10:47:48 -0700 Subject: [PATCH 3/3] update punctuation --- src/cleanlab_tlm/utils/rag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cleanlab_tlm/utils/rag.py b/src/cleanlab_tlm/utils/rag.py index ebc08d9..632db87 100644 --- a/src/cleanlab_tlm/utils/rag.py +++ b/src/cleanlab_tlm/utils/rag.py @@ -437,7 +437,7 @@ def __repr__(self) -> str: A Response is not helpful if it: - Is not useful, incomplete, or unclear - Abstains or refuses to answer the question -- Contains statements which are similar to 'I don't know', 'Sorry', 'N.A.', or 'No information available' +- Contains statements which are similar to 'I don't know', 'Sorry', 'N.A', or 'No information available' - Leaves part of the original User Query unresolved""", "query_identifier": "User Query", "context_identifier": None,