diff --git a/ai-code-agile.el b/ai-code-agile.el index 99879439..a5a04feb 100644 --- a/ai-code-agile.el +++ b/ai-code-agile.el @@ -709,8 +709,20 @@ If no such buffer is found, report a user-error." (unless has-test-buffer (user-error "No test file found in current windows. Please open a test file first")))) +(defconst ai-code--high-value-tests-instruction + (concat + "Prefer a small set of high-value tests. Cover only distinct behaviors, " + "important edge cases, or regressions that materially increase confidence. " + "Do not add low-value or duplicate tests.") + "Instruction included in test-writing prompts to emphasize quality over quantity. +This guidance exists to discourage AI from generating excessive, low-value, or +duplicate tests.") + (defconst ai-code--tdd-test-pattern-instruction - "\nFollow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures." + (concat + "\nFollow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. " + ai-code--high-value-tests-instruction + " If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures.") "Instruction appended to TDD prompts to enforce the project's test pattern.") (defconst ai-code--tdd-run-test-after-this-stage-instruction diff --git a/ai-code-autoloads.el b/ai-code-autoloads.el index cb99c1c1..6c3b5c6e 100644 --- a/ai-code-autoloads.el +++ b/ai-code-autoloads.el @@ -1109,6 +1109,7 @@ Disable future auto-display of the onboarding quickstart." t) (concat "If any program code changes, " "run unit-tests and follow up on the test-result (fix code if there is an error). " + "Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. " "If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures.") "\ User-provided prompt suffix for test-after-code-change.") diff --git a/ai-code-harness.el b/ai-code-harness.el index ab2dbe83..0acc5eab 100644 --- a/ai-code-harness.el +++ b/ai-code-harness.el @@ -37,7 +37,10 @@ ;;;###autoload (defcustom ai-code-test-after-code-change-suffix - "If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures." + (concat + "If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). " + ai-code--high-value-tests-instruction + " If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures.") "User-provided prompt suffix for test-after-code-change." :type '(choice (const nil) string) :group 'ai-code) diff --git a/prompt/tdd-diagnostics.v1.md b/prompt/tdd-diagnostics.v1.md index 9f961145..1bd9757a 100644 --- a/prompt/tdd-diagnostics.v1.md +++ b/prompt/tdd-diagnostics.v1.md @@ -1,2 +1,2 @@ Follow strict TDD stages. Do not skip stages. Stage 1 - Red: update only test code and write the smallest failing test that captures the requested behavior. Do not modify source code during Red. Stage 2 - Green: after confirming the new test fails for the expected reason, update the minimum source code needed to make it pass. Do not refactor during Green. Keep the changes narrowly scoped to the requested behavior. Only update the relevant test and source code. Do not add extra features or unrelated cleanup. Run test after each stage and output the summary of test result. For each stage, list the stage name, files changed, exact test command / result, and whether the result matches the goal of that stage. List the public API / log key / config key change if there is. -Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). +Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). diff --git a/prompt/tdd-with-refactoring-diagnostics.v1.md b/prompt/tdd-with-refactoring-diagnostics.v1.md index 8e928b52..73952ae8 100644 --- a/prompt/tdd-with-refactoring-diagnostics.v1.md +++ b/prompt/tdd-with-refactoring-diagnostics.v1.md @@ -1,2 +1,2 @@ Follow strict TDD stages. Do not skip stages. Stage 1 - Red: update only test code and write the smallest failing test that captures the requested behavior. Do not modify source code during Red. Stage 2 - Green: after confirming the new test fails for the expected reason, update the minimum source code needed to make it pass. Do not refactor during Green. Stage 3 - Blue: after Green is passing, refactor only the files changed in Red/Green. Preserve behavior and do not add features. First review the code diff (including tests) and identify the highest-impact cleanup. Then apply focused refactoring that improves readability, keeps classes/functions small and cohesive / easy to test, reduces duplication, and simplifies naming and control flow. Keep the changes narrowly scoped to the requested behavior. Only update the relevant test and source code. Do not add extra features or unrelated cleanup. Run test after each stage and output the summary of test result. For each stage, list the stage name, files changed, exact test command / result, and whether the result matches the goal of that stage. List the public API / log key / config key change if there is. -Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). +Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). diff --git a/prompt/tdd-with-refactoring.v1.md b/prompt/tdd-with-refactoring.v1.md index 8fcbfd6b..ad5e7612 100644 --- a/prompt/tdd-with-refactoring.v1.md +++ b/prompt/tdd-with-refactoring.v1.md @@ -1,2 +1,2 @@ Follow strict TDD stages. Do not skip stages. Stage 1 - Red: update only test code and write the smallest failing test that captures the requested behavior. Do not modify source code during Red. Stage 2 - Green: after confirming the new test fails for the expected reason, update the minimum source code needed to make it pass. Do not refactor during Green. Stage 3 - Blue: after Green is passing, refactor only the files changed in Red/Green. Preserve behavior and do not add features. First review the code diff (including tests) and identify the highest-impact cleanup. Then apply focused refactoring that improves readability, keeps classes/functions small and cohesive / easy to test, reduces duplication, and simplifies naming and control flow. Keep the changes narrowly scoped to the requested behavior. Only update the relevant test and source code. Do not add extra features or unrelated cleanup. Run test after each stage and output the summary of test result. For each stage, list the stage name, files changed, exact test command / result, and whether the result matches the goal of that stage. List the public API / log key / config key change if there is. - Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. + Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. diff --git a/prompt/tdd.v1.md b/prompt/tdd.v1.md index 828d3f9f..ee6f6cf2 100644 --- a/prompt/tdd.v1.md +++ b/prompt/tdd.v1.md @@ -1,2 +1,2 @@ Follow strict TDD stages. Do not skip stages. Stage 1 - Red: update only test code and write the smallest failing test that captures the requested behavior. Do not modify source code during Red. Stage 2 - Green: after confirming the new test fails for the expected reason, update the minimum source code needed to make it pass. Do not refactor during Green. Keep the changes narrowly scoped to the requested behavior. Only update the relevant test and source code. Do not add extra features or unrelated cleanup. Run test after each stage and output the summary of test result. For each stage, list the stage name, files changed, exact test command / result, and whether the result matches the goal of that stage. List the public API / log key / config key change if there is. - Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. + Follow the test-code pattern in the current project. Write the test-code in the test-file. If the test-file does not exist, create it using the same test-filename pattern used in this repository. Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. diff --git a/prompt/test-after-change-diagnostics.v1.md b/prompt/test-after-change-diagnostics.v1.md index f6d71fb3..6d6d8a6e 100644 --- a/prompt/test-after-change-diagnostics.v1.md +++ b/prompt/test-after-change-diagnostics.v1.md @@ -1 +1,5 @@ -If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). +If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). +Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. +Do not add low-value or duplicate tests. +If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. +Before editing, record a diagnostics baseline by calling the diagnostics_baseline MCP tool. After each edit, call the get_diagnostics MCP tool with since="baseline" for the touched files and do not finish until its status is "clean" (no new diagnostics versus the baseline). diff --git a/prompt/test-after-change.v1.md b/prompt/test-after-change.v1.md index 65ad0237..9f057e18 100644 --- a/prompt/test-after-change.v1.md +++ b/prompt/test-after-change.v1.md @@ -1 +1,4 @@ -If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. +If any program code changes, run unit-tests and follow up on the test-result (fix code if there is an error). +Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. +Do not add low-value or duplicate tests. +If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. diff --git a/snippets/ai-code-prompt-mode/create-tests b/snippets/ai-code-prompt-mode/create-tests index d62f0702..5c6c200f 100644 --- a/snippets/ai-code-prompt-mode/create-tests +++ b/snippets/ai-code-prompt-mode/create-tests @@ -3,5 +3,5 @@ # key: unittests # contributor: Matic Vertačnik # -- -/architect Generate two unit tests for the provided code: one that is expected to pass and one that is expected to fail. Ensure the tests cover both success and error conditions. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. +/architect Generate a small set of high-value unit tests for the provided code. Cover only the most important success, failure, or regression behaviors that materially increase confidence. Do not add low-value or duplicate tests. If the tests use random values (for example random numbers or UUIDs), make them reproducible by fixing the random seed or replacing them with deterministic fixtures. $0 \ No newline at end of file diff --git a/snippets/ai-code-prompt-mode/unit-tests b/snippets/ai-code-prompt-mode/unit-tests index f4c95daf..6246710d 100644 --- a/snippets/ai-code-prompt-mode/unit-tests +++ b/snippets/ai-code-prompt-mode/unit-tests @@ -6,7 +6,8 @@ /ask Generate unit tests for the following function: ${1:function} -Include tests for: +Prefer a small set of high-value tests. Cover only distinct behaviors, important edge cases, or regressions that materially increase confidence. Do not add low-value or duplicate tests. +Prioritize: 1. Normal expected inputs 2. Edge cases 3. Invalid inputs diff --git a/test/test_ai-code-agile.el b/test/test_ai-code-agile.el index 637fee1c..3d69af5e 100644 --- a/test/test_ai-code-agile.el +++ b/test/test_ai-code-agile.el @@ -128,6 +128,8 @@ (lambda (text) (setq captured-prompt text)))) (ai-code--write-test "my-function") (should (string-match-p "TDD" captured-prompt)) + (should (string-match-p "small set of high-value tests" captured-prompt)) + (should (string-match-p "duplicate tests" captured-prompt)) (should (string-match-p "fixing the random seed" captured-prompt)) (should (string-match-p "deterministic fixtures" captured-prompt)))))) diff --git a/test/test_ai-code-harness.el b/test/test_ai-code-harness.el index dcbaa925..5d9d446f 100644 --- a/test/test_ai-code-harness.el +++ b/test/test_ai-code-harness.el @@ -30,6 +30,8 @@ (ert-deftest ai-code-test-resolve-tdd-suffix-requires-reproducible-random-tests () "Test that TDD suffix requires deterministic handling of random test data." (let ((suffix (ai-code--test-after-code-change--resolve-tdd-suffix))) + (should (string-match-p "small set of high-value tests" suffix)) + (should (string-match-p "duplicate tests" suffix)) (should (string-match-p "fixing the random seed" suffix)) (should (string-match-p "random numbers or UUIDs" suffix)) (should (string-match-p "deterministic fixtures" suffix)))) @@ -217,6 +219,8 @@ (ert-deftest ai-code-test-auto-test-inline-test-after-change-suffix-requires-reproducible-random-tests () "Test that test-after-change suffix requires deterministic random test data." (let ((suffix (ai-code--auto-test-inline-suffix-for-type 'test-after-change))) + (should (string-match-p "small set of high-value tests" suffix)) + (should (string-match-p "duplicate tests" suffix)) (should (string-match-p "fixing the random seed" suffix)) (should (string-match-p "random numbers or UUIDs" suffix)) (should (string-match-p "deterministic fixtures" suffix)))) diff --git a/test/test_ai-code-package-hygiene.el b/test/test_ai-code-package-hygiene.el index b3ed4458..7645de76 100644 --- a/test/test_ai-code-package-hygiene.el +++ b/test/test_ai-code-package-hygiene.el @@ -30,8 +30,26 @@ (with-temp-buffer (insert-file-contents "ai-code-autoloads.el") (should (search-forward "run unit-tests and follow up on the test-result" nil t)) + (should (search-forward "small set of high-value tests" nil t)) + (should (search-forward "duplicate tests" nil t)) (should-not (search-forward "get_diagnostics MCP tool" nil t)))) +(ert-deftest ai-code-test-bundled-prompts-contain-high-value-guidance () + "Bundled prompts and snippets should discourage low-value duplicate tests." + (dolist (file '("prompt/test-after-change.v1.md" + "prompt/test-after-change-diagnostics.v1.md" + "prompt/tdd.v1.md" + "prompt/tdd-diagnostics.v1.md" + "prompt/tdd-with-refactoring.v1.md" + "prompt/tdd-with-refactoring-diagnostics.v1.md" + "snippets/ai-code-prompt-mode/create-tests" + "snippets/ai-code-prompt-mode/unit-tests")) + (with-temp-buffer + (insert-file-contents file) + (goto-char (point-min)) + (should (search-forward "small set of high-value" nil t)) + (should (search-forward "duplicate tests" nil t))))) + (ert-deftest ai-code-test-ai-code-el-does-not-autoload-private-diagnostics-constant () "Private diagnostics helper constants should not be marked for autoload." (with-temp-buffer