Add analysis for gold path exists in problem statement

notebooks/contamination.ipynb

@@ -0,0 +1,188 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "definitions",
+   "metadata": {},
+   "source": [
+    "# Problem-statement localization hints\n",
+    "\n",
+    "For each bug-fix task, this notebook derives three signals from the problem statement and gold patch:\n",
+    "\n",
+    "- **Gold path:** a complete repository-relative path modified by the gold patch appears in the statement.\n",
+    "- **Gold object name:** the gold AL filename without its object-type suffix (for example, `SalesHeader.Table.al` → `SalesHeader`) appears after spaces and punctuation are ignored. This is an approximate indicator, not a contamination label.\n",
+    "- **Call stack:** the statement contains “call stack” or “stack trace”."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "build-dataset",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-08-31T14:31:48.673098Z",
+     "iopub.status.busy": "2026-08-31T14:31:48.672729Z",
+     "iopub.status.idle": "2026-08-31T14:31:49.916071Z",
+     "shell.execute_reply": "2026-08-31T14:31:49.914782Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded 101 bug-fix tasks\n"
+     ]
+    }
+   ],
+   "source": [
+    "import re\n",
+    "from pathlib import PurePosixPath\n",
+    "\n",
+    "import pandas as pd\n",
+    "\n",
+    "from bcbench.collection.patch_utils import extract_file_paths_from_patch\n",
+    "from bcbench.config import get_config\n",
+    "from bcbench.dataset import BugFixEntry\n",
+    "\n",
+    "_config = get_config()\n",
+    "_object_type_suffix = re.compile(r\"\\.(table|page|codeunit|report|xmlport|query|enum|interface|permissionset|profile|controladdin|dotnet)\\.al$\", re.IGNORECASE)\n",
+    "\n",
+    "\n",
+    "def contains_gold_path(statement: str, gold_paths: list[str]) -> bool:\n",
+    "    normalized_statement = statement.replace(\"\\\\\", \"/\").casefold()\n",
+    "    return any(path.replace(\"\\\\\", \"/\").casefold() in normalized_statement for path in gold_paths)\n",
+    "\n",
+    "\n",
+    "def mentions_gold_object_name(statement: str, gold_paths: list[str]) -> bool:\n",
+    "    normalized_statement = re.sub(r\"[^a-z0-9]\", \"\", statement.casefold())\n",
+    "    object_names = [re.sub(r\"[^a-z0-9]\", \"\", _object_type_suffix.sub(\"\", PurePosixPath(path).name).casefold()) for path in gold_paths]\n",
+    "    return any(len(name) >= 8 and name in normalized_statement for name in object_names)\n",
+    "\n",
+    "\n",
+    "entries: list[BugFixEntry] = BugFixEntry.load(_config.paths.dataset_dir / \"bcbench.jsonl\")\n",
+    "dataset_df = pd.DataFrame(\n",
+    "    [\n",
+    "        {\n",
+    "            \"instance_id\": entry.instance_id,\n",
+    "            \"contains_gold_path\": contains_gold_path(entry.get_task(), extract_file_paths_from_patch(entry.patch)),\n",
+    "            \"mentions_gold_object_name\": mentions_gold_object_name(entry.get_task(), extract_file_paths_from_patch(entry.patch)),\n",
+    "            \"contains_call_stack\": bool(re.search(r\"call\\s*stack|stack\\s*trace\", entry.get_task(), re.IGNORECASE)),\n",
+    "        }\n",
+    "        for entry in entries\n",
+    "    ]\n",
+    ")\n",
+    "print(f\"Loaded {len(dataset_df)} bug-fix tasks\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "signal-counts",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-08-31T14:31:49.918888Z",
+     "iopub.status.busy": "2026-08-31T14:31:49.918412Z",
+     "iopub.status.idle": "2026-08-31T14:31:49.934958Z",
+     "shell.execute_reply": "2026-08-31T14:31:49.933787Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "            Signal  Tasks  Percentage\n",
+      "Complete gold path      1         1.0\n",
+      "  Gold object name     28        27.7\n",
+      "        Call stack      7         6.9\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>instance_id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>microsoft__BCApps-4699</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "               instance_id\n",
+       "34  microsoft__BCApps-4699"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "signals = {\n",
+    "    \"contains_gold_path\": \"Complete gold path\",\n",
+    "    \"mentions_gold_object_name\": \"Gold object name\",\n",
+    "    \"contains_call_stack\": \"Call stack\",\n",
+    "}\n",
+    "signal_stats = pd.DataFrame([{\"Signal\": label, \"Tasks\": int(dataset_df[column].sum()), \"Percentage\": round(dataset_df[column].mean() * 100, 1)} for column, label in signals.items()])\n",
+    "print(signal_stats.to_string(index=False))\n",
+    "dataset_df.loc[dataset_df[\"contains_gold_path\"], [\"instance_id\"]]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "findings",
+   "metadata": {},
+   "source": [
+    "## Findings\n",
+    "\n",
+    "- Only 1 of 101 tasks (1.0%), `microsoft__BCApps-4699`, contains a complete gold path, so direct path disclosure cannot explain a broad benchmark-level contamination signal.\n",
+    "- Gold object names occur in 28 tasks (27.7%) and call stacks in 7 (6.9%), so genuine deduction from the statement is a meaningful confounder for context-free file localization."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bcbench",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}