bert_lime_faithfulness.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating Faithfulness on our model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "import torch.nn.functional as F\n",
    "from lime.lime_text import LimeTextExplainer\n",
    "import torch.nn.functional as F\n",
    "import os \n",
    "import numpy as np\n",
    "\n",
    "torch.cuda.empty_cache()\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# device = \"cpu\"\n",
    "data_dir = \"output/\"\n",
    "destination_dir = \"./\"\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "test_short_path = \"data/test_10_top50_short.csv\"\n",
    "labels_10_top50 = pd.read_csv('data/icd10_codes_top50.csv')\n",
    "code_labels_10 = pd.read_csv(\"data/icd10_codes.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Parameters\n",
    "MAX_POSITION_EMBEDDINGS = 512\n",
    "MODEL = \"emilyalsentzer/Bio_ClinicalBERT\"\n",
    "CKPT = os.path.join(data_dir,\"best_model_state.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classes\n",
      "bert model and tokenizer initialized\n"
     ]
    }
   ],
   "source": [
    "# Create class dictionaries\n",
    "classes = [class_ for class_ in code_labels_10[\"icd_code\"] if class_]\n",
    "class2id = {class_: id for id, class_ in enumerate(classes)}\n",
    "id2class = {id: class_ for class_, id in class2id.items()}\n",
    "\n",
    "print(\"classes\")\n",
    "\n",
    "config, unused_kwargs = AutoConfig.from_pretrained(\n",
    "    MODEL,\n",
    "    num_labels=len(classes),\n",
    "    id2label=id2class,\n",
    "    label2id=class2id,\n",
    "    problem_type=\"multi_label_classification\",\n",
    "    return_unused_kwargs=True,\n",
    ")\n",
    "\n",
    "tokenizer_bert = AutoTokenizer.from_pretrained(MODEL)\n",
    "model_bert = AutoModel.from_pretrained(MODEL, config=config, cache_dir='./model_ckpt/')\n",
    "print(\"bert model and tokenizer initialized\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset loaded\n"
     ]
    }
   ],
   "source": [
    "class TokenizerWrapper:\n",
    "    def __init__(self, tokenizer, length, classes):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_length = length\n",
    "        self.classes = classes\n",
    "        self.class2id = {class_: id for id, class_ in enumerate(self.classes)}\n",
    "        self.id2class = {id: class_ for class_, id in self.class2id.items()}\n",
    "        \n",
    "    def multi_labels_to_ids(self, labels: list[str]) -> list[float]:\n",
    "        ids = [0.0] * len(self.class2id)  # BCELoss requires float as target type\n",
    "        for label in labels:\n",
    "            ids[self.class2id[label]] = 1.0\n",
    "        return ids\n",
    "    \n",
    "    def tokenize_function(self, example):\n",
    "        result = self.tokenizer(\n",
    "            example[\"text\"],\n",
    "            max_length = self.max_length,\n",
    "            padding = 'max_length',\n",
    "            truncation = True,\n",
    "            return_tensors='pt'\n",
    "        ).to(device)\n",
    "        result[\"label\"] = torch.tensor([self.multi_labels_to_ids(eval(label)) for label in example[\"label\"]])\n",
    "        return result\n",
    "        \n",
    "data_files = {\n",
    "        \"test\": test_short_path,\n",
    "    }\n",
    "\n",
    "tokenizer_wrapper = TokenizerWrapper(tokenizer_bert, MAX_POSITION_EMBEDDINGS, classes)\n",
    "dataset = load_dataset(\"csv\", data_files=data_files)\n",
    "dataset = dataset.map(tokenizer_wrapper.tokenize_function, batched=True, num_proc=1)\n",
    "dataset = dataset.with_format(\"torch\")\n",
    "print(\"dataset loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.config = config\n",
    "        self.device = device\n",
    "        self.bert_model = model_bert\n",
    "        self.can_generate = model_bert.can_generate\n",
    "        self.base_model_prefix = model_bert.base_model_prefix\n",
    "        self.get_input_embeddings = model_bert.get_input_embeddings\n",
    "        self.dropout = torch.nn.Dropout(0.2)\n",
    "        self.linear = torch.nn.Linear(self.bert_model.config.hidden_size, 50)\n",
    "    \n",
    "    def forward(self, input_ids, attn_mask, token_type_ids):\n",
    "        output = self.bert_model(\n",
    "            input_ids, \n",
    "            attention_mask=attn_mask, \n",
    "            token_type_ids=token_type_ids\n",
    "        )\n",
    "        output_dropout = self.dropout(output.pooler_output)\n",
    "        output = self.linear(output_dropout)\n",
    "        return output\n",
    "    \n",
    "model_bert = BERTClass()\n",
    "model_bert.load_state_dict(torch.load(CKPT))\n",
    "model_bert = model_bert.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = LimeTextExplainer(class_names=classes, bow=False)\n",
    "\n",
    "def predictor_bert(texts):\n",
    "    tk = tokenizer_bert(\n",
    "            texts,\n",
    "            max_length = MAX_POSITION_EMBEDDINGS,\n",
    "            padding = 'max_length',\n",
    "            truncation = True,\n",
    "            return_tensors='pt'\n",
    "        ).to(device)\n",
    "    ids = tk['input_ids'].to(device, dtype = torch.long)\n",
    "    mask = tk['attention_mask'].to(device, dtype = torch.long)\n",
    "    token_type_ids = tk['token_type_ids'].to(device, dtype = torch.long)\n",
    "    outputs = model_bert(ids, mask, token_type_ids)\n",
    "    probas = F.sigmoid(outputs).detach().cpu().numpy()\n",
    "    del tk, outputs\n",
    "    return probas\n",
    "\n",
    "\n",
    "def predictor_model(texts, model, tokenizer):\n",
    "    tk = tokenizer(\n",
    "            texts,\n",
    "            max_length = MAX_POSITION_EMBEDDINGS,\n",
    "            padding = 'max_length',\n",
    "            truncation = True,\n",
    "            return_tensors='pt'\n",
    "        ).to(device)\n",
    "    ids = tk['input_ids'].to(device, dtype = torch.long)\n",
    "    mask = tk['attention_mask'].to(device, dtype = torch.long)\n",
    "    token_type_ids = tk['token_type_ids'].to(device, dtype = torch.long)\n",
    "    outputs = model(ids, mask, token_type_ids)\n",
    "    probas = F.sigmoid(outputs).detach().cpu().numpy()\n",
    "    del tk, outputs\n",
    "    return probas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The instances are formatted as a list of strings, where each string is one word used by lime. The rationales mask is a list of indices, where the first list refers to the index of the sample the label corresponds to and the second list is the index of string used in that rationale."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test code for faithfulness calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faithfulness_lime as faithfulness\n",
    "# this reimports the library for easy testing in the notebook\n",
    "import importlib\n",
    "import numpy as np\n",
    "importlib.reload(faithfulness)\n",
    "\n",
    "explainer = LimeTextExplainer(class_names=classes, bow=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of rationale removed list: 10\n",
      "length of others removed list: 10\n",
      "Calculating Sufficiency\n",
      "Calculating Comprehensiveness\n",
      "\n",
      "-- Metrics -------------------------------------------------------------\n",
      "\n",
      "\n",
      "Faithfulness:  0.37830418\n",
      "Comprehensiveness:  0.4156465\n",
      "Sufficency:  0.9101585\n",
      "\n",
      "Sufficiency list: [1.1997775  0.43490794 1.3844305  0.6592898  0.3310731  0.7969022\n",
      " 1.1339489  1.4446026  0.52336097 1.1932914 ]\n",
      "Comprehensiveness list: [0.12781498 0.15973037 0.47970444 0.09612808 0.17070179 0.5854102\n",
      " 0.6944684  0.4119725  0.7578804  0.6726538 ]\n",
      "\n",
      "Comprehensiveness Median:  0.44583845\n",
      "Comprehensiveness q1 (25% percentile):  0.16247322782874107\n",
      "Comprehensiveness q3 (75% percentile):  0.6508428901433945\n",
      "\n",
      "\n",
      "Sufficency Median:  0.96542555\n",
      "Sufficency q1 (25% percentile):  0.5573431700468063\n",
      "Sufficency q3 (75% percentile):  1.1981559693813324\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Fix N to the first 10 examples and pick k = 5\n",
    "samples_start = 0\n",
    "samples_end = 10\n",
    "input_data = dataset[\"test\"][\"text\"][samples_start:samples_end]\n",
    "k = 5\n",
    "\n",
    "indexed_text, index_array_rationalle = faithfulness.lime_create_index_arrays(input_data, predictor_bert, explainer, k_labels = k)\n",
    "rationale_removed = faithfulness.remove_rationale_words(indexed_text, index_array_rationalle)\n",
    "others_removed = faithfulness.remove_other_words(indexed_text, index_array_rationalle)\n",
    "\n",
    "# We expect a list of rationales and others to be removed that correspond to the number of texts in input_data\n",
    "print('length of rationale removed list:', len(rationale_removed))\n",
    "print('length of others removed list:', len(others_removed))\n",
    "_, faith_5 = faithfulness.calculate_faithfulness(input_data, rationale_removed, others_removed, model_bert, tokenizer_bert, predictor_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of rationale removed list: 10\n",
      "length of others removed list: 10\n",
      "Calculating Sufficiency\n",
      "Calculating Comprehensiveness\n",
      "\n",
      "-- Metrics -------------------------------------------------------------\n",
      "\n",
      "\n",
      "Faithfulness:  0.21069297\n",
      "Comprehensiveness:  0.20763528\n",
      "Sufficency:  1.0147263\n",
      "\n",
      "Sufficiency list: [0.66150486 0.50363344 1.3346727  1.0085003  0.56927866 0.8266784\n",
      " 1.367616   1.479163   1.1131557  1.2830596 ]\n",
      "Comprehensiveness list: [0.10953143 0.12813778 0.14402504 0.16545348 0.06882977 0.31301817\n",
      " 0.31437403 0.362963   0.23182683 0.2381933 ]\n",
      "\n",
      "Comprehensiveness Median:  0.19864015\n",
      "Comprehensiveness q1 (25% percentile):  0.132109597325325\n",
      "Comprehensiveness q3 (75% percentile):  0.2943119555711746\n",
      "\n",
      "\n",
      "Sufficency Median:  1.060828\n",
      "Sufficency q1 (25% percentile):  0.7027982473373413\n",
      "Sufficency q3 (75% percentile):  1.3217694163322449\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Fix N to the first 10 examples and pick k = 10\n",
    "samples_start = 0\n",
    "samples_end = 10\n",
    "input_data = dataset[\"test\"][\"text\"][samples_start:samples_end]\n",
    "k = 10\n",
    "\n",
    "indexed_text, index_array_rationalle = faithfulness.lime_create_index_arrays(input_data, predictor_bert, explainer, k_labels = k)\n",
    "rationale_removed = faithfulness.remove_rationale_words(indexed_text, index_array_rationalle)\n",
    "others_removed = faithfulness.remove_other_words(indexed_text, index_array_rationalle)\n",
    "\n",
    "# We expect a list of rationales and others to be removed that correspond to the number of texts in input_data\n",
    "print('length of rationale removed list:', len(rationale_removed))\n",
    "print('length of others removed list:', len(others_removed))\n",
    "_, faith_10 = faithfulness.calculate_faithfulness(input_data, rationale_removed, others_removed, model_bert, tokenizer_bert, predictor_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of rationale removed list: 10\n",
      "length of others removed list: 10\n",
      "Calculating Sufficiency\n",
      "Calculating Comprehensiveness\n",
      "\n",
      "-- Metrics -------------------------------------------------------------\n",
      "\n",
      "\n",
      "Faithfulness:  0.19094196\n",
      "Comprehensiveness:  0.19981489\n",
      "Sufficency:  0.95559424\n",
      "\n",
      "Sufficiency list: [0.5098805  0.4577699  1.1453369  0.73992354 0.5434567  0.774903\n",
      " 1.2848253  1.570421   1.1037817  1.4256439 ]\n",
      "Comprehensiveness list: [0.10799496 0.1121013  0.22378412 0.09614211 0.10092095 0.25278768\n",
      " 0.24575733 0.28585857 0.2403104  0.3324914 ]\n",
      "\n",
      "Comprehensiveness Median:  0.23204726\n",
      "Comprehensiveness q1 (25% percentile):  0.10902154445648193\n",
      "Comprehensiveness q3 (75% percentile):  0.2510300911962986\n",
      "\n",
      "\n",
      "Sufficency Median:  0.9393424\n",
      "Sufficency q1 (25% percentile):  0.5925733894109726\n",
      "Sufficency q3 (75% percentile):  1.2499532103538513\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Fix N to the first 10 examples and pick k = 15\n",
    "samples_start = 0\n",
    "samples_end = 10\n",
    "input_data = dataset[\"test\"][\"text\"][samples_start:samples_end]\n",
    "k = 15\n",
    "\n",
    "indexed_text, index_array_rationalle = faithfulness.lime_create_index_arrays(input_data, predictor_bert, explainer, k_labels = k)\n",
    "rationale_removed = faithfulness.remove_rationale_words(indexed_text, index_array_rationalle)\n",
    "others_removed = faithfulness.remove_other_words(indexed_text, index_array_rationalle)\n",
    "\n",
    "# We expect a list of rationales and others to be removed that correspond to the number of texts in input_data\n",
    "print('length of rationale removed list:', len(rationale_removed))\n",
    "print('length of others removed list:', len(others_removed))\n",
    "_, faith_15 = faithfulness.calculate_faithfulness(input_data, rationale_removed, others_removed, model_bert, tokenizer_bert, predictor_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For k = 5:\n",
      "avg faith: [0.37830418]\n",
      "For k = 10:\n",
      "avg faith: [0.21069297]\n",
      "For k = 15:\n",
      "avg faith: [0.19094196]\n"
     ]
    }
   ],
   "source": [
    "print('For k = 5:')\n",
    "print('avg faith:', faith_5)\n",
    "print('For k = 10:')\n",
    "print('avg faith:', faith_10)\n",
    "print('For k = 15:')\n",
    "print('avg faith:', faith_15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2599797"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "faiths = faith_5 + faith_10 + faith_15\n",
    "np.mean(faiths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "##################################################################################################################################\n",
    "# Returns the faithfulness results for choice of:\n",
    "# - input_data being the same dimension/characteristics as the test/val/train dataset used for our classifier of choice Bert or OPT.\n",
    "# - start_index being the starting point index of the input_data\n",
    "# - N being the size of input dataset (this is how many texts you want an explanation for)\n",
    "# - B being the size of explanation batch (this is how many texts your machine can explain at a given instance)\n",
    "# - k being the top k features defined as our rationales for explanation\n",
    "# Precondition: pipeline with tokenizer and model correctly initialized along with explainer for LIME\n",
    "######################################################################################################################################\n",
    "def get_faith_lime(input_data, start_index, N, B, k):\n",
    "    num_steps = math.ceil(N/B)\n",
    "    tail_n =  N % B    \n",
    "    overall_ind = []\n",
    "    overall_faith = []\n",
    "\n",
    "    i = start_index\n",
    "    while i < N + tail_n:\n",
    "        if i >= (N-tail_n) and tail_n > 0:\n",
    "            input_subset = input_data[i: i+tail_n]\n",
    "        else:    \n",
    "            input_subset = input_data[i: i+B]\n",
    "\n",
    "\n",
    "        indexed_text, index_array_rationalle = faithfulness.lime_create_index_arrays(input_subset, predictor_bert, explainer, k_labels = k)   \n",
    "        rationale_removed = faithfulness.remove_rationale_words(indexed_text, index_array_rationalle)\n",
    "        others_removed = faithfulness.remove_other_words(indexed_text, index_array_rationalle)\n",
    "        \n",
    "        ind, faith = faithfulness.calculate_faithfulness(input_data, rationale_removed, others_removed, model_bert, tokenizer_bert, predictor_model)\n",
    "        overall_ind.append(ind)\n",
    "        overall_faith.extend(faith)\n",
    "\n",
    "        i += B\n",
    "        \n",
    "    return overall_ind, overall_faith, np.mean(overall_faith)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_21196\\310016248.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moverall_faith_5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mavg_faith_5\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_faith_lime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstart_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mB\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moverall_faith_10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mavg_faith_10\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_faith_lime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstart_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mB\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_21196\\1264753054.py\u001b[0m in \u001b[0;36mget_faith_lime\u001b[1;34m(input_data, start_index, N, B, k)\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[0mothers_removed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfaithfulness\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mremove_other_words\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexed_text\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex_array_rationalle\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 29\u001b[1;33m         \u001b[0mind\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfaith\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfaithfulness\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcalculate_faithfulness\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrationale_removed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mothers_removed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel_bert\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtokenizer_bert\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredictor_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     30\u001b[0m         \u001b[0moverall_ind\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mind\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m         \u001b[0moverall_faith\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfaith\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\user\\Documents\\CSC413\\CSC413-Project\\faithfulness_lime.py\u001b[0m in \u001b[0;36mcalculate_faithfulness\u001b[1;34m(instances, instances_rationalle_removed, instances_other_removed, model, tokenizer, predictor_func)\u001b[0m\n\u001b[0;32m    277\u001b[0m         \u001b[1;31m# print(len(instances_batch))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    278\u001b[0m         \u001b[1;31m# print(instances_batch)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 279\u001b[1;33m         \u001b[0moutput_batch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredictor_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minstances_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    280\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    281\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_21196\\3307639056.py\u001b[0m in \u001b[0;36mpredictor_model\u001b[1;34m(texts, model, tokenizer)\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mpredictor_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m     tk = tokenizer(\n\u001b[0m\u001b[0;32m     22\u001b[0m             \u001b[0mtexts\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m             \u001b[0mmax_length\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMAX_POSITION_EMBEDDINGS\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   2870\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_target_context_manager\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2871\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_switch_to_input_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2872\u001b[1;33m             \u001b[0mencodings\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_one\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtext_pair\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mall_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2873\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtext_target\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2874\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_switch_to_target_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36m_call_one\u001b[1;34m(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   2956\u001b[0m                 \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2957\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2958\u001b[1;33m             return self.batch_encode_plus(\n\u001b[0m\u001b[0;32m   2959\u001b[0m                 \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2960\u001b[0m                 \u001b[0madd_special_tokens\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36mbatch_encode_plus\u001b[1;34m(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   3147\u001b[0m         \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3148\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3149\u001b[1;33m         return self._batch_encode_plus(\n\u001b[0m\u001b[0;32m   3150\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3151\u001b[0m             \u001b[0madd_special_tokens\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\transformers\\tokenization_utils_fast.py\u001b[0m in \u001b[0;36m_batch_encode_plus\u001b[1;34m(self, batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose)\u001b[0m\n\u001b[0;32m    502\u001b[0m         \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    503\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 504\u001b[1;33m         encodings = self._tokenizer.encode_batch(\n\u001b[0m\u001b[0;32m    505\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    506\u001b[0m             \u001b[0madd_special_tokens\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#  For our evaluation, we fix for 10 examples and choose k = 5, 10, 15.\n",
    "start_index = 0\n",
    "N = 1\n",
    "B = 1\n",
    "explainer = LimeTextExplainer(class_names=classes, bow=False)\n",
    "input_data = dataset['test']['text']\n",
    "\n",
    "k = 5\n",
    "_, overall_faith_5, avg_faith_5 = get_faith_lime(input_data, start_index, N, B, k)\n",
    "k = 10\n",
    "_, overall_faith_10, avg_faith_10 = get_faith_lime(input_data, start_index, N, B, k)\n",
    "k = 15\n",
    "_, overall_faith_15, avg_faith_15 = get_faith_lime(input_data, start_index, N, B, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('For k = 5:')\n",
    "print('overall_faith:', overall_faith_5)\n",
    "print('avg faith:', avg_faith_5)\n",
    "print('For k = 10:')\n",
    "print('overall_faith:', overall_faith_10)\n",
    "print('avg faith:', avg_faith_10)\n",
    "print('For k = 15:')\n",
    "print('overall_faith:', overall_faith_15)\n",
    "print('avg faith:', avg_faith_15)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}