First commit

Author: wade3han

README.md

@@ -0,0 +1,98 @@
+# Meet Your Favorite Character: Open-domain Chatbot Mimicking Fictional Characters with only a Few Utterances (NAACL 2022)
+
+- [Paper link](https://arxiv.org/pdf/2204.10825.pdf)
+
+## Env Setup
+```
+conda create -n [env_name] python=3.8
+conda activate [env_name]
+
+conda install cudatoolkit=11.0 pytorch=1.7.1
+pip install git+https://github.com/huggingface/transformers
+pip install datasets pandas pyarrow sklearn
+pip install -r requirements.txt
+```
+
+## Run inference
+- We assume you run your language model and launch server on `http://${1}/generate`.
+- Below is an example for `Dynamic Match`.
+
+```
+python3 methods/inference_dynamic_match.py \
+  --model-file $retriever_model_path \
+  --megatron-endpoint http://${1}/generate \
+  --character-name $character_name \
+  --response-selection-strategy top1 \
+  --max-num-exemplars 8 \
+  --evaluate-set resources/dailydialog_test_utterances.tsv \
+  --all-styles-path resources/all_styles.tsv \
+  --save-results-path results \
+  --styles $character
+```
+
+## Run Character StyleProb Evaluation
+```
+srun --gres=gpu:1 eval_scripts/eval_character.sh [jsonl_input_file_path] [character_name] [classfier_model_path]
+```
+
+## Run Other Styles StyleProb Evaluation
+Styles including `positive`, `negative`, `Modern`, `Shakespearean`, `joy`, `anger`.
+```
+srun --gres=gpu:1 eval_scripts/eval_style.sh [jsonl_input_file_path] [expected_label] [classfier_model_path]
+```
+
+### Note: Example of Jsonl file
+```
+{"context": ["that's awesome! Do you spend a lot of time there?", "i do! it's a lot of fun but it can be tiring sometimes", "I can imagine. what kind of restaurant do they own?"], "response": "The restaurant the restaurant"}
+{"context": ["I got some great news today! My husband got a better paying job offer!", "Holy cow that's awesome!!!  What are you going to do with all that extra moneys??", "Not sure yet, but itll help us life more comforatbly! We move to his hometown in November when he gets out of Army!"], "response": "You must be so thrilled. There are so many lonely life out there. He must be thrilled."}
+...
+```
+
+## Run MaUdE
+- Clone [MaUdE Repo](https://github.com/facebookresearch/online_dialog_eval) and setup environment
+- Run following script
+
+```
+cat maude_inference.sh
+
+>>>
+
+#!/bin/zsh
+MODEL_SAVE_DIR=full_runs/
+DATA_NAME=convai2
+DATA_LOC=$1
+FINE_TUNE_MODEL=convai2_data/distilbert_lm
+TRAIN_MODE=nce
+
+VERSION=20488119
+MODEL_ID=na_all
+
+
+for DATA_LOC in "$@"
+do
+  python3 codes/inference.py \
+    --id $MODEL_ID \
+    --model_save_dir $MODEL_SAVE_DIR \
+    --model_version $VERSION \
+    --train_mode nce \
+    --corrupt_pre $DATA_LOC \
+    --test_suffix true_response \
+    --test_column response
+done
+```
+```
+srun --gres=gpu:1 maude_inference.sh [jsonl_path]
+```
+
+## Citation
+
+If you find our paper or this project helps your research, please kindly consider citing our paper in your publications.
+
+```
+@article{han2022meet,
+  title={Meet Your Favorite Character: Open-domain Chatbot Mimicking Fictional Characters with only a Few Utterances},
+  author={Han, Seungju and Kim, Beomsu and Yoo, Jin Yong and Seo, Seokjun and Kim, Sangbum and Erdenee, Enkhbayar and Chang, Buru},
+  journal={arXiv preprint arXiv:2204.10825},
+  year={2022}
+}
+```

eval_character.py

@@ -0,0 +1,78 @@
+import argparse
+import json
+
+import torch
+from scipy.special import softmax
+from transformers import AutoModelForSequenceClassification
+from transformers import AutoTokenizer
+
+CHARACTERS = [
+    "BMO",
+    "Rachel",
+    "Burke",
+    "Barney",
+    "Spock",
+    "Sheldon",
+    "Dwight",
+    "Michael",
+    "BartSimpson",
+    "MargeSimpson",
+]
+CHARACTER_TO_IDX = {c: i for i, c in enumerate(CHARACTERS)}
+
+
+def transform_input(texts, tokenizer):
+    result = tokenizer(texts, max_length=256, truncation=True, padding="max_length")
+    result = {k: torch.LongTensor(v) for k, v in result.items()}
+    # Optional when you run your model in GPU
+    result = {k: v.to("cuda:0") for k, v in result.items()}
+    return result
+
+
+def run_model(texts, model, tokenizer):
+    transformed = transform_input(texts, tokenizer)
+    with torch.no_grad():
+        logits = model(**transformed).logits
+        logits = logits.cpu().numpy()
+        probs = softmax(logits, axis=-1)
+    return probs
+
+
+def main(args):
+    try:
+        character_idx = CHARACTER_TO_IDX[args.character_name]
+    except KeyError:
+        raise ValueError(f"Unsupported character name: {args.character_name}")
+
+    with open(args.input_path) as f:
+        sentences = [json.loads(line.strip())["response"] for line in f]
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_dir)
+    model = AutoModelForSequenceClassification.from_pretrained(args.model_dir, from_tf=False)
+    # Optional when you run your model in GPU
+    model = model.to("cuda:0")
+
+    sum_probs = 0.
+    num_instances = 0.
+
+    for start_idx in range(0, len(sentences), args.batch_size):
+        end_idx = min(start_idx + args.batch_size, len(sentences))
+        batch = sentences[start_idx: end_idx]
+
+        model_preds = run_model(batch, model, tokenizer)
+        sum_probs += model_preds[:, character_idx].sum()
+        num_instances += len(model_preds)
+
+    avg_prob = sum_probs / num_instances
+    print(f"Avg prob for predicting as character {args.character_name}: {avg_prob:.8f}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", type=str)
+    parser.add_argument("--input-path", type=str)
+    parser.add_argument("--character-name", type=str, choices=CHARACTERS)
+    parser.add_argument("--batch-size", type=int, default=64)
+    args = parser.parse_args()
+
+    main(args)

eval_scripts/eval_character.sh

@@ -0,0 +1,10 @@
+#!/bin/zsh
+
+EVAL_INPUT_PATH=$1  # Path for input jsonl file containing pairs for evaluation
+CHARACTER_NAME=$2
+CLASSIFIER_MODEL_DIR=$3
+
+python3 eval_character.py \
+  --model-dir $CLASSIFIER_MODEL_DIR \
+  --input-path $EVAL_INPUT_PATH \
+  --character-name $CHARACTER_NAME

eval_scripts/eval_style.sh

@@ -0,0 +1,10 @@
+#!/bin/zsh
+
+EVAL_INPUT_PATH=$1  # Path for input jsonl file containing pairs for evaluation
+EXPECTED_STYLE_LABEL=$2
+CLASSIFIER_MODEL_DIR=$3
+
+python3 eval_style.py \
+  --model-dir $CLASSIFIER_MODEL_DIR \
+  --input-path $EVAL_INPUT_PATH\
+  --expected-label $EXPECTED_STYLE_LABEL

eval_style.py

@@ -0,0 +1,74 @@
+import argparse
+import json
+
+import numpy as np
+import torch
+from scipy.special import softmax
+from transformers import AutoModelForSequenceClassification
+from transformers import AutoTokenizer
+
+
+def transform_input(texts, tokenizer):
+    result = tokenizer(texts, max_length=256, truncation=True, padding="max_length")
+    result = {k: torch.LongTensor(v) for k, v in result.items()}
+    # Optional when you run your model in GPU
+    result = {k: v.to("cuda:0") for k, v in result.items()}
+    return result
+
+
+def run_model(texts, model, tokenizer):
+    transformed = transform_input(texts, tokenizer)
+    with torch.no_grad():
+        logits = model(**transformed).logits
+        logits = logits.cpu().numpy()
+        probs = softmax(logits, axis=-1)
+    preds = np.argmax(probs, axis=-1)
+    return preds
+
+
+def main(args):
+    LABEL_STR_TO_INT = {
+        "modern": 0,
+        "shakespearen": 1,
+        "negative": 0,
+        "positive": 1,
+        "anger": 0,
+        "joy": 1,
+    }
+    expected_label = LABEL_STR_TO_INT[args.expected_label]
+
+    with open(args.input_path) as f:
+        sentences = [json.loads(line.strip())["response"] for line in f]
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_dir)
+    model = AutoModelForSequenceClassification.from_pretrained(args.model_dir, from_tf=False)
+    # Optional when you run your model in GPU
+    model = model.to("cuda:0")
+
+    num_right = 0
+
+    for start_idx in range(0, len(sentences), args.batch_size):
+        end_idx = min(start_idx + args.batch_size, len(sentences))
+        batch = sentences[start_idx: end_idx]
+
+        model_preds = run_model(batch, model, tokenizer)
+        num_right += (model_preds == expected_label).sum()
+
+    accuracy = num_right * 100. / len(sentences)
+    print(f"Accuracy: {accuracy:.6f}%")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", type=str)
+    parser.add_argument("--input-path", type=str)
+    parser.add_argument("--expected-label", type=str, choices=["modern",
+                                                               "shakespearen",
+                                                               "negative",
+                                                               "positive",
+                                                               "anger",
+                                                               "joy"])
+    parser.add_argument("--batch-size", type=int, default=64)
+    args = parser.parse_args()
+
+    main(args)