From 1a9cce57e302b78723d98bc3af8c27cf93039a0e Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Thu, 20 Feb 2025 15:03:32 -0800 Subject: [PATCH 1/3] update chat and system template match for llama --- examples/python/model-chat.py | 13 ++++++------- examples/python/model-qa.py | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/python/model-chat.py b/examples/python/model-chat.py index 6068d7524..b3866bc8e 100644 --- a/examples/python/model-chat.py +++ b/examples/python/model-chat.py @@ -50,10 +50,8 @@ def main(args): args.chat_template = '<|user|>\n{input} <|end|>\n<|assistant|>' elif model_type.startswith("phi4"): args.chat_template = '<|im_start|>user<|im_sep|>\n{input}<|im_end|>\n<|im_start|>assistant<|im_sep|>' - elif model_type.startswith("llama3"): + elif model_type.startswith("llama"): args.chat_template = '<|start_header_id|>user<|end_header_id|>\n{input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>' - elif model_type.startswith("llama2"): - args.chat_template = '{input}' elif model_type.startswith("qwen2"): args.chat_template = '<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n' else: @@ -77,10 +75,8 @@ def main(args): system_prompt = f"<|system|>\n{args.system_prompt}<|end|>" elif model_type.startswith('phi4'): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" - elif model_type.startswith("llama3"): + elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" - elif model_type.startswith("llama2"): - system_prompt = f"[INST] <>\n{args.system_prompt}\n<>" elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: @@ -92,11 +88,14 @@ def main(args): # Keep asking for input prompts in a loop while True: - text = input("Input: ") + text = input("Prompt (Use quit() to exit): ") if not text: print("Error, input cannot be empty") continue + if text == "quit()": + break + if args.timings: started_timestamp = time.time() prompt = f'{args.chat_template.format(input=text)}' diff --git a/examples/python/model-qa.py b/examples/python/model-qa.py index 47d45b6df..958014540 100644 --- a/examples/python/model-qa.py +++ b/examples/python/model-qa.py @@ -47,10 +47,8 @@ def main(args): args.chat_template = '<|user|>\n{input} <|end|>\n<|assistant|>' elif model_type.startswith("phi4"): args.chat_template = '<|im_start|>user<|im_sep|>\n{input}<|im_end|>\n<|im_start|>assistant<|im_sep|>' - elif model_type.startswith("llama3"): + elif model_type.startswith("llama"): args.chat_template = '<|start_header_id|>user<|end_header_id|>\n{input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>' - elif model_type.startswith("llama2"): - args.chat_template = '{input}' elif model_type.startswith("qwen2"): args.chat_template = '<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n' else: @@ -65,10 +63,8 @@ def main(args): system_prompt = f"<|system|>\n{args.system_prompt}<|end|>" elif model_type.startswith('phi4'): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" - elif model_type.startswith("llama3"): + elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" - elif model_type.startswith("llama2"): - system_prompt = f"[INST] <>\n{args.system_prompt}\n<>" elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: @@ -79,11 +75,14 @@ def main(args): # Keep asking for input prompts in a loop while True: - text = input("Input: ") + text = input("Prompt (Use quit() to exit): ") if not text: print("Error, input cannot be empty") continue + if text == "quit()": + break + if args.timings: started_timestamp = time.time() prompt = f'{args.chat_template.format(input=text)}' From 2cd3f75bca4b5b1a069c0072630b9ad499fda05b Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Fri, 21 Feb 2025 15:32:11 -0800 Subject: [PATCH 2/3] add LLAMA 2 templates --- examples/python/model-chat.py | 2 ++ examples/python/model-qa.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/python/model-chat.py b/examples/python/model-chat.py index b3866bc8e..421d578a7 100644 --- a/examples/python/model-chat.py +++ b/examples/python/model-chat.py @@ -52,6 +52,7 @@ def main(args): args.chat_template = '<|im_start|>user<|im_sep|>\n{input}<|im_end|>\n<|im_start|>assistant<|im_sep|>' elif model_type.startswith("llama"): args.chat_template = '<|start_header_id|>user<|end_header_id|>\n{input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>' + print("Using Chat Template for LLAMA 3, if you are using LLAMA 2 please pass the argument --chat_template '{input} [/INST]')") elif model_type.startswith("qwen2"): args.chat_template = '<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n' else: @@ -77,6 +78,7 @@ def main(args): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" + print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\n{args.system_prompt}\n<>')") elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: diff --git a/examples/python/model-qa.py b/examples/python/model-qa.py index 958014540..0eeb2dcf4 100644 --- a/examples/python/model-qa.py +++ b/examples/python/model-qa.py @@ -49,6 +49,7 @@ def main(args): args.chat_template = '<|im_start|>user<|im_sep|>\n{input}<|im_end|>\n<|im_start|>assistant<|im_sep|>' elif model_type.startswith("llama"): args.chat_template = '<|start_header_id|>user<|end_header_id|>\n{input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>' + print("Using Chat Template for LLAMA 3, if you are using LLAMA 2 please pass the argument --chat_template '{input} [/INST]')") elif model_type.startswith("qwen2"): args.chat_template = '<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n' else: @@ -65,6 +66,7 @@ def main(args): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" + print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\n{args.system_prompt}\n<>')") elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: From 4722d11ca68e6030c82703d08fa75b630b91a08a Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Fri, 21 Feb 2025 15:37:36 -0800 Subject: [PATCH 3/3] minor fix --- examples/python/model-chat.py | 2 +- examples/python/model-qa.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python/model-chat.py b/examples/python/model-chat.py index 421d578a7..f95667a18 100644 --- a/examples/python/model-chat.py +++ b/examples/python/model-chat.py @@ -78,7 +78,7 @@ def main(args): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" - print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\n{args.system_prompt}\n<>')") + print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\\n{args.system_prompt}\\n<>')") elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: diff --git a/examples/python/model-qa.py b/examples/python/model-qa.py index 0eeb2dcf4..f2adf9387 100644 --- a/examples/python/model-qa.py +++ b/examples/python/model-qa.py @@ -66,7 +66,7 @@ def main(args): system_prompt = f"<|im_start|>system<|im_sep|>\n{args.system_prompt}<|im_end|>" elif model_type.startswith("llama"): system_prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n{args.system_prompt}<|eot_id|>" - print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\n{args.system_prompt}\n<>')") + print("Using System Prompt for LLAMA 3, if you are using LLAMA 2 please pass the argument --system_prompt '[INST] <>\\n{args.system_prompt}\\n<>')") elif model_type.startswith("qwen2"): system_prompt = f"<|im_start|>system\n{args.system_prompt}<|im_end|>\n" else: