Finishing generation + running_eval

Author: ruggsea

agent_factory.py

@@ -2,6 +2,25 @@
 from llm_engines import LLMApi
 import json 
 import random
+import itertools
+
+# set up logging
+import logging
+
+# Create a custom logger
+angent_factory_logger = logging.getLogger('agent_factory')
+angent_factory_logger.setLevel(logging.INFO)
+
+# Create handlers
+agent_factory_file_handler = logging.FileHandler('agent_factory.log', mode='w')
+agent_factory_file_handler.setLevel(logging.INFO)
+
+# Create formatters and add it to handlers
+formatter = logging.Formatter('%(asctime)-15s %(message)s')
+agent_factory_file_handler.setFormatter(formatter)
+
+# Add handlers to the logger
+angent_factory_logger.addHandler(agent_factory_file_handler)
 
 def read_personas():
     list_personas=[]
@@ -36,11 +55,22 @@ def gen_name(persona, neutral_llm=LLMApi()):
     name=""
     while len(name)<2 or len(name)>20:
         name=neutral_llm.generate_response(f"Generate a name for me appropriate for a persona with the following description: {persona}.\nGenerate only a valid first name and nothing else. If your answer contains anything more than a first name, you will be terminated. Follow the name with ## to indicate the end of the name.")
+        # if something like "Sure i can" is in the name, start again
+        if "sure i can" in name.lower() and "can" in name.lower():
+            agent_factory_log.info(f"Invalid name: {name}")
+            name=""
+            continue
         if "##" in name:
             name=name.split("##")[0]
             name=name.strip()
-            print(f"Generated name: {name}")
+            if len(name)<2 or len(name)>20:
+                agent_factory_logger.info(f"Invalid name: {name}")
+                name=""
+                continue
             break      
+        else:
+            agent_factory_log.info(f"Invalid name: {name}")
+            name=""
     return name
 
 def get_persona_by_topics(topics:list, domain=None):
@@ -140,4 +170,18 @@ def create_groupchat(topics_to_include, n_agents=2, agent_type=DialogueReactAgen
 
     ## check the topics of the agents
     print(f"Group chat topics: {gc_covered_topics}")
+    # check that agents have different personas
+    for agents_comb in itertools.combinations(agents, 2):
+        while agents_comb[0].persona==agents_comb[1].persona:
+            print("Agents have the same persona")
+            # substitute the persona of the second agent
+            agent=get_agent_by_topics(topics, agent_type=agent_type, neutral_llm=neutral_llm, **agent_args)
+        # same for names
+        while agents_comb[0].name==agents_comb[1].name:
+            print("Agents have the same name")
+            agent.name=gen_name(agent.persona, neutral_llm=neutral_llm)
+        # substitute the persona instead of $name$ in the persona desc
+        agent.persona=agent.persona.replace("$name$", agent.name)
+            
+        
     return agents

dialogue_react_agent.py

@@ -47,9 +47,23 @@ def read_dialogue_act_list(file_path:str):
 dialogue_acts_list=read_dialogue_act_list("prompts/dialogue_acts.json")
 
 
-logging.basicConfig(level=logging.INFO, filename="chat.log", filemode="w", format="%(asctime)-15s %(message)s")
+# Create a custom logger
+internal_agent_logger = logging.getLogger('agent_internal')
+internal_agent_logger.setLevel(logging.INFO)
 
+# Create handlers
+internal_agent_file_handler = logging.FileHandler('agent_internal.log', mode='w')
+internal_agent_file_handler.setLevel(logging.INFO)
 
+# Create formatters and add it to handlers
+formatter = logging.Formatter('%(asctime)-15s %(message)s')
+internal_agent_file_handler.setFormatter(formatter)
+
+# Add handlers to the internal_agent_logge
+internal_agent_logger.addHandler(internal_agent_file_handler)
+
+# prevent logging from propagating to the root logger
+internal_agent_logger.propagate = False
 
 
 class DialogueReactAgent(ReflectingAgent):
@@ -134,7 +148,7 @@ def gen_memories(self, last_messages, n_memories, **kwargs):
 
 
         # log compiled memory prompt
-        logging.info(f"Memory prompt: {memory_prompt}")
+        internal_agent_logger.info(f"Memory prompt: {memory_prompt}")
 
         ## generate until you get the desired number of memories
         memories = []
@@ -202,7 +216,7 @@ def gen_reflections(self, n_memories, n_reflections, **kwargs):
                                                        agent_list=agent_list)
 
         # log compiled reflection prompt
-        logging.info(f"Reflection prompt: {reflection_prompt}")
+        internal_agent_logger.info(f"Reflection prompt: {reflection_prompt}")
 
         ## generate until you get the desired number of reflections
         reflections = []
@@ -292,7 +306,7 @@ def get_answer(self, last_messages, extra_context="", **kwargs):
             )
 
         # log rendered prompt
-        logging.info(f"Answer generation prompt: {prompt}")
+        internal_agent_logger.info(f"Answer generation prompt: {prompt}")
 
         # get the answer for the dialogue react method
         if self.ablation == None:
@@ -312,20 +326,24 @@ def get_answer(self, last_messages, extra_context="", **kwargs):
                         ## check that thought is a dialogue act
                         observation = observation[0].strip()
                         thought = thought[0].strip()
+                        # remove everything after the first > in the thought
+                        thought = thought.split(">")[0]
+                        # readd the > at the end
+                        thought = thought + ">"     
                         action = action[0].strip()
                         if thought not in self.dialogue_acts:
-                            logging.info(f"Invalid dialogue act: {thought}.")
+                            internal_agent_logger.info(f"Invalid dialogue act: {thought}.")
                             continue
                         answer = action
                         #print(f"Valid answer: {answer_candidate}.")
-                        logging.info(f"Valid dialogue_act: {thought}.")
-                        logging.info(f"Valid answer: {answer}.")
+                        internal_agent_logger.info(f"Valid dialogue_act: {thought}.")
+                        internal_agent_logger.info(f"Valid answer: {answer}.")
                         break
                     else:
-                        logging.info(f"Invalid answer: {answer_candidate}.")
+                        internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
                 except Exception as e:
-                    logging.info(f"Error generating response: {e}.")
-                    logging.info(f"Invalid answer: {answer_candidate}.")
+                    internal_agent_logger.info(f"Error generating response: {e}.")
+                    internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
 
             answer_with_dialogueAct = f"Following the observation: {observation}, I wanted to commit the following dialogue act: {thought}. Therefore, I wrote the message: {action}."                
             ## save the answer in memory
@@ -350,13 +368,13 @@ def get_answer(self, last_messages, extra_context="", **kwargs):
                         thought = thought[0].strip()
                         action = action[0].strip()
                         answer = action
-                        logging.info(f"Valid answer: {answer}.")
+                        internal_agent_logger.info(f"Valid answer: {answer}.")
                         break
                     else:
-                        logging.info(f"Invalid answer: {answer_candidate}.")
+                        internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
                 except Exception as e:
-                    logging.info(f"Error generating response: {e}.")
-                    logging.info(f"Invalid answer: {answer_candidate}.")
+                    internal_agent_logger.info(f"Error generating response: {e}.")
+                    internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
         # generate answer for the ablation no_dialogue_no_react
         elif self.ablation == "no_dialogue_no_react":
             # in this case, the message is a string that ends with ##
@@ -367,13 +385,13 @@ def get_answer(self, last_messages, extra_context="", **kwargs):
                         answer = answer_candidate
                         # remove the ##
                         answer = answer[:-2]
-                        logging.info(f"Valid answer: {answer}.")
+                        internal_agent_logger.info(f"Valid answer: {answer}.")
                         break
                     else:
-                        logging.info(f"Invalid answer: {answer_candidate}.")
+                        internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
                 except Exception as e:
-                    logging.info(f"Error generating response: {e}.")
-                    logging.info(f"Invalid answer: {answer_candidate}.")
+                    internal_agent_logger.info(f"Error generating response: {e}.")
+                    internal_agent_logger.info(f"Invalid answer: {answer_candidate}.")
             ## save the answer in memory
             self.memory.upsert([{"turn_count":turn_count, "text":answer}])
 

groupchat_thread.py

@@ -4,9 +4,27 @@
 
 eval_prompt=load_base_prompt("prompts/chat_evaluation.j2")
 
-logging.basicConfig(level=logging.INFO, filename="chat.log", filemode="w", format="%(asctime)-15s %(message)s")
 
-class GroupchatThread:
+# Create a custom logger
+group_chat_logger = logging.getLogger('chat_thread')
+group_chat_logger.setLevel(logging.INFO)
+
+# Create handlers
+group_chat_file_handler = logging.FileHandler('chat.log', mode='w')
+group_chat_file_handler.setLevel(logging.INFO)
+
+# Create formatters and add it to handlers
+formatter = logging.Formatter('%(asctime)-15s %(message)s')
+group_chat_file_handler.setFormatter(formatter)
+
+# Add handlers to the logger
+group_chat_logger.addHandler(group_chat_file_handler)
+
+# prevent logging from propagating to the root logger
+group_chat_logger.propagate = False
+
+
+class GroupChatThread:
     """
     A group chat thread that simulates a conversation between multiple agents.    
     """
@@ -69,7 +87,7 @@ def start_conversation(self):
         assert self.turn == 0, "The conversation has already started."
 
         # log the start of the conversation and agent list
-        logging.info(f"Starting conversation {self.chat_id} with agents: {[agent.name for agent in self.agent_list]}")
+        group_chat_logger.info(f"Starting conversation {self.chat_id} with agents: {[agent.name for agent in self.agent_list]}")
 
         random_agent = self.pick_random_agent()
         first_message = (1, random_agent.name, random.choice(self.conversation_starters))
@@ -142,7 +160,7 @@ def evaluate_chat(self, start_index=0, end_index=-1):
                 if eval_score >= 0 and eval_score <= 10:
                     eval_results.append(eval_score)
             except Exception as e:
-                logging.error(f"Error in evaluation response: {eval_result}")
+                group_chat_logger.error(f"Error in evaluation response: {eval_result}")
                 print(e)
 
         eval_score = sum(eval_results) / 3
@@ -221,7 +239,7 @@ def run_chat(self, max_turns=50):
                 raise ValueError("The selection method is not valid.")
 
         # log the end of the conversation
-        logging.info(f"Ending conversation {self.chat_id} after {self.turn} turns.")
+        group_chat_logger.info(f"Ending conversation {self.chat_id} after {self.turn} turns.")
         # at the end of the chat, evaluate the chat
         if self.n_eval != -1:
             self.evaluate_chat()

llm_engines.py

@@ -18,6 +18,10 @@
 
 url= "http://127.0.0.1:1200/v1/chat/completions"
 
+model_info_url= "http://127.0.0.1:1200/v1/internal/model/info"
+
+load_model_url = "http://127.0.0.1:1200/v1/internal/model/load"
+
 headers = {
     "Content-Type": "application/json"
 }
@@ -83,18 +87,30 @@ def __init__(self, history=[], model="turboderp_Mixtral-8x7B-instruct-exl2_5.0bp
         self.url = url
         self.headers = headers
         self.model = model
+    
+    def get_current_model(self):
+        # send a request to get the current model
+        response = requests.get(model_info_url, headers=self.headers, verify=False)
+        return response.json()["model_name"]
+    
+    
     def generate_response(self, user_prompt):
 
         assistant_message = ""
 
+        # make sure the model is loaded
+        if self.get_current_model() != self.model:
+            data = {
+                "model_name": self.model
+            }
+            response = requests.post(load_model_url, headers=self.headers, json=data, verify=False)
 
         # append user prompt to history
         self.history.append({"role":"user","content": user_prompt})
         # query api for response
         data = {
         "mode":  "instruct",
         "messages": self.history,
-        "model": self.model,
         }
         response = requests.post(url, headers=headers, json=data, verify=False, )
 

test_name_dist.py name_dist_experiment.py

@@ -4,7 +4,7 @@
 import random, logging, time,json, os, textwrap
 
 
-logging.basicConfig(level=logging.INFO, filename="chat.log", filemode="w", format="%(asctime)-15s %(message)s")
+logging.basicConfig(level=logging.INFO, filename="chat.log", filemode="w", format="%(asctime)-15s %(message)s", force=True)
 
 
 # few-shot examples for the PLACES paper are contained in a jsonl file at prompts/places_examples.jsonl, let's load them, eacj line is a few-shot example conversation
@@ -97,7 +97,33 @@ def dump_chat(self):
             os.makedirs("chat_logs")
         with open(f"chat_logs/{self.chat_id}.json", "w") as f:
             json.dump(chat_data, f)    
+    
+    def summarize_personas_into_oneliner(self):
+        """
+        Summarizes the personas of the agents into a one-liner.
+        """
+        # get the unique personas
+        personas=set([agent.persona for agent in self.agent_list])
+        
+        # make it a string
+        personas_string = ". ".join(personas)
+        
+        examples="The following is a conversation between Alice and Bob about grocery shopping. Alice has a shopping list for Bob.\nThe following is a conversation between Alice and Bob about relationships. Bob recently got engaged.\nThe following is a conversation between Alice and Bob about their hobbies. Alice enjoys tennis and Bob likes playing soccer."
+        
+        prompt = f"Generate a sentence like the following one line summary of a convesation premise, but using the personas of the agents in the conversation. Remember to always use 'The following is a conversation between' and use the exact names from the personas. Include some details from topics the speakers are interested in based on their personas.\n Personas:\n\n{personas_string}\n\nFew-shot examples:\n{examples}. Answer (one line ending with ##):"
 
+        answer = ""
+        
+        # keep asking until the answer is valid
+        while len(answer)<10 or len(answer)>100:
+            answer = self.neutral_llm.generate_response(prompt)
+            if "##" in answer and "The following is a conversation between" in answer and self.agent_list[0].name in answer and self.agent_list[1].name in answer:
+                answer=answer.split("##")[0]
+                answer=answer.strip()
+                return answer
+            else:
+                logging.info(f"Invalid answer: {answer}")
+                answer = ""
 
     def generate_conversation(self, min_turns :int=10, start_conversation:bool=True):
         """
@@ -124,20 +150,12 @@ def generate_conversation(self, min_turns :int=10, start_conversation:bool=True)
             conversation_starter_string = f"{speaker.name}: {conversation_starter}"
         else:
             conversation_starter_string = ""
+            
 
-
 
 
-        # fill out the prompt template
-        prompt = self.generation_template.render(
-            agent_names=[agent.name for agent in self.agent_list],
-            agent_personas=[agent.persona for agent in self.agent_list],
-            conversation_starter_string=conversation_starter_string,
-            first_speaker=speaker.name,
-            few_shot_examples=random.sample(self.few_shot_examples,3)
-        )
+
 
-        logging.info(f"Naive generation prompt: {prompt}")
 
         # the answer should a list of strings, one message per line until the max_turns is reached
 
@@ -146,21 +164,35 @@ def generate_conversation(self, min_turns :int=10, start_conversation:bool=True)
         # keep generating until you have a conversation of min max_turns length
 
         while self.turn < min_turns:
+            # fill out the prompt template
+            prompt = self.generation_template.render(
+                agent_names=[agent.name for agent in self.agent_list],
+                conversation_premise=self.summarize_personas_into_oneliner(),
+                conversation_starter_string=conversation_starter_string,
+                first_speaker=speaker.name,
+                few_shot_examples=random.sample(self.few_shot_examples,3)
+            )
+            logging.info(f"Naive generation prompt: {prompt}")
+            
             self.turn = 1 if start_conversation else 0
 
             # reset the chat history for each turn
             temp_chat_history = []
 
             # generate the answer
             answer = self.neutral_llm.generate_response(prompt)
+            logging.info(f"Naive generation answer: {answer}")
 
+            # remove double newlines just in case
+            answer = answer.replace("\n\n", "\n")
             # split the answer into lines, each line should start with the agent name and a colon
             answer_lines = answer.split("\n")
 
             # add the answer to the chat history
             for line in answer_lines:
                 # split the line into agent name and message only at the first colon
                 try:
+                    line=line.strip()
                     agent_name, message = line.split(":", 1)
                     if agent_name in [agent.name for agent in self.agent_list]:
                         temp_chat_history.append((self.turn, agent_name, message))