新增代码文件 脚本 场景生成 量化评估文件 (#95)

* Update hutbthesis_main.tex

* Add files via upload

* 添加scripts目录

包含一些运行脚本

* retrieve文件夹

retrieve 文件夹在项目中用于存储与场景描述检索相关的数据和代码，帮助从数据库中获取并生成动态场景。

* 添加 undergraduate/demo 文件夹

* Add files via upload

* Add files via upload

* 添加docs文件夹

* Add files via upload

* 添加safebench.egg-info文件夹

* 添加 safebench 文件夹

* 添加 undergraduate/tools 文件夹

* 添加 undergraduate/Scenic 文件夹

* 为 Python 包 safebench 创建一个可安装的包

为 Python 包 safebench 创建一个可安装的包，并指定了依赖项（gym 和 pygame）以及包的基本信息。

* 这是原作者的一个许可证

MIT 许可证是一个非常宽松的开源许可协议，允许他人自由地使用和修改代码，只要他们保留版权声明并且了解代码的“原样”提供，不承担任何责任。

* 包含用于构建和运行 Docker 容器的相关配置文件

docker 文件夹包含用于构建和运行 Docker 容器的相关配置文件（如 Dockerfile 和 run_docker.sh），目的是创建一个可重复的环境来运行项目或应用，通常用于依赖管理和环境配置。

* 用于存储和管理与自然语言描述生成场景相关的数据和代码。

用于存储和管理与自然语言描述生成场景相关的数据和代码。

* Update README.md

* 输出文件outputs

* 更新存放量化评估的文件夹

* Update README.md

* Update README.md

* Update README.md

* Delete scene/undergraduate/demo/.idea directory

* Delete scene/undergraduate/retrieve/__pycache__ directory

* Delete scene/undergraduate/safebench.egg-info directory

* Delete scene/undergraduate/outputs/screenshots directory

* Delete scene/undergraduate/setup.py

* Delete scene/undergraduate/safebench/__pycache__ directory

* Delete scene/undergraduate/scripts/__pycache__ directory

* Delete scene/undergraduate/safebench/agent/__pycache__ directory

* Delete scene/undergraduate/Scenic directory

* Delete scene/undergraduate/demo directory

* Delete scene/undergraduate/docker directory

* Delete scene/undergraduate/evaluation directory

* Delete scene/undergraduate/docs directory

* Delete scene/undergraduate/retrieve directory

* Delete scene/undergraduate/safebench directory

* Delete scene/undergraduate/scripts directory

* Delete scene/undergraduate/tools directory

* Delete scene/undergraduate/LICENSE

* 增加代码文件夹

一些脚本 一些场景生成文件 一些量化评估文件

* Delete scene/retrieve/__pycache__ directory

* Delete scene/retrieve/database_v1.pkl

* Delete scene/scripts/__pycache__ directory

Author: zrx0829222

scene/evaluation/evaluate_scene_quality.py

@@ -0,0 +1,105 @@
+import os
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.metrics import pairwise_distances_argmin_min
+import cv2
+
+# 文件路径
+DESCRIPTION_FILE = 'D:/sceneMain/chatScene/retrieve/scenario_descriptions.txt'
+HISTORY_FILE = 'D:/sceneMain/chatScene/retrieve/scenario_history.txt'
+SCENE_IMAGE_DIR = 'D:/sceneMain/chatScene/outputs/'
+
+# 加载最新的场景描述（只读取文件的第一行）
+def load_latest_description(path):
+    """只读取描述文件中的第一行"""
+    with open(path, 'r', encoding='utf-8') as f:
+        first_line = f.readline().strip()  # 读取第一行
+    return first_line
+
+# 将新的场景描述追加到历史记录文件
+def append_to_history(new_description, history_path):
+    """将新的场景描述追加到历史文件"""
+    with open(history_path, 'a', encoding='utf-8') as f:
+        f.write(new_description + '\n')
+
+# 计算图像相似度（使用结构相似度）
+def calculate_image_similarity(image1, image2):
+    """计算两张图像之间的相似度"""
+    gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+    gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+    score, _ = cv2.quality.QualitySSIM_compute(gray1, gray2)
+    return score
+
+# 计算场景的多样性（使用生成图像之间的距离）
+def calculate_scene_diversity(image_dir):
+    """计算所有图像之间的多样性"""
+    images = []
+    for filename in os.listdir(image_dir):
+        if filename.endswith('.png'):
+            img = cv2.imread(os.path.join(image_dir, filename))
+            images.append(img)
+
+    # 转换为数组（每个图像的特征）
+    image_features = [np.reshape(img, (-1, 3)) for img in images]
+    image_features = np.concatenate(image_features, axis=0)
+
+    # 计算每对图像的最小距离
+    distances = pairwise_distances_argmin_min(image_features, image_features)
+    avg_distance = np.mean(distances[1])  # 平均最小距离
+    return avg_distance
+
+# 评估场景质量：语义一致性，图像质量，多样性
+def evaluate_scene_quality(image_dir):
+    """评估场景的质量"""
+
+    # 语义一致性（假设为手动指定或从其他方法中获得）
+    semantic_consistency = 0.9  # 假设的值，通常需要根据具体情况进行计算
+
+    # 图像质量：假设使用已有的参考图像进行评估（此处为一个示例）
+    reference_image = cv2.imread('D:/sceneMain/chatScene/reference_image.png')  # 参考图像
+    image_files = [f for f in os.listdir(image_dir) if f.endswith('.png')]
+    avg_image_quality = 0
+    for image_file in image_files:
+        img = cv2.imread(os.path.join(image_dir, image_file))
+        similarity = calculate_image_similarity(reference_image, img)
+        avg_image_quality += similarity
+    avg_image_quality /= len(image_files)
+
+    # 多样性
+    diversity = calculate_scene_diversity(image_dir)
+
+    # 打印评估结果
+    print(f"语义一致性: {semantic_consistency}")
+    print(f"平均图像质量: {avg_image_quality}")
+    print(f"场景多样性: {diversity}")
+
+    return semantic_consistency, avg_image_quality, diversity
+
+# 主函数
+def main():
+    # 读取最新的场景描述
+    latest_description = load_latest_description(DESCRIPTION_FILE)
+
+    # 将描述追加到历史记录
+    append_to_history(latest_description, HISTORY_FILE)
+
+    # 打印最新描述
+    print(f"最新的场景描述: {latest_description}")
+
+    # 评估生成的场景质量
+    semantic_consistency, avg_image_quality, diversity = evaluate_scene_quality(SCENE_IMAGE_DIR)
+
+    # 可以根据需要将评估结果保存为JSON或其他格式
+    evaluation_results = {
+        "semantic_consistency": semantic_consistency,
+        "avg_image_quality": avg_image_quality,
+        "diversity": diversity
+    }
+
+    # 保存评估结果到文件
+    with open('D:/sceneMain/chatScene/outputs/evaluation_results.json', 'w') as f:
+        json.dump(evaluation_results, f, indent=4)
+
+if __name__ == "__main__":
+    main()

scene/retrieve/architecture.py

@@ -0,0 +1,35 @@
+import os
+import openai
+import torch
+import transformers
+os.environ["OPENAI_API_KEY"] = 'sk-proj-xxx'
+ 
+class LLMChat():
+    def __init__(self, model_name = 'meta-llama/Meta-Llama-3-8B-Instruct'):
+        super(LLMChat, self).__init__()
+        self.model_name = model_name
+        if model_name.startswith('gpt'):
+            self.client = openai.OpenAI()
+        else:
+            self.pipeline = transformers.pipeline(
+                "text-generation",
+                model=model_name,
+                model_kwargs={"torch_dtype": torch.bfloat16},
+                device="cuda",
+            )
+
+    def generate(self, messages, max_new_tokens = 500):
+        if self.model_name.startswith('gpt'):
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+                temperature=0,
+            )
+            return response.choices[0].message.content
+        else:
+            outputs = self.pipeline(
+                messages,
+                max_new_tokens=max_new_tokens,
+                do_sample=False
+            )
+            return outputs[0]["generated_text"][-1]

scene/retrieve/prompts/behavior.txt

@@ -0,0 +1,5 @@
+Now, your task is to help write part of the Scenic code for defining the adversarial behavior of an agent given the corresponding description. Here are some related description-snippet pairs:
+{content}
+Now, provide the snippet for the following description. Note that if there is already one in the example that matches the description, directly return snippet. Otherwise, please construct the snippet following the same format as above. Note that the name of the behavior function should always be AdvBehavior, and any hyperparameter should be defined as param OPT_xxx, and use it inside the behavior function with globalParameters.OPT_xxx. Please strictly follow the syntax as those used in the examples.
+
+Description: {current_description}

scene/retrieve/prompts/extraction.txt

@@ -0,0 +1,46 @@
+Your task is to decompose full descriptions of safety-critical scenarios into sub-descriptions for the following distinct components:
+
+Adversarial Object: Indicate the type of the adversarial agent, restricted to Car, Pedestrian, Bicycle, or Motorcycle.
+Behavior: Describe the behavior of the adversarial agent.
+Geometry: Specify the road condition where the scenario occurs (e.g., straight road, four-way intersection), including traffic light conditions.
+Spawn Position: Indicate the initial relative position of the adversarial object to the ego vehicle, including any potential obstructions like a vending machine.
+
+Here are some examples:
+Scenario: The ego vehicle is driving on a straight road, and the car in front brakes suddenly as the ego approaches.
+Adversarial Object: Car
+Behavior: The adversarial car suddenly brakes as the ego approaches.
+Geometry: A straight road.
+Spawn Position: The adversarial agent is directly in front of the ego vehicle on the same straight road, heading in the same direction.
+
+Scenario: The ego vehicle attempts a right turn at a four-way intersection, and an adversarial pedestrian crosses the road and suddenly stops.
+Adversarial Object: Pedestrian
+Behavior: The adversarial pedestrian deliberately steps onto the road in front of the ego vehicle.
+Geometry: The ego vehicle drives across a four-way intersection.
+Spawn Position: The adversarial agent is on the right front of the ego vehicle at the end of the ego's initial lane for crossing.
+
+Scenario: The ego vehicle navigates around a parked car, and an oncoming car suddenly turns into its path.
+Adversarial Object: Car
+Behavior: The adversarial car suddenly turns into the ego's path.
+Geometry: The ego vehicle is positioned on a two-lane road with traffic flowing in opposite directions.
+Spawn Position: The adversarial agent comes from the near opposite oncoming lane, with a parked car blocking the ego vehicle's lane.
+
+Scenario: The ego vehicle is traveling along a straight road when a pedestrian, initially hidden behind a bus stop on the sidewalk to the right, unexpectedly dashes onto the road directly in front of the ego vehicle and comes to an abrupt stop.
+Adversarial Object: Pedestrian
+Behavior: The adversarial pedestrian suddenly sprints from the right, stopping abruptly in front of the ego vehicle.
+Geometry: A straight road.
+Spawn Position: The adversarial agent spawns from behind a bus stop on the right front of the ego vehicle on the same straight road for crossing.
+
+Scenario: The ego vehicle is changing to the right lane when an adversarial vehicle approaches rapidly from the right.
+Adversarial Object: Car
+Behavior: The adversarial car approaches rapidly.
+Geometry: The ego vehicle is placed in a straight lane that includes a right lane.
+Spawn Position: The adversarial car drives straight from the rear right of the ego.
+
+Scenario: The ego vehicle is turning right at an intersection, and a crossing car from the left violates the red light and suddenly brakes.
+Adversarial Object: Car
+Behavior: The adversarial car suddenly brakes near the ego vehicle.
+Geometry: The ego vehicle drives straight across a four-way signalized intersection; the light is red for the adversarial agent.
+Spawn Position: The adversarial car is crossing the intersection from the left.
+
+Now, extract the adversarial behavior, geometry and spawn position in the same format from the following scenario, notice Adversarial Object is restricted to Car, Pedestrian, Bicycle, or Motorcycle.
+Scenario: {scenario}

scene/retrieve/prompts/geometry.txt

@@ -0,0 +1,5 @@
+Now, your task is to help write part of the Scenic code for defining the geometry for the ego agent given the corresponding description. Here are some related description-snippet pairs:
+{content}
+Now, provide the snippet for the following description. Note that if there is already one in the example that matches the description, directly return snippet. Otherwise, please construct the snippet following the same format as above. Please do not use the syntax outside of those used in the examples.
+
+Description: {current_description}

scene/retrieve/prompts/spawn.txt

@@ -0,0 +1,5 @@
+Now, your task is to help write part of the Scenic code for defining the spawn point for the adversarial agent given the corresponding description. Here are some related description-snippet pairs:
+{content}
+Now, provide the snippet for the following description. Note that if there is already a snippet in the example that matches the description, directly return snippet. Otherwise, please construct a new snippet following the format provided above. Note that any hyperparameters should be defined as param OPT_xxx, and use them as globalParameters.OPT_xxx. If there is a blocker involved, you must also define it. The blocker can be chosen only from the following options: 'Car', 'ATM', 'Barrel', 'Barrier', 'Bench', 'BusStop', 'Chair', 'Debris', 'Garbage', 'Mailbox', 'VendingMachine'. Please do not use the syntax outside of those used in the examples.
+
+Description: {current_description}

scene/retrieve/retrieve.py

@@ -0,0 +1,106 @@
+import setGPU
+import os
+import csv
+import pickle
+import re
+from sentence_transformers import SentenceTransformer
+from os import path as osp
+from tqdm import tqdm
+import argparse
+from architecture import LLMChat
+from utils import load_file, retrieve_topk, generate_code_snippet, save_scenic_code
+
+# no need for faiss currently
+# import faiss
+
+parser = argparse.ArgumentParser(description="Set up configurations for your script.")
+parser.add_argument('--port_ip', type=int, default=2000, help='Port IP address (default: 2000)')
+parser.add_argument('--topk', type=int, default=3, help='Top K value (default: 3) for retrieval')
+parser.add_argument('--model', type=str, default='gpt-4o', help="Model name (default: 'gpt-4o'), also support transformers model")
+parser.add_argument('--use_llm', action='store_true', help='if use llm for generating new snippets')
+args = parser.parse_args()
+
+port_ip = args.port_ip
+topk = args.topk
+use_llm = args.use_llm
+
+llm_model = LLMChat(args.model)
+local_path = osp.abspath(osp.dirname(osp.dirname(osp.realpath(__file__))))
+extraction_prompt = load_file(osp.join(local_path, 'retrieve', 'prompts', 'extraction.txt'))
+behavior_prompt = load_file(osp.join(local_path, 'retrieve', 'prompts', 'behavior.txt'))
+geometry_prompt = load_file(osp.join(local_path, 'retrieve', 'prompts', 'geometry.txt'))
+spawn_prompt = load_file(osp.join(local_path, 'retrieve', 'prompts', 'spawn.txt'))
+scenario_descriptions = load_file(osp.join(local_path, 'retrieve', 'scenario_descriptions.txt')).split('\n')
+encoder = SentenceTransformer('sentence-transformers/sentence-t5-large', device='cuda')
+
+# Load the database
+with open(osp.join(local_path, 'retrieve/database_v1.pkl'), 'rb') as file:
+    database = pickle.load(file)
+
+behavior_descriptions = database['behavior']['description']
+geometry_descriptions = database['geometry']['description']
+spawn_descriptions = database['spawn']['description']
+behavior_snippets = database['behavior']['snippet']
+geometry_snippets = database['geometry']['snippet']
+spawn_snippets = database['spawn']['snippet']
+
+behavior_embeddings = encoder.encode(behavior_descriptions, device='cuda', convert_to_tensor=True)
+geometry_embeddings = encoder.encode(geometry_descriptions, device='cuda', convert_to_tensor=True)
+spawn_embeddings = encoder.encode(spawn_descriptions, device='cuda', convert_to_tensor=True)
+
+## This is the head for scenic file, you can modify the carla map or ego model here
+head = '''param map = localPath(f'../maps/{Town}.xodr') 
+param carla_map = Town
+model scenic.simulators.carla.model
+EGO_MODEL = "vehicle.lincoln.mkz_2017"
+'''
+
+log_file_path = osp.join(local_path, 'safebench', 'scenario', 'scenario_data', 'scenic_data', 'dynamic_scenario', 'dynamic_log.csv')
+
+with open(log_file_path, mode='w', newline='') as file:
+    log_writer = csv.writer(file)
+    log_writer.writerow(['Scenario', 'AdvObject', 'Behavior Description', 'Behavior Snippet', 'Geometry Description', 'Geometry Snippet', 'Spawn Description', 'Spawn Snippet', 'Success'])
+
+    for q, current_scenario in tqdm(enumerate(scenario_descriptions)):
+        messages = [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": extraction_prompt.format(scenario=current_scenario)},
+        ]
+
+        response = llm_model.generate(messages)
+
+        try:
+            match = re.search(r"Adversarial Object:(.*?)Behavior:(.*?)Geometry:(.*?)Spawn Position:(.*)", response, re.DOTALL)
+            if not match:
+                raise ValueError("Failed to extract components from the response")
+
+            current_adv_object, current_behavior, current_geometry, current_spawn = [s.strip() for s in match.groups()]
+
+            # Retrieve the top K relevant snippets
+            top_behavior_descriptions, top_behavior_snippets = retrieve_topk(encoder, topk, behavior_descriptions, behavior_snippets, behavior_embeddings, current_behavior)
+            top_geometry_descriptions, top_geometry_snippets = retrieve_topk(encoder, topk, geometry_descriptions, geometry_snippets, geometry_embeddings, current_geometry)
+            top_spawn_descriptions, top_spawn_snippets = retrieve_topk(encoder, topk, spawn_descriptions, spawn_snippets, spawn_embeddings, current_spawn)
+
+            # Generate code snippets using the LLM
+            generated_behavior_code = generate_code_snippet(
+                llm_model, behavior_prompt, top_behavior_descriptions, top_behavior_snippets, current_behavior, topk, use_llm
+            )
+
+            generated_geometry_code = generate_code_snippet(
+                llm_model, geometry_prompt, top_geometry_descriptions, top_geometry_snippets, current_geometry, topk, use_llm
+            )
+
+            generated_spawn_code = generate_code_snippet(
+                llm_model, spawn_prompt, top_spawn_descriptions, top_spawn_snippets, current_spawn, topk, use_llm
+            )
+
+            # Log the results
+            log_writer.writerow([current_scenario, current_adv_object, current_behavior, generated_behavior_code, current_geometry, generated_geometry_code, current_spawn, generated_spawn_code, 1])
+
+            Town, generated_geometry_code = generated_geometry_code.split('\n', 1)
+            scenic_code = '\n'.join([f"'''{current_scenario}'''", Town, head, generated_behavior_code, generated_geometry_code, generated_spawn_code.format(AdvObject=current_adv_object)])
+            save_scenic_code(local_path, port_ip, scenic_code, q)
+
+        except Exception as e:
+            log_writer.writerow([current_scenario, '', '', '', '', '', '', '', 0])
+            print(f"Failure for scenario: {current_scenario} - Error: {e}")