Rename and refactor

Author: tiginamaria

README.md

@@ -11,33 +11,34 @@ pip install requirements.txt
 ## Evaluation Pipeline Configuration
 
 We use [Hydra](https://hydra.cc/docs/intro/) library for evaluation pipeline.
-Each configuration is specified in `config.yaml` format:
+Each configuration is specified in `eval.yaml` format:
 
 ```yaml
 # @package _global_
 hydra:
   job:
-    name: planning_${agent.model_name}
+    name: ${agent.name}_${agent.model_name}_[YOUR_ADDITIONAL_TOKEN_OR_NOTHING]
   run:
     dir:[YOUR_PATH_TO_OUTPUT_DIR]/${hydra:job.name}
   job_logging:
     root:
-      handlers: [ console, file ]
+      handlers: [console, file]
 defaults:
   - _self_
   - data_source: hf
-  - env: http
+  - env: code_engine
   - agent: planning
 ```
 
 Where you can define the datasource, env and agent you want to evaluate.
 We present several implementations for each defined in sub yamls:\
 
-| field         | options                                                                                                                                                                                                                                                                                                                                                          |
-|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `data_source` | [hf.yaml](configs/template_generation/data_source/hf.yaml)                                                                                                                                                                                                                                                                                                       |
-| `env`         | [http.yaml](configs/template_generation/env/http.yaml)                                                                                                                                                                                                                                                                                                           |
-| `agent`       | [vanilla.yaml](configs/template_generation/agent/vanilla.yaml)<br> [planning.yaml](configs/template_generation/agent/planning.yaml) <br> [reflexion.yaml](configs/template_generation/agent/reflexion.yaml)<br> [tree_of_thoughts.yaml](configs/template_generation/agent/tree_of_thoughts.yaml) <br> [adapt.yaml](configs/template_generation/agent/adapt.yaml) |
+| field         | options                                                                                                                                                                                                                                                                                                                                                                |
+|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `data_source` | [hf.yaml](configs/template_generation/data_source/hf.yaml)                                                                                                                                                                                                                                                                                                             |
+| `env`         | [code_engine.yaml](configs/template_generation/env/code_engine.yaml) <br> [http.yaml](configs/template_generation/env/http.yaml) <br> [few_shot.yaml](configs/template_generation/env/few_shot.yaml)                                                                                                                                                                   |
+| `agent`       | [few_shot.yaml](configs/template_generation/agent/few_shot.yaml) <br> [planning.yaml](configs/template_generation/agent/planning.yaml) <br> [vanilla.yaml](configs/template_generation/agent/vanilla.yaml) <br>  [reflexion.yaml](configs/template_generation/agent/reflexion.yaml) <br> [tree_of_thoughts.yaml](configs/template_generation/agent/tree_of_thoughts.yaml) <br> [adapt.yaml](configs/template_generation/agent/adapt.yaml) |
+
 
 # Project Template Generation Evaluation
 

configs/template_generation/config.yaml configs/template_generation/eval.yaml

@@ -148,7 +148,7 @@ def calc_stats(config: DictConfig):
         df.to_csv(os.path.join(config.data_path, f'{category}_template_repos.csv'), index=False)
 
 
-@hydra.main(config_path="../../configs/template_generation", config_name="data", version_base=None)
+@hydra.main(config_path="../../../configs/template_generation", config_name="data", version_base=None)
 def main(config: DictConfig):
     calc_stats(config)
 

src/template_generation/dataset_collection/__init__.py

@@ -17,7 +17,6 @@
 from src.utils.github.github_utils import clone_repo
 from src.utils.hf_utils import CATEGORIES, FEATURES, HUGGINGFACE_REPO
 from src.utils.jsonl_utils import read_jsonl
-from src.template_generation.prompts import get_gpt_description_system_prompt
 
 logging.basicConfig(
     format='%(asctime)s %(message)s',
@@ -49,6 +48,14 @@
                        'BSD 2-Clause "Simplified" License']
 
 
+def get_gpt_description_system_prompt() -> str:
+    return """Rewrite in 1-2 small sentences maximum template description and README heading asif user wants to generate template code using GPT.\n 
+    Input:\n
+    This scaffold project is written in Kotlin and will serve an example of implementing a Selenium test project with FluentLenium (Selenium3) and Gradle (with kotlin DSL). Everything is set up and tests can be added straight away. Used Testrunner is JUnit 5. Since Kotlin has an excellent Java interop it's perfectly fine to write your Tests in Java if you want, it will work out of the box. To execute the tests just browse to the path where the selenium-kotlin-example is located via terminal and type ./gradlew clean test or execute the tests in your IDE. The Project will use Chrome Browser in Headless mode by default / if no other browser is stated (see list of implemented browsers for more info on how to use them).\n
+    Output:\n
+    Kotlin template with gradle.kts and Selenium tests with FluentLenium and testrunner is JUnit 5"""
+
+
 def load_repos_data(config: DictConfig):
     for category in ['py']:
         with open(os.path.join(config.data_path, f"{category}_search.json"), "r", errors='ignore') as f:
@@ -222,7 +229,7 @@ def add_gpt_description(project, test_full_names: list[str]):
         df.to_csv(os.path.join(config.data_path, f"{category}_template_repos.csv"), index=False)
 
 
-@hydra.main(config_path="../../configs/template_generation", config_name="data", version_base=None)
+@hydra.main(config_path="../../../configs/template_generation", config_name="data", version_base=None)
 def main(config: DictConfig):
     load_dotenv()
     # load_repos_data(config)

src/template_generation/add_data_metrics.py src/template_generation/dataset_collection/add_data_metrics.py

@@ -71,7 +71,7 @@ def upload_projects(config: DictConfig):
     )
 
 
-@hydra.main(config_path="../../configs/template_generation", config_name="data", version_base=None)
+@hydra.main(config_path="../../../configs/template_generation", config_name="data", version_base=None)
 def main(config: DictConfig):
     load_dotenv()
     upload_projects(config)

src/template_generation/run_git_data_collection.py src/template_generation/dataset_collection/run_git_data_collection.py

@@ -98,7 +98,7 @@ async def run_template_generation(agent: BaseAgent, env: BaseEnv, data_source: B
             writer.writerow(results_dict.values())
 
 
-@hydra.main(config_path="../../configs/template_generation", config_name="config.yaml", version_base="1.2")
+@hydra.main(config_path="../../configs/template_generation", config_name="eval.yaml", version_base="1.2")
 def main(cfg: EvalConfig) -> None:
     agent: BaseAgent = hydra.utils.instantiate(cfg.agent)
     env: BaseEnv = hydra.utils.instantiate(cfg.env)