Filelist-log file to yaml and with checksum, to be used with esm_tests for the new filedicts.py

src/esm_runscripts/filedicts.py

@@ -1207,17 +1207,18 @@ def from_config(cls, config: dict):
                 sim_files[file_key] = SimulationFile.from_config(
                     config, f"{config_address}.{file_key}"
                 )
+                sim_files[file_key]["component"] = component
         return sim_files
 
     def _gather_file_movements(self) -> None:
         """Puts the methods for each file movement into the dictionary as callable values behind the `_filesystem_op` key"""
         for sim_file_id, sim_file_obj in self.items():
-            movement_type = sim_file_obj.get("movement_type", "cp")
-            if movement_type == "mv":
+            movement_type = sim_file_obj.get("movement_type", "copy")
+            if movement_type == "move":
                 self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "mv")
-            elif movement_type == "cp":
+            elif movement_type == "copy":
                 self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "cp")
-            elif movement_type == "ln":
+            elif movement_type == "link":
                 self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "ln")
             else:
                 raise ValueError(
@@ -1231,12 +1232,18 @@ def execute_filesystem_operation(
         for sim_file_id, sim_file_obj in self.items():
             logger.info(f"Processing {sim_file_id}")
             if config["general"]["jobtype"] == "prepcompute":
-                src, dest = "pool", "work"
+                src, dest = "computer", "work"
             elif config["general"]["jobtype"] == "tidy":
                 src, dest = "work", "exp_tree"
             else:
                 raise ValueError(f"Incorrect jobtype specified for {sim_file_obj}")
             sim_file_obj["_filesystem_op"](src, dest)
+            config[sim_file_obj["component"]]["files"][sim_file_id]["src"] = (
+                sim_file_obj.paths[src]
+            )
+            config[sim_file_obj["component"]]["files"][sim_file_id]["dest"] = (
+                sim_file_obj.paths[dest]
+            )
         return config
 
 
@@ -1258,3 +1265,21 @@ def resolve_file_movements(config: ConfigSetup) -> ConfigSetup:
     sim_file_collection = SimulationFileCollection.from_config(config)
     config = sim_file_collection.execute_filesystem_operation(config)
     return config
+
+
+def log_used_files(config: ConfigSetup) -> ConfigSetup:
+    """
+    Logs the files moved on this current phase.
+
+    Parameters
+    ----------
+    config : ConfigSetup
+        The complete simulation configuration.
+
+    Returns
+    -------
+    config : ConfigSetup
+        The complete simulation configuration, potentially modified.
+    """
+
+    return config

tests/test_esm_runscripts/test_filedicts.py

@@ -1115,3 +1115,59 @@ def test_globbing_ln(fs):
 
     for nf in expected_new_paths:
         assert os.path.exists(nf)
+
+
+def test_file_log(fs):
+    """Checks that the file log is produced correctly"""
+    dummy_config = """
+    general:
+        expid: expid
+        base_dir: /some/dummy/location/
+        thisrun_work_dir: "/work/ollie/pgierz/some_exp/run_20010101-20011231/work"
+        exp_dir: "/work/ollie/pgierz/some_exp"
+        thisrun_dir: "/work/ollie/pgierz/some_exp/run_20010101-20011231"
+        all_model_filetypes: [analysis, bin, config, forcing, input, couple, log, mon, outdata, restart, viz, ignore]
+        jobtype: "prepcompute"
+        valid_model_names: ["echam"]
+    computer:
+        pool_dir: "/work/ollie/pool"
+    echam:
+        experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam
+        thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20011231/input/echam
+        files:
+            human_readable_tag_001:
+                kind: input
+                allowed_to_be_missing: True
+                name_in_computer: foo
+                path_in_computer: /work/data/pool
+                name_in_work: foo
+                path_in_work: .
+                movement_type: move
+    """
+
+    check_log_file = """
+    echam:
+        human_readable_tag_001:
+            checksum: d41d8cd98f00b204e9800998ecf8427e
+            intermediate: null
+            source: /work/data/pool/foo
+            target: /work/ollie/pgierz/some_exp/run_20010101-20011231/work/foo
+            kind: input
+    """
+    date = esm_calendar.Date("2000-01-01T00:00:00")
+    config = yaml.safe_load(dummy_config)
+    config["general"]["current_date"] = date
+    fs.create_dir("/work/data/pool")
+    fs.create_file("/work/data/pool/foo")
+    fs.create_dir("/work/ollie/pgierz/some_exp/run_20010101-20011231/work")
+
+    sim_files = esm_runscripts.filedicts.SimulationFileCollection.from_config(
+        config
+    )
+    config = sim_files.execute_filesystem_operation(config)
+
+    esm_runscripts.filedicts.log_used_files(config)
+
+    log_file = open("/work/ollie/pgierz/some_exp/run_20010101-20011231/log/expid_filelist_20010101-20011231.yaml", "r").read()
+
+    assert log_file==check_log_file