Allow to customize bulk delete behavior

pyiron_base/jobs/job/generic.py

@@ -10,6 +10,7 @@
 import os
 import posixpath
 import signal
+import shutil
 import warnings
 
 from pyiron_base.state import state
@@ -1470,6 +1471,32 @@ def _reload_update_master(self, project, master_id):
             self._logger.info("busy master: {} {}".format(master_id, self.get_job_id()))
             del self
 
+    @staticmethod
+    def _bulk_remove_jobs(project, job_df, progress):
+        """
+        """
+        def del_files(row):
+            if progress is not None:
+                progress.update(1)
+            project_path, project, job = row[["projectpath", "project", "job"]]
+            if project_path is not None:
+                base_path = os.path.join(project_path, project, job)
+            else:
+                base_path = os.path.join(project, job)
+            if os.path.isfile(base_path + ".h5"):
+                os.remove(base_path + ".h5")
+            shutil.rmtree(base_path + "_hdf5", ignore_errors=True)
+
+        project.db.delete_item(job_df.id)
+        # if job doesn't match subjob, it's stored inside another job's HDF5 file and will be delete from there.
+        n_total = len(job_df)
+        job_df = job_df.query("job == subjob.str.slice(1, None)")
+        # sub jobs won't have their files deleted, so advance progress bar on its own
+        n_sub = n_total - len(job_df)
+        if progress is not None:
+            progress.update(n_sub)
+        job_df.apply(del_files, axis="columns")
+
 
 class GenericError(object):
     def __init__(self, job):

pyiron_base/jobs/master/generic.py

@@ -504,6 +504,13 @@ def _init_child_job(self, parent):
         """
         self.ref_job = parent
 
+    @staticmethod
+    def _bulk_remove_jobs(project, job_df, progress):
+        for job_id in job_df["id"]:
+            job = project.load(job_id)
+            job.child_project.remove(recursive=True, silently=True)
+
+        super()._bulk_remove_jobs(project, job_df, progress)
 
 def get_function_from_string(function_str):
     """

pyiron_base/project/generic.py

@@ -34,7 +34,7 @@
 from pyiron_base.utils.deprecate import deprecate
 from pyiron_base.jobs.job.util import _special_symbol_replacements, _get_safe_job_name
 from pyiron_base.interfaces.has_groups import HasGroups
-from pyiron_base.jobs.job.jobtype import JobType, JobTypeChoice, JobFactory
+from pyiron_base.jobs.job.jobtype import JOB_CLASS_DICT, JobType, JobTypeChoice, JobFactory
 from pyiron_base.jobs.job.extension.server.queuestatus import (
     queue_delete_job,
     queue_is_empty,
@@ -1591,20 +1591,24 @@ def _remove_jobs_helper(self, recursive=False, progress=True):
             raise ValueError("recursive must be a boolean")
         if self.db.view_mode:
             raise RuntimeError("copy_to: is not available in Viewermode !")
-        job_id_lst = self.get_job_ids(recursive=recursive)
-        if progress and len(job_id_lst) > 0:
-            job_id_lst = tqdm(job_id_lst)
-        for job_id in job_id_lst:
-            if job_id not in self.get_job_ids(recursive=recursive):
-                continue
-            else:
-                try:
-                    self.remove_job(job_specifier=job_id)
-                    state.logger.debug("Remove job with ID {0} ".format(job_id))
-                except (IndexError, Exception):
-                    state.logger.debug(
-                        "Could not remove job with ID {0} ".format(job_id)
-                    )
+        job_df = self.job_table(
+                recursive=recursive,
+                columns=["id", "hamilton", "parentid", "masterid", "projectpath", "project", "job", "subjob"]
+        )
+        job_id_lst = job_df["id"]
+        parents = set(job_df.parentid.dropna())
+        masters = set(job_df.masterid.dropna())
+        if not (parents.issubset(job_id_lst) and masters.issubset(job_id_lst)):
+            assert False, "Somehow sort out out of project jobs"
+
+        progress = tqdm(total=len(job_id_lst)) if progress else None
+        for hamilton, sub_df in job_df.groupby("hamilton"):
+            try:
+                job_class = JobType.convert_str_to_class(JOB_CLASS_DICT, hamilton)
+            except ValueError: # if job class is not registered in JOB_CLASS_DICT use generic routine
+                from pyiron_base.jobs.job.generic import GenericJob
+                job_class = GenericJob
+            job_class._bulk_remove_jobs(self, sub_df, progress)
 
     def _remove_files(self, pattern="*"):
         """