Mb/script

metisp/pymetis/pyproject.toml

@@ -1,13 +1,50 @@
+[project]
+name = "eso-pymetis"
+version = "0.1.0"
+description = "METIS pipeline"
+authors = [
+    {name = "[email protected]"}
+]
+import-names = ["pymetis"]
+license = {text = "GPLv3"}
+readme = "README.md"
+requires-python = ">=3.11, <3.14"
+dependencies = [
+  "pycpl==1.0.3.post4",   # prefer ivh's re-packaged pycpl
+  "edps",
+  "pyesorex",
+  "adari_core",
+]
+
+[dependency-groups]
+dev = [
+    "pytest (>=9.0.1,<10.0.0)"
+]
+
 [build-system]
-requires = [
-    "setuptools >= 45",
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+packages = [
+    { include = "pymetis", from = "src" },
 ]
-build-backend = "setuptools.build_meta"
 
-[project]
-name = "pymetis"
-version = "0.0.1"
-dynamic = ["description","requires-python","license","authors","classifiers","urls","dependencies","optional-dependencies"]
+[tool.poetry.dependencies]
+pycpl = { version = "1.0.3.post4", source = "ivh" }
+pyesorex = { source = "eso" }
+edps = { source = "eso" }
+adari_core = { source = "eso" }
+
+[[tool.poetry.source]]
+name = "eso"
+url = "https://ftp.eso.org/pub/dfs/pipelines/repositories/stable/src"
+priority = "explicit"
+
+[[tool.poetry.source]]
+name = "ivh"
+url = "https://ivh.github.io/pycpl/simple/"
+priority = "explicit"
 
 [tool.pytest.ini_options]
 addopts = "--strict-markers"
@@ -22,4 +59,4 @@ markers = [
     "pyesorex: marks tests that depend on `pyesorex`",
     "slow: marks tasts with full-size data",
     "metadata: marks tests that verify recipe metadata",
-]
+]

metisp/pymetis/src/pymetis/classes/dataitems/dataitem.py

@@ -37,7 +37,7 @@
 PIPELINE = rf'METIS/1'
 
 
-class DataItem(Parametrizable, ABC):
+class DataItem(Parametrizable):
     """
     The `DataItem` class encapsulates a single data item:
     the smallest standalone unit of detector data or a product of a recipe.
@@ -48,13 +48,13 @@ class DataItem(Parametrizable, ABC):
     Multiple files with the same tag should correspond to multiple instances of the same DataItem class.
     """
     # Class registry: all derived classes are automatically registered here (unless declared abstract)
-    __registry: dict[str, type['DataItem']] = {}
+    _registry: dict[str, type['DataItem']] = {}
 
     # Printable title of the data item. Not used internally, only for human-oriented output
     _title_template: str = None                     # No universal title makes sense
     # Actual ID of the data item. Used internally for identification. Should mirror DRLD `name`.
-    _name_template: str = None                      # No universal name makes sense
-    # Description for man page
+    _name_template: str = "DataItem"                # No universal name makes sense
+    # A long description that will be used in the man page
     _description_template: Optional[str] = None     # A verbose string; should correspond to the DRLD description
 
     # CPL frame group and level
@@ -85,34 +85,28 @@ def __init_subclass__(cls,
                           abstract: bool = False,
                           **kwargs):
         """
-        Register every subclass of DataItem in a global registry.
-        Classes marked as abstract are not registered and should never be instantiated.
-        # FixMe: Hugo says it might be useful for database views and such. But for now it is so.
+        Register every subclass of DataItem in a global registry based on their tags.
         """
+
+        super().__init_subclass__(**kwargs)
+
         cls.__abstract = abstract
 
-        if cls.__abstract:
-            # If the class is not fully specialized, skip it
+        if cls.name() in DataItem._registry:
+            # If the class is already registered, warn about it and do nothing.
             Msg.debug(cls.__qualname__,
-                      f"Class is abstract, skipping registration")
+                      f"A class with tag {cls.name()} is already registered, "
+                      f"skipping: {DataItem._registry[cls.name()].__qualname__}")
         else:
-            # Otherwise, add it to the global registry
-            assert cls.__regex_pattern.match(cls.name()) is not None, \
-                (f"Tried to register {cls.__name__} ({cls.name()}) which is not fully specialized "
-                 f"(did you mean to set `abstract=True` in the class declaration?)")
-
-            if cls.name().format() in DataItem.__registry:
-                # If the class is already registered, warn about it and do nothing
-                Msg.warning(cls.__qualname__,
-                            f"A class with tag {cls.name()} is already registered, "
-                            f"skipping: {DataItem.__registry[cls.name()]}")
-            else:
-                # Otherwise add it to the registry
-                Msg.debug(cls.__qualname__,
-                          f"Registered a new class {cls.name()}: {cls}")
-                DataItem.__registry[cls.name()] = cls
+            # Otherwise add the class to the global registry
+            Msg.debug(cls.__qualname__,
+                      f"Registered a new class {cls.name()}: {cls}")
+            DataItem._registry[cls.name()] = cls
 
-        super().__init_subclass__(**kwargs)
+    @classmethod
+    @final
+    def schema(cls) -> dict[str, Union[None, type[Image], type[Table]]]:
+        return cls._schema
 
     @classmethod
     @final
@@ -122,17 +116,16 @@ def find(cls, key: str) -> Optional[type['DataItem']]:
 
         If not found, return ``None`` instead (and leave it to the caller to raise an exception if this is not desired).
         """
-        if key in DataItem.__registry:
-            return DataItem.__registry[key]
+        if key in DataItem._registry:
+            return DataItem._registry[key]
         else:
             return None
 
     @classmethod
-    def name_template(cls) -> str:
-        return cls._name_template
-
-    @classmethod
-    def specialize(cls, **parameters) -> str:
+    def specialize(cls, **parameters: str) -> str:
+        """
+        Specialize the data item's name template with given parameters
+        """
         cls._name_template = partial_format(cls._name_template, **parameters)
         return cls._name_template
 
@@ -155,7 +148,7 @@ def title(cls) -> str:
         """
         assert cls._title_template is not None, \
             f"{cls.__name__} title template is None"
-        return cls._title_template.format(**cls.__replace_empty_tags(**cls.tag_parameters()))
+        return partial_format(cls._title_template, **cls.__replace_empty_tags(**cls.tag_parameters()))
 
     @classmethod
     def name(cls) -> str:
@@ -164,7 +157,7 @@ def name(cls) -> str:
         """
         assert cls._name_template is not None, \
             f"{cls.__name__} name template is None"
-        return cls._name_template.format(**cls.__replace_empty_tags(**cls.tag_parameters()))
+        return partial_format(cls._name_template, **cls.__replace_empty_tags(**cls.tag_parameters()))
 
     @classmethod
     def description(cls) -> str:
@@ -175,7 +168,8 @@ def description(cls) -> str:
         """
         assert cls._description_template is not None, \
             f"{cls.__name__} description template is None"
-        return cls._description_template.format(**cls.__replace_empty_tags(**cls.tag_parameters()))
+        description = partial_format(cls._description_template, **cls.__replace_empty_tags(**cls.tag_parameters()))
+        return description
 
     @classmethod
     @final
@@ -224,8 +218,9 @@ def __init__(self,
                  primary_header: CplPropertyList = CplPropertyList(),
                  *hdus: Hdu,
                  filename: Optional[Path] = None):
-        if self.__abstract:
-            raise TypeError(f"Tried to instantiate an abstract data item {self.__class__.__qualname__}")
+        if self.__abstract or not self.__regex_pattern.match(self.name()):
+            raise TypeError(f"Tried to instantiate an abstract data item "
+                            f"{self.__class__.__qualname__} for {self.name()}")
 
         # Check if the title is defined
         if self.title() is None:
@@ -234,7 +229,7 @@ def __init__(self,
         if self.name() is None:
             raise NotImplementedError(f"DataItem {self.__class__.__qualname__} has no name defined!")
 
-        # Check if frame_group is defined (if not, this gives rise to strange errors deep within CPL
+        # Check if frame_group is defined (if not, it gives rise to strange errors deep within CPL
         # that you really do not want to deal with)
         if self.frame_group() is None:
             raise NotImplementedError(f"DataItem {self.__class__.__qualname__} has no group defined!")
@@ -245,15 +240,16 @@ def __init__(self,
         # Internal usage marker (for used_frames)
         self._used: bool = False
 
-        self.primary_header = primary_header
-
         self.filename = filename
+        self.primary_header = primary_header
+        # Currently all items are expected to have an empty primary HDU
         self._hdus: dict[str, Hdu] = {}
 
         for index, hdu in enumerate(hdus, start=1):
-            assert hdu.name in self._schema, \
-                (f"Schema for {self.__class__.__qualname__} does not specify HDU '{hdu.name}'. "
-                 f"Accepted extension names are {list(self._schema.keys())}.")
+            if hdu.name not in self._schema:
+                Msg.error(self.__class__.__qualname__,
+                          f"Found a HDU '{hdu.name}', which is not defined by the schema for {self.__class__.__qualname__}. "
+                          f"Accepted extension names are {list(self._schema.keys())}.")
 
             assert hdu.klass == self._schema[hdu.name], \
                 (f"Schema for {self.__class__.__qualname__} specifies that HDU '{hdu.name}' "
@@ -307,6 +303,7 @@ def load(cls,
                         extname = header['EXTNAME'].value
                     except KeyError:
                         try:
+                            # FixMe: this is not reliable but XTENSION is sometimes found in the simulated data
                             extname = header['XTENSION'].value
                         except KeyError:
                             extname = 'PRIMARY'
@@ -350,9 +347,9 @@ def load(cls,
     def load_data(self,
                   extension: int | str) -> Image | Table | None:
         """
-        Actually load the associated data (image or a table).
+        Load the associated data (image or a table).
 
-        This might be expensive and therefore the call is better deferred until actually needed.
+        This might be an expensive operation and therefore the call is better deferred until actually needed.
 
         Parameters
         ----------
@@ -367,20 +364,17 @@ def load_data(self,
         Raises
         ------
         KeyError
-            If requested extension is not available
+            If the requested extension is not available
         """
 
-        if self[extension].klass is None:
-            self[extension].klass = Image
-
         try:
             if self[extension].klass == Image:
                 return self[extension].klass.load(self.filename, cpl.core.Type.FLOAT, self._hdus[extension].extno)
             elif self[extension].klass == Table:
                 return self[extension].klass.load(self.filename, self._hdus[extension].extno)
         except cpl.core.DataNotFoundError as exc:
             Msg.error(self.__class__.__qualname__,
-                      f"Failed to load data from extension '{extension}' in file {self.filename}")
+                      f"Failed to load data from extension '{extension}' from file {self.filename}")
             raise exc
 
     @property
@@ -408,11 +402,10 @@ def _get_file_name(self, override: Optional[str] = None):
     def add_properties(self) -> None:
         """
         Hook for adding custom properties.
-        Currently only adds ESO PRO CATG to every product,
+
+        Currently only adds/replaces ESO PRO CATG to/in every product,
         but derived classes are more than welcome to add their own stuff.
         Do not forget to call super().add_properties() then.
-
-        #ToDo: this should not be called for raws, those do not have a PRO CATG.
         """
         # Some data products actually have FrameGroup RAW because they are
         # input to other recipes (to prevent the cryptic empty set-of-frames
@@ -518,39 +511,6 @@ def as_dict(self) -> dict[str, str]:
             'group': self.frame_group().name,
         }
 
-    def _verify_same_detector_from_header(self) -> None:
-        """
-        Verification for headers, currently disabled
-        """
-        detectors = []
-        for frame in self.frameset:
-            header = cpl.core.PropertyList.load(frame.file, 0)
-            try:
-                det = header['ESO DPR TECH'].value
-                try:
-                    detectors.append({
-                        'IMAGE,LM': '2RG',
-                        'IMAGE,N': 'GEO',
-                        'IFU': 'IFU',
-                    }[det])
-                except KeyError as e:
-                    raise KeyError(f"Invalid detector name! In {frame.file}, ESO DPR TECH is '{det}'") from e
-            except KeyError:
-                Msg.warning(self.__class__.__qualname__, "No detector (ESO DPR TECH) set!")
-
-        # Check if all the raws have the same detector, if not, we have a problem
-        if (detector_count := len(unique := list(set(detectors)))) == 1:
-            self._detector = unique[0]
-            Msg.debug(self.__class__.__qualname__,
-                      f"Detector determined: {self._detector}")
-        elif detector_count == 0:
-            Msg.warning(self.__class__.__qualname__,
-                        "No detectors specified (this is probably fine in skeleton stage)")
-        else:
-            # raise ValueError(f"Frames from more than one detector found: {set(detectors)}!")
-            Msg.warning(self.__class__.__qualname__,
-                        f"Frames from more than one detector found: {unique}!")
-
     @classmethod
     def input_for_recipes(cls) -> Generator['PipelineRecipe', None, None]:
         """
@@ -588,7 +548,7 @@ def _extended_description_line(cls, name: str = None) -> str:
 
         Includes leading space.
         """
-        return f"    {cls.name():39s}{cls.description() or '<no description defined>'}"
+        return f"    {cls.name():49s}{cls.description() or '<no description defined>'}"
 
     def __str__(self):
         return f"{self.name()}"
@@ -630,4 +590,4 @@ def __getitem__(self, item: int | str) -> Hdu:
     def get_name(self, index: int) -> str:
         for name, hdu in self._hdus.items():
             if self._hdus[name].extno == index:
-                return name
+                return name

metisp/pymetis/src/pymetis/classes/inputs/__init__.py

@@ -22,7 +22,6 @@
 from .input import PipelineInput
 from .single import SinglePipelineInput
 from .multiple import MultiplePipelineInput
-from .mixins import PersistenceInputSetMixin, GainMapInputSetMixin, LinearityInputSetMixin, BadPixMapInputSetMixin
 
 from .common import (RawInput,
                      MasterDarkInput,
@@ -51,5 +50,4 @@
     'PinholeTableInput', 'DistortionTableInput', 'LsfKernelInput', 'AtmProfileInput', 'MasterRsrfInput',
     'WavecalInput', 'OptionalInputMixin',
     'LsfKernelInput', 'AtmLineCatInput', 'LaserTableInput', 'SynthTransInput',
-    'PersistenceInputSetMixin', 'GainMapInputSetMixin', 'LinearityInputSetMixin', 'BadPixMapInputSetMixin',
 ]