From 81233aaad3fbe6450b4952483a61e83ad92e36f5 Mon Sep 17 00:00:00 2001 From: Hugo Buddelmeijer Date: Sat, 16 Dec 2023 18:01:10 +0100 Subject: [PATCH 1/2] In place convolution to reduce memory consumption --- scopesim/effects/psfs.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/psfs.py b/scopesim/effects/psfs.py index d7457841..a430acc0 100644 --- a/scopesim/effects/psfs.py +++ b/scopesim/effects/psfs.py @@ -82,7 +82,12 @@ def apply_to(self, obj, **kwargs): # kernel /= np.sum(kernel) # kernel[kernel < 0.] = 0. - image = obj.hdu.data.astype(float) + # This line copied the entire image, while below obj.hdu.data + # is updated anyway. So better do all modifications in place. + #image = obj.hdu.data.astype(float) + if obj.hdu.data.dtype != float: + obj.hdu.data = obj.hdu.data.astype(float) + image = obj.hdu.data # subtract background level before convolving, re-add afterwards bkg_level = pu.get_bkg_level(image, self.meta["bkg_width"]) @@ -104,7 +109,10 @@ def apply_to(self, obj, **kwargs): image[iplane,] - bkg_level[iplane,], kernel[iplane,], mode=mode) - obj.hdu.data = new_image + bkg_level + # This line also copied all the data, better to do the + # modifications in place. + #obj.hdu.data = new_image + bkg_level + obj.hdu.data -= bkg_level # ..todo: careful with which dimensions mean what d_x = new_image.shape[-1] - image.shape[-1] From ae69082f0211484d74cb75263b32952626df4f92 Mon Sep 17 00:00:00 2001 From: Hugo Buddelmeijer Date: Sat, 16 Dec 2023 18:01:33 +0100 Subject: [PATCH 2/2] In place median to reduce memory consumption. --- scopesim/effects/psf_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/psf_utils.py b/scopesim/effects/psf_utils.py index e411be5a..675b397f 100644 --- a/scopesim/effects/psf_utils.py +++ b/scopesim/effects/psf_utils.py @@ -307,8 +307,13 @@ def get_bkg_level(obj, bg_w): mask = np.zeros_like(obj, dtype=bool) if bg_w > 0: mask[:, bg_w:-bg_w, bg_w:-bg_w] = True - bkg_level = np.ma.median(np.ma.masked_array(obj, mask=mask), - axis=(2, 1)).data + + # Using overwrite_input=True reduces memory consumption. + # Nevertheless, the computation of the median here can be + # responsible for more than 40% of the memory consumption. + bkg_masked_array = np.ma.masked_array(obj, mask=mask) + bkg_level_temp = np.ma.median(bkg_masked_array, axis=(2, 1), overwrite_input=True) + bkg_level = bkg_level_temp.data else: raise ValueError("Unsupported dimension:", obj.ndim)