From 12f15301f422489bdd9162eb83b8f42bf123d48a Mon Sep 17 00:00:00 2001 From: Kris Thielemans Date: Tue, 17 Sep 2024 17:21:01 +0100 Subject: [PATCH 1/5] update instructions for creating a new dataset --- SIRF_data_preparation/README.md | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/SIRF_data_preparation/README.md b/SIRF_data_preparation/README.md index 9944050..2d187eb 100644 --- a/SIRF_data_preparation/README.md +++ b/SIRF_data_preparation/README.md @@ -25,3 +25,38 @@ for the Siemens mMR NEMA IQ data (on Zenodo): - `download_Siemens_mMR_NEMA_IQ.py`: download and extract - `prepare_mMR_NEMA_IQ_data.py`: prepare the data (prompts etc) - `BSREM_*.py`: functions with specific settings for a particular data-set + +# Steps to follow to prepare data +If starting from Siemens list-mode data and letting SIRF take care of scatter etc, check for instance [Siemens_mMR_ACR/README.md](steps for Siemens mMR ACR). If pre-prepared data are given, check that naming of all files is correct. KT normally puts all data +in `~/devel/PETRIC/data/` with `datasetname` following convention of `scanner_phantom-name` as others (instructions below and indeed some scripts might assume this location). Change working directory to where data sits and add PETRIC to your python-path, e.g. +``` +PYTHONPATH=~/devel/PETRIC:$PYTHONPATH` +``` + +1. Run initial [data_QC.py](data_QC) + ``` + python ../../SIRF_data_preparation/data_QC.py + ``` + +2. Run [create_initial_images.py](create_initial_images). + ``` + python ../../SIRF_data_preparation/create_initial_images.py --template_image= + ``` + where the template image is one of the given VOIs (it does not matter which one, as they should all have the same geometry). (If you need to create VOIs yourself, you can use `None` or the vendor image). +3. Edit `OSEM_image.hv` to add modality, radionuclide and duration info which got lost (copy from `prompts.hs`) +4. Edit [dataset_settings.py](dataset_settings.py) for subsets (used by our reference reconstructions only, not by participants). +5. Edit [../petric.py](petric.py) for slices to use for creating figures (`DATA_SLICES`). Note that `data_QC.py` outputs centre-of-mass of the VOIs, which can be helpful for this. +6. Run [data_QC.py](data_QC) which should now make more plots. Check VOI alignment etc. + ``` + python ../../SIRF_data_preparation/data_QC.py --dataset= + ``` +7. `cd ../..` +8. `python SIRF_data_preparation/run_OSEM.py ` +9. `python SIRF_data_preparation/run_BSREM.py ` +10. Adapt [plot_BSREM_metrics.py](plot_BSREM_metrics.py) (probably only the ``) and run interactively. +11. Copy the BSREM ` iter_final` to `data//PETRIC/reference_image`, e.g. + ``` + stir_math data//PETRIC/reference_image.hv output//iter_final.hv + ``` +12. `rm output//*ahv`, check its `README.md` etc +13. Transfer to web-server From 8eab8599dd530c89b4af02acf66c098ae12b81bc Mon Sep 17 00:00:00 2001 From: Kris Thielemans Date: Thu, 19 Sep 2024 12:17:31 +0100 Subject: [PATCH 2/5] correct default for VOI file --- SIRF_data_preparation/create_initial_images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIRF_data_preparation/create_initial_images.py b/SIRF_data_preparation/create_initial_images.py index e1d7522..936aeb5 100644 --- a/SIRF_data_preparation/create_initial_images.py +++ b/SIRF_data_preparation/create_initial_images.py @@ -8,7 +8,7 @@ Options: -t , --template_image= filename (relative to ) of image to use - for data sizes [default: PETRIC/VOI_whole_phantom.hv] + for data sizes [default: PETRIC/VOI_whole_object.hv] -s , --xy-size= force xy-size (do not use when using VOIs as init) [default: -1] -S , --subsets= number of subsets [default: 2] -i , --subiterations= number of sub-iterations [default: 14] @@ -16,7 +16,7 @@ # Copyright 2024 Rutherford Appleton Laboratory STFC # Copyright 2024 University College London # Licence: Apache-2.0 -__version__ = '0.1.0' +__version__ = '0.2.0' import logging import math From d89e34802ba13eaa557be2d668f34859e4ad5fda Mon Sep 17 00:00:00 2001 From: Kris Thielemans Date: Thu, 19 Sep 2024 12:18:22 +0100 Subject: [PATCH 3/5] git ignore some more output files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 48e71f3..e71f246 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ data/ orgdata/ +orgdata output/ tmp*/ err*.txt @@ -13,3 +14,6 @@ __pycache__/ *.ahv *.hv *.v +*.csv +# Tensorboard files +events.* \ No newline at end of file From f867014689cfe54a6f5cd0c52e39de40b13650db Mon Sep 17 00:00:00 2001 From: Kris Thielemans Date: Thu, 19 Sep 2024 12:21:34 +0100 Subject: [PATCH 4/5] Use dataset_name to get default path get default location of files from data_utilities --- SIRF_data_preparation/data_QC.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/SIRF_data_preparation/data_QC.py b/SIRF_data_preparation/data_QC.py index cdffe03..9176da6 100644 --- a/SIRF_data_preparation/data_QC.py +++ b/SIRF_data_preparation/data_QC.py @@ -6,7 +6,7 @@ Options: -h, --help - --srcdir= pathname (must have trailing slash!) [default: ./] + --srcdir= pathname. Will default to current directory unless dataset is set --skip_sino_profiles do not plot the sinogram profiles --transverse_slice= idx [default: -1] --coronal_slice= idx [default: -1] @@ -17,7 +17,7 @@ """ # Copyright 2024 University College London # Licence: Apache-2.0 -__version__ = '0.3.0' +__version__ = '0.4.0' import os import os.path @@ -31,6 +31,7 @@ import sirf.STIR as STIR from SIRF_data_preparation.dataset_settings import get_settings +from SIRF_data_preparation.data_utilities import the_data_path STIR.AcquisitionData.set_storage_scheme('memory') @@ -163,11 +164,15 @@ def main(argv=None): slices["sagittal_slice"] = literal_eval(args['--sagittal_slice']) if (dataset): + if srcdir is None: + srcdir = the_data_path(dataset) settings = get_settings(dataset) for key in slices.keys(): if slices[key] == -1 and key in settings.slices: slices[key] = settings.slices[key] - print(slices) + else: + if srcdir is None: + srcdir = os.getcwd() if not skip_sino_profiles: acquired_data = STIR.AcquisitionData(os.path.join(srcdir, 'prompts.hs')) From be0c011f10f35b57ab54e1dc0ba07c77921de5dc Mon Sep 17 00:00:00 2001 From: Kris Thielemans Date: Thu, 19 Sep 2024 12:24:43 +0100 Subject: [PATCH 5/5] use more Python modules Using relative path in some places --- SIRF_data_preparation/README.md | 14 +++++++------- SIRF_data_preparation/Siemens_mMR_ACR/README.md | 8 ++++---- SIRF_data_preparation/Siemens_mMR_ACR/prepare.py | 2 +- .../prepare_mMR_NEMA_IQ_data.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/SIRF_data_preparation/README.md b/SIRF_data_preparation/README.md index 2d187eb..d17d37d 100644 --- a/SIRF_data_preparation/README.md +++ b/SIRF_data_preparation/README.md @@ -35,24 +35,24 @@ PYTHONPATH=~/devel/PETRIC:$PYTHONPATH` 1. Run initial [data_QC.py](data_QC) ``` - python ../../SIRF_data_preparation/data_QC.py + python -m SIRF_data_preparation.data_QC ``` 2. Run [create_initial_images.py](create_initial_images). ``` - python ../../SIRF_data_preparation/create_initial_images.py --template_image= + python -m SIRF_data_preparation.create_initial_images --template_image= ``` where the template image is one of the given VOIs (it does not matter which one, as they should all have the same geometry). (If you need to create VOIs yourself, you can use `None` or the vendor image). 3. Edit `OSEM_image.hv` to add modality, radionuclide and duration info which got lost (copy from `prompts.hs`) 4. Edit [dataset_settings.py](dataset_settings.py) for subsets (used by our reference reconstructions only, not by participants). -5. Edit [../petric.py](petric.py) for slices to use for creating figures (`DATA_SLICES`). Note that `data_QC.py` outputs centre-of-mass of the VOIs, which can be helpful for this. -6. Run [data_QC.py](data_QC) which should now make more plots. Check VOI alignment etc. +5. Edit [../petric.py](petric.py) for slices to use for creating figures (`DATA_SLICES`). Note that `data_QC` outputs centre-of-mass of the VOIs, which can be helpful for this. +6. Run [data_QC](data_QC) which should now make more plots. Check VOI alignment etc. ``` - python ../../SIRF_data_preparation/data_QC.py --dataset= + python -m SIRF_data_preparation.data_QC --dataset= ``` 7. `cd ../..` -8. `python SIRF_data_preparation/run_OSEM.py ` -9. `python SIRF_data_preparation/run_BSREM.py ` +8. `python -m SIRF_data_preparation.run_OSEM ` +9. `python -m SIRF_data_preparation.run_BSREM ` 10. Adapt [plot_BSREM_metrics.py](plot_BSREM_metrics.py) (probably only the ``) and run interactively. 11. Copy the BSREM ` iter_final` to `data//PETRIC/reference_image`, e.g. ``` diff --git a/SIRF_data_preparation/Siemens_mMR_ACR/README.md b/SIRF_data_preparation/Siemens_mMR_ACR/README.md index 8d1562d..ad419f4 100644 --- a/SIRF_data_preparation/Siemens_mMR_ACR/README.md +++ b/SIRF_data_preparation/Siemens_mMR_ACR/README.md @@ -4,7 +4,7 @@ ugly and temporary Steps to follow: 1. `python SIRF_data_preparation/Siemens_mMR_ACR/download.py` 2. `python SIRF_data_preparation/Siemens_mMR_ACR/prepare.py` (As there is no useful mumap, this will **not** do attenuation and scatter estimation) -3. `python SIRF_data_preparation/create_initial_images.py data/Siemens_mMR_ACR/final --template_image=None` (Reconstruct at full FOV size) +3. `python SIRF_data_preparation.create_initial_images data/Siemens_mMR_ACR/final --template_image=None` (Reconstruct at full FOV size) 4. `mv data/Siemens_mMR_ACR/final/OSEM_image.* data/Siemens_mMR_ACR/processing` (this is really an NAC image and ideally would be renamed) 5. `python SIRF_data_preparation/Siemens_mMR_ACR/register_mumap.py` (output is orgdata/Siemens_mMR_ACR/processing/reg_mumap.hv etc) 6. However, registration failed, so this needs a manual intervention step: @@ -24,9 +24,9 @@ Steps to follow: stir_math --accumulate orgdata/Siemens_mMR_ACR/processing/reg_mumap.hv orgdata/Siemens_mMR_ACR/output/ACR_hardware-to-STIR.nii.gz ``` 7. 11. `python SIRF_data_preparation/run_OSEM.py Siemens_mMR_ACR`--end 200` (As there is now a useful mumap, this will now do attenuation and scatter estimation) -8. `python SIRF_data_preparation/create_initial_images.py data/Siemens_mMR_ACR --template_image=../../orgdata/Siemens_mMR_ACR/output/sampling_masks/acr-all-sampling-0-2mm_dipy.nii` -9. `python SIRF_data_preparation/data_QC.py --srcdir='data/Siemens_mMR_ACR' --transverse_slice=99` -10. edit `SIRF_data_prepatation/dataset_settings.py` for subsets etc. edit `OSEM_image.hv` to add modality, radionuclide and duration info which got lost. +8. `python -m SIRF_data_preparation.create_initial_images data/Siemens_mMR_ACR --template_image=../../orgdata/Siemens_mMR_ACR/output/sampling_masks/acr-all-sampling-0-2mm_dipy.nii` +9. `python -m SIRF_data_preparation.data_QC --srcdir='data/Siemens_mMR_ACR' --transverse_slice=99` +10. edit `petric.py` and `SIRF_data_preparation/dataset_settings.py` for subsets etc. edit `OSEM_image.hv` to add modality, radionuclide and duration info which got lost. 11. `python SIRF_data_preparation/Siemens_mMR_ACR/VOI_prep.py` 12. `python SIRF_data_preparation/run_OSEM.py Siemens_mMR_ACR` 13. `python SIRF_data_preparation/run_BSREM.py Siemens_mMR_ACR` diff --git a/SIRF_data_preparation/Siemens_mMR_ACR/prepare.py b/SIRF_data_preparation/Siemens_mMR_ACR/prepare.py index 7cce5c5..884b1c7 100644 --- a/SIRF_data_preparation/Siemens_mMR_ACR/prepare.py +++ b/SIRF_data_preparation/Siemens_mMR_ACR/prepare.py @@ -2,7 +2,7 @@ import logging import os -from SIRF_data_preparation.data_utilities import prepare_challenge_Siemens_data, the_data_path, the_orgdata_path +from ..data_utilities import prepare_challenge_Siemens_data, the_data_path, the_orgdata_path scanID = 'Siemens_mMR_ACR' diff --git a/SIRF_data_preparation/Siemens_mMR_NEMA_IQ/prepare_mMR_NEMA_IQ_data.py b/SIRF_data_preparation/Siemens_mMR_NEMA_IQ/prepare_mMR_NEMA_IQ_data.py index 99464c1..c88ec51 100644 --- a/SIRF_data_preparation/Siemens_mMR_NEMA_IQ/prepare_mMR_NEMA_IQ_data.py +++ b/SIRF_data_preparation/Siemens_mMR_NEMA_IQ/prepare_mMR_NEMA_IQ_data.py @@ -2,7 +2,7 @@ import logging import os -from .data_utilities import prepare_challenge_Siemens_data, the_data_path +from ..data_utilities import prepare_challenge_Siemens_data, the_data_path, the_orgdata_path this_directory = os.path.dirname(__file__) repo_directory = os.path.dirname(this_directory)