diff --git a/docs/examples/caiman_notebooks/__init__.py b/docs/examples/caiman_notebooks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/docs/examples/caiman_notebooks/basic_caiman_notebook.ipynb b/docs/examples/caiman_notebooks/basic_caiman_notebook.ipynb index 5efaf4a..c30de59 100644 --- a/docs/examples/caiman_notebooks/basic_caiman_notebook.ipynb +++ b/docs/examples/caiman_notebooks/basic_caiman_notebook.ipynb @@ -27,15 +27,15 @@ { "metadata": {}, "cell_type": "code", - "outputs": [], - "execution_count": null, "source": [ "!mkdir -p ./conda_env\n", "\n", "!wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -O Miniforge3.sh\n", "!bash Miniforge3.sh -b -p ./conda_env/miniforge3" ], - "id": "82300e0095d28b33" + "id": "82300e0095d28b33", + "outputs": [], + "execution_count": null }, { "metadata": {}, @@ -135,7 +135,7 @@ "metadata": {}, "cell_type": "code", "source": "import caiman", - "id": "291935e9df7c9ab8", + "id": "20a56778ced6d6df", "outputs": [], "execution_count": null }, @@ -162,13 +162,21 @@ "id": "a909f9e56d38a488", "outputs": [], "execution_count": null + }, + { + "metadata": {}, + "cell_type": "code", + "source": "pipeline.caiman.cnmf()", + "id": "9922b1042346aff2", + "outputs": [], + "execution_count": null } ], "metadata": { "kernelspec": { - "display_name": "Python (caiman)", + "display_name": "Python (caiman-env)", "language": "python", - "name": "caiman" + "name": "caiman-env" }, "language_info": { "codemirror_mode": { diff --git a/src/ci_pipe/modules/caiman_module.py b/src/ci_pipe/modules/caiman_module.py index 1977d0e..b525220 100644 --- a/src/ci_pipe/modules/caiman_module.py +++ b/src/ci_pipe/modules/caiman_module.py @@ -5,6 +5,8 @@ class CaimanModule: MOTION_CORRECTION_STEP = "Caiman Motion Correction" MOTION_CORRECTION_VIDEOS_SUFFIX = "MC" + CNMF_STEP = "Caiman Constrained Non-negative Matrix Factorization" + CNMF_VIDEOS_SUFFIX = "CNMF" def __init__(self, caiman, ci_pipe): if caiman is None: @@ -58,3 +60,174 @@ def motion_correction( output.append({'ids': input_data['ids'], 'value': tif_output_path}) return {"videos-tif": output} + + @step(CNMF_STEP) + def cnmf( + self, + inputs, + *, + caiman_n_processes=None, + caiman_k=5, + caiman_gSig=[4, 4], + caiman_gSiz=None, + caiman_merge_thresh=0.8, + caiman_p=2, + caiman_dview=None, + caiman_Ain=None, + caiman_Cin=None, + caiman_b_in=None, + caiman_f_in=None, + caiman_do_merge=True, + caiman_ssub=2, + caiman_tsub=2, + caiman_p_ssub=1, + caiman_p_tsub=1, + caiman_method_init='greedy_roi', + caiman_alpha_snmf=0.5, + caiman_rf=None, + caiman_stride=None, + caiman_memory_fact=1, + caiman_gnb=1, + caiman_nb_patch=1, + caiman_only_init_patch=False, + caiman_method_deconvolution='oasis', + caiman_n_pixels_per_process=4000, + caiman_block_size_temp=5000, + caiman_num_blocks_per_run_temp=20, + caiman_num_blocks_per_run_spat=20, + caiman_check_nan=True, + caiman_skip_refinement=False, + caiman_normalize_init=True, + caiman_options_local_NMF=None, + caiman_minibatch_shape=100, + caiman_minibatch_suff_stat=3, + caiman_update_num_comps=True, + caiman_rval_thr=0.9, + caiman_thresh_fitness_delta=-20, + caiman_thresh_fitness_raw=None, + caiman_thresh_overlap=.5, + caiman_batch_update_suff_stat=False, + caiman_s_min=None, + caiman_remove_very_bad_comps=False, + caiman_border_pix=0, + caiman_low_rank_background=True, + caiman_update_background_components=True, + caiman_rolling_sum=True, + caiman_rolling_length=100, + caiman_min_corr=.85, + caiman_min_pnr=20, + caiman_ring_size_factor=1.5, + caiman_center_psf=False, + caiman_use_dense=True, + caiman_deconv_flag=True, + caiman_simultaneously=False, + caiman_n_refit=0, + caiman_del_duplicates=False, + caiman_N_samples_exceptionality=None, + caiman_max_num_added=3, + caiman_min_num_trial=2, + caiman_thresh_CNN_noisy=0.5, + caiman_fr=30, + caiman_decay_time=0.4, + caiman_min_SNR=2.5, + caiman_ssub_B=2, + caiman_init_iter=2, + caiman_sniper_mode=False, + caiman_use_peak_max=False, + caiman_test_both=False, + caiman_expected_comps=500, + caiman_params=None + ): + output = [] + output_dir = self._ci_pipe.create_output_directory_for_next_step(self.CNMF_STEP) + + for input_data in inputs('videos-tif'): + cnmf_model = self._caiman.source_extraction.cnmf.CNMF( + n_processes=caiman_n_processes, + k=caiman_k, + gSig=caiman_gSig, + gSiz=caiman_gSiz, + merge_thresh=caiman_merge_thresh, + p=caiman_p, + dview=caiman_dview, + Ain=caiman_Ain, + Cin=caiman_Cin, + b_in=caiman_b_in, + f_in=caiman_f_in, + do_merge=caiman_do_merge, + ssub=caiman_ssub, + tsub=caiman_tsub, + p_ssub=caiman_p_ssub, + p_tsub=caiman_p_tsub, + method_init=caiman_method_init, + alpha_snmf=caiman_alpha_snmf, + rf=caiman_rf, + stride=caiman_stride, + memory_fact=caiman_memory_fact, + gnb=caiman_gnb, + nb_patch=caiman_nb_patch, + only_init_patch=caiman_only_init_patch, + method_deconvolution=caiman_method_deconvolution, + n_pixels_per_process=caiman_n_pixels_per_process, + block_size_temp=caiman_block_size_temp, + num_blocks_per_run_temp=caiman_num_blocks_per_run_temp, + num_blocks_per_run_spat=caiman_num_blocks_per_run_spat, + check_nan=caiman_check_nan, + skip_refinement=caiman_skip_refinement, + normalize_init=caiman_normalize_init, + options_local_NMF=caiman_options_local_NMF, + minibatch_shape=caiman_minibatch_shape, + minibatch_suff_stat=caiman_minibatch_suff_stat, + update_num_comps=caiman_update_num_comps, + rval_thr=caiman_rval_thr, + thresh_fitness_delta=caiman_thresh_fitness_delta, + thresh_fitness_raw=caiman_thresh_fitness_raw, + thresh_overlap=caiman_thresh_overlap, + batch_update_suff_stat=caiman_batch_update_suff_stat, + s_min=caiman_s_min, + remove_very_bad_comps=caiman_remove_very_bad_comps, + border_pix=caiman_border_pix, + low_rank_background=caiman_low_rank_background, + update_background_components=caiman_update_background_components, + rolling_sum=caiman_rolling_sum, + rolling_length=caiman_rolling_length, + min_corr=caiman_min_corr, + min_pnr=caiman_min_pnr, + ring_size_factor=caiman_ring_size_factor, + center_psf=caiman_center_psf, + use_dense=caiman_use_dense, + deconv_flag=caiman_deconv_flag, + simultaneously=caiman_simultaneously, + n_refit=caiman_n_refit, + del_duplicates=caiman_del_duplicates, + N_samples_exceptionality=caiman_N_samples_exceptionality, + max_num_added=caiman_max_num_added, + min_num_trial=caiman_min_num_trial, + thresh_CNN_noisy=caiman_thresh_CNN_noisy, + fr=caiman_fr, + decay_time=caiman_decay_time, + min_SNR=caiman_min_SNR, + ssub_B=caiman_ssub_B, + init_iter=caiman_init_iter, + sniper_mode=caiman_sniper_mode, + use_peak_max=caiman_use_peak_max, + test_both=caiman_test_both, + expected_comps=caiman_expected_comps, + params=caiman_params + ) + + # Note: Values for this algorithm are changed within estimates object of cnmf model + memmapped_movie = self._caiman.load(input_data['value']) + cnmf_model.fit(images=memmapped_movie) + + hdf5_output_path = self._ci_pipe.make_output_file_path( + input_data['value'], + output_dir, + self.CNMF_VIDEOS_SUFFIX, + ext="hdf5", + ) + + cnmf_model.save(hdf5_output_path) + output.append({'ids': input_data['ids'], 'value': hdf5_output_path}) + + return {"files-hdf5": output} diff --git a/src/external_dependencies/caiman/in_memory_caiman.py b/src/external_dependencies/caiman/in_memory_caiman.py index 5949a0e..9782bcf 100644 --- a/src/external_dependencies/caiman/in_memory_caiman.py +++ b/src/external_dependencies/caiman/in_memory_caiman.py @@ -1,11 +1,13 @@ -from external_dependencies.caiman.mocked_motion_correction_submodule import MockMotionCorrectionModule +from external_dependencies.caiman.mocked_motion_correction_submodule import MockedMotionCorrectionSubModule from external_dependencies.caiman.mocked_movie import MockedMovie +from external_dependencies.caiman.mocked_source_extraction import MockedSourceExtractionModule class InMemoryCaiman: def __init__(self, file_system=None): self._file_system = file_system - self.motion_correction = MockMotionCorrectionModule(file_system) + self.motion_correction = MockedMotionCorrectionSubModule(file_system) + self.source_extraction = MockedSourceExtractionModule(file_system) def load(self, fname): return MockedMovie(fname, self._file_system) \ No newline at end of file diff --git a/src/external_dependencies/caiman/mocked_cnmf_submodule.py b/src/external_dependencies/caiman/mocked_cnmf_submodule.py new file mode 100644 index 0000000..8e290cc --- /dev/null +++ b/src/external_dependencies/caiman/mocked_cnmf_submodule.py @@ -0,0 +1,90 @@ +class MockedCNMFModule: + def __init__(self, file_system): + self._file_system = file_system + + def CNMF( + self, + n_processes=None, + k=5, + gSig=[4, 4], + gSiz=None, + merge_thresh=0.8, + p=2, + dview=None, + Ain=None, + Cin=None, + b_in=None, + f_in=None, + do_merge=True, + ssub=2, + tsub=2, + p_ssub=1, + p_tsub=1, + method_init='greedy_roi', + alpha_snmf=0.5, + rf=None, + stride=None, + memory_fact=1, + gnb=1, + nb_patch=1, + only_init_patch=False, + method_deconvolution='oasis', + n_pixels_per_process=4000, + block_size_temp=5000, + num_blocks_per_run_temp=20, + num_blocks_per_run_spat=20, + check_nan=True, + skip_refinement=False, + normalize_init=True, + options_local_NMF=None, + minibatch_shape=100, + minibatch_suff_stat=3, + update_num_comps=True, + rval_thr=0.9, + thresh_fitness_delta=-20, + thresh_fitness_raw=None, + thresh_overlap=.5, + batch_update_suff_stat=False, + s_min=None, + remove_very_bad_comps=False, + border_pix=0, + low_rank_background=True, + update_background_components=True, + rolling_sum=True, + rolling_length=100, + min_corr=.85, + min_pnr=20, + ring_size_factor=1.5, + center_psf=False, + use_dense=True, + deconv_flag=True, + simultaneously=False, + n_refit=0, + del_duplicates=False, + N_samples_exceptionality=None, + max_num_added=3, + min_num_trial=2, + thresh_CNN_noisy=0.5, + fr=30, + decay_time=0.4, + min_SNR=2.5, + ssub_B=2, + init_iter=2, + sniper_mode=False, + use_peak_max=False, + test_both=False, + expected_comps=500, + params=None + ): + return MockedCNMFModule( + file_system=self._file_system + ) + + def fit(self, images=None): + pass + + def estimates(self): + pass + + def save(self, path): + self._file_system.write(path, "") diff --git a/src/external_dependencies/caiman/mocked_motion_correction_submodule.py b/src/external_dependencies/caiman/mocked_motion_correction_submodule.py index 467219b..e5254ee 100644 --- a/src/external_dependencies/caiman/mocked_motion_correction_submodule.py +++ b/src/external_dependencies/caiman/mocked_motion_correction_submodule.py @@ -1,7 +1,7 @@ from external_dependencies.caiman.mocked_motion_correct import MockedMotionCorrect -class MockMotionCorrectionModule: +class MockedMotionCorrectionSubModule: def __init__(self, file_system): self._file_system = file_system diff --git a/src/external_dependencies/caiman/mocked_source_extraction.py b/src/external_dependencies/caiman/mocked_source_extraction.py new file mode 100644 index 0000000..95f6f1f --- /dev/null +++ b/src/external_dependencies/caiman/mocked_source_extraction.py @@ -0,0 +1,7 @@ +from external_dependencies.caiman.mocked_cnmf_submodule import MockedCNMFModule + + +class MockedSourceExtractionModule: + def __init__(self, file_system=None): + self._file_system = file_system + self.cnmf = MockedCNMFModule(file_system) \ No newline at end of file diff --git a/tests/caiman_test.py b/tests/caiman_test.py index 9f484cc..4667025 100644 --- a/tests/caiman_test.py +++ b/tests/caiman_test.py @@ -42,6 +42,40 @@ def test_02_a_pipeline_with_caiman_can_execute_motion_correction_algorithm(self) self._file_system, ) + + def test_03_a_pipeline_with_caiman_can_execute_constrained_non_negative_matrix_factorization_algorithm(self): + # Given + pipeline_input = 'input_dir' + self._file_system.makedirs(pipeline_input) + self._file_system.write(f'{pipeline_input}/file1.tif', '') + pipeline = CIPipe.with_videos_from_directory( + pipeline_input, + file_system=self._file_system, + caiman=InMemoryCaiman(self._file_system) + ) + pipeline.caiman.motion_correction() + + # When + pipeline.caiman.cnmf() + + # Then + self._assert_output_files( + pipeline, + 'videos-tif', + [ + 'output/Main Branch - Step 1 - Caiman Motion Correction/file1-MC.tif', + ], + self._file_system, + ) + self._assert_output_files( + pipeline, + 'files-hdf5', + [ + 'output/Main Branch - Step 2 - Caiman Constrained Non-negative Matrix Factorization/file1-MC-CNMF.hdf5', + ], + self._file_system, + ) + def _assert_output_files(self, pipeline, key, expected_paths, file_system): output = pipeline.output(key) self.assertEqual(len(output), len(expected_paths))