From f1066818b50f585260844ba735d6315ba1f34fe8 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Thu, 24 Oct 2024 12:48:25 +0200 Subject: [PATCH 001/157] instllation using uv --- installation_tips/README.md | 75 ++++++++++++++++------ installation_tips/check_your_install.py | 28 +++++--- installation_tips/requirements_rolling.txt | 16 +++++ installation_tips/requirements_stable.txt | 13 ++++ 4 files changed, 104 insertions(+), 28 deletions(-) create mode 100644 installation_tips/requirements_rolling.txt create mode 100644 installation_tips/requirements_stable.txt diff --git a/installation_tips/README.md b/installation_tips/README.md index 1257cb6786..87684d30b8 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -1,36 +1,53 @@ ## Installation tips -If you are not (yet) an expert in Python installations (conda vs pip, mananging environements, etc.), -here we propose a simple recipe to install `spikeinterface` and several sorters inside an anaconda -environment for windows/mac users. +If you are not (yet) an expert in Python installations, a main difficulty is choosing the installation procedure. +The main ideas you need to know before starting: + * python itself can be distributed and installed many many ways. + * python itself do not contain so many features for scientifique computing you need to install "packages". + Numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. "uv" + * installing package can be distributed and installed several ways (pip, conda, uv, mamba, ...) + * installing many packages at once is challenging (because of the depenency graph) so you need to do it in an "isolated environement" + to not destroy any previous installation. You need to see an "environement" as a sub installtion in dedicated folder. + +Choosing the installator + a environement manager + a package installer is a nightmare for beginners. +The main options are: + * use "anaconda" that do everything. The most popular but bad idea because : ultra slow and agressive licensing (not free anymore) + * use python from the system or python.org + venv + pip : good idea for linux users. + * use "uv" : a new, fast and simple. We recommand this for beginners on evry os. + +Here we propose a steps by step recipe for beginers based on "uv". +We used to propose here a solution based on anaconda. It is kept here for a while but we do not recommand it anymore. + This environment will install: * spikeinterface full option * spikeinterface-gui - * phy - * tridesclous + * kilosort4 Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from source. ### Quick installation -Steps: +1. On macOS and Linux. Open a terminal and do + `$ curl -LsSf https://astral.sh/uv/install.sh | sh` +1. On windows. Open a powershell and do + `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` +2. exit session and log again. +3. Download with right click + save this file corresponding in "Documents" folder: + * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) +4. open terminal or powershell +5. `uv venv si_env --python 3.11` +6. `source `source si_env/bin/activate` (you should have `(si_env)` in your terminal) +7. `uv pip install -r Documents/requirements_stable.txt` -1. Download anaconda individual edition [here](https://www.anaconda.com/download) -2. Run the installer. Check the box “Add anaconda3 to my Path environment variable”. It makes life easier for beginners. -3. Download with right click + save the file corresponding to your OS, and put it in "Documents" folder - * [`full_spikeinterface_environment_windows.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_windows.yml) - * [`full_spikeinterface_environment_mac.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_mac.yml) -4. Then open the "Anaconda Command Prompt" (if Windows, search in your applications) or the Terminal (for Mac users) -5. If not in the "Documents" folder type `cd Documents` -6. Then run this depending on your OS: - * `conda env create --file full_spikeinterface_environment_windows.yml` - * `conda env create --file full_spikeinterface_environment_mac.yml` +More detail on [uv here](https://github.com/astral-sh/uv). -Done! Before running a spikeinterface script you will need to "select" this "environment" with `conda activate si_env`. +## Installing before release -Note for **linux** users : this conda recipe should work but we recommend strongly to use **pip + virtualenv**. +Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, python-neo, sortingview). +We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `requirements_rolling.txt` file to create the environment. This will install the packages of the ecosystem from source. +This is a good way to test if patch fix your issue. ### Check the installation @@ -62,6 +79,28 @@ This script tests the following: * opening the spikeinterface-gui * exporting to Phy +### Legacy installation using anaconda (not recomanded) + +Steps: + +1. Download anaconda individual edition [here](https://www.anaconda.com/download) +2. Run the installer. Check the box “Add anaconda3 to my Path environment variable”. It makes life easier for beginners. +3. Download with right click + save the file corresponding to your OS, and put it in "Documents" folder + * [`full_spikeinterface_environment_windows.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_windows.yml) + * [`full_spikeinterface_environment_mac.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_mac.yml) +4. Then open the "Anaconda Command Prompt" (if Windows, search in your applications) or the Terminal (for Mac users) +5. If not in the "Documents" folder type `cd Documents` +6. Then run this depending on your OS: + * `conda env create --file full_spikeinterface_environment_windows.yml` + * `conda env create --file full_spikeinterface_environment_mac.yml` + + +Done! Before running a spikeinterface script you will need to "select" this "environment" with `conda activate si_env`. + +Note for **linux** users : this conda recipe should work but we recommend strongly to use **pip + virtualenv**. + + + ## Installing before release diff --git a/installation_tips/check_your_install.py b/installation_tips/check_your_install.py index f3f80961e8..64ab9cca8a 100644 --- a/installation_tips/check_your_install.py +++ b/installation_tips/check_your_install.py @@ -4,6 +4,9 @@ import shutil import argparse + +job_kwargs = dict(n_jobs=-1, progress_bar=True, chunk_duration="1s") + def check_import_si(): import spikeinterface as si @@ -13,15 +16,20 @@ def check_import_si_full(): def _create_recording(): import spikeinterface.full as si - rec, sorting = si.toy_example(num_segments=1, duration=200, seed=1, num_channels=16, num_columns=2) - rec.save(folder='./toy_example_recording') + rec, sorting = si.generate_ground_truth_recording( + durations=[200.], + sampling_frequency=30_000., + num_channels=16, + num_units=10, + seed=2205 + ) + rec.save(folder='./toy_example_recording', **job_kwargs) def _run_one_sorter_and_analyzer(sorter_name): - job_kwargs = dict(n_jobs=-1, progress_bar=True, chunk_duration="1s") import spikeinterface.full as si recording = si.load_extractor('./toy_example_recording') - sorting = si.run_sorter(sorter_name, recording, output_folder=f'./sorter_with_{sorter_name}', verbose=False) + sorting = si.run_sorter(sorter_name, recording, folder=f'./sorter_with_{sorter_name}', verbose=False) sorting_analyzer = si.create_sorting_analyzer(sorting, recording, format="binary_folder", folder=f"./analyzer_with_{sorter_name}", @@ -36,12 +44,12 @@ def _run_one_sorter_and_analyzer(sorter_name): sorting_analyzer.compute("quality_metrics", metric_names=["snr", "firing_rate"]) -def run_tridesclous(): - _run_one_sorter_and_analyzer('tridesclous') - def run_tridesclous2(): _run_one_sorter_and_analyzer('tridesclous2') +def run_kilosort4(): + _run_one_sorter_and_analyzer('kilosort4') + def open_sigui(): @@ -75,10 +83,10 @@ def _clean(): # clean folders = [ "./toy_example_recording", - "./sorter_with_tridesclous", - "./analyzer_with_tridesclous", "./sorter_with_tridesclous2", "./analyzer_with_tridesclous2", + "./sorter_with_kilosort4", + "./analyzer_with_kilosort4", "./phy_example" ] for folder in folders: @@ -100,8 +108,8 @@ def _clean(): steps = [ ('Import spikeinterface', check_import_si), ('Import spikeinterface.full', check_import_si_full), - ('Run tridesclous', run_tridesclous), ('Run tridesclous2', run_tridesclous2), + ('Run kilosort4', run_kilosort4), ] # backwards logic because default is True for end-user diff --git a/installation_tips/requirements_rolling.txt b/installation_tips/requirements_rolling.txt new file mode 100644 index 0000000000..d35009f1ea --- /dev/null +++ b/installation_tips/requirements_rolling.txt @@ -0,0 +1,16 @@ +numpy<2 +jupyterlab +PySide6<6.8 +numba +zarr +hdbscan +pyqtgraph +ipywidgets +ipympl +ephyviewer +https://github.com/NeuralEnsemble/python-neo/archive/master.zip +https://github.com/SpikeInterface/probeinterface/archive/main.zip +https://github.com/SpikeInterface/spikeinterface/archive/main.zip[full,widgets] +https://github.com/SpikeInterface/spikeinterface-gui/archive/main.zip +https://github.com/magland/sortingview/archive/main.zip +kilosort diff --git a/installation_tips/requirements_stable.txt b/installation_tips/requirements_stable.txt new file mode 100644 index 0000000000..3939e10562 --- /dev/null +++ b/installation_tips/requirements_stable.txt @@ -0,0 +1,13 @@ +numpy<2 +jupyterlab +PySide6<6.8 +numba +zarr +hdbscan +pyqtgraph +ipywidgets +ipympl +ephyviewer +spikeinterface[full,widgets] +spikeinterface-gui +kilosort From 888c7851a1c7fa96538da43b244a9ede8cddbc28 Mon Sep 17 00:00:00 2001 From: Garcia Samuel Date: Thu, 24 Oct 2024 17:20:48 +0200 Subject: [PATCH 002/157] Merci Zach and Joe Co-authored-by: Zach McKenzie <92116279+zm711@users.noreply.github.com> Co-authored-by: Joe Ziminski <55797454+JoeZiminski@users.noreply.github.com> --- installation_tips/README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index 87684d30b8..2cb605c442 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -3,20 +3,20 @@ If you are not (yet) an expert in Python installations, a main difficulty is choosing the installation procedure. The main ideas you need to know before starting: * python itself can be distributed and installed many many ways. - * python itself do not contain so many features for scientifique computing you need to install "packages". - Numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. "uv" - * installing package can be distributed and installed several ways (pip, conda, uv, mamba, ...) - * installing many packages at once is challenging (because of the depenency graph) so you need to do it in an "isolated environement" - to not destroy any previous installation. You need to see an "environement" as a sub installtion in dedicated folder. + * python itself does not contain so many features for scientific computing you need to install "packages". + numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. "uv" + * packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) + * installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environement" + to not destroy any previous installation. You need to see an "environment" as a sub installation in a dedicated folder. -Choosing the installator + a environement manager + a package installer is a nightmare for beginners. +Choosing the installer + an environment manager + a package installer is a nightmare for beginners. The main options are: - * use "anaconda" that do everything. The most popular but bad idea because : ultra slow and agressive licensing (not free anymore) + * use "anaconda", which does everything. The most popular but bad idea because : ultra slow and aggressive licensing (not always free anymore) * use python from the system or python.org + venv + pip : good idea for linux users. - * use "uv" : a new, fast and simple. We recommand this for beginners on evry os. + * use "uv" : a new, fast and simple package manager. We recommend this for beginners on every os. -Here we propose a steps by step recipe for beginers based on "uv". -We used to propose here a solution based on anaconda. It is kept here for a while but we do not recommand it anymore. +Here we propose a step by step recipe for beginers based on "uv". +We used to recommend installing with anaconda. It will be kept here for a while but we do not recommend it anymore. This environment will install: @@ -33,7 +33,7 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so 1. On windows. Open a powershell and do `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` 2. exit session and log again. -3. Download with right click + save this file corresponding in "Documents" folder: +3. Download with right click and save this file corresponding in "Documents" folder: * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell 5. `uv venv si_env --python 3.11` @@ -47,7 +47,7 @@ More detail on [uv here](https://github.com/astral-sh/uv). Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, python-neo, sortingview). We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `requirements_rolling.txt` file to create the environment. This will install the packages of the ecosystem from source. -This is a good way to test if patch fix your issue. +This is a good way to test if a patch fixes your issue. ### Check the installation @@ -79,7 +79,7 @@ This script tests the following: * opening the spikeinterface-gui * exporting to Phy -### Legacy installation using anaconda (not recomanded) +### Legacy installation using anaconda (not recommended) Steps: From 971ce4577e6257c6a343ee971cb30720a7257cf5 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Thu, 24 Oct 2024 17:48:00 +0200 Subject: [PATCH 003/157] updates --- installation_tips/README.md | 40 +++++++++---------- ...spikeinterface_environment_linux_dandi.yml | 2 +- .../full_spikeinterface_environment_mac.yml | 2 +- ...einterface_environment_rolling_updates.yml | 2 +- ...ull_spikeinterface_environment_windows.yml | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index 2cb605c442..f7b8ed4445 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -4,18 +4,21 @@ If you are not (yet) an expert in Python installations, a main difficulty is cho The main ideas you need to know before starting: * python itself can be distributed and installed many many ways. * python itself does not contain so many features for scientific computing you need to install "packages". - numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. "uv" + numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. * packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) * installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environement" to not destroy any previous installation. You need to see an "environment" as a sub installation in a dedicated folder. Choosing the installer + an environment manager + a package installer is a nightmare for beginners. The main options are: - * use "anaconda", which does everything. The most popular but bad idea because : ultra slow and aggressive licensing (not always free anymore) - * use python from the system or python.org + venv + pip : good idea for linux users. * use "uv" : a new, fast and simple package manager. We recommend this for beginners on every os. + * use "anaconda", which does everything. The most popular but theses days it is becoming + a bad idea because : ultra slow by default and aggressive licensing by default (not always free anymore). + You need to play with "community channels" to make it free again, which is too complicated for beginners. + Do not go this way. + * use python from the system or python.org + venv + pip : good idea for linux users. -Here we propose a step by step recipe for beginers based on "uv". +Here we propose a step by step recipe for beginers based on **"uv"**. We used to recommend installing with anaconda. It will be kept here for a while but we do not recommend it anymore. @@ -26,7 +29,7 @@ This environment will install: Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from source. -### Quick installation +### Quick installation using "uv" (recommended) 1. On macOS and Linux. Open a terminal and do `$ curl -LsSf https://astral.sh/uv/install.sh | sh` @@ -37,7 +40,7 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell 5. `uv venv si_env --python 3.11` -6. `source `source si_env/bin/activate` (you should have `(si_env)` in your terminal) +6. `source si_env/bin/activate` (you should have `(si_env)` in your terminal) 7. `uv pip install -r Documents/requirements_stable.txt` @@ -57,29 +60,26 @@ If you want to test the spikeinterface install you can: 1. Download with right click + save the file [`check_your_install.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/check_your_install.py) and put it into the "Documents" folder - -2. Open the Anaconda Command Prompt (Windows) or Terminal (Mac) -3. If not in your "Documents" folder type `cd Documents` -4. Run this: - ``` - conda activate si_env - python check_your_install.py - ``` -5. If a windows user to clean-up you will also need to right click + save [`cleanup_for_windows.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/cleanup_for_windows.py) +2. Open the CMD (Windows) or Terminal (Mac/Linux) +3. Acticate your is_env : `source si_env/bin/activate` +4. Go to your "Documents" folder with `cd Documents` or the place where you downloaded the `check_your_install.py` +5. Run this: + `python check_your_install.py` +6. If a windows user to clean-up you will also need to right click + save [`cleanup_for_windows.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/cleanup_for_windows.py) Then transfer `cleanup_for_windows.py` into your "Documents" folder. Finally run : ``` python cleanup_for_windows.py ``` -This script tests the following: +This script tests the following steps: * importing spikeinterface - * running tridesclous - * running spyking-circus (not on mac) - * running herdingspikes (not on windows) + * running tridesclous2 + * running kilosort4 * opening the spikeinterface-gui * exporting to Phy -### Legacy installation using anaconda (not recommended) + +### Legacy installation using anaconda (not recommended anymore) Steps: diff --git a/installation_tips/full_spikeinterface_environment_linux_dandi.yml b/installation_tips/full_spikeinterface_environment_linux_dandi.yml index 5f276b0d20..197109cd9c 100755 --- a/installation_tips/full_spikeinterface_environment_linux_dandi.yml +++ b/installation_tips/full_spikeinterface_environment_linux_dandi.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.11 - pip - - numpy + - numpy<2 - scipy - joblib - tqdm diff --git a/installation_tips/full_spikeinterface_environment_mac.yml b/installation_tips/full_spikeinterface_environment_mac.yml index 2522fda78d..1df2c6878c 100755 --- a/installation_tips/full_spikeinterface_environment_mac.yml +++ b/installation_tips/full_spikeinterface_environment_mac.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.11 - pip - - numpy + - numpy<2 - scipy - joblib - tqdm diff --git a/installation_tips/full_spikeinterface_environment_rolling_updates.yml b/installation_tips/full_spikeinterface_environment_rolling_updates.yml index b4479aa20f..a2b51546f1 100644 --- a/installation_tips/full_spikeinterface_environment_rolling_updates.yml +++ b/installation_tips/full_spikeinterface_environment_rolling_updates.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.11 - pip - - numpy + - numpy<2 - scipy - joblib - tqdm diff --git a/installation_tips/full_spikeinterface_environment_windows.yml b/installation_tips/full_spikeinterface_environment_windows.yml index 2522fda78d..1df2c6878c 100755 --- a/installation_tips/full_spikeinterface_environment_windows.yml +++ b/installation_tips/full_spikeinterface_environment_windows.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.11 - pip - - numpy + - numpy<2 - scipy - joblib - tqdm From e298384832d81ad44b026e5cd4e8d7af4a231429 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Fri, 25 Oct 2024 10:15:00 +0200 Subject: [PATCH 004/157] work around --- installation_tips/requirements_stable.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation_tips/requirements_stable.txt b/installation_tips/requirements_stable.txt index 3939e10562..d77343fbba 100644 --- a/installation_tips/requirements_stable.txt +++ b/installation_tips/requirements_stable.txt @@ -1,3 +1,4 @@ +spikeinterface[full,widgets] numpy<2 jupyterlab PySide6<6.8 @@ -8,6 +9,5 @@ pyqtgraph ipywidgets ipympl ephyviewer -spikeinterface[full,widgets] spikeinterface-gui kilosort From 081a54af9d5503f50406ccf249ae0ac94d626d5c Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 28 Oct 2024 18:59:44 +0100 Subject: [PATCH 005/157] job_kwargs --- installation_tips/check_your_install.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/installation_tips/check_your_install.py b/installation_tips/check_your_install.py index 64ab9cca8a..1144bda577 100644 --- a/installation_tips/check_your_install.py +++ b/installation_tips/check_your_install.py @@ -28,19 +28,21 @@ def _create_recording(): def _run_one_sorter_and_analyzer(sorter_name): import spikeinterface.full as si + + si.set_global_job_kwargs(**job_kwargs) + recording = si.load_extractor('./toy_example_recording') sorting = si.run_sorter(sorter_name, recording, folder=f'./sorter_with_{sorter_name}', verbose=False) sorting_analyzer = si.create_sorting_analyzer(sorting, recording, - format="binary_folder", folder=f"./analyzer_with_{sorter_name}", - **job_kwargs) + format="binary_folder", folder=f"./analyzer_with_{sorter_name}") sorting_analyzer.compute("random_spikes", method="uniform", max_spikes_per_unit=500) - sorting_analyzer.compute("waveforms", **job_kwargs) + sorting_analyzer.compute("waveforms") sorting_analyzer.compute("templates") sorting_analyzer.compute("noise_levels") sorting_analyzer.compute("unit_locations", method="monopolar_triangulation") sorting_analyzer.compute("correlograms", window_ms=100, bin_ms=5.) - sorting_analyzer.compute("principal_components", n_components=3, mode='by_channel_global', whiten=True, **job_kwargs) + sorting_analyzer.compute("principal_components", n_components=3, mode='by_channel_global', whiten=True) sorting_analyzer.compute("quality_metrics", metric_names=["snr", "firing_rate"]) @@ -117,9 +119,6 @@ def _clean(): steps.append(('Open spikeinterface-gui', open_sigui)) steps.append(('Export to phy', export_to_phy)), - # phy is removed from the env because it force a pip install PyQt5 - # which break the conda env - # ('Open phy', open_phy), # if platform.system() == "Windows": # pass From 286226cb51babe1e377bc53c7ca0873a1e7e877c Mon Sep 17 00:00:00 2001 From: Garcia Samuel Date: Wed, 30 Oct 2024 16:37:48 +0100 Subject: [PATCH 006/157] merci zach Co-authored-by: Zach McKenzie <92116279+zm711@users.noreply.github.com> --- installation_tips/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index f7b8ed4445..7827dd370e 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -40,7 +40,7 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell 5. `uv venv si_env --python 3.11` -6. `source si_env/bin/activate` (you should have `(si_env)` in your terminal) +6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) or for Powershell `si_env\Scripts\activate` 7. `uv pip install -r Documents/requirements_stable.txt` @@ -61,7 +61,7 @@ If you want to test the spikeinterface install you can: 1. Download with right click + save the file [`check_your_install.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/check_your_install.py) and put it into the "Documents" folder 2. Open the CMD (Windows) or Terminal (Mac/Linux) -3. Acticate your is_env : `source si_env/bin/activate` +3. Activate your si_env : `source si_env/bin/activate` (Max/Linux), `si_env\Scripts\activate` (CMD prompt) 4. Go to your "Documents" folder with `cd Documents` or the place where you downloaded the `check_your_install.py` 5. Run this: `python check_your_install.py` From ef2c00c0838d71e2926ed8ea9e9b58a7addb06c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:38:26 +0000 Subject: [PATCH 007/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- installation_tips/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index 7827dd370e..e86185b503 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -40,7 +40,7 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell 5. `uv venv si_env --python 3.11` -6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) or for Powershell `si_env\Scripts\activate` +6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) or for Powershell `si_env\Scripts\activate` 7. `uv pip install -r Documents/requirements_stable.txt` From a42b572e68b71c271c067199d8cd465f938f93f1 Mon Sep 17 00:00:00 2001 From: Max Lim Date: Sun, 5 Jan 2025 20:15:17 -0800 Subject: [PATCH 008/157] Implemented RT-Sort as an external sorter --- doc/modules/sorters.rst | 1 + .../sorters/external/rt_sort.py | 199 ++++++++++++++++++ src/spikeinterface/sorters/sorterlist.py | 2 + 3 files changed, 202 insertions(+) create mode 100644 src/spikeinterface/sorters/external/rt_sort.py diff --git a/doc/modules/sorters.rst b/doc/modules/sorters.rst index a58fba1c98..86a9286b8d 100644 --- a/doc/modules/sorters.rst +++ b/doc/modules/sorters.rst @@ -466,6 +466,7 @@ Here is the list of external sorters accessible using the run_sorter wrapper: * **Klusta** :code:`run_sorter(sorter_name='klusta')` * **Mountainsort4** :code:`run_sorter(sorter_name='mountainsort4')` * **Mountainsort5** :code:`run_sorter(sorter_name='mountainsort5')` +* **RT-Sort** :code:`run_sorter(sorter_name='rt-sort')` * **SpyKING Circus** :code:`run_sorter(sorter_name='spykingcircus')` * **Tridesclous** :code:`run_sorter(sorter_name='tridesclous')` * **Wave clus** :code:`run_sorter(sorter_name='waveclus')` diff --git a/src/spikeinterface/sorters/external/rt_sort.py b/src/spikeinterface/sorters/external/rt_sort.py new file mode 100644 index 0000000000..31306bea80 --- /dev/null +++ b/src/spikeinterface/sorters/external/rt_sort.py @@ -0,0 +1,199 @@ +import importlib.util +import os +import numpy as np + +from ..basesorter import BaseSorter + +from spikeinterface.extractors import NumpySorting # TODO: Create separate sorting extractor for RT-Sort + + +class RTSortSorter(BaseSorter): + """RTSort sorter object""" + + sorter_name = "rt-sort" + + _default_params = { + "detection_model": "neuropixels", + "recording_window_ms": None, + "stringent_thresh": 0.175, + "loose_thresh": 0.075, + "inference_scaling_numerator": 15.4, + "ms_before": 0.5, + "ms_after": 0.5, + "pre_median_ms": 50, + "inner_radius": 50, + "outer_radius": 100, + "min_elecs_for_array_noise_n": 100, + "min_elecs_for_array_noise_f": 0.1, + "min_elecs_for_seq_noise_n": 50, + "min_elecs_for_seq_noise_f": 0.05, + "min_activity_root_cocs": 2, + "min_activity_hz": 0.05, + "max_n_components_latency": 4, + "min_coc_n": 10, + "min_coc_p": 10, + "min_extend_comp_p": 50, + "elec_patience": 6, + "split_coc_clusters_amps": True, + "min_amp_dist_p": 0.1, + "max_n_components_amp": 4, + "min_loose_elec_prob": 0.03, + "min_inner_loose_detections": 3, + "min_loose_detections_n": 4, + "min_loose_detections_r_spikes": 1 / 3, + "min_loose_detections_r_sequences": 1 / 3, + "max_latency_diff_spikes": 2.5, + "max_latency_diff_sequences": 2.5, + "clip_latency_diff_factor": 2, + "max_amp_median_diff_spikes": 0.45, + "max_amp_median_diff_sequences": 0.45, + "clip_amp_median_diff_factor": 2, + "max_root_amp_median_std_spikes": 2.5, + "max_root_amp_median_std_sequences": 2.5, + "repeated_detection_overlap_time": 0.2, + "min_seq_spikes_n": 10, + "min_seq_spikes_hz": 0.05, + "relocate_root_min_amp": 0.8, + "relocate_root_max_latency": -2, + "device": "cuda", + "num_processes": None, + "ignore_warnings": True, + "debug": False, + } + + _params_description = { + "detection_model": "`mea` or `neuropixels` to use the mea- or neuropixels-trained detection models. Or, path to saved detection model (see https://braingeneers.github.io/braindance/docs/RT-sort/usage/training-models) (`str`, `neuropixels`)", + "recording_window_ms": "A tuple `(start_ms, end_ms)` defining the portion of the recording (in milliseconds) to process. (`tuple`, `None`)", + "stringent_thresh": "The stringent threshold for spike detection. (`float`, `0.175`)", + "loose_thresh": "The loose threshold for spike detection. (`float`, `0.075`)", + "inference_scaling_numerator": "Scaling factor for inference. (`float`, `15.4`)", + "ms_before": "Time (in milliseconds) to consider before each detected spike for sequence formation. (`float`, `0.5`)", + "ms_after": "Time (in milliseconds) to consider after each detected spike for sequence formation. (`float`, `0.5`)", + "pre_median_ms": "Duration (in milliseconds) to compute the median for normalization. (`float`, `50`)", + "inner_radius": "Inner radius (in micrometers). (`float`, `50`)", + "outer_radius": "Outer radius (in micrometers). (`float`, `100`)", + "min_elecs_for_array_noise_n": "Minimum number of electrodes for array-wide noise filtering. (`int`, `100`)", + "min_elecs_for_array_noise_f": "Minimum fraction of electrodes for array-wide noise filtering. (`float`, `0.1`)", + "min_elecs_for_seq_noise_n": "Minimum number of electrodes for sequence-wide noise filtering. (`int`, `50`)", + "min_elecs_for_seq_noise_f": "Minimum fraction of electrodes for sequence-wide noise filtering. (`float`, `0.05`)", + "min_activity_root_cocs": "Minimum number of stringent spike detections on inner electrodes within the maximum propagation window that cause a stringent spike detection on a root electrode to be counted as a stringent codetection. (`int`, `2`)", + "min_activity_hz": "Minimum activity rate of root detections (in Hz) for an electrode to be used as a root electrode. (`float`, `0.05`)", + "max_n_components_latency": "Maximum number of latency components for Gaussian mixture model used for splitting latency distribution. (`int`, `4`)", + "min_coc_n": "After splitting a cluster of codetections, a cluster is discarded if it does not have at least min_coc_n codetections. (`int`, `10`)", + "min_coc_p": "After splitting a cluster of codetections, a cluster is discarded if it does not have at least (min_coc_p * the total number of codetections before splitting) codetections. (`int`, `10`)", + "min_extend_comp_p": "The required percentage of codetections before splitting that is preserved after the split in order for the inner electrodes of the current splitting electrode to be added to the total list of electrodes used to further split the cluster. (`int`, `50`)", + "elec_patience": "Number of electrodes considered for splitting that do not lead to a split before terminating the splitting process. (`int`, `6`)", + "split_coc_clusters_amps": "Whether to split clusters based on amplitude. (`bool`, `True`)", + "min_amp_dist_p": "The minimum Hartigan's dip test p-value for a distribution to be considered unimodal. (`float`, `0.1`)", + "max_n_components_amp": "Maximum number of components for Gaussian mixture model used for splitting amplitude distribution. (`int`, `4`)", + "min_loose_elec_prob": "Minimum average detection score (smaller values are set to 0) in decimal form (ranging from 0 to 1). (`float`, `0.03`)", + "min_inner_loose_detections": "Minimum inner loose electrode detections for assigning spikes / overlaps for merging. (`int`, `3`)", + "min_loose_detections_n": "Minimum loose electrode detections for assigning spikes / overlaps for merging. (`int`, `4`)", + "min_loose_detections_r_spikes": "Minimum ratio of loose electrode detections for assigning spikes. (`float`, `1/3`)", + "min_loose_detections_r_sequences": "Minimum ratio of loose electrode detections overlaps for merging. (`float`, `1/3`)", + "max_latency_diff_spikes": "Maximum allowed weighted latency difference for spike assignment. (`float`, `2.5`)", + "max_latency_diff_sequences": "Maximum allowed weighted latency difference for sequence merging. (`float`, `2.5`)", + "clip_latency_diff_factor": "Latency clip = clip_latency_diff_factor * max_latency_diff. (`float`, `2`)", + "max_amp_median_diff_spikes": "Maximum allowed weighted percent amplitude difference for spike assignment. (`float`, `0.45`)", + "max_amp_median_diff_sequences": "Maximum allowed weighted percent amplitude difference for sequence merging. (`float`, `0.45`)", + "clip_amp_median_diff_factor": "Amplitude clip = clip_amp_median_diff_factor * max_amp_median_diff. (`float`, `2`)", + "max_root_amp_median_std_spikes": "Maximum allowed root amplitude standard deviation for spike assignment. (`float`, `2.5`)", + "max_root_amp_median_std_sequences": "Maximum allowed root amplitude standard deviation for sequence merging. (`float`, `2.5`)", + "repeated_detection_overlap_time": "Time window (in seconds) for overlapping repeated detections. (`float`, `0.2`)", + "min_seq_spikes_n": "Minimum number of spikes required for a valid sequence. (`int`, `10`)", + "min_seq_spikes_hz": "Minimum spike rate for a valid sequence. (`float`, `0.05`)", + "relocate_root_min_amp": "Minimum amplitude ratio for relocating a root electrode before first merging. (`float`, `0.8`)", + "relocate_root_max_latency": "Maximum latency for relocating a root electrode before first merging. (`float`, `-2`)", + "device": "The device for PyTorch operations ('cuda' or 'cpu'). (`str`, `cuda`)", + "num_processes": "Number of processes to use for parallelization. (`int`, `None`)", + "ignore_warnings": "Whether to suppress warnings during execution. (`bool`, `True`)", + "debug": "Whether to enable debugging features such as saving intermediate steps. (`bool`, `False`)", + } + + sorter_description = """RT-Sort is a real-time spike sorting algorithm that enables the sorted detection of action potentials within 7.5ms±1.5ms (mean±STD) after the waveform trough while the recording remains ongoing. + It utilizes unique propagation patterns of action potentials along axons detected as high-fidelity sequential activations on adjacent electrodes, together with a convolutional neural network-based spike detection algorithm. + This implementation in SpikeInterface only implements RT-Sort's offline sorting. + For more information see https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0312438""" + + installation_mesg = f"""\nTo use RTSort run:\n + >>> pip install git+https://github.com/braingeneers/braindance + + And install the following additional dependencies: + >>> pip install {'torch' if os.name == 'nt' else 'torch_tensorrt'} + >>> pip install diptest + >>> pip install pynvml + >>> pip install scikit-learn + + More information on RTSort at: + *https://github.com/braingeneers/braindance + """ + + handle_multi_segment = False + + @classmethod + def get_sorter_version(cls): + import braindance + + try: + return braindance.__version__ + except AttributeError: + return "0.0.1" + + @classmethod + def is_installed(cls): + libraries = ["braindance", "torch" if os.name == "nt" else "torch_tensorrt", "diptest", "pynvml", "sklearn"] + + HAVE_RTSORT = True + for lib in libraries: + if importlib.util.find_spec(lib) is None: + HAVE_RTSORT = False + break + + return HAVE_RTSORT + + @classmethod + def _check_params(cls, recording, output_folder, params): + return params + + @classmethod + def _check_apply_filter_in_params(cls, params): + return False + + @classmethod + def _setup_recording(cls, recording, sorter_output_folder, params, verbose): + # nothing to copy inside the folder : RTSort uses spikeinterface natively + pass + + @classmethod + def _run_from_folder(cls, sorter_output_folder, params, verbose): + from braindance.core.spikesorter.rt_sort import detect_sequences + from braindance.core.spikedetector.model import ModelSpikeSorter + + recording = cls.load_recording_from_folder(sorter_output_folder.parent, with_warnings=False) + + params = params.copy() + params["recording"] = recording + rt_sort_inter = sorter_output_folder / "rt_sort_inter" + params["inter_path"] = rt_sort_inter + params["verbose"] = verbose + + if params["detection_model"] == "mea": + params["detection_model"] = ModelSpikeSorter.load_mea() + elif params["detection_model"] == "neuropixels": + params["detection_model"] = ModelSpikeSorter.load_neuropixels() + else: + params["detection_model"] = ModelSpikeSorter.load(params["detection_model"]) + + rt_sort = detect_sequences(**params, delete_inter=False, return_spikes=False) + np_sorting = rt_sort.sort_offline( + rt_sort_inter / "scaled_traces.npy", + verbose=verbose, + recording_window_ms=params.get("recording_window_ms", None), + return_spikeinterface_sorter=True, + ) # type: NumpySorting + rt_sort.save(sorter_output_folder / "rt_sort.pickle") + np_sorting.save(folder=sorter_output_folder / "rt_sorting") + + @classmethod + def _get_result_from_folder(cls, sorter_output_folder): + return NumpySorting.load_from_folder(sorter_output_folder / "rt_sorting") diff --git a/src/spikeinterface/sorters/sorterlist.py b/src/spikeinterface/sorters/sorterlist.py index e1a6816133..f5291f4ea4 100644 --- a/src/spikeinterface/sorters/sorterlist.py +++ b/src/spikeinterface/sorters/sorterlist.py @@ -13,6 +13,7 @@ from .external.klusta import KlustaSorter from .external.mountainsort4 import Mountainsort4Sorter from .external.mountainsort5 import Mountainsort5Sorter +from .external.rt_sort import RTSortSorter from .external.spyking_circus import SpykingcircusSorter from .external.tridesclous import TridesclousSorter from .external.waveclus import WaveClusSorter @@ -39,6 +40,7 @@ KlustaSorter, Mountainsort4Sorter, Mountainsort5Sorter, + RTSortSorter, SpykingcircusSorter, TridesclousSorter, WaveClusSorter, From 4f0bee99b4d84897062915b9aeeb0f66ec94a0be Mon Sep 17 00:00:00 2001 From: Max Lim Date: Mon, 6 Jan 2025 17:52:50 -0800 Subject: [PATCH 009/157] Added reference and sorter test suite --- doc/references.rst | 3 +++ .../sorters/external/tests/test_rtsort.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/spikeinterface/sorters/external/tests/test_rtsort.py diff --git a/doc/references.rst b/doc/references.rst index ce4672a9ca..f807d85671 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -40,6 +40,7 @@ please include the appropriate citation for the :code:`sorter_name` parameter yo - :code:`herdingspikes` [Muthmann]_ [Hilgen]_ - :code:`kilosort` [Pachitariu]_ - :code:`mountainsort` [Chung]_ +- :code:`rt-sort` [van der Molen]_ - :code:`spykingcircus` [Yger]_ - :code:`wavclus` [Chaure]_ - :code:`yass` [Lee]_ @@ -126,6 +127,8 @@ References .. [UMS] `UltraMegaSort2000 - Spike sorting and quality metrics for extracellular spike data. 2011. `_ +.. [van der Molen] `RT-Sort: An action potential propagation-based algorithm for real time spike detection and sorting with millisecond latencies. 2024. `_ + .. [Varol] `Decentralized Motion Inference and Registration of Neuropixel Data. 2021. `_ .. [Windolf] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. `_ diff --git a/src/spikeinterface/sorters/external/tests/test_rtsort.py b/src/spikeinterface/sorters/external/tests/test_rtsort.py new file mode 100644 index 0000000000..1eb28e7591 --- /dev/null +++ b/src/spikeinterface/sorters/external/tests/test_rtsort.py @@ -0,0 +1,16 @@ +import unittest +import pytest + +from spikeinterface.sorters import RTSortSorter +from spikeinterface.sorters.tests.common_tests import SorterCommonTestSuite + + +@pytest.mark.skipif(not RTSortSorter.is_installed(), reason="rt-sort not installed") +class RTSortSorterCommonTestSuite(SorterCommonTestSuite, unittest.TestCase): + SorterClass = RTSortSorter + + +if __name__ == "__main__": + test = RTSortSorterCommonTestSuite() + test.setUp() + test.test_with_run() From a38d2f82ef3778b68750ac339fa73957dc207101 Mon Sep 17 00:00:00 2001 From: Max Lim Date: Sun, 19 Jan 2025 16:11:05 -0800 Subject: [PATCH 010/157] Updated installation --- src/spikeinterface/sorters/external/rt_sort.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/spikeinterface/sorters/external/rt_sort.py b/src/spikeinterface/sorters/external/rt_sort.py index 31306bea80..23e1ca5ffb 100644 --- a/src/spikeinterface/sorters/external/rt_sort.py +++ b/src/spikeinterface/sorters/external/rt_sort.py @@ -116,16 +116,12 @@ class RTSortSorter(BaseSorter): For more information see https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0312438""" installation_mesg = f"""\nTo use RTSort run:\n - >>> pip install git+https://github.com/braingeneers/braindance + >>> pip install git+https://github.com/braingeneers/braindance#egg=braindance[rt-sort] - And install the following additional dependencies: - >>> pip install {'torch' if os.name == 'nt' else 'torch_tensorrt'} - >>> pip install diptest - >>> pip install pynvml - >>> pip install scikit-learn + Additionally, install PyTorch (https://pytorch.org/get-started/locally/) with any version of CUDA as the compute platform. + If running on a Linux machine, install Torch-TensorRT (https://pytorch.org/TensorRT/getting_started/installation.html) for faster computations. - More information on RTSort at: - *https://github.com/braingeneers/braindance + More information on RTSort at: https://github.com/braingeneers/braindance """ handle_multi_segment = False @@ -134,10 +130,7 @@ class RTSortSorter(BaseSorter): def get_sorter_version(cls): import braindance - try: - return braindance.__version__ - except AttributeError: - return "0.0.1" + return braindance.__version__ @classmethod def is_installed(cls): From 4e655933b0f97325598f49c61c593ce6dad4df85 Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Tue, 25 Mar 2025 15:21:15 +0000 Subject: [PATCH 011/157] Add multi-segment support for amplitudes widget --- src/spikeinterface/widgets/amplitudes.py | 84 +++++++++++++++++++----- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 197fefbab2..fb3bfbd4db 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -73,34 +73,84 @@ def __init__( if unit_ids is None: unit_ids = sorting.unit_ids - if sorting.get_num_segments() > 1: + num_segments = sorting.get_num_segments() + + # Handle segment_index input + if num_segments > 1: if segment_index is None: warn("More than one segment available! Using `segment_index = 0`.") segment_index = 0 else: segment_index = 0 + + # Convert segment_index to list for consistent processing + if isinstance(segment_index, int): + segment_indices = [segment_index] + elif isinstance(segment_index, list): + segment_indices = segment_index + else: + raise ValueError("segment_index must be an int or a list of ints") + + # Validate segment indices + for idx in segment_indices: + if not isinstance(idx, int): + raise ValueError(f"Each segment index must be an integer, got {type(idx)}") + if idx < 0 or idx >= num_segments: + raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") + + # Initialize dictionaries for concatenated data + all_spiketrains = {unit_id: [] for unit_id in unit_ids} + all_amplitudes = {unit_id: [] for unit_id in unit_ids} + + # Calculate cumulative durations for spike time adjustments + cumulative_durations = [0] + for i in range(len(segment_indices) - 1): + segment_idx = segment_indices[i] + duration = sorting_analyzer.get_num_samples(segment_idx) / sorting_analyzer.sampling_frequency + cumulative_durations.append(cumulative_durations[-1] + duration) + + # Calculate total duration across all segments + total_duration = cumulative_durations[-1] + if segment_indices: # Check if there are any segments + total_duration += sorting_analyzer.get_num_samples(segment_indices[-1]) / sorting_analyzer.sampling_frequency + + # Concatenate spike trains and amplitudes across segments + for i, segment_idx in enumerate(segment_indices): + amplitudes_segment = amplitudes[segment_idx] + offset = cumulative_durations[i] + + for unit_id in unit_ids: + # Get spike times for this unit in this segment + spike_times = sorting.get_unit_spike_train(unit_id, segment_index=segment_idx, return_times=True) + + # Adjust spike times by adding cumulative duration of previous segments + if offset > 0: + spike_times = spike_times + offset + + # Get amplitudes for this unit in this segment + amps = amplitudes_segment[unit_id] + + # Concatenate with any existing data + if len(all_spiketrains[unit_id]) > 0: + all_spiketrains[unit_id] = np.concatenate([all_spiketrains[unit_id], spike_times]) + all_amplitudes[unit_id] = np.concatenate([all_amplitudes[unit_id], amps]) + else: + all_spiketrains[unit_id] = spike_times + all_amplitudes[unit_id] = amps - amplitudes_segment = amplitudes[segment_index] - total_duration = sorting_analyzer.get_num_samples(segment_index) / sorting_analyzer.sampling_frequency - - all_spiketrains = { - unit_id: sorting.get_unit_spike_train(unit_id, segment_index=segment_index, return_times=True) - for unit_id in sorting.unit_ids - } - - all_amplitudes = amplitudes_segment if max_spikes_per_unit is not None: spiketrains_to_plot = dict() amplitudes_to_plot = dict() - for unit, st in all_spiketrains.items(): - amps = all_amplitudes[unit] + for unit_id in unit_ids: + st = all_spiketrains[unit_id] + amps = all_amplitudes[unit_id] if len(st) > max_spikes_per_unit: random_idxs = np.random.choice(len(st), size=max_spikes_per_unit, replace=False) - spiketrains_to_plot[unit] = st[random_idxs] - amplitudes_to_plot[unit] = amps[random_idxs] + spiketrains_to_plot[unit_id] = st[random_idxs] + amplitudes_to_plot[unit_id] = amps[random_idxs] else: - spiketrains_to_plot[unit] = st - amplitudes_to_plot[unit] = amps + spiketrains_to_plot[unit_id] = st + amplitudes_to_plot[unit_id] = amps else: spiketrains_to_plot = all_spiketrains amplitudes_to_plot = all_amplitudes @@ -124,7 +174,7 @@ def __init__( ) BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) - + def plot_sortingview(self, data_plot, **backend_kwargs): import sortingview.views as vv from .utils_sortingview import generate_unit_table_view, make_serializable, handle_display_and_url From 677f90c18bee7b0c0ca86b3f9f2eb3c5a3ab689b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 25 Mar 2025 11:21:26 -0400 Subject: [PATCH 012/157] wip --- src/spikeinterface/curation/curation_model.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index 0e2f1da870..f7c7415543 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -5,6 +5,14 @@ supported_curation_format_versions = {"1"} +# TODO: splitting +# - add split_units to curation model +# - add split_mode to curation model +# - add split to apply_curation +# - add split_units to SortingAnalyzer +# - add _split_units to extensions + + class LabelDefinition(BaseModel): name: str = Field(..., description="Name of the label") label_options: List[str] = Field(..., description="List of possible label options") @@ -21,11 +29,14 @@ class CurationModel(BaseModel): unit_ids: List[Union[int, str]] = Field(..., description="List of unit IDs") label_definitions: Dict[str, LabelDefinition] = Field(..., description="Dictionary of label definitions") manual_labels: List[ManualLabel] = Field(..., description="List of manual labels") - merge_unit_groups: List[List[Union[int, str]]] = Field(..., description="List of groups of units to be merged") removed_units: List[Union[int, str]] = Field(..., description="List of removed unit IDs") + merge_unit_groups: List[List[Union[int, str]]] = Field(..., description="List of groups of units to be merged") merge_new_unit_ids: Optional[List[Union[int, str]]] = Field( default=None, description="List of new unit IDs after merging" ) + split_units: Optional[Dict[Union[int, str], List[List[int]]]] = Field( + default=None, description="Dictionary of units to be split" + ) @field_validator("format_version") def check_format_version(cls, v): From 2539a8948345743297bae037d44ebc22303ff8a7 Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Tue, 25 Mar 2025 17:28:55 +0000 Subject: [PATCH 013/157] Update base raster widget and children to handle multi-segment --- src/spikeinterface/widgets/amplitudes.py | 96 +++++------ src/spikeinterface/widgets/motion.py | 105 +++++++++--- src/spikeinterface/widgets/rasters.py | 209 ++++++++++++++++++++--- 3 files changed, 303 insertions(+), 107 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index fb3bfbd4db..3b99f4dd50 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -25,8 +25,9 @@ class AmplitudesWidget(BaseRasterWidget): unit_colors : dict | None, default: None Dict of colors with unit ids as keys and colors as values. Colors can be any type accepted by matplotlib. If None, default colors are chosen using the `get_some_colors` function. - segment_index : int or None, default: None - The segment index (or None if mono-segment) + segment_index : int or list of int or None, default: None + Segment index or indices to plot. If None and there are multiple segments, defaults to 0. + If list, spike trains and amplitudes are concatenated across the specified segments. max_spikes_per_unit : int or None, default: None Number of max spikes per unit to display. Use None for all spikes y_lim : tuple or None, default: None @@ -64,10 +65,10 @@ def __init__( ): sorting_analyzer = self.ensure_sorting_analyzer(sorting_analyzer) - sorting = sorting_analyzer.sorting self.check_extensions(sorting_analyzer, "spike_amplitudes") + # Get amplitudes by segment amplitudes = sorting_analyzer.get_extension("spike_amplitudes").get_data(outputs="by_unit") if unit_ids is None: @@ -98,70 +99,57 @@ def __init__( if idx < 0 or idx >= num_segments: raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") - # Initialize dictionaries for concatenated data - all_spiketrains = {unit_id: [] for unit_id in unit_ids} - all_amplitudes = {unit_id: [] for unit_id in unit_ids} - - # Calculate cumulative durations for spike time adjustments - cumulative_durations = [0] - for i in range(len(segment_indices) - 1): - segment_idx = segment_indices[i] - duration = sorting_analyzer.get_num_samples(segment_idx) / sorting_analyzer.sampling_frequency - cumulative_durations.append(cumulative_durations[-1] + duration) + # Create multi-segment data structure (dict of dicts) + spiketrains_by_segment = {} + amplitudes_by_segment = {} - # Calculate total duration across all segments - total_duration = cumulative_durations[-1] - if segment_indices: # Check if there are any segments - total_duration += sorting_analyzer.get_num_samples(segment_indices[-1]) / sorting_analyzer.sampling_frequency - - # Concatenate spike trains and amplitudes across segments - for i, segment_idx in enumerate(segment_indices): - amplitudes_segment = amplitudes[segment_idx] - offset = cumulative_durations[i] + for idx in segment_indices: + amplitudes_segment = amplitudes[idx] + + # Initialize for this segment + spiketrains_by_segment[idx] = {} + amplitudes_by_segment[idx] = {} for unit_id in unit_ids: # Get spike times for this unit in this segment - spike_times = sorting.get_unit_spike_train(unit_id, segment_index=segment_idx, return_times=True) - - # Adjust spike times by adding cumulative duration of previous segments - if offset > 0: - spike_times = spike_times + offset - - # Get amplitudes for this unit in this segment + spike_times = sorting.get_unit_spike_train(unit_id, segment_index=idx, return_times=True) amps = amplitudes_segment[unit_id] - # Concatenate with any existing data - if len(all_spiketrains[unit_id]) > 0: - all_spiketrains[unit_id] = np.concatenate([all_spiketrains[unit_id], spike_times]) - all_amplitudes[unit_id] = np.concatenate([all_amplitudes[unit_id], amps]) - else: - all_spiketrains[unit_id] = spike_times - all_amplitudes[unit_id] = amps - + # Store data in dict of dicts format + spiketrains_by_segment[idx][unit_id] = spike_times + amplitudes_by_segment[idx][unit_id] = amps + + # Apply max_spikes_per_unit limit if specified if max_spikes_per_unit is not None: - spiketrains_to_plot = dict() - amplitudes_to_plot = dict() - for unit_id in unit_ids: - st = all_spiketrains[unit_id] - amps = all_amplitudes[unit_id] - if len(st) > max_spikes_per_unit: - random_idxs = np.random.choice(len(st), size=max_spikes_per_unit, replace=False) - spiketrains_to_plot[unit_id] = st[random_idxs] - amplitudes_to_plot[unit_id] = amps[random_idxs] - else: - spiketrains_to_plot[unit_id] = st - amplitudes_to_plot[unit_id] = amps - else: - spiketrains_to_plot = all_spiketrains - amplitudes_to_plot = all_amplitudes + for idx in segment_indices: + for unit_id in unit_ids: + st = spiketrains_by_segment[idx][unit_id] + amps = amplitudes_by_segment[idx][unit_id] + if len(st) > max_spikes_per_unit: + # Scale down the number of spikes proportionally per segment + # to ensure we have max_spikes_per_unit total after concatenation + segment_count = len(segment_indices) + segment_max = max(1, max_spikes_per_unit // segment_count) + + if len(st) > segment_max: + random_idxs = np.random.choice(len(st), size=segment_max, replace=False) + spiketrains_by_segment[idx][unit_id] = st[random_idxs] + amplitudes_by_segment[idx][unit_id] = amps[random_idxs] if plot_histograms and bins is None: bins = 100 + # Calculate total duration across all segments for x-axis limits + total_duration = 0 + for idx in segment_indices: + duration = sorting_analyzer.get_num_samples(idx) / sorting_analyzer.sampling_frequency + total_duration += duration + plot_data = dict( - spike_train_data=spiketrains_to_plot, - y_axis_data=amplitudes_to_plot, + spike_train_data=spiketrains_by_segment, + y_axis_data=amplitudes_by_segment, unit_colors=unit_colors, + segment_index=segment_indices, plot_histograms=plot_histograms, bins=bins, total_duration=total_duration, diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index a0c7e1e28c..564373b704 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -117,14 +117,14 @@ class DriftRasterMapWidget(BaseRasterWidget): "spike_locations" extension computed. direction : "x" or "y", default: "y" The direction to display. "y" is the depth direction. - segment_index : int, default: None - The segment index to display. recording : RecordingExtractor | None, default: None The recording extractor object (only used to get "real" times). - segment_index : int, default: 0 - The segment index to display. sampling_frequency : float, default: None The sampling frequency (needed if recording is None). + segment_index : int or list of int or None, default: None + The segment index or indices to display. If None and there's only one segment, it's used. + If None and there are multiple segments, you must specify which to use. + If a list of indices is provided, peaks and locations are concatenated across the segments. depth_lim : tuple or None, default: None The min and max depth to display, if None (min and max of the recording). scatter_decimate : int, default: None @@ -149,7 +149,7 @@ def __init__( direction: str = "y", recording: BaseRecording | None = None, sampling_frequency: float | None = None, - segment_index: int | None = None, + segment_index: int | list | None = None, depth_lim: tuple[float, float] | None = None, color_amplitude: bool = True, scatter_decimate: int | None = None, @@ -160,7 +160,11 @@ def __init__( backend: str | None = None, **backend_kwargs, ): + from matplotlib.pyplot import colormaps + from matplotlib.colors import Normalize + assert peaks is not None or sorting_analyzer is not None + if peaks is not None: assert peak_locations is not None if recording is None: @@ -168,6 +172,7 @@ def __init__( else: sampling_frequency = recording.sampling_frequency peak_amplitudes = peaks["amplitude"] + if sorting_analyzer is not None: if sorting_analyzer.has_recording(): recording = sorting_analyzer.recording @@ -190,32 +195,62 @@ def __init__( else: peak_amplitudes = None + unique_segments = np.unique(peaks["segment_index"]) + if segment_index is None: - assert ( - len(np.unique(peaks["segment_index"])) == 1 - ), "segment_index must be specified if there are multiple segments" - segment_index = 0 + if len(unique_segments) == 1: + segment_indices = [int(unique_segments[0])] + else: + raise ValueError("segment_index must be specified if there are multiple segments") + elif isinstance(segment_index, int): + segment_indices = [segment_index] + elif isinstance(segment_index, list): + segment_indices = segment_index else: - peak_mask = peaks["segment_index"] == segment_index - peaks = peaks[peak_mask] - peak_locations = peak_locations[peak_mask] - if peak_amplitudes is not None: - peak_amplitudes = peak_amplitudes[peak_mask] - - from matplotlib.pyplot import colormaps - - if color_amplitude: - amps = peak_amplitudes + raise ValueError("segment_index must be an int or a list of ints") + + # Validate all segment indices exist in the data + for idx in segment_indices: + if idx not in unique_segments: + raise ValueError(f"segment_index {idx} not found in peaks data") + + # Filter data for the selected segments + # Note: For simplicity, we'll filter all data first, then construct dict of dicts + segment_mask = np.isin(peaks["segment_index"], segment_indices) + filtered_peaks = peaks[segment_mask] + filtered_locations = peak_locations[segment_mask] + if peak_amplitudes is not None: + filtered_amplitudes = peak_amplitudes[segment_mask] + + # Create dict of dicts structure for the base class + spike_train_data = {} + y_axis_data = {} + + # Process each segment separately + for seg_idx in segment_indices: + segment_mask = filtered_peaks["segment_index"] == seg_idx + segment_peaks = filtered_peaks[segment_mask] + segment_locations = filtered_locations[segment_mask] + + # Convert peak times to seconds + spike_times = segment_peaks["sample_index"] / sampling_frequency + + # Store in dict of dicts format (using 0 as the "unit" id) + spike_train_data[seg_idx] = {0: spike_times} + y_axis_data[seg_idx] = {0: segment_locations[direction]} + + if color_amplitude and peak_amplitudes is not None: + amps = filtered_amplitudes amps_abs = np.abs(amps) q_95 = np.quantile(amps_abs, 0.95) - cmap = colormaps[cmap] + cmap_obj = colormaps[cmap] if clim is None: amps = amps_abs amps /= q_95 - c = cmap(amps) + c = cmap_obj(amps) else: - norm_function = Normalize(vmin=dp.clim[0], vmax=dp.clim[1], clip=True) - c = cmap(norm_function(amps)) + norm_function = Normalize(vmin=clim[0], vmax=clim[1], clip=True) + c = cmap_obj(norm_function(amps)) color_kwargs = dict( color=None, c=c, @@ -223,19 +258,33 @@ def __init__( ) else: color_kwargs = dict(color=color, c=None, alpha=alpha) - - # convert data into format that `BaseRasterWidget` can take it in - spike_train_data = {0: peaks["sample_index"] / sampling_frequency} - y_axis_data = {0: peak_locations[direction]} - + + # Calculate total duration for x-axis limits + total_duration = 0 + for seg_idx in segment_indices: + if recording is not None and hasattr(recording, "get_duration"): + duration = recording.get_duration(seg_idx) + else: + # Estimate from spike times + segment_mask = filtered_peaks["segment_index"] == seg_idx + segment_peaks = filtered_peaks[segment_mask] + if len(segment_peaks) > 0: + max_sample = np.max(segment_peaks["sample_index"]) + duration = (max_sample + 1) / sampling_frequency + else: + duration = 0 + total_duration += duration + plot_data = dict( spike_train_data=spike_train_data, y_axis_data=y_axis_data, + segment_index=segment_indices, y_lim=depth_lim, color_kwargs=color_kwargs, scatter_decimate=scatter_decimate, title="Peak depth", y_label="Depth [um]", + total_duration=total_duration, ) BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 398ae4d728..42b5876da4 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -15,12 +15,17 @@ class BaseRasterWidget(BaseWidget): Parameters ---------- - spike_train_data : dict - A dict of spike trains, indexed by the unit_id - y_axis_data : dict - A dict of the y-axis data, indexed by the unit_id + spike_train_data : dict of dicts + A dict of dicts where the structure is spike_train_data[segment_index][unit_id]. + y_axis_data : dict of dicts + A dict of dicts where the structure is y_axis_data[segment_index][unit_id]. + For backwards compatibility, a flat dict indexed by unit_id will be internally + converted to a dict of dicts with segment 0. unit_ids : array-like | None, default: None List of unit_ids to plot + segment_index : int | list | None, default: None + For multi-segment data, specifies which segment(s) to plot. If None, uses all available segments. + For single-segment data, this parameter is ignored. total_duration : int | None, default: None Duration of spike_train_data in seconds. plot_histograms : bool, default: False @@ -48,6 +53,8 @@ class BaseRasterWidget(BaseWidget): Ticks on y-axis, passed to `set_yticks`. If None, default ticks are used. hide_unit_selector : bool, default: False For sortingview backend, if True the unit selector is not displayed + segment_boundary_kwargs : dict | None, default: None + Additional arguments for the segment boundary lines, passed to `matplotlib.axvline` backend : str | None, default None Which plotting backend to use e.g. 'matplotlib', 'ipywidgets'. If None, uses default from `get_default_plotter_backend`. @@ -58,6 +65,7 @@ def __init__( spike_train_data: dict, y_axis_data: dict, unit_ids: list | None = None, + segment_index: int | list | None = None, total_duration: int | None = None, plot_histograms: bool = False, bins: int | None = None, @@ -71,13 +79,103 @@ def __init__( y_label: str | None = None, y_ticks: bool = False, hide_unit_selector: bool = True, + segment_boundary_kwargs: dict | None = None, backend: str | None = None, **backend_kwargs, ): + # Set default segment boundary kwargs if not provided + if segment_boundary_kwargs is None: + segment_boundary_kwargs = {"color": "gray", "linestyle": "--", "alpha": 0.7} + + # Process the data + available_segments = list(spike_train_data.keys()) + available_segments.sort() # Ensure consistent ordering + + # Determine which segments to use + if segment_index is None: + # Use all segments by default + segments_to_use = available_segments + elif isinstance(segment_index, int): + # Single segment specified + if segment_index not in available_segments: + raise ValueError(f"segment_index {segment_index} not found in data") + segments_to_use = [segment_index] + elif isinstance(segment_index, list): + # Multiple segments specified + for idx in segment_index: + if idx not in available_segments: + raise ValueError(f"segment_index {idx} not found in data") + segments_to_use = segment_index + else: + raise ValueError("segment_index must be int, list, or None") + + # Get all unit IDs present in any segment if not specified + if unit_ids is None: + all_units = set() + for seg_idx in segments_to_use: + all_units.update(spike_train_data[seg_idx].keys()) + unit_ids = list(all_units) + + # Calculate segment durations and boundaries + segment_durations = [] + for seg_idx in segments_to_use: + max_time = 0 + for unit_id in unit_ids: + if unit_id in spike_train_data[seg_idx]: + unit_times = spike_train_data[seg_idx][unit_id] + if len(unit_times) > 0: + max_time = max(max_time, np.max(unit_times)) + segment_durations.append(max_time) + + # Calculate cumulative durations for segment boundaries + cumulative_durations = [0] + for duration in segment_durations[:-1]: + cumulative_durations.append(cumulative_durations[-1] + duration) + + # Segment boundaries for visualization (only internal boundaries) + segment_boundaries = cumulative_durations[1:] if len(segments_to_use) > 1 else None + + # Concatenate data across segments with proper time offsets + concatenated_spike_trains = {unit_id: [] for unit_id in unit_ids} + concatenated_y_axis = {unit_id: [] for unit_id in unit_ids} + + for i, seg_idx in enumerate(segments_to_use): + offset = cumulative_durations[i] + + for unit_id in unit_ids: + if unit_id in spike_train_data[seg_idx]: + # Get spike times for this unit in this segment + spike_times = spike_train_data[seg_idx][unit_id] + + # Adjust spike times by adding cumulative duration of previous segments + if offset > 0: + adjusted_times = spike_times + offset + else: + adjusted_times = spike_times + + # Get y-axis data for this unit in this segment + y_values = y_axis_data[seg_idx][unit_id] + + # Concatenate with any existing data + if len(concatenated_spike_trains[unit_id]) > 0: + concatenated_spike_trains[unit_id] = np.concatenate([concatenated_spike_trains[unit_id], adjusted_times]) + concatenated_y_axis[unit_id] = np.concatenate([concatenated_y_axis[unit_id], y_values]) + else: + concatenated_spike_trains[unit_id] = adjusted_times + concatenated_y_axis[unit_id] = y_values + + # Update spike train and y-axis data with concatenated values + processed_spike_train_data = concatenated_spike_trains + processed_y_axis_data = concatenated_y_axis + + # Calculate total duration from the data if not provided + if total_duration is None: + total_duration = cumulative_durations[-1] + segment_durations[-1] + plot_data = dict( - spike_train_data=spike_train_data, - y_axis_data=y_axis_data, + spike_train_data=processed_spike_train_data, + y_axis_data=processed_y_axis_data, unit_ids=unit_ids, plot_histograms=plot_histograms, y_lim=y_lim, @@ -92,6 +190,8 @@ def __init__( bins=bins, y_ticks=y_ticks, hide_unit_selector=hide_unit_selector, + segment_boundaries=segment_boundaries, + segment_boundary_kwargs=segment_boundary_kwargs, ) BaseWidget.__init__(self, plot_data, backend=backend, **backend_kwargs) @@ -134,7 +234,9 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): y_axis_data = dp.y_axis_data for unit_id in unit_ids: - + if unit_id not in spike_train_data: + continue # Skip this unit if not in data + unit_spike_train = spike_train_data[unit_id][:: dp.scatter_decimate] unit_y_data = y_axis_data[unit_id][:: dp.scatter_decimate] @@ -155,6 +257,11 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): count, bins = np.histogram(unit_y_data, bins=bins) ax_hist.plot(count, bins[:-1], color=unit_colors[unit_id], alpha=0.8) + # Add segment boundary lines if provided + if getattr(dp, 'segment_boundaries', None) is not None: + for boundary in dp.segment_boundaries: + scatter_ax.axvline(boundary, **dp.segment_boundary_kwargs) + if dp.plot_histograms: ax_hist = self.axes.flatten()[1] ax_hist.set_ylim(scatter_ax.get_ylim()) @@ -282,8 +389,9 @@ class RasterWidget(BaseRasterWidget): A sorting object sorting_analyzer : SortingAnalyzer | None, default: None A sorting analyzer object - segment_index : None or int - The segment index. + segment_index : int or list of int or None, default: None + The segment index or indices to use. If None and there are multiple segments, defaults to 0. + If a list of indices is provided, spike trains are concatenated across the specified segments. unit_ids : list List of unit ids time_range : list @@ -303,39 +411,88 @@ def __init__( backend=None, **backend_kwargs, ): + recording = None if sorting is None and sorting_analyzer is None: raise Exception("Must supply either a sorting or a sorting_analyzer") elif sorting is not None and sorting_analyzer is not None: raise Exception("Should supply either a sorting or a sorting_analyzer, not both") elif sorting_analyzer is not None: sorting = sorting_analyzer.sorting + recording = sorting_analyzer.recording sorting = self.ensure_sorting(sorting) - if sorting.get_num_segments() > 1: + num_segments = sorting.get_num_segments() + + # Handle segment_index input + if num_segments > 1: if segment_index is None: warn("More than one segment available! Using `segment_index = 0`.") segment_index = 0 else: segment_index = 0 + + # Convert segment_index to list for consistent processing + if isinstance(segment_index, int): + segment_indices = [segment_index] + elif isinstance(segment_index, list): + segment_indices = segment_index + else: + raise ValueError("segment_index must be an int or a list of ints") + + # Validate segment indices + for idx in segment_indices: + if not isinstance(idx, int): + raise ValueError(f"Each segment index must be an integer, got {type(idx)}") + if idx < 0 or idx >= num_segments: + raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") if unit_ids is None: unit_ids = sorting.unit_ids - all_spiketrains = { - unit_id: sorting.get_unit_spike_train(unit_id, segment_index=segment_index, return_times=True) - for unit_id in unit_ids - } - + # Create dict of dicts structure + spike_train_data = {} + y_axis_data = {} + + # Create a lookup dictionary for unit indices + unit_indices_map = {unit_id: i for i, unit_id in enumerate(unit_ids)} + + # Calculate total duration across all segments + total_duration = 0 + for seg_idx in segment_indices: + # Try to get duration from recording if available + if recording is not None: + duration = recording.get_duration(seg_idx) + else: + # Fallback: estimate from max spike time + max_time = 0 + for unit_id in unit_ids: + st = sorting.get_unit_spike_train(unit_id, segment_index=seg_idx, return_times=True) + if len(st) > 0: + max_time = max(max_time, np.max(st)) + duration = max_time + + total_duration += duration + + # Initialize dicts for this segment + spike_train_data[seg_idx] = {} + y_axis_data[seg_idx] = {} + + # Get spike trains for each unit in this segment + for unit_id in unit_ids: + spike_times = sorting.get_unit_spike_train(unit_id, segment_index=seg_idx, return_times=True) + + # Store spike trains + spike_train_data[seg_idx][unit_id] = spike_times + + # Create raster locations (y-values for plotting) + unit_index = unit_indices_map[unit_id] + y_axis_data[seg_idx][unit_id] = unit_index * np.ones(len(spike_times)) + + # Apply time range filtering if specified if time_range is not None: assert len(time_range) == 2, "'time_range' should be a list with start and end time in seconds" - for unit_id in unit_ids: - unit_st = all_spiketrains[unit_id] - all_spiketrains[unit_id] = unit_st[(time_range[0] < unit_st) & (unit_st < time_range[1])] - - raster_locations = { - unit_id: unit_index * np.ones(len(all_spiketrains[unit_id])) for unit_index, unit_id in enumerate(unit_ids) - } + # Let BaseRasterWidget handle the filtering unit_indices = list(range(len(unit_ids))) @@ -346,14 +503,16 @@ def __init__( y_ticks = {"ticks": unit_indices, "labels": unit_ids} plot_data = dict( - spike_train_data=all_spiketrains, - y_axis_data=raster_locations, + spike_train_data=spike_train_data, + y_axis_data=y_axis_data, + segment_index=segment_indices, x_lim=time_range, y_label="Unit id", unit_ids=unit_ids, unit_colors=unit_colors, plot_histograms=None, y_ticks=y_ticks, + total_duration=total_duration, ) - BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) + BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) \ No newline at end of file From 11a1845c61b456f5071c203913ec1db3cc3cd1c0 Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Tue, 25 Mar 2025 17:37:19 +0000 Subject: [PATCH 014/157] Retain sortingview compatibility --- src/spikeinterface/widgets/amplitudes.py | 27 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 3b99f4dd50..063641e40f 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -84,6 +84,14 @@ def __init__( else: segment_index = 0 + # Check for SortingView backend + is_sortingview = backend == "sortingview" + + # For SortingView, ensure we're only using a single segment + if is_sortingview and isinstance(segment_index, list) and len(segment_index) > 1: + warn("SortingView backend currently supports only single segment. Using first segment.") + segment_index = segment_index[0] + # Convert segment_index to list for consistent processing if isinstance(segment_index, int): segment_indices = [segment_index] @@ -144,12 +152,10 @@ def __init__( for idx in segment_indices: duration = sorting_analyzer.get_num_samples(idx) / sorting_analyzer.sampling_frequency total_duration += duration - + + # Build the plot data with the full dict of dicts structure plot_data = dict( - spike_train_data=spiketrains_by_segment, - y_axis_data=amplitudes_by_segment, unit_colors=unit_colors, - segment_index=segment_indices, plot_histograms=plot_histograms, bins=bins, total_duration=total_duration, @@ -160,7 +166,18 @@ def __init__( y_lim=y_lim, scatter_decimate=scatter_decimate, ) - + + # If using SortingView, extract just the first segment's data as flat dicts + if is_sortingview: + first_segment = segment_indices[0] + plot_data["spike_train_data"] = spiketrains_by_segment[first_segment] + plot_data["y_axis_data"] = amplitudes_by_segment[first_segment] + else: + # Otherwise use the full dict of dicts structure with all segments + plot_data["spike_train_data"] = spiketrains_by_segment + plot_data["y_axis_data"] = amplitudes_by_segment + plot_data["segment_index"] = segment_indices + BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) def plot_sortingview(self, data_plot, **backend_kwargs): From 16e6272f39359eba4df9556082f46ecb04ce8283 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:39:53 +0000 Subject: [PATCH 015/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/widgets/amplitudes.py | 30 ++++++------ src/spikeinterface/widgets/motion.py | 26 +++++----- src/spikeinterface/widgets/rasters.py | 60 ++++++++++++------------ 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 063641e40f..ef85f9ca30 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -75,7 +75,7 @@ def __init__( unit_ids = sorting.unit_ids num_segments = sorting.get_num_segments() - + # Handle segment_index input if num_segments > 1: if segment_index is None: @@ -83,15 +83,15 @@ def __init__( segment_index = 0 else: segment_index = 0 - + # Check for SortingView backend is_sortingview = backend == "sortingview" - + # For SortingView, ensure we're only using a single segment if is_sortingview and isinstance(segment_index, list) and len(segment_index) > 1: warn("SortingView backend currently supports only single segment. Using first segment.") segment_index = segment_index[0] - + # Convert segment_index to list for consistent processing if isinstance(segment_index, int): segment_indices = [segment_index] @@ -99,7 +99,7 @@ def __init__( segment_indices = segment_index else: raise ValueError("segment_index must be an int or a list of ints") - + # Validate segment indices for idx in segment_indices: if not isinstance(idx, int): @@ -110,23 +110,23 @@ def __init__( # Create multi-segment data structure (dict of dicts) spiketrains_by_segment = {} amplitudes_by_segment = {} - + for idx in segment_indices: amplitudes_segment = amplitudes[idx] - + # Initialize for this segment spiketrains_by_segment[idx] = {} amplitudes_by_segment[idx] = {} - + for unit_id in unit_ids: # Get spike times for this unit in this segment spike_times = sorting.get_unit_spike_train(unit_id, segment_index=idx, return_times=True) amps = amplitudes_segment[unit_id] - + # Store data in dict of dicts format spiketrains_by_segment[idx][unit_id] = spike_times amplitudes_by_segment[idx][unit_id] = amps - + # Apply max_spikes_per_unit limit if specified if max_spikes_per_unit is not None: for idx in segment_indices: @@ -138,7 +138,7 @@ def __init__( # to ensure we have max_spikes_per_unit total after concatenation segment_count = len(segment_indices) segment_max = max(1, max_spikes_per_unit // segment_count) - + if len(st) > segment_max: random_idxs = np.random.choice(len(st), size=segment_max, replace=False) spiketrains_by_segment[idx][unit_id] = st[random_idxs] @@ -152,7 +152,7 @@ def __init__( for idx in segment_indices: duration = sorting_analyzer.get_num_samples(idx) / sorting_analyzer.sampling_frequency total_duration += duration - + # Build the plot data with the full dict of dicts structure plot_data = dict( unit_colors=unit_colors, @@ -166,7 +166,7 @@ def __init__( y_lim=y_lim, scatter_decimate=scatter_decimate, ) - + # If using SortingView, extract just the first segment's data as flat dicts if is_sortingview: first_segment = segment_indices[0] @@ -177,9 +177,9 @@ def __init__( plot_data["spike_train_data"] = spiketrains_by_segment plot_data["y_axis_data"] = amplitudes_by_segment plot_data["segment_index"] = segment_indices - + BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) - + def plot_sortingview(self, data_plot, **backend_kwargs): import sortingview.views as vv from .utils_sortingview import generate_unit_table_view, make_serializable, handle_display_and_url diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index 564373b704..1a1512545b 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -162,9 +162,9 @@ def __init__( ): from matplotlib.pyplot import colormaps from matplotlib.colors import Normalize - + assert peaks is not None or sorting_analyzer is not None - + if peaks is not None: assert peak_locations is not None if recording is None: @@ -172,7 +172,7 @@ def __init__( else: sampling_frequency = recording.sampling_frequency peak_amplitudes = peaks["amplitude"] - + if sorting_analyzer is not None: if sorting_analyzer.has_recording(): recording = sorting_analyzer.recording @@ -196,7 +196,7 @@ def __init__( peak_amplitudes = None unique_segments = np.unique(peaks["segment_index"]) - + if segment_index is None: if len(unique_segments) == 1: segment_indices = [int(unique_segments[0])] @@ -208,12 +208,12 @@ def __init__( segment_indices = segment_index else: raise ValueError("segment_index must be an int or a list of ints") - + # Validate all segment indices exist in the data for idx in segment_indices: if idx not in unique_segments: raise ValueError(f"segment_index {idx} not found in peaks data") - + # Filter data for the selected segments # Note: For simplicity, we'll filter all data first, then construct dict of dicts segment_mask = np.isin(peaks["segment_index"], segment_indices) @@ -221,24 +221,24 @@ def __init__( filtered_locations = peak_locations[segment_mask] if peak_amplitudes is not None: filtered_amplitudes = peak_amplitudes[segment_mask] - + # Create dict of dicts structure for the base class spike_train_data = {} y_axis_data = {} - + # Process each segment separately for seg_idx in segment_indices: segment_mask = filtered_peaks["segment_index"] == seg_idx segment_peaks = filtered_peaks[segment_mask] segment_locations = filtered_locations[segment_mask] - + # Convert peak times to seconds spike_times = segment_peaks["sample_index"] / sampling_frequency - + # Store in dict of dicts format (using 0 as the "unit" id) spike_train_data[seg_idx] = {0: spike_times} y_axis_data[seg_idx] = {0: segment_locations[direction]} - + if color_amplitude and peak_amplitudes is not None: amps = filtered_amplitudes amps_abs = np.abs(amps) @@ -258,7 +258,7 @@ def __init__( ) else: color_kwargs = dict(color=color, c=None, alpha=alpha) - + # Calculate total duration for x-axis limits total_duration = 0 for seg_idx in segment_indices: @@ -274,7 +274,7 @@ def __init__( else: duration = 0 total_duration += duration - + plot_data = dict( spike_train_data=spike_train_data, y_axis_data=y_axis_data, diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 42b5876da4..3d4470c249 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -87,11 +87,11 @@ def __init__( # Set default segment boundary kwargs if not provided if segment_boundary_kwargs is None: segment_boundary_kwargs = {"color": "gray", "linestyle": "--", "alpha": 0.7} - + # Process the data available_segments = list(spike_train_data.keys()) available_segments.sort() # Ensure consistent ordering - + # Determine which segments to use if segment_index is None: # Use all segments by default @@ -109,14 +109,14 @@ def __init__( segments_to_use = segment_index else: raise ValueError("segment_index must be int, list, or None") - + # Get all unit IDs present in any segment if not specified if unit_ids is None: all_units = set() for seg_idx in segments_to_use: all_units.update(spike_train_data[seg_idx].keys()) unit_ids = list(all_units) - + # Calculate segment durations and boundaries segment_durations = [] for seg_idx in segments_to_use: @@ -127,52 +127,54 @@ def __init__( if len(unit_times) > 0: max_time = max(max_time, np.max(unit_times)) segment_durations.append(max_time) - + # Calculate cumulative durations for segment boundaries cumulative_durations = [0] for duration in segment_durations[:-1]: cumulative_durations.append(cumulative_durations[-1] + duration) - + # Segment boundaries for visualization (only internal boundaries) segment_boundaries = cumulative_durations[1:] if len(segments_to_use) > 1 else None - + # Concatenate data across segments with proper time offsets concatenated_spike_trains = {unit_id: [] for unit_id in unit_ids} concatenated_y_axis = {unit_id: [] for unit_id in unit_ids} - + for i, seg_idx in enumerate(segments_to_use): offset = cumulative_durations[i] - + for unit_id in unit_ids: if unit_id in spike_train_data[seg_idx]: # Get spike times for this unit in this segment spike_times = spike_train_data[seg_idx][unit_id] - + # Adjust spike times by adding cumulative duration of previous segments if offset > 0: adjusted_times = spike_times + offset else: adjusted_times = spike_times - + # Get y-axis data for this unit in this segment y_values = y_axis_data[seg_idx][unit_id] - + # Concatenate with any existing data if len(concatenated_spike_trains[unit_id]) > 0: - concatenated_spike_trains[unit_id] = np.concatenate([concatenated_spike_trains[unit_id], adjusted_times]) + concatenated_spike_trains[unit_id] = np.concatenate( + [concatenated_spike_trains[unit_id], adjusted_times] + ) concatenated_y_axis[unit_id] = np.concatenate([concatenated_y_axis[unit_id], y_values]) else: concatenated_spike_trains[unit_id] = adjusted_times concatenated_y_axis[unit_id] = y_values - + # Update spike train and y-axis data with concatenated values processed_spike_train_data = concatenated_spike_trains processed_y_axis_data = concatenated_y_axis - + # Calculate total duration from the data if not provided if total_duration is None: total_duration = cumulative_durations[-1] + segment_durations[-1] - + plot_data = dict( spike_train_data=processed_spike_train_data, y_axis_data=processed_y_axis_data, @@ -236,7 +238,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): for unit_id in unit_ids: if unit_id not in spike_train_data: continue # Skip this unit if not in data - + unit_spike_train = spike_train_data[unit_id][:: dp.scatter_decimate] unit_y_data = y_axis_data[unit_id][:: dp.scatter_decimate] @@ -258,7 +260,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): ax_hist.plot(count, bins[:-1], color=unit_colors[unit_id], alpha=0.8) # Add segment boundary lines if provided - if getattr(dp, 'segment_boundaries', None) is not None: + if getattr(dp, "segment_boundaries", None) is not None: for boundary in dp.segment_boundaries: scatter_ax.axvline(boundary, **dp.segment_boundary_kwargs) @@ -423,7 +425,7 @@ def __init__( sorting = self.ensure_sorting(sorting) num_segments = sorting.get_num_segments() - + # Handle segment_index input if num_segments > 1: if segment_index is None: @@ -431,7 +433,7 @@ def __init__( segment_index = 0 else: segment_index = 0 - + # Convert segment_index to list for consistent processing if isinstance(segment_index, int): segment_indices = [segment_index] @@ -439,7 +441,7 @@ def __init__( segment_indices = segment_index else: raise ValueError("segment_index must be an int or a list of ints") - + # Validate segment indices for idx in segment_indices: if not isinstance(idx, int): @@ -453,10 +455,10 @@ def __init__( # Create dict of dicts structure spike_train_data = {} y_axis_data = {} - + # Create a lookup dictionary for unit indices unit_indices_map = {unit_id: i for i, unit_id in enumerate(unit_ids)} - + # Calculate total duration across all segments total_duration = 0 for seg_idx in segment_indices: @@ -471,20 +473,20 @@ def __init__( if len(st) > 0: max_time = max(max_time, np.max(st)) duration = max_time - + total_duration += duration - + # Initialize dicts for this segment spike_train_data[seg_idx] = {} y_axis_data[seg_idx] = {} - + # Get spike trains for each unit in this segment for unit_id in unit_ids: spike_times = sorting.get_unit_spike_train(unit_id, segment_index=seg_idx, return_times=True) - + # Store spike trains spike_train_data[seg_idx][unit_id] = spike_times - + # Create raster locations (y-values for plotting) unit_index = unit_indices_map[unit_id] y_axis_data[seg_idx][unit_id] = unit_index * np.ones(len(spike_times)) @@ -515,4 +517,4 @@ def __init__( total_duration=total_duration, ) - BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) \ No newline at end of file + BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) From 86a3ab47b7fbffd67bb5f53d46074edaa0dce69a Mon Sep 17 00:00:00 2001 From: "jain.anoushka24" Date: Wed, 26 Mar 2025 13:57:39 +0100 Subject: [PATCH 016/157] Enhance CurationModel: Add split_units validation --- src/spikeinterface/curation/curation_model.py | 71 +++++++++++++++---- .../curation/tests/test_curation_model.py | 29 ++++++++ 2 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 src/spikeinterface/curation/tests/test_curation_model.py diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index f7c7415543..b8802bb9a1 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -1,12 +1,11 @@ from pydantic import BaseModel, Field, field_validator, model_validator from typing import List, Dict, Union, Optional -from itertools import combinations - +from itertools import combinations, chain supported_curation_format_versions = {"1"} # TODO: splitting -# - add split_units to curation model +# - add split_units to curation model done # - add split_mode to curation model # - add split to apply_curation # - add split_units to SortingAnalyzer @@ -27,17 +26,20 @@ class ManualLabel(BaseModel): class CurationModel(BaseModel): format_version: str = Field(..., description="Version of the curation format") unit_ids: List[Union[int, str]] = Field(..., description="List of unit IDs") - label_definitions: Dict[str, LabelDefinition] = Field(..., description="Dictionary of label definitions") - manual_labels: List[ManualLabel] = Field(..., description="List of manual labels") - removed_units: List[Union[int, str]] = Field(..., description="List of removed unit IDs") - merge_unit_groups: List[List[Union[int, str]]] = Field(..., description="List of groups of units to be merged") + label_definitions: Optional[Dict[str, LabelDefinition]] = Field(default = None, description="Dictionary of label definitions") + manual_labels: Optional[List[ManualLabel]]= Field(default = None, description="List of manual labels") + removed_units: Optional[List[Union[int, str]]] = Field(default = None, description="List of removed unit IDs") + merge_unit_groups: Optional[List[List[Union[int, str]]]] = Field(default = None, description="List of groups of units to be merged") merge_new_unit_ids: Optional[List[Union[int, str]]] = Field( default=None, description="List of new unit IDs after merging" ) - split_units: Optional[Dict[Union[int, str], List[List[int]]]] = Field( + split_units: Optional[Dict[Union[int, str], Union[List[List[int]], List[int]]]] = Field( default=None, description="Dictionary of units to be split" ) + + + @field_validator("format_version") def check_format_version(cls, v): if v not in supported_curation_format_versions: @@ -56,7 +58,7 @@ def add_label_definition_name(cls, v): @model_validator(mode="before") def check_manual_labels(cls, values): unit_ids = values["unit_ids"] - manual_labels = values["manual_labels"] + manual_labels = values.get("manual_labels") if manual_labels is None: values["manual_labels"] = [] else: @@ -84,6 +86,8 @@ def check_merge_unit_groups(cls, values): raise ValueError(f"Merge unit group unit_id {unit_id} is not in the unit list") if len(merge_group) < 2: raise ValueError("Merge unit groups must have at least 2 elements") + else: + values["merge_unit_groups"] = merge_unit_groups return values @model_validator(mode="before") @@ -101,19 +105,60 @@ def check_merge_new_unit_ids(cls, values): raise ValueError(f"New unit ID {new_unit_id} is already in the unit list") return values + + + @model_validator(mode="before") + def check_split_units(cls, values): + # we want to get split_units as a dictionary + # if method 1 Union[List[List[int] is used we want to check there no duplicates in any list of split_units: contacenate the list number of unique elements should be equal to the length of the list + # if method 2 Union[List[int] is used we want to check list dont have duplicate + # both these methods are possible + + split_units = values.get("split_units", {}) + unit_ids = values["unit_ids"] + for unit_id, split in split_units.items(): + if unit_id not in unit_ids: + raise ValueError(f"Split unit_id {unit_id} is not in the unit list") + if len(split) == 0: + raise ValueError(f"Split unit_id {unit_id} has no split") + if not isinstance(split[0], list): # uses method 1 + split = [split] + if len(split) > 1: # uses method 2 + # concatenate the list and check if the number of unique elements is equal to the length of the list + flatten = list(chain.from_iterable(split)) + if len(flatten) != len(set(flatten)): + raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") + # if len(set(sum(split))) != len(sum(split)): + # raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") + elif len(split) == 1: # uses method 1 + # check the list dont have duplicates + if len(split[0]) != len(set(split[0])): + raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") + return values + + @model_validator(mode="before") def check_removed_units(cls, values): unit_ids = values["unit_ids"] removed_units = values.get("removed_units", []) - for unit_id in removed_units: - if unit_id not in unit_ids: - raise ValueError(f"Removed unit_id {unit_id} is not in the unit list") + if removed_units is None: + + for unit_id in removed_units: + if unit_id not in unit_ids: + raise ValueError(f"Removed unit_id {unit_id} is not in the unit list") + + else: + values["removed_units"] = removed_units + return values @model_validator(mode="after") def validate_curation_dict(cls, values): labeled_unit_set = set([lbl.unit_id for lbl in values.manual_labels]) - merged_units_set = set(sum(values.merge_unit_groups, [])) + if len(values.merge_unit_groups)>0: + merged_units_set = set(sum(values.merge_unit_groups)) + else: + merged_units_set = set() removed_units_set = set(values.removed_units) unit_ids = values.unit_ids diff --git a/src/spikeinterface/curation/tests/test_curation_model.py b/src/spikeinterface/curation/tests/test_curation_model.py new file mode 100644 index 0000000000..648189a650 --- /dev/null +++ b/src/spikeinterface/curation/tests/test_curation_model.py @@ -0,0 +1,29 @@ +import pytest + +from pydantic import BaseModel, ValidationError, field_validator + + +from pathlib import Path +import json +import numpy as np + +from spikeinterface.curation.curation_model import CurationModel + +values_1 = { "format_version": "1", + "unit_ids": [1, 2, 3], + "split_units": {1: [1, 2], 2: [2, 3],3: [4,5]} +} + + +values_2 = { "format_version": "1", + "unit_ids": [1, 2, 3, 4], + "split_units": { + 1: [[1, 2], [3, 4]], + 2: [[2, 3], [4, 1]] + } +} + +curation_model1 = CurationModel(**values_1) +curation_model = CurationModel(**values_2) + + \ No newline at end of file From d4e0f84d4ea1f670a5f2cc54a718503b67fa0ca7 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 26 Mar 2025 10:45:07 -0400 Subject: [PATCH 017/157] Add splitting sorting to curation format --- src/spikeinterface/core/__init__.py | 7 +- src/spikeinterface/core/sorting_tools.py | 129 ++++++++++++++++++ .../curation/curation_format.py | 102 ++++++++------ src/spikeinterface/curation/curation_model.py | 70 +++++----- .../curation/tests/test_curation_format.py | 26 +++- .../curation/tests/test_curation_model.py | 38 +++--- 6 files changed, 279 insertions(+), 93 deletions(-) diff --git a/src/spikeinterface/core/__init__.py b/src/spikeinterface/core/__init__.py index fb2e173b3e..fdcfc73c27 100644 --- a/src/spikeinterface/core/__init__.py +++ b/src/spikeinterface/core/__init__.py @@ -109,7 +109,12 @@ get_chunk_with_margin, order_channels_by_depth, ) -from .sorting_tools import spike_vector_to_spike_trains, random_spikes_selection, apply_merges_to_sorting +from .sorting_tools import ( + spike_vector_to_spike_trains, + random_spikes_selection, + apply_merges_to_sorting, + apply_splits_to_sorting, +) from .waveform_tools import extract_waveforms_to_buffers, estimate_templates, estimate_templates_with_accumulator from .snippets_tools import snippets_from_sorting diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index ca8c731040..b60b0c1b94 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -231,6 +231,7 @@ def random_spikes_selection( return random_spikes_indices +### MERGING ZONE ### def apply_merges_to_sorting( sorting: BaseSorting, merge_unit_groups: list[list[int | str]] | list[tuple[int | str]], @@ -445,3 +446,131 @@ def generate_unit_ids_for_merge_group(old_unit_ids, merge_unit_groups, new_unit_ raise ValueError("wrong new_id_strategy") return new_unit_ids + + +### SPLITTING ZONE ### +def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extra=False, new_id_strategy="append"): + spikes = sorting.to_spike_vector().copy() + + num_spikes = sorting.count_num_spikes_per_unit() + + # take care of single-list splits + full_unit_splits = {} + for unit_id, split_indices in unit_splits.items(): + if not isinstance(split_indices[0], (list, np.ndarray)): + split_2 = np.arange(num_spikes[unit_id]) + split_2 = split_2[~np.isin(split_2, split_indices)] + new_split_indices = [split_indices, split_2] + else: + new_split_indices = split_indices + full_unit_splits[unit_id] = new_split_indices + + new_unit_ids = generate_unit_ids_for_split( + sorting.unit_ids, full_unit_splits, new_unit_ids=new_unit_ids, new_id_strategy=new_id_strategy + ) + old_unit_ids = sorting.unit_ids + all_unit_ids = list(old_unit_ids) + for split_unit, split_new_units in zip(full_unit_splits, new_unit_ids): + all_unit_ids.remove(split_unit) + all_unit_ids.extend(split_new_units) + + num_seg = sorting.get_num_segments() + assert num_seg == 1 + seg_lims = np.searchsorted(spikes["segment_index"], np.arange(0, num_seg + 2)) + segment_slices = [(seg_lims[i], seg_lims[i + 1]) for i in range(num_seg)] + + # using this function vaoid to use the mask approach and simplify a lot the algo + spike_vector_list = [spikes[s0:s1] for s0, s1 in segment_slices] + spike_indices = spike_vector_to_indices(spike_vector_list, sorting.unit_ids, absolute_index=True) + + # TODO deal with segments in splits + for unit_id in old_unit_ids: + if unit_id in full_unit_splits: + split_indices = full_unit_splits[unit_id] + new_split_ids = new_unit_ids[list(full_unit_splits.keys()).index(unit_id)] + + for split, new_unit_id in zip(split_indices, new_split_ids): + new_unit_index = all_unit_ids.index(new_unit_id) + for segment_index in range(num_seg): + spike_inds = spike_indices[segment_index][unit_id] + spikes["unit_index"][spike_inds[split]] = new_unit_index + else: + new_unit_index = all_unit_ids.index(unit_id) + for segment_index in range(num_seg): + spike_inds = spike_indices[segment_index][unit_id] + spikes["unit_index"][spike_inds] = new_unit_index + sorting = NumpySorting(spikes, sorting.sampling_frequency, all_unit_ids) + + if return_extra: + return sorting, new_unit_ids + else: + return sorting + + +def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, new_id_strategy="append"): + """ + Function to generate new units ids during a merging procedure. If new_units_ids + are provided, it will return these unit ids, checking that they have the the same + length as `merge_unit_groups`. + + Parameters + ---------- + old_unit_ids : np.array + The old unit_ids. + unit_splits : dict + + new_unit_ids : list | None, default: None + Optional new unit_ids for merged units. If given, it needs to have the same length as `merge_unit_groups`. + If None, new ids will be generated. + new_id_strategy : "append" | "take_first" | "join", default: "append" + The strategy that should be used, if `new_unit_ids` is None, to create new unit_ids. + + * "append" : new_units_ids will be added at the end of max(sorging.unit_ids) + * "split" : new_unit_ids will join unit_ids of groups with a "-". + Only works if unit_ids are str otherwise switch to "append" + + Returns + ------- + new_unit_ids : list of lists + The new units_ids associated with the merges. + """ + assert new_id_strategy in ["append", "split"], "new_id_strategy should be 'append' or 'split'" + old_unit_ids = np.asarray(old_unit_ids) + + if new_unit_ids is not None: + for split_unit, new_split_ids in zip(unit_splits.values(), new_unit_ids): + # then only doing a consistency check + assert len(split_unit) == len(new_split_ids), "new_unit_ids should have the same len as unit_splits.values" + # new_unit_ids can also be part of old_unit_ids only inside the same group: + assert all( + new_split_id not in old_unit_ids for new_split_id in new_split_ids + ), "new_unit_ids already exists but outside the split groups" + else: + dtype = old_unit_ids.dtype + new_unit_ids = [] + for unit_to_split, split_indices in unit_splits.items(): + num_splits = len(split_indices) + # select new_unit_ids greater that the max id, event greater than the numerical str ids + if new_id_strategy == "append": + if np.issubdtype(dtype, np.character): + # dtype str + if all(p.isdigit() for p in old_unit_ids): + # All str are digit : we can generate a max + m = max(int(p) for p in old_unit_ids) + 1 + new_unit_ids.append([str(m + i) for i in range(num_splits)]) + else: + # we cannot automatically find new names + new_unit_ids.append([f"split{i}" for i in range(num_splits)]) + else: + # dtype int + new_unit_ids.append(list(max(old_unit_ids) + 1 + np.arange(num_splits, dtype=dtype))) + old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) + elif new_id_strategy == "split": + if np.issubdtype(dtype, np.character): + new_unit_ids.append([f"{unit_to_split}-{i}" for i in np.arange(len(split_indices))]) + else: + # dtype int + new_unit_ids.append(list(max(old_unit_ids) + 1 + np.arange(num_splits, dtype=dtype))) + old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) + + return new_unit_ids diff --git a/src/spikeinterface/curation/curation_format.py b/src/spikeinterface/curation/curation_format.py index 186bb34568..540c508575 100644 --- a/src/spikeinterface/curation/curation_format.py +++ b/src/spikeinterface/curation/curation_format.py @@ -2,9 +2,9 @@ import copy import numpy as np +from itertools import chain -from spikeinterface import curation -from spikeinterface.core import BaseSorting, SortingAnalyzer, apply_merges_to_sorting +from spikeinterface.core import BaseSorting, SortingAnalyzer, apply_merges_to_sorting, apply_splits_to_sorting from spikeinterface.curation.curation_model import CurationModel @@ -187,16 +187,15 @@ def curation_label_to_dataframe(curation_dict_or_model: dict | CurationModel): return labels -def apply_curation_labels( - sorting: BaseSorting, new_unit_ids: list[int, str], curation_dict_or_model: dict | CurationModel -): +def apply_curation_labels(sorting: BaseSorting, curation_dict_or_model: dict | CurationModel): """ - Apply manual labels after merges. + Apply manual labels after merges/splits. Rules: - * label for non merge is applied first + * label for non merged units is applied first * for merged group, when exclusive=True, if all have the same label then this label is applied * for merged group, when exclusive=False, if one unit has the label then the new one have also it + * for split units, the original label is applied to all split units """ if isinstance(curation_dict_or_model, dict): curation_model = CurationModel(**curation_dict_or_model) @@ -206,37 +205,51 @@ def apply_curation_labels( # Please note that manual_labels is done on the unit_ids before the merge!!! manual_labels = curation_label_to_vectors(curation_model) - # apply on non merged + # apply on non merged / split + merge_new_unit_ids = curation_model.merge_new_unit_ids if curation_model.merge_new_unit_ids is not None else [] + split_new_unit_ids = ( + list(chain(*curation_model.split_new_unit_ids)) if curation_model.split_new_unit_ids is not None else [] + ) + merged_split_units = merge_new_unit_ids + split_new_unit_ids for key, values in manual_labels.items(): all_values = np.zeros(sorting.unit_ids.size, dtype=values.dtype) for unit_ind, unit_id in enumerate(sorting.unit_ids): - if unit_id not in new_unit_ids: + if unit_id not in merged_split_units: ind = list(curation_model.unit_ids).index(unit_id) all_values[unit_ind] = values[ind] sorting.set_property(key, all_values) - for new_unit_id, old_group_ids in zip(new_unit_ids, curation_model.merge_unit_groups): - for label_key, label_def in curation_model.label_definitions.items(): - if label_def.exclusive: - group_values = [] - for unit_id in old_group_ids: - ind = list(curation_model.unit_ids).index(unit_id) - value = manual_labels[label_key][ind] - if value != "": - group_values.append(value) - if len(set(group_values)) == 1: - # all group has the same label or empty - sorting.set_property(key, values=group_values[:1], ids=[new_unit_id]) - else: - - for key in label_def.label_options: + # merges + if len(merge_new_unit_ids) > 0: + for new_unit_id, old_group_ids in zip(curation_model.merge_new_unit_ids, curation_model.merge_unit_groups): + for label_key, label_def in curation_model.label_definitions.items(): + if label_def.exclusive: group_values = [] for unit_id in old_group_ids: ind = list(curation_model.unit_ids).index(unit_id) - value = manual_labels[key][ind] - group_values.append(value) - new_value = np.any(group_values) - sorting.set_property(key, values=[new_value], ids=[new_unit_id]) + value = manual_labels[label_key][ind] + if value != "": + group_values.append(value) + if len(set(group_values)) == 1: + # all group has the same label or empty + sorting.set_property(key, values=group_values[:1], ids=[new_unit_id]) + else: + for key in label_def.label_options: + group_values = [] + for unit_id in old_group_ids: + ind = list(curation_model.unit_ids).index(unit_id) + value = manual_labels[key][ind] + group_values.append(value) + new_value = np.any(group_values) + sorting.set_property(key, values=[new_value], ids=[new_unit_id]) + + # splits + if len(split_new_unit_ids) > 0: + for new_unit_index, old_unit in enumerate(curation_model.split_units): + for label_key, label_def in curation_model.label_definitions.items(): + ind = list(curation_model.unit_ids).index(old_unit) + value = manual_labels[label_key][ind] + sorting.set_property(label_key, values=[value], ids=[curation_model.split_new_unit_ids[new_unit_index]]) def apply_curation( @@ -255,7 +268,8 @@ def apply_curation( Steps are done in this order: 1. Apply removal using curation_dict["removed_units"] 2. Apply merges using curation_dict["merge_unit_groups"] - 3. Set labels using curation_dict["manual_labels"] + 3. Apply splits using curation_dict["split_units"] + 4. Set labels using curation_dict["manual_labels"] A new Sorting or SortingAnalyzer (in memory) is returned. The user (an adult) has the responsability to save it somewhere (or not). @@ -304,14 +318,25 @@ def apply_curation( if isinstance(sorting_or_analyzer, BaseSorting): sorting = sorting_or_analyzer sorting = sorting.remove_units(curation_model.removed_units) - sorting, _, new_unit_ids = apply_merges_to_sorting( - sorting, - curation_model.merge_unit_groups, - censor_ms=censor_ms, - return_extra=True, - new_id_strategy=new_id_strategy, - ) - apply_curation_labels(sorting, new_unit_ids, curation_model) + new_unit_ids = sorting.unit_ids + if len(curation_model.merge_unit_groups) > 0: + sorting, _, new_unit_ids = apply_merges_to_sorting( + sorting, + curation_model.merge_unit_groups, + censor_ms=censor_ms, + return_extra=True, + new_id_strategy=new_id_strategy, + ) + curation_model.merge_new_unit_ids = new_unit_ids + if len(curation_model.split_units) > 0: + sorting, new_unit_ids = apply_splits_to_sorting( + sorting, + curation_model.split_units, + new_id_strategy=new_id_strategy, + return_extra=True, + ) + curation_model.split_new_unit_ids = new_unit_ids + apply_curation_labels(sorting, curation_model) return sorting elif isinstance(sorting_or_analyzer, SortingAnalyzer): @@ -330,9 +355,10 @@ def apply_curation( verbose=verbose, **job_kwargs, ) + curation_model.merge_new_unit_ids = new_unit_ids else: new_unit_ids = [] - apply_curation_labels(analyzer.sorting, new_unit_ids, curation_model) + apply_curation_labels(analyzer.sorting, curation_model) return analyzer else: raise TypeError( diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index b8802bb9a1..0ada9c4777 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -1,13 +1,14 @@ from pydantic import BaseModel, Field, field_validator, model_validator from typing import List, Dict, Union, Optional from itertools import combinations, chain + supported_curation_format_versions = {"1"} # TODO: splitting -# - add split_units to curation model done -# - add split_mode to curation model -# - add split to apply_curation +# - add split_units to curation model done V +# - add split_mode to curation model X +# - add split to apply_curation V # - add split_units to SortingAnalyzer # - add _split_units to extensions @@ -26,19 +27,23 @@ class ManualLabel(BaseModel): class CurationModel(BaseModel): format_version: str = Field(..., description="Version of the curation format") unit_ids: List[Union[int, str]] = Field(..., description="List of unit IDs") - label_definitions: Optional[Dict[str, LabelDefinition]] = Field(default = None, description="Dictionary of label definitions") - manual_labels: Optional[List[ManualLabel]]= Field(default = None, description="List of manual labels") - removed_units: Optional[List[Union[int, str]]] = Field(default = None, description="List of removed unit IDs") - merge_unit_groups: Optional[List[List[Union[int, str]]]] = Field(default = None, description="List of groups of units to be merged") + label_definitions: Optional[Dict[str, LabelDefinition]] = Field( + default=None, description="Dictionary of label definitions" + ) + manual_labels: Optional[List[ManualLabel]] = Field(default=None, description="List of manual labels") + removed_units: Optional[List[Union[int, str]]] = Field(default=None, description="List of removed unit IDs") + merge_unit_groups: Optional[List[List[Union[int, str]]]] = Field( + default=None, description="List of groups of units to be merged" + ) merge_new_unit_ids: Optional[List[Union[int, str]]] = Field( - default=None, description="List of new unit IDs after merging" + default=None, description="List of new unit IDs for each merge group" ) split_units: Optional[Dict[Union[int, str], Union[List[List[int]], List[int]]]] = Field( - default=None, description="Dictionary of units to be split" + default=None, description="Dictionary of units to be split. TODO more description needed" + ) + split_new_unit_ids: Optional[List[Union[int, str]]] = Field( + default=None, description="List of new unit IDs for each unit split" ) - - - @field_validator("format_version") def check_format_version(cls, v): @@ -46,14 +51,16 @@ def check_format_version(cls, v): raise ValueError(f"Format version ({v}) not supported. Only {supported_curation_format_versions} are valid") return v - @field_validator("label_definitions", mode="before") - def add_label_definition_name(cls, v): - if v is None: - v = {} + @model_validator(mode="before") + def add_label_definition_name(cls, values): + label_definitions = values.get("label_definitions") + if label_definitions is None: + label_definitions = {} else: - for key in list(v.keys()): - v[key]["name"] = key - return v + for key in list(label_definitions.keys()): + label_definitions[key]["name"] = key + values["label_definitions"] = label_definitions + return values @model_validator(mode="before") def check_manual_labels(cls, values): @@ -105,15 +112,9 @@ def check_merge_new_unit_ids(cls, values): raise ValueError(f"New unit ID {new_unit_id} is already in the unit list") return values - - @model_validator(mode="before") def check_split_units(cls, values): - # we want to get split_units as a dictionary - # if method 1 Union[List[List[int] is used we want to check there no duplicates in any list of split_units: contacenate the list number of unique elements should be equal to the length of the list - # if method 2 Union[List[int] is used we want to check list dont have duplicate # both these methods are possible - split_units = values.get("split_units", {}) unit_ids = values["unit_ids"] for unit_id, split in split_units.items(): @@ -121,32 +122,30 @@ def check_split_units(cls, values): raise ValueError(f"Split unit_id {unit_id} is not in the unit list") if len(split) == 0: raise ValueError(f"Split unit_id {unit_id} has no split") - if not isinstance(split[0], list): # uses method 1 + if not isinstance(split[0], list): # uses method 1 split = [split] - if len(split) > 1: # uses method 2 + if len(split) > 1: # uses method 2 # concatenate the list and check if the number of unique elements is equal to the length of the list flatten = list(chain.from_iterable(split)) if len(flatten) != len(set(flatten)): raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") - # if len(set(sum(split))) != len(sum(split)): - # raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") - elif len(split) == 1: # uses method 1 + elif len(split) == 1: # uses method 1 # check the list dont have duplicates if len(split[0]) != len(set(split[0])): raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") + values["split_units"] = split_units return values - @model_validator(mode="before") def check_removed_units(cls, values): unit_ids = values["unit_ids"] removed_units = values.get("removed_units", []) if removed_units is None: - + for unit_id in removed_units: if unit_id not in unit_ids: raise ValueError(f"Removed unit_id {unit_id} is not in the unit list") - + else: values["removed_units"] = removed_units @@ -155,10 +154,7 @@ def check_removed_units(cls, values): @model_validator(mode="after") def validate_curation_dict(cls, values): labeled_unit_set = set([lbl.unit_id for lbl in values.manual_labels]) - if len(values.merge_unit_groups)>0: - merged_units_set = set(sum(values.merge_unit_groups)) - else: - merged_units_set = set() + merged_units_set = set(chain.from_iterable(values.merge_unit_groups)) removed_units_set = set(values.removed_units) unit_ids = values.unit_ids diff --git a/src/spikeinterface/curation/tests/test_curation_format.py b/src/spikeinterface/curation/tests/test_curation_format.py index 0d9562f404..45c1ad6b25 100644 --- a/src/spikeinterface/curation/tests/test_curation_format.py +++ b/src/spikeinterface/curation/tests/test_curation_format.py @@ -87,6 +87,16 @@ "removed_units": ["u31", "u42"], # Can not be in the merged_units } +curation_with_split = { + "format_version": "1", + "unit_ids": [1, 2, 3, 6, 10, 14, 20, 31, 42], + "split_units": { + 1: np.arange(10), + 2: np.arange(10, 20), + }, +} + + # This is a failure example with duplicated merge duplicate_merge = curation_ids_int.copy() duplicate_merge["merge_unit_groups"] = [[3, 6, 10], [10, 14, 20]] @@ -173,7 +183,7 @@ def test_curation_label_to_dataframe(): def test_apply_curation(): recording, sorting = generate_ground_truth_recording(durations=[10.0], num_units=9, seed=2205) - sorting._main_ids = np.array([1, 2, 3, 6, 10, 14, 20, 31, 42]) + sorting = sorting.rename_units(np.array([1, 2, 3, 6, 10, 14, 20, 31, 42])) analyzer = create_sorting_analyzer(sorting, recording, sparse=False) sorting_curated = apply_curation(sorting, curation_ids_int) @@ -185,6 +195,20 @@ def test_apply_curation(): assert "quality" in analyzer_curated.sorting.get_property_keys() +def test_apply_curation_with_split(): + recording, sorting = generate_ground_truth_recording(durations=[10.0], num_units=9, seed=2205) + sorting = sorting.rename_units(np.array([1, 2, 3, 6, 10, 14, 20, 31, 42])) + analyzer = create_sorting_analyzer(sorting, recording, sparse=False) + + sorting_curated = apply_curation(sorting, curation_with_split) + assert len(sorting_curated.unit_ids) == len(sorting.unit_ids) + 2 + + assert 1 not in sorting_curated.unit_ids + assert 2 not in sorting_curated.unit_ids + assert 43 in sorting_curated.unit_ids + assert 44 in sorting_curated.unit_ids + + if __name__ == "__main__": test_curation_format_validation() test_curation_format_validation() diff --git a/src/spikeinterface/curation/tests/test_curation_model.py b/src/spikeinterface/curation/tests/test_curation_model.py index 648189a650..9dbc6fac22 100644 --- a/src/spikeinterface/curation/tests/test_curation_model.py +++ b/src/spikeinterface/curation/tests/test_curation_model.py @@ -1,29 +1,35 @@ import pytest -from pydantic import BaseModel, ValidationError, field_validator - - +from pydantic import ValidationError from pathlib import Path -import json import numpy as np from spikeinterface.curation.curation_model import CurationModel -values_1 = { "format_version": "1", - "unit_ids": [1, 2, 3], - "split_units": {1: [1, 2], 2: [2, 3],3: [4,5]} -} +valid_split_1 = {"format_version": "1", "unit_ids": [1, 2, 3], "split_units": {1: [1, 2], 2: [2, 3], 3: [4, 5]}} -values_2 = { "format_version": "1", +valid_split_2 = { + "format_version": "1", "unit_ids": [1, 2, 3, 4], - "split_units": { - 1: [[1, 2], [3, 4]], - 2: [[2, 3], [4, 1]] - } + "split_units": {1: [[1, 2], [3, 4]], 2: [[2, 3], [4, 1]]}, } -curation_model1 = CurationModel(**values_1) -curation_model = CurationModel(**values_2) +invalid_split_1 = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "split_units": {1: [[1, 2], [2, 3]], 2: [2, 3], 3: [4, 5]}, +} + +invalid_split_2 = {"format_version": "1", "unit_ids": [1, 2, 3], "split_units": {4: [[1, 2], [2, 3]]}} + + +def test_unit_split(): + CurationModel(**valid_split_1) + CurationModel(**valid_split_2) - \ No newline at end of file + # shold raise error + with pytest.raises(ValidationError): + CurationModel(**invalid_split_1) + with pytest.raises(ValidationError): + CurationModel(**invalid_split_2) From c7316bb2fb803b59bb99c67a3eb93a3d809da107 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 26 Mar 2025 11:55:13 -0400 Subject: [PATCH 018/157] (wip) Add split_units to SortingAnalyzer --- src/spikeinterface/core/sorting_tools.py | 48 +++++- src/spikeinterface/core/sortinganalyzer.py | 162 ++++++++++++++++++--- 2 files changed, 185 insertions(+), 25 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index b60b0c1b94..292780bb38 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -468,11 +468,7 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr new_unit_ids = generate_unit_ids_for_split( sorting.unit_ids, full_unit_splits, new_unit_ids=new_unit_ids, new_id_strategy=new_id_strategy ) - old_unit_ids = sorting.unit_ids - all_unit_ids = list(old_unit_ids) - for split_unit, split_new_units in zip(full_unit_splits, new_unit_ids): - all_unit_ids.remove(split_unit) - all_unit_ids.extend(split_new_units) + all_unit_ids = _get_ids_after_splitting(sorting.unit_ids, full_unit_splits, new_unit_ids) num_seg = sorting.get_num_segments() assert num_seg == 1 @@ -574,3 +570,45 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) return new_unit_ids + + +def _get_ids_after_splitting(old_unit_ids, split_units, new_unit_ids): + """ + Function to get the list of unique unit_ids after some splits, with given new_units_ids would + be provided. + + Every new unit_id will be added at the end if not already present. + + Parameters + ---------- + old_unit_ids : np.array + The old unit_ids. + split_units : dict + A dict of split units. Each element needs to have at least two elements (two units to split). + new_unit_ids : list | None + A new unit_ids for split units. If given, it needs to have the same length as `split_units` values. + + Returns + ------- + + all_unit_ids : The unit ids in the split sorting + The units_ids that will be present after splits + + """ + old_unit_ids = np.asarray(old_unit_ids) + dtype = old_unit_ids.dtype + if dtype.kind == "U": + # the new dtype can be longer + dtype = "U" + + assert len(new_unit_ids) == len(split_units), "new_unit_ids should have the same len as merge_unit_groups" + for new_unit_in_split, unit_to_split in zip(new_unit_ids, split_units.keys()): + assert len(new_unit_in_split) == len( + split_units[unit_to_split] + ), "new_unit_ids should have the same len as split_units values" + + all_unit_ids = list(old_unit_ids.copy()) + for split_unit, split_new_units in zip(split_units, new_unit_ids): + all_unit_ids.remove(split_unit) + all_unit_ids.extend(split_new_units) + return np.array(all_unit_ids, dtype=dtype) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index a53b4c5cb9..62a8e3b6aa 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -31,7 +31,12 @@ is_path_remote, clean_zarr_folder_name, ) -from .sorting_tools import generate_unit_ids_for_merge_group, _get_ids_after_merging +from .sorting_tools import ( + generate_unit_ids_for_merge_group, + _get_ids_after_merging, + generate_unit_ids_for_split, + _get_ids_after_splitting, +) from .job_tools import split_job_kwargs from .numpyextractors import NumpySorting from .sparsity import ChannelSparsity, estimate_sparsity @@ -867,17 +872,19 @@ def are_units_mergeable( else: return mergeable - def _save_or_select_or_merge( + def _save_or_select_or_merge_or_split( self, format="binary_folder", folder=None, unit_ids=None, merge_unit_groups=None, + split_units=None, censor_ms=None, merging_mode="soft", sparsity_overlap=0.75, verbose=False, - new_unit_ids=None, + merge_new_unit_ids=None, + split_new_unit_ids=None, backend_options=None, **job_kwargs, ) -> "SortingAnalyzer": @@ -896,6 +903,8 @@ def _save_or_select_or_merge( merge_unit_groups : list/tuple of lists/tuples or None, default: None A list of lists for every merge group. Each element needs to have at least two elements (two units to merge). If `merge_unit_groups` is not None, `new_unit_ids` must be given. + split_units : dict or None, default: None + A dictionary with the keys being the unit ids to split and the values being the split indices. censor_ms : None or float, default: None When merging units, any spikes violating this refractory period will be discarded. merging_mode : "soft" | "hard", default: "soft" @@ -904,8 +913,10 @@ def _save_or_select_or_merge( sparsity_overlap : float, default 0.75 The percentage of overlap that units should share in order to accept merges. If this criteria is not achieved, soft merging will not be performed. - new_unit_ids : list or None, default: None + merge_new_unit_ids : list or None, default: None The new unit ids for merged units. Required if `merge_unit_groups` is not None. + split_new_unit_ids : list or None, default: None + The new unit ids for split units. Required if `split_units` is not None. verbose : bool, default: False If True, output is verbose. backend_options : dict | None, default: None @@ -943,8 +954,8 @@ def _save_or_select_or_merge( ) for unit_index, unit_id in enumerate(all_unit_ids): - if unit_id in new_unit_ids: - merge_unit_group = tuple(merge_unit_groups[new_unit_ids.index(unit_id)]) + if unit_id in merge_new_unit_ids: + merge_unit_group = tuple(merge_unit_groups[merge_new_unit_ids.index(unit_id)]) if not mergeable[merge_unit_group]: raise Exception( f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " @@ -967,25 +978,35 @@ def _save_or_select_or_merge( # if the original sorting object is not available anymore (kilosort folder deleted, ....), take the copy sorting_provenance = self.sorting - if merge_unit_groups is None: + if merge_unit_groups is None and split_units is None: # when only some unit_ids then the sorting must be sliced # TODO check that unit_ids are in same order otherwise many extension do handle it properly!!!! sorting_provenance = sorting_provenance.select_units(unit_ids) - else: + elif merge_unit_groups is not None: + assert split_units is None, "split_units must be None when merge_unit_groups is None" from spikeinterface.core.sorting_tools import apply_merges_to_sorting sorting_provenance, keep_mask, _ = apply_merges_to_sorting( sorting=sorting_provenance, merge_unit_groups=merge_unit_groups, - new_unit_ids=new_unit_ids, + new_unit_ids=merge_new_unit_ids, censor_ms=censor_ms, return_extra=True, ) if censor_ms is None: # in this case having keep_mask None is faster instead of having a vector of ones keep_mask = None - # TODO: sam/pierre would create a curation field / curation.json with the applied merges. - # What do you think? + elif split_units is not None: + assert merge_unit_groups is None, "merge_unit_groups must be None when split_units is not None" + from spikeinterface.core.sorting_tools import apply_splits_to_sorting + + sorting_provenance = apply_splits_to_sorting( + sorting=sorting_provenance, + split_units=split_units, + new_unit_ids=split_new_unit_ids, + ) + # TODO: sam/pierre would create a curation field / curation.json with the applied merges. + # What do you think? backend_options = {} if backend_options is None else backend_options @@ -1034,24 +1055,31 @@ def _save_or_select_or_merge( recompute_dict = {} for extension_name, extension in sorted_extensions.items(): - if merge_unit_groups is None: + if merge_unit_groups is None and split_units is None: # copy full or select new_sorting_analyzer.extensions[extension_name] = extension.copy( new_sorting_analyzer, unit_ids=unit_ids ) - else: + elif merge_unit_groups is not None: # merge if merging_mode == "soft": new_sorting_analyzer.extensions[extension_name] = extension.merge( new_sorting_analyzer, merge_unit_groups=merge_unit_groups, - new_unit_ids=new_unit_ids, + new_unit_ids=merge_new_unit_ids, keep_mask=keep_mask, verbose=verbose, **job_kwargs, ) elif merging_mode == "hard": recompute_dict[extension_name] = extension.params + else: + # split + # TODO + print("Splitting extension needs to be implemented") + # new_sorting_analyzer.extensions[extension_name] = extension.split( + # new_sorting_analyzer, split_units=split_units, new_unit_ids=split_new_unit_ids, verbose=verbose + # ) if merge_unit_groups is not None and merging_mode == "hard" and len(recompute_dict) > 0: new_sorting_analyzer.compute_several_extensions(recompute_dict, save=True, verbose=verbose, **job_kwargs) @@ -1081,7 +1109,7 @@ def save_as(self, format="memory", folder=None, backend_options=None) -> "Sortin """ if format == "zarr": folder = clean_zarr_folder_name(folder) - return self._save_or_select_or_merge(format=format, folder=folder, backend_options=backend_options) + return self._save_or_select_or_merge_or_split(format=format, folder=folder, backend_options=backend_options) def select_units(self, unit_ids, format="memory", folder=None) -> "SortingAnalyzer": """ @@ -1108,7 +1136,7 @@ def select_units(self, unit_ids, format="memory", folder=None) -> "SortingAnalyz # TODO check that unit_ids are in same order otherwise many extension do handle it properly!!!! if format == "zarr": folder = clean_zarr_folder_name(folder) - return self._save_or_select_or_merge(format=format, folder=folder, unit_ids=unit_ids) + return self._save_or_select_or_merge_or_split(format=format, folder=folder, unit_ids=unit_ids) def remove_units(self, remove_unit_ids, format="memory", folder=None) -> "SortingAnalyzer": """ @@ -1136,7 +1164,7 @@ def remove_units(self, remove_unit_ids, format="memory", folder=None) -> "Sortin unit_ids = self.unit_ids[~np.isin(self.unit_ids, remove_unit_ids)] if format == "zarr": folder = clean_zarr_folder_name(folder) - return self._save_or_select_or_merge(format=format, folder=folder, unit_ids=unit_ids) + return self._save_or_select_or_merge_or_split(format=format, folder=folder, unit_ids=unit_ids) def merge_units( self, @@ -1222,7 +1250,7 @@ def merge_units( ) all_unit_ids = _get_ids_after_merging(self.unit_ids, merge_unit_groups, new_unit_ids=new_unit_ids) - new_analyzer = self._save_or_select_or_merge( + new_analyzer = self._save_or_select_or_merge_or_split( format=format, folder=folder, merge_unit_groups=merge_unit_groups, @@ -1231,7 +1259,80 @@ def merge_units( merging_mode=merging_mode, sparsity_overlap=sparsity_overlap, verbose=verbose, - new_unit_ids=new_unit_ids, + merge_new_unit_ids=new_unit_ids, + **job_kwargs, + ) + if return_new_unit_ids: + return new_analyzer, new_unit_ids + else: + return new_analyzer + + def split_units( + self, + split_units: dict[list[str | int], list[int] | list[list[int]]], + new_unit_ids: list[list[int | str]] | None = None, + new_id_strategy: str = "append", + return_new_unit_ids: bool = False, + format: str = "memory", + folder: Path | str | None = None, + verbose: bool = False, + **job_kwargs, + ) -> "SortingAnalyzer | tuple[SortingAnalyzer, list[int | str]]": + """ + This method is equivalent to `save_as()` but with a list of splits that have to be achieved. + Split units by creating a new SortingAnalyzer object with the appropriate splits + + Extensions are also updated to display the split `unit_ids`. + + Parameters + ---------- + split_units : dict + A dictionary with the keys being the unit ids to split and the values being the split indices. + new_unit_ids : None | list, default: None + A new unit_ids for split units. If given, it needs to have the same length as `merge_unit_groups`. If None, + merged units will have the first unit_id of every lists of merges + new_id_strategy : "append" | "split", default: "append" + The strategy that should be used, if `new_unit_ids` is None, to create new unit_ids. + + * "append" : new_units_ids will be added at the end of max(sorting.unit_ids) + * "split" : new_unit_ids will be the original unit_id to split with -{subsplit} + return_new_unit_ids : bool, default False + Alse return new_unit_ids which are the ids of the new units. + folder : Path | None, default: None + The new folder where the analyzer with merged units is copied for `format` "binary_folder" or "zarr" + format : "memory" | "binary_folder" | "zarr", default: "memory" + The format of SortingAnalyzer + verbose : bool, default: False + Whether to display calculations (such as sparsity estimation) + + Returns + ------- + analyzer : SortingAnalyzer + The newly create `SortingAnalyzer` with the selected units + """ + + if format == "zarr": + folder = clean_zarr_folder_name(folder) + + if len(split_units) == 0: + # TODO I think we should raise an error or at least make a copy and not return itself + if return_new_unit_ids: + return self, [] + else: + return self + + # TODO: add some checks + + new_unit_ids = generate_unit_ids_for_split(self.unit_ids, split_units, new_unit_ids, new_id_strategy) + all_unit_ids = _get_ids_after_splitting(self.unit_ids, split_units, new_unit_ids=new_unit_ids) + + new_analyzer = self._save_or_select_or_merge_or_split( + format=format, + folder=folder, + split_units=split_units, + unit_ids=all_unit_ids, + verbose=verbose, + split_new_unit_ids=new_unit_ids, **job_kwargs, ) if return_new_unit_ids: @@ -1243,7 +1344,7 @@ def copy(self): """ Create a a copy of SortingAnalyzer with format "memory". """ - return self._save_or_select_or_merge(format="memory", folder=None) + return self._save_or_select_or_merge_or_split(format="memory", folder=None) def is_read_only(self) -> bool: if self.format == "memory": @@ -2048,6 +2149,10 @@ def _merge_extension_data( # must be implemented in subclass raise NotImplementedError + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # must be implemented in subclass + raise NotImplementedError + def _get_pipeline_nodes(self): # must be implemented in subclass only if use_nodepipeline=True raise NotImplementedError @@ -2283,6 +2388,23 @@ def merge( new_extension.save() return new_extension + def split( + self, + new_sorting_analyzer, + split_units, + new_unit_ids, + verbose=False, + **job_kwargs, + ): + new_extension = self.__class__(new_sorting_analyzer) + new_extension.params = self.params.copy() + new_extension.data = self._split_extension_data( + split_units, new_unit_ids, new_sorting_analyzer, verbose=verbose, **job_kwargs + ) + new_extension.run_info = copy(self.run_info) + new_extension.save() + return new_extension + def run(self, save=True, **kwargs): if save and not self.sorting_analyzer.is_read_only(): # NB: this call to _save_params() also resets the folder or zarr group From 2e21923412319a1c496a463ef17f65d6214f3b91 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 26 Mar 2025 15:32:48 -0400 Subject: [PATCH 019/157] Add split_units to sorting analyzer --- .../core/analyzer_extension_core.py | 52 ++++++++- src/spikeinterface/core/sorting_tools.py | 30 +++--- src/spikeinterface/core/sortinganalyzer.py | 100 ++++++++++++------ .../postprocessing/amplitude_scalings.py | 3 + .../postprocessing/correlograms.py | 9 +- src/spikeinterface/postprocessing/isi.py | 24 +++++ .../postprocessing/principal_component.py | 4 + .../postprocessing/spike_amplitudes.py | 4 + .../postprocessing/spike_locations.py | 4 + .../postprocessing/template_metrics.py | 21 ++++ .../postprocessing/template_similarity.py | 53 ++++++++++ .../postprocessing/unit_locations.py | 26 ++++- .../quality_metric_calculator.py | 28 +++++ 13 files changed, 305 insertions(+), 53 deletions(-) diff --git a/src/spikeinterface/core/analyzer_extension_core.py b/src/spikeinterface/core/analyzer_extension_core.py index 447bbe562e..4834e864d5 100644 --- a/src/spikeinterface/core/analyzer_extension_core.py +++ b/src/spikeinterface/core/analyzer_extension_core.py @@ -94,6 +94,11 @@ def _merge_extension_data( new_data["random_spikes_indices"] = np.flatnonzero(selected_mask[keep_mask]) return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + new_data = dict() + new_data["random_spikes_indices"] = self.data["random_spikes_indices"].copy() + return new_data + def _get_data(self): return self.data["random_spikes_indices"] @@ -245,8 +250,6 @@ def _select_extension_data(self, unit_ids): def _merge_extension_data( self, merge_unit_groups, new_unit_ids, new_sorting_analyzer, keep_mask=None, verbose=False, **job_kwargs ): - new_data = dict() - waveforms = self.data["waveforms"] some_spikes = self.sorting_analyzer.get_extension("random_spikes").get_random_spikes() if keep_mask is not None: @@ -277,6 +280,11 @@ def _merge_extension_data( return dict(waveforms=waveforms) + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # splitting only affects random spikes, not waveforms + new_data = dict(waveforms=self.data["waveforms"].copy()) + return new_data + def get_waveforms_one_unit(self, unit_id, force_dense: bool = False): """ Returns the waveforms of a unit id. @@ -556,6 +564,42 @@ def _merge_extension_data( return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + new_data = dict() + for operator, arr in self.data.items(): + # we first copy the unsplit units + new_array = np.zeros((len(new_sorting_analyzer.unit_ids), arr.shape[1], arr.shape[2]), dtype=arr.dtype) + new_analyzer_unit_ids = list(new_sorting_analyzer.unit_ids) + unsplit_unit_ids = [unit_id for unit_id in self.sorting_analyzer.unit_ids if unit_id not in split_units] + new_indices = np.array([new_analyzer_unit_ids.index(unit_id) for unit_id in unsplit_unit_ids]) + old_indices = self.sorting_analyzer.sorting.ids_to_indices(unsplit_unit_ids) + new_array[new_indices, ...] = arr[old_indices, ...] + + for split_unit_id, new_splits in zip(split_units, new_unit_ids): + if new_sorting_analyzer.has_extension("waveforms"): + for new_unit_id in new_splits: + split_unit_index = new_sorting_analyzer.sorting.id_to_index(new_unit_id) + wfs = new_sorting_analyzer.get_extension("waveforms").get_waveforms_one_unit( + new_unit_id, force_dense=True + ) + + if operator == "average": + arr = np.average(wfs, axis=0) + elif operator == "std": + arr = np.std(wfs, axis=0) + elif operator == "median": + arr = np.median(wfs, axis=0) + elif "percentile" in operator: + _, percentile = operator.splot("_") + arr = np.percentile(wfs, float(percentile), axis=0) + new_array[split_unit_index, ...] = arr + else: + old_template = arr[self.sorting_analyzer.sorting.ids_to_indices([split_unit_id])[0], ...] + new_indices = np.array([new_unit_ids.index(unit_id) for unit_id in new_splits]) + new_array[new_indices, ...] = np.tile(old_template, (len(new_splits), 1, 1)) + new_data[operator] = new_array + return new_data + def _get_data(self, operator="average", percentile=None, outputs="numpy"): if operator != "percentile": key = operator @@ -729,6 +773,10 @@ def _merge_extension_data( # this does not depend on units return self.data.copy() + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # this does not depend on units + return self.data.copy() + def _run(self, verbose=False): self.data["noise_levels"] = get_noise_levels( self.sorting_analyzer.recording, return_scaled=self.sorting_analyzer.return_scaled, **self.params diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index 292780bb38..d6ba6a7e6b 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -452,23 +452,14 @@ def generate_unit_ids_for_merge_group(old_unit_ids, merge_unit_groups, new_unit_ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extra=False, new_id_strategy="append"): spikes = sorting.to_spike_vector().copy() - num_spikes = sorting.count_num_spikes_per_unit() - # take care of single-list splits - full_unit_splits = {} - for unit_id, split_indices in unit_splits.items(): - if not isinstance(split_indices[0], (list, np.ndarray)): - split_2 = np.arange(num_spikes[unit_id]) - split_2 = split_2[~np.isin(split_2, split_indices)] - new_split_indices = [split_indices, split_2] - else: - new_split_indices = split_indices - full_unit_splits[unit_id] = new_split_indices + full_unit_splits = _get_full_unit_splits(unit_splits, sorting) new_unit_ids = generate_unit_ids_for_split( sorting.unit_ids, full_unit_splits, new_unit_ids=new_unit_ids, new_id_strategy=new_id_strategy ) all_unit_ids = _get_ids_after_splitting(sorting.unit_ids, full_unit_splits, new_unit_ids) + all_unit_ids = list(all_unit_ids) num_seg = sorting.get_num_segments() assert num_seg == 1 @@ -480,7 +471,7 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr spike_indices = spike_vector_to_indices(spike_vector_list, sorting.unit_ids, absolute_index=True) # TODO deal with segments in splits - for unit_id in old_unit_ids: + for unit_id in sorting.unit_ids: if unit_id in full_unit_splits: split_indices = full_unit_splits[unit_id] new_split_ids = new_unit_ids[list(full_unit_splits.keys()).index(unit_id)] @@ -572,6 +563,21 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne return new_unit_ids +def _get_full_unit_splits(unit_splits, sorting): + # take care of single-list splits + full_unit_splits = {} + num_spikes = sorting.count_num_spikes_per_unit() + for unit_id, split_indices in unit_splits.items(): + if not isinstance(split_indices[0], (list, np.ndarray)): + split_2 = np.arange(num_spikes[unit_id]) + split_2 = split_2[~np.isin(split_2, split_indices)] + new_split_indices = [split_indices, split_2] + else: + new_split_indices = split_indices + full_unit_splits[unit_id] = new_split_indices + return full_unit_splits + + def _get_ids_after_splitting(old_unit_ids, split_units, new_unit_ids): """ Function to get the list of unique unit_ids after some splits, with given new_units_ids would diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index 62a8e3b6aa..d7b96b32b3 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -36,6 +36,7 @@ _get_ids_after_merging, generate_unit_ids_for_split, _get_ids_after_splitting, + _get_full_unit_splits, ) from .job_tools import split_job_kwargs from .numpyextractors import NumpySorting @@ -939,36 +940,63 @@ def _save_or_select_or_merge_or_split( else: recording = None - if self.sparsity is not None and unit_ids is None and merge_unit_groups is None: - sparsity = self.sparsity - elif self.sparsity is not None and unit_ids is not None and merge_unit_groups is None: - sparsity_mask = self.sparsity.mask[np.isin(self.unit_ids, unit_ids), :] - sparsity = ChannelSparsity(sparsity_mask, unit_ids, self.channel_ids) - elif self.sparsity is not None and merge_unit_groups is not None: - all_unit_ids = unit_ids - sparsity_mask = np.zeros((len(all_unit_ids), self.sparsity.mask.shape[1]), dtype=bool) - mergeable, masks = self.are_units_mergeable( - merge_unit_groups, - sparsity_overlap=sparsity_overlap, - return_masks=True, - ) + has_removed = unit_ids is not None + has_merges = merge_unit_groups is not None + has_splits = split_units is not None + assert not has_merges if has_splits else True, "Cannot merge and split at the same time" + + if self.sparsity is not None: + if not has_removed and not has_merges and not has_splits: + # no changes in units + sparsity = self.sparsity + elif has_removed and not has_merges and not has_splits: + # remove units + sparsity_mask = self.sparsity.mask[np.isin(self.unit_ids, unit_ids), :] + sparsity = ChannelSparsity(sparsity_mask, unit_ids, self.channel_ids) + elif has_merges: + # merge units + all_unit_ids = unit_ids + sparsity_mask = np.zeros((len(all_unit_ids), self.sparsity.mask.shape[1]), dtype=bool) + mergeable, masks = self.are_units_mergeable( + merge_unit_groups, + sparsity_overlap=sparsity_overlap, + return_masks=True, + ) - for unit_index, unit_id in enumerate(all_unit_ids): - if unit_id in merge_new_unit_ids: - merge_unit_group = tuple(merge_unit_groups[merge_new_unit_ids.index(unit_id)]) - if not mergeable[merge_unit_group]: - raise Exception( - f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " - f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold or use " - "a hard merge." - ) + for unit_index, unit_id in enumerate(all_unit_ids): + if unit_id in merge_new_unit_ids: + merge_unit_group = tuple(merge_unit_groups[merge_new_unit_ids.index(unit_id)]) + if not mergeable[merge_unit_group]: + raise Exception( + f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " + f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold or use " + "a hard merge." + ) + else: + sparsity_mask[unit_index] = masks[merge_unit_group] else: - sparsity_mask[unit_index] = masks[merge_unit_group] - else: - # This means that the unit is already in the previous sorting - index = self.sorting.id_to_index(unit_id) - sparsity_mask[unit_index] = self.sparsity.mask[index] - sparsity = ChannelSparsity(sparsity_mask, list(all_unit_ids), self.channel_ids) + # This means that the unit is already in the previous sorting + index = self.sorting.id_to_index(unit_id) + sparsity_mask[unit_index] = self.sparsity.mask[index] + sparsity = ChannelSparsity(sparsity_mask, list(all_unit_ids), self.channel_ids) + elif has_splits: + # split units + all_unit_ids = unit_ids + original_unit_ids = self.unit_ids + sparsity_mask = np.zeros((len(all_unit_ids), self.sparsity.mask.shape[1]), dtype=bool) + for unit_index, unit_id in enumerate(all_unit_ids): + if unit_id not in original_unit_ids: + # then it is a new unit + # we assign the original sparsity + for split_unit, new_unit_ids in zip(split_units, split_new_unit_ids): + if unit_id in new_unit_ids: + original_unit_index = self.sorting.id_to_index(split_unit) + sparsity_mask[unit_index] = self.sparsity.mask[original_unit_index] + break + else: + original_unit_index = self.sorting.id_to_index(unit_id) + sparsity_mask[unit_index] = self.sparsity.mask[original_unit_index] + sparsity = ChannelSparsity(sparsity_mask, list(all_unit_ids), self.channel_ids) else: sparsity = None @@ -1002,7 +1030,7 @@ def _save_or_select_or_merge_or_split( sorting_provenance = apply_splits_to_sorting( sorting=sorting_provenance, - split_units=split_units, + unit_splits=split_units, new_unit_ids=split_new_unit_ids, ) # TODO: sam/pierre would create a curation field / curation.json with the applied merges. @@ -1075,13 +1103,14 @@ def _save_or_select_or_merge_or_split( recompute_dict[extension_name] = extension.params else: # split - # TODO - print("Splitting extension needs to be implemented") - # new_sorting_analyzer.extensions[extension_name] = extension.split( - # new_sorting_analyzer, split_units=split_units, new_unit_ids=split_new_unit_ids, verbose=verbose - # ) + try: + new_sorting_analyzer.extensions[extension_name] = extension.split( + new_sorting_analyzer, split_units=split_units, new_unit_ids=split_new_unit_ids, verbose=verbose + ) + except NotImplementedError: + recompute_dict[extension_name] = extension.params - if merge_unit_groups is not None and merging_mode == "hard" and len(recompute_dict) > 0: + if len(recompute_dict) > 0: new_sorting_analyzer.compute_several_extensions(recompute_dict, save=True, verbose=verbose, **job_kwargs) return new_sorting_analyzer @@ -1322,6 +1351,7 @@ def split_units( return self # TODO: add some checks + split_units = _get_full_unit_splits(split_units, self.sorting) new_unit_ids = generate_unit_ids_for_split(self.unit_ids, split_units, new_unit_ids, new_id_strategy) all_unit_ids = _get_ids_after_splitting(self.unit_ids, split_units, new_unit_ids=new_unit_ids) diff --git a/src/spikeinterface/postprocessing/amplitude_scalings.py b/src/spikeinterface/postprocessing/amplitude_scalings.py index 278151a930..ff926a998d 100644 --- a/src/spikeinterface/postprocessing/amplitude_scalings.py +++ b/src/spikeinterface/postprocessing/amplitude_scalings.py @@ -128,6 +128,9 @@ def _merge_extension_data( return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + return self.data.copy() + def _get_pipeline_nodes(self): recording = self.sorting_analyzer.recording diff --git a/src/spikeinterface/postprocessing/correlograms.py b/src/spikeinterface/postprocessing/correlograms.py index 5e30d7c68b..d41beb595f 100644 --- a/src/spikeinterface/postprocessing/correlograms.py +++ b/src/spikeinterface/postprocessing/correlograms.py @@ -154,9 +154,6 @@ def _merge_extension_data( if unit_involved_in_merge is False: old_to_new_unit_index_map[old_unit_index] = new_sorting_analyzer.sorting.id_to_index(old_unit) - need_to_append = False - delete_from = 1 - correlograms, new_bins = deepcopy(self.get_data()) for new_unit_id, merge_unit_group in zip(new_unit_ids, merge_unit_groups): @@ -188,6 +185,12 @@ def _merge_extension_data( new_data = dict(ccgs=new_correlograms, bins=new_bins) return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # TODO: for now we just copy + new_ccgs, new_bins = _compute_correlograms_on_sorting(new_sorting_analyzer.sorting, **self.params) + new_data = dict(ccgs=new_ccgs, bins=new_bins) + return new_data + def _run(self, verbose=False): ccgs, bins = _compute_correlograms_on_sorting(self.sorting_analyzer.sorting, **self.params) self.data["ccgs"] = ccgs diff --git a/src/spikeinterface/postprocessing/isi.py b/src/spikeinterface/postprocessing/isi.py index 542f829f21..03bd9d71a8 100644 --- a/src/spikeinterface/postprocessing/isi.py +++ b/src/spikeinterface/postprocessing/isi.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np +from itertools import chain from spikeinterface.core.sortinganalyzer import register_result_extension, AnalyzerExtension @@ -80,6 +81,29 @@ def _merge_extension_data( new_extension_data = dict(isi_histograms=new_isi_hists, bins=new_bins) return new_extension_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + new_bins = self.data["bins"] + arr = self.data["isi_histograms"] + num_dims = arr.shape[1] + all_new_units = new_sorting_analyzer.unit_ids + new_isi_hists = np.zeros((len(all_new_units), num_dims), dtype=arr.dtype) + + # compute all new isi at once + new_unit_ids_f = list(chain(*new_unit_ids)) + new_sorting = new_sorting_analyzer.sorting.select_units(new_unit_ids_f) + only_new_hist, _ = _compute_isi_histograms(new_sorting, **self.params) + + for unit_ind, unit_id in enumerate(all_new_units): + if unit_id not in new_unit_ids_f: + keep_unit_index = self.sorting_analyzer.sorting.id_to_index(unit_id) + new_isi_hists[unit_ind, :] = arr[keep_unit_index, :] + else: + new_unit_index = new_sorting.id_to_index(unit_id) + new_isi_hists[unit_ind, :] = only_new_hist[new_unit_index, :] + + new_extension_data = dict(isi_histograms=new_isi_hists, bins=new_bins) + return new_extension_data + def _run(self, verbose=False): isi_histograms, bins = _compute_isi_histograms(self.sorting_analyzer.sorting, **self.params) self.data["isi_histograms"] = isi_histograms diff --git a/src/spikeinterface/postprocessing/principal_component.py b/src/spikeinterface/postprocessing/principal_component.py index dd3a8febd7..c340b7ff50 100644 --- a/src/spikeinterface/postprocessing/principal_component.py +++ b/src/spikeinterface/postprocessing/principal_component.py @@ -149,6 +149,10 @@ def _merge_extension_data( new_data[k] = v return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # splitting only changes random spikes assignments + return self.data.copy() + def get_pca_model(self): """ Returns the scikit-learn PCA model objects. diff --git a/src/spikeinterface/postprocessing/spike_amplitudes.py b/src/spikeinterface/postprocessing/spike_amplitudes.py index 577dc948c3..4b7a4e8eae 100644 --- a/src/spikeinterface/postprocessing/spike_amplitudes.py +++ b/src/spikeinterface/postprocessing/spike_amplitudes.py @@ -92,6 +92,10 @@ def _merge_extension_data( return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # splitting only changes random spikes assignments + return self.data.copy() + def _get_pipeline_nodes(self): recording = self.sorting_analyzer.recording diff --git a/src/spikeinterface/postprocessing/spike_locations.py b/src/spikeinterface/postprocessing/spike_locations.py index 6995fc04da..c33b9bb8aa 100644 --- a/src/spikeinterface/postprocessing/spike_locations.py +++ b/src/spikeinterface/postprocessing/spike_locations.py @@ -105,6 +105,10 @@ def _merge_extension_data( ### in a merged could be different. Should be discussed return dict(spike_locations=new_spike_locations) + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + # splitting only changes random spikes assignments + return self.data.copy() + def _get_pipeline_nodes(self): from spikeinterface.sortingcomponents.peak_localization import get_localization_pipeline_nodes diff --git a/src/spikeinterface/postprocessing/template_metrics.py b/src/spikeinterface/postprocessing/template_metrics.py index d78b1e3809..e077dab482 100644 --- a/src/spikeinterface/postprocessing/template_metrics.py +++ b/src/spikeinterface/postprocessing/template_metrics.py @@ -8,6 +8,7 @@ import numpy as np import warnings +from itertools import chain from copy import deepcopy from spikeinterface.core.sortinganalyzer import register_result_extension, AnalyzerExtension @@ -195,6 +196,26 @@ def _merge_extension_data( new_data = dict(metrics=metrics) return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + import pandas as pd + + metric_names = self.params["metric_names"] + old_metrics = self.data["metrics"] + + all_unit_ids = new_sorting_analyzer.unit_ids + new_unit_ids_f = list(chain(*new_unit_ids)) + not_new_ids = all_unit_ids[~np.isin(all_unit_ids, new_unit_ids_f)] + + metrics = pd.DataFrame(index=all_unit_ids, columns=old_metrics.columns) + + metrics.loc[not_new_ids, :] = old_metrics.loc[not_new_ids, :] + metrics.loc[new_unit_ids_f, :] = self._compute_metrics( + new_sorting_analyzer, new_unit_ids_f, verbose, metric_names, **job_kwargs + ) + + new_data = dict(metrics=metrics) + return new_data + def _compute_metrics(self, sorting_analyzer, unit_ids=None, verbose=False, metric_names=None, **job_kwargs): """ Compute template metrics. diff --git a/src/spikeinterface/postprocessing/template_similarity.py b/src/spikeinterface/postprocessing/template_similarity.py index 1928e12edc..5469c7fe5a 100644 --- a/src/spikeinterface/postprocessing/template_similarity.py +++ b/src/spikeinterface/postprocessing/template_similarity.py @@ -2,6 +2,7 @@ import numpy as np import warnings +from itertools import chain from spikeinterface.core.sortinganalyzer import register_result_extension, AnalyzerExtension from spikeinterface.core.template_tools import get_dense_templates_array @@ -128,6 +129,58 @@ def _merge_extension_data( return dict(similarity=similarity) + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + num_shifts = int(self.params["max_lag_ms"] * self.sorting_analyzer.sampling_frequency / 1000) + all_templates_array = get_dense_templates_array( + new_sorting_analyzer, return_scaled=self.sorting_analyzer.return_scaled + ) + + new_unit_ids_f = list(chain(*new_unit_ids)) + keep = np.isin(new_sorting_analyzer.unit_ids, new_unit_ids_f) + new_templates_array = all_templates_array[keep, :, :] + if new_sorting_analyzer.sparsity is None: + new_sparsity = None + else: + new_sparsity = ChannelSparsity( + new_sorting_analyzer.sparsity.mask[keep, :], new_unit_ids_f, new_sorting_analyzer.channel_ids + ) + + new_similarity = compute_similarity_with_templates_array( + new_templates_array, + all_templates_array, + method=self.params["method"], + num_shifts=num_shifts, + support=self.params["support"], + sparsity=new_sparsity, + other_sparsity=new_sorting_analyzer.sparsity, + ) + + old_similarity = self.data["similarity"] + + all_new_unit_ids = new_sorting_analyzer.unit_ids + n = all_new_unit_ids.size + similarity = np.zeros((n, n), dtype=old_similarity.dtype) + + # copy old similarity + for unit_ind1, unit_id1 in enumerate(all_new_unit_ids): + if unit_id1 not in new_unit_ids_f: + old_ind1 = self.sorting_analyzer.sorting.id_to_index(unit_id1) + for unit_ind2, unit_id2 in enumerate(all_new_unit_ids): + if unit_id2 not in new_unit_ids_f: + old_ind2 = self.sorting_analyzer.sorting.id_to_index(unit_id2) + s = self.data["similarity"][old_ind1, old_ind2] + similarity[unit_ind1, unit_ind2] = s + similarity[unit_ind1, unit_ind2] = s + + # insert new similarity both way + for unit_ind, unit_id in enumerate(all_new_unit_ids): + if unit_id in new_unit_ids_f: + new_index = list(new_unit_ids_f).index(unit_id) + similarity[unit_ind, :] = new_similarity[new_index, :] + similarity[:, unit_ind] = new_similarity[new_index, :] + + return dict(similarity=similarity) + def _run(self, verbose=False): num_shifts = int(self.params["max_lag_ms"] * self.sorting_analyzer.sampling_frequency / 1000) templates_array = get_dense_templates_array( diff --git a/src/spikeinterface/postprocessing/unit_locations.py b/src/spikeinterface/postprocessing/unit_locations.py index 5618499770..ea297f7b6c 100644 --- a/src/spikeinterface/postprocessing/unit_locations.py +++ b/src/spikeinterface/postprocessing/unit_locations.py @@ -1,7 +1,7 @@ from __future__ import annotations import numpy as np -import warnings +from itertools import chain from spikeinterface.core.sortinganalyzer import register_result_extension, AnalyzerExtension from .localization_tools import _unit_location_methods @@ -88,6 +88,30 @@ def _merge_extension_data( return dict(unit_locations=unit_location) + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + old_unit_locations = self.data["unit_locations"] + num_dims = old_unit_locations.shape[1] + + method = self.params.get("method") + method_kwargs = self.params.copy() + method_kwargs.pop("method") + func = _unit_location_methods[method] + new_unit_ids_f = list(chain(*new_unit_ids)) + new_unit_locations = func(new_sorting_analyzer, unit_ids=new_unit_ids_f, **method_kwargs) + assert new_unit_locations.shape[0] == len(new_unit_ids_f) + + all_new_unit_ids = new_sorting_analyzer.unit_ids + unit_location = np.zeros((len(all_new_unit_ids), num_dims), dtype=old_unit_locations.dtype) + for unit_index, unit_id in enumerate(all_new_unit_ids): + if unit_id not in new_unit_ids_f: + old_index = self.sorting_analyzer.sorting.id_to_index(unit_id) + unit_location[unit_index] = old_unit_locations[old_index] + else: + new_index = list(new_unit_ids_f).index(unit_id) + unit_location[unit_index] = new_unit_locations[new_index] + + return dict(unit_locations=unit_location) + def _run(self, verbose=False): method = self.params.get("method") method_kwargs = self.params.copy() diff --git a/src/spikeinterface/qualitymetrics/quality_metric_calculator.py b/src/spikeinterface/qualitymetrics/quality_metric_calculator.py index 134849e70f..055fefc78c 100644 --- a/src/spikeinterface/qualitymetrics/quality_metric_calculator.py +++ b/src/spikeinterface/qualitymetrics/quality_metric_calculator.py @@ -3,6 +3,7 @@ from __future__ import annotations import warnings +from itertools import chain from copy import deepcopy import numpy as np @@ -158,6 +159,33 @@ def _merge_extension_data( new_data = dict(metrics=metrics) return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + import pandas as pd + + metric_names = self.params["metric_names"] + old_metrics = self.data["metrics"] + + all_unit_ids = new_sorting_analyzer.unit_ids + new_unit_ids_f = list(chain(*new_unit_ids)) + not_new_ids = all_unit_ids[~np.isin(all_unit_ids, new_unit_ids_f)] + + # this creates a new metrics dictionary, but the dtype for everything will be + # object. So we will need to fix this later after computing metrics + metrics = pd.DataFrame(index=all_unit_ids, columns=old_metrics.columns) + metrics.loc[not_new_ids, :] = old_metrics.loc[not_new_ids, :] + metrics.loc[new_unit_ids_f, :] = self._compute_metrics( + new_sorting_analyzer, new_unit_ids_f, verbose, metric_names, **job_kwargs + ) + + # we need to fix the dtypes after we compute everything because we have nans + # we can iterate through the columns and convert them back to the dtype + # of the original quality dataframe. + for column in old_metrics.columns: + metrics[column] = metrics[column].astype(old_metrics[column].dtype) + + new_data = dict(metrics=metrics) + return new_data + def _compute_metrics(self, sorting_analyzer, unit_ids=None, verbose=False, metric_names=None, **job_kwargs): """ Compute quality metrics. From 4afdb805c04bbdbe2ea6cc099341532824337150 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 26 Mar 2025 15:50:26 -0400 Subject: [PATCH 020/157] Propagate SortingAnalyzer.split_units to apply_curation --- src/spikeinterface/curation/curation_format.py | 11 +++++++++-- src/spikeinterface/curation/curation_model.py | 8 -------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/curation/curation_format.py b/src/spikeinterface/curation/curation_format.py index 540c508575..bc121090f1 100644 --- a/src/spikeinterface/curation/curation_format.py +++ b/src/spikeinterface/curation/curation_format.py @@ -356,8 +356,15 @@ def apply_curation( **job_kwargs, ) curation_model.merge_new_unit_ids = new_unit_ids - else: - new_unit_ids = [] + if len(curation_model.split_units) > 0: + analyzer, new_unit_ids = analyzer.split_units( + curation_model.split_units, + new_id_strategy=new_id_strategy, + return_new_unit_ids=True, + format="memory", + verbose=verbose, + ) + curation_model.split_new_unit_ids = new_unit_ids apply_curation_labels(analyzer.sorting, curation_model) return analyzer else: diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index 0ada9c4777..f5cc035676 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -5,14 +5,6 @@ supported_curation_format_versions = {"1"} -# TODO: splitting -# - add split_units to curation model done V -# - add split_mode to curation model X -# - add split to apply_curation V -# - add split_units to SortingAnalyzer -# - add _split_units to extensions - - class LabelDefinition(BaseModel): name: str = Field(..., description="Name of the label") label_options: List[str] = Field(..., description="List of possible label options") From 62bfb7f955718136511d7bdcbdb0986bd4f1a490 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 27 Mar 2025 09:22:33 -0400 Subject: [PATCH 021/157] Extend CurationModel tests --- src/spikeinterface/curation/curation_model.py | 28 ++- .../curation/tests/test_curation_model.py | 221 ++++++++++++++++-- 2 files changed, 221 insertions(+), 28 deletions(-) diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index f5cc035676..b50db1e69c 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -1,13 +1,14 @@ from pydantic import BaseModel, Field, field_validator, model_validator from typing import List, Dict, Union, Optional from itertools import combinations, chain +import numpy as np supported_curation_format_versions = {"1"} class LabelDefinition(BaseModel): name: str = Field(..., description="Name of the label") - label_options: List[str] = Field(..., description="List of possible label options") + label_options: List[str] = Field(..., description="List of possible label options", min_length=2) exclusive: bool = Field(..., description="Whether the label is exclusive") @@ -47,11 +48,16 @@ def check_format_version(cls, v): def add_label_definition_name(cls, values): label_definitions = values.get("label_definitions") if label_definitions is None: - label_definitions = {} - else: - for key in list(label_definitions.keys()): - label_definitions[key]["name"] = key - values["label_definitions"] = label_definitions + values["label_definitions"] = {} + return values + if isinstance(values["label_definitions"], dict): + if label_definitions is None: + label_definitions = {} + else: + for key in list(label_definitions.keys()): + if isinstance(label_definitions[key], dict): + label_definitions[key]["name"] = key + values["label_definitions"] = label_definitions return values @model_validator(mode="before") @@ -70,7 +76,11 @@ def check_manual_labels(cls, values): for label in labels: if label not in values["label_definitions"]: raise ValueError(f"Manual label {unit_id} has an unknown label {label}") - manual_label["labels"][label] = manual_label[label] + if label not in manual_label["labels"]: + if label in manual_label: + manual_label["labels"][label] = manual_label[label] + else: + raise ValueError(f"Manual label {unit_id} has no value for label {label}") if unit_id not in unit_ids: raise ValueError(f"Manual label unit_id {unit_id} is not in the unit list") return values @@ -114,7 +124,7 @@ def check_split_units(cls, values): raise ValueError(f"Split unit_id {unit_id} is not in the unit list") if len(split) == 0: raise ValueError(f"Split unit_id {unit_id} has no split") - if not isinstance(split[0], list): # uses method 1 + if not isinstance(split[0], (list, np.ndarray)): # uses method 1 split = [split] if len(split) > 1: # uses method 2 # concatenate the list and check if the number of unique elements is equal to the length of the list @@ -123,7 +133,7 @@ def check_split_units(cls, values): raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") elif len(split) == 1: # uses method 1 # check the list dont have duplicates - if len(split[0]) != len(set(split[0])): + if len(split[0]) != len(set(list(split[0]))): raise ValueError(f"Split unit_id {unit_id} has duplicate units in the split") values["split_units"] = split_units return values diff --git a/src/spikeinterface/curation/tests/test_curation_model.py b/src/spikeinterface/curation/tests/test_curation_model.py index 9dbc6fac22..12db0984c8 100644 --- a/src/spikeinterface/curation/tests/test_curation_model.py +++ b/src/spikeinterface/curation/tests/test_curation_model.py @@ -1,35 +1,218 @@ import pytest from pydantic import ValidationError -from pathlib import Path import numpy as np -from spikeinterface.curation.curation_model import CurationModel +from spikeinterface.curation.curation_model import CurationModel, LabelDefinition -valid_split_1 = {"format_version": "1", "unit_ids": [1, 2, 3], "split_units": {1: [1, 2], 2: [2, 3], 3: [4, 5]}} +# Test data for format version +def test_format_version(): + # Valid format version + CurationModel(format_version="1", unit_ids=[1, 2, 3]) + + # Invalid format version + with pytest.raises(ValidationError): + CurationModel(format_version="2", unit_ids=[1, 2, 3]) + with pytest.raises(ValidationError): + CurationModel(format_version="0", unit_ids=[1, 2, 3]) -valid_split_2 = { - "format_version": "1", - "unit_ids": [1, 2, 3, 4], - "split_units": {1: [[1, 2], [3, 4]], 2: [[2, 3], [4, 1]]}, -} -invalid_split_1 = { - "format_version": "1", - "unit_ids": [1, 2, 3], - "split_units": {1: [[1, 2], [2, 3]], 2: [2, 3], 3: [4, 5]}, -} +# Test data for label definitions +def test_label_definitions(): + valid_label_def = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "label_definitions": { + "quality": LabelDefinition(name="quality", label_options=["good", "noise"], exclusive=True), + "tags": LabelDefinition(name="tags", label_options=["burst", "slow", "fast"], exclusive=False), + }, + } -invalid_split_2 = {"format_version": "1", "unit_ids": [1, 2, 3], "split_units": {4: [[1, 2], [2, 3]]}} + model = CurationModel(**valid_label_def) + assert "quality" in model.label_definitions + assert model.label_definitions["quality"].name == "quality" + assert model.label_definitions["quality"].exclusive is True + + # Test invalid label definition + with pytest.raises(ValidationError): + LabelDefinition(name="quality", label_options=[], exclusive=True) # Empty options should be invalid +# Test manual labels +def test_manual_labels(): + valid_labels = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "label_definitions": { + "quality": LabelDefinition(name="quality", label_options=["good", "noise"], exclusive=True), + "tags": LabelDefinition(name="tags", label_options=["burst", "slow", "fast"], exclusive=False), + }, + "manual_labels": [ + {"unit_id": 1, "labels": {"quality": ["good"], "tags": ["burst", "fast"]}}, + {"unit_id": 2, "labels": {"quality": ["noise"]}}, + ], + } + + model = CurationModel(**valid_labels) + assert len(model.manual_labels) == 2 + + # Test invalid unit ID + invalid_unit = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "label_definitions": { + "quality": LabelDefinition(name="quality", label_options=["good", "noise"], exclusive=True) + }, + "manual_labels": [{"unit_id": 4, "labels": {"quality": ["good"]}}], # Non-existent unit + } + with pytest.raises(ValidationError): + CurationModel(**invalid_unit) + + # Test violation of exclusive label + invalid_exclusive = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "label_definitions": { + "quality": LabelDefinition(name="quality", label_options=["good", "noise"], exclusive=True) + }, + "manual_labels": [ + {"unit_id": 1, "labels": {"quality": ["good", "noise"]}} # Multiple values for exclusive label + ], + } + with pytest.raises(ValidationError): + CurationModel(**invalid_exclusive) + + +# Test merge functionality +def test_merge_units(): + valid_merge = { + "format_version": "1", + "unit_ids": [1, 2, 3, 4], + "merge_unit_groups": [[1, 2], [3, 4]], + "merge_new_unit_ids": [5, 6], + } + + model = CurationModel(**valid_merge) + assert len(model.merge_unit_groups) == 2 + assert len(model.merge_new_unit_ids) == 2 + + # Test invalid merge group (single unit) + invalid_merge_group = {"format_version": "1", "unit_ids": [1, 2, 3], "merge_unit_groups": [[1], [2, 3]]} + with pytest.raises(ValidationError): + CurationModel(**invalid_merge_group) + + # Test overlapping merge groups + invalid_overlap = {"format_version": "1", "unit_ids": [1, 2, 3], "merge_unit_groups": [[1, 2], [2, 3]]} + with pytest.raises(ValidationError): + CurationModel(**invalid_overlap) + + # Test merge new unit IDs length mismatch + invalid_new_ids = { + "format_version": "1", + "unit_ids": [1, 2, 3, 4], + "merge_unit_groups": [[1, 2], [3, 4]], + "merge_new_unit_ids": [5], # Missing one ID + } + with pytest.raises(ValidationError): + CurationModel(**invalid_new_ids) + + +# Test removed units +def test_removed_units(): + valid_remove = {"format_version": "1", "unit_ids": [1, 2, 3], "removed_units": [2]} + + model = CurationModel(**valid_remove) + assert len(model.removed_units) == 1 + + # Test removing non-existent unit + invalid_remove = {"format_version": "1", "unit_ids": [1, 2, 3], "removed_units": [4]} # Non-existent unit + with pytest.raises(ValidationError): + CurationModel(**invalid_remove) + + # Test conflict between merge and remove + invalid_merge_remove = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "merge_unit_groups": [[1, 2]], + "removed_units": [1], # Unit is both merged and removed + } + with pytest.raises(ValidationError): + CurationModel(**invalid_merge_remove) + + +# Test complete model with multiple operations +def test_complete_model(): + complete_model = { + "format_version": "1", + "unit_ids": [1, 2, 3, 4, 5], + "label_definitions": { + "quality": LabelDefinition(name="quality", label_options=["good", "noise"], exclusive=True), + "tags": LabelDefinition(name="tags", label_options=["burst", "slow"], exclusive=False), + }, + "manual_labels": [{"unit_id": 1, "labels": {"quality": ["good"], "tags": ["burst"]}}], + "merge_unit_groups": [[2, 3]], + "merge_new_unit_ids": [6], + "split_units": {4: [[1, 2], [3, 4]]}, + "removed_units": [5], + } + + model = CurationModel(**complete_model) + assert model.format_version == "1" + assert len(model.unit_ids) == 5 + assert len(model.label_definitions) == 2 + assert len(model.manual_labels) == 1 + assert len(model.merge_unit_groups) == 1 + assert len(model.merge_new_unit_ids) == 1 + assert len(model.split_units) == 1 + assert len(model.removed_units) == 1 + + +# Test unit splitting functionality def test_unit_split(): - CurationModel(**valid_split_1) - CurationModel(**valid_split_2) + # Test simple split (method 1) + valid_simple_split = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "split_units": { + 1: [1, 2], # Split unit 1 into two parts + 2: [2, 3], # Split unit 2 into two parts + 3: [4, 5], # Split unit 3 into two parts + }, + } + model = CurationModel(**valid_simple_split) + assert len(model.split_units) == 3 - # shold raise error + # Test complex split with multiple groups (method 2) + valid_complex_split = { + "format_version": "1", + "unit_ids": [1, 2, 3, 4], + "split_units": { + 1: [[1, 2], [3, 4]], # Split unit 1 into two groups + 2: [[2, 3], [4, 1]], # Split unit 2 into two groups + }, + } + model = CurationModel(**valid_complex_split) + assert len(model.split_units) == 2 + + # Test invalid mixing of methods + invalid_mixed_methods = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "split_units": { + 1: [[1, 2], [2, 3]], # Using method 2 + 2: [2, 3], # Using method 1 + 3: [4, 5], # Using method 1 + }, + } with pytest.raises(ValidationError): - CurationModel(**invalid_split_1) + CurationModel(**invalid_mixed_methods) + + # Test invalid unit ID + invalid_unit_id = { + "format_version": "1", + "unit_ids": [1, 2, 3], + "split_units": {4: [[1, 2], [2, 3]]}, # Unit 4 doesn't exist in unit_ids + } with pytest.raises(ValidationError): - CurationModel(**invalid_split_2) + CurationModel(**invalid_unit_id) From 40fe01be5e1a57e3824f1eae856af60316d8c82f Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 27 Mar 2025 09:32:33 -0400 Subject: [PATCH 022/157] Add analyzer split to curation tests --- src/spikeinterface/curation/tests/test_curation_format.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/spikeinterface/curation/tests/test_curation_format.py b/src/spikeinterface/curation/tests/test_curation_format.py index 45c1ad6b25..23fc69925e 100644 --- a/src/spikeinterface/curation/tests/test_curation_format.py +++ b/src/spikeinterface/curation/tests/test_curation_format.py @@ -208,6 +208,14 @@ def test_apply_curation_with_split(): assert 43 in sorting_curated.unit_ids assert 44 in sorting_curated.unit_ids + analyzer_curated = apply_curation(analyzer, curation_with_split) + assert len(analyzer_curated.sorting.unit_ids) == len(analyzer.sorting.unit_ids) + 2 + + assert 1 not in analyzer_curated.unit_ids + assert 2 not in analyzer_curated.unit_ids + assert 43 in analyzer_curated.unit_ids + assert 44 in analyzer_curated.unit_ids + if __name__ == "__main__": test_curation_format_validation() From ca6f2e0b17de9162c9615eb3e841d69613d59d02 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 27 Mar 2025 10:42:07 -0400 Subject: [PATCH 023/157] wip: add split tests in postprocessing --- .../tests/test_multi_extensions.py | 158 +++++++++++++----- 1 file changed, 113 insertions(+), 45 deletions(-) diff --git a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py index be0070d94a..0c8c2649af 100644 --- a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py +++ b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py @@ -11,8 +11,46 @@ ) from spikeinterface.core.generate import inject_some_split_units +# even if this is in postprocessing, we make an extension for quality metrics +extension_dict = { + "noise_levels": dict(), + "random_spikes": dict(), + "waveforms": dict(), + "templates": dict(), + "principal_components": dict(), + "spike_amplitudes": dict(), + "template_similarity": dict(), + "correlograms": dict(), + "isi_histograms": dict(), + "amplitude_scalings": dict(handle_collisions=False), # otherwise hard mode could fail due to dropped spikes + "spike_locations": dict(method="center_of_mass"), # trick to avoid UserWarning + "unit_locations": dict(), + "template_metrics": dict(), + "quality_metrics": dict(metric_names=["firing_rate", "isi_violation", "snr"]), +} +extension_data_type = { + "noise_levels": None, + "templates": "unit", + "isi_histograms": "unit", + "unit_locations": "unit", + "spike_amplitudes": "spike", + "amplitude_scalings": "spike", + "spike_locations": "spike", + "quality_metrics": "pandas", + "template_metrics": "pandas", + "correlograms": "matrix", + "template_similarity": "matrix", + "principal_components": "random", + "waveforms": "random", + "random_spikes": "random_spikes", +} +data_with_miltiple_returns = ["isi_histograms", "correlograms"] +# due to incremental PCA, hard computation could result in different results for PCA +# the model is differents always +random_computation = ["principal_components"] -def get_dataset(): + +def get_dataset_with_splits(): recording, sorting = generate_ground_truth_recording( durations=[30.0], sampling_frequency=16000.0, @@ -36,15 +74,15 @@ def get_dataset(): sort_by_amp = np.argsort(list(get_template_extremum_amplitude(analyzer_raw).values()))[::-1] split_ids = sorting.unit_ids[sort_by_amp][:3] - sorting_with_splits, other_ids = inject_some_split_units( + sorting_with_splits, split_unit_ids = inject_some_split_units( sorting, num_split=3, split_ids=split_ids, output_ids=True, seed=0 ) - return recording, sorting_with_splits, other_ids + return recording, sorting_with_splits, split_unit_ids @pytest.fixture(scope="module") def dataset(): - return get_dataset() + return get_dataset_with_splits() @pytest.mark.parametrize("sparse", [False, True]) @@ -54,52 +92,14 @@ def test_SortingAnalyzer_merge_all_extensions(dataset, sparse): recording, sorting, other_ids = dataset sorting_analyzer = create_sorting_analyzer(sorting, recording, format="memory", sparse=sparse) + extension_dict_merge = extension_dict.copy() # we apply the merges according to the artificial splits merges = [list(v) for v in other_ids.values()] split_unit_ids = np.ravel(merges) unmerged_unit_ids = sorting_analyzer.unit_ids[~np.isin(sorting_analyzer.unit_ids, split_unit_ids)] - # even if this is in postprocessing, we make an extension for quality metrics - extension_dict = { - "noise_levels": dict(), - "random_spikes": dict(), - "waveforms": dict(), - "templates": dict(), - "principal_components": dict(), - "spike_amplitudes": dict(), - "template_similarity": dict(), - "correlograms": dict(), - "isi_histograms": dict(), - "amplitude_scalings": dict(handle_collisions=False), # otherwise hard mode could fail due to dropped spikes - "spike_locations": dict(method="center_of_mass"), # trick to avoid UserWarning - "unit_locations": dict(), - "template_metrics": dict(), - "quality_metrics": dict(metric_names=["firing_rate", "isi_violation", "snr"]), - } - extension_data_type = { - "noise_levels": None, - "templates": "unit", - "isi_histograms": "unit", - "unit_locations": "unit", - "spike_amplitudes": "spike", - "amplitude_scalings": "spike", - "spike_locations": "spike", - "quality_metrics": "pandas", - "template_metrics": "pandas", - "correlograms": "matrix", - "template_similarity": "matrix", - "principal_components": "random", - "waveforms": "random", - "random_spikes": "random_spikes", - } - data_with_miltiple_returns = ["isi_histograms", "correlograms"] - - # due to incremental PCA, hard computation could result in different results for PCA - # the model is differents always - random_computation = ["principal_components"] - - sorting_analyzer.compute(extension_dict, n_jobs=1) + sorting_analyzer.compute(extension_dict_merge, n_jobs=1) # TODO: still some UserWarnings for n_jobs, where from? t0 = time.perf_counter() @@ -165,6 +165,74 @@ def test_SortingAnalyzer_merge_all_extensions(dataset, sparse): assert np.allclose(data_hard_merged[f], data_soft_merged[f], rtol=0.1) +@pytest.mark.parametrize("sparse", [False, True]) +def test_SortingAnalyzer_split_all_extensions(dataset, sparse): + set_global_job_kwargs(n_jobs=1) + + recording, sorting, _ = dataset + + sorting_analyzer = create_sorting_analyzer(sorting, recording, format="memory", sparse=sparse) + extension_dict_split = extension_dict.copy() + sorting_analyzer.compute(extension_dict, n_jobs=1) + + # we randomly apply splits (at half of spiketrain) + num_spikes = sorting.count_num_spikes_per_unit() + + units_to_split = [sorting_analyzer.unit_ids[1], sorting_analyzer.unit_ids[5]] + unsplit_unit_ids = sorting_analyzer.unit_ids[~np.isin(sorting_analyzer.unit_ids, units_to_split)] + splits = {} + for unit in units_to_split: + splits[unit] = np.arange(num_spikes[unit] // 2) + + analyzer_split, split_unit_ids = sorting_analyzer.split_units(split_units=splits, return_new_unit_ids=True) + split_unit_ids = list(np.concatenate(split_unit_ids)) + + # also do a full recopute + analyzer_hard = create_sorting_analyzer(analyzer_split.sorting, recording, format="memory", sparse=sparse) + # we propagate random spikes to avoid random spikes to be recomputed + extension_dict_ = extension_dict_split.copy() + extension_dict_.pop("random_spikes") + analyzer_hard.extensions["random_spikes"] = analyzer_split.extensions["random_spikes"] + analyzer_hard.compute(extension_dict_, n_jobs=1) + + for ext in extension_dict: + # 1. check that data are exactly the same for unchanged units between original/split + data_original = sorting_analyzer.get_extension(ext).get_data() + data_split = analyzer_split.get_extension(ext).get_data() + data_recompute = analyzer_hard.get_extension(ext).get_data() + if ext in data_with_miltiple_returns: + data_original = data_original[0] + data_split = data_split[0] + data_recompute = data_recompute[0] + data_original_unsplit = get_extension_data_for_units( + sorting_analyzer, data_original, unsplit_unit_ids, extension_data_type[ext] + ) + data_split_unsplit = get_extension_data_for_units( + analyzer_split, data_split, unsplit_unit_ids, extension_data_type[ext] + ) + + np.testing.assert_array_equal(data_original_unsplit, data_split_unsplit) + + # 2. check that split data are the same for extension split and recompute + data_split_soft = get_extension_data_for_units( + analyzer_split, data_split, split_unit_ids, extension_data_type[ext] + ) + data_split_hard = get_extension_data_for_units( + analyzer_hard, data_recompute, split_unit_ids, extension_data_type[ext] + ) + # TODO: fix amplitude scalings + failing_extensions = [] + if ext not in random_computation + failing_extensions: + if extension_data_type[ext] == "pandas": + data_split_soft = data_split_soft.dropna().to_numpy().astype("float") + data_split_hard = data_split_hard.dropna().to_numpy().astype("float") + if data_split_hard.dtype.fields is None: + assert np.allclose(data_split_hard, data_split_soft, rtol=0.1) + else: + for f in data_split_hard.dtype.fields: + assert np.allclose(data_split_hard[f], data_split_soft[f], rtol=0.1) + + def get_extension_data_for_units(sorting_analyzer, data, unit_ids, ext_data_type): unit_indices = sorting_analyzer.sorting.ids_to_indices(unit_ids) spike_vector = sorting_analyzer.sorting.to_spike_vector() @@ -191,5 +259,5 @@ def get_extension_data_for_units(sorting_analyzer, data, unit_ids, ext_data_type if __name__ == "__main__": - dataset = get_dataset() + dataset = get_dataset_with_splits() test_SortingAnalyzer_merge_all_extensions(dataset, False) From 58b62fb00f5010a4fb9b8a51f1d5ec301825ed1b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 27 Mar 2025 13:18:14 -0400 Subject: [PATCH 024/157] wip - modify model --- src/spikeinterface/curation/curation_model.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index b50db1e69c..9b4672f904 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -1,8 +1,9 @@ from pydantic import BaseModel, Field, field_validator, model_validator -from typing import List, Dict, Union, Optional +from typing import List, Dict, Union, Optional, Literal from itertools import combinations, chain import numpy as np + supported_curation_format_versions = {"1"} @@ -17,6 +18,21 @@ class ManualLabel(BaseModel): labels: Dict[str, List[str]] = Field(..., description="Dictionary of labels for the unit") +class Merges(BaseModel): + merge_unit_groups: List[List[Union[int, str]]] = Field(..., description="List of groups of units to be merged") + merge_new_unit_ids: List[Union[int, str]] = Field(..., description="List of new unit IDs for each merge group") + + +class Split(BaseModel): + unit_id: Union[int, str] = Field(..., description="ID of the unit") + split_mode: Literal["indices", "labels"] = Field(default="indices", description="Mode of the split") + split_indices: Optional[Union[List[List[int]]]] = Field(default=None, description="Information about the split") + split_labels = Optional[List[int]] = Field(default=None, description="List of labels for the split") + split_new_unit_ids: Optional[List[Union[int, str]]] = Field( + default=None, description="List of new unit IDs for each unit split" + ) + + class CurationModel(BaseModel): format_version: str = Field(..., description="Version of the curation format") unit_ids: List[Union[int, str]] = Field(..., description="List of unit IDs") From d2f220a0abfefa9712f9fa0f9810f8b8ed0a6025 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 28 Mar 2025 16:41:48 -0400 Subject: [PATCH 025/157] Fix test-multi-extensions --- .../tests/test_multi_extensions.py | 92 +++++++++++++++---- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py index 0c8c2649af..8c512c2109 100644 --- a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py +++ b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py @@ -48,9 +48,22 @@ # due to incremental PCA, hard computation could result in different results for PCA # the model is differents always random_computation = ["principal_components"] +# for some extensions (templates, amplitude_scalings), since the templates slightly change for merges/splits +# we allow a relative tolerance +# (amplitud_scalings are the moste sensitive!) +extensions_with_rel_tolerance_merge = { + "amplitude_scalings": 1e-1, + "templates": 1e-3, + "template_similarity": 1e-3, + "unit_locations": 1e-3, + "template_metrics": 1e-3, + "quality_metrics": 1e-3, +} +extensions_with_rel_tolerance_splits = {"amplitude_scalings": 1e-1} -def get_dataset_with_splits(): +def get_dataset_to_merge(): + # generate a dataset with some split units to minimize merge errors recording, sorting = generate_ground_truth_recording( durations=[30.0], sampling_frequency=16000.0, @@ -58,6 +71,7 @@ def get_dataset_with_splits(): num_units=10, generate_sorting_kwargs=dict(firing_rates=10.0, refractory_period_ms=4.0), noise_kwargs=dict(noise_levels=5.0, strategy="tile_pregenerated"), + generate_unit_locations_kwargs=dict(margin_um=10.0, minimum_z=2.0, maximum_z=15.0, minimum_distance=20), seed=2205, ) @@ -80,16 +94,49 @@ def get_dataset_with_splits(): return recording, sorting_with_splits, split_unit_ids +def get_dataset_to_split(): + # generate a dataset and return large unit to split to minimize split errors + recording, sorting = generate_ground_truth_recording( + durations=[30.0], + sampling_frequency=16000.0, + num_channels=10, + num_units=10, + generate_sorting_kwargs=dict(firing_rates=10.0, refractory_period_ms=4.0), + noise_kwargs=dict(noise_levels=5.0, strategy="tile_pregenerated"), + seed=2205, + ) + + channel_ids_as_integers = [id for id in range(recording.get_num_channels())] + unit_ids_as_integers = [id for id in range(sorting.get_num_units())] + recording = recording.rename_channels(new_channel_ids=channel_ids_as_integers) + sorting = sorting.rename_units(new_unit_ids=unit_ids_as_integers) + + # since templates are going to be averaged and this might be a problem for amplitude scaling + # we select the 3 units with the largest templates to split + analyzer_raw = create_sorting_analyzer(sorting, recording, format="memory", sparse=False) + analyzer_raw.compute(["random_spikes", "templates"]) + # select 3 largest templates to split + sort_by_amp = np.argsort(list(get_template_extremum_amplitude(analyzer_raw).values()))[::-1] + large_units = sorting.unit_ids[sort_by_amp][:2] + + return recording, sorting, large_units + + +@pytest.fixture(scope="module") +def dataset_to_merge(): + return get_dataset_to_merge() + + @pytest.fixture(scope="module") -def dataset(): - return get_dataset_with_splits() +def dataset_to_split(): + return get_dataset_to_split() @pytest.mark.parametrize("sparse", [False, True]) -def test_SortingAnalyzer_merge_all_extensions(dataset, sparse): +def test_SortingAnalyzer_merge_all_extensions(dataset_to_merge, sparse): set_global_job_kwargs(n_jobs=1) - recording, sorting, other_ids = dataset + recording, sorting, other_ids = dataset_to_merge sorting_analyzer = create_sorting_analyzer(sorting, recording, format="memory", sparse=sparse) extension_dict_merge = extension_dict.copy() @@ -155,21 +202,29 @@ def test_SortingAnalyzer_merge_all_extensions(dataset, sparse): ) if ext not in random_computation: + if ext in extensions_with_rel_tolerance_merge: + rtol = extensions_with_rel_tolerance_merge[ext] + else: + rtol = 0 if extension_data_type[ext] == "pandas": data_hard_merged = data_hard_merged.dropna().to_numpy().astype("float") data_soft_merged = data_soft_merged.dropna().to_numpy().astype("float") if data_hard_merged.dtype.fields is None: - assert np.allclose(data_hard_merged, data_soft_merged, rtol=0.1) + if not np.allclose(data_hard_merged, data_soft_merged, rtol=rtol): + max_error = np.max(np.abs(data_hard_merged - data_soft_merged)) + raise Exception(f"Failed for {ext} - max error {max_error}") else: for f in data_hard_merged.dtype.fields: - assert np.allclose(data_hard_merged[f], data_soft_merged[f], rtol=0.1) + if not np.allclose(data_hard_merged[f], data_soft_merged[f], rtol=rtol): + max_error = np.max(np.abs(data_hard_merged[f] - data_soft_merged[f])) + raise Exception(f"Failed for {ext} - field {f} - max error {max_error}") @pytest.mark.parametrize("sparse", [False, True]) -def test_SortingAnalyzer_split_all_extensions(dataset, sparse): +def test_SortingAnalyzer_split_all_extensions(dataset_to_split, sparse): set_global_job_kwargs(n_jobs=1) - recording, sorting, _ = dataset + recording, sorting, units_to_split = dataset_to_split sorting_analyzer = create_sorting_analyzer(sorting, recording, format="memory", sparse=sparse) extension_dict_split = extension_dict.copy() @@ -178,7 +233,6 @@ def test_SortingAnalyzer_split_all_extensions(dataset, sparse): # we randomly apply splits (at half of spiketrain) num_spikes = sorting.count_num_spikes_per_unit() - units_to_split = [sorting_analyzer.unit_ids[1], sorting_analyzer.unit_ids[5]] unsplit_unit_ids = sorting_analyzer.unit_ids[~np.isin(sorting_analyzer.unit_ids, units_to_split)] splits = {} for unit in units_to_split: @@ -220,17 +274,23 @@ def test_SortingAnalyzer_split_all_extensions(dataset, sparse): data_split_hard = get_extension_data_for_units( analyzer_hard, data_recompute, split_unit_ids, extension_data_type[ext] ) - # TODO: fix amplitude scalings - failing_extensions = [] - if ext not in random_computation + failing_extensions: + if ext not in random_computation: + if ext in extensions_with_rel_tolerance_splits: + rtol = extensions_with_rel_tolerance_splits[ext] + else: + rtol = 0 if extension_data_type[ext] == "pandas": data_split_soft = data_split_soft.dropna().to_numpy().astype("float") data_split_hard = data_split_hard.dropna().to_numpy().astype("float") if data_split_hard.dtype.fields is None: - assert np.allclose(data_split_hard, data_split_soft, rtol=0.1) + if not np.allclose(data_split_hard, data_split_soft, rtol=rtol): + max_error = np.max(np.abs(data_split_hard - data_split_soft)) + raise Exception(f"Failed for {ext} - max error {max_error}") else: for f in data_split_hard.dtype.fields: - assert np.allclose(data_split_hard[f], data_split_soft[f], rtol=0.1) + if not np.allclose(data_split_hard[f], data_split_soft[f], rtol=rtol): + max_error = np.max(np.abs(data_split_hard[f] - data_split_soft[f])) + raise Exception(f"Failed for {ext} - field {f} - max error {max_error}") def get_extension_data_for_units(sorting_analyzer, data, unit_ids, ext_data_type): @@ -259,5 +319,5 @@ def get_extension_data_for_units(sorting_analyzer, data, unit_ids, ext_data_type if __name__ == "__main__": - dataset = get_dataset_with_splits() + dataset = get_dataset_to_merge() test_SortingAnalyzer_merge_all_extensions(dataset, False) From d4fa8bf0da57d9449c8ca91de8a64d03075696a4 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 28 Mar 2025 16:52:25 -0400 Subject: [PATCH 026/157] Deal with multi-segment --- src/spikeinterface/core/sorting_tools.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index d6ba6a7e6b..f9d05d29ca 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -452,17 +452,16 @@ def generate_unit_ids_for_merge_group(old_unit_ids, merge_unit_groups, new_unit_ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extra=False, new_id_strategy="append"): spikes = sorting.to_spike_vector().copy() - # take care of single-list splits - full_unit_splits = _get_full_unit_splits(unit_splits, sorting) + # here we assume that unit_splits split_indices are already full. + # this is true when running via apply_curation new_unit_ids = generate_unit_ids_for_split( - sorting.unit_ids, full_unit_splits, new_unit_ids=new_unit_ids, new_id_strategy=new_id_strategy + sorting.unit_ids, unit_splits, new_unit_ids=new_unit_ids, new_id_strategy=new_id_strategy ) - all_unit_ids = _get_ids_after_splitting(sorting.unit_ids, full_unit_splits, new_unit_ids) + all_unit_ids = _get_ids_after_splitting(sorting.unit_ids, unit_splits, new_unit_ids) all_unit_ids = list(all_unit_ids) num_seg = sorting.get_num_segments() - assert num_seg == 1 seg_lims = np.searchsorted(spikes["segment_index"], np.arange(0, num_seg + 2)) segment_slices = [(seg_lims[i], seg_lims[i + 1]) for i in range(num_seg)] @@ -470,17 +469,18 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr spike_vector_list = [spikes[s0:s1] for s0, s1 in segment_slices] spike_indices = spike_vector_to_indices(spike_vector_list, sorting.unit_ids, absolute_index=True) - # TODO deal with segments in splits + # split_indices are a concatenation across segments for unit_id in sorting.unit_ids: - if unit_id in full_unit_splits: - split_indices = full_unit_splits[unit_id] - new_split_ids = new_unit_ids[list(full_unit_splits.keys()).index(unit_id)] + if unit_id in unit_splits: + split_indices = unit_splits[unit_id] + new_split_ids = new_unit_ids[list(unit_splits.keys()).index(unit_id)] for split, new_unit_id in zip(split_indices, new_split_ids): new_unit_index = all_unit_ids.index(new_unit_id) - for segment_index in range(num_seg): - spike_inds = spike_indices[segment_index][unit_id] - spikes["unit_index"][spike_inds[split]] = new_unit_index + spike_indices_unit = np.concatenate( + spike_indices[segment_index][unit_id] for segment_index in range(num_seg) + ) + spikes["unit_index"][spike_indices_unit[split]] = new_unit_index else: new_unit_index = all_unit_ids.index(unit_id) for segment_index in range(num_seg): From 659ecff46fee32ce59671c56af0dd57245da2d0c Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Sat, 29 Mar 2025 11:52:05 -0400 Subject: [PATCH 027/157] Extend splitting-tests to multi-segment and mask labels --- src/spikeinterface/core/sorting_tools.py | 5 +- .../curation/tests/test_curation_format.py | 121 ++++++++++++++++-- 2 files changed, 116 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index f9d05d29ca..24718d4e5d 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -469,7 +469,6 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr spike_vector_list = [spikes[s0:s1] for s0, s1 in segment_slices] spike_indices = spike_vector_to_indices(spike_vector_list, sorting.unit_ids, absolute_index=True) - # split_indices are a concatenation across segments for unit_id in sorting.unit_ids: if unit_id in unit_splits: split_indices = unit_splits[unit_id] @@ -477,8 +476,10 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr for split, new_unit_id in zip(split_indices, new_split_ids): new_unit_index = all_unit_ids.index(new_unit_id) + # split_indices are a concatenation across segments with absolute indices + # so we need to concatenate the spike indices across segments spike_indices_unit = np.concatenate( - spike_indices[segment_index][unit_id] for segment_index in range(num_seg) + [spike_indices[segment_index][unit_id] for segment_index in range(num_seg)] ) spikes["unit_index"][spike_indices_unit[split]] = new_unit_index else: diff --git a/src/spikeinterface/curation/tests/test_curation_format.py b/src/spikeinterface/curation/tests/test_curation_format.py index fef1f9d6f7..d0126d5460 100644 --- a/src/spikeinterface/curation/tests/test_curation_format.py +++ b/src/spikeinterface/curation/tests/test_curation_format.py @@ -127,7 +127,6 @@ # Test dictionary format for merges with string IDs curation_ids_str_dict = {**curation_ids_str, "merges": {"u50": ["u3", "u6"], "u51": ["u10", "u14", "u20"]}} - # This is a failure example with duplicated merge duplicate_merge = curation_ids_int.copy() duplicate_merge["merge_unit_groups"] = [[3, 6, 10], [10, 14, 20]] @@ -292,11 +291,117 @@ def test_apply_curation_with_split(): assert analyzer_curated.sorting.get_property("pyramidal", ids=[unit_id])[0] +def test_apply_curation_with_split_multi_segment(): + recording, sorting = generate_ground_truth_recording(durations=[10.0, 10.0], num_units=9, seed=2205) + sorting = sorting.rename_units(np.array([1, 2, 3, 6, 10, 14, 20, 31, 42])) + analyzer = create_sorting_analyzer(sorting, recording, sparse=False) + num_segments = sorting.get_num_segments() + + curation_with_splits_multi_segment = curation_with_splits.copy() + + # we make a split so that each subsplit will have all spikes from different segments + split_unit_id = curation_with_splits_multi_segment["splits"][0]["unit_id"] + sv = sorting.to_spike_vector() + unit_index = sorting.id_to_index(split_unit_id) + spikes_from_split_unit = sv[sv["unit_index"] == unit_index] + + split_indices = [] + cum_spikes = 0 + for segment_index in range(num_segments): + spikes_in_segment = spikes_from_split_unit[spikes_from_split_unit["segment_index"] == segment_index] + split_indices.append(np.arange(0, len(spikes_in_segment)) + cum_spikes) + cum_spikes += len(spikes_in_segment) + + curation_with_splits_multi_segment["splits"][0]["split_indices"] = split_indices + + sorting_curated = apply_curation(sorting, curation_with_splits_multi_segment) + + assert len(sorting_curated.unit_ids) == len(sorting.unit_ids) + 1 + assert 2 not in sorting_curated.unit_ids + assert 43 in sorting_curated.unit_ids + assert 44 in sorting_curated.unit_ids + + # check that spike trains are correctly split across segments + for seg_index in range(num_segments): + st_43 = sorting_curated.get_unit_spike_train(43, segment_index=seg_index) + st_44 = sorting_curated.get_unit_spike_train(44, segment_index=seg_index) + if seg_index == 0: + assert len(st_43) > 0 + assert len(st_44) == 0 + else: + assert len(st_43) == 0 + assert len(st_44) > 0 + + +def test_apply_curation_splits_with_mask(): + recording, sorting = generate_ground_truth_recording(durations=[10.0], num_units=9, seed=2205) + sorting = sorting.rename_units(np.array([1, 2, 3, 6, 10, 14, 20, 31, 42])) + analyzer = create_sorting_analyzer(sorting, recording, sparse=False) + + # Get number of spikes for unit 2 + num_spikes = sorting.count_num_spikes_per_unit()[2] + + # Create split labels that assign spikes to 3 different clusters + split_labels = np.zeros(num_spikes, dtype=int) + split_labels[: num_spikes // 3] = 0 # First third to cluster 0 + split_labels[num_spikes // 3 : 2 * num_spikes // 3] = 1 # Second third to cluster 1 + split_labels[2 * num_spikes // 3 :] = 2 # Last third to cluster 2 + + curation_with_mask_split = { + "format_version": "2", + "unit_ids": [1, 2, 3, 6, 10, 14, 20, 31, 42], + "label_definitions": { + "quality": {"label_options": ["good", "noise", "MUA", "artifact"], "exclusive": True}, + "putative_type": { + "label_options": ["excitatory", "inhibitory", "pyramidal", "mitral"], + "exclusive": False, + }, + }, + "manual_labels": [ + {"unit_id": 2, "quality": ["good"], "putative_type": ["excitatory", "pyramidal"]}, + ], + "splits": [ + { + "unit_id": 2, + "split_mode": "labels", + "split_labels": split_labels.tolist(), + "split_new_unit_ids": [43, 44, 45], + } + ], + } + + sorting_curated = apply_curation(sorting, curation_with_mask_split) + + # Check results + assert len(sorting_curated.unit_ids) == len(sorting.unit_ids) + 2 # Original units - 1 (split) + 3 (new) + assert 2 not in sorting_curated.unit_ids # Original unit should be removed + + # Check new split units + split_unit_ids = [43, 44, 45] + for unit_id in split_unit_ids: + assert unit_id in sorting_curated.unit_ids + # Check properties are propagated + assert sorting_curated.get_property("quality", ids=[unit_id])[0] == "good" + assert sorting_curated.get_property("excitatory", ids=[unit_id])[0] + assert sorting_curated.get_property("pyramidal", ids=[unit_id])[0] + + # Check analyzer + analyzer_curated = apply_curation(analyzer, curation_with_mask_split) + assert len(analyzer_curated.sorting.unit_ids) == len(analyzer.sorting.unit_ids) + 2 + + # Verify split sizes + spike_counts = analyzer_curated.sorting.count_num_spikes_per_unit() + assert spike_counts[43] == num_spikes // 3 # First third + assert spike_counts[44] == num_spikes // 3 # Second third + assert spike_counts[45] == num_spikes - 2 * (num_spikes // 3) # Remainder + + if __name__ == "__main__": - # test_curation_format_validation() - # test_to_from_json() - # test_convert_from_sortingview_curation_format_v0() - # test_curation_label_to_vectors() - # test_curation_label_to_dataframe() - # test_apply_curation() - test_apply_curation_with_split() + test_curation_format_validation() + test_to_from_json() + test_convert_from_sortingview_curation_format_v0() + test_curation_label_to_vectors() + test_curation_label_to_dataframe() + test_apply_curation() + test_apply_curation_with_split_multi_segment() + test_apply_curation_splits_with_mask() From e073dcb67ad4a1e756e9b47c1c12c25138e1d12a Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:48:32 +0200 Subject: [PATCH 028/157] Better Neuroscope support: - fetches the xml with channel groups - automatically adds it as a group in the recording - parses the real colors and positions (offsets) directly from Neuroscope and adds them as properties of the object (handy for ephyviewer) Will allow for parallel spike sorting, processing... as if it were separate recordings/structures (for now it's only anatomy-based, but we could do the same with the "spike groups" identified after spike sorting by Neuroscope). --- .../extractors/neoextractors/neuroscope.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 70a110eced..e50e6da2b2 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -3,6 +3,7 @@ import warnings from pathlib import Path from typing import Union, Optional +from xml.etree import ElementTree as Etree import numpy as np @@ -64,6 +65,7 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) + self.split_recording_by_channel_groups(xml_file_path=xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml")) @classmethod def map_to_neo_kwargs(cls, file_path, xml_file_path=None): @@ -78,6 +80,65 @@ def map_to_neo_kwargs(cls, file_path, xml_file_path=None): return neo_kwargs + def _parse_xml_file(self, xml_file_path): + """ + Comes from NeuroPhy package by Diba Lab + """ + tree = Etree.parse(xml_file_path) + myroot = tree.getroot() + + for sf in myroot.findall("acquisitionSystem"): + n_channels = int(sf.find("nChannels").text) + + channel_groups, skipped_channels, anatomycolors = [], [], {} + for x in myroot.findall("anatomicalDescription"): + for y in x.findall("channelGroups"): + for z in y.findall("group"): + chan_group = [] + for chan in z.findall("channel"): + if int(chan.attrib["skip"]) == 1: + skipped_channels.append(int(chan.text)) + + chan_group.append(int(chan.text)) + if chan_group: + channel_groups.append(np.array(chan_group)) + + for x in myroot.findall("neuroscope"): + for y in x.findall("channels"): + for i, z in enumerate(y.findall("channelColors")): + try: + channel_id = str(z.find("channel").text) + color = z.find("color").text + + except AttributeError: + channel_id = i + color = "#0080ff" + anatomycolors[channel_id] = color + + discarded_channels = np.setdiff1d(np.arange(n_channels), np.concatenate(channel_groups)) + kept_channels = np.setdiff1d(np.arange(n_channels),np.concatenate([skipped_channels, discarded_channels])) + + return channel_groups, kept_channels, discarded_channels, anatomycolors + + def split_recording_by_channel_groups(self): + n = self.get_num_channels() + group_ids = np.full(n, -1, dtype=int) # Initialize all positions to -1 + + channel_groups, kept_channels, discarded_channels, colors = ( + self._parse_xml_file(self.xml_file_path) + ) + for group_id, numbers in enumerate(channel_groups): + group_ids[numbers] = ( + group_id # Assign group_id to the positions in `numbers` + ) + self.set_property("group", group_ids) + discarded_ppty = np.full(n, False, dtype=bool) + discarded_ppty[discarded_channels] = True + self.set_property("discarded_channels", discarded_ppty) + self.set_property( + "colors", values=list(colors.values()), ids=list(colors.keys()) + ) + class NeuroScopeSortingExtractor(BaseSorting): """ From 588c550075013f43a574a47d85bad5a5ae31c688 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:54:59 +0000 Subject: [PATCH 029/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../extractors/neoextractors/neuroscope.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index e50e6da2b2..5c2944c208 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -65,7 +65,9 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) - self.split_recording_by_channel_groups(xml_file_path=xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml")) + self.split_recording_by_channel_groups( + xml_file_path=xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") + ) @classmethod def map_to_neo_kwargs(cls, file_path, xml_file_path=None): @@ -111,12 +113,12 @@ def _parse_xml_file(self, xml_file_path): color = z.find("color").text except AttributeError: - channel_id = i + channel_id = i color = "#0080ff" anatomycolors[channel_id] = color discarded_channels = np.setdiff1d(np.arange(n_channels), np.concatenate(channel_groups)) - kept_channels = np.setdiff1d(np.arange(n_channels),np.concatenate([skipped_channels, discarded_channels])) + kept_channels = np.setdiff1d(np.arange(n_channels), np.concatenate([skipped_channels, discarded_channels])) return channel_groups, kept_channels, discarded_channels, anatomycolors @@ -124,20 +126,14 @@ def split_recording_by_channel_groups(self): n = self.get_num_channels() group_ids = np.full(n, -1, dtype=int) # Initialize all positions to -1 - channel_groups, kept_channels, discarded_channels, colors = ( - self._parse_xml_file(self.xml_file_path) - ) + channel_groups, kept_channels, discarded_channels, colors = self._parse_xml_file(self.xml_file_path) for group_id, numbers in enumerate(channel_groups): - group_ids[numbers] = ( - group_id # Assign group_id to the positions in `numbers` - ) + group_ids[numbers] = group_id # Assign group_id to the positions in `numbers` self.set_property("group", group_ids) discarded_ppty = np.full(n, False, dtype=bool) discarded_ppty[discarded_channels] = True self.set_property("discarded_channels", discarded_ppty) - self.set_property( - "colors", values=list(colors.values()), ids=list(colors.keys()) - ) + self.set_property("colors", values=list(colors.values()), ids=list(colors.keys())) class NeuroScopeSortingExtractor(BaseSorting): From 36953ded59358d69929e0cf4ae97561bdb6a8eca Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Mon, 14 Apr 2025 15:05:34 +0200 Subject: [PATCH 030/157] fix bad copy-paste --> now the arguments to split by group are good. --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 5c2944c208..40c3a797bf 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -65,9 +65,11 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) - self.split_recording_by_channel_groups( - xml_file_path=xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") + xml_file_path=( + xml_file_path if xml_file_path is not None + else Path(file_path).with_suffix(".xml") ) + self.split_recording_by_channel_groups() @classmethod def map_to_neo_kwargs(cls, file_path, xml_file_path=None): From b969a9dbe1bc6fbe4227287b9c5253ce497e5c33 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 13:07:05 +0000 Subject: [PATCH 031/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 40c3a797bf..96ea572763 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -65,10 +65,7 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) - xml_file_path=( - xml_file_path if xml_file_path is not None - else Path(file_path).with_suffix(".xml") - ) + xml_file_path = xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") self.split_recording_by_channel_groups() @classmethod From fa3084403eccd8e05e7da9cb94f4693afae60f06 Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Mon, 14 Apr 2025 15:15:56 +0200 Subject: [PATCH 032/157] forgot to add xml_file_path as Neuroscope attribute --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 96ea572763..60c3387047 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -65,7 +65,10 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) - xml_file_path = xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") + self.xml_file_path=( + xml_file_path if xml_file_path is not None + else Path(file_path).with_suffix(".xml") + ) self.split_recording_by_channel_groups() @classmethod From d8475eb8115262a308986a6e47eb55dda93fd676 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 13:17:42 +0000 Subject: [PATCH 033/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 60c3387047..965c31017b 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -65,10 +65,7 @@ def __init__( if xml_file_path is not None: xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) - self.xml_file_path=( - xml_file_path if xml_file_path is not None - else Path(file_path).with_suffix(".xml") - ) + self.xml_file_path = xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") self.split_recording_by_channel_groups() @classmethod From 9b0595f13fa26a44913d6324818bbca82731db7e Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Wed, 16 Apr 2025 13:21:12 +0200 Subject: [PATCH 034/157] resolve comment about np setdiff1d --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 965c31017b..96eb9823b9 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -116,8 +116,8 @@ def _parse_xml_file(self, xml_file_path): color = "#0080ff" anatomycolors[channel_id] = color - discarded_channels = np.setdiff1d(np.arange(n_channels), np.concatenate(channel_groups)) - kept_channels = np.setdiff1d(np.arange(n_channels), np.concatenate([skipped_channels, discarded_channels])) + discarded_channels = [ch for ch in range(n_channels) if all(ch not in group for group in channel_groups)] + kept_channels = [ch for ch in range(n_channels) if ch not in skipped_channels and ch not in discarded_channels] return channel_groups, kept_channels, discarded_channels, anatomycolors From 963a811f7a2a77413d6d0f4208088f1193149c9b Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Wed, 16 Apr 2025 17:05:51 +0200 Subject: [PATCH 035/157] rename split recording groups with private name --- .../extractors/neoextractors/neuroscope.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index 96eb9823b9..efbcb57e59 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -66,7 +66,7 @@ def __init__( xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) self.xml_file_path = xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") - self.split_recording_by_channel_groups() + self._set_groups() @classmethod def map_to_neo_kwargs(cls, file_path, xml_file_path=None): @@ -121,7 +121,12 @@ def _parse_xml_file(self, xml_file_path): return channel_groups, kept_channels, discarded_channels, anatomycolors - def split_recording_by_channel_groups(self): + def _set_groups(self): + """ + Set the group ids and colors based on the xml file. + These group ids are usually different brain/body anatomical areas, or shanks from multi-shank probes. + The group ids are set as a property of the recording extractor. + """ n = self.get_num_channels() group_ids = np.full(n, -1, dtype=int) # Initialize all positions to -1 From cbc790ce0f46a6680f73d32ae161db6b43f54f15 Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Tue, 29 Apr 2025 13:24:22 +0100 Subject: [PATCH 036/157] Improve segment validation to list only. Add unitls function for validation --- src/spikeinterface/widgets/amplitudes.py | 34 +++------------- src/spikeinterface/widgets/motion.py | 13 +++--- src/spikeinterface/widgets/rasters.py | 52 ++++++------------------ src/spikeinterface/widgets/utils.py | 44 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 76 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index ef85f9ca30..a6fb9948ae 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -5,7 +5,7 @@ from .rasters import BaseRasterWidget from .base import BaseWidget, to_attr -from .utils import get_some_colors +from .utils import get_some_colors, validate_segment_indices from spikeinterface.core.sortinganalyzer import SortingAnalyzer @@ -25,7 +25,7 @@ class AmplitudesWidget(BaseRasterWidget): unit_colors : dict | None, default: None Dict of colors with unit ids as keys and colors as values. Colors can be any type accepted by matplotlib. If None, default colors are chosen using the `get_some_colors` function. - segment_index : int or list of int or None, default: None + segment_indices : list of int or None, default: None Segment index or indices to plot. If None and there are multiple segments, defaults to 0. If list, spike trains and amplitudes are concatenated across the specified segments. max_spikes_per_unit : int or None, default: None @@ -52,7 +52,7 @@ def __init__( sorting_analyzer: SortingAnalyzer, unit_ids=None, unit_colors=None, - segment_index=None, + segment_indices=None, max_spikes_per_unit=None, y_lim=None, scatter_decimate=1, @@ -74,38 +74,16 @@ def __init__( if unit_ids is None: unit_ids = sorting.unit_ids - num_segments = sorting.get_num_segments() - # Handle segment_index input - if num_segments > 1: - if segment_index is None: - warn("More than one segment available! Using `segment_index = 0`.") - segment_index = 0 - else: - segment_index = 0 + segment_indices = validate_segment_indices(segment_indices, sorting) # Check for SortingView backend is_sortingview = backend == "sortingview" # For SortingView, ensure we're only using a single segment - if is_sortingview and isinstance(segment_index, list) and len(segment_index) > 1: + if is_sortingview and len(segment_indices) > 1: warn("SortingView backend currently supports only single segment. Using first segment.") - segment_index = segment_index[0] - - # Convert segment_index to list for consistent processing - if isinstance(segment_index, int): - segment_indices = [segment_index] - elif isinstance(segment_index, list): - segment_indices = segment_index - else: - raise ValueError("segment_index must be an int or a list of ints") - - # Validate segment indices - for idx in segment_indices: - if not isinstance(idx, int): - raise ValueError(f"Each segment index must be an integer, got {type(idx)}") - if idx < 0 or idx >= num_segments: - raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") + segment_indices = segment_indices[0] # Create multi-segment data structure (dict of dicts) spiketrains_by_segment = {} diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index 1a1512545b..f932ed44f6 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -121,7 +121,7 @@ class DriftRasterMapWidget(BaseRasterWidget): The recording extractor object (only used to get "real" times). sampling_frequency : float, default: None The sampling frequency (needed if recording is None). - segment_index : int or list of int or None, default: None + segment_indices : list of int or None, default: None The segment index or indices to display. If None and there's only one segment, it's used. If None and there are multiple segments, you must specify which to use. If a list of indices is provided, peaks and locations are concatenated across the segments. @@ -149,7 +149,7 @@ def __init__( direction: str = "y", recording: BaseRecording | None = None, sampling_frequency: float | None = None, - segment_index: int | list | None = None, + segment_indices: list[int] | None = None, depth_lim: tuple[float, float] | None = None, color_amplitude: bool = True, scatter_decimate: int | None = None, @@ -197,16 +197,13 @@ def __init__( unique_segments = np.unique(peaks["segment_index"]) - if segment_index is None: + if segment_indices is None: if len(unique_segments) == 1: segment_indices = [int(unique_segments[0])] else: raise ValueError("segment_index must be specified if there are multiple segments") - elif isinstance(segment_index, int): - segment_indices = [segment_index] - elif isinstance(segment_index, list): - segment_indices = segment_index - else: + + if not isinstance(segment_indices, list): raise ValueError("segment_index must be an int or a list of ints") # Validate all segment indices exist in the data diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 3d4470c249..f03f980f24 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -4,7 +4,7 @@ from warnings import warn from .base import BaseWidget, to_attr, default_backend_kwargs -from .utils import get_some_colors +from .utils import get_some_colors, validate_segment_indices class BaseRasterWidget(BaseWidget): @@ -23,7 +23,7 @@ class BaseRasterWidget(BaseWidget): converted to a dict of dicts with segment 0. unit_ids : array-like | None, default: None List of unit_ids to plot - segment_index : int | list | None, default: None + segment_indices : list | None, default: None For multi-segment data, specifies which segment(s) to plot. If None, uses all available segments. For single-segment data, this parameter is ignored. total_duration : int | None, default: None @@ -65,7 +65,7 @@ def __init__( spike_train_data: dict, y_axis_data: dict, unit_ids: list | None = None, - segment_index: int | list | None = None, + segment_indices: list | None = None, total_duration: int | None = None, plot_histograms: bool = False, bins: int | None = None, @@ -93,22 +93,17 @@ def __init__( available_segments.sort() # Ensure consistent ordering # Determine which segments to use - if segment_index is None: + if segment_indices is None: # Use all segments by default segments_to_use = available_segments - elif isinstance(segment_index, int): - # Single segment specified - if segment_index not in available_segments: - raise ValueError(f"segment_index {segment_index} not found in data") - segments_to_use = [segment_index] - elif isinstance(segment_index, list): + elif isinstance(segment_indices, list): # Multiple segments specified - for idx in segment_index: + for idx in segment_indices: if idx not in available_segments: - raise ValueError(f"segment_index {idx} not found in data") - segments_to_use = segment_index + raise ValueError(f"segment_index {idx} not found in avialable segments {available_segments}") + segments_to_use = segment_indices else: - raise ValueError("segment_index must be int, list, or None") + raise ValueError("segment_index must be `list` or `None`") # Get all unit IDs present in any segment if not specified if unit_ids is None: @@ -391,7 +386,7 @@ class RasterWidget(BaseRasterWidget): A sorting object sorting_analyzer : SortingAnalyzer | None, default: None A sorting analyzer object - segment_index : int or list of int or None, default: None + segment_indices : list of int or None, default: None The segment index or indices to use. If None and there are multiple segments, defaults to 0. If a list of indices is provided, spike trains are concatenated across the specified segments. unit_ids : list @@ -406,7 +401,7 @@ def __init__( self, sorting=None, sorting_analyzer=None, - segment_index=None, + segment_indices=None, unit_ids=None, time_range=None, color="k", @@ -424,30 +419,7 @@ def __init__( sorting = self.ensure_sorting(sorting) - num_segments = sorting.get_num_segments() - - # Handle segment_index input - if num_segments > 1: - if segment_index is None: - warn("More than one segment available! Using `segment_index = 0`.") - segment_index = 0 - else: - segment_index = 0 - - # Convert segment_index to list for consistent processing - if isinstance(segment_index, int): - segment_indices = [segment_index] - elif isinstance(segment_index, list): - segment_indices = segment_index - else: - raise ValueError("segment_index must be an int or a list of ints") - - # Validate segment indices - for idx in segment_indices: - if not isinstance(idx, int): - raise ValueError(f"Each segment index must be an integer, got {type(idx)}") - if idx < 0 or idx >= num_segments: - raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") + segment_indices = validate_segment_indices(sorting, segment_indices) if unit_ids is None: unit_ids = sorting.unit_ids diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index a1ac9d4af9..dd9bb20065 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -3,6 +3,7 @@ from warnings import warn import numpy as np +from spikeinterface.core import BaseSorting def get_some_colors( keys, @@ -349,3 +350,46 @@ def make_units_table_from_analyzer( ) return units_table + +def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSorting): + """ + Validate a list of segment indices for a sorting object. + + Parameters + ---------- + segment_indices : list of int + The segment index or indices to validate. + sorting : BaseSorting + The sorting object to validate against. + + Returns + ------- + list of int + A list of valid segment indices. + + Raises + ------ + ValueError + If the segment indices are not valid. + """ + num_segments = sorting.get_num_segments() + + # Handle segment_indices input + if segment_indices is None: + if num_segments > 1: + warn("Segment indices not specified. Using first available segment only.") + return [0] + + # Convert segment_index to list for consistent processing + if not isinstance(segment_indices, list): + raise ValueError("segment_indices must be a list of ints - available segments are: " + list(range(num_segments))) + + # Validate segment indices + for idx in segment_indices: + if not isinstance(idx, int): + raise ValueError(f"Each segment index must be an integer, got {type(idx)}") + if idx < 0 or idx >= num_segments: + raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") + + + return segment_indices \ No newline at end of file From 540db00a14517abbea6f59a383aa5f4f0a58e35c Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Wed, 30 Apr 2025 11:15:10 +0100 Subject: [PATCH 037/157] minor fixes --- src/spikeinterface/widgets/amplitudes.py | 6 ++++-- src/spikeinterface/widgets/motion.py | 8 ++++---- src/spikeinterface/widgets/rasters.py | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index a6fb9948ae..c09f3d82be 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -83,7 +83,7 @@ def __init__( # For SortingView, ensure we're only using a single segment if is_sortingview and len(segment_indices) > 1: warn("SortingView backend currently supports only single segment. Using first segment.") - segment_indices = segment_indices[0] + segment_indices = [segment_indices[0]] # Create multi-segment data structure (dict of dicts) spiketrains_by_segment = {} @@ -150,11 +150,13 @@ def __init__( first_segment = segment_indices[0] plot_data["spike_train_data"] = spiketrains_by_segment[first_segment] plot_data["y_axis_data"] = amplitudes_by_segment[first_segment] + print(plot_data["spike_train_data"]) + print(plot_data["y_axis_data"]) else: # Otherwise use the full dict of dicts structure with all segments plot_data["spike_train_data"] = spiketrains_by_segment plot_data["y_axis_data"] = amplitudes_by_segment - plot_data["segment_index"] = segment_indices + plot_data["segment_indices"] = segment_indices BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index f932ed44f6..9071698b2f 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -201,10 +201,10 @@ def __init__( if len(unique_segments) == 1: segment_indices = [int(unique_segments[0])] else: - raise ValueError("segment_index must be specified if there are multiple segments") + raise ValueError("segment_indices must be specified if there are multiple segments") if not isinstance(segment_indices, list): - raise ValueError("segment_index must be an int or a list of ints") + raise ValueError("segment_indices must be a list of ints") # Validate all segment indices exist in the data for idx in segment_indices: @@ -275,7 +275,7 @@ def __init__( plot_data = dict( spike_train_data=spike_train_data, y_axis_data=y_axis_data, - segment_index=segment_indices, + segment_indices=segment_indices, y_lim=depth_lim, color_kwargs=color_kwargs, scatter_decimate=scatter_decimate, @@ -417,7 +417,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): commpon_drift_map_kwargs = dict( direction=dp.motion.direction, recording=dp.recording, - segment_index=dp.segment_index, + segment_indices=list(dp.segment_index), depth_lim=dp.depth_lim, scatter_decimate=dp.scatter_decimate, color_amplitude=dp.color_amplitude, diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index f03f980f24..6012cdaed2 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -419,7 +419,7 @@ def __init__( sorting = self.ensure_sorting(sorting) - segment_indices = validate_segment_indices(sorting, segment_indices) + segment_indices = validate_segment_indices(segment_indices, sorting) if unit_ids is None: unit_ids = sorting.unit_ids @@ -479,7 +479,7 @@ def __init__( plot_data = dict( spike_train_data=spike_train_data, y_axis_data=y_axis_data, - segment_index=segment_indices, + segment_indices=segment_indices, x_lim=time_range, y_label="Unit id", unit_ids=unit_ids, From afdea786de73f7a068e85e27e6af8577737cd4b8 Mon Sep 17 00:00:00 2001 From: jakeswann1 Date: Wed, 30 Apr 2025 11:50:10 +0100 Subject: [PATCH 038/157] Update durations to use a list --- src/spikeinterface/widgets/amplitudes.py | 12 ++- src/spikeinterface/widgets/motion.py | 16 ++-- src/spikeinterface/widgets/rasters.py | 102 +++++++++-------------- 3 files changed, 51 insertions(+), 79 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index c09f3d82be..5c0f304bba 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -125,18 +125,18 @@ def __init__( if plot_histograms and bins is None: bins = 100 - # Calculate total duration across all segments for x-axis limits - total_duration = 0 + # Calculate durations for all segments for x-axis limits + durations = [] for idx in segment_indices: duration = sorting_analyzer.get_num_samples(idx) / sorting_analyzer.sampling_frequency - total_duration += duration + durations.append(duration) # Build the plot data with the full dict of dicts structure plot_data = dict( unit_colors=unit_colors, plot_histograms=plot_histograms, bins=bins, - total_duration=total_duration, + durations=durations, unit_ids=unit_ids, hide_unit_selector=hide_unit_selector, plot_legend=plot_legend, @@ -150,8 +150,6 @@ def __init__( first_segment = segment_indices[0] plot_data["spike_train_data"] = spiketrains_by_segment[first_segment] plot_data["y_axis_data"] = amplitudes_by_segment[first_segment] - print(plot_data["spike_train_data"]) - print(plot_data["y_axis_data"]) else: # Otherwise use the full dict of dicts structure with all segments plot_data["spike_train_data"] = spiketrains_by_segment @@ -178,7 +176,7 @@ def plot_sortingview(self, data_plot, **backend_kwargs): ] self.view = vv.SpikeAmplitudes( - start_time_sec=0, end_time_sec=dp.total_duration, plots=sa_items, hide_unit_selector=dp.hide_unit_selector + start_time_sec=0, end_time_sec=np.sum(dp.durations), plots=sa_items, hide_unit_selector=dp.hide_unit_selector ) self.url = handle_display_and_url(self, self.view, **backend_kwargs) diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index 9071698b2f..afdb4a6963 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -256,8 +256,8 @@ def __init__( else: color_kwargs = dict(color=color, c=None, alpha=alpha) - # Calculate total duration for x-axis limits - total_duration = 0 + # Calculate segment durations for x-axis limits + durations = [] for seg_idx in segment_indices: if recording is not None and hasattr(recording, "get_duration"): duration = recording.get_duration(seg_idx) @@ -270,7 +270,7 @@ def __init__( duration = (max_sample + 1) / sampling_frequency else: duration = 0 - total_duration += duration + durations.append(duration) plot_data = dict( spike_train_data=spike_train_data, @@ -281,7 +281,7 @@ def __init__( scatter_decimate=scatter_decimate, title="Peak depth", y_label="Depth [um]", - total_duration=total_duration, + durations=durations, ) BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) @@ -414,10 +414,10 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): dp.recording, ) - commpon_drift_map_kwargs = dict( + common_drift_map_kwargs = dict( direction=dp.motion.direction, recording=dp.recording, - segment_indices=list(dp.segment_index), + segment_indices=[dp.segment_index], depth_lim=dp.depth_lim, scatter_decimate=dp.scatter_decimate, color_amplitude=dp.color_amplitude, @@ -434,7 +434,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): dp.peak_locations, ax=ax0, immediate_plot=True, - **commpon_drift_map_kwargs, + **common_drift_map_kwargs, ) _ = DriftRasterMapWidget( @@ -442,7 +442,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): corrected_location, ax=ax1, immediate_plot=True, - **commpon_drift_map_kwargs, + **common_drift_map_kwargs, ) ax2.plot(temporal_bins_s, displacement, alpha=0.2, color="black") diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 6012cdaed2..534a30d7d7 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -26,8 +26,8 @@ class BaseRasterWidget(BaseWidget): segment_indices : list | None, default: None For multi-segment data, specifies which segment(s) to plot. If None, uses all available segments. For single-segment data, this parameter is ignored. - total_duration : int | None, default: None - Duration of spike_train_data in seconds. + durations : list | None, default: None + List of durations per segment of spike_train_data in seconds. plot_histograms : bool, default: False Plot histogram of y-axis data in another subplot bins : int | None, default: None @@ -66,7 +66,7 @@ def __init__( y_axis_data: dict, unit_ids: list | None = None, segment_indices: list | None = None, - total_duration: int | None = None, + durations: list | None = None, plot_histograms: bool = False, bins: int | None = None, scatter_decimate: int = 1, @@ -112,67 +112,41 @@ def __init__( all_units.update(spike_train_data[seg_idx].keys()) unit_ids = list(all_units) - # Calculate segment durations and boundaries - segment_durations = [] - for seg_idx in segments_to_use: - max_time = 0 - for unit_id in unit_ids: - if unit_id in spike_train_data[seg_idx]: - unit_times = spike_train_data[seg_idx][unit_id] - if len(unit_times) > 0: - max_time = max(max_time, np.max(unit_times)) - segment_durations.append(max_time) - # Calculate cumulative durations for segment boundaries - cumulative_durations = [0] - for duration in segment_durations[:-1]: - cumulative_durations.append(cumulative_durations[-1] + duration) - - # Segment boundaries for visualization (only internal boundaries) - segment_boundaries = cumulative_durations[1:] if len(segments_to_use) > 1 else None + segment_boundaries = np.cumsum(durations) + cumulative_durations = np.concatenate([[0], segment_boundaries]) # Concatenate data across segments with proper time offsets - concatenated_spike_trains = {unit_id: [] for unit_id in unit_ids} - concatenated_y_axis = {unit_id: [] for unit_id in unit_ids} - - for i, seg_idx in enumerate(segments_to_use): - offset = cumulative_durations[i] - - for unit_id in unit_ids: - if unit_id in spike_train_data[seg_idx]: - # Get spike times for this unit in this segment - spike_times = spike_train_data[seg_idx][unit_id] - - # Adjust spike times by adding cumulative duration of previous segments - if offset > 0: - adjusted_times = spike_times + offset - else: - adjusted_times = spike_times - - # Get y-axis data for this unit in this segment - y_values = y_axis_data[seg_idx][unit_id] - - # Concatenate with any existing data - if len(concatenated_spike_trains[unit_id]) > 0: - concatenated_spike_trains[unit_id] = np.concatenate( - [concatenated_spike_trains[unit_id], adjusted_times] - ) - concatenated_y_axis[unit_id] = np.concatenate([concatenated_y_axis[unit_id], y_values]) - else: - concatenated_spike_trains[unit_id] = adjusted_times - concatenated_y_axis[unit_id] = y_values - - # Update spike train and y-axis data with concatenated values - processed_spike_train_data = concatenated_spike_trains - processed_y_axis_data = concatenated_y_axis - - # Calculate total duration from the data if not provided - if total_duration is None: - total_duration = cumulative_durations[-1] + segment_durations[-1] + concatenated_spike_trains = {unit_id: np.array([]) for unit_id in unit_ids} + concatenated_y_axis = {unit_id: np.array([]) for unit_id in unit_ids} + + for offset, spike_train_segment, y_axis_segment in zip( + cumulative_durations, + [spike_train_data[idx] for idx in segments_to_use], + [y_axis_data[idx] for idx in segments_to_use] + ): + # Process each unit in the current segment + for unit_id, spike_times in spike_train_segment.items(): + if unit_id not in unit_ids: + continue + + # Get y-axis values for this unit + y_values = y_axis_segment[unit_id] + + # Apply offset to spike times + adjusted_times = spike_times + offset + + # Add to concatenated data + concatenated_spike_trains[unit_id] = np.concatenate( + [concatenated_spike_trains[unit_id], adjusted_times] + ) + concatenated_y_axis[unit_id] = np.concatenate( + [concatenated_y_axis[unit_id], y_values] + ) plot_data = dict( - spike_train_data=processed_spike_train_data, - y_axis_data=processed_y_axis_data, + spike_train_data=concatenated_spike_trains, + y_axis_data=concatenated_y_axis, unit_ids=unit_ids, plot_histograms=plot_histograms, y_lim=y_lim, @@ -182,7 +156,7 @@ def __init__( unit_colors=unit_colors, y_label=y_label, title=title, - total_duration=total_duration, + durations=durations, plot_legend=plot_legend, bins=bins, y_ticks=y_ticks, @@ -275,7 +249,7 @@ def plot_matplotlib(self, data_plot, **backend_kwargs): scatter_ax.set_ylim(*dp.y_lim) x_lim = dp.x_lim if x_lim is None: - x_lim = [0, dp.total_duration] + x_lim = [0, np.sum(dp.durations)] scatter_ax.set_xlim(x_lim) if dp.y_ticks: @@ -432,7 +406,7 @@ def __init__( unit_indices_map = {unit_id: i for i, unit_id in enumerate(unit_ids)} # Calculate total duration across all segments - total_duration = 0 + durations = [] for seg_idx in segment_indices: # Try to get duration from recording if available if recording is not None: @@ -446,7 +420,7 @@ def __init__( max_time = max(max_time, np.max(st)) duration = max_time - total_duration += duration + durations.append(duration) # Initialize dicts for this segment spike_train_data[seg_idx] = {} @@ -486,7 +460,7 @@ def __init__( unit_colors=unit_colors, plot_histograms=None, y_ticks=y_ticks, - total_duration=total_duration, + durations=durations, ) BaseRasterWidget.__init__(self, **plot_data, backend=backend, **backend_kwargs) From 65a52803365359ca8afafb570d65027cedb5fd2c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Apr 2025 10:53:05 +0000 Subject: [PATCH 039/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/widgets/amplitudes.py | 5 ++++- src/spikeinterface/widgets/motion.py | 2 +- src/spikeinterface/widgets/rasters.py | 18 ++++++++---------- src/spikeinterface/widgets/utils.py | 11 +++++++---- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 5c0f304bba..3d4d1d41fd 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -176,7 +176,10 @@ def plot_sortingview(self, data_plot, **backend_kwargs): ] self.view = vv.SpikeAmplitudes( - start_time_sec=0, end_time_sec=np.sum(dp.durations), plots=sa_items, hide_unit_selector=dp.hide_unit_selector + start_time_sec=0, + end_time_sec=np.sum(dp.durations), + plots=sa_items, + hide_unit_selector=dp.hide_unit_selector, ) self.url = handle_display_and_url(self, self.view, **backend_kwargs) diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index c93a0d2eeb..024baff29a 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -202,7 +202,7 @@ def __init__( segment_indices = [int(unique_segments[0])] else: raise ValueError("segment_indices must be specified if there are multiple segments") - + if not isinstance(segment_indices, list): raise ValueError("segment_indices must be a list of ints") diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 534a30d7d7..55d12e6102 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -113,36 +113,34 @@ def __init__( unit_ids = list(all_units) # Calculate cumulative durations for segment boundaries - segment_boundaries = np.cumsum(durations) - cumulative_durations = np.concatenate([[0], segment_boundaries]) + segment_boundaries = np.cumsum(durations) + cumulative_durations = np.concatenate([[0], segment_boundaries]) # Concatenate data across segments with proper time offsets concatenated_spike_trains = {unit_id: np.array([]) for unit_id in unit_ids} concatenated_y_axis = {unit_id: np.array([]) for unit_id in unit_ids} for offset, spike_train_segment, y_axis_segment in zip( - cumulative_durations, + cumulative_durations, [spike_train_data[idx] for idx in segments_to_use], - [y_axis_data[idx] for idx in segments_to_use] + [y_axis_data[idx] for idx in segments_to_use], ): # Process each unit in the current segment for unit_id, spike_times in spike_train_segment.items(): if unit_id not in unit_ids: continue - + # Get y-axis values for this unit y_values = y_axis_segment[unit_id] - + # Apply offset to spike times adjusted_times = spike_times + offset - + # Add to concatenated data concatenated_spike_trains[unit_id] = np.concatenate( [concatenated_spike_trains[unit_id], adjusted_times] ) - concatenated_y_axis[unit_id] = np.concatenate( - [concatenated_y_axis[unit_id], y_values] - ) + concatenated_y_axis[unit_id] = np.concatenate([concatenated_y_axis[unit_id], y_values]) plot_data = dict( spike_train_data=concatenated_spike_trains, diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index dd9bb20065..898142f515 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -5,6 +5,7 @@ from spikeinterface.core import BaseSorting + def get_some_colors( keys, color_engine="auto", @@ -351,7 +352,8 @@ def make_units_table_from_analyzer( return units_table -def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSorting): + +def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSorting): """ Validate a list of segment indices for a sorting object. @@ -382,7 +384,9 @@ def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSo # Convert segment_index to list for consistent processing if not isinstance(segment_indices, list): - raise ValueError("segment_indices must be a list of ints - available segments are: " + list(range(num_segments))) + raise ValueError( + "segment_indices must be a list of ints - available segments are: " + list(range(num_segments)) + ) # Validate segment indices for idx in segment_indices: @@ -391,5 +395,4 @@ def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSo if idx < 0 or idx >= num_segments: raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") - - return segment_indices \ No newline at end of file + return segment_indices From c43fa7c13943236150bfd761d4ba2b023e172f51 Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Mon, 12 May 2025 11:56:31 -0400 Subject: [PATCH 040/157] add test for validate_segment_indices --- .../widgets/tests/test_widgets_utils.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index 2131969c2c..de096d0197 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -1,4 +1,7 @@ -from spikeinterface.widgets.utils import get_some_colors +import pytest + +from spikeinterface import generate_sorting +from spikeinterface.widgets.utils import get_some_colors, validate_segment_indices def test_get_some_colors(): @@ -19,5 +22,34 @@ def test_get_some_colors(): # print(colors) +def test_validate_segment_indices(): + # Setup + sorting_single = generate_sorting(durations=[5]) # 1 segment + sorting_multiple = generate_sorting(durations=[5, 10, 15, 20, 25]) # 5 segments + + # Test None with single segment + assert validate_segment_indices(None, sorting_single) == [0] + + # Test None with multiple segments + with pytest.warns(UserWarning): + assert validate_segment_indices(None, sorting_multiple) == [0] + + # Test valid indices + assert validate_segment_indices([0], sorting_single) == [0] + assert validate_segment_indices([0, 1, 4], sorting_multiple) == [0, 1, 4] + + # Test invalid type + with pytest.raises(TypeError): + validate_segment_indices(0, sorting_multiple) + + # Test invalid index type + with pytest.raises(ValueError): + validate_segment_indices([0, "1"], sorting_multiple) + + # Test out of range + with pytest.raises(ValueError): + validate_segment_indices([5], sorting_multiple) + if __name__ == "__main__": test_get_some_colors() + test_validate_segment_indices() From 55e773e81df3879abb1405a9adeb0b73687b2038 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 16:24:21 +0000 Subject: [PATCH 041/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../widgets/tests/test_widgets_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index de096d0197..db6f1bf537 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -26,30 +26,31 @@ def test_validate_segment_indices(): # Setup sorting_single = generate_sorting(durations=[5]) # 1 segment sorting_multiple = generate_sorting(durations=[5, 10, 15, 20, 25]) # 5 segments - + # Test None with single segment assert validate_segment_indices(None, sorting_single) == [0] - + # Test None with multiple segments with pytest.warns(UserWarning): assert validate_segment_indices(None, sorting_multiple) == [0] - + # Test valid indices assert validate_segment_indices([0], sorting_single) == [0] assert validate_segment_indices([0, 1, 4], sorting_multiple) == [0, 1, 4] - + # Test invalid type with pytest.raises(TypeError): validate_segment_indices(0, sorting_multiple) - + # Test invalid index type with pytest.raises(ValueError): validate_segment_indices([0, "1"], sorting_multiple) - + # Test out of range with pytest.raises(ValueError): validate_segment_indices([5], sorting_multiple) + if __name__ == "__main__": test_get_some_colors() test_validate_segment_indices() From b0985576695aab738f1a9498225f828ae6f318b7 Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Mon, 12 May 2025 12:56:13 -0400 Subject: [PATCH 042/157] simplify segment duration computation --- src/spikeinterface/widgets/amplitudes.py | 7 +-- src/spikeinterface/widgets/motion.py | 27 ++++++------ src/spikeinterface/widgets/rasters.py | 44 +++++++------------ .../widgets/tests/test_widgets_utils.py | 39 +++++++++++++++- src/spikeinterface/widgets/utils.py | 29 ++++++++++++ 5 files changed, 97 insertions(+), 49 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 3d4d1d41fd..6f36baf521 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -5,7 +5,7 @@ from .rasters import BaseRasterWidget from .base import BaseWidget, to_attr -from .utils import get_some_colors, validate_segment_indices +from .utils import get_some_colors, validate_segment_indices, get_segment_durations from spikeinterface.core.sortinganalyzer import SortingAnalyzer @@ -126,10 +126,7 @@ def __init__( bins = 100 # Calculate durations for all segments for x-axis limits - durations = [] - for idx in segment_indices: - duration = sorting_analyzer.get_num_samples(idx) / sorting_analyzer.sampling_frequency - durations.append(duration) + durations = get_segment_durations(sorting) # Build the plot data with the full dict of dicts structure plot_data = dict( diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index 024baff29a..6b2936afb7 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -6,6 +6,7 @@ from spikeinterface.core import BaseRecording, SortingAnalyzer from .rasters import BaseRasterWidget +from .utils import get_segment_durations from spikeinterface.core.motion import Motion @@ -259,20 +260,18 @@ def __init__( color_kwargs = dict(color=color, c=None, alpha=alpha) # Calculate segment durations for x-axis limits - durations = [] - for seg_idx in segment_indices: - if recording is not None and hasattr(recording, "get_duration"): - duration = recording.get_duration(seg_idx) - else: - # Estimate from spike times - segment_mask = filtered_peaks["segment_index"] == seg_idx - segment_peaks = filtered_peaks[segment_mask] - if len(segment_peaks) > 0: - max_sample = np.max(segment_peaks["sample_index"]) - duration = (max_sample + 1) / sampling_frequency - else: - duration = 0 - durations.append(duration) + if recording is not None: + durations = [recording.get_duration(seg_idx) for seg_idx in segment_indices] + else: + # Find boundaries between segments using searchsorted + segment_boundaries = [np.searchsorted(filtered_peaks["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices] + + # Calculate durations from max sample in each segment + durations = [ + (np.max(filtered_peaks["sample_index"][start:end]) + 1) / sampling_frequency + if start < end else 0 + for (start, end) in segment_boundaries + ] plot_data = dict( spike_train_data=spike_train_data, diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 55d12e6102..d1452b15cb 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -4,7 +4,7 @@ from warnings import warn from .base import BaseWidget, to_attr, default_backend_kwargs -from .utils import get_some_colors, validate_segment_indices +from .utils import get_some_colors, validate_segment_indices, get_segment_durations class BaseRasterWidget(BaseWidget): @@ -380,14 +380,12 @@ def __init__( backend=None, **backend_kwargs, ): - recording = None if sorting is None and sorting_analyzer is None: raise Exception("Must supply either a sorting or a sorting_analyzer") elif sorting is not None and sorting_analyzer is not None: raise Exception("Should supply either a sorting or a sorting_analyzer, not both") elif sorting_analyzer is not None: sorting = sorting_analyzer.sorting - recording = sorting_analyzer.recording sorting = self.ensure_sorting(sorting) @@ -403,37 +401,25 @@ def __init__( # Create a lookup dictionary for unit indices unit_indices_map = {unit_id: i for i, unit_id in enumerate(unit_ids)} - # Calculate total duration across all segments - durations = [] - for seg_idx in segment_indices: - # Try to get duration from recording if available - if recording is not None: - duration = recording.get_duration(seg_idx) - else: - # Fallback: estimate from max spike time - max_time = 0 - for unit_id in unit_ids: - st = sorting.get_unit_spike_train(unit_id, segment_index=seg_idx, return_times=True) - if len(st) > 0: - max_time = max(max_time, np.max(st)) - duration = max_time + # Get all spikes at once + spikes = sorting.to_spike_vector() - durations.append(duration) + # Estimate segment duration from max spike time in each segment + durations = get_segment_durations(sorting) - # Initialize dicts for this segment - spike_train_data[seg_idx] = {} - y_axis_data[seg_idx] = {} + # Extract spike data for all segments and units at once + spike_train_data = {seg_idx: {} for seg_idx in segment_indices} + y_axis_data = {seg_idx: {} for seg_idx in segment_indices} - # Get spike trains for each unit in this segment + for seg_idx in segment_indices: for unit_id in unit_ids: - spike_times = sorting.get_unit_spike_train(unit_id, segment_index=seg_idx, return_times=True) - - # Store spike trains + # Get spikes for this segment and unit + mask = (spikes['segment_index'] == seg_idx) & (spikes['unit_index'] == unit_id) + spike_times = spikes['sample_index'][mask] / sorting.sampling_frequency + + # Store data spike_train_data[seg_idx][unit_id] = spike_times - - # Create raster locations (y-values for plotting) - unit_index = unit_indices_map[unit_id] - y_axis_data[seg_idx][unit_id] = unit_index * np.ones(len(spike_times)) + y_axis_data[seg_idx][unit_id] = unit_indices_map[unit_id] * np.ones(len(spike_times)) # Apply time range filtering if specified if time_range is not None: diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index db6f1bf537..e29309bea0 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -1,7 +1,7 @@ import pytest from spikeinterface import generate_sorting -from spikeinterface.widgets.utils import get_some_colors, validate_segment_indices +from spikeinterface.widgets.utils import get_some_colors, validate_segment_indices, get_segment_durations def test_get_some_colors(): @@ -50,7 +50,44 @@ def test_validate_segment_indices(): with pytest.raises(ValueError): validate_segment_indices([5], sorting_multiple) +def test_get_segment_durations(): + from spikeinterface import generate_sorting + + # Test with a normal multi-segment sorting + durations = [5.0, 10.0, 15.0] + + # Create sorting with high fr to ensure spikes near the end segments + sorting = generate_sorting( + durations=durations, + firing_rates=15.0, + ) + + # Calculate durations + calculated_durations = get_segment_durations(sorting) + + # Check results + assert len(calculated_durations) == len(durations) + # Durations should be approximately correct + for calculated_duration, expected_duration in zip(calculated_durations, durations): + # Duration should be <= expected (spikes can't be after the end) + assert calculated_duration <= expected_duration + # And reasonably close + tolerance = max(0.1 * expected_duration, 0.1) + assert expected_duration - calculated_duration < tolerance + + # Test with single-segment sorting + sorting_single = generate_sorting( + durations=[7.0], + firing_rates=15.0, + ) + + single_duration = get_segment_durations(sorting_single)[0] + + # Test that the calculated duration is reasonable + assert single_duration <= 7.0 + assert 7.0 - single_duration < 0.7 # Within 10% if __name__ == "__main__": test_get_some_colors() test_validate_segment_indices() + test_get_segment_durations() diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 898142f515..3e55b71bd5 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -396,3 +396,32 @@ def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSor raise ValueError(f"segment_index {idx} out of range (0 to {num_segments - 1})") return segment_indices + +def get_segment_durations(sorting: BaseSorting) -> list[float]: + """ + Calculate the duration of each segment in a sorting object. + + Parameters + ---------- + sorting : BaseSorting + The sorting object containing spike data + + Returns + ------- + list[float] + List of segment durations in seconds + """ + spikes = sorting.to_spike_vector() + segment_indices = np.unique(spikes['segment_index']) + + durations = [] + for seg_idx in segment_indices: + segment_mask = spikes['segment_index'] == seg_idx + if np.any(segment_mask): + max_sample = np.max(spikes['sample_index'][segment_mask]) + duration = max_sample / sorting.sampling_frequency + else: + duration = 0 + durations.append(duration) + + return durations \ No newline at end of file From 9dfd1a4150dd6102347f82bcf44b5f25e959787b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 16:56:41 +0000 Subject: [PATCH 043/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/widgets/motion.py | 9 +++++---- src/spikeinterface/widgets/rasters.py | 6 +++--- .../widgets/tests/test_widgets_utils.py | 16 +++++++++------- src/spikeinterface/widgets/utils.py | 17 +++++++++-------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index 6b2936afb7..dbc271f305 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -264,12 +264,13 @@ def __init__( durations = [recording.get_duration(seg_idx) for seg_idx in segment_indices] else: # Find boundaries between segments using searchsorted - segment_boundaries = [np.searchsorted(filtered_peaks["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices] - + segment_boundaries = [ + np.searchsorted(filtered_peaks["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices + ] + # Calculate durations from max sample in each segment durations = [ - (np.max(filtered_peaks["sample_index"][start:end]) + 1) / sampling_frequency - if start < end else 0 + (np.max(filtered_peaks["sample_index"][start:end]) + 1) / sampling_frequency if start < end else 0 for (start, end) in segment_boundaries ] diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index d1452b15cb..4219b34c3d 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -414,9 +414,9 @@ def __init__( for seg_idx in segment_indices: for unit_id in unit_ids: # Get spikes for this segment and unit - mask = (spikes['segment_index'] == seg_idx) & (spikes['unit_index'] == unit_id) - spike_times = spikes['sample_index'][mask] / sorting.sampling_frequency - + mask = (spikes["segment_index"] == seg_idx) & (spikes["unit_index"] == unit_id) + spike_times = spikes["sample_index"][mask] / sorting.sampling_frequency + # Store data spike_train_data[seg_idx][unit_id] = spike_times y_axis_data[seg_idx][unit_id] = unit_indices_map[unit_id] * np.ones(len(spike_times)) diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index e29309bea0..ff4bfd957c 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -50,21 +50,22 @@ def test_validate_segment_indices(): with pytest.raises(ValueError): validate_segment_indices([5], sorting_multiple) + def test_get_segment_durations(): from spikeinterface import generate_sorting - + # Test with a normal multi-segment sorting durations = [5.0, 10.0, 15.0] - + # Create sorting with high fr to ensure spikes near the end segments sorting = generate_sorting( durations=durations, firing_rates=15.0, ) - + # Calculate durations calculated_durations = get_segment_durations(sorting) - + # Check results assert len(calculated_durations) == len(durations) # Durations should be approximately correct @@ -74,19 +75,20 @@ def test_get_segment_durations(): # And reasonably close tolerance = max(0.1 * expected_duration, 0.1) assert expected_duration - calculated_duration < tolerance - + # Test with single-segment sorting sorting_single = generate_sorting( durations=[7.0], firing_rates=15.0, ) - + single_duration = get_segment_durations(sorting_single)[0] - + # Test that the calculated duration is reasonable assert single_duration <= 7.0 assert 7.0 - single_duration < 0.7 # Within 10% + if __name__ == "__main__": test_get_some_colors() test_validate_segment_indices() diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 3e55b71bd5..9c5892a937 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -397,31 +397,32 @@ def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSor return segment_indices + def get_segment_durations(sorting: BaseSorting) -> list[float]: """ Calculate the duration of each segment in a sorting object. - + Parameters ---------- sorting : BaseSorting The sorting object containing spike data - + Returns ------- list[float] List of segment durations in seconds """ spikes = sorting.to_spike_vector() - segment_indices = np.unique(spikes['segment_index']) - + segment_indices = np.unique(spikes["segment_index"]) + durations = [] for seg_idx in segment_indices: - segment_mask = spikes['segment_index'] == seg_idx + segment_mask = spikes["segment_index"] == seg_idx if np.any(segment_mask): - max_sample = np.max(spikes['sample_index'][segment_mask]) + max_sample = np.max(spikes["sample_index"][segment_mask]) duration = max_sample / sorting.sampling_frequency else: duration = 0 durations.append(duration) - - return durations \ No newline at end of file + + return durations From b0d33d53d4c0a80e238e709e1d1a5d826dcdd4d5 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 20 May 2025 14:07:42 -0600 Subject: [PATCH 044/157] add tests --- src/spikeinterface/core/tests/test_time_handling.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/spikeinterface/core/tests/test_time_handling.py b/src/spikeinterface/core/tests/test_time_handling.py index ffdb121316..69a6a330c0 100644 --- a/src/spikeinterface/core/tests/test_time_handling.py +++ b/src/spikeinterface/core/tests/test_time_handling.py @@ -435,3 +435,12 @@ def _get_sorting_with_recording_attached(self, recording_for_durations, recordin assert sorting.has_recording() return sorting + + +def test_shift_times_with_None_as_t_start(): + + recording = generate_recording(num_channels=4, durations=[10]) + + assert recording._recording_segments[0].t_start is None + recording.shift_times(shift=1.0) # Shift by one seconds should not generate an error + assert recording.get_t_start() == 1.0 From a812163bf790fc17fe474c597cbb9160a7b3b56f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 20 May 2025 14:10:27 -0600 Subject: [PATCH 045/157] fix tests --- src/spikeinterface/core/baserecording.py | 3 ++- src/spikeinterface/core/tests/test_time_handling.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/core/baserecording.py b/src/spikeinterface/core/baserecording.py index d944ea22a4..8104569db7 100644 --- a/src/spikeinterface/core/baserecording.py +++ b/src/spikeinterface/core/baserecording.py @@ -547,7 +547,8 @@ def shift_times(self, shift: int | float, segment_index: int | None = None) -> N if self.has_time_vector(segment_index=idx): rs.time_vector += shift else: - rs.t_start += shift + new_start_time = 0 + shift if rs.t_start is None else rs.t_start + shift + rs.t_start = new_start_time def sample_index_to_time(self, sample_ind, segment_index=None): """ diff --git a/src/spikeinterface/core/tests/test_time_handling.py b/src/spikeinterface/core/tests/test_time_handling.py index 69a6a330c0..b70140b4b7 100644 --- a/src/spikeinterface/core/tests/test_time_handling.py +++ b/src/spikeinterface/core/tests/test_time_handling.py @@ -443,4 +443,4 @@ def test_shift_times_with_None_as_t_start(): assert recording._recording_segments[0].t_start is None recording.shift_times(shift=1.0) # Shift by one seconds should not generate an error - assert recording.get_t_start() == 1.0 + assert recording.get_start_time() == 1.0 From 0d89d2832538a4b25ba0b3c3c02ac30c024c67c2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 20 May 2025 14:10:54 -0600 Subject: [PATCH 046/157] fix tests --- src/spikeinterface/core/tests/test_time_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/core/tests/test_time_handling.py b/src/spikeinterface/core/tests/test_time_handling.py index b70140b4b7..2ae435ea84 100644 --- a/src/spikeinterface/core/tests/test_time_handling.py +++ b/src/spikeinterface/core/tests/test_time_handling.py @@ -438,7 +438,7 @@ def _get_sorting_with_recording_attached(self, recording_for_durations, recordin def test_shift_times_with_None_as_t_start(): - + """Ensures we can shift times even when t_stat is None which is interpeted as zero""" recording = generate_recording(num_channels=4, durations=[10]) assert recording._recording_segments[0].t_start is None From 1313c55f0aee736c5d1d344e7f6cf7dc07f167e4 Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:10:23 +0200 Subject: [PATCH 047/157] fix: - change the channel and color property name to neuroscope_groups - method now need to be called by the user --- .../extractors/neoextractors/neuroscope.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index efbcb57e59..ddda070d25 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -66,7 +66,6 @@ def __init__( xml_file_path = str(Path(xml_file_path).absolute()) self._kwargs.update(dict(file_path=str(Path(file_path).absolute()), xml_file_path=xml_file_path)) self.xml_file_path = xml_file_path if xml_file_path is not None else Path(file_path).with_suffix(".xml") - self._set_groups() @classmethod def map_to_neo_kwargs(cls, file_path, xml_file_path=None): @@ -121,7 +120,7 @@ def _parse_xml_file(self, xml_file_path): return channel_groups, kept_channels, discarded_channels, anatomycolors - def _set_groups(self): + def _set_neuroscope_groups(self): """ Set the group ids and colors based on the xml file. These group ids are usually different brain/body anatomical areas, or shanks from multi-shank probes. @@ -139,6 +138,13 @@ def _set_groups(self): self.set_property("discarded_channels", discarded_ppty) self.set_property("colors", values=list(colors.values()), ids=list(colors.keys())) + def prepare_neuroscope_for_ephyviewer(self): + """ + Prepare the recording extractor for ephyviewer by setting the group ids and colors. + This function is not called when the extractor is initialized, and the user must call it manually. + """ + self._set_neuroscope_groups() + class NeuroScopeSortingExtractor(BaseSorting): """ From 63af579a08cbe60d0fa49b6cb8b2064d8e6b3012 Mon Sep 17 00:00:00 2001 From: theodchrn <48409582+theodchrn@users.noreply.github.com> Date: Fri, 13 Jun 2025 15:31:34 +0200 Subject: [PATCH 048/157] change group attribute to neuroscope group for more clarity --- src/spikeinterface/extractors/neoextractors/neuroscope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/neuroscope.py b/src/spikeinterface/extractors/neoextractors/neuroscope.py index ddda070d25..298a2d6109 100644 --- a/src/spikeinterface/extractors/neoextractors/neuroscope.py +++ b/src/spikeinterface/extractors/neoextractors/neuroscope.py @@ -132,7 +132,7 @@ def _set_neuroscope_groups(self): channel_groups, kept_channels, discarded_channels, colors = self._parse_xml_file(self.xml_file_path) for group_id, numbers in enumerate(channel_groups): group_ids[numbers] = group_id # Assign group_id to the positions in `numbers` - self.set_property("group", group_ids) + self.set_property("neuroscope_group", group_ids) discarded_ppty = np.full(n, False, dtype=bool) discarded_ppty[discarded_channels] = True self.set_property("discarded_channels", discarded_ppty) From d4321ed15b1510a5e2087ced754498d594d0817b Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:09:07 -0400 Subject: [PATCH 049/157] First pass of deprecations for 0.103.0 --- doc/get_started/quickstart.rst | 2 +- doc/how_to/analyze_neuropixels.rst | 2 +- doc/how_to/process_by_channel_group.rst | 2 +- doc/modules/sorters.rst | 34 +++++++++--------- .../qualitymetrics/plot_3_quality_metrics.py | 4 +-- installation_tips/check_your_install.py | 2 +- .../benchmark_motion_interpolation.py | 2 +- src/spikeinterface/core/baserecording.py | 36 ------------------- .../core/baserecordingsnippets.py | 23 ------------ src/spikeinterface/core/basesnippets.py | 8 ----- src/spikeinterface/core/sparsity.py | 28 --------------- .../extractors/nwbextractors.py | 24 ------------- .../tests/test_nwbextractors_streaming.py | 4 +-- .../qualitymetrics/pca_metrics.py | 22 ------------ .../qualitymetrics/quality_metric_list.py | 1 - .../qualitymetrics/tests/test_pca_metrics.py | 2 +- src/spikeinterface/sorters/launcher.py | 9 ----- src/spikeinterface/sorters/runsorter.py | 30 ---------------- 18 files changed, 26 insertions(+), 209 deletions(-) diff --git a/doc/get_started/quickstart.rst b/doc/get_started/quickstart.rst index bfa4335ac1..1d532c9387 100644 --- a/doc/get_started/quickstart.rst +++ b/doc/get_started/quickstart.rst @@ -336,7 +336,7 @@ Alternatively we can pass a full dictionary containing the parameters: # parameters set by params dictionary sorting_TDC_2 = ss.run_sorter( - sorter_name="tridesclous", recording=recording_preprocessed, output_folder="tdc_output2", **other_params + sorter_name="tridesclous", recording=recording_preprocessed, folder="tdc_output2", **other_params ) print(sorting_TDC_2) diff --git a/doc/how_to/analyze_neuropixels.rst b/doc/how_to/analyze_neuropixels.rst index 1fe741ea48..04a9736b80 100644 --- a/doc/how_to/analyze_neuropixels.rst +++ b/doc/how_to/analyze_neuropixels.rst @@ -567,7 +567,7 @@ In this example: # run kilosort2.5 without drift correction params_kilosort2_5 = {'do_correction': False} - sorting = si.run_sorter('kilosort2_5', rec, output_folder=base_folder / 'kilosort2.5_output', + sorting = si.run_sorter('kilosort2_5', rec, folder=base_folder / 'kilosort2.5_output', docker_image=True, verbose=True, **params_kilosort2_5) .. code:: ipython3 diff --git a/doc/how_to/process_by_channel_group.rst b/doc/how_to/process_by_channel_group.rst index 334f83b247..0e6ae49d37 100644 --- a/doc/how_to/process_by_channel_group.rst +++ b/doc/how_to/process_by_channel_group.rst @@ -160,7 +160,7 @@ sorting objects in a dictionary for later use. sorting = run_sorter( sorter_name='kilosort2', recording=split_preprocessed_recording, - output_folder=f"folder_KS2_group{group}" + folder=f"folder_KS2_group{group}" ) sortings[group] = sorting diff --git a/doc/modules/sorters.rst b/doc/modules/sorters.rst index d8a4708236..6bf3a60e46 100644 --- a/doc/modules/sorters.rst +++ b/doc/modules/sorters.rst @@ -55,15 +55,15 @@ to easily run spike sorters: from spikeinterface.sorters import run_sorter # run Tridesclous - sorting_TDC = run_sorter(sorter_name="tridesclous", recording=recording, output_folder="/folder_TDC") + sorting_TDC = run_sorter(sorter_name="tridesclous", recording=recording, folder="/folder_TDC") # run Kilosort2.5 - sorting_KS2_5 = run_sorter(sorter_name="kilosort2_5", recording=recording, output_folder="/folder_KS2_5") + sorting_KS2_5 = run_sorter(sorter_name="kilosort2_5", recording=recording, folder="/folder_KS2_5") # run IronClust - sorting_IC = run_sorter(sorter_name="ironclust", recording=recording, output_folder="/folder_IC") + sorting_IC = run_sorter(sorter_name="ironclust", recording=recording, folder="/folder_IC") # run pyKilosort - sorting_pyKS = run_sorter(sorter_name="pykilosort", recording=recording, output_folder="/folder_pyKS") + sorting_pyKS = run_sorter(sorter_name="pykilosort", recording=recording, folder="/folder_pyKS") # run SpykingCircus - sorting_SC = run_sorter(sorter_name="spykingcircus", recording=recording, output_folder="/folder_SC") + sorting_SC = run_sorter(sorter_name="spykingcircus", recording=recording, folder="/folder_SC") Then the output, which is a :py:class:`~spikeinterface.core.BaseSorting` object, can be easily @@ -87,10 +87,10 @@ Spike-sorter-specific parameters can be controlled directly from the .. code-block:: python - sorting_TDC = run_sorter(sorter_name='tridesclous', recording=recording, output_folder="/folder_TDC", + sorting_TDC = run_sorter(sorter_name='tridesclous', recording=recording, folder="/folder_TDC", detect_threshold=8.) - sorting_KS2_5 = run_sorter(sorter_name="kilosort2_5", recording=recording, output_folder="/folder_KS2_5" + sorting_KS2_5 = run_sorter(sorter_name="kilosort2_5", recording=recording, folder="/folder_KS2_5" do_correction=False, preclust_threshold=6, freq_min=200.) @@ -193,7 +193,7 @@ The following code creates a test recording and runs a containerized spike sorte sorting = ss.run_sorter(sorter_name='kilosort3', recording=test_recording, - output_folder="kilosort3", + folder="kilosort3", singularity_image=True) print(sorting) @@ -208,7 +208,7 @@ To run in Docker instead of Singularity, use ``docker_image=True``. .. code-block:: python sorting = run_sorter(sorter_name='kilosort3', recording=test_recording, - output_folder="/tmp/kilosort3", docker_image=True) + folder="/tmp/kilosort3", docker_image=True) To use a specific image, set either ``docker_image`` or ``singularity_image`` to a string, e.g. ``singularity_image="spikeinterface/kilosort3-compiled-base:0.1.0"``. @@ -217,7 +217,7 @@ e.g. ``singularity_image="spikeinterface/kilosort3-compiled-base:0.1.0"``. sorting = run_sorter(sorter_name="kilosort3", recording=test_recording, - output_folder="kilosort3", + folder="kilosort3", singularity_image="spikeinterface/kilosort3-compiled-base:0.1.0") @@ -301,10 +301,10 @@ an :code:`engine` that supports parallel processing (such as :code:`joblib` or : another_recording = ... job_list = [ - {'sorter_name': 'tridesclous', 'recording': recording, 'output_folder': 'folder1','detect_threshold': 5.}, - {'sorter_name': 'tridesclous', 'recording': another_recording, 'output_folder': 'folder2', 'detect_threshold': 5.}, - {'sorter_name': 'herdingspikes', 'recording': recording, 'output_folder': 'folder3', 'clustering_bandwidth': 8., 'docker_image': True}, - {'sorter_name': 'herdingspikes', 'recording': another_recording, 'output_folder': 'folder4', 'clustering_bandwidth': 8., 'docker_image': True}, + {'sorter_name': 'tridesclous', 'recording': recording, 'folder': 'folder1','detect_threshold': 5.}, + {'sorter_name': 'tridesclous', 'recording': another_recording, 'folder': 'folder2', 'detect_threshold': 5.}, + {'sorter_name': 'herdingspikes', 'recording': recording, 'folder': 'folder3', 'clustering_bandwidth': 8., 'docker_image': True}, + {'sorter_name': 'herdingspikes', 'recording': another_recording, 'folder': 'folder4', 'clustering_bandwidth': 8., 'docker_image': True}, ] # run in loop @@ -380,7 +380,7 @@ In this example, we create a 16-channel recording with 4 tetrodes: # here the result is a dict of a sorting object sortings = {} for group, sub_recording in recordings.items(): - sorting = run_sorter(sorter_name='kilosort2', recording=recording, output_folder=f"folder_KS2_group{group}") + sorting = run_sorter(sorter_name='kilosort2', recording=recording, folder=f"folder_KS2_group{group}") sortings[group] = sorting **Option 2 : Automatic splitting** @@ -390,7 +390,7 @@ In this example, we create a 16-channel recording with 4 tetrodes: # here the result is one sorting that aggregates all sub sorting objects aggregate_sorting = run_sorter_by_property(sorter_name='kilosort2', recording=recording_4_tetrodes, grouping_property='group', - working_folder='working_path') + folder='working_path') Handling multi-segment recordings @@ -546,7 +546,7 @@ From the user's perspective, they behave exactly like the external sorters: .. code-block:: python - sorting = run_sorter(sorter_name="spykingcircus2", recording=recording, output_folder="/tmp/folder") + sorting = run_sorter(sorter_name="spykingcircus2", recording=recording, folder="/tmp/folder") Contributing diff --git a/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py b/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py index bfa6880cb0..a6b0da67ac 100644 --- a/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py +++ b/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py @@ -9,12 +9,10 @@ import spikeinterface.core as si import spikeinterface.extractors as se -from spikeinterface.postprocessing import compute_principal_components from spikeinterface.qualitymetrics import ( compute_snrs, compute_firing_rates, compute_isi_violations, - calculate_pc_metrics, compute_quality_metrics, ) @@ -70,7 +68,7 @@ ############################################################################## # Some metrics are based on the principal component scores, so the exwtension -# need to be computed before. For instance: +# must be computed before. For instance: analyzer.compute("principal_components", n_components=3, mode="by_channel_global", whiten=True) diff --git a/installation_tips/check_your_install.py b/installation_tips/check_your_install.py index f3f80961e8..92bafd3d55 100644 --- a/installation_tips/check_your_install.py +++ b/installation_tips/check_your_install.py @@ -21,7 +21,7 @@ def _run_one_sorter_and_analyzer(sorter_name): job_kwargs = dict(n_jobs=-1, progress_bar=True, chunk_duration="1s") import spikeinterface.full as si recording = si.load_extractor('./toy_example_recording') - sorting = si.run_sorter(sorter_name, recording, output_folder=f'./sorter_with_{sorter_name}', verbose=False) + sorting = si.run_sorter(sorter_name, recording, folder=f'./sorter_with_{sorter_name}', verbose=False) sorting_analyzer = si.create_sorting_analyzer(sorting, recording, format="binary_folder", folder=f"./analyzer_with_{sorter_name}", diff --git a/src/spikeinterface/benchmark/benchmark_motion_interpolation.py b/src/spikeinterface/benchmark/benchmark_motion_interpolation.py index ab72a1f9bd..c0969ed32a 100644 --- a/src/spikeinterface/benchmark/benchmark_motion_interpolation.py +++ b/src/spikeinterface/benchmark/benchmark_motion_interpolation.py @@ -53,7 +53,7 @@ def run(self, **job_kwargs): sorting = run_sorter( sorter_name, recording, - output_folder=self.sorter_folder, + folder=self.sorter_folder, **sorter_params, delete_output_folder=False, ) diff --git a/src/spikeinterface/core/baserecording.py b/src/spikeinterface/core/baserecording.py index b32893981f..ac299f52e7 100644 --- a/src/spikeinterface/core/baserecording.py +++ b/src/spikeinterface/core/baserecording.py @@ -369,21 +369,6 @@ def get_traces( traces = traces.astype("float32", copy=False) * gains + offsets return traces - def has_scaled_traces(self) -> bool: - """Checks if the recording has scaled traces - - Returns - ------- - bool - True if the recording has scaled traces, False otherwise - """ - warnings.warn( - "`has_scaled_traces` is deprecated and will be removed in 0.103.0. Use has_scaleable_traces() instead", - category=DeprecationWarning, - stacklevel=2, - ) - return self.has_scaled() - def get_time_info(self, segment_index=None) -> dict: """ Retrieves the timing attributes for a given segment index. As with @@ -725,17 +710,6 @@ def rename_channels(self, new_channel_ids: list | np.array | tuple) -> "BaseReco return ChannelSliceRecording(self, renamed_channel_ids=new_channel_ids) - def _channel_slice(self, channel_ids, renamed_channel_ids=None): - from .channelslice import ChannelSliceRecording - - warnings.warn( - "Recording.channel_slice will be removed in version 0.103, use `select_channels` or `rename_channels` instead.", - DeprecationWarning, - stacklevel=2, - ) - sub_recording = ChannelSliceRecording(self, channel_ids, renamed_channel_ids=renamed_channel_ids) - return sub_recording - def _remove_channels(self, remove_channel_ids): from .channelslice import ChannelSliceRecording @@ -878,8 +852,6 @@ def binary_compatible_with( time_axis=None, file_paths_length=None, file_offset=None, - file_suffix=None, - file_paths_lenght=None, ): """ Check is the recording is binary compatible with some constrain on @@ -891,14 +863,6 @@ def binary_compatible_with( * file_suffix """ - # spelling typo need to fix - if file_paths_lenght is not None: - warnings.warn( - "`file_paths_lenght` is deprecated and will be removed in 0.103.0 please use `file_paths_length`" - ) - if file_paths_length is None: - file_paths_length = file_paths_lenght - if not self.is_binary_compatible(): return False diff --git a/src/spikeinterface/core/baserecordingsnippets.py b/src/spikeinterface/core/baserecordingsnippets.py index ea1f9c4542..3be8ef6938 100644 --- a/src/spikeinterface/core/baserecordingsnippets.py +++ b/src/spikeinterface/core/baserecordingsnippets.py @@ -51,14 +51,6 @@ def has_scaleable_traces(self) -> bool: else: return True - def has_scaled(self): - warn( - "`has_scaled` has been deprecated and will be removed in 0.103.0. Please use `has_scaleable_traces()`", - category=DeprecationWarning, - stacklevel=2, - ) - return self.has_scaleable_traces() - def has_probe(self) -> bool: return "contact_vector" in self.get_property_keys() @@ -234,21 +226,6 @@ def _set_probes(self, probe_or_probegroup, group_mode="by_probe", in_place=False return sub_recording - def set_probes(self, probe_or_probegroup, group_mode="by_probe", in_place=False): - - warning_msg = ( - "`set_probes` is now a private function and the public function will be " - "removed in 0.103.0. Please use `set_probe` or `set_probegroup` instead" - ) - - warn(warning_msg, category=DeprecationWarning, stacklevel=2) - - sub_recording = self._set_probes( - probe_or_probegroup=probe_or_probegroup, group_mode=group_mode, in_place=in_place - ) - - return sub_recording - def get_probe(self): probes = self.get_probes() assert len(probes) == 1, "there are several probe use .get_probes() or get_probegroup()" diff --git a/src/spikeinterface/core/basesnippets.py b/src/spikeinterface/core/basesnippets.py index 872f3fa8e1..e20fe09e11 100644 --- a/src/spikeinterface/core/basesnippets.py +++ b/src/spikeinterface/core/basesnippets.py @@ -79,14 +79,6 @@ def is_aligned(self): def get_num_segments(self): return len(self._snippets_segments) - def has_scaled_snippets(self): - warn( - "`has_scaled_snippets` is deprecated and will be removed in version 0.103.0. Please use `has_scaleable_traces()` instead", - category=DeprecationWarning, - stacklevel=2, - ) - return self.has_scaleable_traces() - def get_frames(self, indices=None, segment_index: Union[int, None] = None): segment_index = self._check_segment_index(segment_index) spts = self._snippets_segments[segment_index] diff --git a/src/spikeinterface/core/sparsity.py b/src/spikeinterface/core/sparsity.py index 0e30760262..6fa300c126 100644 --- a/src/spikeinterface/core/sparsity.py +++ b/src/spikeinterface/core/sparsity.py @@ -30,7 +30,6 @@ * "by_property" : sparsity is given by a property of the recording and sorting (e.g. "group"). In this case the sparsity for each unit is given by the channels that have the same property value as the unit. Use the "by_property" argument to specify the property name. - * "ptp: : deprecated, use the 'snr' method with the 'peak_to_peak' amplitude mode instead. peak_sign : "neg" | "pos" | "both" Sign of the template to compute best channels. @@ -454,33 +453,6 @@ def from_snr( mask[unit_ind, chan_inds] = True return cls(mask, unit_ids, channel_ids) - @classmethod - def from_ptp(cls, templates_or_sorting_analyzer, threshold, noise_levels=None): - """ - Construct sparsity from a thresholds based on template peak-to-peak values. - Use the "threshold" argument to specify the peak-to-peak threshold. - - Parameters - ---------- - templates_or_sorting_analyzer : Templates | SortingAnalyzer - A Templates or a SortingAnalyzer object. - threshold : float - Threshold for "ptp" method (in units of amplitude). - - Returns - ------- - sparsity : ChannelSparsity - The estimated sparsity. - """ - warnings.warn( - "The 'ptp' method is deprecated and will be removed in version 0.103.0. " - "Please use the 'snr' method with the 'peak_to_peak' amplitude mode instead.", - DeprecationWarning, - ) - return cls.from_snr( - templates_or_sorting_analyzer, threshold, amplitude_mode="peak_to_peak", noise_levels=noise_levels - ) - @classmethod def from_amplitude(cls, templates_or_sorting_analyzer, threshold, amplitude_mode="extremum", peak_sign="neg"): """ diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 65125efbcc..8006eb4d7f 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -53,16 +53,6 @@ def read_file_from_backend( else: raise RuntimeError(f"{file_path} is not a valid HDF5 file!") - elif stream_mode == "ros3": - import h5py - - assert file_path is not None, "file_path must be specified when using stream_mode='ros3'" - - drivers = h5py.registered_drivers() - assertion_msg = "ROS3 support not enbabled, use: install -c conda-forge h5py>=3.2 to enable streaming" - assert "ros3" in drivers, assertion_msg - open_file = h5py.File(name=file_path, mode="r", driver="ros3") - elif stream_mode == "remfile": import remfile import h5py @@ -535,13 +525,6 @@ def __init__( use_pynwb: bool = False, ): - if stream_mode == "ros3": - warnings.warn( - "The 'ros3' stream_mode is deprecated and will be removed in version 0.103.0. " - "Use 'fsspec' stream_mode instead.", - DeprecationWarning, - ) - if file_path is not None and file is not None: raise ValueError("Provide either file_path or file, not both") if file_path is None and file is None: @@ -1062,13 +1045,6 @@ def __init__( use_pynwb: bool = False, ): - if stream_mode == "ros3": - warnings.warn( - "The 'ros3' stream_mode is deprecated and will be removed in version 0.103.0. " - "Use 'fsspec' stream_mode instead.", - DeprecationWarning, - ) - self.stream_mode = stream_mode self.stream_cache_path = stream_cache_path self.electrical_series_path = electrical_series_path diff --git a/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py b/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py index 84ae3c03bf..404a598713 100644 --- a/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py +++ b/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py @@ -73,7 +73,7 @@ def test_recording_s3_nwb_remfile(): assert full_traces.shape == (num_frames, num_chans) assert full_traces.dtype == dtype - if rec.has_scaled(): + if rec.has_scaleable_traces(): trace_scaled = rec.get_traces(segment_index=segment_index, return_scaled=True, end_frame=2) assert trace_scaled.dtype == "float32" @@ -103,7 +103,7 @@ def test_recording_s3_nwb_remfile_file_like(tmp_path): assert full_traces.shape == (num_frames, num_chans) assert full_traces.dtype == dtype - if rec.has_scaled(): + if rec.has_scaleable_traces(): trace_scaled = rec.get_traces(segment_index=segment_index, return_scaled=True, end_frame=2) assert trace_scaled.dtype == "float32" diff --git a/src/spikeinterface/qualitymetrics/pca_metrics.py b/src/spikeinterface/qualitymetrics/pca_metrics.py index f4e36b24c0..9b8618f990 100644 --- a/src/spikeinterface/qualitymetrics/pca_metrics.py +++ b/src/spikeinterface/qualitymetrics/pca_metrics.py @@ -229,28 +229,6 @@ def compute_pc_metrics( return pc_metrics -def calculate_pc_metrics( - sorting_analyzer, metric_names=None, metric_params=None, unit_ids=None, seed=None, n_jobs=1, progress_bar=False -): - warnings.warn( - "The `calculate_pc_metrics` function is deprecated and will be removed in 0.103.0. Please use compute_pc_metrics instead", - category=DeprecationWarning, - stacklevel=2, - ) - - pc_metrics = compute_pc_metrics( - sorting_analyzer, - metric_names=metric_names, - metric_params=metric_params, - unit_ids=unit_ids, - seed=seed, - n_jobs=n_jobs, - progress_bar=progress_bar, - ) - - return pc_metrics - - ################################################################# # Code from spikemetrics diff --git a/src/spikeinterface/qualitymetrics/quality_metric_list.py b/src/spikeinterface/qualitymetrics/quality_metric_list.py index 23b781eb9d..f7411f6376 100644 --- a/src/spikeinterface/qualitymetrics/quality_metric_list.py +++ b/src/spikeinterface/qualitymetrics/quality_metric_list.py @@ -22,7 +22,6 @@ from .pca_metrics import ( compute_pc_metrics, - calculate_pc_metrics, # remove after 0.103.0 mahalanobis_metrics, lda_metrics, nearest_neighbors_metrics, diff --git a/src/spikeinterface/qualitymetrics/tests/test_pca_metrics.py b/src/spikeinterface/qualitymetrics/tests/test_pca_metrics.py index 287439a4f7..1491b9eac1 100644 --- a/src/spikeinterface/qualitymetrics/tests/test_pca_metrics.py +++ b/src/spikeinterface/qualitymetrics/tests/test_pca_metrics.py @@ -4,7 +4,7 @@ from spikeinterface.qualitymetrics import compute_pc_metrics, get_quality_pca_metric_list -def test_calculate_pc_metrics(small_sorting_analyzer): +def test_compute_pc_metrics(small_sorting_analyzer): import pandas as pd sorting_analyzer = small_sorting_analyzer diff --git a/src/spikeinterface/sorters/launcher.py b/src/spikeinterface/sorters/launcher.py index 137ff98cdb..a6b049c182 100644 --- a/src/spikeinterface/sorters/launcher.py +++ b/src/spikeinterface/sorters/launcher.py @@ -239,7 +239,6 @@ def run_sorter_by_property( verbose=False, docker_image=None, singularity_image=None, - working_folder: None = None, **sorter_params, ): """ @@ -301,14 +300,6 @@ def run_sorter_by_property( stacklevel=2, ) - if working_folder is not None: - warnings.warn( - "`working_folder` is deprecated and will be removed in 0.103. Please use folder instead", - category=DeprecationWarning, - stacklevel=2, - ) - folder = working_folder - working_folder = Path(folder).absolute() assert grouping_property in recording.get_property_keys(), ( diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index bd5d9b3529..5c44db2d58 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -95,8 +95,6 @@ If True, the output Sorting is returned as a Sorting delete_container_files : bool, default: True If True, the container temporary files are deleted after the sorting is done - output_folder : None, default: None - Do not use. Deprecated output function to be removed in 0.103. **sorter_params : keyword args Spike sorter specific arguments (they can be retrieved with `get_default_sorter_params(sorter_name_or_class)`) @@ -119,7 +117,6 @@ def run_sorter( singularity_image: Optional[Union[bool, str]] = False, delete_container_files: bool = True, with_output: bool = True, - output_folder: None = None, **sorter_params, ): """ @@ -132,13 +129,6 @@ def run_sorter( >>> sorting = run_sorter("tridesclous", recording) """ - if output_folder is not None and folder is None: - deprecation_msg = ( - "`output_folder` is deprecated and will be removed in version 0.103.0 Please use folder instead" - ) - folder = output_folder - warn(deprecation_msg, category=DeprecationWarning, stacklevel=2) - common_kwargs = dict( sorter_name=sorter_name, recording=recording, @@ -210,7 +200,6 @@ def run_sorter_local( verbose=False, raise_error=True, with_output=True, - output_folder=None, **sorter_params, ): """ @@ -235,20 +224,11 @@ def run_sorter_local( If False, the process continues and the error is logged in the log file with_output : bool, default: True If True, the output Sorting is returned as a Sorting - output_folder : None, default: None - Do not use. Deprecated output function to be removed in 0.103. **sorter_params : keyword args """ if isinstance(recording, list): raise Exception("If you want to run several sorters/recordings use run_sorter_jobs(...)") - if output_folder is not None and folder is None: - deprecation_msg = ( - "`output_folder` is deprecated and will be removed in version 0.103.0 Please use folder instead" - ) - folder = output_folder - warn(deprecation_msg, category=DeprecationWarning, stacklevel=2) - SorterClass = sorter_dict[sorter_name] # only classmethod call not instance (stateless at instance level but state is in folder) @@ -294,7 +274,6 @@ def run_sorter_container( installation_mode="auto", spikeinterface_version=None, spikeinterface_folder_source=None, - output_folder: None = None, **sorter_params, ): """ @@ -309,8 +288,6 @@ def run_sorter_container( The container mode : "docker" or "singularity" container_image : str, default: None The container image name and tag. If None, the default container image is used - output_folder : str, default: None - Path to output folder remove_existing_folder : bool, default: True If True and output_folder exists yet then delete delete_output_folder : bool, default: False @@ -345,13 +322,6 @@ def run_sorter_container( """ assert installation_mode in ("auto", "pypi", "github", "folder", "dev", "no-install") - - if output_folder is not None and folder is None: - deprecation_msg = ( - "`output_folder` is deprecated and will be removed in version 0.103.0 Please use folder instead" - ) - folder = output_folder - warn(deprecation_msg, category=DeprecationWarning, stacklevel=2) spikeinterface_version = spikeinterface_version or si_version if extra_requirements is None: From e4b9d07aaf16461fddfbc04d8823420b9a0b00ba Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:21:03 -0400 Subject: [PATCH 050/157] fix ptp deprecation and tests --- src/spikeinterface/core/sparsity.py | 39 +++++++------------ .../core/tests/test_sparsity.py | 21 ---------- 2 files changed, 13 insertions(+), 47 deletions(-) diff --git a/src/spikeinterface/core/sparsity.py b/src/spikeinterface/core/sparsity.py index 6fa300c126..f760243fb5 100644 --- a/src/spikeinterface/core/sparsity.py +++ b/src/spikeinterface/core/sparsity.py @@ -21,8 +21,7 @@ * "snr" : threshold based on template signal-to-noise ratio. Use the "threshold" argument to specify the SNR threshold (in units of noise levels) and the "amplitude_mode" argument to specify the mode to compute the amplitude of the templates. - * "amplitude" : threshold based on the amplitude values on every channels. Use the "threshold" argument - to specify the ptp threshold (in units of amplitude) and the "amplitude_mode" argument + * "amplitude" : threshold based on the amplitude values on every channels. Use the "amplitude_mode" argument to specify the mode to compute the amplitude of the templates. * "energy" : threshold based on the expected energy that should be present on the channels, given their noise levels. Use the "threshold" argument to specify the energy threshold @@ -38,7 +37,7 @@ radius_um : float Radius in um for "radius" method. threshold : float - Threshold for "snr", "energy" (in units of noise levels) and "ptp" methods (in units of amplitude). + Threshold for "snr" and "energy" (in units of noise levels) (in units of amplitude). For the "snr" method, the template amplitude mode is controlled by the "amplitude_mode" argument. amplitude_mode : "extremum" | "at_index" | "peak_to_peak" Mode to compute the amplitude of the templates for the "snr", "amplitude", and "best_channels" methods. @@ -607,9 +606,7 @@ def create_dense(cls, sorting_analyzer): def compute_sparsity( templates_or_sorting_analyzer: "Templates | SortingAnalyzer", noise_levels: np.ndarray | None = None, - method: ( - "radius" | "best_channels" | "closest_channels" | "snr" | "amplitude" | "energy" | "by_property" | "ptp" - ) = "radius", + method: "radius" | "best_channels" | "closest_channels" | "snr" | "amplitude" | "energy" | "by_property" = "radius", peak_sign: "neg" | "pos" | "both" = "neg", num_channels: int | None = 5, radius_um: float | None = 100.0, @@ -644,7 +641,13 @@ def compute_sparsity( # to keep backward compatibility templates_or_sorting_analyzer = templates_or_sorting_analyzer.sorting_analyzer - if method in ("best_channels", "closest_channels", "radius", "snr", "amplitude", "ptp"): + if method in ( + "best_channels", + "closest_channels", + "radius", + "snr", + "amplitude", + ): assert isinstance( templates_or_sorting_analyzer, (Templates, SortingAnalyzer) ), f"compute_sparsity(method='{method}') need Templates or SortingAnalyzer" @@ -687,14 +690,6 @@ def compute_sparsity( sparsity = ChannelSparsity.from_property( templates_or_sorting_analyzer.sorting, templates_or_sorting_analyzer.recording, by_property ) - elif method == "ptp": - # TODO: remove after deprecation - assert threshold is not None, "For the 'ptp' method, 'threshold' needs to be given" - sparsity = ChannelSparsity.from_ptp( - templates_or_sorting_analyzer, - threshold, - noise_levels=noise_levels, - ) else: raise ValueError(f"compute_sparsity() method={method} does not exists") @@ -710,7 +705,7 @@ def estimate_sparsity( num_spikes_for_sparsity: int = 100, ms_before: float = 1.0, ms_after: float = 2.5, - method: "radius" | "best_channels" | "closest_channels" | "amplitude" | "snr" | "by_property" | "ptp" = "radius", + method: "radius" | "best_channels" | "closest_channels" | "amplitude" | "snr" | "by_property" = "radius", peak_sign: "neg" | "pos" | "both" = "neg", radius_um: float = 100.0, num_channels: int = 5, @@ -759,9 +754,9 @@ def estimate_sparsity( # Can't be done at module because this is a cyclic import, too bad from .template import Templates - assert method in ("radius", "best_channels", "closest_channels", "snr", "amplitude", "by_property", "ptp"), ( + assert method in ("radius", "best_channels", "closest_channels", "snr", "amplitude", "by_property"), ( f"method={method} is not available for `estimate_sparsity()`. " - "Available methods are 'radius', 'best_channels', 'snr', 'amplitude', 'by_property', 'ptp' (deprecated)" + "Available methods are 'radius', 'best_channels', 'snr', 'amplitude', 'by_property'" ) if recording.get_probes() == 1: @@ -838,14 +833,6 @@ def estimate_sparsity( sparsity = ChannelSparsity.from_amplitude( templates, threshold, amplitude_mode=amplitude_mode, peak_sign=peak_sign ) - elif method == "ptp": - # TODO: remove after deprecation - assert threshold is not None, "For the 'ptp' method, 'threshold' needs to be given" - assert noise_levels is not None, ( - "For the 'snr' method, 'noise_levels' needs to be given. You can use the " - "`get_noise_levels()` function to compute them." - ) - sparsity = ChannelSparsity.from_ptp(templates, threshold, noise_levels=noise_levels) else: raise ValueError(f"compute_sparsity() method={method} does not exists") else: diff --git a/src/spikeinterface/core/tests/test_sparsity.py b/src/spikeinterface/core/tests/test_sparsity.py index 6ed311b5d8..c865068e4a 100644 --- a/src/spikeinterface/core/tests/test_sparsity.py +++ b/src/spikeinterface/core/tests/test_sparsity.py @@ -268,24 +268,8 @@ def test_estimate_sparsity(): progress_bar=True, n_jobs=1, ) - # ptp: just run it print(noise_levels) - with pytest.warns(DeprecationWarning): - sparsity = estimate_sparsity( - sorting, - recording, - num_spikes_for_sparsity=50, - ms_before=1.0, - ms_after=2.0, - method="ptp", - threshold=5, - noise_levels=noise_levels, - chunk_duration="1s", - progress_bar=True, - n_jobs=1, - ) - def test_compute_sparsity(): recording, sorting = get_dataset() @@ -310,8 +294,6 @@ def test_compute_sparsity(): sparsity = compute_sparsity(sorting_analyzer, method="amplitude", threshold=5, amplitude_mode="peak_to_peak") sparsity = compute_sparsity(sorting_analyzer, method="energy", threshold=5) sparsity = compute_sparsity(sorting_analyzer, method="by_property", by_property="group") - with pytest.warns(DeprecationWarning): - sparsity = compute_sparsity(sorting_analyzer, method="ptp", threshold=5) # using object Templates templates = sorting_analyzer.get_extension("templates").get_data(outputs="Templates") @@ -322,9 +304,6 @@ def test_compute_sparsity(): sparsity = compute_sparsity(templates, method="amplitude", threshold=5, amplitude_mode="peak_to_peak") sparsity = compute_sparsity(templates, method="closest_channels", num_channels=2) - with pytest.warns(DeprecationWarning): - sparsity = compute_sparsity(templates, method="ptp", noise_levels=noise_levels, threshold=5) - if __name__ == "__main__": # test_ChannelSparsity() From 82360063c3fd79cd1ba1e3ad92cf9385df4826e8 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:31:45 -0400 Subject: [PATCH 051/157] fix channel_slice deprecation --- .../core/baserecordingsnippets.py | 22 ------------------- .../core/tests/test_sortinganalyzer.py | 2 +- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/src/spikeinterface/core/baserecordingsnippets.py b/src/spikeinterface/core/baserecordingsnippets.py index 3be8ef6938..6be1766dbc 100644 --- a/src/spikeinterface/core/baserecordingsnippets.py +++ b/src/spikeinterface/core/baserecordingsnippets.py @@ -61,9 +61,6 @@ def is_filtered(self): # the is_filtered is handle with annotation return self._annotations.get("is_filtered", False) - def _channel_slice(self, channel_ids, renamed_channel_ids=None): - raise NotImplementedError - def set_probe(self, probe, group_mode="by_probe", in_place=False): """ Attach a list of Probe object to a recording. @@ -418,25 +415,6 @@ def planarize(self, axes: str = "xy"): return recording2d - # utils - def channel_slice(self, channel_ids, renamed_channel_ids=None): - """ - Returns a new object with sliced channels. - - Parameters - ---------- - channel_ids : np.array or list - The list of channels to keep - renamed_channel_ids : np.array or list, default: None - A list of renamed channels - - Returns - ------- - BaseRecordingSnippets - The object with sliced channels - """ - return self._channel_slice(channel_ids, renamed_channel_ids=renamed_channel_ids) - def select_channels(self, channel_ids): """ Returns a new object with sliced channels. diff --git a/src/spikeinterface/core/tests/test_sortinganalyzer.py b/src/spikeinterface/core/tests/test_sortinganalyzer.py index c5c4e9db63..7074c054b5 100644 --- a/src/spikeinterface/core/tests/test_sortinganalyzer.py +++ b/src/spikeinterface/core/tests/test_sortinganalyzer.py @@ -250,7 +250,7 @@ def test_SortingAnalyzer_tmp_recording(dataset): assert not sorting_analyzer_saved.has_temporary_recording() assert isinstance(sorting_analyzer_saved.recording, type(recording)) - recording_sliced = recording.channel_slice(recording.channel_ids[:-1]) + recording_sliced = recording.select_channels(recording.channel_ids[:-1]) # wrong channels with pytest.raises(ValueError): From f54ca4d72d2a159e016578f8f4240a8b1a7fdaa5 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:50:50 -0400 Subject: [PATCH 052/157] fix sorter tests --- src/spikeinterface/sorters/tests/test_runsorter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index b995520f26..1f2ec373a9 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -34,7 +34,7 @@ def test_run_sorter_local(generate_recording, create_cache_folder): sorting = run_sorter( "tridesclous2", recording, - output_folder=cache_folder / "sorting_tdc_local", + folder=cache_folder / "sorting_tdc_local", remove_existing_folder=True, delete_output_folder=False, verbose=True, @@ -61,7 +61,7 @@ def test_run_sorter_docker(generate_recording, create_cache_folder): sorting = run_sorter( "tridesclous", recording, - output_folder=output_folder, + folder=output_folder, remove_existing_folder=True, delete_output_folder=False, verbose=True, @@ -96,7 +96,7 @@ def test_run_sorter_singularity(generate_recording, create_cache_folder): sorting = run_sorter( "tridesclous", recording, - output_folder=output_folder, + folder=output_folder, remove_existing_folder=True, delete_output_folder=False, verbose=True, From c370f3104d364df01e7f203e57830c32d1fc0446 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:19:24 -0400 Subject: [PATCH 053/157] doc fix --- examples/tutorials/core/plot_1_recording_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorials/core/plot_1_recording_extractor.py b/examples/tutorials/core/plot_1_recording_extractor.py index e7d773e9e6..39520f2195 100644 --- a/examples/tutorials/core/plot_1_recording_extractor.py +++ b/examples/tutorials/core/plot_1_recording_extractor.py @@ -122,7 +122,7 @@ ############################################################################## # You can also get a recording with a subset of channels (i.e. a channel slice): -recording4 = recording3.channel_slice(channel_ids=["a", "c", "e"]) +recording4 = recording3.select_channels(channel_ids=["a", "c", "e"]) print(recording4) print(recording4.get_channel_ids()) From ba817c0b83a269374df17c33e3bf446c06769452 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Mon, 23 Jun 2025 11:28:13 +0100 Subject: [PATCH 054/157] allow for sort_by_dict --- src/spikeinterface/sorters/runsorter.py | 69 +++++++++++++++++-- .../sorters/tests/test_runsorter.py | 43 ++++++++++++ 2 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index bd5d9b3529..e285f35295 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -100,10 +100,6 @@ **sorter_params : keyword args Spike sorter specific arguments (they can be retrieved with `get_default_sorter_params(sorter_name_or_class)`) - Returns - ------- - BaseSorting | None - The spike sorted data (it `with_output` is True) or None (if `with_output` is False) """ @@ -124,8 +120,11 @@ def run_sorter( ): """ Generic function to run a sorter via function approach. - {} + Returns + ------- + BaseSorting | None + The spike sorted data (it `with_output` is True) or None (if `with_output` is False) Examples -------- @@ -151,6 +150,16 @@ def run_sorter( **sorter_params, ) + if isinstance(recording, dict): + + all_kwargs = common_kwargs + all_kwargs["docker_image"] = docker_image + all_kwargs["singularity_image"] = singularity_image + all_kwargs["delete_container_files"] = delete_container_files + + dict_of_sorters = _run_sorter_by_dict(recording, **all_kwargs) + return dict_of_sorters + if docker_image or singularity_image: common_kwargs.update(dict(delete_container_files=delete_container_files)) if docker_image: @@ -201,6 +210,56 @@ def run_sorter( run_sorter.__doc__ = run_sorter.__doc__.format(_common_param_doc) +def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = None, **run_sorter_params): + """ + Applies `run_sorter` to each recording in a dict of recordings and saves + the results. + {} + Returns + ------- + dict + Dictionary of `BaseSorting`s, with the same keys as the input dict of `BaseRecording`s. + """ + + sorter_name = run_sorter_params["sorter_name"] + remove_existing_folder = run_sorter_params["remove_existing_folder"] + + if folder is None: + folder = Path(sorter_name + "_output") + + folder.mkdir(exist_ok=remove_existing_folder) + + # If we know how the recording was split, save this in the info file + first_recording = next(iter(dict_of_recordings.values())) + split_by_property = first_recording.get_annotation("split_by_property") + if split_by_property is None: + split_by_property = "Unknown" + + info_file = folder / "spikeinterface_info.json" + info = dict( + version=spikeinterface.__version__, + dev_mode=spikeinterface.DEV_MODE, + object="dict of BaseSorting", + dict_keys=list(dict_of_recordings.keys()), + split_by_property=split_by_property, + ) + with open(info_file, mode="w") as f: + json.dump(check_json(info), f, indent=4) + + sorter_dict = {} + for group_key, recording in dict_of_recordings.items(): + + if "recording" in run_sorter_params: + run_sorter_params.pop("recording") + + sorter_dict[group_key] = run_sorter(recording=recording, folder=folder / f"{group_key}", **run_sorter_params) + + return sorter_dict + + +_run_sorter_by_dict.__doc__ = _run_sorter_by_dict.__doc__.format(_common_param_doc) + + def run_sorter_local( sorter_name, recording, diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index b995520f26..c418bbf40d 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -4,6 +4,7 @@ from pathlib import Path import shutil from packaging.version import parse +import json from spikeinterface import generate_ground_truth_recording from spikeinterface.sorters import run_sorter @@ -45,6 +46,48 @@ def test_run_sorter_local(generate_recording, create_cache_folder): print(sorting) +def test_run_sorter_dict(generate_recording, create_cache_folder): + recording = generate_recording + cache_folder = create_cache_folder + + recording.set_property(key="split_property", values=[4, 4, "g", "g", 4, 4, 4, "g"]) + dict_of_recordings = recording.split_by("split_property") + + sorter_params = {"detection": {"detect_threshold": 4.9}} + + output_folder = cache_folder / "sorting_tdc_local_dict" + + dict_of_sortings = run_sorter( + "tridesclous2", + dict_of_recordings, + output_folder=output_folder, + remove_existing_folder=True, + delete_output_folder=False, + verbose=True, + raise_error=True, + **sorter_params, + ) + + assert set(list(dict_of_sortings.keys())) == set(["g", "4"]) + assert (output_folder / "g").is_dir() + assert (output_folder / "4").is_dir() + + assert dict_of_sortings["g"]._recording.get_num_channels() == 3 + assert dict_of_sortings["4"]._recording.get_num_channels() == 5 + + info_filepath = output_folder / "spikeinterface_info.json" + assert info_filepath.is_file() + + with open(info_filepath) as f: + spikeinterface_info = json.load(f) + + si_info_keys = spikeinterface_info.keys() + for key in ["version", "dev_mode", "object", "dict_keys", "split_by_property"]: + assert key in si_info_keys + + assert spikeinterface_info["split_by_property"] == "split_property" + + @pytest.mark.skipif(ON_GITHUB, reason="Docker tests don't run on github: test locally") def test_run_sorter_docker(generate_recording, create_cache_folder): recording = generate_recording From 39461e3c4635f607ecf855e19742f534074aa0b6 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Tue, 24 Jun 2025 09:16:55 +0100 Subject: [PATCH 055/157] add a Path, and use dict of Sorting --- src/spikeinterface/sorters/runsorter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index e285f35295..ee5afef615 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -226,7 +226,8 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No if folder is None: folder = Path(sorter_name + "_output") - + + folder = Path(folder) folder.mkdir(exist_ok=remove_existing_folder) # If we know how the recording was split, save this in the info file @@ -239,7 +240,7 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No info = dict( version=spikeinterface.__version__, dev_mode=spikeinterface.DEV_MODE, - object="dict of BaseSorting", + object="dict of Sorting", dict_keys=list(dict_of_recordings.keys()), split_by_property=split_by_property, ) From c07905fc8d016a064ea98d1b7a912c9723de7cbe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 08:17:56 +0000 Subject: [PATCH 056/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/sorters/runsorter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index ee5afef615..295f23496b 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -226,7 +226,7 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No if folder is None: folder = Path(sorter_name + "_output") - + folder = Path(folder) folder.mkdir(exist_ok=remove_existing_folder) From 5ea7c2c065b5e5119b7ce01094f7b7a14300f923 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 24 Jun 2025 20:29:40 -0600 Subject: [PATCH 057/157] add get_unit_spike_train_in_seconds --- src/spikeinterface/core/basesorting.py | 95 ++++++++++++++-- .../extractors/nwbextractors.py | 61 +++++++++-- .../extractors/tests/test_nwbextractors.py | 101 ++++++++++++++++++ 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index c092de3387..664466e2e5 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -162,17 +162,100 @@ def get_unit_spike_train( ).astype("int64") if return_times: - if self.has_recording(): - times = self.get_times(segment_index=segment_index) - return times[spike_frames] - else: + # Use the new get_unit_spike_train_in_seconds method for better precision + start_time = None + end_time = None + + if start_frame is not None: + segment = self._sorting_segments[segment_index] + t_start = segment._t_start if segment._t_start is not None else 0 + start_time = start_frame / self.get_sampling_frequency() + t_start + + if end_frame is not None: segment = self._sorting_segments[segment_index] t_start = segment._t_start if segment._t_start is not None else 0 - spike_times = spike_frames / self.get_sampling_frequency() - return t_start + spike_times + end_time = end_frame / self.get_sampling_frequency() + t_start + + return self.get_unit_spike_train_in_seconds( + unit_id=unit_id, + segment_index=segment_index, + start_time=start_time, + end_time=end_time, + ) else: return spike_frames + def get_unit_spike_train_in_seconds( + self, + unit_id: str | int, + segment_index: Union[int, None] = None, + start_time: Union[float, None] = None, + end_time: Union[float, None] = None, + ): + """ + Get spike train for a unit in seconds. + + This method avoids double conversion for extractors that already store + spike times in seconds (e.g., NWB format). If the segment implements + get_unit_spike_train_in_seconds(), it uses that directly. Otherwise, + it falls back to the standard frame-to-time conversion. + + Parameters + ---------- + unit_id : str or int + The unit id to retrieve spike train for + segment_index : int or None, default: None + The segment index to retrieve spike train from. + For multi-segment objects, it is required + start_time : float or None, default: None + The start time in seconds for spike train extraction + end_time : float or None, default: None + The end time in seconds for spike train extraction + + Returns + ------- + spike_times : np.ndarray + Spike times in seconds + """ + segment_index = self._check_segment_index(segment_index) + segment = self._sorting_segments[segment_index] + + # Try to use segment-specific method if available + if hasattr(segment, "get_unit_spike_train_in_seconds"): + return segment.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) + + # Fall back to frame-based conversion + start_frame = None + end_frame = None + + if start_time is not None: + t_start = segment._t_start if segment._t_start is not None else 0 + start_frame = int((start_time - t_start) * self.get_sampling_frequency()) + + if end_time is not None: + t_start = segment._t_start if segment._t_start is not None else 0 + end_frame = int((end_time - t_start) * self.get_sampling_frequency()) + + # Get spike train in frames and convert to times using traditional method + spike_frames = self.get_unit_spike_train( + unit_id=unit_id, + segment_index=segment_index, + start_frame=start_frame, + end_frame=end_frame, + return_times=False, + use_cache=True, + ) + + # Convert frames to times + if self.has_recording(): + times = self.get_times(segment_index=segment_index) + return times[spike_frames] + else: + segment = self._sorting_segments[segment_index] + t_start = segment._t_start if segment._t_start is not None else 0 + spike_times = spike_frames / self.get_sampling_frequency() + return t_start + spike_times + def register_recording(self, recording, check_spike_frames=True): """ Register a recording to the sorting. If the sorting and recording both contain diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 65125efbcc..fdc85a589a 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -1360,6 +1360,49 @@ def get_unit_spike_train( start_frame: Optional[int] = None, end_frame: Optional[int] = None, ) -> np.ndarray: + # Convert frame boundaries to time boundaries + start_time = None + end_time = None + + if start_frame is not None: + start_time = start_frame / self._sampling_frequency + self._t_start + + if end_frame is not None: + end_time = end_frame / self._sampling_frequency + self._t_start + + # Get spike times in seconds + spike_times = self.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) + + # Convert to frames + frames = np.round((spike_times - self._t_start) * self._sampling_frequency) + return frames.astype("int64", copy=False) + + def get_unit_spike_train_in_seconds( + self, + unit_id, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + ) -> np.ndarray: + """Get the spike train for a unit in seconds. + + This method returns spike times directly in seconds without conversion + to frames, avoiding double conversion for NWB files that already store + spike times as timestamps. + + Parameters + ---------- + unit_id + The unit id to retrieve spike train for + start_time : float, default: None + The start time in seconds for spike train extraction + end_time : float, default: None + The end time in seconds for spike train extraction + + Returns + ------- + spike_times : np.ndarray + Spike times in seconds + """ # Extract the spike times for the unit unit_index = self.parent_extractor.id_to_index(unit_id) if unit_index == 0: @@ -1369,19 +1412,17 @@ def get_unit_spike_train( end_index = self.spike_times_index_data[unit_index] spike_times = self.spike_times_data[start_index:end_index] - # Transform spike times to frames and subset - frames = np.round((spike_times - self._t_start) * self._sampling_frequency) + # Filter by time range if specified + start_idx = 0 + if start_time is not None: + start_idx = np.searchsorted(spike_times, start_time, side="left") - start_index = 0 - if start_frame is not None: - start_index = np.searchsorted(frames, start_frame, side="left") - - if end_frame is not None: - end_index = np.searchsorted(frames, end_frame, side="left") + if end_time is not None: + end_idx = np.searchsorted(spike_times, end_time, side="left") else: - end_index = frames.size + end_idx = spike_times.size - return frames[start_index:end_index].astype("int64", copy=False) + return spike_times[start_idx:end_idx].astype("float64", copy=False) def _find_timeseries_from_backend(group, path="", result=None, backend="hdf5"): diff --git a/src/spikeinterface/extractors/tests/test_nwbextractors.py b/src/spikeinterface/extractors/tests/test_nwbextractors.py index 15d3e8fee9..5d089073c9 100644 --- a/src/spikeinterface/extractors/tests/test_nwbextractors.py +++ b/src/spikeinterface/extractors/tests/test_nwbextractors.py @@ -542,6 +542,107 @@ def test_sorting_extraction_start_time_from_series(tmp_path, use_pynwb): np.testing.assert_allclose(extracted_spike_times1, expected_spike_times1) +@pytest.mark.parametrize("use_pynwb", [True, False]) +def test_get_unit_spike_train_in_seconds(tmp_path, use_pynwb): + """Test that get_unit_spike_train_in_seconds returns accurate timestamps without double conversion.""" + from pynwb import NWBHDF5IO + from pynwb.testing.mock.file import mock_NWBFile + + nwbfile = mock_NWBFile() + + # Add units with known spike times + t_start = 5.0 + sampling_frequency = 1000.0 + spike_times_unit_a = np.array([5.1, 5.2, 5.3, 6.0, 6.5]) # Absolute times + spike_times_unit_b = np.array([5.05, 5.15, 5.25, 5.35, 6.1]) # Absolute times + + nwbfile.add_unit(spike_times=spike_times_unit_a) + nwbfile.add_unit(spike_times=spike_times_unit_b) + + file_path = tmp_path / "test.nwb" + with NWBHDF5IO(path=file_path, mode="w") as io: + io.write(nwbfile) + + sorting_extractor = NwbSortingExtractor( + file_path=file_path, + sampling_frequency=sampling_frequency, + t_start=t_start, + use_pynwb=use_pynwb, + ) + + # Test full spike trains + spike_times_a_direct = sorting_extractor.get_unit_spike_train_in_seconds(unit_id=0) + spike_times_a_legacy = sorting_extractor.get_unit_spike_train(unit_id=0, return_times=True) + + spike_times_b_direct = sorting_extractor.get_unit_spike_train_in_seconds(unit_id=1) + spike_times_b_legacy = sorting_extractor.get_unit_spike_train(unit_id=1, return_times=True) + + # Both methods should return exact timestamps since return_times now uses get_unit_spike_train_in_seconds + np.testing.assert_array_equal(spike_times_a_direct, spike_times_unit_a) + np.testing.assert_array_equal(spike_times_b_direct, spike_times_unit_b) + np.testing.assert_array_equal(spike_times_a_legacy, spike_times_unit_a) + np.testing.assert_array_equal(spike_times_b_legacy, spike_times_unit_b) + + # Test time filtering + start_time = 5.2 + end_time = 6.1 + + # Direct method with time filtering + spike_times_a_filtered = sorting_extractor.get_unit_spike_train_in_seconds( + unit_id=0, start_time=start_time, end_time=end_time + ) + spike_times_b_filtered = sorting_extractor.get_unit_spike_train_in_seconds( + unit_id=1, start_time=start_time, end_time=end_time + ) + + # Expected filtered results + expected_a = spike_times_unit_a[(spike_times_unit_a >= start_time) & (spike_times_unit_a < end_time)] + expected_b = spike_times_unit_b[(spike_times_unit_b >= start_time) & (spike_times_unit_b < end_time)] + + np.testing.assert_array_equal(spike_times_a_filtered, expected_a) + np.testing.assert_array_equal(spike_times_b_filtered, expected_b) + + # Test edge cases + # Start time filtering only + spike_times_from_start = sorting_extractor.get_unit_spike_train_in_seconds(unit_id=0, start_time=5.25) + expected_from_start = spike_times_unit_a[spike_times_unit_a >= 5.25] + np.testing.assert_array_equal(spike_times_from_start, expected_from_start) + + # End time filtering only + spike_times_to_end = sorting_extractor.get_unit_spike_train_in_seconds(unit_id=0, end_time=6.0) + expected_to_end = spike_times_unit_a[spike_times_unit_a < 6.0] + np.testing.assert_array_equal(spike_times_to_end, expected_to_end) + + # Test that direct method avoids frame conversion rounding errors + # by comparing exact values that would be lost in frame conversion + precise_times = np.array([5.1001, 5.1002, 5.1003]) + nwbfile_precise = mock_NWBFile() + nwbfile_precise.add_unit(spike_times=precise_times) + + file_path_precise = tmp_path / "test_precise.nwb" + with NWBHDF5IO(path=file_path_precise, mode="w") as io: + io.write(nwbfile_precise) + + sorting_precise = NwbSortingExtractor( + file_path=file_path_precise, + sampling_frequency=sampling_frequency, + t_start=t_start, + use_pynwb=use_pynwb, + ) + + # Direct method should preserve exact precision + direct_precise = sorting_precise.get_unit_spike_train_in_seconds(unit_id=0) + np.testing.assert_array_equal(direct_precise, precise_times) + + # Both methods should now preserve exact precision since return_times uses get_unit_spike_train_in_seconds + legacy_precise = sorting_precise.get_unit_spike_train(unit_id=0, return_times=True) + # Both methods should be exactly equal since return_times now avoids double conversion + np.testing.assert_array_equal(direct_precise, precise_times) + np.testing.assert_array_equal(legacy_precise, precise_times) + # Verify both methods return identical results + np.testing.assert_array_equal(direct_precise, legacy_precise) + + @pytest.mark.parametrize("use_pynwb", [True, False]) def test_multiple_unit_tables(tmp_path, use_pynwb): from pynwb.misc import Units From c5000d4dc0512b9db1b909f2f747b946bd7164b1 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Wed, 25 Jun 2025 10:04:18 +0100 Subject: [PATCH 058/157] add dict_key_types --- src/spikeinterface/sorters/runsorter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index 295f23496b..e4060dcd81 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -236,12 +236,16 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No if split_by_property is None: split_by_property = "Unknown" + dict_keys = dict_of_recordings.keys() + dict_key_types = [type(key).__name__ for key in dict_keys] + info_file = folder / "spikeinterface_info.json" info = dict( version=spikeinterface.__version__, dev_mode=spikeinterface.DEV_MODE, object="dict of Sorting", dict_keys=list(dict_of_recordings.keys()), + dict_key_types=dict_key_types, split_by_property=split_by_property, ) with open(info_file, mode="w") as f: From 6302fff1adde2470f77b02abc1ed5ebab0df8030 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Wed, 25 Jun 2025 10:42:21 +0100 Subject: [PATCH 059/157] add docs and docstrings --- doc/how_to/process_by_channel_group.rst | 48 +++++++++++++++---------- doc/modules/sorters.rst | 27 +++++++------- src/spikeinterface/sorters/runsorter.py | 16 +++++---- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/doc/how_to/process_by_channel_group.rst b/doc/how_to/process_by_channel_group.rst index 334f83b247..4bbd2b08cc 100644 --- a/doc/how_to/process_by_channel_group.rst +++ b/doc/how_to/process_by_channel_group.rst @@ -99,7 +99,8 @@ to any preprocessing function. referenced_recording = spre.common_reference(filtered_recording) good_channels_recording = spre.detect_and_remove_bad_channels(filtered_recording) -We can then aggregate the recordings back together using the ``aggregate_channels`` function +We can then aggregate the recordings back together using the ``aggregate_channels`` function. +Note that we do not need to do this to sort the data (see :ref:`sorting a recording by channel group`) .. code-block:: python @@ -141,16 +142,38 @@ Sorting a Recording by Channel Group We can also sort a recording for each channel group separately. It is not necessary to preprocess a recording by channel group in order to sort by channel group. -There are two ways to sort a recording by channel group. First, we can split the preprocessed -recording (or, if it was already split during preprocessing as above, skip the :py:func:`~aggregate_channels` step -directly use the :py:func:`~split_recording_dict`). +There are two ways to sort a recording by channel group. First, we can simply pass the output from +our preprocessing-by-group method above. Second, for more control, we can loop over the recordings +ourselves. -**Option 1: Manual splitting** +**Option 1 : Automatic splitting** -In this example, similar to above we loop over all preprocessed recordings that +Simply pass the split recording to the `run_sorter` function, as if it was a non-split recording. +This will return a dict of sortings, with the keys corresponding to the groups. + +.. code-block:: python + + split_recording = raw_recording.split_by("group") + + # do preprocessing if needed + pp_recording = spre.bandpass_filter(split_recording) + + dict_of_sortings = run_sorter( + sorter_name='kilosort2', + recording=pp_recording, + working_folder='working_path' + ) + + +**Option 2: Manual splitting** + +In this example, we loop over all preprocessed recordings that are grouped by channel, and apply the sorting separately. We store the sorting objects in a dictionary for later use. +You might do this if you want extra control e.g. to apply bespoke steps +to different groups. + .. code-block:: python split_preprocessed_recording = preprocessed_recording.split_by("group") @@ -163,16 +186,3 @@ sorting objects in a dictionary for later use. output_folder=f"folder_KS2_group{group}" ) sortings[group] = sorting - -**Option 2 : Automatic splitting** - -Alternatively, SpikeInterface provides a convenience function to sort the recording by property: - -.. code-block:: python - - aggregate_sorting = run_sorter_by_property( - sorter_name='kilosort2', - recording=preprocessed_recording, - grouping_property='group', - working_folder='working_path' - ) diff --git a/doc/modules/sorters.rst b/doc/modules/sorters.rst index d8a4708236..2ec4cc0046 100644 --- a/doc/modules/sorters.rst +++ b/doc/modules/sorters.rst @@ -339,8 +339,8 @@ Running spike sorting by group is indeed a very common need. A :py:class:`~spikeinterface.core.BaseRecording` object has the ability to split itself into a dictionary of sub-recordings given a certain property (see :py:meth:`~spikeinterface.core.BaseRecording.split_by`). So it is easy to loop over this dictionary and sequentially run spike sorting on these sub-recordings. -SpikeInterface also provides a high-level function to automate the process of splitting the -recording and then aggregating the results with the :py:func:`~spikeinterface.sorters.run_sorter_by_property` function. +The :py:func:`~spikeinterface.sorters.run_sorter` method can also accept the dictionary which is returned +by :py:meth:`~spikeinterface.core.BaseRecording.split_by` and will return a dictionary of sortings. In this example, we create a 16-channel recording with 4 tetrodes: @@ -368,7 +368,19 @@ In this example, we create a 16-channel recording with 4 tetrodes: # >>> [0 0 0 0 1 1 1 1 2 2 2 2 3 3 3 3] -**Option 1: Manual splitting** +**Option 1 : Automatic splitting** + +.. code-block:: python + + # here the result is a dict of sortings + dict_of_sortings = run_sorter( + sorter_name='kilosort2', + recording=recording_4_tetrodes, + working_folder='working_path' + ) + + +**Option 2: Manual splitting** .. code-block:: python @@ -383,15 +395,6 @@ In this example, we create a 16-channel recording with 4 tetrodes: sorting = run_sorter(sorter_name='kilosort2', recording=recording, output_folder=f"folder_KS2_group{group}") sortings[group] = sorting -**Option 2 : Automatic splitting** - -.. code-block:: python - - # here the result is one sorting that aggregates all sub sorting objects - aggregate_sorting = run_sorter_by_property(sorter_name='kilosort2', recording=recording_4_tetrodes, - grouping_property='group', - working_folder='working_path') - Handling multi-segment recordings --------------------------------- diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index e4060dcd81..8f072153c0 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -70,7 +70,7 @@ ---------- sorter_name : str The sorter name - recording : RecordingExtractor + recording : RecordingExtractor | dict of RecordingExtractor The recording extractor to be spike sorted folder : str or Path Path to output folder @@ -105,7 +105,7 @@ def run_sorter( sorter_name: str, - recording: BaseRecording, + recording: BaseRecording | dict, folder: Optional[str] = None, remove_existing_folder: bool = False, delete_output_folder: bool = False, @@ -123,7 +123,7 @@ def run_sorter( {} Returns ------- - BaseSorting | None + BaseSorting | dict of BaseSorting | None The spike sorted data (it `with_output` is True) or None (if `with_output` is False) Examples @@ -153,9 +153,13 @@ def run_sorter( if isinstance(recording, dict): all_kwargs = common_kwargs - all_kwargs["docker_image"] = docker_image - all_kwargs["singularity_image"] = singularity_image - all_kwargs["delete_container_files"] = delete_container_files + all_kwargs.update( + dict( + docker_image=docker_image, + singularity_image=singularity_image, + delete_container_files=delete_container_files, + ) + ) dict_of_sorters = _run_sorter_by_dict(recording, **all_kwargs) return dict_of_sorters From 9ca1ef39ffba24c8ef471d7818473015ae9e4b0c Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Mon, 30 Jun 2025 11:32:44 +0100 Subject: [PATCH 060/157] change format and add load for Groups, and load tests --- src/spikeinterface/core/loading.py | 25 ++++++++++++++++++- src/spikeinterface/sorters/runsorter.py | 7 +----- .../sorters/tests/test_runsorter.py | 15 ++++++++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/spikeinterface/core/loading.py b/src/spikeinterface/core/loading.py index d5845f033e..33a2b935fc 100644 --- a/src/spikeinterface/core/loading.py +++ b/src/spikeinterface/core/loading.py @@ -196,6 +196,12 @@ def _guess_object_from_local_folder(folder): with open(folder / "spikeinterface_info.json", "r") as f: spikeinterface_info = json.load(f) return _guess_object_from_dict(spikeinterface_info) + elif ( + (folder / "sorter_output").is_dir() + and (folder / "spikeinterface_params.json").is_file() + and (folder / "spikeinterface_log.json").is_file() + ): + return "SorterOutput" elif (folder / "waveforms").is_dir(): # before the SortingAnlazer, it was WaveformExtractor (v<0.101) return "WaveformExtractor" @@ -212,13 +218,20 @@ def _guess_object_from_local_folder(folder): return "Recording|Sorting" -def _load_object_from_folder(folder, object_type, **kwargs): +def _load_object_from_folder(folder, object_type: str, **kwargs): + if object_type == "SortingAnalyzer": from .sortinganalyzer import load_sorting_analyzer analyzer = load_sorting_analyzer(folder, **kwargs) return analyzer + elif object_type == "SorterOutput": + from spikeinterface.sorters import read_sorter_folder + + sorting = read_sorter_folder(folder) + return sorting + elif object_type == "Motion": from spikeinterface.core.motion import Motion @@ -244,6 +257,16 @@ def _load_object_from_folder(folder, object_type, **kwargs): si_file = f return BaseExtractor.load(si_file, base_folder=folder) + elif object_type.startswith("Group"): + + sub_object_type = object_type.split("[")[1].split("]")[0] + with open(folder / "spikeinterface_info.json", "r") as f: + spikeinterface_info = json.load(f) + group_keys = spikeinterface_info.get("dict_keys") + + group_of_objects = {key: _load_object_from_folder(folder / str(key), sub_object_type) for key in group_keys} + return group_of_objects + def _guess_object_from_zarr(zarr_folder): # here it can be a zarr folder for Recording|Sorting|SortingAnalyzer|Template diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index 8f072153c0..fad18fcff6 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -240,17 +240,12 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No if split_by_property is None: split_by_property = "Unknown" - dict_keys = dict_of_recordings.keys() - dict_key_types = [type(key).__name__ for key in dict_keys] - info_file = folder / "spikeinterface_info.json" info = dict( version=spikeinterface.__version__, dev_mode=spikeinterface.DEV_MODE, - object="dict of Sorting", + object="Group[SorterOutput]", dict_keys=list(dict_of_recordings.keys()), - dict_key_types=dict_key_types, - split_by_property=split_by_property, ) with open(info_file, mode="w") as f: json.dump(check_json(info), f, indent=4) diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index c418bbf40d..c2a0c7c891 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -5,8 +5,9 @@ import shutil from packaging.version import parse import json +import numpy as np -from spikeinterface import generate_ground_truth_recording +from spikeinterface import generate_ground_truth_recording, load from spikeinterface.sorters import run_sorter ON_GITHUB = bool(os.getenv("GITHUB_ACTIONS")) @@ -50,6 +51,8 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): recording = generate_recording cache_folder = create_cache_folder + recording = recording.time_slice(start_time=0, end_time=3) + recording.set_property(key="split_property", values=[4, 4, "g", "g", 4, 4, 4, "g"]) dict_of_recordings = recording.split_by("split_property") @@ -58,7 +61,7 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): output_folder = cache_folder / "sorting_tdc_local_dict" dict_of_sortings = run_sorter( - "tridesclous2", + "simple", dict_of_recordings, output_folder=output_folder, remove_existing_folder=True, @@ -82,10 +85,14 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): spikeinterface_info = json.load(f) si_info_keys = spikeinterface_info.keys() - for key in ["version", "dev_mode", "object", "dict_keys", "split_by_property"]: + for key in ["version", "dev_mode", "object"]: assert key in si_info_keys - assert spikeinterface_info["split_by_property"] == "split_property" + loaded_sortings = load(output_folder) + assert loaded_sortings.keys() == dict_of_sortings.keys() + for key, sorting in loaded_sortings.items(): + assert np.all(sorting.unit_ids == dict_of_sortings[key].unit_ids) + assert np.all(sorting.to_spike_vector() == dict_of_sortings[key].to_spike_vector()) @pytest.mark.skipif(ON_GITHUB, reason="Docker tests don't run on github: test locally") From 3b0bf56566aac3e5a4c99baf11dd8bfb9b3eba0d Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Mon, 30 Jun 2025 11:38:21 +0100 Subject: [PATCH 061/157] SorterOutput to SorterFolder --- src/spikeinterface/core/loading.py | 4 ++-- src/spikeinterface/sorters/runsorter.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spikeinterface/core/loading.py b/src/spikeinterface/core/loading.py index 33a2b935fc..97f104d08f 100644 --- a/src/spikeinterface/core/loading.py +++ b/src/spikeinterface/core/loading.py @@ -201,7 +201,7 @@ def _guess_object_from_local_folder(folder): and (folder / "spikeinterface_params.json").is_file() and (folder / "spikeinterface_log.json").is_file() ): - return "SorterOutput" + return "SorterFolder" elif (folder / "waveforms").is_dir(): # before the SortingAnlazer, it was WaveformExtractor (v<0.101) return "WaveformExtractor" @@ -226,7 +226,7 @@ def _load_object_from_folder(folder, object_type: str, **kwargs): analyzer = load_sorting_analyzer(folder, **kwargs) return analyzer - elif object_type == "SorterOutput": + elif object_type == "SorterFolder": from spikeinterface.sorters import read_sorter_folder sorting = read_sorter_folder(folder) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index fad18fcff6..7a97ecec6f 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -244,7 +244,7 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No info = dict( version=spikeinterface.__version__, dev_mode=spikeinterface.DEV_MODE, - object="Group[SorterOutput]", + object="Group[SorterFolder]", dict_keys=list(dict_of_recordings.keys()), ) with open(info_file, mode="w") as f: From 7aca913890fac729dc85994d73cc38f43edd1fcd Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Mon, 30 Jun 2025 12:12:40 +0100 Subject: [PATCH 062/157] add split_by_property to basesorter load --- src/spikeinterface/sorters/basesorter.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/spikeinterface/sorters/basesorter.py b/src/spikeinterface/sorters/basesorter.py index 83f700af8a..b2f694c3fc 100644 --- a/src/spikeinterface/sorters/basesorter.py +++ b/src/spikeinterface/sorters/basesorter.py @@ -344,6 +344,17 @@ def get_result_from_folder(cls, output_folder, register_recording=True, sorting_ if recording is not None: sorting.register_recording(recording) + if recording.get_annotation("split_by_property") is not None: + split_by_property = recording.get_annotation("split_by_property") + property_values = set(recording.get_property(split_by_property)) + if len(property_values) > 1: + warnings.warn( + f"Registered Recording has non-unique {split_by_property} keys. They are {property_values}." + ) + elif len(property_values) == 1: + property_value = next(iter(property_values)) + sorting.set_property("split_by_property", values=[property_value] * sorting.get_num_units()) + if sorting_info: # set sorting info to Sorting object if (output_folder / "spikeinterface_recording.json").exists(): From 83c17772883444860d77e6ed83bda14345c225be Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 30 Jun 2025 21:31:28 -0600 Subject: [PATCH 063/157] add axon recording --- .../extractors/neoextractors/__init__.py | 2 + .../extractors/neoextractors/axon.py | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/spikeinterface/extractors/neoextractors/axon.py diff --git a/src/spikeinterface/extractors/neoextractors/__init__.py b/src/spikeinterface/extractors/neoextractors/__init__.py index d781d8c5ae..a90e0954f3 100644 --- a/src/spikeinterface/extractors/neoextractors/__init__.py +++ b/src/spikeinterface/extractors/neoextractors/__init__.py @@ -1,4 +1,5 @@ from .alphaomega import AlphaOmegaRecordingExtractor, AlphaOmegaEventExtractor, read_alphaomega, read_alphaomega_event +from .axon import AxonRecordingExtractor, read_axon from .axona import AxonaRecordingExtractor, read_axona from .biocam import BiocamRecordingExtractor, read_biocam from .blackrock import BlackrockRecordingExtractor, BlackrockSortingExtractor, read_blackrock, read_blackrock_sorting @@ -44,6 +45,7 @@ neo_recording_extractors_dict = { AlphaOmegaRecordingExtractor: dict(wrapper_string="read_alphaomega", wrapper_class=read_alphaomega), + AxonRecordingExtractor: dict(wrapper_string="read_axon", wrapper_class=read_axon), AxonaRecordingExtractor: dict(wrapper_string="read_axona", wrapper_class=read_axona), BiocamRecordingExtractor: dict(wrapper_string="read_biocam", wrapper_class=read_biocam), BlackrockRecordingExtractor: dict(wrapper_string="read_blackrock", wrapper_class=read_blackrock), diff --git a/src/spikeinterface/extractors/neoextractors/axon.py b/src/spikeinterface/extractors/neoextractors/axon.py new file mode 100644 index 0000000000..b2c47b697f --- /dev/null +++ b/src/spikeinterface/extractors/neoextractors/axon.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from pathlib import Path + +from spikeinterface.core.core_tools import define_function_from_class + +from .neobaseextractor import NeoBaseRecordingExtractor + + +class AxonRecordingExtractor(NeoBaseRecordingExtractor): + """ + Class for reading Axon Binary Format (ABF) files. + + Based on :py:class:`neo.rawio.AxonRawIO` + + Supports both ABF1 (pClamp ≤9) and ABF2 (pClamp ≥10) formats. + Can read data from pCLAMP and AxoScope software. + + Parameters + ---------- + file_path : str or Path + The ABF file path to load the recordings from. + stream_id : str or None, default: None + If there are several streams, specify the stream id you want to load. + stream_name : str or None, default: None + If there are several streams, specify the stream name you want to load. + all_annotations : bool, default: False + Load exhaustively all annotations from neo. + use_names_as_ids : bool, default: False + Determines the format of the channel IDs used by the extractor. + + Examples + -------- + >>> from spikeinterface.extractors import read_axon + >>> recording = read_axon(file_path='path/to/file.abf') + """ + + NeoRawIOClass = "AxonRawIO" + + def __init__( + self, + file_path, + stream_id=None, + stream_name=None, + all_annotations: bool = False, + use_names_as_ids: bool = False, + ): + neo_kwargs = self.map_to_neo_kwargs(file_path) + NeoBaseRecordingExtractor.__init__( + self, + stream_id=stream_id, + stream_name=stream_name, + all_annotations=all_annotations, + use_names_as_ids=use_names_as_ids, + **neo_kwargs, + ) + + self._kwargs.update(dict(file_path=str(Path(file_path).absolute()))) + + @classmethod + def map_to_neo_kwargs(cls, file_path): + neo_kwargs = {"filename": str(file_path)} + return neo_kwargs + + +read_axon = define_function_from_class(source_class=AxonRecordingExtractor, name="read_axon") From 54ce47504c9d2a9c41bb7f2ff02032cf766b5afa Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Tue, 1 Jul 2025 08:47:18 +0100 Subject: [PATCH 064/157] docs link fix --- doc/how_to/process_by_channel_group.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/how_to/process_by_channel_group.rst b/doc/how_to/process_by_channel_group.rst index 4bbd2b08cc..b1929eec3b 100644 --- a/doc/how_to/process_by_channel_group.rst +++ b/doc/how_to/process_by_channel_group.rst @@ -100,7 +100,7 @@ to any preprocessing function. good_channels_recording = spre.detect_and_remove_bad_channels(filtered_recording) We can then aggregate the recordings back together using the ``aggregate_channels`` function. -Note that we do not need to do this to sort the data (see :ref:`sorting a recording by channel group`) +Note that we do not need to do this to sort the data (see :ref:`sorting-by-channel-group`). .. code-block:: python @@ -135,6 +135,7 @@ back together under the hood). In general, it is not recommended to apply :py:func:`~aggregate_channels` more than once. This will slow down :py:func:`~get_traces` calls and may result in unpredictable behaviour. +.. _sorting-by-channel-group: Sorting a Recording by Channel Group ------------------------------------ From 38f321b9943d3f47401fee0a97d907a3c29422ca Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 10:57:22 -0600 Subject: [PATCH 065/157] typing --- src/spikeinterface/core/baserecording.py | 3 ++- src/spikeinterface/core/basesorting.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/core/baserecording.py b/src/spikeinterface/core/baserecording.py index b32893981f..53d92df224 100644 --- a/src/spikeinterface/core/baserecording.py +++ b/src/spikeinterface/core/baserecording.py @@ -2,6 +2,7 @@ import warnings from pathlib import Path +from typing import Optional import numpy as np from probeinterface import read_probeinterface, write_probeinterface @@ -467,7 +468,7 @@ def get_end_time(self, segment_index=None) -> float: rs = self._recording_segments[segment_index] return rs.get_end_time() - def has_time_vector(self, segment_index=None): + def has_time_vector(self, segment_index: Optional[int] = None): """Check if the segment of the recording has a time vector. Parameters diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 664466e2e5..777df9d5d1 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -188,9 +188,9 @@ def get_unit_spike_train( def get_unit_spike_train_in_seconds( self, unit_id: str | int, - segment_index: Union[int, None] = None, - start_time: Union[float, None] = None, - end_time: Union[float, None] = None, + segment_index: Optional[int] = None, + start_time: Optional[float] = None, + end_time: Optional[float] = None, ): """ Get spike train for a unit in seconds. @@ -298,7 +298,7 @@ def set_sorting_info(self, recording_dict, params_dict, log_dict): def has_recording(self) -> bool: return self._recording is not None - def has_time_vector(self, segment_index=None) -> bool: + def has_time_vector(self, segment_index: Optional[int] = None) -> bool: """ Check if the segment of the registered recording has a time vector. """ From 12e9a7ce71ce4bd373dcf55434874acfeb896a38 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 11:10:55 -0600 Subject: [PATCH 066/157] fix --- src/spikeinterface/core/basesorting.py | 64 +++++++++++++++++--------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 777df9d5d1..0153f656ef 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -136,7 +136,8 @@ def get_unit_spike_train( end_frame: Union[int, None] = None, return_times: bool = False, use_cache: bool = True, - ): + ) -> np.ndarray: + segment_index = self._check_segment_index(segment_index) if use_cache: if segment_index not in self._cached_spike_trains: @@ -191,14 +192,18 @@ def get_unit_spike_train_in_seconds( segment_index: Optional[int] = None, start_time: Optional[float] = None, end_time: Optional[float] = None, - ): + ) -> np.ndarray: """ Get spike train for a unit in seconds. - This method avoids double conversion for extractors that already store - spike times in seconds (e.g., NWB format). If the segment implements - get_unit_spike_train_in_seconds(), it uses that directly. Otherwise, - it falls back to the standard frame-to-time conversion. + This method uses a three-tier approach to get spike times: + 1. If the sorting has a recording, use the recording's time conversion + 2. If the segment implements get_unit_spike_train_in_seconds(), use that directly These are the native timestamps of the format + 3. Fall back to standard frame-to-time conversion + + This approach avoids double conversion for extractors that already store + spike times in seconds (e.g., NWB format) and ensures consistent timing + when a recording is associated with the sorting. Parameters ---------- @@ -220,21 +225,43 @@ def get_unit_spike_train_in_seconds( segment_index = self._check_segment_index(segment_index) segment = self._sorting_segments[segment_index] - # Try to use segment-specific method if available + # If sorting has a registered recording, get the frames and get the times from the recording + if self.has_time_vector(): + + start_frame = None + end_frame = None + + if start_time is not None: + start_frame = self.time_to_sample_index(start_time, segment_index=segment_index) + + if end_time is not None: + end_frame = self.time_to_sample_index(end_time, segment_index=segment_index) + + spike_frames = self.get_unit_spike_train( + unit_id=unit_id, + segment_index=segment_index, + start_frame=start_frame, + end_frame=end_frame, + return_times=False, + use_cache=True, + ) + + times = self.get_times(segment_index=segment_index) + return times[spike_frames] + + # Use the native spiking times if available if hasattr(segment, "get_unit_spike_train_in_seconds"): return segment.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) - # Fall back to frame-based conversion + # all back to frame-based conversion start_frame = None end_frame = None if start_time is not None: - t_start = segment._t_start if segment._t_start is not None else 0 - start_frame = int((start_time - t_start) * self.get_sampling_frequency()) + start_frame = self.time_to_sample_index(start_time, segment_index=segment_index) if end_time is not None: - t_start = segment._t_start if segment._t_start is not None else 0 - end_frame = int((end_time - t_start) * self.get_sampling_frequency()) + end_frame = self.time_to_sample_index(end_time, segment_index=segment_index) # Get spike train in frames and convert to times using traditional method spike_frames = self.get_unit_spike_train( @@ -247,16 +274,11 @@ def get_unit_spike_train_in_seconds( ) # Convert frames to times - if self.has_recording(): - times = self.get_times(segment_index=segment_index) - return times[spike_frames] - else: - segment = self._sorting_segments[segment_index] - t_start = segment._t_start if segment._t_start is not None else 0 - spike_times = spike_frames / self.get_sampling_frequency() - return t_start + spike_times + t_start = segment._t_start if segment._t_start is not None else 0 + spike_times = spike_frames / self.get_sampling_frequency() + return t_start + spike_times - def register_recording(self, recording, check_spike_frames=True): + def register_recording(self, recording, check_spike_frames: bool = True): """ Register a recording to the sorting. If the sorting and recording both contain time information, the recording’s time information will be used. From cdc42441668757c842de25d41840aceb959b5ab2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 11:13:41 -0600 Subject: [PATCH 067/157] base sorting docstring --- src/spikeinterface/core/basesorting.py | 30 +++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 0153f656ef..86adadf40a 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -131,12 +131,36 @@ def get_total_duration(self) -> float: def get_unit_spike_train( self, unit_id: str | int, - segment_index: Union[int, None] = None, - start_frame: Union[int, None] = None, - end_frame: Union[int, None] = None, + segment_index: Optional[int] = None, + start_frame: Optional[int] = None, + end_frame: Optional[int] = None, return_times: bool = False, use_cache: bool = True, ) -> np.ndarray: + """ + Get spike train for a unit. + + Parameters + ---------- + unit_id : str or int + The unit id to retrieve spike train for + segment_index : int or None, default: None + The segment index to retrieve spike train from. + For multi-segment objects, it is required + start_frame : int or None, default: None + The start frame for spike train extraction + end_frame : int or None, default: None + The end frame for spike train extraction + return_times : bool, default: False + If True, returns spike times in seconds instead of frames + use_cache : bool, default: True + If True, uses cached spike trains when available + + Returns + ------- + spike_train : np.ndarray + Spike frames (or times if return_times=True) + """ segment_index = self._check_segment_index(segment_index) if use_cache: From 7cf800be5ea4c8d1445172deeec0cb94ab89e590 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 11:19:08 -0600 Subject: [PATCH 068/157] other improvements --- src/spikeinterface/extractors/nwbextractors.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index fdc85a589a..8b8b22f354 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -1383,7 +1383,7 @@ def get_unit_spike_train_in_seconds( start_time: Optional[float] = None, end_time: Optional[float] = None, ) -> np.ndarray: - """Get the spike train for a unit in seconds. + """Get the spike train times for a unit in seconds. This method returns spike times directly in seconds without conversion to frames, avoiding double conversion for NWB files that already store @@ -1413,16 +1413,16 @@ def get_unit_spike_train_in_seconds( spike_times = self.spike_times_data[start_index:end_index] # Filter by time range if specified - start_idx = 0 + start_index = 0 if start_time is not None: - start_idx = np.searchsorted(spike_times, start_time, side="left") + start_index = np.searchsorted(spike_times, start_time, side="left") if end_time is not None: - end_idx = np.searchsorted(spike_times, end_time, side="left") + end_index = np.searchsorted(spike_times, end_time, side="left") else: - end_idx = spike_times.size + end_index = spike_times.size - return spike_times[start_idx:end_idx].astype("float64", copy=False) + return spike_times[start_index:end_index].astype("float64", copy=False) def _find_timeseries_from_backend(group, path="", result=None, backend="hdf5"): From 016912602c503b09b547a9e40babf1cefb738898 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 13:10:06 -0600 Subject: [PATCH 069/157] fix tests --- src/spikeinterface/core/basesorting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 86adadf40a..6e0f43b16a 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -250,8 +250,7 @@ def get_unit_spike_train_in_seconds( segment = self._sorting_segments[segment_index] # If sorting has a registered recording, get the frames and get the times from the recording - if self.has_time_vector(): - + if self.has_time_vector(segment_index=segment_index): start_frame = None end_frame = None From 96bd39b7f7425c18f75752b3648b9fa22da7fc63 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 14:32:20 -0600 Subject: [PATCH 070/157] test time handling --- src/spikeinterface/core/basesorting.py | 25 +++++++++++-------- .../core/tests/test_time_handling.py | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 6e0f43b16a..e788f5d2b2 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -250,16 +250,12 @@ def get_unit_spike_train_in_seconds( segment = self._sorting_segments[segment_index] # If sorting has a registered recording, get the frames and get the times from the recording - if self.has_time_vector(segment_index=segment_index): + # Note that this take into account the segment start time of the recording + if self.has_recording(): + + # Get all the spike times and then slice them start_frame = None end_frame = None - - if start_time is not None: - start_frame = self.time_to_sample_index(start_time, segment_index=segment_index) - - if end_time is not None: - end_frame = self.time_to_sample_index(end_time, segment_index=segment_index) - spike_frames = self.get_unit_spike_train( unit_id=unit_id, segment_index=segment_index, @@ -268,15 +264,22 @@ def get_unit_spike_train_in_seconds( return_times=False, use_cache=True, ) + + recording_times = self.get_times() + spike_times = recording_times[spike_frames] + if start_time is not None: + spike_times = spike_times[spike_times >= start_time] + if end_time is not None: + spike_times = spike_times[spike_times <= end_time] + + return spike_times - times = self.get_times(segment_index=segment_index) - return times[spike_frames] # Use the native spiking times if available if hasattr(segment, "get_unit_spike_train_in_seconds"): return segment.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) - # all back to frame-based conversion + # If no recording attached and all back to frame-based conversion start_frame = None end_frame = None diff --git a/src/spikeinterface/core/tests/test_time_handling.py b/src/spikeinterface/core/tests/test_time_handling.py index 2ae435ea84..f22939c33c 100644 --- a/src/spikeinterface/core/tests/test_time_handling.py +++ b/src/spikeinterface/core/tests/test_time_handling.py @@ -411,9 +411,10 @@ def _check_spike_times_are_correct(self, sorting, times_recording, segment_index spike_indexes = sorting.get_unit_spike_train(unit_id, segment_index=segment_index) rec_times = times_recording.get_times(segment_index=segment_index) + times_in_recording = rec_times[spike_indexes] assert np.array_equal( spike_times, - rec_times[spike_indexes], + times_in_recording, ) def _get_sorting_with_recording_attached(self, recording_for_durations, recording_to_attach): From d0ee2b7b3f9456953c427d571316c71e51be3b2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 20:32:49 +0000 Subject: [PATCH 071/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/basesorting.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index e788f5d2b2..739c49cee4 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -252,7 +252,7 @@ def get_unit_spike_train_in_seconds( # If sorting has a registered recording, get the frames and get the times from the recording # Note that this take into account the segment start time of the recording if self.has_recording(): - + # Get all the spike times and then slice them start_frame = None end_frame = None @@ -264,16 +264,15 @@ def get_unit_spike_train_in_seconds( return_times=False, use_cache=True, ) - + recording_times = self.get_times() spike_times = recording_times[spike_frames] if start_time is not None: spike_times = spike_times[spike_times >= start_time] if end_time is not None: spike_times = spike_times[spike_times <= end_time] - - return spike_times + return spike_times # Use the native spiking times if available if hasattr(segment, "get_unit_spike_train_in_seconds"): From 8d0da66a3938bdd466de02b9da37fdcac755e528 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 1 Jul 2025 14:38:10 -0600 Subject: [PATCH 072/157] get times index --- src/spikeinterface/core/basesorting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 739c49cee4..3d980d4e21 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -265,7 +265,7 @@ def get_unit_spike_train_in_seconds( use_cache=True, ) - recording_times = self.get_times() + recording_times = self.get_times(segment_index=segment_index) spike_times = recording_times[spike_frames] if start_time is not None: spike_times = spike_times[spike_times >= start_time] From a83fa8a95ff0713d86ede1784d2d4f8af0a7468d Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Wed, 2 Jul 2025 08:41:49 +0100 Subject: [PATCH 073/157] respond to sam --- doc/how_to/process_by_channel_group.rst | 1 + src/spikeinterface/sorters/basesorter.py | 11 ----------- src/spikeinterface/sorters/runsorter.py | 24 +++++++++--------------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/doc/how_to/process_by_channel_group.rst b/doc/how_to/process_by_channel_group.rst index b1929eec3b..9a68bdbdc3 100644 --- a/doc/how_to/process_by_channel_group.rst +++ b/doc/how_to/process_by_channel_group.rst @@ -155,6 +155,7 @@ This will return a dict of sortings, with the keys corresponding to the groups. .. code-block:: python split_recording = raw_recording.split_by("group") + # is a dict of recordings # do preprocessing if needed pp_recording = spre.bandpass_filter(split_recording) diff --git a/src/spikeinterface/sorters/basesorter.py b/src/spikeinterface/sorters/basesorter.py index b2f694c3fc..83f700af8a 100644 --- a/src/spikeinterface/sorters/basesorter.py +++ b/src/spikeinterface/sorters/basesorter.py @@ -344,17 +344,6 @@ def get_result_from_folder(cls, output_folder, register_recording=True, sorting_ if recording is not None: sorting.register_recording(recording) - if recording.get_annotation("split_by_property") is not None: - split_by_property = recording.get_annotation("split_by_property") - property_values = set(recording.get_property(split_by_property)) - if len(property_values) > 1: - warnings.warn( - f"Registered Recording has non-unique {split_by_property} keys. They are {property_values}." - ) - elif len(property_values) == 1: - property_value = next(iter(property_values)) - sorting.set_property("split_by_property", values=[property_value] * sorting.get_num_units()) - if sorting_info: # set sorting info to Sorting object if (output_folder / "spikeinterface_recording.json").exists(): diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index 7a97ecec6f..77c979bb29 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -225,8 +225,8 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No Dictionary of `BaseSorting`s, with the same keys as the input dict of `BaseRecording`s. """ - sorter_name = run_sorter_params["sorter_name"] - remove_existing_folder = run_sorter_params["remove_existing_folder"] + sorter_name = run_sorter_params.get("sorter_name") + remove_existing_folder = run_sorter_params.get("remove_existing_folder") if folder is None: folder = Path(sorter_name + "_output") @@ -234,11 +234,13 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No folder = Path(folder) folder.mkdir(exist_ok=remove_existing_folder) - # If we know how the recording was split, save this in the info file - first_recording = next(iter(dict_of_recordings.values())) - split_by_property = first_recording.get_annotation("split_by_property") - if split_by_property is None: - split_by_property = "Unknown" + sorter_dict = {} + for group_key, recording in dict_of_recordings.items(): + + if "recording" in run_sorter_params: + run_sorter_params.pop("recording") + + sorter_dict[group_key] = run_sorter(recording=recording, folder=folder / f"{group_key}", **run_sorter_params) info_file = folder / "spikeinterface_info.json" info = dict( @@ -250,14 +252,6 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No with open(info_file, mode="w") as f: json.dump(check_json(info), f, indent=4) - sorter_dict = {} - for group_key, recording in dict_of_recordings.items(): - - if "recording" in run_sorter_params: - run_sorter_params.pop("recording") - - sorter_dict[group_key] = run_sorter(recording=recording, folder=folder / f"{group_key}", **run_sorter_params) - return sorter_dict From 4c4d8a73db42948b04dc23fde5b870b5e7d07193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20WYNGAARD?= Date: Wed, 2 Jul 2025 16:13:17 +0200 Subject: [PATCH 074/157] Fix sampling rate issue when aggregating Different versions of Kilosort output different sampling rate (sometimes rounding it to 2 decimal places, sometimes not) This makes a crash when trying to aggregate both sortings together This PR fixes this --- src/spikeinterface/core/unitsaggregationsorting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/core/unitsaggregationsorting.py b/src/spikeinterface/core/unitsaggregationsorting.py index 8f4a2732c3..300d668982 100644 --- a/src/spikeinterface/core/unitsaggregationsorting.py +++ b/src/spikeinterface/core/unitsaggregationsorting.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math import warnings import numpy as np @@ -62,7 +63,7 @@ def __init__(self, sorting_list, renamed_unit_ids=None): sampling_frequency = sorting_list[0].get_sampling_frequency() num_segments = sorting_list[0].get_num_segments() - ok1 = all(sampling_frequency == sort.get_sampling_frequency() for sort in sorting_list) + ok1 = all(math.isclose(sampling_frequency, sort.get_sampling_frequency(), abs_tol=1e-2) for sort in sorting_list) ok2 = all(num_segments == sort.get_num_segments() for sort in sorting_list) if not (ok1 and ok2): raise ValueError("Sortings don't have the same sampling_frequency/num_segments") From 8167f65a5d6e5d9d54f6c61fdc25b66227363978 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 14:13:54 +0000 Subject: [PATCH 075/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/unitsaggregationsorting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/core/unitsaggregationsorting.py b/src/spikeinterface/core/unitsaggregationsorting.py index 300d668982..cb824abd49 100644 --- a/src/spikeinterface/core/unitsaggregationsorting.py +++ b/src/spikeinterface/core/unitsaggregationsorting.py @@ -63,7 +63,9 @@ def __init__(self, sorting_list, renamed_unit_ids=None): sampling_frequency = sorting_list[0].get_sampling_frequency() num_segments = sorting_list[0].get_num_segments() - ok1 = all(math.isclose(sampling_frequency, sort.get_sampling_frequency(), abs_tol=1e-2) for sort in sorting_list) + ok1 = all( + math.isclose(sampling_frequency, sort.get_sampling_frequency(), abs_tol=1e-2) for sort in sorting_list + ) ok2 = all(num_segments == sort.get_num_segments() for sort in sorting_list) if not (ok1 and ok2): raise ValueError("Sortings don't have the same sampling_frequency/num_segments") From dff4da4d6bd37b045980907dfb2e84ce06f0f8bc Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 2 Jul 2025 14:01:55 -0400 Subject: [PATCH 076/157] don't force soft merge check on hard merge --- src/spikeinterface/core/sortinganalyzer.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index d1a0ef7be4..f1b704911d 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -967,11 +967,19 @@ def _save_or_select_or_merge( if unit_id in new_unit_ids: merge_unit_group = tuple(merge_unit_groups[new_unit_ids.index(unit_id)]) if not mergeable[merge_unit_group]: - raise Exception( - f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " - f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold or use " - "a hard merge." - ) + # if someone wants soft mode we error if they try to use an inappropriate sparsity overlap + # if hard mode we can still use a sparsity if all units have good sparsity overlap + # but if any merge_unit_group doesn't have an overlap then we have to not use the old + # sparsity and instead use a new sparsity + if merging_mode == "soft": + raise Exception( + f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " + f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold or use " + "a hard merge." + ) + else: + sparsity = None + break else: sparsity_mask[unit_index] = masks[merge_unit_group] else: From 9399f6277f8c2ce65440a9a9884d4e98e001f024 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 2 Jul 2025 12:55:29 -0600 Subject: [PATCH 077/157] base sorting --- src/spikeinterface/core/basesorting.py | 49 ++++++++++++-------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 3d980d4e21..23d45def2d 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Optional, Union +from typing import Optional import numpy as np @@ -187,19 +187,8 @@ def get_unit_spike_train( ).astype("int64") if return_times: - # Use the new get_unit_spike_train_in_seconds method for better precision - start_time = None - end_time = None - - if start_frame is not None: - segment = self._sorting_segments[segment_index] - t_start = segment._t_start if segment._t_start is not None else 0 - start_time = start_frame / self.get_sampling_frequency() + t_start - - if end_frame is not None: - segment = self._sorting_segments[segment_index] - t_start = segment._t_start if segment._t_start is not None else 0 - end_time = end_frame / self.get_sampling_frequency() + t_start + start_time = self.sample_index_to_time(start_frame, segment_index=segment_index) if start_frame is not None else None + end_time = self.sample_index_to_time(end_frame, segment_index=segment_index) if end_frame is not None else None return self.get_unit_spike_train_in_seconds( unit_id=unit_id, @@ -265,8 +254,9 @@ def get_unit_spike_train_in_seconds( use_cache=True, ) - recording_times = self.get_times(segment_index=segment_index) - spike_times = recording_times[spike_frames] + spike_times = self.sample_index_to_time(spike_frames, segment_index=segment_index) + + # Filter to return only the spikes within the specified time range if start_time is not None: spike_times = spike_times[spike_times >= start_time] if end_time is not None: @@ -279,16 +269,10 @@ def get_unit_spike_train_in_seconds( return segment.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) # If no recording attached and all back to frame-based conversion - start_frame = None - end_frame = None - - if start_time is not None: - start_frame = self.time_to_sample_index(start_time, segment_index=segment_index) - - if end_time is not None: - end_frame = self.time_to_sample_index(end_time, segment_index=segment_index) - # Get spike train in frames and convert to times using traditional method + start_frame = self.time_to_sample_index(start_time, segment_index=segment_index) if start_time else None + end_frame = self.time_to_sample_index(end_time, segment_index=segment_index) if end_time else None + spike_frames = self.get_unit_spike_train( unit_id=unit_id, segment_index=segment_index, @@ -298,7 +282,6 @@ def get_unit_spike_train_in_seconds( use_cache=True, ) - # Convert frames to times t_start = segment._t_start if segment._t_start is not None else 0 spike_times = spike_frames / self.get_sampling_frequency() return t_start + spike_times @@ -306,7 +289,7 @@ def get_unit_spike_train_in_seconds( def register_recording(self, recording, check_spike_frames: bool = True): """ Register a recording to the sorting. If the sorting and recording both contain - time information, the recording’s time information will be used. + time information, the recording's time information will be used. Parameters ---------- @@ -650,6 +633,18 @@ def time_to_sample_index(self, time, segment_index=0): return sample_index + def sample_index_to_time(self, sample_index: int | np.ndarray, segment_index: Optional[int] = None) -> float | np.ndarray: + """ + Transform sample index into time in seconds + """ + segment_index = self._check_segment_index(segment_index) + if self.has_recording(): + return self._recording.sample_index_to_time(sample_index, segment_index=segment_index) + else: + segment = self._sorting_segments[segment_index] + t_start = segment._t_start if segment._t_start is not None else 0 + return (sample_index / self.get_sampling_frequency()) + t_start + def precompute_spike_trains(self, from_spike_vector=None): """ Pre-computes and caches all spike trains for this sorting From bd2166e8482ecccc96b8e07d0864ba49f9f4788f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:55:59 +0000 Subject: [PATCH 078/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/basesorting.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 23d45def2d..c9f244fa76 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -187,8 +187,12 @@ def get_unit_spike_train( ).astype("int64") if return_times: - start_time = self.sample_index_to_time(start_frame, segment_index=segment_index) if start_frame is not None else None - end_time = self.sample_index_to_time(end_frame, segment_index=segment_index) if end_frame is not None else None + start_time = ( + self.sample_index_to_time(start_frame, segment_index=segment_index) if start_frame is not None else None + ) + end_time = ( + self.sample_index_to_time(end_frame, segment_index=segment_index) if end_frame is not None else None + ) return self.get_unit_spike_train_in_seconds( unit_id=unit_id, @@ -255,7 +259,7 @@ def get_unit_spike_train_in_seconds( ) spike_times = self.sample_index_to_time(spike_frames, segment_index=segment_index) - + # Filter to return only the spikes within the specified time range if start_time is not None: spike_times = spike_times[spike_times >= start_time] @@ -633,7 +637,9 @@ def time_to_sample_index(self, time, segment_index=0): return sample_index - def sample_index_to_time(self, sample_index: int | np.ndarray, segment_index: Optional[int] = None) -> float | np.ndarray: + def sample_index_to_time( + self, sample_index: int | np.ndarray, segment_index: Optional[int] = None + ) -> float | np.ndarray: """ Transform sample index into time in seconds """ From 006d6908f15c861e67345a045e11145fc1a44972 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Wed, 2 Jul 2025 20:26:28 +0100 Subject: [PATCH 079/157] update version ibllib for passing tests --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a8005fbfc8..e20f8b3b62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,13 +68,13 @@ extractors = [ "sonpy;python_version<'3.10'", "lxml", # lxml for neuroscope "scipy", - "ibllib==3.3.1", # streaming IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL "pymatreader>=0.0.32", # For cell explorer matlab files "zugbruecke>=0.2; sys_platform!='win32'", # For plexon2 ] streaming_extractors = [ - "ibllib==3.3.1", # streaming IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL # Following dependencies are for streaming with nwb files "pynwb>=2.6.0", "fsspec", @@ -143,7 +143,7 @@ test_extractors = [ ] test_preprocessing = [ - "ibllib==3.3.1", # for IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL "torch", ] @@ -155,7 +155,7 @@ test = [ "psutil", # preprocessing - "ibllib==3.3.1", # for IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming templates "s3fs", From ca5ff71c3775a4ac509971f664143c66386a4377 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Wed, 2 Jul 2025 20:26:28 +0100 Subject: [PATCH 080/157] update version ibllib for passing tests --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6554423890..d1d492973d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,13 +69,13 @@ extractors = [ "sonpy;python_version<'3.10'", "lxml", # lxml for neuroscope "scipy", - "ibllib==3.3.1", # streaming IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL "pymatreader>=0.0.32", # For cell explorer matlab files "zugbruecke>=0.2; sys_platform!='win32'", # For plexon2 ] streaming_extractors = [ - "ibllib==3.3.1", # streaming IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL # Following dependencies are for streaming with nwb files "pynwb>=2.6.0", "fsspec", @@ -144,7 +144,7 @@ test_extractors = [ ] test_preprocessing = [ - "ibllib==3.3.1", # for IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming IBL "torch", ] @@ -156,7 +156,7 @@ test = [ "psutil", # preprocessing - "ibllib==3.3.1", # for IBL + "ibllib>=3.4.1;python_version>='3.10'", # streaming templates "s3fs", From 963335a5307f8176a9e78eab3812022fd58c8df5 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 2 Jul 2025 15:12:49 -0600 Subject: [PATCH 081/157] try test --- .../extractors/tests/test_neoextractors.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 0eb43535aa..6a33a4e180 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -40,6 +40,7 @@ Spike2RecordingExtractor, EDFRecordingExtractor, Plexon2RecordingExtractor, + AxonRecordingExtractor, ) from spikeinterface.extractors.extractor_classes import KiloSortSortingExtractor @@ -294,6 +295,12 @@ class TdTRecordingTest(RecordingCommonTestSuite, unittest.TestCase): entities = [("tdt/aep_05", {"stream_id": "1"})] +class AxonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): + ExtractorClass = AxonRecordingExtractor + downloads = ["axon"] + entities = ["axon/extracellular_data/four_electrodes/24606005_SampleData.abf"] + + class AxonaRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = AxonaRecordingExtractor downloads = ["axona"] @@ -422,6 +429,8 @@ class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): entities = [("plexon/4chDemoPL2.pl2", {"sampling_frequency": 40000})] + + if __name__ == "__main__": # test = MearecSortingTest() # test = SpikeGLXRecordingTest() From 6d914d342c27fd4b44e4cb961aa639b160ed3b6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 21:13:16 +0000 Subject: [PATCH 082/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/extractors/tests/test_neoextractors.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 6a33a4e180..622469ca51 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -429,8 +429,6 @@ class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): entities = [("plexon/4chDemoPL2.pl2", {"sampling_frequency": 40000})] - - if __name__ == "__main__": # test = MearecSortingTest() # test = SpikeGLXRecordingTest() From 1c1d7ab476514196031ff9be3d7e452162c0e049 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Jul 2025 08:06:12 +0200 Subject: [PATCH 083/157] Apply suggestions from code review Co-authored-by: Zach McKenzie <92116279+zm711@users.noreply.github.com> --- doc/references.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/references.rst b/doc/references.rst index b4d853207a..c456c7833a 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -40,7 +40,7 @@ please include the appropriate citation for the :code:`sorter_name` parameter yo - :code:`herdingspikes` [Muthmann]_ [Hilgen]_ - :code:`kilosort` [Pachitariu]_ - :code:`mountainsort` [Chung]_ -- :code:`rt-sort` [van der Molen]_ +- :code:`rt-sort` [van_der_Molen]_ - :code:`spykingcircus` [Yger]_ - :code:`wavclus` [Chaure]_ - :code:`yass` [Lee]_ @@ -136,7 +136,7 @@ References .. [UMS] `UltraMegaSort2000 - Spike sorting and quality metrics for extracellular spike data. 2011. `_ -.. [van der Molen] `RT-Sort: An action potential propagation-based algorithm for real time spike detection and sorting with millisecond latencies. 2024. `_ +.. [van_der_Molen] `RT-Sort: An action potential propagation-based algorithm for real time spike detection and sorting with millisecond latencies. 2024. `_ .. [Varol] `Decentralized Motion Inference and Registration of Neuropixel Data. 2021. `_ From b45cb79f72b80c9b79d8eb54e4061fd4d9e42755 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Thu, 3 Jul 2025 08:49:31 +0100 Subject: [PATCH 084/157] ignore AppleDouble format files on extension load --- src/spikeinterface/core/sortinganalyzer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index d1a0ef7be4..b3f2a5d83b 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -2219,6 +2219,7 @@ def load_data(self): ext_data_file.name == "params.json" or ext_data_file.name == "info.json" or ext_data_file.name == "run_info.json" + or str(ext_data_file.name).startswith("._") # ignore AppleDouble format files ): continue ext_data_name = ext_data_file.stem From 0fe9580b1eccdd30fa169f57c1a772e78b34b6b0 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 3 Jul 2025 09:45:10 -0600 Subject: [PATCH 085/157] fix order --- src/spikeinterface/core/basesorting.py | 31 +++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index c9f244fa76..330a36f5ad 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -162,6 +162,21 @@ def get_unit_spike_train( Spike frames (or times if return_times=True) """ + if return_times: + start_time = ( + self.sample_index_to_time(start_frame, segment_index=segment_index) if start_frame is not None else None + ) + end_time = ( + self.sample_index_to_time(end_frame, segment_index=segment_index) if end_frame is not None else None + ) + + return self.get_unit_spike_train_in_seconds( + unit_id=unit_id, + segment_index=segment_index, + start_time=start_time, + end_time=end_time, + ) + segment_index = self._check_segment_index(segment_index) if use_cache: if segment_index not in self._cached_spike_trains: @@ -186,22 +201,8 @@ def get_unit_spike_train( unit_id=unit_id, start_frame=start_frame, end_frame=end_frame ).astype("int64") - if return_times: - start_time = ( - self.sample_index_to_time(start_frame, segment_index=segment_index) if start_frame is not None else None - ) - end_time = ( - self.sample_index_to_time(end_frame, segment_index=segment_index) if end_frame is not None else None - ) - return self.get_unit_spike_train_in_seconds( - unit_id=unit_id, - segment_index=segment_index, - start_time=start_time, - end_time=end_time, - ) - else: - return spike_frames + return spike_frames def get_unit_spike_train_in_seconds( self, From a69f58ff241e6362ba0d373daf926a0caddbbe3b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:45:51 +0000 Subject: [PATCH 086/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/basesorting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 330a36f5ad..ef50d39721 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -201,7 +201,6 @@ def get_unit_spike_train( unit_id=unit_id, start_frame=start_frame, end_frame=end_frame ).astype("int64") - return spike_frames def get_unit_spike_train_in_seconds( From 1a735f963160cee84eebb00f503b87a9783b485c Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Jul 2025 18:22:18 +0200 Subject: [PATCH 087/157] Fix tests --- .../curation/curation_format.py | 18 ++++++------- src/spikeinterface/curation/curation_model.py | 25 +++++++++++-------- .../curation/tests/test_curation_format.py | 23 ++++++++++++----- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/spikeinterface/curation/curation_format.py b/src/spikeinterface/curation/curation_format.py index 94f6670832..baea4399ff 100644 --- a/src/spikeinterface/curation/curation_format.py +++ b/src/spikeinterface/curation/curation_format.py @@ -126,8 +126,8 @@ def apply_curation_labels( manual_labels = curation_label_to_vectors(curation_model) # apply on non merged / split - merge_new_unit_ids = [m.merge_new_unit_id for m in curation_model.merges] - split_new_unit_ids = [m.split_new_unit_ids for m in curation_model.splits] + merge_new_unit_ids = [m.new_unit_id for m in curation_model.merges] + split_new_unit_ids = [m.new_unit_ids for m in curation_model.splits] split_new_unit_ids = list(chain(*split_new_unit_ids)) merged_split_units = merge_new_unit_ids + split_new_unit_ids @@ -139,7 +139,7 @@ def apply_curation_labels( all_values[unit_ind] = values[ind] sorting.set_property(key, all_values) - for new_unit_id, merge in zip(new_unit_ids, curation_model.merges): + for new_unit_id, merge in zip(merge_new_unit_ids, curation_model.merges): old_group_ids = merge.unit_ids for label_key, label_def in curation_model.label_definitions.items(): if label_def.exclusive: @@ -166,7 +166,7 @@ def apply_curation_labels( for split in curation_model.splits: # propagate property of splut unit to new units old_unit = split.unit_id - new_unit_ids = split.split_new_unit_ids + new_unit_ids = split.new_unit_ids for label_key, label_def in curation_model.label_definitions.items(): if label_def.exclusive: ind = list(curation_model.unit_ids).index(old_unit) @@ -194,9 +194,9 @@ def apply_curation( Apply curation dict to a Sorting or a SortingAnalyzer. Steps are done in this order: - 1. Apply removal using curation_dict["removed_units"] - 2. Apply merges using curation_dict["merge_unit_groups"] - 3. Apply splits using curation_dict["split_units"] + 1. Apply removal using curation_dict["removed"] + 2. Apply merges using curation_dict["merges"] + 3. Apply splits using curation_dict["splits"] 4. Set labels using curation_dict["manual_labels"] A new Sorting or SortingAnalyzer (in memory) is returned. @@ -281,7 +281,7 @@ def apply_curation( **job_kwargs, ) for i, merge_unit_id in enumerate(new_unit_ids): - curation_model.merges[i].merge_new_unit_id = merge_unit_id + curation_model.merges[i].new_unit_id = merge_unit_id # 3. Split units if len(curation_model.splits) > 0: @@ -314,7 +314,7 @@ def apply_curation( verbose=verbose, ) for i, split_unit_ids in enumerate(new_unit_ids): - curation_model.splits[i].split_new_unit_ids = split_unit_ids + curation_model.splits[i].new_unit_ids = split_unit_ids # 4. Apply labels apply_curation_labels(curated_sorting_or_analyzer, curation_model) diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index a701ffec01..e32aa6e275 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -51,26 +51,26 @@ def get_full_spike_indices(self, sorting: BaseSorting): Get the full indices of the spikes in the split for different split modes. """ num_spikes = sorting.count_num_spikes_per_unit()[self.unit_id] - if self.split_mode == "indices": + if self.mode == "indices": # check the sum of split_indices is equal to num_spikes - num_spikes_in_split = sum(len(indices) for indices in self.split_indices) + num_spikes_in_split = sum(len(indices) for indices in self.indices) if num_spikes_in_split != num_spikes: # add remaining spike indices - full_spike_indices = list(self.split_indices) - existing_indices = np.concatenate(self.split_indices) + full_spike_indices = list(self.indices) + existing_indices = np.concatenate(self.indices) remaining_indices = np.setdiff1d(np.arange(num_spikes), existing_indices) full_spike_indices.append(remaining_indices) else: - full_spike_indices = self.split_indices - elif self.split_mode == "labels": - assert len(self.split_labels) == num_spikes, ( - f"In 'labels' mode, the number of split_labels ({len(self.split_labels)}) " + full_spike_indices = self.indices + elif self.mode == "labels": + assert len(self.labels) == num_spikes, ( + f"In 'labels' mode, the number of.labels ({len(self.labels)}) " f"must match the number of spikes in the unit ({num_spikes})" ) # convert to spike indices full_spike_indices = [] - for label in np.unique(self.split_labels): - label_indices = np.where(self.split_labels == label)[0] + for label in np.unique(self.labels): + label_indices = np.where(self.labels == label)[0] full_spike_indices.append(label_indices) return full_spike_indices @@ -258,7 +258,10 @@ def check_splits(cls, values): # Validate new unit IDs if split.new_unit_ids is not None: if split.mode == "indices": - if len(split.new_unit_ids) != len(split.indices): + if ( + len(split.new_unit_ids) != len(split.indices) + and len(split.new_unit_ids) != len(split.indices) + 1 + ): raise ValueError( f"Number of new unit IDs does not match number of splits for unit {split.unit_id}" ) diff --git a/src/spikeinterface/curation/tests/test_curation_format.py b/src/spikeinterface/curation/tests/test_curation_format.py index 12001cf1b2..9707b0f082 100644 --- a/src/spikeinterface/curation/tests/test_curation_format.py +++ b/src/spikeinterface/curation/tests/test_curation_format.py @@ -133,8 +133,19 @@ # Test with splits curation_with_splits = { - **curation_ids_int, - "splits": [{"unit_id": 2, "mode": "indices", "indices": [[0, 1, 2], [3, 4, 5]], "new_unit_ids": [50, 51]}], + "format_version": "2", + "unit_ids": [1, 2, 3, 6, 10, 14, 20, 31, 42], + "label_definitions": { + "quality": {"label_options": ["good", "noise", "MUA", "artifact"], "exclusive": True}, + "putative_type": { + "label_options": ["excitatory", "inhibitory", "pyramidal", "mitral"], + "exclusive": False, + }, + }, + "manual_labels": [ + {"unit_id": 2, "quality": ["good"], "putative_type": ["excitatory", "pyramidal"]}, + ], + "splits": [{"unit_id": 2, "mode": "indices", "indices": [[0, 1, 2], [3, 4, 5]]}], } # Test dictionary format for splits @@ -301,7 +312,7 @@ def test_apply_curation_with_split_multi_segment(): split_indices.append(np.arange(0, len(spikes_in_segment)) + cum_spikes) cum_spikes += len(spikes_in_segment) - curation_with_splits_multi_segment["splits"][0]["split_indices"] = split_indices + curation_with_splits_multi_segment["splits"][0]["indices"] = split_indices sorting_curated = apply_curation(sorting, curation_with_splits_multi_segment) @@ -352,9 +363,9 @@ def test_apply_curation_splits_with_mask(): "splits": [ { "unit_id": 2, - "split_mode": "labels", - "split_labels": split_labels.tolist(), - "split_new_unit_ids": [43, 44, 45], + "mode": "labels", + "labels": split_labels.tolist(), + "new_unit_ids": [43, 44, 45], } ], } From 9684e0aaadfd35b7f0a2076484d3c484a9c9b8bb Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Thu, 3 Jul 2025 22:22:41 +0100 Subject: [PATCH 088/157] Address Chris' comments --- src/spikeinterface/widgets/amplitudes.py | 15 +++++++++++++++ src/spikeinterface/widgets/motion.py | 22 ++++++++++++++++++---- src/spikeinterface/widgets/rasters.py | 21 ++++++++++++++++++--- src/spikeinterface/widgets/utils.py | 14 +++++--------- 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 6f36baf521..57fd846ff4 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -60,9 +60,24 @@ def __init__( plot_histograms=False, bins=None, plot_legend=True, + segment_index=None, backend=None, **backend_kwargs, ): + import warnings + # Handle deprecation of segment_index parameter + if segment_index is not None: + warnings.warn( + "The 'segment_index' parameter is deprecated and will be removed in a future version. " + "Use 'segment_indices' instead.", + DeprecationWarning, + stacklevel=2 + ) + if segment_indices is None: + if isinstance(segment_index, int): + segment_indices = [segment_index] + else: + segment_indices = segment_index sorting_analyzer = self.ensure_sorting_analyzer(sorting_analyzer) sorting = sorting_analyzer.sorting diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index dbc271f305..aebaf2340a 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -158,12 +158,28 @@ def __init__( color: str = "Gray", clim: tuple[float, float] | None = None, alpha: float = 1, + segment_index: int | list[int] | None = None, backend: str | None = None, **backend_kwargs, ): + import warnings from matplotlib.pyplot import colormaps from matplotlib.colors import Normalize + # Handle deprecation of segment_index parameter + if segment_index is not None: + warnings.warn( + "The 'segment_index' parameter is deprecated and will be removed in a future version. " + "Use 'segment_indices' instead.", + DeprecationWarning, + stacklevel=2 + ) + if segment_indices is None: + if isinstance(segment_index, int): + segment_indices = [segment_index] + else: + segment_indices = segment_index + assert peaks is not None or sorting_analyzer is not None if peaks is not None: @@ -269,10 +285,8 @@ def __init__( ] # Calculate durations from max sample in each segment - durations = [ - (np.max(filtered_peaks["sample_index"][start:end]) + 1) / sampling_frequency if start < end else 0 - for (start, end) in segment_boundaries - ] + durations = [(filtered_peaks["sample_index"][end-1]+1) / sampling_frequency for (_, end) in segment_boundaries ] + plot_data = dict( spike_train_data=spike_train_data, diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 9926948215..a517eafad7 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -374,16 +374,32 @@ class RasterWidget(BaseRasterWidget): def __init__( self, sorting_analyzer_or_sorting: SortingAnalyzer | BaseSorting | None = None, - segment_index: int | None = None, + segment_indices: int | None = None, unit_ids: list | None = None, time_range: list | None = None, color="k", backend: str | None = None, sorting: BaseSorting | None = None, sorting_analyzer: SortingAnalyzer | None = None, + segment_index: int | None = None, **backend_kwargs, ): + import warnings + # Handle deprecation of segment_index parameter + if segment_index is not None: + warnings.warn( + "The 'segment_index' parameter is deprecated and will be removed in a future version. " + "Use 'segment_indices' instead.", + DeprecationWarning, + stacklevel=2 + ) + if segment_indices is None: + if isinstance(segment_index, int): + segment_indices = [segment_index] + else: + segment_indices = segment_index + if sorting is not None: # When removed, make `sorting_analyzer_or_sorting` a required argument rather than None. deprecation_msg = "`sorting` argument is deprecated and will be removed in version 0.105.0. Please use `sorting_analyzer_or_sorting` instead" @@ -421,8 +437,7 @@ def __init__( for seg_idx in segment_indices: for unit_id in unit_ids: # Get spikes for this segment and unit - mask = (spikes["segment_index"] == seg_idx) & (spikes["unit_index"] == unit_id) - spike_times = spikes["sample_index"][mask] / sorting.sampling_frequency + spike_times = sorting.get_unit_spike_train(unit_id=unit_id, segment_index=seg_idx) / sorting.sampling_frequency # Store data spike_train_data[seg_idx][unit_id] = spike_times diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 9c5892a937..53fa527276 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -415,14 +415,10 @@ def get_segment_durations(sorting: BaseSorting) -> list[float]: spikes = sorting.to_spike_vector() segment_indices = np.unique(spikes["segment_index"]) - durations = [] - for seg_idx in segment_indices: - segment_mask = spikes["segment_index"] == seg_idx - if np.any(segment_mask): - max_sample = np.max(spikes["sample_index"][segment_mask]) - duration = max_sample / sorting.sampling_frequency - else: - duration = 0 - durations.append(duration) + segment_boundaries = [ + np.searchsorted(spikes["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices + ] + + durations = [(spikes["sample_index"][end-1] + 1) / sorting.sampling_frequency for (_, end) in segment_boundaries] return durations From 6c20215d322ab72e31e0d1693a72e14cd6dbe01c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 21:24:30 +0000 Subject: [PATCH 089/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/widgets/amplitudes.py | 3 ++- src/spikeinterface/widgets/motion.py | 7 ++++--- src/spikeinterface/widgets/rasters.py | 7 +++++-- src/spikeinterface/widgets/utils.py | 8 ++++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 57fd846ff4..1f4c1eab47 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -65,13 +65,14 @@ def __init__( **backend_kwargs, ): import warnings + # Handle deprecation of segment_index parameter if segment_index is not None: warnings.warn( "The 'segment_index' parameter is deprecated and will be removed in a future version. " "Use 'segment_indices' instead.", DeprecationWarning, - stacklevel=2 + stacklevel=2, ) if segment_indices is None: if isinstance(segment_index, int): diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index aebaf2340a..eac1b2713c 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -172,7 +172,7 @@ def __init__( "The 'segment_index' parameter is deprecated and will be removed in a future version. " "Use 'segment_indices' instead.", DeprecationWarning, - stacklevel=2 + stacklevel=2, ) if segment_indices is None: if isinstance(segment_index, int): @@ -285,8 +285,9 @@ def __init__( ] # Calculate durations from max sample in each segment - durations = [(filtered_peaks["sample_index"][end-1]+1) / sampling_frequency for (_, end) in segment_boundaries ] - + durations = [ + (filtered_peaks["sample_index"][end - 1] + 1) / sampling_frequency for (_, end) in segment_boundaries + ] plot_data = dict( spike_train_data=spike_train_data, diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index a517eafad7..55032b38eb 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -386,13 +386,14 @@ def __init__( ): import warnings + # Handle deprecation of segment_index parameter if segment_index is not None: warnings.warn( "The 'segment_index' parameter is deprecated and will be removed in a future version. " "Use 'segment_indices' instead.", DeprecationWarning, - stacklevel=2 + stacklevel=2, ) if segment_indices is None: if isinstance(segment_index, int): @@ -437,7 +438,9 @@ def __init__( for seg_idx in segment_indices: for unit_id in unit_ids: # Get spikes for this segment and unit - spike_times = sorting.get_unit_spike_train(unit_id=unit_id, segment_index=seg_idx) / sorting.sampling_frequency + spike_times = ( + sorting.get_unit_spike_train(unit_id=unit_id, segment_index=seg_idx) / sorting.sampling_frequency + ) # Store data spike_train_data[seg_idx][unit_id] = spike_times diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 53fa527276..a003742939 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -415,10 +415,10 @@ def get_segment_durations(sorting: BaseSorting) -> list[float]: spikes = sorting.to_spike_vector() segment_indices = np.unique(spikes["segment_index"]) - segment_boundaries = [ - np.searchsorted(spikes["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices - ] + segment_boundaries = [ + np.searchsorted(spikes["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices + ] - durations = [(spikes["sample_index"][end-1] + 1) / sorting.sampling_frequency for (_, end) in segment_boundaries] + durations = [(spikes["sample_index"][end - 1] + 1) / sorting.sampling_frequency for (_, end) in segment_boundaries] return durations From 96f5d8b86e0647416401150f123722d00ab15b61 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Fri, 4 Jul 2025 08:50:21 +0100 Subject: [PATCH 090/157] move pop --- src/spikeinterface/sorters/runsorter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index 77c979bb29..a81da0e38a 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -160,8 +160,9 @@ def run_sorter( delete_container_files=delete_container_files, ) ) + all_kwargs.pop("recording") - dict_of_sorters = _run_sorter_by_dict(recording, **all_kwargs) + dict_of_sorters = _run_sorter_by_dict(dict_of_recordings=recording, **all_kwargs) return dict_of_sorters if docker_image or singularity_image: @@ -236,10 +237,6 @@ def _run_sorter_by_dict(dict_of_recordings: dict, folder: str | Path | None = No sorter_dict = {} for group_key, recording in dict_of_recordings.items(): - - if "recording" in run_sorter_params: - run_sorter_params.pop("recording") - sorter_dict[group_key] = run_sorter(recording=recording, folder=folder / f"{group_key}", **run_sorter_params) info_file = folder / "spikeinterface_info.json" From f927649621656a0d77b80d6cd58fa320245ef4cb Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Fri, 4 Jul 2025 09:11:16 +0100 Subject: [PATCH 091/157] output_folder -> folder in test --- src/spikeinterface/sorters/tests/test_runsorter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index f5e77e8f19..7b88de0266 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -58,12 +58,12 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): sorter_params = {"detection": {"detect_threshold": 4.9}} - output_folder = cache_folder / "sorting_tdc_local_dict" + folder = cache_folder / "sorting_tdc_local_dict" dict_of_sortings = run_sorter( "simple", dict_of_recordings, - output_folder=output_folder, + folder=folder, remove_existing_folder=True, delete_output_folder=False, verbose=True, @@ -72,13 +72,13 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): ) assert set(list(dict_of_sortings.keys())) == set(["g", "4"]) - assert (output_folder / "g").is_dir() - assert (output_folder / "4").is_dir() + assert (folder / "g").is_dir() + assert (folder / "4").is_dir() assert dict_of_sortings["g"]._recording.get_num_channels() == 3 assert dict_of_sortings["4"]._recording.get_num_channels() == 5 - info_filepath = output_folder / "spikeinterface_info.json" + info_filepath = folder / "spikeinterface_info.json" assert info_filepath.is_file() with open(info_filepath) as f: @@ -88,7 +88,7 @@ def test_run_sorter_dict(generate_recording, create_cache_folder): for key in ["version", "dev_mode", "object"]: assert key in si_info_keys - loaded_sortings = load(output_folder) + loaded_sortings = load(folder) assert loaded_sortings.keys() == dict_of_sortings.keys() for key, sorting in loaded_sortings.items(): assert np.all(sorting.unit_ids == dict_of_sortings[key].unit_ids) From 8f3a81c47b1548e1960ade675369fe5e41d99198 Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Fri, 4 Jul 2025 12:39:46 +0100 Subject: [PATCH 092/157] add concatenated to _get_data --- .../postprocessing/spike_amplitudes.py | 12 +++++++++++- .../postprocessing/spike_locations.py | 12 +++++++++++- src/spikeinterface/qualitymetrics/misc_metrics.py | 13 ++++--------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/spikeinterface/postprocessing/spike_amplitudes.py b/src/spikeinterface/postprocessing/spike_amplitudes.py index cd20f48ffc..61c09977b0 100644 --- a/src/spikeinterface/postprocessing/spike_amplitudes.py +++ b/src/spikeinterface/postprocessing/spike_amplitudes.py @@ -111,7 +111,7 @@ def _run(self, verbose=False, **job_kwargs): ) self.data["amplitudes"] = amps - def _get_data(self, outputs="numpy"): + def _get_data(self, outputs="numpy", concatenated=False): all_amplitudes = self.data["amplitudes"] if outputs == "numpy": return all_amplitudes @@ -125,6 +125,16 @@ def _get_data(self, outputs="numpy"): for unit_id in unit_ids: inds = spike_indices[segment_index][unit_id] amplitudes_by_units[segment_index][unit_id] = all_amplitudes[inds] + + if concatenated: + amplitudes_by_units_concatenated = { + unit_id: np.concatenate( + [amps_in_segment[unit_id] for amps_in_segment in amplitudes_by_units.values()] + ) + for unit_id in unit_ids + } + return amplitudes_by_units_concatenated + return amplitudes_by_units else: raise ValueError(f"Wrong .get_data(outputs={outputs}); possibilities are `numpy` or `by_unit`") diff --git a/src/spikeinterface/postprocessing/spike_locations.py b/src/spikeinterface/postprocessing/spike_locations.py index 54c4a2c164..214aabcf46 100644 --- a/src/spikeinterface/postprocessing/spike_locations.py +++ b/src/spikeinterface/postprocessing/spike_locations.py @@ -142,7 +142,7 @@ def _run(self, verbose=False, **job_kwargs): ) self.data["spike_locations"] = spike_locations - def _get_data(self, outputs="numpy"): + def _get_data(self, outputs="numpy", concatenated=False): all_spike_locations = self.data["spike_locations"] if outputs == "numpy": return all_spike_locations @@ -156,6 +156,16 @@ def _get_data(self, outputs="numpy"): for unit_id in unit_ids: inds = spike_indices[segment_index][unit_id] spike_locations_by_units[segment_index][unit_id] = all_spike_locations[inds] + + if concatenated: + locations_by_units_concatenated = { + unit_id: np.concatenate( + [locs_in_segment[unit_id] for locs_in_segment in spike_locations_by_units.values()] + ) + for unit_id in unit_ids + } + return locations_by_units_concatenated + return spike_locations_by_units else: raise ValueError(f"Wrong .get_data(outputs={outputs})") diff --git a/src/spikeinterface/qualitymetrics/misc_metrics.py b/src/spikeinterface/qualitymetrics/misc_metrics.py index 3a3aab0bf4..a2a04a656a 100644 --- a/src/spikeinterface/qualitymetrics/misc_metrics.py +++ b/src/spikeinterface/qualitymetrics/misc_metrics.py @@ -805,17 +805,12 @@ def compute_amplitude_cv_metrics( def _get_amplitudes_by_units(sorting_analyzer, unit_ids, peak_sign): # used by compute_amplitude_cutoffs and compute_amplitude_medians - amplitudes_by_units = {} - if sorting_analyzer.has_extension("spike_amplitudes"): - spikes = sorting_analyzer.sorting.to_spike_vector() - ext = sorting_analyzer.get_extension("spike_amplitudes") - all_amplitudes = ext.get_data() - for unit_id in unit_ids: - unit_index = sorting_analyzer.sorting.id_to_index(unit_id) - spike_mask = spikes["unit_index"] == unit_index - amplitudes_by_units[unit_id] = all_amplitudes[spike_mask] + + if (spike_amplitudes_extension := sorting_analyzer.get_extension("spike_amplitudes")) is not None: + return spike_amplitudes_extension.get_data(outputs="by_unit", concatenated=True) elif sorting_analyzer.has_extension("waveforms"): + amplitudes_by_units = {} waveforms_ext = sorting_analyzer.get_extension("waveforms") before = waveforms_ext.nbefore extremum_channels_ids = get_template_extremum_channel(sorting_analyzer, peak_sign=peak_sign) From c5bdfc2443a9c6345fec36262eb203045630ac41 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 4 Jul 2025 16:09:15 +0200 Subject: [PATCH 093/157] Add merging_mode input to are_units_mergeable --- src/spikeinterface/core/sortinganalyzer.py | 24 ++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index f1b704911d..0b4dbfdf28 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -960,6 +960,7 @@ def _save_or_select_or_merge( mergeable, masks = self.are_units_mergeable( merge_unit_groups, sparsity_overlap=sparsity_overlap, + merging_mode=merging_mode, return_masks=True, ) @@ -967,21 +968,14 @@ def _save_or_select_or_merge( if unit_id in new_unit_ids: merge_unit_group = tuple(merge_unit_groups[new_unit_ids.index(unit_id)]) if not mergeable[merge_unit_group]: - # if someone wants soft mode we error if they try to use an inappropriate sparsity overlap - # if hard mode we can still use a sparsity if all units have good sparsity overlap - # but if any merge_unit_group doesn't have an overlap then we have to not use the old - # sparsity and instead use a new sparsity - if merging_mode == "soft": - raise Exception( - f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " - f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold or use " - "a hard merge." - ) - else: - sparsity = None - break - else: - sparsity_mask[unit_index] = masks[merge_unit_group] + # unit groups can be not mergeable only in "soft" mode + # see are_units_mergeable() function + raise Exception( + f"The sparsity of {merge_unit_group} do not overlap enough for a soft merge using " + f"a sparsity threshold of {sparsity_overlap}. You can either lower the threshold " + "or use a hard merge." + ) + sparsity_mask[unit_index] = masks[merge_unit_group] else: # This means that the unit is already in the previous sorting index = self.sorting.id_to_index(unit_id) From 96a9f2b9283d3b989fb911f34bb28ecb1c341325 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Fri, 4 Jul 2025 10:26:54 -0400 Subject: [PATCH 094/157] Fix NeuroNexus stream_id --- src/spikeinterface/extractors/tests/test_neoextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 622469ca51..37bea41979 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -219,7 +219,7 @@ class NeuroNexusRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = NeuroNexusRecordingExtractor downloads = ["neuronexus"] entities = [ - ("neuronexus/allego_1/allego_2__uid0701-13-04-49.xdat.json", {"stream_id": "0"}), + ("neuronexus/allego_1/allego_2__uid0701-13-04-49.xdat.json", {"stream_id": "ai-pri"}), ] From d058b54ae58685e8955c800aea9b6f25cc101343 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 4 Jul 2025 15:19:43 -0500 Subject: [PATCH 095/157] fix: update old argname `output_folder` --- src/spikeinterface/sorters/runsorter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index 408bec65c2..2799222e28 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -451,7 +451,7 @@ def run_sorter_container( # run in container output_folder = '{output_folder_unix}' sorting = run_sorter_local( - '{sorter_name}', recording, output_folder=output_folder, + '{sorter_name}', recording, folder=output_folder, remove_existing_folder={remove_existing_folder}, delete_output_folder=False, verbose={verbose}, raise_error={raise_error}, with_output=True, **sorter_params ) From 13a36acecca09d782bcd115cd68cead01bf718c5 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 5 Jul 2025 15:39:11 -0400 Subject: [PATCH 096/157] more deprecations --- .../benchmark/benchmark_plot_tools.py | 1 + src/spikeinterface/comparison/multicomparisons.py | 2 ++ src/spikeinterface/core/loading.py | 2 +- ...waveforms_extractor_backwards_compatibility.py | 2 +- src/spikeinterface/curation/auto_merge.py | 3 ++- src/spikeinterface/extractors/cbin_ibl.py | 2 +- .../extractors/neoextractors/openephys.py | 7 ++++--- src/spikeinterface/extractors/nwbextractors.py | 15 --------------- src/spikeinterface/sorters/launcher.py | 11 ----------- .../sortingcomponents/peak_pipeline.py | 8 ++++++-- src/spikeinterface/widgets/sorting_summary.py | 2 +- 11 files changed, 19 insertions(+), 36 deletions(-) diff --git a/src/spikeinterface/benchmark/benchmark_plot_tools.py b/src/spikeinterface/benchmark/benchmark_plot_tools.py index 9370bcc0bb..d07be05838 100644 --- a/src/spikeinterface/benchmark/benchmark_plot_tools.py +++ b/src/spikeinterface/benchmark/benchmark_plot_tools.py @@ -374,6 +374,7 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None, axs=None): return fig +# what's the dperecation strategy for this function in general? def plot_performances(study, mode="ordered", performance_names=("accuracy", "precision", "recall"), case_keys=None): """ Plot performances over case for a study. diff --git a/src/spikeinterface/comparison/multicomparisons.py b/src/spikeinterface/comparison/multicomparisons.py index 388e9e4b6f..c93ba63c98 100644 --- a/src/spikeinterface/comparison/multicomparisons.py +++ b/src/spikeinterface/comparison/multicomparisons.py @@ -190,6 +190,7 @@ def get_agreement_sorting(self, minimum_agreement_count=1, minimum_agreement_cou ) return sorting + # strategy for this dep? def save_to_folder(self, save_folder): warnings.warn( "save_to_folder() is deprecated. " @@ -221,6 +222,7 @@ def save_to_folder(self, save_folder): with (save_folder / "sortings.json").open("w") as f: json.dump(sortings, f) + # and here too. Strategy for this dep? @staticmethod def load_from_folder(folder_path): warnings.warn( diff --git a/src/spikeinterface/core/loading.py b/src/spikeinterface/core/loading.py index 97f104d08f..09e82e1026 100644 --- a/src/spikeinterface/core/loading.py +++ b/src/spikeinterface/core/loading.py @@ -130,7 +130,7 @@ def load( def load_extractor(file_or_folder_or_dict, base_folder=None) -> "BaseExtractor": warnings.warn( - "load_extractor() is deprecated and will be removed in the future. Please use load() instead.", + "load_extractor() is deprecated and will be removed in version 0.104.0. Please use load() instead.", DeprecationWarning, stacklevel=2, ) diff --git a/src/spikeinterface/core/waveforms_extractor_backwards_compatibility.py b/src/spikeinterface/core/waveforms_extractor_backwards_compatibility.py index d5697fabcb..70c1cf55e3 100644 --- a/src/spikeinterface/core/waveforms_extractor_backwards_compatibility.py +++ b/src/spikeinterface/core/waveforms_extractor_backwards_compatibility.py @@ -30,7 +30,7 @@ # extract_waveforms() and WaveformExtractor() have been replaced by the `SortingAnalyzer` since version 0.101.0. # You should use `spikeinterface.create_sorting_analyzer()` instead. # `spikeinterface.extract_waveforms()` is now mocking the old behavior for backwards compatibility only, -# and will be removed with version 0.103.0 +# and may potentially be removed in a future version. ####""" diff --git a/src/spikeinterface/curation/auto_merge.py b/src/spikeinterface/curation/auto_merge.py index 8447728216..f9ace831d8 100644 --- a/src/spikeinterface/curation/auto_merge.py +++ b/src/spikeinterface/curation/auto_merge.py @@ -635,8 +635,9 @@ def get_potential_auto_merge( done by Aurelien Wyngaard and Victor Llobet. https://github.com/BarbourLab/lussac/blob/v1.0.0/postprocessing/merge_units.py """ + # deprecation moved to 0.105.0 for @zm711 warnings.warn( - "get_potential_auto_merge() is deprecated. Use compute_merge_unit_groups() instead", + "get_potential_auto_merge() is deprecated and will be removed in version 0.105.0. Use compute_merge_unit_groups() instead", DeprecationWarning, stacklevel=2, ) diff --git a/src/spikeinterface/extractors/cbin_ibl.py b/src/spikeinterface/extractors/cbin_ibl.py index 728d352973..c70c49e8f8 100644 --- a/src/spikeinterface/extractors/cbin_ibl.py +++ b/src/spikeinterface/extractors/cbin_ibl.py @@ -55,7 +55,7 @@ def __init__( raise ImportError(self.installation_mesg) if cbin_file is not None: warnings.warn( - "The `cbin_file` argument is deprecated, please use `cbin_file_path` instead", + "The `cbin_file` argument is deprecated and will be removed in version 0.104.0, please use `cbin_file_path` instead", DeprecationWarning, stacklevel=2, ) diff --git a/src/spikeinterface/extractors/neoextractors/openephys.py b/src/spikeinterface/extractors/neoextractors/openephys.py index 0db3cd4426..8fede18e00 100644 --- a/src/spikeinterface/extractors/neoextractors/openephys.py +++ b/src/spikeinterface/extractors/neoextractors/openephys.py @@ -80,8 +80,9 @@ def __init__( ignore_timestamps_errors: bool = None, ): if ignore_timestamps_errors is not None: + dep_msg = "OpenEphysLegacyRecordingExtractor: `ignore_timestamps_errors` is deprecated. It will be removed in version 0.104.0 and is currently ignored" warnings.warn( - "OpenEphysLegacyRecordingExtractor: ignore_timestamps_errors is deprecated and is ignored", + dep_msg, DeprecationWarning, stacklevel=2, ) @@ -161,8 +162,8 @@ def __init__( if load_sync_channel: warning_message = ( - "OpenEphysBinaryRecordingExtractor: load_sync_channel is deprecated and will" - "be removed in version 0.104, use the stream_name or stream_id to load the sync stream if needed" + "OpenEphysBinaryRecordingExtractor: `load_sync_channel` is deprecated and will" + "be removed in version 0.104, use the `stream_name` or `stream_id` to load the sync stream if needed" ) warnings.warn(warning_message, DeprecationWarning, stacklevel=2) diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 8006eb4d7f..65531f7236 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -442,8 +442,6 @@ class NwbRecordingExtractor(BaseRecording, _BaseNWBExtractor): file_path : str, Path, or None Path to the NWB file or an s3 URL. Use this parameter to specify the file location if not using the `file` parameter. - electrical_series_name : str or None, default: None - Deprecated, use `electrical_series_path` instead. electrical_series_path : str or None, default: None The name of the ElectricalSeries object within the NWB file. This parameter is crucial when the NWB file contains multiple ElectricalSeries objects. It helps in identifying @@ -511,7 +509,6 @@ class NwbRecordingExtractor(BaseRecording, _BaseNWBExtractor): def __init__( self, file_path: str | Path | None = None, # provide either this or file - electrical_series_name: str | None = None, # deprecated load_time_vector: bool = False, samples_for_rate_estimation: int = 1_000, stream_mode: Optional[Literal["fsspec", "remfile", "zarr"]] = None, @@ -530,18 +527,6 @@ def __init__( if file_path is None and file is None: raise ValueError("Provide either file_path or file") - if electrical_series_name is not None: - warning_msg = ( - "The `electrical_series_name` parameter is deprecated and will be removed in version 0.101.0.\n" - "Use `electrical_series_path` instead." - ) - if electrical_series_path is None: - warning_msg += f"\nSetting `electrical_series_path` to 'acquisition/{electrical_series_name}'." - electrical_series_path = f"acquisition/{electrical_series_name}" - else: - warning_msg += f"\nIgnoring `electrical_series_name` and using the provided `electrical_series_path`." - warnings.warn(warning_msg, DeprecationWarning, stacklevel=2) - self.file_path = file_path self.stream_mode = stream_mode self.stream_cache_path = stream_cache_path diff --git a/src/spikeinterface/sorters/launcher.py b/src/spikeinterface/sorters/launcher.py index a6b049c182..122c5d4908 100644 --- a/src/spikeinterface/sorters/launcher.py +++ b/src/spikeinterface/sorters/launcher.py @@ -233,7 +233,6 @@ def run_sorter_by_property( recording, grouping_property, folder, - mode_if_folder_exists=None, engine="loop", engine_kwargs=None, verbose=False, @@ -260,10 +259,6 @@ def run_sorter_by_property( Property to split by before sorting folder : str | Path The working directory. - mode_if_folder_exists : bool or None, default: None - Must be None. This is deprecated. - If not None then a warning is raise. - Will be removed in next release. engine : "loop" | "joblib" | "dask" | "slurm", default: "loop" Which engine to use to run sorter. engine_kwargs : dict @@ -293,12 +288,6 @@ def run_sorter_by_property( engine_kwargs={"n_jobs": 4}) """ - if mode_if_folder_exists is not None: - warnings.warn( - "run_sorter_by_property(): mode_if_folder_exists is not used anymore and will be removed in 0.102.0", - DeprecationWarning, - stacklevel=2, - ) working_folder = Path(folder).absolute() diff --git a/src/spikeinterface/sortingcomponents/peak_pipeline.py b/src/spikeinterface/sortingcomponents/peak_pipeline.py index d9ae76f283..a0b2b263c7 100644 --- a/src/spikeinterface/sortingcomponents/peak_pipeline.py +++ b/src/spikeinterface/sortingcomponents/peak_pipeline.py @@ -16,10 +16,14 @@ def run_peak_pipeline( folder=None, names=None, ): - # TODO remove this soon + # TODO remove this soon. Will be removed in 0.104.0 import warnings - warnings.warn("run_peak_pipeline() is deprecated use run_node_pipeline() instead", DeprecationWarning, stacklevel=2) + warnings.warn( + "run_peak_pipeline() is deprecated and will be removed in version 0.104.0, `use run_node_pipeline()` instead", + DeprecationWarning, + stacklevel=2, + ) node0 = PeakRetriever(recording, peaks) # because nodes are modified inplace (insert parent) they need to copy incase diff --git a/src/spikeinterface/widgets/sorting_summary.py b/src/spikeinterface/widgets/sorting_summary.py index 67322398fb..b021f98563 100644 --- a/src/spikeinterface/widgets/sorting_summary.py +++ b/src/spikeinterface/widgets/sorting_summary.py @@ -88,7 +88,7 @@ def __init__( if unit_table_properties is not None: warnings.warn( - "plot_sorting_summary() : unit_table_properties is deprecated, use displayed_unit_properties instead", + "plot_sorting_summary() : `unit_table_properties` is deprecated and will be removed in version 0.104.0, use `displayed_unit_properties` instead", category=DeprecationWarning, stacklevel=2, ) From 032bdb558c9c0f98d8c804b48e801f22f93ee1e1 Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Sun, 6 Jul 2025 22:50:24 +0100 Subject: [PATCH 097/157] address Chris' comments --- src/spikeinterface/widgets/amplitudes.py | 2 +- src/spikeinterface/widgets/motion.py | 1 - src/spikeinterface/widgets/rasters.py | 5 +---- src/spikeinterface/widgets/tests/test_widgets_utils.py | 7 +++++-- src/spikeinterface/widgets/utils.py | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/widgets/amplitudes.py b/src/spikeinterface/widgets/amplitudes.py index 1f4c1eab47..c3aeb221ab 100644 --- a/src/spikeinterface/widgets/amplitudes.py +++ b/src/spikeinterface/widgets/amplitudes.py @@ -142,7 +142,7 @@ def __init__( bins = 100 # Calculate durations for all segments for x-axis limits - durations = get_segment_durations(sorting) + durations = get_segment_durations(sorting, segment_indices) # Build the plot data with the full dict of dicts structure plot_data = dict( diff --git a/src/spikeinterface/widgets/motion.py b/src/spikeinterface/widgets/motion.py index eac1b2713c..3c08217300 100644 --- a/src/spikeinterface/widgets/motion.py +++ b/src/spikeinterface/widgets/motion.py @@ -6,7 +6,6 @@ from spikeinterface.core import BaseRecording, SortingAnalyzer from .rasters import BaseRasterWidget -from .utils import get_segment_durations from spikeinterface.core.motion import Motion diff --git a/src/spikeinterface/widgets/rasters.py b/src/spikeinterface/widgets/rasters.py index 55032b38eb..757401d77c 100644 --- a/src/spikeinterface/widgets/rasters.py +++ b/src/spikeinterface/widgets/rasters.py @@ -425,11 +425,8 @@ def __init__( # Create a lookup dictionary for unit indices unit_indices_map = {unit_id: i for i, unit_id in enumerate(unit_ids)} - # Get all spikes at once - spikes = sorting.to_spike_vector() - # Estimate segment duration from max spike time in each segment - durations = get_segment_durations(sorting) + durations = get_segment_durations(sorting, segment_indices) # Extract spike data for all segments and units at once spike_train_data = {seg_idx: {} for seg_idx in segment_indices} diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index ff4bfd957c..dfb611119d 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -1,3 +1,4 @@ +from matplotlib.lines import segment_hits import pytest from spikeinterface import generate_sorting @@ -63,8 +64,10 @@ def test_get_segment_durations(): firing_rates=15.0, ) + segment_indices = list(range(sorting.get_num_segments())) + # Calculate durations - calculated_durations = get_segment_durations(sorting) + calculated_durations = get_segment_durations(sorting, segment_indices) # Check results assert len(calculated_durations) == len(durations) @@ -82,7 +85,7 @@ def test_get_segment_durations(): firing_rates=15.0, ) - single_duration = get_segment_durations(sorting_single)[0] + single_duration = get_segment_durations(sorting_single, [0])[0] # Test that the calculated duration is reasonable assert single_duration <= 7.0 diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index a003742939..3adcc05636 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -4,6 +4,7 @@ import numpy as np from spikeinterface.core import BaseSorting +from traitlets import Int def get_some_colors( @@ -398,7 +399,7 @@ def validate_segment_indices(segment_indices: list[int] | None, sorting: BaseSor return segment_indices -def get_segment_durations(sorting: BaseSorting) -> list[float]: +def get_segment_durations(sorting: BaseSorting, segment_indices: list[int]) -> list[float]: """ Calculate the duration of each segment in a sorting object. @@ -413,7 +414,6 @@ def get_segment_durations(sorting: BaseSorting) -> list[float]: List of segment durations in seconds """ spikes = sorting.to_spike_vector() - segment_indices = np.unique(spikes["segment_index"]) segment_boundaries = [ np.searchsorted(spikes["segment_index"], [seg_idx, seg_idx + 1]) for seg_idx in segment_indices From d889d41062cd6415335435e8d34e1621d2b3d276 Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Sun, 6 Jul 2025 22:56:33 +0100 Subject: [PATCH 098/157] oops --- src/spikeinterface/widgets/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 3adcc05636..26b5ec2e59 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -4,8 +4,6 @@ import numpy as np from spikeinterface.core import BaseSorting -from traitlets import Int - def get_some_colors( keys, From 42f509731295d6ea340d2bf86624aac0f70b04b6 Mon Sep 17 00:00:00 2001 From: Jake Swann Date: Sun, 6 Jul 2025 23:00:28 +0100 Subject: [PATCH 099/157] remove unused import --- src/spikeinterface/widgets/tests/test_widgets_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/spikeinterface/widgets/tests/test_widgets_utils.py b/src/spikeinterface/widgets/tests/test_widgets_utils.py index dfb611119d..3adf31c189 100644 --- a/src/spikeinterface/widgets/tests/test_widgets_utils.py +++ b/src/spikeinterface/widgets/tests/test_widgets_utils.py @@ -1,4 +1,3 @@ -from matplotlib.lines import segment_hits import pytest from spikeinterface import generate_sorting From c172ce163ed09d559aae5ce30a3230ac79dedf3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:02:05 +0000 Subject: [PATCH 100/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/widgets/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spikeinterface/widgets/utils.py b/src/spikeinterface/widgets/utils.py index 26b5ec2e59..75fb74cfae 100644 --- a/src/spikeinterface/widgets/utils.py +++ b/src/spikeinterface/widgets/utils.py @@ -5,6 +5,7 @@ from spikeinterface.core import BaseSorting + def get_some_colors( keys, color_engine="auto", From ca0d67e98dee7d6c8ae12e11088c4403236a5faf Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 7 Jul 2025 10:29:54 +0200 Subject: [PATCH 101/157] Update on uv installation --- installation_tips/README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index e86185b503..e0728de5f7 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -12,11 +12,11 @@ The main ideas you need to know before starting: Choosing the installer + an environment manager + a package installer is a nightmare for beginners. The main options are: * use "uv" : a new, fast and simple package manager. We recommend this for beginners on every os. - * use "anaconda", which does everything. The most popular but theses days it is becoming - a bad idea because : ultra slow by default and aggressive licensing by default (not always free anymore). + * use "anaconda", which does everything. Used to be very popular but theses days it is becoming + a bad idea because : slow by default and aggressive licensing on the default channel (not always free anymore). You need to play with "community channels" to make it free again, which is too complicated for beginners. Do not go this way. - * use python from the system or python.org + venv + pip : good idea for linux users. + * use python from the system or python.org + venv + pip : good and simple idea for linux users. Here we propose a step by step recipe for beginers based on **"uv"**. We used to recommend installing with anaconda. It will be kept here for a while but we do not recommend it anymore. @@ -33,26 +33,30 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so 1. On macOS and Linux. Open a terminal and do `$ curl -LsSf https://astral.sh/uv/install.sh | sh` -1. On windows. Open a powershell and do +1. On windows. Open a terminal using with CMD `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` 2. exit session and log again. 3. Download with right click and save this file corresponding in "Documents" folder: * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell 5. `uv venv si_env --python 3.11` -6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) or for Powershell `si_env\Scripts\activate` -7. `uv pip install -r Documents/requirements_stable.txt` +6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) +6. For windows `si_env\Scripts\activate` +7. `uv pip install -r Documents/beginner_requirements_stable.txt` or `uv pip install -r Documents/beginner_requirements_rolling.txt` -More detail on [uv here](https://github.com/astral-sh/uv). +More details on [uv here](https://github.com/astral-sh/uv). + ## Installing before release Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, python-neo, sortingview). -We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `requirements_rolling.txt` file to create the environment. This will install the packages of the ecosystem from source. +We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `beginner_requirements_rolling.txt` file to create the environment. This will install the packages of the ecosystem from source. This is a good way to test if a patch fixes your issue. + + ### Check the installation From 2c9d81f17395c5b354eb1a47dc19aa5036d7e0f3 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 7 Jul 2025 10:35:52 +0200 Subject: [PATCH 102/157] yep --- installation_tips/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index e0728de5f7..21696bbd05 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -32,14 +32,14 @@ Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from so ### Quick installation using "uv" (recommended) 1. On macOS and Linux. Open a terminal and do - `$ curl -LsSf https://astral.sh/uv/install.sh | sh` + `curl -LsSf https://astral.sh/uv/install.sh | sh` 1. On windows. Open a terminal using with CMD `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` 2. exit session and log again. 3. Download with right click and save this file corresponding in "Documents" folder: * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) 4. open terminal or powershell -5. `uv venv si_env --python 3.11` +5. `uv venv si_env --python 3.12` 6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) 6. For windows `si_env\Scripts\activate` 7. `uv pip install -r Documents/beginner_requirements_stable.txt` or `uv pip install -r Documents/beginner_requirements_rolling.txt` From a327c0725cf7d9134b11523449c17a0f2c3e45f1 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Mon, 7 Jul 2025 08:45:54 -0400 Subject: [PATCH 103/157] remove deprecation test from nwb tests --- .../extractors/tests/test_nwbextractors.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/spikeinterface/extractors/tests/test_nwbextractors.py b/src/spikeinterface/extractors/tests/test_nwbextractors.py index 15d3e8fee9..1bc5cf9a13 100644 --- a/src/spikeinterface/extractors/tests/test_nwbextractors.py +++ b/src/spikeinterface/extractors/tests/test_nwbextractors.py @@ -207,24 +207,6 @@ def test_nwb_extractor_channel_ids_retrieval(generate_nwbfile, use_pynwb): assert np.array_equal(extracted_channel_ids, expected_channel_ids) -@pytest.mark.parametrize("use_pynwb", [True, False]) -def test_electrical_series_name_backcompatibility(generate_nwbfile, use_pynwb): - """ - Test that the channel_ids are retrieved from the electrodes table ONLY from the corresponding - region of the electrical series - """ - path_to_nwbfile, nwbfile_with_ecephys_content = generate_nwbfile - electrical_series_name_list = ["ElectricalSeries1", "ElectricalSeries2"] - for electrical_series_name in electrical_series_name_list: - with pytest.deprecated_call(): - recording_extractor = NwbRecordingExtractor( - path_to_nwbfile, - electrical_series_name=electrical_series_name, - use_pynwb=use_pynwb, - ) - assert recording_extractor.electrical_series_path == f"acquisition/{electrical_series_name}" - - @pytest.mark.parametrize("use_pynwb", [True, False]) def test_nwb_extractor_property_retrieval(generate_nwbfile, use_pynwb): """ From 0c2f71318c044c519cd4114a0d1605708c0a5213 Mon Sep 17 00:00:00 2001 From: Pierre Yger Date: Tue, 8 Jul 2025 16:03:44 +0200 Subject: [PATCH 104/157] Shared memory for the overlapps while matching in circus and wobble --- .../sortingcomponents/matching/base.py | 8 ++++ .../sortingcomponents/matching/circus.py | 48 +++++++++++++++++-- .../sortingcomponents/matching/main.py | 5 ++ .../sortingcomponents/matching/wobble.py | 43 ++++++++++++++++- 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/sortingcomponents/matching/base.py b/src/spikeinterface/sortingcomponents/matching/base.py index 0e60a9e864..2dc93b52aa 100644 --- a/src/spikeinterface/sortingcomponents/matching/base.py +++ b/src/spikeinterface/sortingcomponents/matching/base.py @@ -43,6 +43,14 @@ def compute(self, traces, start_frame, end_frame, segment_index, max_margin): def compute_matching(self, traces, start_frame, end_frame, segment_index): raise NotImplementedError + def clean(self): + """ + Clean the matching output. + This is called at the end of the matching process. + """ + # can be overwritten if needed + pass + def get_extra_outputs(self): # can be overwritten if need to ouput some variables with a dict return None diff --git a/src/spikeinterface/sortingcomponents/matching/circus.py b/src/spikeinterface/sortingcomponents/matching/circus.py index faf73465ff..6bbcb77136 100644 --- a/src/spikeinterface/sortingcomponents/matching/circus.py +++ b/src/spikeinterface/sortingcomponents/matching/circus.py @@ -137,6 +137,9 @@ class CircusOMPSVDPeeler(BaseTemplateMatching): The engine to use for the convolutions torch_device : string in ["cpu", "cuda", None]. Default "cpu" Controls torch device if the torch engine is selected + shared_memory : bool, default True + If True, the overlaps are stored in shared memory, which is more efficient when + using numerous cores ----- """ @@ -167,6 +170,7 @@ def __init__( vicinity=2, precomputed=None, engine="numpy", + shared_memory=True, torch_device="cpu", ): @@ -176,6 +180,7 @@ def __init__( self.num_samples = templates.num_samples self.nbefore = templates.nbefore self.nafter = templates.nafter + self.shared_memory = shared_memory self.sampling_frequency = recording.get_sampling_frequency() self.vicinity = vicinity * self.num_samples assert engine in ["numpy", "torch", "auto"], "engine should be numpy, torch or auto" @@ -208,6 +213,18 @@ def __init__( assert precomputed[key] is not None, "If templates are provided, %d should also be there" % key setattr(self, key, precomputed[key]) + if self.shared_memory: + self.max_overlaps = max([len(o) for o in self.overlaps]) + num_samples = len(self.overlaps[0][0]) + from spikeinterface.core.core_tools import make_shared_array + + arr, shm = make_shared_array((self.num_templates, self.max_overlaps, num_samples), dtype=np.float32) + for i in range(self.num_templates): + n_overlaps = len(self.unit_overlaps_indices[i]) + arr[i, :n_overlaps] = self.overlaps[i] + self.overlaps = arr + self.shm = shm + self.ignore_inds = np.array(ignore_inds) self.unit_overlaps_tables = {} @@ -299,7 +316,10 @@ def _push_to_torch(self): def get_extra_outputs(self): output = {} for key in self._more_output_keys: - output[key] = getattr(self, key) + if key == "overlaps" and self.shared_memory: + output[key] = self.overlaps.copy() + else: + output[key] = getattr(self, key) return output def get_trace_margin(self): @@ -409,7 +429,12 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index): myline = neighbor_window + delta_t[idx] myindices = selection[0, idx] - local_overlaps = self.overlaps[best_cluster_ind] + if self.shared_memory: + n_overlaps = len(self.unit_overlaps_indices[best_cluster_ind]) + local_overlaps = self.overlaps[best_cluster_ind, :n_overlaps] + else: + local_overlaps = self.overlaps[best_cluster_ind] + overlapping_templates = self.unit_overlaps_indices[best_cluster_ind] table = self.unit_overlaps_tables[best_cluster_ind] @@ -487,7 +512,11 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index): for i in modified: tmp_best, tmp_peak = sub_selection[:, i] diff_amp = diff_amplitudes[i] * self.norms[tmp_best] - local_overlaps = self.overlaps[tmp_best] + if self.shared_memory: + n_overlaps = len(self.unit_overlaps_indices[tmp_best]) + local_overlaps = self.overlaps[tmp_best, :n_overlaps] + else: + local_overlaps = self.overlaps[tmp_best] overlapping_templates = self.units_overlaps[tmp_best] tmp = tmp_peak - neighbor_window idx = [max(0, tmp), min(num_peaks, tmp_peak + self.num_samples)] @@ -532,6 +561,19 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index): return spikes + def clean(self): + if self.shared_memory and self.shm is not None: + self.overlaps = None + self.shm.close() + self.shm.unlink() + self.shm = None + + def __del__(self): + if self.shared_memory and self.shm is not None: + self.overlaps = None + self.shm.close() + self.shm = None + class CircusPeeler(BaseTemplateMatching): """ diff --git a/src/spikeinterface/sortingcomponents/matching/main.py b/src/spikeinterface/sortingcomponents/matching/main.py index 5895dc433a..6d88396159 100644 --- a/src/spikeinterface/sortingcomponents/matching/main.py +++ b/src/spikeinterface/sortingcomponents/matching/main.py @@ -58,8 +58,13 @@ def find_spikes_from_templates( gather_mode="memory", squeeze_output=True, ) + if extra_outputs: outputs = node0.get_extra_outputs() + + node0.clean() + + if extra_outputs: return spikes, outputs else: return spikes diff --git a/src/spikeinterface/sortingcomponents/matching/wobble.py b/src/spikeinterface/sortingcomponents/matching/wobble.py index 59e171fe52..a0c926cafa 100644 --- a/src/spikeinterface/sortingcomponents/matching/wobble.py +++ b/src/spikeinterface/sortingcomponents/matching/wobble.py @@ -54,6 +54,9 @@ class WobbleParameters: The engine to use for the convolutions torch_device : string in ["cpu", "cuda", None]. Default "cpu" Controls torch device if the torch engine is selected + shared_memory : bool, default True + If True, the overlaps are stored in shared memory, which is more efficient when + using numerous cores Notes ----- @@ -77,6 +80,7 @@ class WobbleParameters: scale_amplitudes: bool = False engine: str = "numpy" torch_device: str = "cpu" + shared_memory: bool = True def __post_init__(self): assert self.amplitude_variance >= 0, "amplitude_variance must be a non-negative scalar" @@ -361,6 +365,7 @@ def __init__( parameters={}, engine="numpy", torch_device="cpu", + shared_memory=True, ): BaseTemplateMatching.__init__(self, recording, templates, return_output=True, parents=None) @@ -399,6 +404,24 @@ def __init__( compressed_templates, params.jitter_factor, params.approx_rank, template_meta.jittered_indices, sparsity ) + self.shared_memory = shared_memory + + if self.shared_memory: + self.max_overlaps = max([len(o) for o in pairwise_convolution]) + num_samples = len(pairwise_convolution[0][0]) + num_templates = len(templates_array) + num_jittered = num_templates * params.jitter_factor + from spikeinterface.core.core_tools import make_shared_array + + arr, shm = make_shared_array((num_jittered, self.max_overlaps, num_samples), dtype=np.float32) + for jittered_index in range(num_jittered): + units_are_overlapping = sparsity.unit_overlap[jittered_index, :] + overlapping_units = np.where(units_are_overlapping)[0] + n_overlaps = len(overlapping_units) + arr[jittered_index, :n_overlaps] = pairwise_convolution[jittered_index] + pairwise_convolution = [arr] + self.shm = shm + norm_squared = compute_template_norm(sparsity.visible_channels, templates_array) spatial = np.moveaxis(spatial, [0, 1, 2], [1, 0, 2]) @@ -424,6 +447,19 @@ def __init__( # self.margin = int(buffer_ms*1e-3 * recording.sampling_frequency) self.margin = 300 # To ensure equivalence with spike-psvae version of the algorithm + def clean(self): + if self.shared_memory and self.shm is not None: + self.template_meta = None + self.shm.close() + self.shm.unlink() + self.shm = None + + def __del__(self): + if self.shared_memory and self.shm is not None: + self.template_meta = None + self.shm.close() + self.shm = None + def _push_to_torch(self): if self.engine == "torch": temporal, singular, spatial, temporal_jittered = self.template_data.compressed_templates @@ -644,7 +680,12 @@ def subtract_spike_train( id_scaling = scalings[id_mask] overlapping_templates = sparsity.unit_overlap[jittered_index] # Note: pairwise_conv only has overlapping template convolutions already - pconv = template_data.pairwise_convolution[jittered_index] + if params.shared_memory: + overlapping_units = np.where(overlapping_templates)[0] + n_overlaps = len(overlapping_units) + pconv = template_data.pairwise_convolution[0][jittered_index, :n_overlaps] + else: + pconv = template_data.pairwise_convolution[jittered_index] # TODO: If optimizing for speed -- check this loop for spike_start_index, spike_scaling in zip(id_spiketrain, id_scaling): spike_stop_index = spike_start_index + convolution_resolution_len From 6583b8c84913c6c1a439066c23e5f1526cf5214f Mon Sep 17 00:00:00 2001 From: chrishalcrow Date: Wed, 9 Jul 2025 13:46:52 +0100 Subject: [PATCH 105/157] add lower bound to numba --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d1d492973d..9caf5a4784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ full = [ "distinctipy", "matplotlib>=3.6", # matplotlib.colormaps "cuda-python; platform_system != 'Darwin'", - "numba", + "numba>=0.59", "skops", "huggingface_hub" ] From 184d64a964ff7de98927959aba7751560433040c Mon Sep 17 00:00:00 2001 From: Pierre Yger Date: Wed, 9 Jul 2025 15:12:54 +0200 Subject: [PATCH 106/157] Fix --- src/spikeinterface/core/baserecording.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spikeinterface/core/baserecording.py b/src/spikeinterface/core/baserecording.py index ac299f52e7..ce95581be8 100644 --- a/src/spikeinterface/core/baserecording.py +++ b/src/spikeinterface/core/baserecording.py @@ -852,6 +852,7 @@ def binary_compatible_with( time_axis=None, file_paths_length=None, file_offset=None, + file_suffix=None, ): """ Check is the recording is binary compatible with some constrain on From 81256d6166de5203ece9ffa66a14b66152fca798 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:43:44 -0400 Subject: [PATCH 107/157] first attempt uv for docs --- readthedocs.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/readthedocs.yml b/readthedocs.yml index c6c44d83a0..dbeecee5fc 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,14 +1,19 @@ version: 2 -sphinx: - # Path to your Sphinx configuration file. - configuration: doc/conf.py - +# Specify os and python version build: - os: ubuntu-22.04 + os: "ubuntu-24.04" tools: - python: "mambaforge-4.10" + python: "3.12" + jobs: + create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras --group docs + install: + - "true" - -conda: - environment: docs_rtd.yml +sphinx: + # Path to your Sphinx configuration file. + configuration: doc/conf.py From 8bb3f4ed0f36e3ee5d24e3b2a4492624a5326e8d Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:50:12 -0400 Subject: [PATCH 108/157] try 2 --- readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yml b/readthedocs.yml index dbeecee5fc..59da5bb10e 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -10,7 +10,7 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras --group docs + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras --optional docs install: - "true" From a1d3cc6b60059f91c9f1fad295c25954f7e8262e Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:52:51 -0400 Subject: [PATCH 109/157] try optional without extras --- readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yml b/readthedocs.yml index 59da5bb10e..ec96997f01 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -10,7 +10,7 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras --optional docs + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --optional docs install: - "true" From ac4c93f945dfc6d736856cd36872a50548cf2ad9 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:59:02 -0400 Subject: [PATCH 110/157] try a different strategy --- readthedocs.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/readthedocs.yml b/readthedocs.yml index ec96997f01..804e06e72b 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -5,14 +5,13 @@ build: os: "ubuntu-24.04" tools: python: "3.12" - jobs: - create_environment: - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --optional docs - install: - - "true" + commands: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + - uv venv $READTHEDOCS_VIRTUALENV_PATH + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv --preview pip install .[docs] + - python -m sphinx -T -b html -d docs/_build/doctrees -D language=en docs/source $READTHEDOCS_OUTPUT/html sphinx: # Path to your Sphinx configuration file. From 1aababbcfeedb7aa1862005ae7a468f6521317e9 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 20:00:00 -0400 Subject: [PATCH 111/157] try on python 3.10 --- readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yml b/readthedocs.yml index 804e06e72b..fd3f42d3bf 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -4,7 +4,7 @@ version: 2 build: os: "ubuntu-24.04" tools: - python: "3.12" + python: "3.10" commands: - asdf plugin add uv - asdf install uv latest From 9cddea50f8ba5844a39021004b85f40033870c6f Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 20:07:39 -0400 Subject: [PATCH 112/157] try the normal argument --- readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yml b/readthedocs.yml index fd3f42d3bf..2bc48e4d7a 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -11,7 +11,7 @@ build: - asdf global uv latest - uv venv $READTHEDOCS_VIRTUALENV_PATH - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv --preview pip install .[docs] - - python -m sphinx -T -b html -d docs/_build/doctrees -D language=en docs/source $READTHEDOCS_OUTPUT/html + - python -m sphinx -T -b html -d _build/doctrees -D language=en . $READTHEDOCS_OUTPUT/html sphinx: # Path to your Sphinx configuration file. From a7c52f8e0fa5921f008ef04fb948baf7afdabb5f Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 20:09:31 -0400 Subject: [PATCH 113/157] more config --- readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs.yml b/readthedocs.yml index 2bc48e4d7a..486ace7d53 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -11,7 +11,7 @@ build: - asdf global uv latest - uv venv $READTHEDOCS_VIRTUALENV_PATH - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv --preview pip install .[docs] - - python -m sphinx -T -b html -d _build/doctrees -D language=en . $READTHEDOCS_OUTPUT/html + - python -m sphinx -T -b html -d doc/_build/doctrees -D language=en doc $READTHEDOCS_OUTPUT/html sphinx: # Path to your Sphinx configuration file. From efb041ef45d9447e886b9c266a8f0ed2892ea5ca Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 11 Jul 2025 20:35:18 -0400 Subject: [PATCH 114/157] remove conda env file --- docs_rtd.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 docs_rtd.yml diff --git a/docs_rtd.yml b/docs_rtd.yml deleted file mode 100644 index c4e1fb378c..0000000000 --- a/docs_rtd.yml +++ /dev/null @@ -1,9 +0,0 @@ -channels: - - conda-forge - - defaults -dependencies: - - python=3.10 - - pip - - datalad - - pip: - - -e .[docs] From 6050edb091e464920de010474e77ecff0ecc835f Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 09:37:12 -0400 Subject: [PATCH 115/157] remove setup.py --- setup.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 0f4aa42fb2..0000000000 --- a/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -import setuptools -import warnings - -warnings.warn("Using `python setup.py` is legacy! See https://spikeinterface.readthedocs.io/en/latest/installation.html for installation") - -if __name__ == "__main__": - setuptools.setup() From deca07f3ddc1f15cec2d3316c20287097994ef86 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 10:45:58 -0400 Subject: [PATCH 116/157] add additional citations --- doc/references.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/references.rst b/doc/references.rst index 6ba6efe6a1..1bf83d11ac 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -20,7 +20,7 @@ If you use one of the following preprocessing methods, please cite the appropria - :code:`common_reference` [Rolston]_ Motion Correction -^^^^^^^^^^^^^^^^^ +----------------- If you use the :code:`correct_motion` method in the preprocessing module, please cite [Garcia]_ as well as the references that correspond to the :code:`preset` you used: @@ -28,6 +28,7 @@ as well as the references that correspond to the :code:`preset` you used: - :code:`nonrigid_fast_and_accurate` [Windolf]_ [Varol]_ [Pachitariu]_ - :code:`rigid_fast` *no additional citation needed* - :code:`kilosort_like` [Pachitariu]_ +- :code:`medicine` [Watters]_ Sorters Module -------------- @@ -47,7 +48,14 @@ please include the appropriate citation for the :code:`sorter_name` parameter yo Postprocessing Module --------------------- -If you use the :code:`acgs_3d` extensions, (i.e. :code:`postprocessing.compute_acgs_3d`, :code:`postprocessing.ComputeACG3D`) please cite [Beau]_ +If you use the :code:`postprocessing module`, i.e. you use the :code:`analyzer.compute()` include the citations for the following +methods: + + - :code:`acgs_3d` [Beau]_ + - :code:`unit_locations` or :code:`spike_locations` with :code:`monopolar_triangulation` based on work from [Boussard]_ + - :code:`unit_locations` or :code:`spike_locations` with :code:`grid_convolution` based on work from [Pachitariu]_ + - :code:`template_metrics` [Jia]_ + Qualitymetrics Module --------------------- @@ -74,6 +82,7 @@ important for your research: - :code:`nearest_neighbor` or :code:`nn_isolation` or :code:`nn_noise_overlap` [Chung]_ [Siegle]_ - :code:`silhouette` [Rousseeuw]_ [Hruschka]_ + Curation Module --------------- If you use the :code:`get_potential_auto_merge` method from the curation module, please cite [Llobet]_ @@ -83,6 +92,8 @@ References .. [Beau] `A deep learning strategy to identify cell types across species from high-density extracellular recordings. 2025. `_ +.. [Boussard] `Three-dimensional spike localization and imporved motion correction for Neuropixels recordings. 2021 `_ + .. [Buccino] `SpikeInterface, a unified framework for spike sorting. 2020. `_ .. [Buzsáki] `The Log-Dynamic Brain: How Skewed Distributions Affect Network Operations. 2014. `_ @@ -109,6 +120,8 @@ References .. [Jackson] Quantitative assessment of extracellular multichannel recording quality using measures of cluster separation. Society of Neuroscience Abstract. 2005. +.. [Jia] `High-density extracellular probes reveal dendritic backpropagation and facilitate neuron classification. 2019 `_ + .. [Lee] `YASS: Yet another spike sorter. 2017. `_ .. [Lemon] Methods for neuronal recording in conscious animals. IBRO Handbook Series. 1984. @@ -137,6 +150,8 @@ References .. [Varol] `Decentralized Motion Inference and Registration of Neuropixel Data. 2021. `_ +.. [Watters] `MEDiCINe: Motion Correction for Neural Electrophysiology Recordings. 2025. `_ + .. [Windolf] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. `_ .. [Yger] `A spike sorting toolbox for up to thousands of electrodes validated with ground truth recordings in vitro and in vivo. 2018. `_ From 2626bb5e9301da59e1e78fd9f9c62aa52089fa2d Mon Sep 17 00:00:00 2001 From: Chris Halcrow <57948917+chrishalcrow@users.noreply.github.com> Date: Sat, 12 Jul 2025 16:19:19 +0100 Subject: [PATCH 117/157] Remove MEArec downloads from docs (#4051) * remove mearec from docs * oups * remove from plot_4_sorting_analyzer * remove packages from docs * readd matplotlib * PLOB * respond to zach * revert pyproject.toml * oups --- doc/how_to/read_various_formats.rst | 173 ++++++++++++++++++ .../read_various_formats_12_0.png | Bin 0 -> 55532 bytes .../read_various_formats_12_1.png | Bin 0 -> 13223 bytes doc/modules/comparison.rst | 11 +- doc/tutorials_custom_index.rst | 5 +- .../read_various_formats.py} | 0 .../tutorials/core/plot_4_sorting_analyzer.py | 17 +- .../qualitymetrics/plot_3_quality_metrics.py | 7 +- .../qualitymetrics/plot_4_curation.py | 11 +- .../widgets/plot_3_waveforms_gallery.py | 7 +- .../tutorials/widgets/plot_4_peaks_gallery.py | 21 +-- 11 files changed, 198 insertions(+), 54 deletions(-) create mode 100644 doc/how_to/read_various_formats.rst create mode 100644 doc/how_to/read_various_formats_files/read_various_formats_12_0.png create mode 100644 doc/how_to/read_various_formats_files/read_various_formats_12_1.png rename examples/{tutorials/extractors/plot_1_read_various_formats.py => how_to/read_various_formats.py} (100%) diff --git a/doc/how_to/read_various_formats.rst b/doc/how_to/read_various_formats.rst new file mode 100644 index 0000000000..1e8ee0d5bc --- /dev/null +++ b/doc/how_to/read_various_formats.rst @@ -0,0 +1,173 @@ +.. code:: ipython3 + + %matplotlib inline + +Read various format into SpikeInterface +======================================= + +SpikeInterface can read various formats of “recording” (traces) and +“sorting” (spike train) data. + +Internally, to read different formats, SpikeInterface either uses: \* a +wrapper to ``neo ``\ \_ +rawio classes \* or a direct implementation + +Note that: + +- file formats contain a “recording”, a “sorting”, or “both” +- file formats can be file-based (NWB, …) or folder based (SpikeGLX, + OpenEphys, …) + +In this example we demonstrate how to read different file formats into +SI + +.. code:: ipython3 + + import matplotlib.pyplot as plt + + import spikeinterface.core as si + import spikeinterface.extractors as se + +Let’s download some datasets in different formats from the +``ephy_testing_data ``\ \_ +repo: + +- MEArec: a simulator format which is hdf5-based. It contains both a + “recording” and a “sorting” in the same file. +- Spike2: file from spike2 devices. It contains “recording” information + only. + +.. code:: ipython3 + + + spike2_file_path = si.download_dataset(remote_path="spike2/130322-1LY.smr") + print(spike2_file_path) + + mearec_folder_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5") + print(mearec_folder_path) + + +.. parsed-literal:: + + Downloading data from 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data/raw/master/mearec/mearec_test_10s.h5' to file '/Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/mearec/mearec_test_10s.h5'. + + +.. parsed-literal:: + + modified: spike2/130322-1LY.smr (file) + 1 annex'd file (15.8 MB recorded total size) + /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/spike2/130322-1LY.smr + 1 annex'd file (59.4 MB recorded total size) + nothing to save, working tree clean + + +.. parsed-literal:: + + 100%|█████████████████████████████████████| 62.3M/62.3M [00:00<00:00, 76.7GB/s] + + +.. parsed-literal:: + + /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/mearec/mearec_test_10s.h5 + + +Now that we have downloaded the files, let’s load them into SI. + +The :py:func:``~spikeinterface.extractors.read_spike2`` function returns +one object, a :py:class:``~spikeinterface.core.BaseRecording``. + +Note that internally this file contains 2 data streams (‘0’ and ‘1’), so +we need to specify which one we want to retrieve (‘0’ in our case). the +stream information can be retrieved by using the +:py:func:``~spikeinterface.extractors.get_neo_streams`` function. + +.. code:: ipython3 + + stream_names, stream_ids = se.get_neo_streams("spike2", spike2_file_path) + print(stream_names) + print(stream_ids) + stream_id = stream_ids[0] + print("stream_id", stream_id) + + recording = se.read_spike2(spike2_file_path, stream_id="0") + print(recording) + print(type(recording)) + print(isinstance(recording, si.BaseRecording)) + + +.. parsed-literal:: + + ['Signal stream 0', 'Signal stream 1'] + ['0', '1'] + stream_id 0 + Spike2RecordingExtractor: 1 channels - 20833.333333 Hz - 1 segments - 4,126,365 samples + 198.07s (3.30 minutes) - int16 dtype - 7.87 MiB + file_path: /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/spike2/130322-1LY.smr + + True + + +The +:py:func::literal:`~spikeinterface.extractors.read_spike2`\` function is equivalent to instantiating a :py:class:`\ ~spikeinterface.extractors.Spike2RecordingExtractor\` +object: + +.. code:: ipython3 + + recording = se.read_spike2(spike2_file_path, stream_id="0") + print(recording) + + +.. parsed-literal:: + + Spike2RecordingExtractor: 1 channels - 20833.333333 Hz - 1 segments - 4,126,365 samples + 198.07s (3.30 minutes) - int16 dtype - 7.87 MiB + file_path: /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/spike2/130322-1LY.smr + + +The :py:func:``~spikeinterface.extractors.read_mearec`` function returns +two objects, a :py:class:``~spikeinterface.core.BaseRecording`` and a +:py:class:``~spikeinterface.core.BaseSorting``: + +.. code:: ipython3 + + recording, sorting = se.read_mearec(mearec_folder_path) + print(recording) + print(type(recording)) + print() + print(sorting) + print(type(sorting)) + + + +.. parsed-literal:: + + MEArecRecordingExtractor: 32 channels - 32.0kHz - 1 segments - 320,000 samples - 10.00s + float32 dtype - 39.06 MiB + file_path: /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/mearec/mearec_test_10s.h5 + + + MEArecSortingExtractor: 10 units - 1 segments - 32.0kHz + file_path: /Users/christopherhalcrow/spikeinterface_datasets/ephy_testing_data/mearec/mearec_test_10s.h5 + + + +SI objects (:py:class:``~spikeinterface.core.BaseRecording`` and +:py:class:``~spikeinterface.core.BaseSorting``) can be plotted quickly +with the :py:mod:``spikeinterface.widgets`` submodule: + +.. code:: ipython3 + + import spikeinterface.widgets as sw + + w_ts = sw.plot_traces(recording, time_range=(0, 5)) + w_rs = sw.plot_rasters(sorting, time_range=(0, 5)) + + plt.show() + + + +.. image:: read_various_formats_files/read_various_formats_12_0.png + + + +.. image:: read_various_formats_files/read_various_formats_12_1.png diff --git a/doc/how_to/read_various_formats_files/read_various_formats_12_0.png b/doc/how_to/read_various_formats_files/read_various_formats_12_0.png new file mode 100644 index 0000000000000000000000000000000000000000..58ea70c4fee3b34d4e44c384c6ed236e1c71f0f4 GIT binary patch literal 55532 zcmagGcQl<{*FH*g!j9-&h~7yE(YuIf5kwDyXwm!Dd+$W=5uyb_lqk`AkKS#)ZQbT9 z&-1?D@0>BdbIuX$+3};kZ={`rPYv-9-ts0Ay;6cgWo)} z{)i9$6LOT%a#Xkd=;&f(Z;GU3O-@e;|Tro9fObCAi2z+yG6fGD830RR7?7!K}PS|3J@k@m$2e)sUO?;U8iK?Dt6j z{)~p>K?;6_?w^y=PK^%!Z#87zs0c>>=dWRcyGFn79})Bc?|-?BnP?;b?)G}_B$o_euZWqs^ zKO_sS7&PfWgOL|7&5iD^7w<-yGX8ynh~=UzXq?!IJI&BoyH5kU>!bg^FKm*>^%opI z!~gGZ4U>rP{*UNV{7Z}8cc-_v*9U&Y^p+JnWxw1R6qEWL1kbqtd9-9KSBuxR^Fc5T zl-XRh3U9qP@r3nVX$liHtp_PHD$4JW%!gsxKR&v795nrOx!Nissb)r(YL=uyJzMnl zF1Pwy#n0t_cZXR1xd=UR-kpBZ!lv;pK@5WOS~BdLMu~fxfNj)`=x#-O_H8TBB;WkJC*RbbKchk)nB=Y#T=+gGlfUY;ANX4tC} z$LoE@nJ&wLB<>Cy{XLZj-Sp9L(c6pFI>c|bHoxCTT{H(&4*fg};att6ZXD-bG~Uut zG*08H3m7z>F%{`SxD@V+N$f}1DBX&MGuC;3ObKV)pbgT@w)SyLlv1_BaD0BO}Zar^T^ZS}lZniB5=TQgR+CH{3lUZuq zxtMx<1|0i^()QZU!i(y$MIq zJA?}t*Ko5D-a0|*>I508!={F&UZPr-kDezrhhz+K9pu6N*5zI!xsZD92vc#Z;FHFH zH15s*Ocu)=Ib4kv>s-gGvU+fTjmh)RSSdQRKZL6OyUkzTJh}kih#kX)f!ArG*vEd- zyENP8BLUlm-IntOkE*P_^F{CTmWvgfJFsGBEr_$(?psC9dg(GVu)7eOcc)VZRd$G4 z>)HGpu)1@22lGKK#xHaL}tSvrg5DXG=V2=yiRs7&*fsCB$E`a;@JE?uCdKY<#un@B^6vV$TDt9eH&4K> zCdha&^G4(wPFY^o+sSCrt8VZ0M9nqw#FM?Bi)UG0XWA@^JIn9q&422bd-c{jYz}sd zlDH$<0&!%#r-wR13FxPMTm3QQ9P{>!rCL7mC0yT}3~dTQ{vI^!aI%nOL(aaxuX(i_ zSalh7YaTztux>X1HU*u~yKFJ*v|406HR`ic_qW5*Y?@^W?s^{i)KT|mVHACPBmP$7 zC551U{@qql)~m&;7UQlc!Oq{H1TofPRbb5&?Y;BR5Il`{c%eC4Vbb$B)?jEezIpg0 zHkDtkaMA-j$u(Hjmb-CvWd%gF$KNVbx*|(1T@uP4*v(_m-lB(7XVs872My6Hy*-6< zKK_d+gdN8^+U$4!@+nwdp}yl@Qdi+#c4Ic@#emD;s(|xKZLH-%-7222$~j&y-8Z=tn|yRU=P)F*;bWM^n!-ly1y{WTg(Q9v z768q=tL750nvUrcHD^0H*RgG4a97VTHtvqRnNwsAKl`+Jq<6qG9#~qYWPHABES2n@ z@iOV|x@D1a^@gkEM$Iv&w%B>KR{gx#>vKkl?fI`khU!4&PX0X;bvM7&ck8D84Xfm^ zn(8#`{keK5D3b_cj`!WCu*~a?e1&-Fmi57L-(cT7DRgr?w6n1mZ*T8~MmM`5PXvqeb2=I7U;woP4=ox7sM-kB-GPG+Ln8Byu+0{_=dz zp@}KewT;c7xltE|N%HO%(PY#f)I(wHDrnQN#dxs)>tIkT?LUn=5KkCWoLuwK#d@;A z8Y49^_|V5fjxbtjs|w~!r%7%7*;?y$WTdLxOc4)v4xL&!F_%93=AzqXh9tc6=<0L_ zXIYQ2yIyy?-KSM0ul7jQmp9NlvsOzkdD`jp0PEw=jz zpI#P|kDlF61sg*7V%P&T$)?HgI&!4mZp__Zf{_{Nm`#2;@3FSwQVYL$d|2y#`srZF zACknBwLMYdq4U$`bWrTh(-xeX#}6;Sr*Q{oq;8n~F^t<~U$vu&_HG!6_=|_`o63|)ayf=fdJyT=Jw0`nFkcq>bH3$2<_K(5~ldl-B z)bs~_&)xux(#xa&b15x&^NT%1_)xqUN^ z+iG2a3M1ob`q4NF#ki({7lU$)XVF7P86=Dh1OO8_fqTu@`#az@WZ@IJGC1YRS)!xF zUlq@1ZN)C%xJ~J~WK30CsE=?R>jR8)+!Jq-A>>3CKmf-t&o5L;V>GSsL9_N1CHZ3L zh*ofr#G#X~tnab+yg1pCyE71n1RIUnJ=e>s+Sk&Y-Zu(V_xy3hq>gRR>2QE#>Z?0& zY0W6pMaM9UP!N9j6*Iy6iIc&X$}^|)E{mbU!pj6~k}ZeTb9r)*5rSF?F?x#X8hW{? z1P1}@nKhBqarG(*lwJb9>l`I?VI|PDxq`EM2KEM%*T7OiBrE}%A?npIJqmh1hxJF*M~=M&x8Hv*u-ZTFXjl4TJRP8B z>wUfVQy_>Q~iOhB7KsaYZw+<*T z)9=dln;JFs-DG+tXr><`9NTdPx?R(FjDsTfK=JHJfnKgBuak;XUxg~0h?De_-Ix4o zCveQw#O$k7#f)9rgK&8l%%|jNyq;c3IzNbXbLy?{?z;KSW3>&MV1gTZno0YcVfu3Y z;$-g+utK6pM6 z*vw`dTy@u{Fh+sTkI2z*yxr_}UI}ID71N7SB{}6Kt{{N_QHtm3DEa6!$oM9m`Shqf zZfa+->ZWLiC1!v(O5c{}YM6=~GXe5RRL~ysTA$T|KA{@*i4g@V^s{ou3|1x=# z8yT@2Ng>b+E&IdagI6D9KTNe6R;wmzI!u$U=&edTb*Qrm?Um)+Pppn^F62yRhc%rq z5Hva9)=KHb#$MuF7m4mF6o;qcR6c1|XPV;9(sNr+{Lt4g0{0*#rK-FK{#E3{uTPe~ ziJrCvq}enmQ2z#-ld(m+B{=xutu(5{wQJ1!$1+H?QEOxt2IsA-j8sk^j&lLT=e_XfFhuir?!851Z?I9lE0~(&Bb- zg+VY|*vh33)0U>3jzC2VJFuJhrm=$YZfR;Iv1`0o{ZC6E@6l-_8K11DNh9xaF}Am0 zvxBD=bE>k?O4+adI7?(`1{MKh2o0tIZnI6&Z9b)^$VC-%_yvZizVN_E?8W4f*lL$< ztySuUOH8QHk;IB|+xcsZcxpdX{P;IxI; zxMcK4-C;k#I=oqt4jIZ;cC+6k*Rx5Ue%M>E7;-MoABo#quAqH!QAx6kDwcWGxFxnN zJ=9p1C+Zo}D4K73=ynpi8-ag?Wx!wCQ`ag6?O{mn@Wc&8ET_qRG-(YE-+Mw)8^U?C zXiPBh5_0e^-1Y&F;w?8BX)rDE$ z9r0{eCyd?-eYUgAl_(t70k5l4aYovkyBriW8ZRkf6d|cM`YkJp8%|u!tNZi6`Yt?i1aKGP8X*X*QTeV#tJj|j4lp8DL%E=EYr_~E5^$NHYp3Gk{i z33VpH)K_rpl5Rg{4T?>p^ZV}hQ}k<33m89s`5-~&Znu6jQ%!W8_ipsvw#h=9J~M{S zxCiQWeG-qf71HK6?=0wsaHN+6-fpm96z{a$F4Z~h2IJZ)shwCqO?qFWjNp*iUE?$= z!cVk9VK)_Of``@}rGF}r^2{P``r(<94jC5~e$d%)0bfA5HcQZQ>bo~Pv|ub}?#gC( zgAU+omC8x;?bS~GSBk{;tBQ8KlW_6|=e=zjcUTTSr|se3%YzGzZ*OByX~WnmP+SIl zwR{d<_PYg=jTI>mwXSC_R?h)k6|qBalo80@^UYwdr6AkJN?)%v4>1 z_u1j$R5G(_uJ{$^0s@T0S5m=v9L|m%U~K!PBEIwfQIF&lqaiLv@B1eAn#F7hluPzU zLLbiX|8|eX;$+r_z}FNQ^@inDGEu|n!lc5KSeL9Z?{<|1*AwbCkD`&T*I4Sm`u()M zNuyUwc^z<5OLB_^Ch`ZX?}hL!srzx@1}3#h=c!6bt@P>m1BqYPk!Hj=n>9c zBU;hXBW^?FfPWArp~F391N9RdO|r3@GwE+@YR`ozHTN4-ooq0fj5~#R|y^5wVyZzKw4{lgFih9x_h#6Dp4Vq&OuX(?Z9IflDnz zUUSUhQo4)PUv1^_3&L@p<54rD%W5td$(7b=+1^u?gtyA0fz!daN9tRr<2k05+BR9J z{zwYtdCKxH^C9vlk~G~idig1_Gizr|OP*@8FAVzV3PKaO3pUosP1@eY;OpfW?8XwZ zTxGh;d$h~u;@Z?a7wy4ioan82-mG3W6F-YV+0H3Xl6R;3Ysqsa)E4r$X5f=leuad# z;tbnr_&DbS%vj%Q`rbSx7|rbH?HiN!8L{^yD%WGqIefiA22__dh}NnKW3|stVw>MM z32!=^W0H?6)i;m|f(!OIhl6J}#adq9la9q9GTcrp=6%%Z8M%JFd?DV&is+q$h2#c*=e65H#LT-Iw+WFh=3qzr&gaH4 z4iyC3>y(b6rLBS!XvL2C=c?$dskO}OwWEb4-z%f*!#p6H7Vgioj%pWghd9Xubmau! zLl1R-z7eg&*}&QO098+d=2CD3E5nPY=$NkT)*<>d30b0!#ruC_$iy|@kaN46cru(E znmBgcZ_sY{!=*lQ^3VJ+86Q$L_17T8mVOmw3XQXF?|)$Z%R9=)G-Kh%h&*()KiRZa zOc6k;{dcd2J*+!gM(4{XAr?2kCnKwoznMwldy@`}KNeUD%}xb#fIt_N|3Q`3<^pam z_7D8uzIXmUq2^tfNl%y=T>EjKvPVz|Z^bhy4;FCwnPPj)9yhdV4$TmU4*DOYWvAXG zDEX6~7`O9Nd*9e$-LQv1mljEV+fi2;JoKtg#EoXP(gK%i9TVjqky=Vu6iTx?$Ms*< z`C^$f9P(3DSuRzfV=~4ix;sU&Z9A`E%)BvtabEMbjEOcW+YF(^O=pLK%F?h|-&%QO z`0zhxRh1Ma=buv?r1D=*^MB=9>DPmO4d;xiUv$t?RXlj54dcpLU|7utBO<=gA1za~ z9X0F;ZrTPlmQQ+rvropq4puCA7JPC-CRzvli^Fc7ZUH;4I%jo4HI9&Gvh)M(ypaml zb%$!>Co7Wl39W8uS6UT@z5?dMv^m`ofm&zTRSob;+V?G=Dc!XSWTG(4g}Nfh2;$o& z(T=6~DDvgr1eo?Eg;Fu$z$|;Hv2f_3a>%{*>o)^F5{WP)c**!p$hgbmWO!?7&{w;n zlQB0<#KnwGDy@?2@SQ1T}iBV;zYn>3<2MpGFJ*wveVMV8ZZ8Z_PcSdHLA?xYSPHedRl(HRnZIz ziNI2q{n3QS+$52?+WsZ5#ftUw{rTBnD`9N*BiT{ZA~PO<59cdnyJ9EW)?k*aX$gO< zyXhCesGh6{y(*;r>IAspkL6}Vne}J%I45k#iAQAyKW#E1a=^H% zF*v0B7whUNEk88}?>TKoS3DUX;PQTTGzSXaL4sw;gf6%dFTi|ld$`p4SbhP_sR z$ZyPrD_Er6P7?nq)%(-rg#H?%tDSo28fWCp`B4xFlz%0XjHYacCF^crO<;x0@ z&Hy?tYkC~H&2`5e=H0m%m~igdRGD5{1?5_Br;zqVGne+qvN!F1?5GEEp2ivV}5@##IQPasla+})V5 zERRIR-_{=BvgpFd$iGc74lxTdX!ZG$eAK?g9XD_tj}fH#4t{LX+pE zJyw|Fna~xyW=fx+p0jT0!94+UR`@!VZf6F=B8{)!3voJCZ*DX~amqA%*A4jUoq)IX zmHgWgll$h^q_e0xj6%wLwrv4e3uk+?^w*--yL^=*R()s^-?ysHD=2DbYOQs0oHqrK zt-=gx3>RAkIc6p+jpYt}Q1CFFY;p$@TMI5CYgn8Yz0Omv>NmbvHUdsdviTQ+cvq_P zeujSvdw0Nd(0pyvc62ml;NA1Qr#k*J-Xq%YLdax%wJ(X|1ZW-sm=n-yXp~$frB%`3 zR0PRJo1q}NN$GV(bZHL#yPUeqP80~1G&0a9mKCO28&ClRJaV43{;AtV%&xUW1<0y< zQiA@Pdl1lTU6G}xx$_P~BJoLoR6hS6bOtQY@TP;h>0vULfoK<$@mkMie&XduiZL&3 zK@PJ|FgCUPJ#SPFxF&sm9|}}R-N=T-hUUf(Uen&E?7_jARi=IJfL&}!&N5D>IM+7I zNE{~}M8CLm$>Y?n%J*b!m|ACQTWGPi{zrx&4P&T5{H}_D(1b2VmFQtReOj@~Z-_6T{9`8>Y7*JCdABV6 zW+gDF5{h;d6PRNy7w>M*Tk59LT~|UaEc*a~m9^@UCF*sP7cDAud_61$8Tb+9H%p|c zyFB$(mA=?AO6>L`U^4*#VumIfyAJ%EG8Tho&oz&F`}G?zWC!k#-Aum+bLoK-Ancl< zK$#jrL1U)uA5jj9Z z;W|b{@C|l5%qOyHgn4!q4hn9jS4S~eN$pGnwPH2%7f|bhSoubD2b=+0zj>ke?QL;& zO)-#^RL>8EQ%rhUW*mz!D9 zH@Dyvm_&Y_{)ULVP#0iw(bv35y zjvGeTbwEME7hG3y|3XZwNByc=zsX;w`_BO+Lrj*q^0?s*b6SoS0+ts|$7{!VdZIHG z>S9``oauS2P#oW!&2F+J)VoQZyb=%9sWw-|do@D4&Uie!sOxe%p{ZEky);9R&Avx6 zvp$1d-LV8@Sol3Pd7Yg#2@TzN@(WOkfuE*KQ&$C?N~nvKXO*=ha2#blFp1mvW9~Un z5|IYS9Lm)%@yepwT03v?G`nV~MH6ij4k~`I=zg*Og8~tLM8=e$=vGZarCKzK_QPP$ zVrTOE0$>syhI8z9!N%#6sSlfP`#`eGMiuNSi4vav&62;yE`}<0j#52#<`Uhpa}Qq_|v4zuRNmV0M(CsXs^WeW&-3j8}R)=-Vw2fg?Ye z0S1lulfhMd6-&7Lizup{bo=fn!SdSEvlmCJO@I_vyw=Ttd4^MhYMB8i*lav(!=Y1; zp-Z`{Y59ubHLToCjyt>tist)$YmeQ^V0c=TLxF5(p;P9#8-%LfI!nmuTmSs-s%{iL zmwT6UjZ#0>O(&jin&cq2e5mhmf&N1j8#U(h-5*0yyLs%9QrW(Ts_Ky5Km$>2!)XjE zN0k7}1+-ot_vwP=*V1G?{wVOy>-}H0U$301`iG{muaHrk>pSN8lYt~RJmMaNOn?k2 zD0R#67fphaekEdk8{!f5=Czed#REouE0!-dW4;xH^GCl;&`f!KiZUra+2zCzGIH0; z<=qWDgLzZKw6mlw{I!4jx$Of&s@IOQcHjyNuGsG^Zx$>CwaeV&@r_y?p+FPc)1dx{R6hBc{@0dQT+-Dj@q=J*$Yww zuB0@G+X+7>Z4cgtDvMv5?`tVdNT6hm8U4seO>KYxu0i& zA`)AdCl`H&CwgJ1a6*Uhx7YVZI4Sh4dYaI293Hkqk`r*(p1e@gwg&QI201Qy`b^69Y^4y?tt(;i4r-}N`2H_{Uk66&ZYLf!rSU3 zLH!Lr!cZVvvs_<4o>8eob~+dlYxLWttW|%QpjJ9$wF3P_Fa+#Z1x?RvT!Sxq@Yt~a9Tkehe{n8(U$KcyyDE+pWV(We1>TIZ9J))qW(A+eM+DUf@76!#a z5^F;|+V)pJQeYSx0!qnJL)FUaA7X>;d zjVpMoi_y-LdeCy?=@sKT*tWeRb%8^yB{YSg77$}39h0(O7c1d^l4@-i_fNslmG;6e zH({s=TZHe76lZ*Z=8=G6_+_dknN;-_5ZDtfD6#w>(1=9gM-U|AQkcOsdp z;yGaQ_v^lH*~%i=_zYkCVWw=sMBybv!_$((wVQU-F5HF)9#>JjWD(f)PZ`0dxSuIh zI*k6J-p>}?q$_RzW3rNG{fLw$kPwFb0qszROQ*Kq1AO{Fkbr13n#Z@?Hxd`AM}4ca zdSF0T323s5nXn;X^i-~%*o z{3S|Hl?(I30k#mX-{oP8Xe}&um=`M{%D=gS?izuqXoaK=vQT5G8H)9r1hsFXH<9g6 z3+mzZz2evFdG^=r&t@K!`9}L9{Lok_g|b$I(SdYD`s7qQCX?ZIWNsw zV)3(xL~TvfmlV4ULRv0>BpQApe`Jx%bfc_`amcpJrv8ZD=K$OL`-3J4o`^f_3~{M& zZy8p!uGTfO=sEt#^Yj>=%VuZ#seBQ z=<5g4m-&=d5Q>#}p6n$7_pa!!ofjWy5C1N|G!g3$?T|DqhhYv+ZEJQahc#sr6kvA- zg_b-yTLIpJuyH4VMMIQOeYp~<>Y~m@G+~KVcIQtpnm_vt5w!6BHj9W(YZeBhLH>|A z1RIy(c7-`_^lM!CW$0?_0t?396fv?cE6O~p`~P$taVOiQ3&!yDdipQY%o^{)8YO5RQVBhyqvug3pZ$F! znKnY?z-G^EjG>r zU_S46+66KN{HM)4c|$2Mu}$6`wA|V6;@mp#l&TfHib4P;z+z2H_y)9WL1+uWNVmXt zhcGz=mxVRHU-ZSGX1>0Efx6Qu8Zmt<7NR=KrY`@?cA=dSILY2zj1(OPbDAh#IIyY7 zZC`Dq*-9Gx1k+`=lU{>ks8AFyhll?1nLkwX?`*Y2e5^Ymjpw06KBx5wFrDq)+r3o# z%*Vf~ayBKgs57$YsY44@fX_m=suAFeTY$0=xglsc!~v5{nNt-;?fKgmT&=?R2SVTD zMK8ek<1kQOyb>*TE3aL)gm_6E$7sF5@+wDoe@k3A>hiT@*4!9cFDvjp}84m@TO43Z!$lT)9 zhO3J5n#~s_go+3APJfoIF3qs0t~4rO>!9DQxuQHGPF?#G{qf~i=1>(@@w8mk)gr?m zx3;Dzfw;y{1rlDK{XDuvrf}c}nFiLoaCPXm7w|UbH!d`JT;Dqq03E}7>5`hI1;iI` z;JCPKxj9yR-K$9?P5-$C8Er z1)vt*0cD3d}9J?9DtMzl7CE24de5C=28N{l_Wk zS@Yo!c1t#oNLZ5;gcwm5fSAFuMnU+nTby(h% zuK7pkml>4{@$rV7i(Kid^0U+qdAH>n!gwYw;>Tzbv0*tZB6H6aFjm}QstQ|l^MP-% z80EZFTgM{Q4z`PQNaC{#ODdwr#n+4^^F`Ins@Z!rKFRTyF;E3e!SW?os_{lkom4@eJ1NH=Qu5oDI?oLtXO@T zbm|GD>alJIlW&PI!kE5n9%Ff1ZpwI=P)74yLdT_L48sb0f?+DAR(2+Ba&J3Dd2l3{ zc&4nlpjtxR^ZC4zsmwvXV8d}e=I!Ju#Wbml&~0`S<3!lF@cH{ri2LtJT?BVhZ20(# z?^0b!Gt}3A<%CGCQ`zQ%d7Had3n6D1X;Em7zk{rVXoeYKEo9f#@j^L-nvdp4*Sa+S zCh}`tCoi&6INm)vfHTn&FwGy`ugAZhLe)u~AY8i>bFGdmF+0|a(c?arqlFp~ADt)Sdv$r>1 zbfx&$4aI~NJJp?Tr>J<ACdfP z#YUIn2^M)s%8U1bBpbDc@P(8e`m<(qDQH`PiSkanI5(;F?sTP~A6uKv2$Gott7$<4 z%aSI9njR_G(+yYjRiW^U*%LYvi6@_x=>7NUeS*bfwVbxeR~srh{psF`pnj^%d?2Cg z7Mr|8Qulivi^~VJ6^@T~ zY*Q_fcJH#`(`El|MT0_P`ro+J2T4Y*N8J7X4NK4sf zG{|}YmYbC={dG+%Q$E-xqDE*nO#1xQ($ViCr!WpNeC9``{i7{0(6}PxF>PvZqm)kp z`lmt%`sqlXT;$DSo6`B1T~|iKXI7Vso+pvjG|hy5k^~t;2o)6Gc_D2sxpP&#otcVT z#Rj-S_ESqsZkIZQ3>E6iQ8#= zygDKIbw)6ivli%Jy7mgax~Fc5GEpj~W}CspI?%^hXJvOZ z`MFy;^V$ljWX|-lO_)K79r?iE+$l(CjI8D9$s|T{>xX@0M^&#rL1Dlte)+5*HLZY8 zyE)GEu{v^j#4_phS4EAm7HL1b5y@8Eu-Bi= zl&3zxd0c5os`xC}&D``7PWw6ZNQIcv0S!3`If}td6}3-0I54UNhpTb8&z2I{to4lo zM(~TUE;S3=uCO^54Hm#Y~|DsA@?MJlKB zivZeElIIKS*LZL0%8OqAh9!aVG`Fo5!_>-cS#hbI|0pioso3)bMcqwKg3%D4nCIT`#GLUG4|#+;gE2?CZ!SIK-We# z+ChM(Z@L?t!ByR&`*P_e8P=BUXW zKmieUG(-b#FBa1VORD}aI2hfPTBi}DJd#o1w4C5mF-ei^+EfctQ`xc30y3cyxEkdj z@P%rRAjr9szj~EHLB`}R--?&)r8K{ux4yuG{*|p!TpNdL!5P^vU(rkJ>-P=`rKN-% zX)NjW*uLOZXupseMydXbL(DF(MQr`Va13{Yo|DCux;1VofZz=#2UFu?)Zto-G1&*c z8qr=?@X&Fh&dK>_>hqh0)%BaBco&*qnY8ebp`6y%(;P1rCeX1KWVvcArv{&dTWa=9 z%81(lVy+1ydO@2o&-5d<%46PX3QNQ3o5Mp2XQEoKBiM$qwX*P0fD7R3J^V2s#M%Tk zSTPNxr4GQ|E9fYf{h1X(W9iT84RQlVoPaDZ2R?T-es_jHgu=isEjT#DOl5a{wubm>yh}mUO&5Reo?U1>6FlzH9bigjse- zXouaR9XvkM_zsBRrryBdiMWxBrWQGtR?T>*@4m%QyaRZsP>+QPO#_ZiT*?>RMyZzy zTZ#t!=EJ|m5b&}1+^BWkH8!Bx+<{BhbLpsonaY4P=U6%Y1y9~-cFVo{v}M0xM>0d~ z(>QZ%iOqaNednYPGRlo#=Y1}v@8i^WolS4R{XtbbHU`q9qDS0Mw@v2??{i3$t0WC; z7Gs5`rOk%(TZQy*wv1@Ym!5nWOc%TbX$;oh8=yPf+;jp9=n8~JRG3iCjr0bboLdm4 z+S+6qUIa2z{yr^`;C@usa6E){0oIIhaU17k_>a%u3L~kn6#MAZ@ObfE+@F zIJIlrobhD2fq}`Cg*>{^!J@2?$v}DsT=7%KM0W2St#-*xu#IOLU5S=@!dO(Z?D4|I@H3#`)8Ob?Lm%ka z@4Us{=XR&f6~-s}{xLZ~;!wB>CpRPSEI3k@ElucN0_L^mdD~pg!A!82V;}jBvsSs9 z52xRpzZ-b@*Gch@KtwBPr$VDs_t~%D2Hma<*}N_kgsh_{DJ{;4nm&uCf3|Q9iIs1& z(XAnSU|=l)g26$r@i-)oNh;WxKJTMY9NSS_ zf3yjeJ*7b1%r23B`$xAgtnt&LQT910Tr!BH8rIXVih_X`(T`E-sOTN+`+w*no_|vU z|986R|M*1Wpro|Zg&@YGKUS#ZFj4^&Sfc`D>6P8Q5I|@>D&l1|YNG(34Hr4$@&qK*%G*3JH}p#rj^nPp^XVsGnHa zd#yV**3z%Ky+fzYx@G%Y$=Y?D-74L&T*)uXsj^rvFrngq>!B@J2Xkf<*#9y!+~&jC zA=Gn1dDc=v4`Wxsu{(JW>l2z^_aOPGS{4HDx?}8if!W+7ay&w%P2h`gQ*-_72HAb4fJYw0*{joRNX+<1+;ust< zin{JudMDVh!~>jDq2xAlZfXsFM?UY{8(rgMyOQe^#q46|O^x|ycn6>cC0~j%E0@+I zf3hJFDkF*(K9+xbwHqKG)XMx&k@Q)y#Ld~Pt)u{DH0~#m7g6oo`~sADPS$q9{y?#t zL;n^~%qUHz$guTzca6dT#-w}J_xE1(JmzP%Wyh8`5d^pZ0TL#%B2nou@!7mZ0h_s$ za4&pDK`k^e)zXL&0VppueF-)mi**QwX}%=mG%2vtWUgL-$me!6Fie~CkF_?)l8vK; zPXoq1Bj!mgRIV0e@rJU5-RxgQgjMhXO)VXN$cU+P4pOg^($rm(kc|7kJA5;x}d9HW` zA)9aZVLXeG!s{^=D{ngcatEZDi@s#8bHF07Ujy!#x;_c&1{RBM9VAhv>eucIy39et zeV<6Ir=za>2*gIULLKV8OkUHlezY*a6&PHtfNpd)hSj+YdVKyZaQ|Kda*6QEl1U0( zrREj99U_+CToeYA zE|Nt;I|bB21c+y>3CVhol>x9LR@huo34%G@_wfq^2y&`)z7htah8Qn(QkgY5MM zup)=Mu`kTUn$Pih--j%r#cBnkxj&?@dWRx!S5Lg}t_{>2n5}7b*hcDsRNDQ+YHM!v zixQ9qKtIhnY_0m&yujTN2xj4b#Do6W@%G=quM(){1kCJ@1CUJ%D0UxcF#6JzZtjjM zHv^fhDoJ363&TGvyW7!%X=kdO>pJb3#fqNYAEMZkIV4_YVvv@ixWrA|1;`>(>fQHX z+!+&-u|+GDiw1boeWq(yY%4#$5s2h({U$LDRcF-n5!=CDG z0rocDXyLbJySuu*QKu=rA$-f8ACPYvE?H{7)kNW=HuVj%?lJG)IzddoLpNsty1{1H z3&0|11i%T0;rZJMT|S46*TpAwt5F0LdGm0b^&gyq2zf!>yx}VlxL&?bkVy9QHZQra z$x%5asFe8K>#dXSMY=NqoG%Rag1pK&6T{LRre73VGa@A&>xm zFthEukh=(3Q5=go`EBM%1vj?;qS;IKMc{td`S}K(Z=5(4G<;L0-z1cqWvFd6?9TR0 zvph3TuMSdaG}E7AfOzY`y{4wztC}6 za|O<}CSsfE>i2C7^7~I`M|yjzo7_{GHPEv4Ik53VtScc@qY&)Qij{u8p@ie%pHUpW zrk)LNyWt{fgJ|Qy^M0W4_BaW(NU*&o8~(vYRMR_Is{Mn~>kd%IvQa#l!!@%l<6VWO zH2~qmd+ato)z~GECcBY@neV=MQL8azb#Ke$1?xhV+6UL2I0w{fij@8a?-?5!0%Xb( zr9R^F&jIUA-stgzb5WnDEv7w77ya>7m4=c%#Fh7nh!Zf!w}XQ?Pq)dbQ~SDCl4&Nn zZ2qgKyVx)SPUH^CzOu>w3IRewV89hMSIs)v%J zj-2&l=ZDS*n0aWl;s>PR9|&FzyDWMMZ1a320OZ2y*Jqhven_&+$=e&w@oxc7f%%+q zJZG5en2J^I>!;HOWE6BvB?iX|BjiYW!n9X?3iZV*$PT#U(JHS+!5s&3lV2lU>V|1- z2cq+9(L76ov1pX){2a{UGkY_BHHHiGTuoG@Xde%6v~4>U9ie;|Gfh zlasy#i8oWpUuX%@3j_l!ZW5*Z4aa8(i(;${EjX~a94onxE@jKNID3vWD43B^)A!XS zs0}5Lp8ubc@(BC?!MOcfW&V#1HsRIA{e0pEcK@0C*)9uC<2f*ixq-p*aKGtX z{{R>V&U=%&4nRffV^c4TNcX z#G{XF0i~{BF$3HL^b(KK9Y9>q6-4i<^9N!wE1mb=AB-w;C02c11@M(X@ZJ(}06Oa_ zxquD(7AfkRm{+o4I4ugw+KsN2>Gxh)#hIrT9{Fd$PcyhPRup}HZyk6U+hWoax4`*p zmu`zO-?=~IrKVb#{N`UE;5-d9xljI`^)O|zH%REKt(c>j0vC*^e&}O{r!s+r4RX|O z9kYPtWH=33!dq(s?c?b{tmA97eQMVwK)AP z@8>|6Q4#wF4W^@ZObu?xz>)@6AN7r)@yO%wsir-PXAJAVw=KAUW*c4#=i-+aj21ad znF36w1Z4$2vRx zM#c#7iuojK5KAZ9LIb=?bQD`raGXt}H zC?ElvST(-MhcOg?A0G%B9HDUTs04r|q*)=jz}p>E3&<0n8xB`x-!jY(X)ui;e&j?k^S|)}g8X>!q<*%g{aqsq&CMP+~WrIs^lNW!_kGmf=c=_LQS|z#5JIO}3 z2SbwWq`s8)%6oNDIz+%z-`>Vibg_Klzg^DtjK ziS4WzEaz^XP9Cq2(g+H8@0|P<1tLg~JK5-Z{Ey&xxIR(8aRgfLlZmK$jF%r@kC*A` zz=xWGgcKClZ%4sMb-4g;EFc&(SSMhZ)?fL55%$($S#58>FWn*CEsb<{cL_*Ihk%r% zbV_$9ARrI$PzIq$N=b@H3rI_Mcb+lVZ=Lh!RCNlki%GX2W(=*r$6ugEwD^KT$fy31x0hM88=@jh0>3}|hj1Q+Y~P^=H1 zPZ|KwbvjaNGQg0V)WD4y9eCxt^ELuTbgUgmeAHlp3Gg1Gxs+%}rKM>hr+0ix%SDBDKdN*ktq z@RUFz%ooS%(cW1 z(<@D>rnz~i>$F7HJ&FC5qkUl#WW5Soz{^nqKIB9cB3jwIp1hB9&tpI4e!+|wBDTb| z@pK1$F{`sJL388nvH4;Ob8DnALQv_2;^AL{>u@@9OHLnxl<ixDdAqG6!BO-+n!k0 zzuRz`>ZjfbRWy@~SGq5xmdW$wU;Z?P<=0ZoeD}T3YUv_h6j+m3j*^y*xtOg9-XY8F z-A~lXntBj)mZ&3qGfF8EF7qzLR@yp9j!4rCyM`_knM5>LD~~X^(4eDFr}>bT2w$OX zOxPPGitBh5ba9!L<6F@#qH9oHa36SmpE9DfzCvu1*|nGCZQT%I>u1l}@A}r@y_d+j zlXLf-PSboxAEx5nF8Pz(qeHYXX@iJLfi&o5%f1hlyy23B&BI{b z8nmC+8L|^|sb)J;P31lGt;`F{trjPuUu;sE?#Nu6)*09J%^!6sziVu5BTpWyU z1lJc^H0GM*tV)*DJ7zYKb4xqcZX>oZ_|=|X1#O`;B)s1K>qFeN;i8`by-UKTR?v326q)nZA0_V6m23qhkC3KeKpHU7pD)Q* z+;0guJ(9UtjO=SwN+7)idi=>T)kYiSY^O78@Ei7isA}Ch-}$3o?nkl_dD;VtK5upZ zv3{Udwx@2eCyii@oUG+77%&pFK;g?H;TD2S$XC22YjQ&H;T&JkZHh)b1AUuEn;AgB z`$?{Y(z67Hg#1x*POkPdC=Ka4z5t|Rl7JHf#3$E}RcJ`OJ}ThiCItvKy(~{-)p|?( z_6-fkF%&(#2LmhNJ8T-s@$tBgl0t<7etUJZCiC)@nxtpH-u?)BZ`>BrqTAft38t7^ za7bzKAsZnPam|uia!whZ!at&Jjy6V**UT=Nt!8O6~xb z!t1w_cUKgSH>%Em3Se+#Wfe0F>;FKD*f9P}Lj9ixvv{0XlxG59WK%!`0oE5xzY=Si zMxVDsBp-Ge>M`8ZCVr6n>GMb7dyz;?GRmnsX1y#Lj%zB-&ZtY9L0H5t9gyhFe6t0l zxP+atE7fKunpc_f8`A7qM~DS^DsM9*a?zaV;Bk)qd2f`q(o3*1f8>K&<)A#R{cA&v$b6x6hfxm}yz9?{(D zwoNEW5TDSoi@|vt7sYqN$4RHPZ`~x)*!?1XPvx4DO9|8Je8AM)cRAdVUfIVQJg9u% z3S!ps3OG;7C-BR&#tnpbUuwxQ=O+ZiC7W*L`b(yBIKPCjnyVJ>j*gp*Hdc$dDE&-9 z(-QLW*?glmH%0HN+?$C1=;!O#Aq+HW#41{yN-w7>OHD`H#yg|mnWbKlXrZ}i^lDTO z(v#=Hc^FvF;@u+G)0LIt$*)K7$Ge8xAv@$sXFDvJH0Mu@SsqobrBp$H`3&wv#iUhe zZ3eOEZ;3@yy?IrUhe=|l*W}HWcx;@0v~8O&^^7(}b}-+Ytk}|aDOb?R&ioWDvHWUg z=egVUXu(y)D`wQG*p>{QkjC=kD?HLBFfXN&EwWBPS~aEE^6QRqa6ezq)T8=YMyJ66 z9=XeLw`z7h+f9l5g`z*178>s#1~5L#tUK6@u$`hi>4Eb*zvr*;g!7=EE=ANz4;H#_ zqDKuzW`5!ym9D=X@VZ$uOZIqoAL=_~yhJN6lkq>p?!FJl zILG-Ry&(;B%A8NPSIjn&)s&Rz;M`s`w?Bi_0yjfPebk25Bwa0IZ^AGV|GhOZ;{8ik z_+Nc`oM`!&?|nZIeX2TiAJU>N?i)UM4%>c&j2nxAW?CKH&@3}j8kY5EZhxZW6kP;K zYKbPLuEXRLqM~(`N8~dMKef`}xM|#T-&!jYT(`%Lx|$)bl+{^@Zu-9wx1V&4%6-JO zxLoxAD1xwPi~nwjlHy`$ac59RYR#VM!4Fh98-Ml`dgB*h!esp|V}1S54;L|Ib2N%- z*kqV0+_O?TTcc>R7uQOwwuThZDnH7x*JMW8lm=yv!}uqoruo2E>uVy~P~2ABvHg!; z1l!aN*O7HAXp7xV+LO$ED`wvP>}3z_*R4e&*hc=AKQ6ZWmBPE~#Z8YO{5IJnru~+N zO9z>qh>@#+zpT=%QSV~mZ!h|bw_jvT`JkK94n)u@z)?=7-pm388ZuT9>LXJl$(q0Q zF<-ybq%8Ka8O`((A9} zv!`6bE>ra4<+_FPv(6A9>ZkDH_ZS_S2~!e)->b6BZYQ6D?&P~c0KIc;#$r2Vj?G95 zjm#AdL%fuLX{V})w3j47A3W-L-_oVG=>qGW>(D5%32={ag zj=5_v84vNrfO`BjUzP2wh360E&&u+p)vhzbZGiC-0lub1LE{5ZR1G)Lc? z);bdSdWra=h53W#G(fZ#%Sz)05r&$l6iAFb_c<0(^4bq0Rd-JYv;auW+b9X@jw=V6-MEh@S z!A$kZ4|iUUJR~)aFE-2k#=9!|$&*4dn+(njQdPAr+|#5AeV{OYZO5>)JTX2TdB1kAzd+GH1t7tPwuyn68bh=w%RRsODBR^58x1RB&CR#n_l&Gp zyocn@)%WpR>NIkbL6@EEzk)-g|3N!OG5`d^|A$V=R1t$15i={)F^5n}A7%58bK@*T zbO75bx-IM_fHu8k7A`Zf9Y2O|w$g4^=z%5O*g*};2m zC=;kdkokVX16iiv*L|EX#RGW`dpa zAiLf<5W=b6bN78fIMyoiH|@18I=lnsk*X~7Nng!i4zIPmq9I#=k9KtYB@kAU{Amz# zyQ&E`vf}P;Kt}Z&IIScZpF$&>!8r`di}59Jd{xZwOWy!VB9;RzGg5+R`;4K zR6xH>+7r-Lp-viv_=2S3gY?k-KHK2!A;RkeaQdu4Msj0v4V_p|3f+DgQCzuDG2;6n zpp`vx<%i9{T48rPqa_KtfgJPT$e_;bzQlCY9{aOa9TpD27nTedi*+g!2(bw7!7q`q4dRzcK7M%* zhQ>R<3R@K*E&A0vbp^ChyN}VR-gBT&tEp;~%k_OYSV5{4@W(=dO`~Rx9_3rQ*#*oJ zkr9oeMpl!JJjDfhC7fTO*&f6$JWXD`jP~*-w2H0&`m$Di+W$Yb{ZxksyrfA<-1-;b zQDElnf%G=i6W#~#hVhM`E@&fV>W!3V&lCim#5be$QOEP_JnZ_jP|{m!(5`cFkl(!b zEq*_SnT3?g{E^e}Bl&CJ3qZ;1C(z2LgGm>y2;e#@7mm@0xS7dsEostoeSLJ+WYvLG zMK32DKV27r5n5E=srQEkyL|G8CgV@m4Ejjy=L>$BI~8GvsB@>lI9RQJ7FL3ssbpv( zzR@O-lrQlz;3bd~bul9S_|ez3dQ?-t(@MM&H*&iTXd( z1+@Re^Ja))!JsN%foGzTgU~`B#kl@?P+}qQWP{=4E%z@M!L0?U^I82Z>lT-gFB9S$ zGxb$BEV|o3r6Hnx(m%P4$;;FbkMJ2_&-y*2dICnUO0J^Nh^`g@MsgtEM7?Zvd3^ z>58a9rg~gr$g_cLg4Y)MZJ`%`W9IdBq?joXfUlPYW36)(gr8s|DxoM5GUU%N}Blj|9dfl%$G+qo68VIiczJfJM2KanIdPhLaUZ$&MOdJQn6KPAW`i6!*_@AqT&TQ6;(-|^b7pB{-GeBdrhE!>&c0Q$iF4KIgo!T%DD-=K1FUnaAM zIqMD%6fLNg&9Cx6fUJscE)eIu35$fo4~V6jD;L1f(0AhWn9C^ zf+~FrHO&Gb;6dWk>7fvBY3GTLef~GkLJR|JT=i?MwjKL8(Bv8KC_b`rk6Bgo{pm%@ zK;FjRshe@5$m&j|oVX_h@EXChgVWq^AfYZK;{7PA)}&KEs}W1 zZY#ifOxQ?x%CYbT6~t&vPXA#KgC{EZeP^W(S%f zk5>rQLcyS?k?AcQwy}C+oE1Jy%a|;n^)lCJm`5?!0U1NpkkY>B{SO>WjSqQT;JrZ; z7slGmkU^giR_Lw?m57~&2jg64d&pZ7Vus}oj&Vy2PYB6rdX+=c> zjJ2N$BN=E3fy&AHWlyv8vI!j{2B;%LGK0aZ_p`1GBavnN`eYF0i&#v2eX4=n64+Qdp(x!LpOVppv2NqOhvy-iMgPb22dgX|*{3z92>>;gC1{tlv=mS*6D=b-%0d z8s3T(ookrQ8|MoB?HBBYmq}dWZofkWVVd?u`@=a2Aixp;v3OBv$9Ohqf*W?91Uu%} z#(dsDKjZw$yHG(l{k#XEZavw38gZ42cy1#sWI0GgmGZ4^k62xVB)2G8u3mm272C(c z0DMMOf80-JfQd;tLUd(!!$e0F{PCD(cRiqI43jmF-iv%W;c7~Umsc)JJ!Ut;KTH}Y znQ1NL#20uqfxxuUS~1Nx!d#&S0LMOgI#lHIsj6e10&d(>F!-^G15$W+BH47*&bS(| ze0rnLSSJ-D`*$M59qZ?N4IFCKT+_u*=8Ni7C498IxT{ILb!1+cvG*(lY=4Rk0u6oy zB@f^V5~0#}MWFGz%zqOb_XiWY`uiJDIaaE~T>`lAegcEP6ABMB1!uwY$oEr#Sh5vl zYwp3l?&ntsji7e^RB&b%*id%&P!L?W5myO&@0%xpQWDpS-T||Z@d%~27gRkdpSz$$ z4j&P@={kuQ%6YHnc_kNMLbue%$QQ(@B|~=m3ztW{2Y`%u2@oF|eijL&k8X&xMLX_6 z;4y>a9||{k;oF4vBvLf~1Ab0R&IZYv9q5N(3A(J={K4Qw8Uk9lw_p^OU>CzdO@JWQ zZJ;uL3$mqAtjYNYok%81}gSTZw>+P_7ZrZZ|bz(3jP?<;WGK$Ri$5+3}u7hU(NPp=9+wt z>s@Eo9IzmVZR)$0$lSxnH+wCs5c-k|(+ht91yDb}xVc7XavimE(veoxD2e=Iy5=3| z_g%l-vA@Q`$T8kk%ToT2KbuN*X7uMD&(z5K!CAQ^1tc zr2bG02v6!n*9qOHy+z7(T2PHs0>F=2fL*9=w<1x4+CZEvwwA5>p3=#pPo5IXN%@6~ z7v3gSpW<75d`fE8UBW+&)=>P$Q;MAd{>cM(QwWN=;E38)g5O+_(V)NH5L8bGa?3Ld zh^P$W`nuC8$$wP;*jSDDvd~To0J29F;A}A}fUBUQ2%J1^2^v2FK+!#&st=G=9%gg_ z1DH8DDiLz_;Mk+mE7QPY{syM9{(V5M2mlGy0waL?smL&pzDfvr1oA|g^*|ZXFo$3B z`3*nXRn&o}KL+^K4`5Kz6ox8r>YsO`e0%odE1ssYj+uAf+xWznB25b^eFns``R`>3;kp=s@KFd0w zfQJI7U6q4axJUX5`<9=e7=1b}c*JNClwH$XC5{IKn6)ZFQ3CpQR`K6db|7WVEOby_ zoSg0rvgYC(x375cCA#K^v73IY(NUG+LdsMV2uF%G)_6!Qr#7i2z};NPu3?&k#rqUq z7uXaMlHV5Ni|;oCuHHzmK2w$&S2LyD?PL*4{zT;7=>WNcW$>)WXc>~XKNMB&Q*F?P z8CHu0T=IO20k_&Gi&#j{pvJt1*r^>lGy3E$`AJg?d`HgnVqC{qr}iL}uCP+*DED+i z2!O&YL5S6!Sunh^iO^qVpK$aL`T=N!-A-z<64oTZrF{dR)Sq8ZD7-Me&?+J}TtR?z z)Xj}>ARL75GS6&?;CfAt^WPfb-_x5CDnTdmLKTbaHhI@6Uhqu&o(BACiX2wTEQEIu zXNvb7>5hr6RL=YGOSSVkPKBIC6?-_E6m-J;|3W;~04I@P9}p1i=Uc;DL+@7>o=$=) ztD2RKq2Q3R00)r08?#T+*(lu<#+B6_2pZO?xXRyd+=n5rd9VDwv0HMFK93AYJE4drV?W*pe>aUjtb!EksoLKvWtKp)R|p_+ZvK z>%Uo&)n8!JBXG>-C9%p9Px1ebv+<(;>c?mb|DeR7!wlrZOGpXZU+q)g6NZrzSdxU8 z1rc7nIKW@V?nrs1gGEQ!Diy{l|Etkig}%$; zfp&3tP9D&C2&K7iV4aRVPu zu{gYaA>AZTb}`U^S0LfJSp5Ay%01l}^~Tx7EM+KT|0H7~Tm&QELn`>y5M?OsA>9Se zVY6IL0#8OjTr<64r1)XM^18fl(6f_A;t%T(=xBQ@ih(aRYIx(m0g!K%sdwG(+PTAI z>()dUS%>UPDaHGT{(onRwN3&-tfpJq4dC4l&k0kIL}R81uV_Q*L!Q~q1Er-aw5$P; zFcf`Vih;*A57GCfZ(63aSwNQoQ>G0ZE=fj((56$&2H6Yk7iUhx+EhlPO6zHAAI2aqV z1d7XRzKGV#q=KwN&!Zo#2|&-?v?3>c^Q&uIm-I6cYx_l~`{2uMI&&nrM=NZhfS;E0MRAT%dl|Ga*lb3o*p$Dw7=LW(g~3cuHV$ zX@<3(vbh~E!YbJ+8IE(3BsnGI8!4G+?~>_JB<)jj{p_9S%|U0$T%zsG85kBwX{05Fw!|=zZJg-CN{5dO`vB5XG6cxJaSdpLOm29AOBPyB;}%9l zGa6J`BOi0IAuoamze0Y!-{BIx|K;Qm-V(T<4Jx5Gf*W|>Ik?<)+h_2LFMW>!Rg5xC zl^ZB)c-CQ1dTs3^Kwi|5Z{*hSH#&k;#YL1f=?v&07LFUSF9Ay}F%!Glm8BX)L>s)j zz+Avh?A>*_#IooV9c%Wk`g>T}gdL1vPVzPSUuhCXG;9j{?KJqns z2h|2}u6ox(*^Xlj2o^ucx(?OXCMw1er~#$=;?xs4l|l-1YU9;=IMi@#e+n<$o$ z=a!Z_9z`~D!m0 z|5HTOd-|j(Vm>5fv4U};$!oa)3id~Ey&AflIDs(3H`|e`8514OBuIT1pZFJbmV;Ip z@xsSxwb(mfs!GAR55<+Y*NHBNO$4Pqj9As%G@oO#M_>WgY%asDhLhB3kRDQ6T*zOM zE7m92Qt6|p$>L=V+oBu(htr@f1CQ>z)3esM~>B30Ic!kxyUfR#wgYTx5p zHSvjve>1NS#h}g2vKU=2+HnE0B`k>YGYA#U;d80dEB$Dy*WTBe+w57b6HTWtz$`E#r|Y}RNm4#y=p>35NG}M##?Z! z7{8HL>T`kfg?4}`S8MR-nyS&7*=Pw{2A`7Ha|LeI+pcG@A1rXCG^#CY>6LepA0>E8?Z~-i`Ux%Z4a>;A($&h~=#xadt+7yPYZUi# z<({6VovJBCD3qV-Xn<2L+w5fdEHvGzkHtp-t!@7Dk5TkHPhL zjT`9?6rlW5wH`GW>qIXaP5)t$4|f9*rJ_0oh9_EwNj%an@sjCa324F@u#{?ks3U0? z%OYX6bYSm%Kq065qn2kv`2lV?GfHv8HD-G_#%ed2rRICdX&iTn?GPeOvNg)Bn4P#q zxrsd%`lO_NN?wv#jMxZ@^u<;`dm^q#a{@<|2k-Y-!xHE6QFu z{?ej9&_~^-Kfrs+LjUIL@ZT^8nea>$kLZ!`N_W}drWAUHh?b&1WUe-@z8A-*v%IGd z+OhVfm*bp`$-=CpkSyk4@9l)mhr(q)R8N*Cr_W)iM%G&*=X%?^V(mo18vQ1Ta8%oz zQyo52E$;(SmF#v?8p+k97bzF_U5XT+iW33@qCd!JWQe7NL{ZTiJ`vq5{uOGb66OSM zCi>|j6Uy|Z(*(zSa`2l8*z*T@G1=%j^}p9_ToVN%c!0p5oizY%!u>)PkyIUi8v@gk zmA8e7pA+fM$-R)myHD`jHg&S?YJ~P49oz}*m+y@>Lffgt$Tgji5eFf`?ju2XKEg!2 zTAj=EKyVtuZ{uE~*MH}cml-%!V>aj+gK?uZ5w2Ge6H#rKYN|@GSde>^1;uw<(D6wU zWk_G~5r?GubmX$WS|CDX{Oltk-5+~XU zCx%~|RHPltahXL3l!7{EK!b7PxTXeHc^o-v5~)(WO9y11ddHEXn?Q2<{@lK2MzM;K z0&L&(T#%lyg=_B7PdN5zhV|JnIv2CQhH(@Df(0HnWJ(d~zpVT*JPME|G%OZhzunkN+xa#k8QwGH)wOs~{T;qpUEL zjhbgR@`Bb`Umys;ojEV>zMLjq$=2zPgP0`u*?I2QKFxWY_Kxx{k`5oCaGrAY;i0PQT zAvb;__L7a+Jk>PMQ@M4Pwko-qT~aL;p?HAiMD zWfxHj|7ghz16dOa$t)Nh!V}l2E0U3NLCmA=x#pK_(kk z`arvH#LmtE1=5qtP)hNayiR}F9+M;f)&P)u{DC=x`P;2L!5nJd%O7IFM3Q}RJCcSM z2^C2{nBfY`yTbrJ^mBpFra4MgC*B;4T?so=7vwWP8l`IYg4m z01#u0Nc|%x8-YKB7(F5o?vKL74z#EYXB^N=l-~3hE72gs6xATp`oKsj#Bb1b!sLZ0 z=Z9iB!mR{4H^px-9|m1l7afe!QWQT zHcBJwQd;;I8o+a(Pz;pv3eN6WGUC8!#;PcdY<~~*3u@BU&d$6}?b?P3(0xqFoNfwt$y03}N53tRQ zHxw+nJ2U!d%y12?@|4=w_u$;W077Ko$~lO>c83I<@6nt$LHqCa0aJ_nR#D;h>a=74 z{Fojx&j5Gk=2xx=>m0PzP5W~mLL3qDB;-6)K?f^PMAxzDF-D5o@qmRCA6Ea?@(g@+Zg0O>T_I!Rw^k?3Dw35RKbRz&ywfX zU<2JPurqo7Hi8F+)cgkgSGC`V;D>hse)*^1SFj9U4FvbE+rYg>*3oAxw_q3H4Y;hY z{@@;ddeC|V?o>Y=vahuc?r5-4l~pu3i0|!il+Yzdj`780Qr#-uZyod?g6?Z4$evF6 zSjJR`9$pl46Yv0G)8ZuZXh#mDDTs}X5&naw@239m2X>dNV89<2#n{x8>^^+jT^DCB z$raYJKwcyf+F>M=R*cRt2h z)VrCxgXt+P7_<;FKVwpR4cYIym@k-~kC1)z+U=)HLZf(Y`~oP-KixMR)1C}5V=$M0 zF*X?E))wD{_nlS@yEkR#!T@p|Sba@r)>rFL-%r6M;gdA!tV3bPtJ88aKfl?ZTCOE-dlp$!s5!yKXUs;)qV9L=-y*jd%!d} zy3MvtPCEvA)jGg9>$S>&bZ+AgNwBiBwLoK zpV@Jnk<^XrHNQsLT{DpST;>26bHxXsZ;Gv41I+Q#G@||UMmdUP@vU-Q;Y_mJKIcD) zKX_LgPMSf#B0bpJvF&FjJCUj3#)rstc36r^+f??bLGwDZ1fCF^Zm7cL9#RXhQE$3b zv3FM_8txH9B;)U9G)OTr^N6~K_O0ArpPxf~>`<`GGbF?YoRT3RPy9+*Fc#|LO>d8; zvU~?#YyoThnqsUW_sJLFX4FHBhxMZI9Z)6Ez{{_`c^wPU?L0fsS2Jl}0yuYq?FPu&LY@pjr;N)Eu*{AZivGoZ85HN_~4@yPZ(b!$}Gks8Pw2Ply8<3O$; zaCkx|Az9THRnL}Jobi#m@jE78MHit4K423ijz&ie`x*ZlBY}zQf4g7T zhF+0PsVB%8eDH|WpBTNlf%iY!3%2UZwVeMt7jblrCZGr`7z6fn+{;+rkp#*lWAb{? z?E$S>(2{KfK{$WzdT^Yj?Ss$pFOQFa6LAOZ*_C}z_##w$e+`hrdHcIOz+9~i_IJGp zCI0NkIgwX5U!Z+)6~{bH8%2^oj4Sk*+kjypc193)({_K$vv4gs_m8{#(+y8-CAEBg z0zk}dJ$IV6D^cIafYe~=oVNa-zISTfm5^rL1`Y<)_nqHdfG40hy%Y8^70gU!T6N-1 z4KWQ==UC_GK!M08yQVf#?(q6=VB=pR?Bup!E9 z326zka?|Z0NbiPC01qBcUHO+fYcH$GNn*A5C-7L=M|n(Hx1EY(0KUf`a+@zfLzeC6 z!$-3jFqbuwzMllofgpxZC}^R*BVaw?8wa}1tj38 zG*7M*z0B35A$r32vmO;?6%e)C+TS6R^W`^WQBHMaJV(Kq1J7@No=%+>?NWMee}Ybmt}wqH`<1}db5 zd#-@@2^A0npvfn15+vp*&;czcgD;VKigF=s=n;{aTjMHeBdvce&+vyj_P3KnR4u@! zWNNF(@M2H7jH;0fG{^F)qY!x3cR6Vh_-P7nc?%j3&gNcPZU#>O5;FU+XjW;D_lo7s z5mdsgx!fEpOTcp^tE)Qs{ro<&)7_Qql!>azPLiQ&nw_714D#@_a`=@79qtSqDW-Nn zOJetvd9&eK2Skj5jFl6HBVG|+!!vM9_I(}KkN_+CR=_{u>=_96_$CIURivjSH93B1 zrn?;$X$d@k;|Uy2W9V5;pw^$!&);UflAG-7EF zY`TlVp8}#y^UxsmS)mx0QrLyfHJfc({Hm{F=no=xGUl&RC??ow5VBG#H>_ z2ZuF)tu2IL4SwcgY~si2Rk_@hqjm=nL=i%ALqnQbZwLtcg#u3M7txeAn zXqB=6zeV;!VMR2fcvTiyFZ)*S$aU=s{RTOf)) zr#Gd#FYmCzNP)lMWx)u~OcEH990T{T$!{ReGW@}yA{|bizBcmdsYn%A{+o{Vq}kV1 zEsmZh!dcap!6&j-ezdaTZyL*IpzNxe0?*>n!`tLrIAuQAaGh=_w`Bu#4(4A8#wwtv zYsB~~ay$vzym1Xeb}gWnq{$Kv#ZN{Jq2P`h(Nh2!e>q{|;18MYnZFx6Z2wsi`&fl$ zwFRIMEWr5R@*{&)gTn%l7asb=( z0fUs(JYD(^xB{xNGw!O4HI>H@tJ;YIJNsc&Lap42D6p(ouR*c<;ir$FP(2T>uFOt6 z@UdZ|+AM=wu=r6`CiSzzv=*4BzJai#Ka@V{)4HN;;sLvU`@$+!^O5@GsMK5)&}7va z#cenBMr}m3scP~ePJ>>isy&p>SiOPB7lE5Pg9=$E>YrJ)A?$p;<6jVWz4}&$8>Q; zs%j6Hc;az@HdMZPv-%>9F2AS@JmYHl8+d;P!Mgki8Y%cEp9rF10npgD(E@Uhqh(Vh6`JtE5s~q@ zU;5m>U4BKlwIv`o#)w__(H4_x_M88gKE(if+ql*etrM!--d6CV>~H9J2L2k%>`(e) z?j&CBarsCAtkC>_Tq6#$x=!PE-(gmISKP);&Z{?KQQo#LNyO}1&+rPRN57GW!ddu3 zbqh)zXuyh?qufJ;?3j(GJt$4?Gj*R5RrH89jRx&C)80Wm8Fg{?tFAYW66-K@?^k=( z43%YEZ<S5W;+jwxTu)l$o3&C3KlbM+bs1$mqIBx~P z_Ib6Oho=-X-O@t&8JAg}oBC@CJwjgW_WbV1SD7Jpv<$d!?_e!?k#03Ub_`FlYx=08 zKc!to?+S3|oL6gX$>Jvhy4nks1zscT$XTNX8JN2sW~0OKv-WQ(<~BrmDYWGoVmN=c z#w(b79t->SPnm-{{RxDpd&rzt6KPR13Eh_0)Os+^+JoY?tX>Y_!awOrpHuF{HRju= zR%D6&1rbQ1TfTQ`Sz-l&ByAK4@Yu2YsdFe+yNvTECvu}6fQ^i&6S&~zqwX6Upkn{# z#7Qr0Gt^s?&Kz(gM-5L>=7tA(|I04s%KazF|3BNs-8f<3vB5lLAdbCII=EeITO#R# zwgj&)#(*X;PxSFYA=^QWGd02;Z_vXFpV_0s4~w>)GKn%M3z)UF7cXJyaPAX3B(HljlTcLaCT0wmg5UVD7*hSPGqsoQY$}o))>(l@~)?Jp$*M}lY zMB)Gn-Zj*F+2D3SRcz@UrNgKCCxH&TP3Df%ZWX|I`eM1DyhvVgdrE@kgEk^*L)3ts zKVQl=5?i~6;@fZ*7(%?lc&KSc+iDYx^e(UuiR48zWFB}G!srf~83gXMlMp-H0AhWnVxms?G>Sku@VXFn$ z(0au>5#|2_bs0&5Ia)S6X}Mly0gLT59dHgP&M&ItFvH)HnLlPd8dhOaPE5TZbit>S zx^u8$J76cG>JSr82CL(WWQY1_%Gkf{ z&&YYeFy3&(vKG&$ZicX&(x?%x^;g89Rl&lB_2BpNIm;eWfwVpE-3W6H#sFPcBh1o< z-OLKR>!TvF=dfFOXhlXNrIhU$)-+IH*lWyXrKYfMwO#TwPzP^1MOl)uFLOAv+cpF3 zfE^q-s}rkZbz_Y^MR{8)tcXLrYd?rR*wOWjudUi+p_4ZDc$kkC#aEt=okr-GZr9Fj z`A%&|o$XbjjDdJ3FLBaLz5CEt{D2;)SGJ+QJc8U?M4hBujPZ+&xIy=?yixxtw-8U} z_YVG|AEW?(;Bo-Cf|q83KMox;*d>h10`o^58JP^yZ=$h8RjV$+`|kq0BM$XDz;Y>u z@hshPsJ?Lrv@m&dbbFTZPaOeZc@Ytm3cTw}V;Yk$i=ze0E%pHp8^Q&j6TQO^W)}wo zPGuVilw?+@zD9$#l%GNY-OPU2p5yp$IiQYH0R+m_GeI#}rxLzyPSf~ztm&U(i9&<6 zI}K+kIM6MLdRc$d{qoaUTuAZbe|X+^psSFXPFq3teg@n+u7T+C%Lg#g>ByRW4}lih zFY4JVd{z|zq0E#F5n9|pn2)iIq}CSPTnTcDPvss=@%bnt2l&kB+(T>J%l(ekeSTi( z%aU9l{p8We0c$oQ?=ldBOT5&L`+Z#ob_2Iw`O>^Kf8+&H^l3FMKsi;tG*#h}s$SSl zunVX03bv42&@%+drl`+mhu~7LVd+^=I5&LUgX9Veg`MHW33lVH>typX@RRYq0o@s%1$Iu-uaX91|6{#P&;NP%UU8ok&Vv@UlM;qo zdMf(tPpO#qh7at>ULlS&yL1HCe7A}9#3SI>hZ7jq%RT%cLOvN!6g$nz#FRmA%VH`6 z`2Ovh3@(<@HEE7H0@l02znwF}3&JlMp04M?Y0+REuV`NrGweP{e0TDWUNwilI}R@r z?{VXIpCg9f>aXQZ(da*+m@4-#XVXsV=sB0u1w1}`{AT{55sxW~Hgg2z7N)LVP{Cve(g4w|DPhM9}z8zELyS;zL~Rp%NW2 zyeBLbDM740iQ)!$zvE^?&ODN_2Mf(nno{S{Rm;LTUVAHyYHhk#GHGF&>V8F(9>MM4 zbEyrllYiXrt`|HTfH2+&w@I6h;#OP#OYW;7{?JK(DmG9{x4=w=lG@>=c*1lx*L51sIGg2tS{~nLvK5i+{6b{ko|kS^8lK9$WX9X_|@-crxlw-(E*adWlWz zM8tgTsbP|e8UP87gx8JW*D9RKsdHVE^HAeY(2PG?3>anZ1s^kZH+T>~`c_`-r}9}y z#5-Z$EU!%BRBE2K1Y0SU@J5fS5l2zPdV+qSsVY45p}eM-N|m;`8y3@Ee{S^b3a_!| zX=RCa@P?km=<#Ke*$D;v_6^z%-gM^v^=MTsN#ll&9Sj#P)D!t8ih5j*>f~L|x53+( zOs{F=Ibr+syM7-aC!`FEH6K$HOv?=L^vQE@uJXGVleLcN1n?}198+$la%5~t;{0Em z$mwXAeU)_csN+QdasL3ji`tG*SCoqXh8Mx%YlJSc2!0tX{B?SucMRvoQ1939cgXiK)Dx%o6O2`y ztX^x0_=<&P8j~|Mi$fYhw2@~t(*&3as{t2B#Wbu3Vx}|$xj&|bo$(MRMAf`2Bfe^U z&+C<9^3y~vGasfs`;{ZWTDTLp=8v>_G+ibK|7nO}YWScW1H&)nqQG}(`(-|bMX+>7auT9daufo%BF&c#MSgP7A>vZ?0@3UBDIFTc078NR zBmwIc4K&5lXwgiOej|^*)nDG8>Sl5y!LHFCpq9vm zUnDMaRF?MW$;zfRLUn+?U?!&#_a-`GaQE_^tw)Et&4{5+0;{F*LtCqlVQ=O9WvD((ng z#4IH!yW%i63jcM|3HGr}TJmGeWLXc{y5Vz%T8H$V=LZrW6LDIw$aCetp!3{U@>kL; z7T^(?xsjEV1O({CvLj{2v5@+RAo5t@DWcOMU{7%saIsTXQ7g<}3;ylb3*EgAkvIU@RO(vq=IDC8mgpeEo@d+0-$fX`(KGyVzYJy+%7T_Fc96u)% zF(iiwZbxVY@_9=LlOY08#v;Ffc&F6b+XEb)^_+kSAOQ@(5{;fcyv`<>EwxR*= zxKOELPZCtv3@CSBLd!dQw*0`==>{}B)~fj?HcL~!TLHBiB499q3HZVmQf==*V9_)G zXVYIOHSQJ9Y5XC0Z6N1PWFaWy; z-W#$7Aa2RfatsIfJV=y}xI-2ka2g-hegYPlOl1<(t-Ay6;4P%9OzHfI+Ya* z6!=FDEqwF$ns&Vb=Gj>cF4da*LaS88Hdv6u=w;3EVt?TedSHMhHK)SL)d~0SA z$gyi@5l&l}RWhd=!SR(vunmkSNqH&=lh85{fh{45u4ZXJoJN>;D=5)>i zmUn4yVDq!V6nuWJ6-^2QWUW?s`X3w@p^C` zEQ9JAjw zC_y9yatpQ#I1Fk!PAWDC=!~E_mIEco1vA}uAN~I+?7hRe?Bl;-o5&_)hO&2A*&{Qg zf$Run?@f`t%g73aDA|PUEwZz>?7g?1_t|gU_kBFS`+5Gkj_az9i+s=X^Lf8tv&((i z67oGotWF@Hd>c%j)LKG?b0n%Ca`Zc%!Kg-(ehECN3d3C+g4U;$a@4Y2FRGmbKVFyO znS`gTecaiijdYT!a%(r6ImxrRCZ4Yh!rx-I{<|BHIOxCJyZ>{s@2YWq1U188J^sGK zu%&Ag5zec5*KIe(>x!B@6Bif{DsT)G3ZK9Z|BQ~%hCRwp>e`WT!~G&w*GVG^G@E|u zhg1=f@1>Ho_j~fV(zrvrc*o&Eq|rLUIL%VQ-dPU5@>W_wdTjoNY~Y`_Q3roaI%s<0 z-;OT7b9$i3GDZg)1sVs^`hXsv+y=zxj1Bx#r$@W4CmNjr6H88vou) zU6xd*w|azB8r#~uTNC0~S$4(c8*@N9UvcJs-E7?LpGxe0#j4qyw&%l$f}JVj=;Wii z!qdT5>%8NJ5B$>ip3!3{uev5E-B~{lyBUNwSygxATqPhjoMbm zF1v#J6sTBz$A~kbRKMYV6plMBgH4bEwQ8bV^=a<~XbKX`tLBK~>c`}7DcDJ8S`b(E zSriZ8e{7$;bC*8^K27D!3N-OQm>vALca3~OLx0bq{S#{HkJ}w`LgskLkHW9rRanZ& zLA`Ax1u8CkF)YbE?7W&8X;rc!PNT2b$}x^@nJ$TU%Nl4upT{hSk;ZH^P0NzWi@iGg z8)`aNpNDdG$LSAyR$iX2J!C%u`s5Il<_oFri;26*XI#D`RG-;yB#O0wY~ZJPd*?1K z?3s#BCB%KfE|5`h0x7%rQHsez*%Zq`DlmDNKj8hHCdcej8oz zU4%Wwo}jAoP0(l7;k?nF^%uxnnbCLgz;LcvbE4In>xUM(SvFV%~sXZ8cw z4JM~L-Y0L1J{N|WRu(ii@wNpPeQT_4ZD~NQ;fp7Y2s52}`A13U3jLiFmaxl$z>}a=Cuo_D@kUU>bx)wx z<_uIMX&k%we83yTcn0?WMaW6~eF>k@$MPjm(YLdHdR9IU$=$nvL(d^}xspCyQCmuB zgOHFmcyg|r^+E2hYA6Z91tC5%DvmTEe{cc(d|tpmn2ODZ3!H=b=5Fy8kQ=JUJjW265Wgnsv^-;ui-fngqYAtfZJPj2B$;f@ z8Ybtx{XoX`MevPNU&$Y(Vq@lYq`%H zR+LX9AWk>&b}=eYX$MFfaQN^Ze7$z&G%i9O4azmkotY-~%_@dsr96D{agp!nNu z@eZn4_$^O{Kamcqm83(5N_SauI_u3~tei%FvX+wo_*M-jj1Zn`%N_Gbm5y_l-w^Dw zwmv}h{kb9bl@9Ce>$}^#mi#jvlxaTvIM;J)QD?JB@k&|p>_1au-YNPxY$1f={qla> zbwX8|KASh8D%_RKi`2N6#ub10NoL`??eWj7tpkoDHf`jEIsf7O<=ytL6Ws5eJ{`RH zqCquzzzpw+)=I*8x$bv~%UmXkzu5w))~--f#_0U{VL>P$XKJgnhDQ*ROMXBvN+{lU z88e@DYm@bQg-%e}f`!R#IyYSkj_y}MKhyPjdzg!NR9C(ObxaI7lgJVEevKJpVbRNJ z3#sjUTm0q8w=W*fGcYaS)76`BG}y6mxWc?a-L zs4tBi!7$|soNe(iun8S?@+`QuC4~C_$h#$>;^wc|b;L`s8X=)${Zkf84g`M(}g=Ykn1bv7l?IL-ZQlKV-D>cf0vcpW1`Gp!4Z=m0&eeE`hL^3uj;nOZ*fL z#y0uxap)LZ_pjlzAnb*@!F;YJKGX;f{)+w1ytZ0+Ubn#CxQ~?n&>S?xhTM1FoX{D% znvK<$7Q1jHdc_pa`^TpIQX{)B^n*PB>z<3$xh3_B3rlt)q~fTIYa8Lord%7B%bG$M!77Rl{M zi|9RHGX2Njn{}wO%z*%lQ5+c1fB^Y95R9UCgq@Bv+snT!&1Ns-^1li)2&P< z>#<+Upl-!>Fdb(`dl88Q0|4^46T8}_D2S@w3(~wFoG9BSCo9SqoSXRT*07ccT*t+B8^6-#EnT#*6K@0Bfcu`6)$X(O z7FCN{n=HBf#%3fi%%)YZ+yyFWp!`x*deU;@%(dFeFOwc?j@%67a*);F30@Ct)Hi`L z*j~?&E=%{hf#L<~hg(Ei!s4FuOJ9YYNj}ufsN<4oKwh9_I`$Qfb z)$aXgHW(x%ifdRCW_NM8nThoL^Gc|YFN~o-SIg?6XpSOdfcoosv>D$ua#Nxd3XU9t z%i}}=^T7B|kRsAQ1qniJVk~v&*a8jm1USvetFZeh5Nj;+XDTgTY2pFBp8ggPGbhFS zyr@}4Il7ZZTxcy2znRjzkUbWq63-bU2wAQzJ!V8kt^D6QH2uu4V}#RzED8}B31~`4 z=kNCbgnOYXL$N6eSJ1oKYu$gfO=wT2Z{1I7PTkYQEM^oCReq!T>#K2Y8205fntYx> z41v_19_Vp==bA%&q;D`;l45*E5*b!~7$ZJym|H@YaFE(=E9)pqL!N-CSL69te^;!~^ySC}x z8;|wLA;TeS3IrWo6aj@{Op(HkKL7yABKWASe_fyG7l$UWv>920f|dJGa&|v%ml-p* zWoSS7c|R*OQ(WUcBbaIv79Qp*O}2XlyH+d$4bk5Bq}%@%*;DOWS#O$e`Wl-3E`PxF zDq3nthSa$cBq9-)D|lmFWy%k5dtdiUqcfheB};&XzcXNJIBPvXIdRwTLKw6Lm1=A` z8CY0)(@_7A;pDt>Zyt4*bYJyPfujXrlTzsjtjI#8Qv6 z?CKmMeDqp+|JJc+hk9(zf>Oe(f<}R&U6FdC7=*E1!QQ+Ev750DMUSnXustXu|L%>1 z@yI>N%6)P-JxX5>Cag-vBT*@csUB%jALd)k z2ng_dips@7Hi$d+vobu!V!vRFF4*~Ya?^!cjRxkA{Rxdn0|Zto@U+u-e?A>5 zQc66{bt*GPnN^47jnOe>_0jlkD0QJ(l1=EU+~0s>(X`r_uw8Mw|5m7#IE_geE!wrs-L4%9$T-iU7pPQd!BxKB!?#eX1Tde*ejUQ&tG*AZ{>E3{h2Wz zSA5`un&rtUj(8o%;dYMQ<(1CYaurP`w+Gcw?;` z^UJl&q_{waDYY-SC>`}GxXoR1_MYVHvDIaa{W*%hA?uLh3Q<4G(XfDAFxNqt5M8Ap zWyC3L5Q{iDD3GEPa-PjuOsNNZC$__S>3~`o`x4?2Z$l`Yn7CbdXGBKZ%-LTVlz?Qv z@=zVbL(6)fPaV!F<5Lt+~yooKXQesLd5+1c*kA< zBi$Kk(h6V~bZXPmjMUafiawdr1fW(iev-{~a{gE3jIU7! zpFN|E3u*n-!Pui0X8uMJgZxw+7!Iz0oEGbY2i5P52|5OPFUf1GH2Ai)S>Jy`rR80O z?SqMm7DI-AnC*QKEVy(T6V7W{6WR>?X(C~j$O0r1`9O`=!LXB=w4=qcpk@)qk|{*M z1KrHTx{89kX*;t&)u{Eq*tCkB0cS>Z;Nc|8dn0x=1O@ZXQCgu6MT#pZRNhqiE_F7c z+~c*GsCcu)C(`hP{?^6M$5^Qia<^afXX-71LzSDN?KYbUyc|EB(H*2iP4YUDs%9;O z{zxJMctg;DhzzmYH87|yN(U&VDz%ic2=D(kYs_ZOqRt)abPmyE#9u)@KN)F<%=?6X zFR3JYHt&^Y0M)qnjEN?;-3rxj6PBo<2yMY^+_MB9A9x7p!S^OLsxA>rTdYf!EPyJNO-K)nq=Wn&eC+c!6OXemlrv1DsM zr4Q-P-jT2r&7>Z!!jkb}SiiXI__f=VIGBl-@~t#L8Msd|{4?q`H1G}XFSeBVpGvygUt z*XlW>madT9OyCiH_<|K$gENAXJ@WN1*alQtQEDR9b&75MwXt9QE*gIDt$Xa(K4K>D zapMHv;^w%H|oZc&w?WEN}xVkERfNe3yQu0badW>9#Eook(m zQ3JMyxgV3@U1zR|7?k|U5D-Fkv!8p)V~wj4G>8UwV#-^-8btnBkGVfAl`h8~P&#c3n+^M|6) zt{^{@VCXYS4V8p6(9;xi+zzK4kMj+vo4>zc$l;2M%n1gnanCU#Bao+KZgzmXmS z9(~mMS+&zj{hZ#qd6O-uGW4=>k&T`zk@UUiGM?23iO*M8O}mVmK#Csg{1N9u&b!_-KCL1`gC+iZ9WY0nPPu#WiX8jF~V^qbQ*FJdlFc=1mP{AUnH%zLEHPQD5^50sGdgMmrpW-ev#~S4;0r4k6I9s zqt(NKJDvxtH_N4Q%>hifnH76R58WD8Zyq@=(zepgr^ec@GStL)Dt&y9`kw_d8Np5e zX#@N}YN{T+oCS!K+sDi+{&;=Ib|$Sk&EJs7m%NsORdQ#qnBI`Xhu;n&-IVfBFU51n zcjBZLLZ{LHUX`H4yi_^&J@(#qo%u9-2U~*vTVV|=&^lszYCaqgRL1SS zpgz-h1G`X>`F|9vKJ1zH%r!;dx7d`@4)kdK>PO(!&+7hMmW-WR!i ze0ue%?pTx|#WdHW(o$^PXI6;0f=Kelk|x1|3HB@R!Cxogd~;WNZ&-h%@ya|7-4SAB zISZj>5X4HB)@oOtCK8bSJLf2WYwr2>{3o72+1Abcy`PUObA#;F>IR07#i+24@9)@? zewEcJ4YOB2i=nL(=DXG&9u9RAYYyBl(t7A7gclkt7g3VeCK>KZXq-2Nd!JH||A0Es zmg!&arX{>X!4De_M+@~$vo5;u_Q|QA_q)tFDBjK1n+uNicZjBPXZ1hESSPwGZw}jJ zu!E2q zX3WQ!?DaX!kldHD!ml#Rm&5WQRcH`r#rbJupkTXsfxT+|Y>+ig_}c2{=|s;$?@fv> zig4LQCnxm@ykR4z<;b(otzx_^eYX~!HtCxW?Fr5*S@>*;edCB$!taG+MrZ~^-nkaM zKD-B9aNMJlO^?k!3oZZsq>yDn0j5g$aa>r zdNb;$J<1d4_xf&>)7Df?dP_)eP6;LzF%t6;yBs_~j!J*4lZEQ~AJgbu2Xbll1tWda zM6y2@%rKZ4Dh0W%UUB4nVrt2WK@rJz^`tRyuwGL3k~n*uK<~`4GsL{)1po31Wija{ zqtPgXY;-kC7n)@vGyTs%`#)wO3riPsb+@O}Lgl)osyn*wh#D~6@^=gT@RNa+Gu2#i zT9&5ZddY=xeRNHIszxvG53cvv<$p;gcyK$@I!Q(q@z%MoBoBA@x?#TZ<+!>{xI=h5 zNxe6^M#Bc>JfH(D?K_%bN#m`@vYZxIHOfaq*5JAMB;>KtC4M!QTVGVYtpsajU!)nW zQ?;sdrnM1FI_JPl2(9c+dv1yA>>fb0dhqI3#_U(m1^vHjSE{_Yw|Ac!cj-%DM;-&V zzh3ErCiFnhAOITKLt~$lU7d)+bnUK2KkZ0J)j4TF9ii&nYUDkFOch+bm(p#{^=?Nn z?YD-1W;3kynpo2bP5O;)Y*hoi^}sEPdCr)PFX5cjAJhsly!gY5|7P*FA|XM!{+>i% z(~CKO_}%DPDo7nCYnPg^~8kEn8}d2m{G_rvS~N06}5Z%Rxqb`Y>^duFu#tVCVNt>Ce6 z{slzzox|v%EWRT=TNzT4OM{DbMwff9rF!`dh^Z$d|hcRC-N25dPX6 zp8b2efjX_cb5h-X6SCBgzH-OB`_3HO;;{rb`#+EI+I5K&*N+DkZL{q|hZV!im9a3* z-e;pmRD=!$7l_&*QTSuouWu?nTKdEb>igo=`?18~%}ZXZ88o$DfAwOI;I92>y3yl` zMLg_C&{tK8{U}eHPr)?*hl}uqH-rhxPoX`)7D9D827~0Er{*cXV`43c3;)^pNZjf- zFce0|LmCl8@}PT^F& z2k+2?sJd@9FvNQ8etpF9&|2ReXUxBf4DT*=dQVWhy{ssVl4JW|+GFb}Q_>fZU6V2t zrN>%(MlaYXses9E|1DBM;@}h(lCA>4@FFXfdImPq{Rwu#G|IGxR=lQLfMgA5o#`jf za7mR0yNM6i+*|Uou`5;B!fyY#v~s>ShM`+3#1i|O?i!~y!%{fxKC9Mp~F@?Z-IH>y9 zhi2sOH$Yqkrh-@O79sjE%_Yw=Z|<*!QpEjC5AMkT*RFli$2hA zm`!5oPm5LLrkqqZ$kX2^*sT=E4SJOC&y#lN&+PXeOYbk16nO(ue@(0CDJ}DIQJ1{8 zDJM>FhcN7-ZYJ%Oev{6WH5+m=iXa+#9O7zFwT2&$Me_Y&}vcPh3v z)OvDj-&JiVzBe%#p4`_*OJ~q~d-bXaRiQ9e*n}K9Ugn>Gi2OoE{o5an6L0Qu)W^E; z?c76`dmvq?p-Nfrkt+%`s-~&09WS;$fcb!8o0_10vHCu$+8OgjzofnsK%n>Wryt8= zT9W1t)J5B#LW=6}lZP4k-GeC&jS`&fk9v)av*eIqJI;@BSG5({aa{ST;o$r^XUAQn z^MuhRU;1tCl_II1IC9rQG_sV*>*PX6V#vM~GFaz=v-+(J2?iTO(pW~b0?@0m(?zDx z5}ZgA)ZVV)CZ)`NH~{jY-eF`1LvZB$^F`oNe#2quj+J?vj6C{-%IOPONsIloUXzs| zIdZDf;nYIhgU>_|e=-cXYlc9g#5}4vM|XVX8-^#HAXW6s@)G5#wtiret=#l+8d_JCYo?hgbbw;5w>poSQ7sp&@}_ z+t-FdTcWFhR#*e>oa1f3TQpl5H$RTL<*{ zdo9i$t*M$RJz*?EbljQoK5E%%?jN(nUs-G*o+4@?o2!qxZCD~mA_E~+xc?03M#qg| z)?K;!Lxe2Pxsa7}5KU`#)Z@QgP}Lsm^`f@-FFr#tpvgJSd$T>Fe>M0XE9Q5V846kP3vcxMrh znZqc{MaFQZpT#H#?yB-IjmdvE$G!g_Oe6noWK-yOYR2GeKpFnwx~15w1m6^M;jL@5 zjITLt7%&=wkf#BVF7^fngc!N#B}c>wN&{RZC@w%OxB(j7?c9ZrkqA}G>&XBcoZAfL z=`h!KO>$TMArbL`AnJg4RMBy5r_SD>DW`$^m8>!**3zO{XmTI*J0;Kp{bF-5Dtpxd zgN@WxhZJRKbR}zT1>V&ip6aVNzjm$)g3%`?fZ;n(9k`e;N-c3F)&YLRSY>ASh3F8* zhY$v^fDq*lf0(g*f>u;k^7`fhOI)r2#h8P4=^i*3J&dtrv!R1R!O0NnixD53{d4fg zxy!{PeO_&pQs48;)5~cmaH^Pd?&NtBr;uenYRMOM-CPI4?&l9+klwMYf3I|0QZBSA z9&Q;!j8#=(_mslDORdMBtPf18YcR}9T#Mind}TxR9aur-+*Aar%PatDn{re=&Ez4t z4WdN8=TtBM02vy|n>M{;FdW}IUW{9bJ<%x05+9ia+Ib**8cA+leE~|nF8$?4ZN#XK zp))DzsNcbW5dM_=(a%tTcqS1zW-u-fnpBgK3vA(o^N@~x!Gezzk(n6#%xqJTicS~( zrjOsgn#e+I@MVZ2Qz5B4+$J-C$?N(t&Ht`*TvDq{Kb_|D#gKIBAJDXP^_o7Q(+pXM-i5Rg;K8ivQq5 z^|tYTGN)VcpIbQl*>;f8;0bR`lvc0N#{!7obcN!1vr;;`bGQ1tNKk=rYc$P%prT`+l@##_ zK7VU_;Ao+-e?ppK3)aQoPaoB_BuVs6;d#fgb}s)fpn!Gm|2HqF2@6~QW`l6A6G?B& z*w_v$`4vtCWn*+|?o|V%@SDt%Fo2O6p>J^CO1)!{w_*IEGD(QZHLP%#K%e8P`{qgS zM&o-0BZX}*3gZ_{@b;|T0gCHOc<(ovU({mPB5&_KILFVu{S(Bp}p`NU~i7X)43;huB^42{XM^`XQe z$+sG$2N;4w2|XY{LPzzaFbv90arC)kJN5i8!7WOW8|65%ba?ks_LsDlFbWk0R@6w0 zQ_L53`koq9hG?X~v~ZqkZYT%^;~kdIh=77pO*IL^ofSMG3=GfOke86XkkD!tX~dnz z5Sz4X!q+q`mC$s57C}#&uoKxwpw+W~#YAvuL=_OYTX{k zwSk4ec)60E8lyLVO{`s<*vOUimTJ4S7Tq`X#(g2`*u_HKRom9TH-cywuU7Wbuenk3 z8c97GZR#g>bdWiRD{()^>l$|NlgX0MVD$@oZ@{rsxz?Z zra=WM3h~<$i2k>S$8^l5bRAH8R#f>5dw9x!qIFYXr1s=fHt480?i2hIh=Xq$%l*N1E1B8;7v&8aXwfdeP~q#H-wM$JksBz z^KvLJaiR{l1hKHmhq$wdC+F+!r zk^~y5H2$nd^6xPcbn;m(b(O;W95mS<$DKy6xCp`BhTLMH2w1-uf5LT}jGbs({uSxp z*elwc7DUkjG%E(zhBYecZ-;V?oLjUDj(iDlM_Ky-WgH764=feKK``qEx*V0-L3z0| zpij8^CPwYhU%iaAm_E-5%W*2cQ%q%QM5Lr2-K*$Bq51~2Kyj+ssz$qiI;J4`CedEm zX8i-U`l`y2tRNLNB0@^Ai9YbRP0kClYJ=~ZGf{h`n6%gva&}mR{^Tj}7f`(&MY3@+ z3qs0~wu-DZx48JoQ~z-!f@Fyw&i*v9godvxGqsS})$y6SAu0D8+~&JqB%b`GfuFxp z0uqM2sFM%oHvKVV9z`0@s9tzN=h-v5ukiildqbF5qPog$m6;r~TKSUuM+c2J_Brf~l+XTfcDVa#( z3ipni32v}%ka!qoP255m;(B)(d@o>i?CR<7+Bu<%pKh*gS4+oAgZ6UpDp{#L2TW{i zL_E^D_ED=oOBO1P<&9E607{NVT+>QJ<)DB56}e1bNY0N?OpJ}v`y$|16D`ly<=iEm zKyaT!hpvlDT74NfK&nKhiNnfM8JE>nAW`07Ce z(c~hDq|Wri%dQ)QzY`+w#9eu5mv!y4&G2HnvpJ0&-J=d%AAwaz5ZHhk^gU}IfEB9g3`=PZs<-5WP7PN@A38b@U3(=34ZjLQ<5449}#cfTtJR70k zwTn1=`k}m^;OG3qw#xDdJ`88k;nnw<^rdyn=BCcMWvGK&4fW54%b`maorkPH@w4`$ z87zG=3_r4W0B`5k!2nV^^e-F@9^7|gBh@m_cR<`HnJPL0=s&l=; z%Q0F)cI4cnI})=>s`lL-CpCsVrAwHn-s z!qdyaczp1^W}X*jkpij~ROpvQx{}$N$$4a`A;?dJH832itbZb{kP?0$n2dvrv961XD1Fw$I zXzZJb4FvrkPe>Wu%2Ivxm=Sd|R`u0yVkG3n7S0%hTQO`k9~`^?Nk?1o-(HzSnJ6BO zgk3&6F47HTnNv(rVsD$iMStVPl9By@xHQcF4tq&F?~B zg>8t6Fsx;cEktBbAg*M*=|AF1Ok^Zq0+Vy>O>T%ZV|t;O znY;5eq?E!-2h2n9WLFL!WM|`%6iO$G?Ev(xZ;OV9bUhMs##SWCO2-(mCF zP1drhcKEGTh!M3d8a5aOBc!foU379RkpVVDv$+2~_aIc`n0-Yc(7 zF`_iES5jf!ugt@<14c58&P~7PYcE_cp?cNInd;gPQ=Aym*&Ox8AqGq2o-&d39NXJ^ z@m0A^cpG*5Yc1%#yDVS+ls+1KO_;VI5<^@0m?bm%4r~ei&wEp|cQVjdZ`N4O)W41) z`}HqtBq2ItF9-JMDI|XW)}^3Rm>OhyF|Xt#H4c1v{59D78AT>VpThNrb|5|p)`zw? z$EmrNu!WUlNZ@)`%CWAs5j3cLniAUyd{~G?^t(gj&e*Vsk9dDFYnIb?i*EGSSqL}? z+$tOZWctCFtUl6Hs@LO;6oU;0aLw(euOr$zG*P82*`<9pldBgsAd2nZB9u5bVzzmu z)v|91LrFDwVRW3FfRhNhxIuprG_n5zWd6-&qfwg{1E32a)6v7GZhC^K4*GPFl*lL$ z$}!tejfLy?5g4RB!&|O^TBs>VFUqoBCLDKz6SYOM$9V*AM`+~MivKb#tx;On%~+*V zigI-kD37BU--LxV)AE7DJXWn&E8n49610?wy|(FP8SXXEj4P5~HCi|3*~c$eGopPR z3|d$6%3dl_so@d&s*kj>3c7^~i$t2`YGCS09H1;2!97$}CH7YN!Ah$8yp~}y!5!b7 zioQ#HpXc!F`F=QRyqj6|_60OV9)QZ7U9>($rML|41{3Miy<6uyezaGQ$6WwYHv#E z9{FrYc{iM8k?&__K`ynVhzhES`97bui~{IdC?=<1{ov;@w%w<-$8E1qsV`f8Bifgd zjq*`;k!`FkMP8X}V?nsQM|(l|)J!6a(BkgVty2P#LVVK zsT_k&ZG35z=U`~O&S!4IfyCvf^0Q=+t{SPCNNzj}XK;lDD9g9d`83*I<)xrtw>y%- zm#M@5842FfyylTh8w`@L$UjE$8!E_gHX^k_H~_ehGU+co|*Yq%A}HH zwT$QD$9&scF`s`*%tkLo^bkmL z`EYs39dB(@+0!n}>bBa#;(QeBOGM9JOLrb{Y_*O5Kyage>05n4mQ#FY$?QM_z0!~E zo3ryR1ws%Gw+b$q;2GAG-}gDZ_;ZQe8;5#Adjb3e@Rnjx73ij&vCeW8%8 zco&oizWSih?mlvl=A+>5j*Ql=HVB|fyw9vN&2vqPjV9O7t>_~E zZ+f6T)8xKXKF?frY{L#Nh_12_nl8F|KGJaRw(oCv?Y8Zdo>dqnrWKl@h)aksy>2Vk zvS(}3Y0+pjg)g&5vjYAKu3%tStiK_Jr?f)*E8DjYMJitH&1W+#Eeh(D%an!!^_MxM z?1oMeb#X-s-h+6yXnU9kM$^l9yy`QC*&E?(P7+?nb~jiXyZb~gB5Ix)YRt>*%TgU1 zVxy=I$l%?g>RrPIv?v*Ni|&|Rdv<_D1^xd(b5pT#}G z8ZCU7>KpuA%4ZR6;#*fpDRE`uc(8wZRFTNOVl{3(eK(CV4XTett6~rVZF55YOcmS1 z?DmwO$EMx&uHil*sn5bm3to415$*?(MDQgDO!vd-E6yG)^osPA2;JE8)3BBHXsIz(&X)SUuk}Y?gG-oGnMPr^Uv$WDV-dIXtf2`d48gG$)Oi2%m z52w9D9P5X|Jtv&^gb>wBGK(hdn^ItMufa?1M+J+~=VdJF86UriaNjbfXU{6=jXyQ> z^hrb#v&b0orLitsj-5d|$Gt@U+*B0CrgL(IZS|TFdW*Y;L9>Y5QI3+LHNvT8Q%}Tj=}150cx;g_zsTR!l_7C&*jq7dM$x+9AM7c zLfpugAj@Kl-9-S=_30v$povfZCO2XGb;G*q9>`tXJmBi{%9n7oUJA$U+2_fSW!HhM zCDnJ-RG~cYmrjCm!xX6?wo`Sn?L2z;>^lh2+XX757l!>&+T*cNqtn7&4jGKvnb)HX z%rv42(K^!Y6p6ug9}TMN4g3$vVLfQ-J82Ql z8xP7?_;~f}|6y{-uzP*xR*i`*8$%$Sgd)#Am=qu~oXEmVXOS+)l_ydrpMfu*MGOC| zABz!Z#kFQ--#E_2DMek@T+rZXn-{}5FnfhIGt14MZi45ybS+MhEY5{2KoG@J;V_-G zG$W^U-d8>ye`gAZ^?yhg{iH2^wJ?@lrFU0H`vKXQ#-lO#CrTdesQ<^_z>ocrlVMWS zZ>4koD|Z~KAXk6=$3Pf3@&EZVKmt-glP+KYn{+iqq|mCP-D)rjh-Kuv|KOSY?;EXXx`2bh7Km&5s74+`M9eCI-*Ey&TB@7XHvO4O27r|}M2iL zdB%Z*uWlTSw4$NlP3Jd8s_LOTy6j&VAB<;3fU<0nC)FyJ?8Y=}sZQBcaGn zbi_rB6*aD`@1H9^#EmeD{h0=6DF$pCX9d3C^*KdiKnM2>-DWX;k-YB`a;F}kjOITe zI`6*_h}==M1#j&U(sNZ;SUL-QRNS7+=ERCQ2mBmv@YjN&g0JVrXmQqbh5bCunHq5E zf3uTqcJUtGE%w%91@wmgFY*^F^^VEGADtqEBc;R_c%EatYnyO#D_@UluL>Zl7G#>|RE$|>6Q`CQpAbZ!3T(D? zEdL05fC0!Hs$pYlLt+XO?jM7aV8Ia_M_hy6#M!i?65Riic){Vi>*BAq5m3a_SbAgp z?6r_ub==j}O5Otne(Uzb%XW^?uBLhz);ZAEU!fDU7o|-XdG#Pi57NX7XVoP2U6;JP zZf#(FY@rm?z!=p!Y{=}%#3_7&HH|2AD1ijDt2JQwDCX-$5;XHaSfRMQK3f3E|6Zi{ z;YjA$M+Osm%@+#gkzt@`86Jo6eKK@n@`l?YL~;%~JM z&!vWV*clToPT5Q22bk)AHXn-6dJaBN2>H9*fKAV-F&aurX=-I739XW&o)FSn4Vjm2 znE03CGvj(d;%dDoE~gz8qx^>@} zYPw1y|MTj`(06L0Y#cVVh__D&?0g;3ilyPMd9^r}zN7t-R>WeU-`@mqa1A4iAib4zdNo-tW(hY{UU zeLn+e!zPlk1)s6+v@P^T8|LeY_yWA^Pz?y}B}5<+i`2oA2JiXa1){G(VC{m6Ib0;n z?GmsodBD>k7VnHEOR~h_y#I98__IfVQ^t~aO-$`^YY(e1YhBr2#bX}Y)D5c zaqJ66ThoT8b8rVA0r9T&;-T?N1;I$XKTPOz@WVYA2Jvzo=*TGkB1V+k-f&J77tuev zK^bC5dCD{@xX&$j5rZ2#*n~m}5n=y=_mPh-^Ewg_qF@fDaM9f#Kp4k3{u=`z#9|*h zch)GFZ~x-1$er-&kDR#!_jarc4G=tU(H_1Z>xBel1U}*~M1pnefF!?!4+l@|CjZ#t zs-T7X4B&#hh;MNPPg4Q}2^$pU$l(noSJdag>V&D1vp?FVzVni-GUQtPf1Y;6wwVDL zl5mnQ$qdrx*2nYyEAq^E|EawhN}&F^1@Zrznf_M{1;6$$Vioy4UNXm#|AaRFZ%M5G z+aJ?tFEs5Sv4!HDe+E&sT>41&Czk%n;=~Z#vh+Jo=b`X1gti_w!p$GZc@I{p8Yu5u z8foj#MX38jIHl2Iav`U63ifo;mwX8z`!jsv3LmXrZ99wsn*RTatLy^Q;(Zo2q3{&`wg*Rw=qa8tYO)JJ*!d0ppE(~GGk_vxBvCMMsCiuyLU zwyZro#)?ZyxY*b_s;a7Fl+QezYzz(4o|~F#>*=M$#l}`R`kLCwpuek ze;#ja^y64tquOwHcQ-D0;q3f-Y-~&>a)oNUrKLrXk8k!Vwg+AHQ*_kC#Kidc_{G-6 z#l?Habr=jJIOwkwUfh566fJ4xE7x|+=g%yz0t6FQkEz}#CKi>H2!usNeQ#`(5%M(D z(Xlt4+20?`&d#22#YBA&Cxv(Eb_^t3G|Cg!Gd1w#hkb-emy z9Q1m^^m-z(wINl#4OGn6GbT$*OPl7uE`Rg4_w*2oL_Njs=;+wGObrSOioW)Ge}7+F zK_Li_J}xe9XmD^p8;yvRbp8tkN_7Y?$~j8IbN9X^Kf5+G-Sf@OO+6)FbMvKN`LAYB zcAhFLhw2*|2GLMozwYqOzokW9*TA5>p+S1sLnE1SW_9&vnJ|hQsWHknZ`l=zlC-~H z2j_>~AG5LY@ciu;{wCmdpB&5E+0|9o((_{@1ATgJN+w;OXTV4>0)O$W3V>H&BHV4PR5&*oct|2yEQN<$gs7mySvK%=`c!t zvX8i=q@?eWJj$`J#ZvvRUlI4!)GiNdzL>FEEIVR3I5=E(9H1VHvZJHani?6^7Z(>x z{4Un;H1YCEVL)-Ci9LhR38B{~5$CU+zjT6ML-?T@>#=+4HF@&MZ1eBk_9_)}**ta8 zAm;`nCEs*rX`G(RQ?e;E>~v2SP`PvQ;y&I>cKW;ah^UtNO}xCNcv7pGUNF769%pd^ zq1i5ILy?)L8QWKRtc`S(B%&n!L$=&c2;htNLLeV*PRu)j$Ncd4+zj|)b=(aCd0KP; zY{mz}y<8B;eSv4p5XiMt@=OrOY5x3Y!CuyP4Gb`gdmxbIPkNo3^CcY@MMZVicR`+R zj$Z|D@pMC>a~A~Cc>aHXyMQF{E`dt_^m5LVZo^LH7y0v$Z)HwtA}H;;-f1m zDj}`Nelx!COB!K;fk%ug-Lg1gzQcXu;C*ad?{sG6^wa$=T3DX*FZ3r%heQ0(dJQ!_G(=FZQ``gWB6 z>Ol1leQT-D1U_3L+1jlI6Fhn8PUlCPvAUPoG7bnt0+VOs){~Z%n=3$c9DaWf>oEAv zAL=l0R@!5+r#?qNFKlGQ93{@j%lpYd{KgHv#vqOi#ZXwbsewU%Qaf{GWMo))c#8B5 z8D}3$%dsSvu|6J6**$nqe~hq&oP)^)$AD~)?Tv4hJPszi1<^Cx7s@;z$B#Lbghq*) z#w+4qR3?1PMJKsj&^X*9hel?iBno-v#Y5a^=gpKeoAlm02K7x#=_U(7?^Y=i^Fw`X z_I5Xh2A4e+o_RMiHjkw_%3 zgk&=t8=E`*-6X_jRNgE9U>*sfFsfpn&3#FV^MeQDg^od`f+JpfumS8|W4iM)(^b*Z zc4zG1Z9IE^d$P5HS;zf8Sm*oD^nG@TIH~=M_`6uo)H$}<>k6{@b&mF>a{bANkQIJI zT}R2g@5ozyj#4FY5b}CdwQM;`WQf6<+Vt z(;H*tP*nEi`H>nzjJRV=(i%2 zL{?@Qn#F*l-W^{lMnY7iW@XvljSi8c-nQ6%4^xJ4btS35sG-BI9kTL*T4t54pFc;m zw`-2p$bmC&iX928w(l3>LYxvXJj)`tk@PuQRK=-s_EN>>RIK&4@lRn95pK%SuTS!k zYby_KHoQK$v;apFUPbKz`0$}mR7y=n#lQCB%w$VA^_3MeeQKb{Dhz{xKCb0U^wECQ z7LzublESTIQSvB=`4$Uo+qn}z9tfc1+o!cAd`@?x;rLC>ETc~34geQG-+R;%I z0&x~^+8bY4DRt$_l`rSgYirGmOG?s`l0HCbTcbO*oN_*f=04w)Jg4J5CaH4;m^5`v zs|mcu+P(_FuB+swnZH9XmbhB4RlChrgWB`m=^EkQlp2HL;^N6)KR)-Jn_pOn%+Jr4 zkJmOe6#y{gi_fxs`SPXB=MtYy1Uu9zj@gRdy2UTu^+a#|L+u%uDLZ;~FCi+ll_^Hf zIR-c8N3i>jOWC1h=7;Y|*~OBQl9h+C3HOIGgP%7vG#u~$Qsr|4ebMo>SMCefBQFql z_KM2IO-SA4O77umap!bGZ%AuaH$~t{OI#-PWjaCY9&!6hcQlaHlIZ8Syx4S;!2gStP!}b)?y_&wUTJhvN41A+{e!{G zva()qz4hEfrpV$}F&K=A9jpJGsSsuqVK?NweDTtyV8aoM!O%wUxwW<4jj`5^9KHc;psrl! zieLBkQ5K?pHqY zdFa96eu>_c$|L>lH%D1M&igyV0{n)!vu@6@&m-ytq1v*FRX0cDLwz!iu(o)ZB(1U! zU_-G6*P2YlYD-M-QG*Y<(?jK!$COW*JP&S~C^9xS&TVW|=5o`;Ra6YBX=#;b=j5!4 zl@s!&X)E1Yc}fANW?8vlzSWVTp_U6HHPZKKva+)JHZ~ar_HcMVDMK^fPES{NLog#f z>!`F_f;B$M2n^|C@?#*&^m}GqmCHz# zVlICI*oTu85xu*2f0f3*vd=g}p-3El1beY%bfjnP_NGb4T`c-NX?O1g5j~`w z4WmRPpWk#^E|=Rwm?DPweczJQQKS4H&%k$Z^j@r(ez-iWc=;6e8y^JRse#P|t~^p^ zj}a3Sn^9>jcOqu{_XgF)iaU1I+f^dSJy|=rqKW-b1EOx)?h^oxogHv)X(J&ii^X!D zOQ{zxUYy~}3g(i&)FGecv!glZix3xvj0kaKe?+{c76B)Zihx8W#{A{vNMYN zvI}yyc7XoGQ%^54A}Wfku1DsM+uq)mKc}Lqrq*TSaY-ZQqI#4-=wOKrzQA{t-;+9< z@tj!_0y#?`@KxrEl1SMDx#?3xV<7Z@L8yy1R(dG=%lKt8MV0R^zDgESX9&{ zM@L7mp)Cqd7@XO9cid(heBKFczVqe3f$6S{Kf&~-R;&~9Y}VQ(!~ zRxe%ic#r3>^$}}j-Zs5uRde3R-w>hTtzfHIIvup;8?q1W$hxN(Zr|$18oL=;X z4xmNf)r2T3O6+J-Ihg(Ph4 zS{h~M?Oiou2vCMo%tA@&<;c@)SlrESm8%>MT!Vvyl_pq+X!w&3pBYoioi<1Oi7QjnP38EYnAp?5{*p}C3zlF72krRRp0mA!FI&v4>Q zTkt05(W;O1*y9T*u`_j>9$ll(5+{1l4VRRZl)8q7vJ;ER6bc5+msvo+%W zzJo(RK!7zE)?S7hJZeOXWx%wgC*J~=GadqjMnrHhGb;Q`OE}-Ew@c5_=B8k-@apPn z(J4nKr>Nx)m<3bPrvNtpYJqm3xsF*lphGnO!~rH|&bI&vEER*78cu4>%m$nieB&v- z3-V#WYKuMI@DEBR3hoUg<>g-pvj2Nz>S#jzI%Ov zkOAEWuvS9c~p>)-t9EYF^uW0~!$bH_5{ z%G+!bQSP6IxP+U6ahnU&+NCDj1@?eMnqj%4fa=|yqB#fcd-p<1u~>3OE_G`Q0G|?- zN_E4;#9ReSjERX!O-oZs*NDA@qpuc_szzz~1=~T;k;@ZJAsCwk&r6x4EOmqE=;*G& z!K}og-4Mv5l2z__vnqF0fb5e~Q>}e@CacaaE=3MQrK&7Bvu`TRp8(>~I#%z`6bVC2 z{>-ch&Q!2BS?wj`?n?S@zc+PlyAcKyXUDI@&hhduKJo&=uZ zGx#%eg)I=u2^GMa`tZ>KU0t1>t)`}~zM7U@q}31J5lDoU_1+yYDhGCsfADK zYxPnq|8)SGCPPOdkhqRMZ?^|Q-oVKDS-4vNG&r;W?P2|IO{58F*G#f(=A-HqOE%S_URsmEXQKmf#pKjxc`VuCLgM-3k(b_Jfm%30PGcB zsbA+uaTdSHrTd*S+grGuDhj2fLryx^8VIC~V?FHgz^RG8W~p%3?7f}6eOhkr z9hUx{o{d*b+8-}Lq5b_qKV^G)dn0di44f0c%FmxxSa^4RPY_2CN#b=k!GB<3^lr#L z$~GP&98zU^0ql)eb=X>L!Cu54UfoI_e_>daUqzX92k!C;I}@@bX= zT9Eth-T9i&i;Ig-|Bxo`V1%42JDKcSVd`a)F#2wOPXKPz@ zhJypyd5<nmcQOef$mq@pJ7a3&rf=gtia`Vz*= z`0|@KZ&J(3^kCM#qocy$4t4eSXGq_e|IIxaD7K)H4Un)1#vB}1Z(lD>5$~n#~ndrx^z`(!jg7BnWn~q6P|Uf^UnQj76F#wx+u4E*mfD8Hixo@Ahm)dquPK3bZ ztg5|AieeZ)`RI8&;P9~dDg?NJq~|PhKkHWk^n9>>@|bY##yj|v^+h68#8H^l#~ih) zM~l@1-?Unkl_Ccc5V)K_+Ba}y2qxbr<2ESE>_OvKQ03sbAN^MkzTH;%V81C%yAHyW5b6207`qcyue1;&5f zx>!_yf~@zuLEyAK9oQFt!@h4O$#u2YKpo7CT4A6!KK$(M*0bg6yP9LsqGw>nYjt$} zJ9ks4RpA51(IcQMrSibT7K+Ynt{+G3%J-$aiwmQLr62L#1)p{##04Pzb#aDpan zu!pu*jthFU#n?DDH-_B%iu@H9Vp{n80KTUW*f(jIM<>fyXF8s>-5}dYdd7DpfSa)Su00(}lJCQCn zpQK}A!VjoaEnii2{hk|QV$I-l%tfop3JVM8g7oC+#_=++QS9-G{>(3RmCa(bwOq3P zj&`Ap)Q%Pym#KAllY6ch)%sM(P-jxYVq-aUX*D%9)cuow@dnn`X+NDhMW#EWC+=8T zb9fT6f-(0Ab8zDBJl9OZg{}qY5CInF`0Gza4k;(stu7;9m24L|A8Ak!E3Y@)sMgch zSJTuCUOev@k~e3q)Ixyc95te^^QK<^8-C`crd}k%ot&^;-Q9U)vYl+(g8Q5&ylO#w zF3QB|Tcn=GXpxo52Pz(qU-eO}0yZq3gqXGLHF!l25)F!UoOHB#@Zdq`!fLsbU!P>p z!tvQPj)euDc5&x7hS)E!w?rbO>W%1z68j&Viy>GF_>7pM9o5B#tfsw`1ZMz{G**tk z;jg<|^^bOU(yVax*Jn4Z8js9@d3gBB%*vm*jgrrJ_f7%s{KGR$12+52sQ`yOT-u)O z6mq+K^f7$L8SAIhP#mN=9Fp9_-oDgSCvxoYmtkhBv1R>x_bjb#Z03Zz)&?rf<#8(0 z{;=(_!iipKTi{BQ0FEyKw-O&G>H17AHgzO-_^*XiSebo)YlrWeC_8j<$IMn5MJ=Uo zag1)f($&-3S0*#^-JUooHDl^NXk8y{;AaW=crkGHXqldutIyede70}t*XJzcw*Z5o zEp5pO zl`3jr2RYezz=926ToC2M|MzTZyoh|$g1b5bQS9d75zsUdT3RY*Yj3}-vD1X!R*9BF zRmLIvjIP^uU96ajv7hWrR{V`|#acbnG&75jh=`bZs$*dx3^ar`FzsUHY}z?J0r4b- zptn;%oMIw?4+Bu`P7>;Fc$tEuqxlB@Wh$> zEA%bOoTx0=uVAT^lz*e9df7=mBtfM6boFw3I3NVnJY0FXL1ks-e&n}Q;HLO1^!k@z zG2Lq#G6h?Pb`y`w`Ss^yWAN;GmGNG88s61JS~(Cq?hiIkd@~dj6x6q}N{Nh!NZo2{ zYt#4gs?;+x%gd-BI*o3F*d#MDD(X?zqmDR9oeqxMo}Lv69*_G62W>y?!Qj*S4OYi; z7f7Y3lO1z%Q9}|#3vYV8LZ%SPIZheeCyO2z*azlq-Nhz+VFz}r=4*2LI5;fPvffxQ z+na#|d^l3==`{pW6x1?$T6#1T5LI_cVljdY(ke)1QZCmZUZ zqtjhzjoTPM3^?$!?IjE@0$9D^ULj|%g7DTq0Wj*z1z_$upE2o}nI9@sKXc~HMJUt& z%P=ng=1r`tkVJd4ww%G|#_6G~au1W;!D^u`0A0fI~Ei3NQcm zMEVM!=dXTD4*`9nx-0_^gQ@KAeTrI`idnW4U{)XpVmT!FCI$(B9i{GrvzmPX8rCM^ExAY6l1x0PzBTCw()Im@^33zoFmLnp5IjE+p`lTM&jGWkcs}R{o8gUXL_x~^(L0mk| zzFcW4B&)IdoE;W+lT_VZm(@S}u7*qF#Ju>={*KnuHfQ4&I9RqGiuH*`5!x0}57(C# zCg0pA`bTjEEj+gG1a9!}7H8)3U@{a6H37U{ABikWm~yY`xSY|*`qNK_9O4cK4AJ10 zG2a*g*NX`6%*xE{41=mV+>{OEj2#l845?iHq8 zJw21LJ78jm17i;l&i@5z_xS@?e!Bp3ZOyI3RfSWRzk#!7kgV;Tf+dIRW&ysVLvwwgc9{m zu-9IqeOIb#*e}1F?kRrnw(t9Zl$H8TQe<~@&J(K)mX?>9UUH@ZPaS8X5-noYm}Co3 zRq8TesuGJoB(!Y%fe8D*n9%<9D5UJr;T0=g>bw^&D4sZRViM%qSS%Y@SK?nxL0<;p ztbS7DFi`lh&wV#w*jE4KAr6(wE~GG1aetzBcm&tX+x*fI27YyU5L_^Y+=Wj3V&vSw3SmpF~#Ps|C^BruGi^sEwe09aV{<{#P)^@gF>xrx%9Bd zF?@rirKSCCIa4}jAa|i}#e0l|y0%NIz@FuED}kSM9tPqjE%QfOiB@jD*JCzhLG8j! zg8ES}x+4~Z5FYBZ15a~mz$-*E_cs;#Y3=`;ZNHfge4}PAkKuE`+-3xD&|&B2 z|67I2r<+o{<+Mqp`0_CvvQ%(h<+i^nR?w+)zm+Gj|4290K*B9U-)hQeZvD&V{rEhM&3zh?F7(O|Ua#M>)is|G1Kj(gYRJSNHNT z%3ApSue(u=D)9%GGX9{Tp_#vr%#cQ(kBlT8KUL(w%iLeMPb1YY^yt4fm&ndQL9 zxHuo>O>l1JK#d`h)@NaE?v$m+$#C&s^_l~^iqG*Cn>t)J{{ea6AD-q($Zhs;H9jRhKM}I!X=g3q5U~Q)&vZYhak+j z4j++`bsTSaZP}0WrFo$S@r<^VO3w%|^QCd+eR`qG&C5EK86eE1JZ0NA_@~z3yg783gTK%S`L`DTwd3&UsrZ%1GMoZSdR@b z?7BKT&sJ1a6eH*KWlcc^=J!PYI#4tqC=WBRK=m4|eI2W(awis`X>;QFV6r)12c-Bi zSv#`w7IJA)6_t`FW#u|QAku=zbBK+$M)D)SdXlp;RtA6R%`wOa2OWva%hTAIVx3QE&(8WVI5N^s2_6hYjS0-NOy9SVfLnk@roX8; z=zvmJU6o#PG8^06-SR_?9hpCDV~Lhv+NzUOsBuVh7QN2EsXq#%nrdoqA~2Xax%p4E z&dh5sOA29e&rz0*wYLkaJ4}Jkdiqj6Be(olz0vVYGYmUN+tBbbAd^eI1-^RBoD%mv z!pFPj3UESREQ8KO#!eVfW z3F{9k86WIYm%8vEQQ+t1JuEUwxSYOxGkXu6@h%`}t!ijY?rf&c^0$!gG|xXmN?gern}NSK>dW?76^_P9*DwuC-Y(L79WrTQ}67apLVZWjpkx~3M_IP57+So zs$=cWcE?VHiBrX=#q@?W(9<+!bj=v2>>O;fMQakwk|>Q>@j+tldzq9H*J(}L$Bz$G zF;8^NFq*N{ov>3vrV=yIf}T+%5{bngbq2DeT^{Cp*gH-RWlUd3?-(Q+>0k=)vmmj{ zs#;q8HqyYeQmBe$#prhGmP=-;8x-!m`3y1@ zULcoxFnORAD7V=Ntvv_ELVhjo+UZ6nF33mPXbyG!!0Hh6oeC;6r+bh<%Md{~fFSc94awf4s9*F*D2QltU(Rd0><)eVr`dxG%pS#7SpPbZ@u1e#y1fJiZc z2cDjq8tN1JaD@>SZ_+5}g2bP7pr7ivfhl|)cxtjT05?$LmY{G%ePq}WT-hrxoB3|B zV& zlaBk`YxPVRx4py;y==9TZ_Zfad(Y6yKZ{My-Xq zPSSny6&O7I@j6*=!y_YEd#Xo}WBs|iY$m!`pJoCPw~8b;UFl^{@SD)RcG_zexSBKM z-W=rMd(Eb2+AN&By|gF7z4hRfTYEPi(x(6l~yaNPG$?)8RMbL(>I!6 zWN+tu9G&`BC|W6n8L|e}1t9>Rt+{4>B*&lOYO%e0Nf}(SUb;*fj2k4?6MYVO zWgcYgN3%4(WPpizPCMh^H4Ua!i7(x+t$ z8mh*O_=GZg4|=t{tgHo&-ZBA6`E}5~Pr`ivEJI>LcLhp<0zv*fp!vz+}KZ^wGVhB5LG43Tlot2 GpZyQ9t*|Qq literal 0 HcmV?d00001 diff --git a/doc/modules/comparison.rst b/doc/modules/comparison.rst index eb8b33edd0..957c974d16 100644 --- a/doc/modules/comparison.rst +++ b/doc/modules/comparison.rst @@ -276,10 +276,8 @@ The :py:func:`~spikeinterface.comparison.compare_two_sorters()` returns the comp import spikeinterface.comparisons as sc import spikinterface.widgets as sw - # First, let's download a simulated dataset - local_path = si.download_dataset(remote_path='mearec/mearec_test_10s.h5') - recording, sorting = se.read_mearec(local_path) - + # First, let's generate a simulated dataset + recording, sorting = si.generate_ground_truth_recording() # Then run two spike sorters and compare their outputs. sorting_HS = ss.run_sorter(sorter_name='herdingspikes', recording=recording) sorting_TDC = ss.run_sorter(sorter_name='tridesclous', recording=recording) @@ -332,9 +330,8 @@ Comparison of multiple sorters uses the following procedure: .. code-block:: python - # Download a simulated dataset - local_path = si.download_dataset(remote_path='mearec/mearec_test_10s.h5') - recording, sorting = se.read_mearec(local_path) + # Generate a simulated dataset + recording, sorting = si.generate_ground_truth_recording() # Then run 3 spike sorters and compare their outputs. sorting_MS4 = ss.run_sorter(sorter_name='mountainsort4', recording=recording) diff --git a/doc/tutorials_custom_index.rst b/doc/tutorials_custom_index.rst index 05da8fa7dd..f146f4a4d0 100755 --- a/doc/tutorials_custom_index.rst +++ b/doc/tutorials_custom_index.rst @@ -101,9 +101,8 @@ The :py:mod:`spikeinterface.extractors` module is designed to load and save reco :gutter: 2 .. grid-item-card:: Read various formats - :link-type: ref - :link: sphx_glr_tutorials_extractors_plot_1_read_various_formats.py - :img-top: /tutorials/extractors/images/thumb/sphx_glr_plot_1_read_various_formats_thumb.png + :link: how_to/read_various_formats.html + :img-top: how_to/read_various_formats_files/read_various_formats_12_0.png :img-alt: Read various formats :class-card: gallery-card :text-align: center diff --git a/examples/tutorials/extractors/plot_1_read_various_formats.py b/examples/how_to/read_various_formats.py similarity index 100% rename from examples/tutorials/extractors/plot_1_read_various_formats.py rename to examples/how_to/read_various_formats.py diff --git a/examples/tutorials/core/plot_4_sorting_analyzer.py b/examples/tutorials/core/plot_4_sorting_analyzer.py index 3b49e35fff..1d0849716e 100644 --- a/examples/tutorials/core/plot_4_sorting_analyzer.py +++ b/examples/tutorials/core/plot_4_sorting_analyzer.py @@ -27,23 +27,12 @@ import matplotlib.pyplot as plt -from spikeinterface import download_dataset -from spikeinterface import create_sorting_analyzer, load_sorting_analyzer -import spikeinterface.extractors as se +from spikeinterface import create_sorting_analyzer, load_sorting_analyzer, generate_ground_truth_recording ############################################################################## -# First let's use the repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data -# to download a MEArec dataset. It is a simulated dataset that contains "ground truth" -# sorting information: +# First let's generate a simulated recording and sorting -repo = "https://gin.g-node.org/NeuralEnsemble/ephy_testing_data" -remote_path = "mearec/mearec_test_10s.h5" -local_path = download_dataset(repo=repo, remote_path=remote_path, local_folder=None) - -############################################################################## -# Let's now instantiate the recording and sorting objects: - -recording, sorting = se.read_mearec(local_path) +recording, sorting = generate_ground_truth_recording() print(recording) print(sorting) diff --git a/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py b/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py index a6b0da67ac..fe71368845 100644 --- a/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py +++ b/examples/tutorials/qualitymetrics/plot_3_quality_metrics.py @@ -8,7 +8,6 @@ """ import spikeinterface.core as si -import spikeinterface.extractors as se from spikeinterface.qualitymetrics import ( compute_snrs, compute_firing_rates, @@ -17,11 +16,9 @@ ) ############################################################################## -# First, let's download a simulated dataset -# from the repo 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data' +# First, let's generate a simulated recording and sorting -local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5") -recording, sorting = se.read_mearec(local_path) +recording, sorting = si.generate_ground_truth_recording() print(recording) print(sorting) diff --git a/examples/tutorials/qualitymetrics/plot_4_curation.py b/examples/tutorials/qualitymetrics/plot_4_curation.py index 6a9253c093..328ebf8f2b 100644 --- a/examples/tutorials/qualitymetrics/plot_4_curation.py +++ b/examples/tutorials/qualitymetrics/plot_4_curation.py @@ -11,20 +11,15 @@ # Import the modules and/or functions necessary from spikeinterface import spikeinterface.core as si -import spikeinterface.extractors as se -from spikeinterface.postprocessing import compute_principal_components from spikeinterface.qualitymetrics import compute_quality_metrics ############################################################################## -# Let's download a simulated dataset -# from the repo 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data' -# -# Let's imagine that the ground-truth sorting is in fact the output of a sorter. +# Let's generate a simulated dataset, and imagine that the ground-truth +# sorting is in fact the output of a sorter. -local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5") -recording, sorting = se.read_mearec(file_path=local_path) +recording, sorting = si.generate_ground_truth_recording() print(recording) print(sorting) diff --git a/examples/tutorials/widgets/plot_3_waveforms_gallery.py b/examples/tutorials/widgets/plot_3_waveforms_gallery.py index 2845dcc62c..d2f4345d14 100644 --- a/examples/tutorials/widgets/plot_3_waveforms_gallery.py +++ b/examples/tutorials/widgets/plot_3_waveforms_gallery.py @@ -9,15 +9,12 @@ import spikeinterface as si import spikeinterface.extractors as se -import spikeinterface.postprocessing as spost import spikeinterface.widgets as sw ############################################################################## -# First, let's download a simulated dataset -# from the repo 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data' +# First, let's generate a simulated dataset -local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5") -recording, sorting = se.read_mearec(local_path) +recording, sorting = si.generate_ground_truth_recording() print(recording) print(sorting) diff --git a/examples/tutorials/widgets/plot_4_peaks_gallery.py b/examples/tutorials/widgets/plot_4_peaks_gallery.py index cce04ae5a0..e6e5f6cd56 100644 --- a/examples/tutorials/widgets/plot_4_peaks_gallery.py +++ b/examples/tutorials/widgets/plot_4_peaks_gallery.py @@ -14,22 +14,19 @@ import spikeinterface.full as si ############################################################################## -# First, let's download a simulated dataset -# from the repo 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data' - -local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5") -rec, sorting = si.read_mearec(local_path) +# First, let's generate a simulated dataset +recording, sorting = si.generate_ground_truth_recording() ############################################################################## # Let's filter and detect peaks on it from spikeinterface.sortingcomponents.peak_detection import detect_peaks -rec_filtred = si.bandpass_filter(recording=rec, freq_min=300.0, freq_max=6000.0, margin_ms=5.0) -print(rec_filtred) +rec_filtered = si.bandpass_filter(recording=recording, freq_min=300.0, freq_max=6000.0, margin_ms=5.0) +print(rec_filtered) peaks = detect_peaks( - recording=rec_filtred, + recording=rec_filtered, method="locally_exclusive", peak_sign="neg", detect_threshold=6, @@ -53,14 +50,14 @@ # This "peaks" vector can be used in several widgets, for instance # plot_peak_activity() -si.plot_peak_activity(recording=rec_filtred, peaks=peaks) +si.plot_peak_activity(recording=rec_filtered, peaks=peaks) plt.show() ############################################################################## -# can be also animated with bin_duration_s=1. - -si.plot_peak_activity(recording=rec_filtred, peaks=peaks, bin_duration_s=1.0) +# can be also animated with bin_duration_s=1. The animation only works if you +# run this code locally +si.plot_peak_activity(recording=rec_filtered, peaks=peaks, bin_duration_s=1.0) plt.show() From 9bdf29137475b22dcb6ebcde2b05824c6f9a9a6d Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 12:05:15 -0400 Subject: [PATCH 118/157] add dredge reference too --- doc/references.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/references.rst b/doc/references.rst index 1bf83d11ac..941fad293b 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -24,10 +24,12 @@ Motion Correction If you use the :code:`correct_motion` method in the preprocessing module, please cite [Garcia]_ as well as the references that correspond to the :code:`preset` you used: -- :code:`nonrigid_accurate` [Windolf]_ [Varol]_ -- :code:`nonrigid_fast_and_accurate` [Windolf]_ [Varol]_ [Pachitariu]_ +- :code:`nonrigid_accurate` [Windolf_a]_ [Varol]_ +- :code:`nonrigid_fast_and_accurate` [Windolf_a]_ [Varol]_ [Pachitariu]_ - :code:`rigid_fast` *no additional citation needed* - :code:`kilosort_like` [Pachitariu]_ +- :code:`dredge_ap` [Windolf_b]_ +- :code:`dredge_lfp` [Windolf_b]_ - :code:`medicine` [Watters]_ Sorters Module @@ -152,6 +154,8 @@ References .. [Watters] `MEDiCINe: Motion Correction for Neural Electrophysiology Recordings. 2025. `_ -.. [Windolf] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. `_ +.. [Windolf_a] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. `_ + +.. [Windolf_b] `DREDge: robust motion correction for high-density extracellular recordings across species. 2023 ` .. [Yger] `A spike sorting toolbox for up to thousands of electrodes validated with ground truth recordings in vitro and in vivo. 2018. `_ From 5e0ff32e7600f1f27bf0f2698626e7697e398fb7 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 12:22:04 -0400 Subject: [PATCH 119/157] add motion functions to api docs --- doc/api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index bc685eee2a..b97df80ec1 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -76,7 +76,7 @@ Low-level .. autoclass:: ChunkRecordingExecutor -Back-compatibility with ``WaveformExtractor`` (version < 0.101.0) +Back-compatibility with ``WaveformExtractor`` (version > 0.100.0) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: spikeinterface.core @@ -179,6 +179,8 @@ spikeinterface.preprocessing .. autofunction:: correct_motion .. autofunction:: get_motion_presets .. autofunction:: get_motion_parameters_preset + .. autofunction:: load_motion_info + .. autofunction:: save_motion_info .. autofunction:: depth_order .. autofunction:: detect_bad_channels .. autofunction:: directional_derivative From 6bf95be230589bc73cf94e2a789a5b8fb3291877 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 12:55:03 -0400 Subject: [PATCH 120/157] update job_kwargs --- src/spikeinterface/core/job_tools.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/core/job_tools.py b/src/spikeinterface/core/job_tools.py index 38a08c0fab..fbca2b0aa6 100644 --- a/src/spikeinterface/core/job_tools.py +++ b/src/spikeinterface/core/job_tools.py @@ -29,13 +29,21 @@ - chunk_duration : str or float or None Chunk duration in s if float or with units if str (e.g. "1s", "500ms") * n_jobs : int | float - Number of jobs to use. With -1 the number of jobs is the same as number of cores. - Using a float between 0 and 1 will use that fraction of the total cores. + Number of workers that will be requested during multiprocessing. Note that + the OS determines how this is distributed, but for convenience one can use + * -1 the number of workers is the same as number of cores (from os.cpu_count()) + * float between 0 and 1 uses fraction of total cores (from os.cpu_count()) * progress_bar : bool If True, a progress bar is printed * mp_context : "fork" | "spawn" | None, default: None Context for multiprocessing. It can be None, "fork" or "spawn". Note that "fork" is only safely available on LINUX systems + * pool_engine : "process" | "thread", default: "process" + Whether to use a ProcessPoolExecutor or ThreadPoolExecutor for multiprocessing + * max_threads_per_worker : int | None, default: 1 + Sets the limit for the number of thread per process using threadpoolctl module + Only applies in an n_jobs>1 context + If None, then no limits are applied. """ From c1756292bd56dfd7484e66dad7d64690be498122 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 13:06:14 -0400 Subject: [PATCH 121/157] add docstrings to load and save motion --- src/spikeinterface/preprocessing/motion.py | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/spikeinterface/preprocessing/motion.py b/src/spikeinterface/preprocessing/motion.py index 9fcb8fdbf3..91418f0a33 100644 --- a/src/spikeinterface/preprocessing/motion.py +++ b/src/spikeinterface/preprocessing/motion.py @@ -568,6 +568,19 @@ def correct_motion( def save_motion_info(motion_info, folder, overwrite=False): + """ + Saves motion info + + Parameters + ---------- + motion_info : dict + The returned motion_info from running `compute_motion` + folder : str | Path + The path for saving the `motion_info` + overwrite : bool, default: False + Whether to overwrite the folder location when saving motion info + + """ folder = Path(folder) if folder.is_dir(): if not overwrite: @@ -590,6 +603,18 @@ def save_motion_info(motion_info, folder, overwrite=False): def load_motion_info(folder): + """ + Loads a motion info dict from folder + Parameters + ---------- + folder : str | Path + The folder containing the motion info to load + + Notes + ----- + Loads both current Motion implmemntation as well as the + legacy Motion format + """ from spikeinterface.core.motion import Motion folder = Path(folder) From 205109cdfb03ff94cbfe515648a45b6d8f6f4bc2 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 13:16:09 -0400 Subject: [PATCH 122/157] Chris edits --- doc/references.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/references.rst b/doc/references.rst index 941fad293b..fa798e77ed 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -88,6 +88,7 @@ important for your research: Curation Module --------------- If you use the :code:`get_potential_auto_merge` method from the curation module, please cite [Llobet]_ +If you use :code:`auto_label_units` or :code:`train_model`, please cite [Jain]_ References ---------- @@ -120,7 +121,9 @@ References .. [IBL] `Spike sorting pipeline for the International Brain Laboratory. 2022. `_ -.. [Jackson] Quantitative assessment of extracellular multichannel recording quality using measures of cluster separation. Society of Neuroscience Abstract. 2005. +.. [Jackson] `Quantitative assessment of extracellular multichannel recording quality using measures of cluster separation. Society of Neuroscience Abstract. 2005. `_ + +.. [Jain] `UnitRefine: A Community Toolbox for Automated Spike Sorting Curation. 2025 `_ .. [Jia] `High-density extracellular probes reveal dendritic backpropagation and facilitate neuron classification. 2019 `_ @@ -134,7 +137,7 @@ References .. [Niediek] `Reliable Analysis of Single-Unit Recordings from the Human Brain under Noisy Conditions: Tracking Neurons over Hours. 2016. `_ -.. [npyx] `NeuroPyxels: loading, processing and plotting Neuropixels data in python. 2021. _` +.. [npyx] `NeuroPyxels: loading, processing and plotting Neuropixels data in python. 2021. `_ .. [Pachitariu] `Spike sorting with Kilosort4. 2024. `_ @@ -156,6 +159,6 @@ References .. [Windolf_a] `Robust Online Multiband Drift Estimation in Electrophysiology Data. 2022. `_ -.. [Windolf_b] `DREDge: robust motion correction for high-density extracellular recordings across species. 2023 ` +.. [Windolf_b] `DREDge: robust motion correction for high-density extracellular recordings across species. 2023 `_ .. [Yger] `A spike sorting toolbox for up to thousands of electrodes validated with ground truth recordings in vitro and in vivo. 2018. `_ From aa3364900d91dff676325b139b5a078e0bb32ad8 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 13:34:49 -0400 Subject: [PATCH 123/157] One more Chris complaint --- doc/references.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/references.rst b/doc/references.rst index fa798e77ed..e4b23f1b76 100644 --- a/doc/references.rst +++ b/doc/references.rst @@ -88,6 +88,7 @@ important for your research: Curation Module --------------- If you use the :code:`get_potential_auto_merge` method from the curation module, please cite [Llobet]_ + If you use :code:`auto_label_units` or :code:`train_model`, please cite [Jain]_ References From 0312e7e5956e23986ab4949cb6a3b88300f8734b Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 13:37:19 -0400 Subject: [PATCH 124/157] fix typo found by Christopher --- src/spikeinterface/preprocessing/motion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/preprocessing/motion.py b/src/spikeinterface/preprocessing/motion.py index 91418f0a33..2243412663 100644 --- a/src/spikeinterface/preprocessing/motion.py +++ b/src/spikeinterface/preprocessing/motion.py @@ -605,6 +605,7 @@ def save_motion_info(motion_info, folder, overwrite=False): def load_motion_info(folder): """ Loads a motion info dict from folder + Parameters ---------- folder : str | Path @@ -612,8 +613,9 @@ def load_motion_info(folder): Notes ----- - Loads both current Motion implmemntation as well as the + Loads both current Motion implementation as well as the legacy Motion format + """ from spikeinterface.core.motion import Motion From a6266d44a214b07a5d2f6b21f76c996e420d80b4 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 12 Jul 2025 16:13:18 -0400 Subject: [PATCH 125/157] updates in the get_started folder --- doc/get_started/import.rst | 2 +- doc/get_started/install_sorters.rst | 18 ++++++++++-------- doc/get_started/installation.rst | 13 ++++++++++--- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/get_started/import.rst b/doc/get_started/import.rst index be3f7d5afb..30e841bdd8 100644 --- a/doc/get_started/import.rst +++ b/doc/get_started/import.rst @@ -73,7 +73,7 @@ For example: .. code-block:: python from spikeinterface.preprocessing import bandpass_filter, common_reference - from spikeinterface.core import extract_waveforms + from spikeinterface.core import create_sorting_analyzer from spikeinterface.extractors import read_binary As mentioned this approach only imports exactly what you plan on using so it is the most minimalist. It does require diff --git a/doc/get_started/install_sorters.rst b/doc/get_started/install_sorters.rst index 61abc0f129..abffd2036b 100644 --- a/doc/get_started/install_sorters.rst +++ b/doc/get_started/install_sorters.rst @@ -12,7 +12,7 @@ and in many cases the easiest way to run them is to do so via Docker or Singular **This is the approach we recommend for all users.** To run containerized sorters see our documentation here: :ref:`containerizedsorters`. -There are some cases where users will need to install the spike sorting algorithms in their own environment. If you +There are some cases where users will need to install the spike sorting algorithms on your own computer. If you are on a system where it is infeasible to run Docker or Singularity containers, or if you are actively developing the spike sorting software, you will likely need to install each spike sorter yourself. @@ -24,7 +24,7 @@ opencl (Tridesclous) to use hardware acceleration (GPU). Here is a list of the implemented wrappers and some instructions to install them on your local machine. Installation instructions are given for an **Ubuntu** platform. Please check the documentation of the different spike sorters to retrieve installation instructions for other operating systems. -We use **pip** to install packages, but **conda** should also work in many cases. +We use **pip** to install packages, but **conda** or **uv** should also work in many cases. Some novel spike sorting algorithms are implemented directly in SpikeInterface using the :py:mod:`spikeinterface.sortingcomponents` module. Checkout the :ref:`get_started/install_sorters:SpikeInterface-based spike sorters` section of this page @@ -140,10 +140,12 @@ Kilosort4 * Python, requires CUDA for GPU acceleration (highly recommended) * Url: https://github.com/MouseLand/Kilosort -* Authors: Marius Pachitariu, Shashwat Sridhar, Carsen Stringer +* Authors: Marius Pachitariu, Shashwat Sridhar, Carsen Stringer, Jacob Pennington * Installation:: - pip install kilosort==4.0 torch + pip install kilosort + pip uninstall torch + pip install torch --index-url https://download.pytorch.org/whl/cu118 * For more installation instruction refer to https://github.com/MouseLand/Kilosort @@ -240,7 +242,7 @@ Waveclus * Also supports Snippets (waveform cutouts) objects (:py:class:`~spikeinterface.core.BaseSnippets`) * Url: https://github.com/csn-le/wave_clus/wiki * Authors: Fernando Chaure, Hernan Rey and Rodrigo Quian Quiroga -* Installation needs Matlab:: +* Installation requires Matlab:: git clone https://github.com/csn-le/wave_clus/ # provide installation path by setting the WAVECLUS_PATH environment variable @@ -270,7 +272,7 @@ with SpikeInterface. SpykingCircus2 ^^^^^^^^^^^^^^ -This is a upgraded version of SpykingCircus, natively written in SpikeInterface. +This is an upgraded version of SpykingCircus, natively written in SpikeInterface. The main differences are located in the clustering (now using on-the-fly features and less prone to finding noise clusters), and in the template-matching procedure, which is now a fully orthogonal matching pursuit, working not only at peak times but at all times, recovering more spikes close to noise thresholds. @@ -289,7 +291,7 @@ Tridesclous2 ^^^^^^^^^^^^ This is an upgraded version of Tridesclous, natively written in SpikeInterface. -#Same add his notes. + * Python * Requires: HDBSCAN and Numba @@ -314,7 +316,7 @@ Klusta (LEGACY) * Authors: Cyrille Rossant, Shabnam Kadir, Dan Goodman, Max Hunter, Kenneth Harris * Installation:: - pip install Cython h5py tqdm + pip install cython h5py tqdm pip install click klusta klustakwik2 * See also: https://github.com/kwikteam/phy diff --git a/doc/get_started/installation.rst b/doc/get_started/installation.rst index 182ce67b94..d432503098 100644 --- a/doc/get_started/installation.rst +++ b/doc/get_started/installation.rst @@ -81,14 +81,16 @@ Requirements * numpy * probeinterface - * neo>=0.9.0 - * joblib + * neo * threadpoolctl * tqdm + * zarr + * pydantic + * numcodecs + * packaging Sub-modules have more dependencies, so you should also install: - * zarr * h5py * scipy * pandas @@ -98,8 +100,13 @@ Sub-modules have more dependencies, so you should also install: * matplotlib * numba * distinctipy + * skops + * huggingface_hub * cuda-python (for non-macOS users) +For developers we offer a :code:`[dev]` option which installs testing, documentation, and linting packages necessary +for testing and building the docs. + All external spike sorters can be either run inside containers (Docker or Singularity - see :ref:`containerizedsorters`) or must be installed independently (see :ref:`get_started/install_sorters:Installing Spike Sorters`). From 549c8dc0875401600678a9a7421d51231b0ac330 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sun, 13 Jul 2025 11:34:27 -0400 Subject: [PATCH 126/157] update drift doc --- doc/how_to/handle_drift.rst | 2 -- examples/how_to/handle_drift.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/doc/how_to/handle_drift.rst b/doc/how_to/handle_drift.rst index 86c146fd02..f1d4dc0a0e 100755 --- a/doc/how_to/handle_drift.rst +++ b/doc/how_to/handle_drift.rst @@ -1322,8 +1322,6 @@ A preset is a nested dict that contains theses methods/parameters. Run motion correction with one function! ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Correcting for drift is easy! You just need to run a single function. We -will try this function with some presets. Here we also save the motion correction results into a folder to be able to load them later. diff --git a/examples/how_to/handle_drift.py b/examples/how_to/handle_drift.py index 02dd12f8a0..a38734991a 100755 --- a/examples/how_to/handle_drift.py +++ b/examples/how_to/handle_drift.py @@ -109,9 +109,6 @@ def preprocess_chain(rec): # ### Run motion correction with one function! # -# Correcting for drift is easy! You just need to run a single function. -# We will try this function with some presets. -# # Here we also save the motion correction results into a folder to be able to load them later. # lets try theses presets From 46e81caf7871afdf1bd4839ed0980bcd3a66d3f7 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 14 Jul 2025 09:30:45 +0200 Subject: [PATCH 127/157] Check for remote paths in check_paths_relative --- src/spikeinterface/core/core_tools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index 5a688e4869..5f17804604 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -429,6 +429,11 @@ def check_paths_relative(input_dict, relative_folder) -> bool: not_possible.append(p) continue + # check path is not a remote path + if is_path_remote(p): + not_possible.append(p) + continue + # If windows path check have same drive if isinstance(p, WindowsPath) and isinstance(relative_folder, WindowsPath): # check that on same drive From 4e090b21b058eb777b06236773ec47efc3b6e5e1 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 14 Jul 2025 09:41:43 +0200 Subject: [PATCH 128/157] Cast samples to int64 after round --- src/spikeinterface/core/baserecording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/core/baserecording.py b/src/spikeinterface/core/baserecording.py index ce95581be8..75c943c647 100644 --- a/src/spikeinterface/core/baserecording.py +++ b/src/spikeinterface/core/baserecording.py @@ -982,7 +982,7 @@ def time_to_sample_index(self, time_s): sample_index = time_s * self.sampling_frequency else: sample_index = (time_s - self.t_start) * self.sampling_frequency - sample_index = np.round(sample_index).astype(int) + sample_index = np.round(sample_index).astype(np.int64) else: sample_index = np.searchsorted(self.time_vector, time_s, side="right") - 1 From f7ff078ee59b860ee6c67c23357550d3d1a774d7 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Mon, 14 Jul 2025 05:11:39 -0400 Subject: [PATCH 129/157] Alessio fix Co-authored-by: Alessio Buccino --- doc/get_started/install_sorters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/get_started/install_sorters.rst b/doc/get_started/install_sorters.rst index abffd2036b..82bef42c14 100644 --- a/doc/get_started/install_sorters.rst +++ b/doc/get_started/install_sorters.rst @@ -12,7 +12,7 @@ and in many cases the easiest way to run them is to do so via Docker or Singular **This is the approach we recommend for all users.** To run containerized sorters see our documentation here: :ref:`containerizedsorters`. -There are some cases where users will need to install the spike sorting algorithms on your own computer. If you +There are some cases where you will need to install the spike sorting algorithms on your own computer. If you are on a system where it is infeasible to run Docker or Singularity containers, or if you are actively developing the spike sorting software, you will likely need to install each spike sorter yourself. From 67e1cf3dc43184e6059f8a8cc2e495924296eb39 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:48:07 -0400 Subject: [PATCH 130/157] add github token for #3371 --- .github/workflows/all-tests.yml | 1 + .github/workflows/full-test-with-codecov.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index 088c703e64..9481bb74f2 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -11,6 +11,7 @@ on: env: KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/full-test-with-codecov.yml b/.github/workflows/full-test-with-codecov.yml index 9d56be8498..03f47bbef1 100644 --- a/.github/workflows/full-test-with-codecov.yml +++ b/.github/workflows/full-test-with-codecov.yml @@ -7,6 +7,7 @@ on: env: KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: full-tests-with-codecov: From 7b90e8de1fd1573ca0e2862e4dcd12e3503ec762 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 14 Jul 2025 16:23:22 +0200 Subject: [PATCH 131/157] Fix Kilosort tests for 4.0.39 --- .github/scripts/test_kilosort4_ci.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_kilosort4_ci.py b/.github/scripts/test_kilosort4_ci.py index 17a55adf5f..349e09eb2f 100644 --- a/.github/scripts/test_kilosort4_ci.py +++ b/.github/scripts/test_kilosort4_ci.py @@ -288,9 +288,11 @@ def test_cluster_spikes_arguments(self): self._check_arguments(cluster_spikes, expected_arguments) def test_save_sorting_arguments(self): - expected_arguments = ["ops", "results_dir", "st", "clu", "tF", "Wall", "imin", "tic0", "save_extra_vars"] - - expected_arguments.append("save_preprocessed_copy") + expected_arguments = [ + "ops", "results_dir", "st", "clu", "tF", "Wall", "imin", "tic0", "save_extra_vars", "save_preprocessed_copy" + ] + if parse(kilosort.__version__) >= parse("4.0.39"): + expected_arguments.append("skip_dat_path") self._check_arguments(save_sorting, expected_arguments) From 7151a30a2e37451d940d7fd435f3cd390e638644 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Mon, 14 Jul 2025 10:42:08 -0400 Subject: [PATCH 132/157] import read_neuroscope --- src/spikeinterface/extractors/extractor_classes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/extractor_classes.py b/src/spikeinterface/extractors/extractor_classes.py index 8d4aa8ed32..a975f2da9e 100644 --- a/src/spikeinterface/extractors/extractor_classes.py +++ b/src/spikeinterface/extractors/extractor_classes.py @@ -19,6 +19,7 @@ # sorting/recording/event from neo from .neoextractors import * +from .neoextractors import read_neuroscope # non-NEO objects implemented in neo folder # keep for reference Currently pulling from neoextractor __init__ @@ -84,7 +85,7 @@ # * A mapping from the original class to its wrapper string (because of __all__) # * A mapping from format to the class wrapper for convenience (exposed to users for ease of use) # -# To achieve these there goals we do the following: +# To achieve these three goals we do the following: # # 1) we line up each class with its wrapper that returns a snakecase version of the class (in some docs called # the "function" version, although this is just a wrapper of the underlying class) @@ -161,7 +162,7 @@ # (e.g. 'intan' , 'kilosort') and values being the appropriate Extractor class returned as its wrapper # (e.g. IntanRecordingExtractor, KiloSortSortingExtractor) # An important note is the the formats are returned after performing `.lower()` so a format like -# SpikeGLX will be a key of 'spikeglx' +# SpikeGLX will have a key of 'spikeglx' # for example if we wanted to create a recording from an intan file we could do the following: # >>> recording = se.recording_extractor_full_dict['intan'](file_path='path/to/data.rhd') @@ -198,5 +199,6 @@ "snippets_extractor_full_dict", "read_binary", # convenience function for binary formats "read_zarr", + "read_neuroscope", # convenience function for neuroscope ] ) From 513463f23ff8e14e39afd511a05ffcbd06ff7c4e Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 14 Jul 2025 17:07:11 +0200 Subject: [PATCH 133/157] Add test_kilosort_ci.py to list of files triggering action --- .github/workflows/test_kilosort4.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_kilosort4.yml b/.github/workflows/test_kilosort4.yml index 42e6140917..10ec9532fe 100644 --- a/.github/workflows/test_kilosort4.yml +++ b/.github/workflows/test_kilosort4.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - '**/kilosort4.py' + - '**/test_kilosort4_ci.py' jobs: versions: From 90217832324713fd5533d07a8a3198a30c7b190a Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 15 Jul 2025 09:30:53 +0200 Subject: [PATCH 134/157] Convert to path after remote check --- src/spikeinterface/core/core_tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index 5f17804604..0e1c9bca9e 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -423,7 +423,6 @@ def check_paths_relative(input_dict, relative_folder) -> bool: relative_folder = Path(relative_folder).resolve().absolute() not_possible = [] for p in path_list: - p = Path(p) # check path is not an URL if "http" in str(p): not_possible.append(p) @@ -434,6 +433,9 @@ def check_paths_relative(input_dict, relative_folder) -> bool: not_possible.append(p) continue + # convert to Path + p = Path(p) + # If windows path check have same drive if isinstance(p, WindowsPath) and isinstance(relative_folder, WindowsPath): # check that on same drive From be9d25ae4a5afd7323111d81121f93907c1a8fe4 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 15 Jul 2025 19:25:24 -0400 Subject: [PATCH 135/157] typo: ephsy -> ephys (#4065) --- src/spikeinterface/extractors/neoextractors/openephys.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/openephys.py b/src/spikeinterface/extractors/neoextractors/openephys.py index 0db3cd4426..f3c80b3d73 100644 --- a/src/spikeinterface/extractors/neoextractors/openephys.py +++ b/src/spikeinterface/extractors/neoextractors/openephys.py @@ -353,20 +353,20 @@ def read_openephys(folder_path, **kwargs): load_sync_channel : bool, default: False If False (default) and a SYNC channel is present (e.g. Neuropixels), this is not loaded. If True, the SYNC channel is loaded and can be accessed in the analog signals. - For open ephsy binary format only + For open ephys binary format only load_sync_timestamps : bool, default: False If True, the synchronized_timestamps are loaded and set as times to the recording. If False (default), only the t_start and sampling rate are set, and timestamps are assumed to be uniform and linearly increasing. - For open ephsy binary format only + For open ephys binary format only experiment_names : str, list, or None, default: None If multiple experiments are available, this argument allows users to select one or more experiments. If None, all experiements are loaded as blocks. E.g. `experiment_names="experiment2"`, `experiment_names=["experiment1", "experiment2"]` - For open ephsy binary format only + For open ephys binary format only ignore_timestamps_errors : bool, default: False Ignore the discontinuous timestamps errors in neo - For open ephsy legacy format only + For open ephys legacy format only Returns From 57ace1695811ae2e161cd6df5a424fe67c91d892 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 16 Jul 2025 10:51:04 +0200 Subject: [PATCH 136/157] Update src/spikeinterface/core/core_tools.py Co-authored-by: Heberto Mayorquin --- src/spikeinterface/core/core_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index 0e1c9bca9e..96c7d66c8c 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -428,7 +428,8 @@ def check_paths_relative(input_dict, relative_folder) -> bool: not_possible.append(p) continue - # check path is not a remote path + # check path is not a remote path, see + # https://github.com/SpikeInterface/spikeinterface/issues/4045 if is_path_remote(p): not_possible.append(p) continue From fa32fbcf7941d6af0c59bdf311e39bc70d69902d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 08:51:30 +0000 Subject: [PATCH 137/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/core_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index 96c7d66c8c..7640168cb7 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -428,7 +428,7 @@ def check_paths_relative(input_dict, relative_folder) -> bool: not_possible.append(p) continue - # check path is not a remote path, see + # check path is not a remote path, see # https://github.com/SpikeInterface/spikeinterface/issues/4045 if is_path_remote(p): not_possible.append(p) From a1f8a53a04b0976b20ea226c7c09e1959ff796a6 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 16 Jul 2025 16:06:48 -0400 Subject: [PATCH 138/157] more deprecations based on review --- .../benchmark/benchmark_plot_tools.py | 50 ---------------- .../comparison/multicomparisons.py | 59 ------------------- .../sortingcomponents/peak_pipeline.py | 49 --------------- 3 files changed, 158 deletions(-) delete mode 100644 src/spikeinterface/sortingcomponents/peak_pipeline.py diff --git a/src/spikeinterface/benchmark/benchmark_plot_tools.py b/src/spikeinterface/benchmark/benchmark_plot_tools.py index d07be05838..b397c7ef74 100644 --- a/src/spikeinterface/benchmark/benchmark_plot_tools.py +++ b/src/spikeinterface/benchmark/benchmark_plot_tools.py @@ -374,56 +374,6 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None, axs=None): return fig -# what's the dperecation strategy for this function in general? -def plot_performances(study, mode="ordered", performance_names=("accuracy", "precision", "recall"), case_keys=None): - """ - Plot performances over case for a study. - - Parameters - ---------- - study : BenchmarkStudy - A study object. - mode : "ordered" | "snr" | "swarm", default: "ordered" - Which plot mode to use: - - * "ordered": plot performance metrics vs unit indices ordered by decreasing accuracy - * "snr": plot performance metrics vs snr - * "swarm": plot performance metrics as a swarm plot (see seaborn.swarmplot for details) - performance_names : list or tuple, default: ("accuracy", "precision", "recall") - Which performances to plot ("accuracy", "precision", "recall") - case_keys : list or None - A selection of cases to plot, if None, then all. - - Returns - ------- - fig : matplotlib.figure.Figure - The resulting figure containing the plots - """ - if mode == "snr": - warnings.warn( - "Use study.plot_performances_vs_snr() instead", - DeprecationWarning, - stacklevel=2, - ) - return plot_performances_vs_snr(study, case_keys=case_keys, performance_names=performance_names) - elif mode == "ordered": - warnings.warn( - "Use study.plot_performances_ordered() instead", - DeprecationWarning, - stacklevel=2, - ) - return plot_performances_ordered(study, case_keys=case_keys, performance_names=performance_names) - elif mode == "swarm": - warnings.warn( - "Use study.plot_performances_swarm() instead", - DeprecationWarning, - stacklevel=2, - ) - return plot_performances_swarm(study, case_keys=case_keys, performance_names=performance_names) - else: - raise ValueError("plot_performances() : wrong mode ") - - def plot_performances_vs_snr( study, case_keys=None, diff --git a/src/spikeinterface/comparison/multicomparisons.py b/src/spikeinterface/comparison/multicomparisons.py index c93ba63c98..2b7180117b 100644 --- a/src/spikeinterface/comparison/multicomparisons.py +++ b/src/spikeinterface/comparison/multicomparisons.py @@ -190,65 +190,6 @@ def get_agreement_sorting(self, minimum_agreement_count=1, minimum_agreement_cou ) return sorting - # strategy for this dep? - def save_to_folder(self, save_folder): - warnings.warn( - "save_to_folder() is deprecated. " - "You should save and load the multi sorting comparison object using pickle." - "\n>>> pickle.dump(mcmp, open('mcmp.pkl', 'wb'))\n>>> mcmp_loaded = pickle.load(open('mcmp.pkl', 'rb'))", - DeprecationWarning, - stacklevel=2, - ) - for sorting in self.object_list: - assert sorting.check_serializability( - "json" - ), "MultiSortingComparison.save_to_folder() needs json serializable sortings" - - save_folder = Path(save_folder) - save_folder.mkdir(parents=True, exist_ok=True) - filename = str(save_folder / "multicomparison.gpickle") - with open(filename, "wb") as f: - pickle.dump(self.graph, f, pickle.HIGHEST_PROTOCOL) - kwargs = { - "delta_time": float(self.delta_time), - "match_score": float(self.match_score), - "chance_score": float(self.chance_score), - } - with (save_folder / "kwargs.json").open("w") as f: - json.dump(kwargs, f) - sortings = {} - for name, sorting in zip(self.name_list, self.object_list): - sortings[name] = sorting.to_dict(recursive=True, relative_to=save_folder) - with (save_folder / "sortings.json").open("w") as f: - json.dump(sortings, f) - - # and here too. Strategy for this dep? - @staticmethod - def load_from_folder(folder_path): - warnings.warn( - "load_from_folder() is deprecated. " - "You should save and load the multi sorting comparison object using pickle." - "\n>>> pickle.dump(mcmp, open('mcmp.pkl', 'wb'))\n>>> mcmp_loaded = pickle.load(open('mcmp.pkl', 'rb'))", - DeprecationWarning, - stacklevel=2, - ) - folder_path = Path(folder_path) - with (folder_path / "kwargs.json").open() as f: - kwargs = json.load(f) - with (folder_path / "sortings.json").open() as f: - dict_sortings = json.load(f) - name_list = list(dict_sortings.keys()) - sorting_list = [load(v, base_folder=folder_path) for v in dict_sortings.values()] - mcmp = MultiSortingComparison(sorting_list=sorting_list, name_list=list(name_list), do_matching=False, **kwargs) - filename = str(folder_path / "multicomparison.gpickle") - with open(filename, "rb") as f: - mcmp.graph = pickle.load(f) - # do step 3 and 4 - mcmp._clean_graph() - mcmp._do_agreement() - mcmp._populate_spiketrains() - return mcmp - class AgreementSortingExtractor(BaseSorting): def __init__( diff --git a/src/spikeinterface/sortingcomponents/peak_pipeline.py b/src/spikeinterface/sortingcomponents/peak_pipeline.py deleted file mode 100644 index a0b2b263c7..0000000000 --- a/src/spikeinterface/sortingcomponents/peak_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import copy - -from spikeinterface.core.node_pipeline import PeakRetriever, run_node_pipeline - - -def run_peak_pipeline( - recording, - peaks, - nodes, - job_kwargs, - job_name="peak_pipeline", - gather_mode="memory", - squeeze_output=True, - folder=None, - names=None, -): - # TODO remove this soon. Will be removed in 0.104.0 - import warnings - - warnings.warn( - "run_peak_pipeline() is deprecated and will be removed in version 0.104.0, `use run_node_pipeline()` instead", - DeprecationWarning, - stacklevel=2, - ) - - node0 = PeakRetriever(recording, peaks) - # because nodes are modified inplace (insert parent) they need to copy incase - # the same pipeline is run several times - nodes = copy.deepcopy(nodes) - - for node in nodes: - if node.parents is None: - node.parents = [node0] - else: - node.parents = [node0] + node.parents - all_nodes = [node0] + nodes - outs = run_node_pipeline( - recording, - all_nodes, - job_kwargs, - job_name=job_name, - gather_mode=gather_mode, - squeeze_output=squeeze_output, - folder=folder, - names=names, - ) - return outs From f81261ed740e9550225df700806c9e01a18e39b6 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 16 Jul 2025 16:09:29 -0400 Subject: [PATCH 139/157] delete tests for peak_pipeline --- .../test_waveform_thresholder.py | 109 ------------------ 1 file changed, 109 deletions(-) delete mode 100644 src/spikeinterface/sortingcomponents/tests/test_waveforms/test_waveform_thresholder.py diff --git a/src/spikeinterface/sortingcomponents/tests/test_waveforms/test_waveform_thresholder.py b/src/spikeinterface/sortingcomponents/tests/test_waveforms/test_waveform_thresholder.py deleted file mode 100644 index 79a9603b8d..0000000000 --- a/src/spikeinterface/sortingcomponents/tests/test_waveforms/test_waveform_thresholder.py +++ /dev/null @@ -1,109 +0,0 @@ -import pytest -import numpy as np -import operator - - -from spikeinterface.sortingcomponents.waveforms.waveform_thresholder import WaveformThresholder -from spikeinterface.core.node_pipeline import ExtractDenseWaveforms -from spikeinterface.sortingcomponents.peak_pipeline import run_peak_pipeline - - -@pytest.fixture(scope="module") -def extract_dense_waveforms_node(generated_recording): - # Parameters - ms_before = 1.0 - ms_after = 1.0 - - # Node initialization - return ExtractDenseWaveforms( - recording=generated_recording, ms_before=ms_before, ms_after=ms_after, return_output=True - ) - - -def test_waveform_thresholder_ptp( - extract_dense_waveforms_node, generated_recording, detected_peaks, chunk_executor_kwargs -): - recording = generated_recording - peaks = detected_peaks - - tresholded_waveforms_ptp = WaveformThresholder( - recording=recording, parents=[extract_dense_waveforms_node], feature="ptp", threshold=3, return_output=True - ) - noise_levels = tresholded_waveforms_ptp.noise_levels - - pipeline_nodes = [extract_dense_waveforms_node, tresholded_waveforms_ptp] - # Extract projected waveforms and compare - waveforms, tresholded_waveforms = run_peak_pipeline( - recording, peaks, nodes=pipeline_nodes, job_kwargs=chunk_executor_kwargs - ) - - data = np.ptp(tresholded_waveforms, axis=1) / noise_levels - assert np.all(data[data != 0] > 3) - - -def test_waveform_thresholder_mean( - extract_dense_waveforms_node, generated_recording, detected_peaks, chunk_executor_kwargs -): - recording = generated_recording - peaks = detected_peaks - - tresholded_waveforms_mean = WaveformThresholder( - recording=recording, parents=[extract_dense_waveforms_node], feature="mean", threshold=0, return_output=True - ) - - pipeline_nodes = [extract_dense_waveforms_node, tresholded_waveforms_mean] - # Extract projected waveforms and compare - waveforms, tresholded_waveforms = run_peak_pipeline( - recording, peaks, nodes=pipeline_nodes, job_kwargs=chunk_executor_kwargs - ) - - assert np.all(tresholded_waveforms.mean(axis=1) >= 0) - - -def test_waveform_thresholder_energy( - extract_dense_waveforms_node, generated_recording, detected_peaks, chunk_executor_kwargs -): - recording = generated_recording - peaks = detected_peaks - - tresholded_waveforms_energy = WaveformThresholder( - recording=recording, parents=[extract_dense_waveforms_node], feature="energy", threshold=3, return_output=True - ) - noise_levels = tresholded_waveforms_energy.noise_levels - - pipeline_nodes = [extract_dense_waveforms_node, tresholded_waveforms_energy] - # Extract projected waveforms and compare - waveforms, tresholded_waveforms = run_peak_pipeline( - recording, peaks, nodes=pipeline_nodes, job_kwargs=chunk_executor_kwargs - ) - - data = np.linalg.norm(tresholded_waveforms, axis=1) / noise_levels - assert np.all(data[data != 0] > 3) - - -def test_waveform_thresholder_operator( - extract_dense_waveforms_node, generated_recording, detected_peaks, chunk_executor_kwargs -): - recording = generated_recording - peaks = detected_peaks - - import operator - - tresholded_waveforms_peak = WaveformThresholder( - recording=recording, - parents=[extract_dense_waveforms_node], - feature="peak_voltage", - threshold=5, - operator=operator.ge, - return_output=True, - ) - noise_levels = tresholded_waveforms_peak.noise_levels - - pipeline_nodes = [extract_dense_waveforms_node, tresholded_waveforms_peak] - # Extract projected waveforms and compare - waveforms, tresholded_waveforms = run_peak_pipeline( - recording, peaks, nodes=pipeline_nodes, job_kwargs=chunk_executor_kwargs - ) - - data = tresholded_waveforms[:, extract_dense_waveforms_node.nbefore, :] / noise_levels - assert np.all(data[data != 0] <= 5) From b6e3536bff79568210fec896491953fd168d0b22 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 16 Jul 2025 15:54:24 -0600 Subject: [PATCH 140/157] add failing tests --- src/spikeinterface/extractors/tests/test_neoextractors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 37bea41979..a870544215 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -105,6 +105,7 @@ class SpikeGLXRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ("spikeglx/Noise4Sam_g0", {"stream_id": "imec0.lf"}), ("spikeglx/Noise4Sam_g0", {"stream_id": "nidq"}), ("spikeglx/Noise4Sam_g0", {"stream_id": "imec0.ap-SYNC"}), + ("spiekglx/onebox/run_with_only_adc", {"stream_id": "obx0"}), ] From bdd242b649872207fca0694a616da8b6b284e7ea Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 16 Jul 2025 16:07:37 -0600 Subject: [PATCH 141/157] fix name in test --- src/spikeinterface/extractors/tests/test_neoextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index a870544215..ecab70a5b3 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -105,7 +105,7 @@ class SpikeGLXRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ("spikeglx/Noise4Sam_g0", {"stream_id": "imec0.lf"}), ("spikeglx/Noise4Sam_g0", {"stream_id": "nidq"}), ("spikeglx/Noise4Sam_g0", {"stream_id": "imec0.ap-SYNC"}), - ("spiekglx/onebox/run_with_only_adc", {"stream_id": "obx0"}), + ("spikeglx/onebox/run_with_only_adc/myRun_g0", {"stream_id": "obx0"}), ] From 980fd85e01d34322f233820b2f78d2121ae398f5 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 16 Jul 2025 16:13:30 -0600 Subject: [PATCH 142/157] fix --- .../extractors/neoextractors/spikeglx.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/spikeglx.py b/src/spikeinterface/extractors/neoextractors/spikeglx.py index df3728fe6a..657eaf0db9 100644 --- a/src/spikeinterface/extractors/neoextractors/spikeglx.py +++ b/src/spikeinterface/extractors/neoextractors/spikeglx.py @@ -74,10 +74,14 @@ def __init__( ) self._kwargs.update(dict(folder_path=str(Path(folder_path).absolute()), load_sync_channel=load_sync_channel)) - - stream_is_nidq_or_sync = "nidq" in self.stream_id or "SYNC" in self.stream_id - if stream_is_nidq_or_sync: - # Do not add probe information for the sync or nidq stream. Early return + + + stream_is_nidq = "nidq" in self.stream_id + stream_is_one_box = "obx" in self.stream_id + stream_is_sync = "SYNC" in self.stream_id + + if stream_is_nidq or stream_is_one_box or stream_is_sync: + # Do not add probe information for the one box, nidq or sync streams. Early return return None # Checks if the probe information is available and adds location, shanks and sample shift if available. From 153ad2e274564b554f84ef566793db55c25e085d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 22:13:57 +0000 Subject: [PATCH 143/157] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/extractors/neoextractors/spikeglx.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/spikeglx.py b/src/spikeinterface/extractors/neoextractors/spikeglx.py index 657eaf0db9..dd51187508 100644 --- a/src/spikeinterface/extractors/neoextractors/spikeglx.py +++ b/src/spikeinterface/extractors/neoextractors/spikeglx.py @@ -74,9 +74,8 @@ def __init__( ) self._kwargs.update(dict(folder_path=str(Path(folder_path).absolute()), load_sync_channel=load_sync_channel)) - - - stream_is_nidq = "nidq" in self.stream_id + + stream_is_nidq = "nidq" in self.stream_id stream_is_one_box = "obx" in self.stream_id stream_is_sync = "SYNC" in self.stream_id From 0f41fac57ff0cf38f48bc3d0e86be9d08dd26592 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:04:13 -0400 Subject: [PATCH 144/157] remove tests for save/load folder --- .../comparison/tests/test_multisortingcomparison.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/spikeinterface/comparison/tests/test_multisortingcomparison.py b/src/spikeinterface/comparison/tests/test_multisortingcomparison.py index dc769f8d59..0b26083bd9 100644 --- a/src/spikeinterface/comparison/tests/test_multisortingcomparison.py +++ b/src/spikeinterface/comparison/tests/test_multisortingcomparison.py @@ -76,10 +76,6 @@ def test_compare_multiple_sorters(setup_module): agreement_2 = msc.get_agreement_sorting(minimum_agreement_count=2, minimum_agreement_count_only=True) assert np.all([agreement_2.get_unit_property(u, "agreement_number")] == 2 for u in agreement_2.get_unit_ids()) - msc.save_to_folder(multicomparison_folder) - - msc = MultiSortingComparison.load_from_folder(multicomparison_folder) - def test_compare_multi_segment(): num_segments = 3 From 9b973cf4d1b0a66cffb99ecef2809ce89fd66cfe Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 17 Jul 2025 15:16:37 +0200 Subject: [PATCH 145/157] Fix installation instructions and check your install --- installation_tips/README.md | 81 ++++++++--------- installation_tips/check_your_install.py | 88 +++++++------------ installation_tips/cleanup_for_windows.py | 14 +-- ...full_spikeinterface_environment_linux.yml} | 2 +- ...ull_spikeinterface_environment_windows.yml | 3 +- installation_tips/requirements_rolling.txt | 14 +-- installation_tips/requirements_stable.txt | 7 +- 7 files changed, 84 insertions(+), 125 deletions(-) rename installation_tips/{full_spikeinterface_environment_linux_dandi.yml => full_spikeinterface_environment_linux.yml} (96%) diff --git a/installation_tips/README.md b/installation_tips/README.md index 21696bbd05..9c5b8fdbdf 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -1,76 +1,69 @@ ## Installation tips If you are not (yet) an expert in Python installations, a main difficulty is choosing the installation procedure. -The main ideas you need to know before starting: - * python itself can be distributed and installed many many ways. - * python itself does not contain so many features for scientific computing you need to install "packages". - numpy, scipy, matplotlib, spikeinterface, ... are python packages that have a complicated dependency graph between then. - * packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) - * installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environement" - to not destroy any previous installation. You need to see an "environment" as a sub installation in a dedicated folder. + +Some main concepts you need to know before starting: + * Python itself can be distributed and installed many many ways. + * Python itself does not contain so many features for scientific computing you need to install "packages". + numpy, scipy, matplotlib, spikeinterface, ... are Python packages that have a complicated dependency graph between then. + * packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) + * installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environement" to not destroy any previous installation. You need to see an "environment" as a sub-installation in a dedicated folder. Choosing the installer + an environment manager + a package installer is a nightmare for beginners. + The main options are: - * use "uv" : a new, fast and simple package manager. We recommend this for beginners on every os. + * use "uv", a new, fast and simple package manager. We recommend this for beginners on every operating system. * use "anaconda", which does everything. Used to be very popular but theses days it is becoming a bad idea because : slow by default and aggressive licensing on the default channel (not always free anymore). You need to play with "community channels" to make it free again, which is too complicated for beginners. Do not go this way. - * use python from the system or python.org + venv + pip : good and simple idea for linux users. + * use Python from the system or Python.org + venv + pip: good and simple idea for linux users. -Here we propose a step by step recipe for beginers based on **"uv"**. +Here we propose a step by step recipe for beginers based on [**"uv"**](https://github.com/astral-sh/uv). We used to recommend installing with anaconda. It will be kept here for a while but we do not recommend it anymore. This environment will install: - * spikeinterface full option + * spikeinterface `full` option * spikeinterface-gui * kilosort4 -Kilosort, Ironclust and HDSort are MATLAB based and need to be installed from source. ### Quick installation using "uv" (recommended) 1. On macOS and Linux. Open a terminal and do `curl -LsSf https://astral.sh/uv/install.sh | sh` -1. On windows. Open a terminal using with CMD +2. On Windows. Open a terminal using CMD `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` -2. exit session and log again. -3. Download with right click and save this file corresponding in "Documents" folder: - * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) -4. open terminal or powershell -5. `uv venv si_env --python 3.12` -6. For Mac/Linux `source si_env/bin/activate` (you should have `(si_env)` in your terminal) -6. For windows `si_env\Scripts\activate` -7. `uv pip install -r Documents/beginner_requirements_stable.txt` or `uv pip install -r Documents/beginner_requirements_rolling.txt` - +3. Exit the session and log in again. +4. Download with right click and save this file in your "Documents" folder: + * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) for stable release +5. Open terminal or powershell and run: +6. `uv venv si_env --python 3.12` +7. Activate your virtual environment by running: + - For Mac/Linux: `source si_env/bin/activate` (you should see `(si_env)` in your terminal) + - For Windows: `si_env\Scripts\activate` +8. Run `uv pip install -r Documents/requirements_stable.txt` -More details on [uv here](https://github.com/astral-sh/uv). +## Installing before release (from source) -## Installing before release - -Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, python-neo, sortingview). -We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `beginner_requirements_rolling.txt` file to create the environment. This will install the packages of the ecosystem from source. +Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, neo). +We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `requirements_rolling.txt` file to create the environment instead of the `requirements_stable.txt` file. This will install the packages of the ecosystem from source. This is a good way to test if a patch fixes your issue. - - ### Check the installation - If you want to test the spikeinterface install you can: 1. Download with right click + save the file [`check_your_install.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/check_your_install.py) and put it into the "Documents" folder 2. Open the CMD (Windows) or Terminal (Mac/Linux) -3. Activate your si_env : `source si_env/bin/activate` (Max/Linux), `si_env\Scripts\activate` (CMD prompt) +3. Activate your si_env : `source si_env/bin/activate` (Max/Linux), `si_env\Scripts\activate` (Windows) 4. Go to your "Documents" folder with `cd Documents` or the place where you downloaded the `check_your_install.py` -5. Run this: - `python check_your_install.py` -6. If a windows user to clean-up you will also need to right click + save [`cleanup_for_windows.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/cleanup_for_windows.py) -Then transfer `cleanup_for_windows.py` into your "Documents" folder. Finally run : +5. Run `python check_your_install.py` +6. If you are a Windows user, you should also right click + save [`cleanup_for_windows.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/cleanup_for_windows.py). Then transfer `cleanup_for_windows.py` into your "Documents" folder and finally run: ``` python cleanup_for_windows.py ``` @@ -80,18 +73,18 @@ This script tests the following steps: * running tridesclous2 * running kilosort4 * opening the spikeinterface-gui - * exporting to Phy -### Legacy installation using anaconda (not recommended anymore) +### Legacy installation using Anaconda (not recommended anymore) Steps: -1. Download anaconda individual edition [here](https://www.anaconda.com/download) +1. Download Anaconda individual edition [here](https://www.anaconda.com/download) 2. Run the installer. Check the box “Add anaconda3 to my Path environment variable”. It makes life easier for beginners. -3. Download with right click + save the file corresponding to your OS, and put it in "Documents" folder +3. Download with right click + save the file corresponding to your operating system, and put it in "Documents" folder * [`full_spikeinterface_environment_windows.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_windows.yml) * [`full_spikeinterface_environment_mac.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_mac.yml) + * [`full_spikeinterface_environment_linux.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_linux.yml) 4. Then open the "Anaconda Command Prompt" (if Windows, search in your applications) or the Terminal (for Mac users) 5. If not in the "Documents" folder type `cd Documents` 6. Then run this depending on your OS: @@ -101,13 +94,11 @@ Steps: Done! Before running a spikeinterface script you will need to "select" this "environment" with `conda activate si_env`. -Note for **linux** users : this conda recipe should work but we recommend strongly to use **pip + virtualenv**. - +Note for **Linux** users: this conda recipe should work but we recommend strongly to use **pip + virtualenv**. +## Installing before release (from source) -## Installing before release - -Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, python-neo, sortingview). +Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, neo). We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `full_spikeinterface_environment_rolling_updates.yml` file to create the environment. This will install the packages of the ecosystem from source. -This is a good way to test if patch fix your issue. +This is a good way to test if a patch fixes your issue. diff --git a/installation_tips/check_your_install.py b/installation_tips/check_your_install.py index 1144bda577..38421ced49 100644 --- a/installation_tips/check_your_install.py +++ b/installation_tips/check_your_install.py @@ -3,27 +3,29 @@ import os import shutil import argparse +import warnings +warnings.filterwarnings("ignore") + + +job_kwargs = dict(n_jobs=-1, progress_bar=False, chunk_duration="1s") -job_kwargs = dict(n_jobs=-1, progress_bar=True, chunk_duration="1s") def check_import_si(): import spikeinterface as si + def check_import_si_full(): import spikeinterface.full as si def _create_recording(): import spikeinterface.full as si - rec, sorting = si.generate_ground_truth_recording( - durations=[200.], - sampling_frequency=30_000., - num_channels=16, - num_units=10, - seed=2205 + + rec, _ = si.generate_ground_truth_recording( + durations=[200.0], sampling_frequency=30_000.0, num_channels=16, num_units=10, seed=2205 ) - rec.save(folder='./toy_example_recording', **job_kwargs) + rec.save(folder="./toy_example_recording", verbose=False, **job_kwargs) def _run_one_sorter_and_analyzer(sorter_name): @@ -31,54 +33,39 @@ def _run_one_sorter_and_analyzer(sorter_name): si.set_global_job_kwargs(**job_kwargs) - recording = si.load_extractor('./toy_example_recording') - sorting = si.run_sorter(sorter_name, recording, folder=f'./sorter_with_{sorter_name}', verbose=False) + recording = si.load("./toy_example_recording") + sorting = si.run_sorter(sorter_name, recording, folder=f"./sorter_with_{sorter_name}", verbose=False) - sorting_analyzer = si.create_sorting_analyzer(sorting, recording, - format="binary_folder", folder=f"./analyzer_with_{sorter_name}") + sorting_analyzer = si.create_sorting_analyzer( + sorting, recording, format="binary_folder", folder=f"./analyzer_with_{sorter_name}" + ) sorting_analyzer.compute("random_spikes", method="uniform", max_spikes_per_unit=500) sorting_analyzer.compute("waveforms") sorting_analyzer.compute("templates") sorting_analyzer.compute("noise_levels") sorting_analyzer.compute("unit_locations", method="monopolar_triangulation") - sorting_analyzer.compute("correlograms", window_ms=100, bin_ms=5.) - sorting_analyzer.compute("principal_components", n_components=3, mode='by_channel_global', whiten=True) + sorting_analyzer.compute("correlograms", window_ms=100, bin_ms=5.0) + sorting_analyzer.compute("principal_components", n_components=3, mode="by_channel_global", whiten=True) sorting_analyzer.compute("quality_metrics", metric_names=["snr", "firing_rate"]) def run_tridesclous2(): - _run_one_sorter_and_analyzer('tridesclous2') + _run_one_sorter_and_analyzer("tridesclous2") -def run_kilosort4(): - _run_one_sorter_and_analyzer('kilosort4') +def run_kilosort4(): + _run_one_sorter_and_analyzer("kilosort4") def open_sigui(): import spikeinterface.full as si - import spikeinterface_gui - app = spikeinterface_gui.mkQApp() + from spikeinterface_gui import run_mainwindow sorter_name = "tridesclous2" folder = f"./analyzer_with_{sorter_name}" analyzer = si.load_sorting_analyzer(folder) - win = spikeinterface_gui.MainWindow(analyzer) - win.show() - app.exec_() - -def export_to_phy(): - import spikeinterface.full as si - sorter_name = "tridesclous2" - folder = f"./analyzer_with_{sorter_name}" - analyzer = si.load_sorting_analyzer(folder) - - phy_folder = "./phy_example" - si.export_to_phy(analyzer, output_folder=phy_folder, verbose=False) - - -def open_phy(): - os.system("phy template-gui ./phy_example/params.py") + win = run_mainwindow(analyzer, start_app=True) def _clean(): @@ -89,18 +76,18 @@ def _clean(): "./analyzer_with_tridesclous2", "./sorter_with_kilosort4", "./analyzer_with_kilosort4", - "./phy_example" ] for folder in folders: if Path(folder).exists(): shutil.rmtree(folder) + parser = argparse.ArgumentParser() # add ci flag so that gui will not be used in ci # end user can ignore -parser.add_argument('--ci', action='store_false') +parser.add_argument("--ci", action="store_false") -if __name__ == '__main__': +if __name__ == "__main__": args = parser.parse_args() @@ -108,31 +95,22 @@ def _clean(): _create_recording() steps = [ - ('Import spikeinterface', check_import_si), - ('Import spikeinterface.full', check_import_si_full), - ('Run tridesclous2', run_tridesclous2), - ('Run kilosort4', run_kilosort4), - ] + ("Import spikeinterface", check_import_si), + ("Import spikeinterface.full", check_import_si_full), + ("Run tridesclous2", run_tridesclous2), + ("Run kilosort4", run_kilosort4), + ] # backwards logic because default is True for end-user if args.ci: - steps.append(('Open spikeinterface-gui', open_sigui)) - - steps.append(('Export to phy', export_to_phy)), - - # if platform.system() == "Windows": - # pass - # elif platform.system() == "Darwin": - # pass - # else: - # pass + steps.append(("Open spikeinterface-gui", open_sigui)) for label, func in steps: try: func() - done = '...OK' + done = "...OK" except Exception as err: - done = f'...Fail, Error: {err}' + done = f"...Fail, Error: {err}" print(label, done) if platform.system() == "Windows": diff --git a/installation_tips/cleanup_for_windows.py b/installation_tips/cleanup_for_windows.py index 2c334f2df2..8d803ae421 100644 --- a/installation_tips/cleanup_for_windows.py +++ b/installation_tips/cleanup_for_windows.py @@ -5,15 +5,17 @@ def _clean(): # clean folders = [ - 'toy_example_recording', - "tridesclous_output", "tridesclous_waveforms", - "spykingcircus_output", "spykingcircus_waveforms", - "phy_example" + "./toy_example_recording", + "./sorter_with_tridesclous2", + "./analyzer_with_tridesclous2", + "./sorter_with_kilosort4", + "./analyzer_with_kilosort4", ] for folder in folders: if Path(folder).exists(): shutil.rmtree(folder) -if __name__ == '__main__': - _clean() +if __name__ == "__main__": + + _clean() diff --git a/installation_tips/full_spikeinterface_environment_linux_dandi.yml b/installation_tips/full_spikeinterface_environment_linux.yml similarity index 96% rename from installation_tips/full_spikeinterface_environment_linux_dandi.yml rename to installation_tips/full_spikeinterface_environment_linux.yml index 197109cd9c..0a220fba4c 100755 --- a/installation_tips/full_spikeinterface_environment_linux_dandi.yml +++ b/installation_tips/full_spikeinterface_environment_linux.yml @@ -34,4 +34,4 @@ dependencies: - spikeinterface[full,widgets] - spikeinterface-gui - tridesclous - # - phy==2.0b5 + - kilosort diff --git a/installation_tips/full_spikeinterface_environment_windows.yml b/installation_tips/full_spikeinterface_environment_windows.yml index 1df2c6878c..2b18be87d8 100755 --- a/installation_tips/full_spikeinterface_environment_windows.yml +++ b/installation_tips/full_spikeinterface_environment_windows.yml @@ -27,8 +27,7 @@ dependencies: - ipympl - pip: - ephyviewer - - MEArec - spikeinterface[full,widgets] - spikeinterface-gui - tridesclous - # - phy==2.0b5 + - kilosort diff --git a/installation_tips/requirements_rolling.txt b/installation_tips/requirements_rolling.txt index d35009f1ea..432a50d51d 100644 --- a/installation_tips/requirements_rolling.txt +++ b/installation_tips/requirements_rolling.txt @@ -1,16 +1,10 @@ -numpy<2 +https://github.com/NeuralEnsemble/python-neo/archive/master.zip +https://github.com/SpikeInterface/probeinterface/archive/main.zip +https://github.com/SpikeInterface/spikeinterface/archive/main.zip[full,widgets] +https://github.com/SpikeInterface/spikeinterface-gui/archive/main.zip jupyterlab PySide6<6.8 -numba -zarr hdbscan pyqtgraph -ipywidgets -ipympl ephyviewer -https://github.com/NeuralEnsemble/python-neo/archive/master.zip -https://github.com/SpikeInterface/probeinterface/archive/main.zip -https://github.com/SpikeInterface/spikeinterface/archive/main.zip[full,widgets] -https://github.com/SpikeInterface/spikeinterface-gui/archive/main.zip -https://github.com/magland/sortingview/archive/main.zip kilosort diff --git a/installation_tips/requirements_stable.txt b/installation_tips/requirements_stable.txt index d77343fbba..faa121f267 100644 --- a/installation_tips/requirements_stable.txt +++ b/installation_tips/requirements_stable.txt @@ -1,13 +1,8 @@ spikeinterface[full,widgets] -numpy<2 jupyterlab PySide6<6.8 -numba -zarr hdbscan pyqtgraph -ipywidgets -ipympl ephyviewer spikeinterface-gui -kilosort +kilosort>4.0.30 From 79968cf6cbf481f18de72f5f5415e6da190bcce4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 17 Jul 2025 08:37:36 -0600 Subject: [PATCH 146/157] add comment --- src/spikeinterface/core/basesorting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index ef50d39721..8a1fa9cf1b 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -269,6 +269,8 @@ def get_unit_spike_train_in_seconds( return spike_times # Use the native spiking times if available + # Some instances might implement a method themselves to access spike times directly without having to convert + # (e.g. NWB extractors) if hasattr(segment, "get_unit_spike_train_in_seconds"): return segment.get_unit_spike_train_in_seconds(unit_id=unit_id, start_time=start_time, end_time=end_time) From a83b166cd6b08e1d7d2c68dfe506a4eaed184b4f Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 17 Jul 2025 16:45:55 +0200 Subject: [PATCH 147/157] Update naming and delet os-specific conda env yaml --- installation_tips/README.md | 26 +++++--------- ...inux.yml => beginner_conda_env_stable.yml} | 10 +++--- ....txt => beginner_requirements_rolling.txt} | 2 +- ...e.txt => beginner_requirements_stable.txt} | 0 installation_tips/check_your_install.py | 1 - .../full_spikeinterface_environment_mac.yml | 34 ------------------ ...einterface_environment_rolling_updates.yml | 35 ------------------- ...ull_spikeinterface_environment_windows.yml | 33 ----------------- 8 files changed, 13 insertions(+), 128 deletions(-) rename installation_tips/{full_spikeinterface_environment_linux.yml => beginner_conda_env_stable.yml} (83%) rename installation_tips/{requirements_rolling.txt => beginner_requirements_rolling.txt} (95%) rename installation_tips/{requirements_stable.txt => beginner_requirements_stable.txt} (100%) delete mode 100755 installation_tips/full_spikeinterface_environment_mac.yml delete mode 100644 installation_tips/full_spikeinterface_environment_rolling_updates.yml delete mode 100755 installation_tips/full_spikeinterface_environment_windows.yml diff --git a/installation_tips/README.md b/installation_tips/README.md index 9c5b8fdbdf..a49d02bb3e 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -37,19 +37,19 @@ This environment will install: `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` 3. Exit the session and log in again. 4. Download with right click and save this file in your "Documents" folder: - * [`requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/requirements_stable.txt) for stable release + * [`beginner_requirements_stable.txt`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/beginner_requirements_stable.txt) for stable release 5. Open terminal or powershell and run: 6. `uv venv si_env --python 3.12` 7. Activate your virtual environment by running: - For Mac/Linux: `source si_env/bin/activate` (you should see `(si_env)` in your terminal) - For Windows: `si_env\Scripts\activate` -8. Run `uv pip install -r Documents/requirements_stable.txt` +8. Run `uv pip install -r Documents/beginner_requirements_stable.txt` ## Installing before release (from source) Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, neo). -We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `requirements_rolling.txt` file to create the environment instead of the `requirements_stable.txt` file. This will install the packages of the ecosystem from source. +We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `beginner_requirements_rolling.txt` file to create the environment instead of the `beginner_requirements_stable.txt` file. This will install the packages of the ecosystem from source. This is a good way to test if a patch fixes your issue. @@ -81,24 +81,14 @@ Steps: 1. Download Anaconda individual edition [here](https://www.anaconda.com/download) 2. Run the installer. Check the box “Add anaconda3 to my Path environment variable”. It makes life easier for beginners. -3. Download with right click + save the file corresponding to your operating system, and put it in "Documents" folder - * [`full_spikeinterface_environment_windows.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_windows.yml) - * [`full_spikeinterface_environment_mac.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_mac.yml) - * [`full_spikeinterface_environment_linux.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/full_spikeinterface_environment_linux.yml) +3. Download with right click + save the enfironment YAML file ([`beginner_conda_env_stable.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/beginner_conda_env_stable.yml)) and put it in "Documents" folder 4. Then open the "Anaconda Command Prompt" (if Windows, search in your applications) or the Terminal (for Mac users) 5. If not in the "Documents" folder type `cd Documents` -6. Then run this depending on your OS: - * `conda env create --file full_spikeinterface_environment_windows.yml` - * `conda env create --file full_spikeinterface_environment_mac.yml` - +6. Run this command to create the environment: + ```bash + conda env create --file beginner_conda_env_stable.yml + ``` Done! Before running a spikeinterface script you will need to "select" this "environment" with `conda activate si_env`. Note for **Linux** users: this conda recipe should work but we recommend strongly to use **pip + virtualenv**. - - -## Installing before release (from source) - -Some tools in the spikeinteface ecosystem are getting regular bug fixes (spikeinterface, spikeinterface-gui, probeinterface, neo). -We are making releases 2 to 4 times a year. In between releases if you want to install from source you can use the `full_spikeinterface_environment_rolling_updates.yml` file to create the environment. This will install the packages of the ecosystem from source. -This is a good way to test if a patch fixes your issue. diff --git a/installation_tips/full_spikeinterface_environment_linux.yml b/installation_tips/beginner_conda_env_stable.yml similarity index 83% rename from installation_tips/full_spikeinterface_environment_linux.yml rename to installation_tips/beginner_conda_env_stable.yml index 0a220fba4c..cd2f36c1bf 100755 --- a/installation_tips/full_spikeinterface_environment_linux.yml +++ b/installation_tips/beginner_conda_env_stable.yml @@ -3,9 +3,9 @@ channels: - conda-forge - defaults dependencies: - - python=3.11 + - python=3.12 - pip - - numpy<2 + - numpy - scipy - joblib - tqdm @@ -13,7 +13,7 @@ dependencies: - h5py - pandas - xarray - - zarr + - zarr<3 - scikit-learn - hdbscan - networkx @@ -30,8 +30,6 @@ dependencies: - libxcb - pip: - ephyviewer - - MEArec - spikeinterface[full,widgets] - spikeinterface-gui - - tridesclous - - kilosort + - kilosort>4.0.30 diff --git a/installation_tips/requirements_rolling.txt b/installation_tips/beginner_requirements_rolling.txt similarity index 95% rename from installation_tips/requirements_rolling.txt rename to installation_tips/beginner_requirements_rolling.txt index 432a50d51d..b4deca3a50 100644 --- a/installation_tips/requirements_rolling.txt +++ b/installation_tips/beginner_requirements_rolling.txt @@ -7,4 +7,4 @@ PySide6<6.8 hdbscan pyqtgraph ephyviewer -kilosort +kilosort>4.0.30 diff --git a/installation_tips/requirements_stable.txt b/installation_tips/beginner_requirements_stable.txt similarity index 100% rename from installation_tips/requirements_stable.txt rename to installation_tips/beginner_requirements_stable.txt diff --git a/installation_tips/check_your_install.py b/installation_tips/check_your_install.py index 38421ced49..e8437743b8 100644 --- a/installation_tips/check_your_install.py +++ b/installation_tips/check_your_install.py @@ -1,6 +1,5 @@ from pathlib import Path import platform -import os import shutil import argparse import warnings diff --git a/installation_tips/full_spikeinterface_environment_mac.yml b/installation_tips/full_spikeinterface_environment_mac.yml deleted file mode 100755 index 1df2c6878c..0000000000 --- a/installation_tips/full_spikeinterface_environment_mac.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: si_env -channels: - - conda-forge - - defaults -dependencies: - - python=3.11 - - pip - - numpy<2 - - scipy - - joblib - - tqdm - - matplotlib - - h5py - - pandas - - xarray - - zarr - - scikit-learn - - hdbscan - - networkx - - pybind11 - - loky - - numba - - jupyter - - pyqt=5 - - pyqtgraph - - ipywidgets - - ipympl - - pip: - - ephyviewer - - MEArec - - spikeinterface[full,widgets] - - spikeinterface-gui - - tridesclous - # - phy==2.0b5 diff --git a/installation_tips/full_spikeinterface_environment_rolling_updates.yml b/installation_tips/full_spikeinterface_environment_rolling_updates.yml deleted file mode 100644 index a2b51546f1..0000000000 --- a/installation_tips/full_spikeinterface_environment_rolling_updates.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: si_env_rolling -channels: - - conda-forge - - defaults -dependencies: - - python=3.11 - - pip - - numpy<2 - - scipy - - joblib - - tqdm - - matplotlib - - h5py - - pandas - - xarray - - hdbscan - - scikit-learn - - networkx - - pybind11 - - loky - - numba - - jupyter - - pyqt=5 - - pyqtgraph - - ipywidgets - - ipympl - - pip: - - ephyviewer - - docker - - https://github.com/SpikeInterface/MEArec/archive/main.zip - - https://github.com/NeuralEnsemble/python-neo/archive/master.zip - - https://github.com/SpikeInterface/probeinterface/archive/main.zip - - https://github.com/SpikeInterface/spikeinterface/archive/main.zip - - https://github.com/SpikeInterface/spikeinterface-gui/archive/main.zip - - https://github.com/magland/sortingview/archive/main.zip diff --git a/installation_tips/full_spikeinterface_environment_windows.yml b/installation_tips/full_spikeinterface_environment_windows.yml deleted file mode 100755 index 2b18be87d8..0000000000 --- a/installation_tips/full_spikeinterface_environment_windows.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: si_env -channels: - - conda-forge - - defaults -dependencies: - - python=3.11 - - pip - - numpy<2 - - scipy - - joblib - - tqdm - - matplotlib - - h5py - - pandas - - xarray - - zarr - - scikit-learn - - hdbscan - - networkx - - pybind11 - - loky - - numba - - jupyter - - pyqt=5 - - pyqtgraph - - ipywidgets - - ipympl - - pip: - - ephyviewer - - spikeinterface[full,widgets] - - spikeinterface-gui - - tridesclous - - kilosort From 9aff21d548d75a847f6fe67ddf8450376bd6a72e Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 17 Jul 2025 16:59:04 +0200 Subject: [PATCH 148/157] Add sampling_frequency_max_diff to aggregate_sortings --- .../tests/test_unitsaggregationsorting.py | 17 ++++++++++++- .../core/unitsaggregationsorting.py | 24 ++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/spikeinterface/core/tests/test_unitsaggregationsorting.py b/src/spikeinterface/core/tests/test_unitsaggregationsorting.py index f60de5b62a..fadac094aa 100644 --- a/src/spikeinterface/core/tests/test_unitsaggregationsorting.py +++ b/src/spikeinterface/core/tests/test_unitsaggregationsorting.py @@ -149,5 +149,20 @@ def test_unit_aggregation_does_not_preserve_ids_not_the_same_type(): assert list(aggregated_sorting.get_unit_ids()) == ["0", "1", "2", "3", "4"] +def test_sampling_frequency_max_diff(): + """Test that the sampling frequency max diff is respected.""" + sorting1 = generate_sorting(sampling_frequency=30000, num_units=3) + sorting2 = generate_sorting(sampling_frequency=30000.01, num_units=3) + sorting3 = generate_sorting(sampling_frequency=30000.001, num_units=3) + + # Default is 0, so should not raise an error + with pytest.raises(ValueError): + aggregate_units([sorting1, sorting2, sorting3]) + + # This should not raise an warning + with pytest.warns(UserWarning): + aggregate_units([sorting1, sorting2, sorting3], sampling_frequency_max_diff=0.02) + + if __name__ == "__main__": - test_unitsaggregationsorting() + test_sampling_frequency_max_diff() diff --git a/src/spikeinterface/core/unitsaggregationsorting.py b/src/spikeinterface/core/unitsaggregationsorting.py index cb824abd49..838660df46 100644 --- a/src/spikeinterface/core/unitsaggregationsorting.py +++ b/src/spikeinterface/core/unitsaggregationsorting.py @@ -4,9 +4,10 @@ import warnings import numpy as np -from .core_tools import define_function_from_class -from .base import BaseExtractor -from .basesorting import BaseSorting, BaseSortingSegment +from spikeinterface.core.core_tools import define_function_from_class +from spikeinterface.core.base import BaseExtractor +from spikeinterface.core.basesorting import BaseSorting, BaseSortingSegment +from spikeinterface.core.segmentutils import _check_sampling_frequencies class UnitsAggregationSorting(BaseSorting): @@ -19,6 +20,8 @@ class UnitsAggregationSorting(BaseSorting): List of BaseSorting objects to aggregate renamed_unit_ids: array-like If given, unit ids are renamed as provided. If None, unit ids are sequential integers. + sampling_frequency_max_diff : float, default: 0 + Maximum allowed difference of sampling frequencies across recordings Returns ------- @@ -26,7 +29,7 @@ class UnitsAggregationSorting(BaseSorting): The aggregated sorting object """ - def __init__(self, sorting_list, renamed_unit_ids=None): + def __init__(self, sorting_list, renamed_unit_ids=None, sampling_frequency_max_diff=0): unit_map = {} num_all_units = sum([sort.get_num_units() for sort in sorting_list]) @@ -60,15 +63,14 @@ def __init__(self, sorting_list, renamed_unit_ids=None): unit_map[unit_ids[u_id]] = {"sorting_id": s_i, "unit_id": unit_id} u_id += 1 - sampling_frequency = sorting_list[0].get_sampling_frequency() + sampling_frequencies = [sort.sampling_frequency for sort in sorting_list] num_segments = sorting_list[0].get_num_segments() - ok1 = all( - math.isclose(sampling_frequency, sort.get_sampling_frequency(), abs_tol=1e-2) for sort in sorting_list - ) - ok2 = all(num_segments == sort.get_num_segments() for sort in sorting_list) - if not (ok1 and ok2): - raise ValueError("Sortings don't have the same sampling_frequency/num_segments") + _check_sampling_frequencies(sampling_frequencies, sampling_frequency_max_diff) + sampling_frequency = sampling_frequencies[0] + num_segments_ok = all(num_segments == sort.get_num_segments() for sort in sorting_list) + if not num_segments_ok: + raise ValueError("Sortings don't have the same num_segments") BaseSorting.__init__(self, sampling_frequency, unit_ids) From 0490d5d515004a670b83dbe8502733372eac7e2e Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:09:21 -0400 Subject: [PATCH 149/157] minor cleanup+test footnote --- installation_tips/README.md | 48 ++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index a49d02bb3e..21b1c92945 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -1,39 +1,45 @@ ## Installation tips -If you are not (yet) an expert in Python installations, a main difficulty is choosing the installation procedure. - -Some main concepts you need to know before starting: - * Python itself can be distributed and installed many many ways. - * Python itself does not contain so many features for scientific computing you need to install "packages". - numpy, scipy, matplotlib, spikeinterface, ... are Python packages that have a complicated dependency graph between then. - * packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) - * installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environement" to not destroy any previous installation. You need to see an "environment" as a sub-installation in a dedicated folder. +If you are not (yet) an expert in Python installations, the first major hurdle is choosing the installation procedure. + +Some key concepts you need to know before starting: + * Python itself can be distributed and installed many, many ways. + * Python itself does not contain many features for scientific computing, so you need to install "packages". For example + numpy, scipy, matplotlib, spikeinterface, ... These are all examples of Python packages that aid in scientific computation. + * All of these packages have their own dependencies which requires figuring out which versions of the dependencies work for + the combination of packages you as the user want to use. + * Packages can be distributed and installed in several ways (pip, conda, uv, mamba, ...) and luckily these methods of installation + typically take care of solving the dependencies for you! + * Installing many packages at once is challenging (because of their dependency graphs) so you need to do it in an "isolated environment" to not destroy any previous installation. You need to see an "environment" as a sub-installation in a dedicated folder. Choosing the installer + an environment manager + a package installer is a nightmare for beginners. The main options are: * use "uv", a new, fast and simple package manager. We recommend this for beginners on every operating system. - * use "anaconda", which does everything. Used to be very popular but theses days it is becoming - a bad idea because : slow by default and aggressive licensing on the default channel (not always free anymore). - You need to play with "community channels" to make it free again, which is too complicated for beginners. - Do not go this way. - * use Python from the system or Python.org + venv + pip: good and simple idea for linux users. - -Here we propose a step by step recipe for beginers based on [**"uv"**](https://github.com/astral-sh/uv). + * use "anaconda" (or its flavors-mamba, miniconda), which does everything. Used to be very popular but theses days it is becoming + harder to use because it is slow by default and has relatively strict licensing on the default channel (not always free anymore). + You need to play with "community channels" to make it free again, which is complicated for beginners. + This way is better for users in organizations that have specific licensing agrees with anaconda already in place. + * use Python from the system or Python.org + venv + pip: good and simple idea for Linux users, but does require familiarity with + the Python ecosystem (so good for intermediate users). + +Here we propose a step by step recipe for beginners based on [**"uv"**](https://github.com/astral-sh/uv). We used to recommend installing with anaconda. It will be kept here for a while but we do not recommend it anymore. -This environment will install: +This recipe will install: * spikeinterface `full` option * spikeinterface-gui * kilosort4 +into our uv venv environment. + ### Quick installation using "uv" (recommended) 1. On macOS and Linux. Open a terminal and do `curl -LsSf https://astral.sh/uv/install.sh | sh` -2. On Windows. Open a terminal using CMD +2. On Windows. Open an instance of the Powershell (Windows has many options this is the recommended one from uv) `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"` 3. Exit the session and log in again. 4. Download with right click and save this file in your "Documents" folder: @@ -59,7 +65,7 @@ If you want to test the spikeinterface install you can: 1. Download with right click + save the file [`check_your_install.py`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/check_your_install.py) and put it into the "Documents" folder -2. Open the CMD (Windows) or Terminal (Mac/Linux) +2. Open the CMD Prompt (Windows)[^1] or Terminal (Mac/Linux) 3. Activate your si_env : `source si_env/bin/activate` (Max/Linux), `si_env\Scripts\activate` (Windows) 4. Go to your "Documents" folder with `cd Documents` or the place where you downloaded the `check_your_install.py` 5. Run `python check_your_install.py` @@ -81,7 +87,7 @@ Steps: 1. Download Anaconda individual edition [here](https://www.anaconda.com/download) 2. Run the installer. Check the box “Add anaconda3 to my Path environment variable”. It makes life easier for beginners. -3. Download with right click + save the enfironment YAML file ([`beginner_conda_env_stable.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/beginner_conda_env_stable.yml)) and put it in "Documents" folder +3. Download with right click + save the environment YAML file ([`beginner_conda_env_stable.yml`](https://raw.githubusercontent.com/SpikeInterface/spikeinterface/main/installation_tips/beginner_conda_env_stable.yml)) and put it in "Documents" folder 4. Then open the "Anaconda Command Prompt" (if Windows, search in your applications) or the Terminal (for Mac users) 5. If not in the "Documents" folder type `cd Documents` 6. Run this command to create the environment: @@ -92,3 +98,7 @@ Steps: Done! Before running a spikeinterface script you will need to "select" this "environment" with `conda activate si_env`. Note for **Linux** users: this conda recipe should work but we recommend strongly to use **pip + virtualenv**. + + + +[^1]: Although uv installation instructions are for the Powershell. Our sorter scripts are for the CMD Prompt. Using both terminals is possible. From 2eab14bf7b1b0224c5a2942c95662aad6689e22c Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:18:11 -0400 Subject: [PATCH 150/157] make Windows shell explanation clearer. --- installation_tips/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installation_tips/README.md b/installation_tips/README.md index 21b1c92945..04c93db2ae 100644 --- a/installation_tips/README.md +++ b/installation_tips/README.md @@ -101,4 +101,5 @@ Note for **Linux** users: this conda recipe should work but we recommend strongl -[^1]: Although uv installation instructions are for the Powershell. Our sorter scripts are for the CMD Prompt. Using both terminals is possible. +[^1]: Although uv installation instructions are for the Powershell, our sorter scripts are for the CMD Prompt. After the initial installation with Powershell, any session that will have sorting requires the CMD Prompt. If you do not +plan to spike sort in a session either shell could be used. From 87d4e36d9ac2eb18d633cceb8578168b94b1a247 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 09:48:27 +0200 Subject: [PATCH 151/157] Solve conflicts, add missing docs, check consistency in analyzer.split_units --- src/spikeinterface/core/sorting_tools.py | 106 ++++++++++++++---- src/spikeinterface/core/sortinganalyzer.py | 28 ++--- .../tests/test_multi_extensions.py | 2 +- 3 files changed, 95 insertions(+), 41 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index fcb7e0113b..d818f5c1a0 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -376,7 +376,7 @@ def _get_ids_after_merging(old_unit_ids, merge_unit_groups, new_unit_ids): def generate_unit_ids_for_merge_group(old_unit_ids, merge_unit_groups, new_unit_ids=None, new_id_strategy="append"): """ - Function to generate new units ids during a merging procedure. If new_units_ids + Function to generate new units ids during a merging procedure. If `new_units_ids` are provided, it will return these unit ids, checking that they have the the same length as `merge_unit_groups`. @@ -444,7 +444,52 @@ def generate_unit_ids_for_merge_group(old_unit_ids, merge_unit_groups, new_unit_ ### SPLITTING ZONE ### -def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extra=False, new_id_strategy="append"): +def apply_splits_to_sorting( + sorting: BaseSorting, + unit_splits: dict[int | str, list[list[int | str]]], + new_unit_ids: list[list[int | str]] | None = None, + return_extra: bool = False, + new_id_strategy: str = "append", +): + """ + Apply a the splits to a sorting object. + + This function is not lazy and creates a new NumpySorting with a compact spike_vector as fast as possible. + The `unit_splits` should be a dict with the unit ids as keys and a list of lists of spike indices as values. + For each split, the list of spike indices should contain the indices of the spikes to be assigned to each split and + it should be complete (i.e. the sum of the lengths of the sublists must equal the number of spikes in the unit). + If `new_unit_ids` is not None, it will use these new unit ids for the split units. + If `new_unit_ids` is None, it will generate new unit ids according to `new_id_strategy`. + + Parameters + ---------- + sorting : BaseSorting + The Sorting object to apply splits. + unit_splits : dict + A dictionary with the split unit id as key and a list of lists of spike indices for each split. + The split indices for each unit MUST be a list of lists, where each sublist (at least two) contains the + indices of the spikes to be assigned to the each split. The sum of the lengths of the sublists must equal + the number of spikes in the unit. + new_unit_ids : list | None, default: None + List of new unit_ids for each split. If given, it needs to have the same length as `unit_splits`. + and each element must have the same length as the corresponding list of split indices. + If None, new ids will be generated. + return_extra : bool, default: False + If True, also return the new_unit_ids. + new_id_strategy : "append" | "split", default: "append" + The strategy that should be used, if `new_unit_ids` is None, to create new unit_ids. + + * "append" : new_units_ids will be added at the end of max(sorging.unit_ids) + * "split" : new_unit_ids will be the created as {split_unit_id]-{split_number} + (e.g. when splitting unit "13" in 2: "13-0" / "13-1"). + Only works if unit_ids are str otherwise switch to "append" + + Returns + ------- + sorting : NumpySorting + The newly create sorting with the split units. + """ + check_unit_splits_consistency(unit_splits, sorting) spikes = sorting.to_spike_vector().copy() # here we assume that unit_splits split_indices are already full. @@ -492,9 +537,9 @@ def apply_splits_to_sorting(sorting, unit_splits, new_unit_ids=None, return_extr def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, new_id_strategy="append"): """ - Function to generate new units ids during a merging procedure. If new_units_ids - are provided, it will return these unit ids, checking that they have the the same - length as `merge_unit_groups`. + Function to generate new units ids during a splitting procedure. If `new_units_ids` + are provided, it will return these unit ids, checking that they are consistent with + `unit_splits`. Parameters ---------- @@ -503,14 +548,15 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne unit_splits : dict new_unit_ids : list | None, default: None - Optional new unit_ids for merged units. If given, it needs to have the same length as `merge_unit_groups`. + Optional new unit_ids for split units. If given, it needs to have the same length as `merge_unit_groups`. If None, new ids will be generated. - new_id_strategy : "append" | "take_first" | "join", default: "append" + new_id_strategy : "append" | "split", default: "append" The strategy that should be used, if `new_unit_ids` is None, to create new unit_ids. * "append" : new_units_ids will be added at the end of max(sorging.unit_ids) - * "split" : new_unit_ids will join unit_ids of groups with a "-". - Only works if unit_ids are str otherwise switch to "append" + * "split" : new_unit_ids will be the created as {split_unit_id]-{split_number} + (e.g. when splitting unit "13" in 2: "13-0" / "13-1"). + Only works if unit_ids are str otherwise switch to "append" Returns ------- @@ -559,19 +605,39 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne return new_unit_ids -def _get_full_unit_splits(unit_splits, sorting): - # take care of single-list splits - full_unit_splits = {} +def check_unit_splits_consistency(unit_splits, sorting): + """ + Function to check the consistency of unit_splits indices with the sorting object. + It checks that the split indices for each unit are a list of lists, where each sublist (at least two) + contains the indices of the spikes to be assigned to each split. The sum of the lengths + of the sublists must equal the number of spikes in the unit. + + Parameters + ---------- + unit_splits : dict + A dictionary with the split unit id as key and a list of numpy arrays or lists of spike indices for each split. + sorting : BaseSorting + The sorting object containing spike information. + + Raises + ------ + ValueError + If the unit_splits are not in the expected format or if the total number of spikes in the splits does not match + the number of spikes in the unit. + """ num_spikes = sorting.count_num_spikes_per_unit() for unit_id, split_indices in unit_splits.items(): - if not isinstance(split_indices[0], (list, np.ndarray)): - split_2 = np.arange(num_spikes[unit_id]) - split_2 = split_2[~np.isin(split_2, split_indices)] - new_split_indices = [split_indices, split_2] - else: - new_split_indices = split_indices - full_unit_splits[unit_id] = new_split_indices - return full_unit_splits + if not isinstance(split_indices, (list, np.ndarray)): + raise ValueError(f"unit_splits[{unit_id}] should be a list or numpy array, got {type(split_indices)}") + if not all(isinstance(indices, (list, np.ndarray)) for indices in split_indices): + raise ValueError(f"unit_splits[{unit_id}] should be a list of lists or numpy arrays") + if len(split_indices) < 2: + raise ValueError(f"unit_splits[{unit_id}] should have at least two splits") + total_spikes_in_split = sum(len(indices) for indices in split_indices) + if total_spikes_in_split != num_spikes[unit_id]: + raise ValueError( + f"Total spikes in unit {unit_id} split ({total_spikes_in_split}) does not match the number of spikes in the unit ({num_spikes[unit_id]})" + ) def _get_ids_after_splitting(old_unit_ids, split_units, new_unit_ids): diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index 7126647d04..6103fce14b 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -33,10 +33,10 @@ ) from .sorting_tools import ( generate_unit_ids_for_merge_group, - _get_ids_after_merging, generate_unit_ids_for_split, + check_unit_splits_consistency, + _get_ids_after_merging, _get_ids_after_splitting, - _get_full_unit_splits, ) from .job_tools import split_job_kwargs from .numpyextractors import NumpySorting @@ -1280,22 +1280,12 @@ def merge_units( assert merging_mode in ["soft", "hard"], "Merging mode should be either soft or hard" if len(merge_unit_groups) == 0: - # TODO I think we should raise an error or at least make a copy and not return itself - if return_new_unit_ids: - return self, [] - else: - return self + raise ValueError("Merging requires at least one group of units to merge") for units in merge_unit_groups: - # TODO more checks like one units is only in one group if len(units) < 2: raise ValueError("Merging requires at least two units to merge") - # TODO : no this function did not exists before - if not isinstance(merge_unit_groups[0], (list, tuple)): - # keep backward compatibility : the previous behavior was only one merge - merge_unit_groups = [merge_unit_groups] - new_unit_ids = generate_unit_ids_for_merge_group( self.unit_ids, merge_unit_groups, new_unit_ids, new_id_strategy ) @@ -1339,6 +1329,9 @@ def split_units( ---------- split_units : dict A dictionary with the keys being the unit ids to split and the values being the split indices. + The split indices for each unit MUST be a list of lists, where each sublist (at least two) contains the + indices of the spikes to be assigned to the each split. The sum of the lengths of the sublists must equal + the number of spikes in the unit. new_unit_ids : None | list, default: None A new unit_ids for split units. If given, it needs to have the same length as `merge_unit_groups`. If None, merged units will have the first unit_id of every lists of merges @@ -1366,14 +1359,9 @@ def split_units( folder = clean_zarr_folder_name(folder) if len(split_units) == 0: - # TODO I think we should raise an error or at least make a copy and not return itself - if return_new_unit_ids: - return self, [] - else: - return self + raise ValueError("Splitting requires at least one unit to split") - # TODO: add some checks - split_units = _get_full_unit_splits(split_units, self.sorting) + check_unit_splits_consistency(split_units, self.sorting) new_unit_ids = generate_unit_ids_for_split(self.unit_ids, split_units, new_unit_ids, new_id_strategy) all_unit_ids = _get_ids_after_splitting(self.unit_ids, split_units, new_unit_ids=new_unit_ids) diff --git a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py index 8c512c2109..87ff3cdeb7 100644 --- a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py +++ b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py @@ -236,7 +236,7 @@ def test_SortingAnalyzer_split_all_extensions(dataset_to_split, sparse): unsplit_unit_ids = sorting_analyzer.unit_ids[~np.isin(sorting_analyzer.unit_ids, units_to_split)] splits = {} for unit in units_to_split: - splits[unit] = np.arange(num_spikes[unit] // 2) + splits[unit] = [np.arange(num_spikes[unit] // 2), np.arange(num_spikes[unit] // 2, num_spikes[unit])] analyzer_split, split_unit_ids = sorting_analyzer.split_units(split_units=splits, return_new_unit_ids=True) split_unit_ids = list(np.concatenate(split_unit_ids)) From a2270cdfe50a101950454d7de2d748310f1522b6 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 09:59:29 +0200 Subject: [PATCH 152/157] Add splitting mode and warning for template split_extension_data --- .../core/analyzer_extension_core.py | 7 +++++++ src/spikeinterface/core/sortinganalyzer.py | 18 +++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/spikeinterface/core/analyzer_extension_core.py b/src/spikeinterface/core/analyzer_extension_core.py index 5170e59426..4d276219a6 100644 --- a/src/spikeinterface/core/analyzer_extension_core.py +++ b/src/spikeinterface/core/analyzer_extension_core.py @@ -9,6 +9,7 @@ * ComputeNoiseLevels which is very convenient to have """ +import warnings import numpy as np from .sortinganalyzer import AnalyzerExtension, register_result_extension @@ -563,6 +564,12 @@ def _merge_extension_data( return new_data def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + if not new_sorting_analyzer.has_extension("waveforms"): + warnings.warn( + "Splitting templates without the 'waveforms' extension will simply copy the template of the unit that " + "was split to the new split units. This is not recommended and may lead to incorrect results. It is " + "recommended to compute the 'waveforms' extension before splitting, or to use 'hard' splitting mode.", + ) new_data = dict() for operator, arr in self.data.items(): # we first copy the unsplit units diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index 6103fce14b..a123e70001 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -900,14 +900,15 @@ def _save_or_select_or_merge_or_split( folder=None, unit_ids=None, merge_unit_groups=None, - split_units=None, censor_ms=None, merging_mode="soft", sparsity_overlap=0.75, - verbose=False, merge_new_unit_ids=None, + split_units=None, + splitting_mode="soft", split_new_unit_ids=None, backend_options=None, + verbose=False, **job_kwargs, ) -> "SortingAnalyzer": """ @@ -925,18 +926,21 @@ def _save_or_select_or_merge_or_split( merge_unit_groups : list/tuple of lists/tuples or None, default: None A list of lists for every merge group. Each element needs to have at least two elements (two units to merge). If `merge_unit_groups` is not None, `new_unit_ids` must be given. - split_units : dict or None, default: None - A dictionary with the keys being the unit ids to split and the values being the split indices. censor_ms : None or float, default: None When merging units, any spikes violating this refractory period will be discarded. merging_mode : "soft" | "hard", default: "soft" How merges are performed. In the "soft" mode, merges will be approximated, with no smart merging - of the extension data. + of the extension data. In the "hard" mode, the extensions for merged units will be recomputed. sparsity_overlap : float, default 0.75 The percentage of overlap that units should share in order to accept merges. If this criteria is not achieved, soft merging will not be performed. merge_new_unit_ids : list or None, default: None The new unit ids for merged units. Required if `merge_unit_groups` is not None. + split_units : dict or None, default: None + A dictionary with the keys being the unit ids to split and the values being the split indices. + splitting_mode : "soft" | "hard", default: "soft" + How splits are performed. In the "soft" mode, splits will be approximated, with no smart splitting. + If `splitting_mode` is "hard", the extensons for split units willbe recomputed. split_new_unit_ids : list or None, default: None The new unit ids for split units. Required if `split_units` is not None. verbose : bool, default: False @@ -1125,11 +1129,11 @@ def _save_or_select_or_merge_or_split( recompute_dict[extension_name] = extension.params else: # split - try: + if splitting_mode == "soft": new_sorting_analyzer.extensions[extension_name] = extension.split( new_sorting_analyzer, split_units=split_units, new_unit_ids=split_new_unit_ids, verbose=verbose ) - except NotImplementedError: + elif splitting_mode == "hard": recompute_dict[extension_name] = extension.params if len(recompute_dict) > 0: From 084419db5806ddee1561afe5dcf77498090bb1f2 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 10:12:24 +0200 Subject: [PATCH 153/157] Imorove logic in new unit ids for splits --- src/spikeinterface/core/sorting_tools.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index d818f5c1a0..4e2ce6afca 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings import importlib.util import numpy as np @@ -576,6 +577,10 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne ), "new_unit_ids already exists but outside the split groups" else: dtype = old_unit_ids.dtype + if np.issubdtype(dtype, np.integer) and new_id_strategy == "split": + warnings.warn("new_id_strategy 'split' is not compatible with integer unit_ids. Switching to 'append'.") + new_id_strategy = "append" + new_unit_ids = [] for unit_to_split, split_indices in unit_splits.items(): num_splits = len(split_indices) @@ -589,18 +594,16 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne new_unit_ids.append([str(m + i) for i in range(num_splits)]) else: # we cannot automatically find new names - new_unit_ids.append([f"split{i}" for i in range(num_splits)]) + new_unit_ids.append([f"{unit_to_split}-split{i}" for i in range(num_splits)]) else: # dtype int new_unit_ids.append(list(max(old_unit_ids) + 1 + np.arange(num_splits, dtype=dtype))) + # we append the last unit id to the list of old unit ids so that the new unit ids + # will continue to increment old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) elif new_id_strategy == "split": - if np.issubdtype(dtype, np.character): - new_unit_ids.append([f"{unit_to_split}-{i}" for i in np.arange(len(split_indices))]) - else: - # dtype int - new_unit_ids.append(list(max(old_unit_ids) + 1 + np.arange(num_splits, dtype=dtype))) - old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) + # we made sure that dtype is not integer + new_unit_ids.append([f"{unit_to_split}-{i}" for i in np.arange(len(split_indices))]) return new_unit_ids From 42f6522bfa5bde76c544124a61634276064f00c5 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 11:55:40 +0200 Subject: [PATCH 154/157] Add property propagation to merge/split sorting --- .../core/analyzer_extension_core.py | 5 +- src/spikeinterface/core/sorting_tools.py | 130 ++++++++++++++++-- .../core/tests/test_sortinganalyzer.py | 56 +++++++- 3 files changed, 178 insertions(+), 13 deletions(-) diff --git a/src/spikeinterface/core/analyzer_extension_core.py b/src/spikeinterface/core/analyzer_extension_core.py index 4d276219a6..4a045c255f 100644 --- a/src/spikeinterface/core/analyzer_extension_core.py +++ b/src/spikeinterface/core/analyzer_extension_core.py @@ -599,8 +599,9 @@ def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, arr = np.percentile(wfs, float(percentile), axis=0) new_array[split_unit_index, ...] = arr else: - old_template = arr[self.sorting_analyzer.sorting.ids_to_indices([split_unit_id])[0], ...] - new_indices = np.array([new_unit_ids.index(unit_id) for unit_id in new_splits]) + split_unit_index = self.sorting_analyzer.sorting.id_to_index(split_unit_id) + old_template = arr[split_unit_index, ...] + new_indices = new_sorting_analyzer.sorting.ids_to_indices(new_splits) new_array[new_indices, ...] = np.tile(old_template, (len(new_splits), 1, 1)) new_data[operator] = new_array return new_data diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index 4e2ce6afca..a4885d3f15 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -5,8 +5,9 @@ import numpy as np -from .basesorting import BaseSorting -from .numpyextractors import NumpySorting +from spikeinterface.core.base import BaseExtractor +from spikeinterface.core.basesorting import BaseSorting +from spikeinterface.core.numpyextractors import NumpySorting numba_spec = importlib.util.find_spec("numba") if numba_spec is not None: @@ -321,12 +322,69 @@ def apply_merges_to_sorting( keep_mask[group_indices[inds + 1]] = False spikes = spikes[keep_mask] - sorting = NumpySorting(spikes, sorting.sampling_frequency, all_unit_ids) + merge_sorting = NumpySorting(spikes, sorting.sampling_frequency, all_unit_ids) + set_properties_after_merging(merge_sorting, sorting, merge_unit_groups, new_unit_ids=new_unit_ids) if return_extra: - return sorting, keep_mask, new_unit_ids + return merge_sorting, keep_mask, new_unit_ids else: - return sorting + return merge_sorting + + +def set_properties_after_merging( + sorting_post_merge: BaseSorting, + sorting_pre_merge: BaseSorting, + merge_unit_groups: list[list[int | str]], + new_unit_ids: list[int | str], +): + """ + Add properties to the merge sorting object after merging units. + The properties of the merged units are propagated only if they are the same + for all units in the merge group. + + Parameters + ---------- + sorting_post_merge : BaseSorting + The Sorting object after merging units. + sorting_pre_merge : BaseSorting + The Sorting object before merging units. + merge_unit_groups : list + The groups of unit ids that were merged. + new_unit_ids : list + A list of new unit_ids for each merge. + """ + prop_keys = sorting_pre_merge.get_property_keys() + pre_unit_ids = sorting_pre_merge.unit_ids + post_unit_ids = sorting_post_merge.unit_ids + + kept_unit_ids = post_unit_ids[np.isin(post_unit_ids, pre_unit_ids)] + keep_pre_inds = sorting_pre_merge.ids_to_indices(kept_unit_ids) + keep_post_inds = sorting_post_merge.ids_to_indices(kept_unit_ids) + + for key in prop_keys: + parent_values = sorting_pre_merge.get_property(key) + + # propagate keep values + shape = (len(sorting_post_merge.unit_ids),) + parent_values.shape[1:] + new_values = np.empty(shape=shape, dtype=parent_values.dtype) + new_values[keep_post_inds] = parent_values[keep_pre_inds] + for new_id, merge_group in zip(new_unit_ids, merge_unit_groups): + merged_indices = sorting_pre_merge.ids_to_indices(merge_group) + merge_values = parent_values[merged_indices] + same_property_values = np.all([np.array_equal(m, merge_values[0]) for m in merge_values[1:]]) + new_index = sorting_post_merge.id_to_index(new_id) + if same_property_values: + # and new values only if they are all similar + new_values[new_index] = merge_values[0] + else: + default_missing_values = BaseExtractor.default_missing_property_values + new_values[new_index] = default_missing_values[parent_values.dtype.kind] + sorting_post_merge.set_property(key, new_values) + + # set is_merged property + is_merged = np.ones(len(sorting_post_merge.unit_ids), dtype=bool) + is_merged[keep_post_inds] = False + sorting_post_merge.set_property("is_merged", is_merged) def _get_ids_after_merging(old_unit_ids, merge_unit_groups, new_unit_ids): @@ -528,12 +586,68 @@ def apply_splits_to_sorting( for segment_index in range(num_seg): spike_inds = spike_indices[segment_index][unit_id] spikes["unit_index"][spike_inds] = new_unit_index - sorting = NumpySorting(spikes, sorting.sampling_frequency, all_unit_ids) + split_sorting = NumpySorting(spikes, sorting.sampling_frequency, all_unit_ids) + set_properties_after_splits( + split_sorting, + sorting, + list(unit_splits.keys()), + new_unit_ids=new_unit_ids, + ) if return_extra: - return sorting, new_unit_ids + return split_sorting, new_unit_ids else: - return sorting + return split_sorting + + +def set_properties_after_splits( + sorting_post_split: BaseSorting, + sorting_pre_split: BaseSorting, + split_unit_ids: list[int | str], + new_unit_ids: list[list[int | str]], +): + """ + Add properties to the split sorting object after splitting units. + The properties of the split units are propagated to the new split units. + + Parameters + ---------- + sorting_post_split : BaseSorting + The Sorting object after splitting units. + sorting_pre_split : BaseSorting + The Sorting object before splitting units. + split_unit_ids : list + The unit ids that were split. + new_unit_ids : list + A list of new unit_ids for each split. + """ + prop_keys = sorting_pre_split.get_property_keys() + pre_unit_ids = sorting_pre_split.unit_ids + post_unit_ids = sorting_post_split.unit_ids + + kept_unit_ids = post_unit_ids[np.isin(post_unit_ids, pre_unit_ids)] + keep_pre_inds = sorting_pre_split.ids_to_indices(kept_unit_ids) + keep_post_inds = sorting_post_split.ids_to_indices(kept_unit_ids) + + for key in prop_keys: + parent_values = sorting_pre_split.get_property(key) + + # propagate keep values + shape = (len(sorting_post_split.unit_ids),) + parent_values.shape[1:] + new_values = np.empty(shape=shape, dtype=parent_values.dtype) + new_values[keep_post_inds] = parent_values[keep_pre_inds] + for split_unit, new_split_ids in zip(split_unit_ids, new_unit_ids): + split_index = sorting_pre_split.id_to_index(split_unit) + split_value = parent_values[split_index] + # propagate the split value to all new unit ids + new_unit_indices = sorting_post_split.ids_to_indices(new_split_ids) + new_values[new_unit_indices] = split_value + sorting_post_split.set_property(key, new_values) + + # set is_merged property + is_split = np.ones(len(sorting_post_split.unit_ids), dtype=bool) + is_split[keep_post_inds] = False + sorting_post_split.set_property("is_split", is_split) def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, new_id_strategy="append"): diff --git a/src/spikeinterface/core/tests/test_sortinganalyzer.py b/src/spikeinterface/core/tests/test_sortinganalyzer.py index 7074c054b5..ce248f00f6 100644 --- a/src/spikeinterface/core/tests/test_sortinganalyzer.py +++ b/src/spikeinterface/core/tests/test_sortinganalyzer.py @@ -378,6 +378,7 @@ def _check_sorting_analyzers(sorting_analyzer, original_sorting, cache_folder): # unit 0, 2, ... should be removed assert np.all(~np.isin(data["result_two"], [0, 2])) + # test merges if format != "memory": if format == "zarr": folder = cache_folder / f"test_SortingAnalyzer_merge_soft_with_{format}.zarr" @@ -387,10 +388,14 @@ def _check_sorting_analyzers(sorting_analyzer, original_sorting, cache_folder): shutil.rmtree(folder) else: folder = None - sorting_analyzer4 = sorting_analyzer.merge_units(merge_unit_groups=[[0, 1]], format=format, folder=folder) + sorting_analyzer4, new_unit_ids = sorting_analyzer.merge_units( + merge_unit_groups=[[0, 1]], format=format, folder=folder, return_new_unit_ids=True + ) assert 0 not in sorting_analyzer4.unit_ids assert 1 not in sorting_analyzer4.unit_ids assert len(sorting_analyzer4.unit_ids) == len(sorting_analyzer.unit_ids) - 1 + is_merged_values = sorting_analyzer4.sorting.get_property("is_merged") + assert is_merged_values[sorting_analyzer4.sorting.ids_to_indices(new_unit_ids)][0] if format != "memory": if format == "zarr": @@ -401,13 +406,50 @@ def _check_sorting_analyzers(sorting_analyzer, original_sorting, cache_folder): shutil.rmtree(folder) else: folder = None - sorting_analyzer5 = sorting_analyzer.merge_units( - merge_unit_groups=[[0, 1]], new_unit_ids=[50], format=format, folder=folder, merging_mode="hard" + sorting_analyzer5, new_unit_ids = sorting_analyzer.merge_units( + merge_unit_groups=[[0, 1]], + new_unit_ids=[50], + format=format, + folder=folder, + merging_mode="hard", + return_new_unit_ids=True, ) assert 0 not in sorting_analyzer5.unit_ids assert 1 not in sorting_analyzer5.unit_ids assert len(sorting_analyzer5.unit_ids) == len(sorting_analyzer.unit_ids) - 1 assert 50 in sorting_analyzer5.unit_ids + is_merged_values = sorting_analyzer5.sorting.get_property("is_merged") + assert is_merged_values[sorting_analyzer5.sorting.id_to_index(50)] + + # test splitting + if format != "memory": + if format == "zarr": + folder = cache_folder / f"test_SortingAnalyzer_split_with_{format}.zarr" + else: + folder = cache_folder / f"test_SortingAnalyzer_split_with_{format}" + if folder.exists(): + shutil.rmtree(folder) + else: + folder = None + split_units = {} + num_spikes = sorting_analyzer.sorting.count_num_spikes_per_unit() + units_to_split = sorting_analyzer.unit_ids[:2] + for unit in units_to_split: + for unit in units_to_split: + split_units[unit] = [ + np.arange(num_spikes[unit] // 2), + np.arange(num_spikes[unit] // 2, num_spikes[unit]), + ] + + sorting_analyzer6, split_new_unit_ids = sorting_analyzer.split_units( + split_units=split_units, format=format, folder=folder, return_new_unit_ids=True + ) + for unit_to_split in units_to_split: + assert unit_to_split not in sorting_analyzer6.unit_ids + assert len(sorting_analyzer6.unit_ids) == len(sorting_analyzer.unit_ids) + 2 + is_split_values = sorting_analyzer6.sorting.get_property("is_split") + for new_unit_ids in split_new_unit_ids: + assert all(is_split_values[sorting_analyzer6.sorting.ids_to_indices(new_unit_ids)]) # test compute with extension-specific params sorting_analyzer.compute(["dummy"], extension_params={"dummy": {"param1": 5.5}}) @@ -491,6 +533,14 @@ def _merge_extension_data( return new_data + def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, verbose=False, **job_kwargs): + new_data = dict() + new_data["result_one"] = self.data["result_one"] + spikes = new_sorting_analyzer.sorting.to_spike_vector() + new_data["result_two"] = spikes["unit_index"].copy() + new_data["result_three"] = np.zeros((len(new_sorting_analyzer.unit_ids), 2)) + return new_data + def _get_data(self): return self.data["result_one"] From c2461fdd3c3dd6f7478e77ef9f2ab7e9a405a6c7 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 12:06:37 +0200 Subject: [PATCH 155/157] Improve readibility on generate_unit_ids_for_split --- src/spikeinterface/core/sorting_tools.py | 19 ++++++++++--------- src/spikeinterface/curation/curation_model.py | 7 +++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/spikeinterface/core/sorting_tools.py b/src/spikeinterface/core/sorting_tools.py index a4885d3f15..c74c76d010 100644 --- a/src/spikeinterface/core/sorting_tools.py +++ b/src/spikeinterface/core/sorting_tools.py @@ -696,28 +696,29 @@ def generate_unit_ids_for_split(old_unit_ids, unit_splits, new_unit_ids=None, ne new_id_strategy = "append" new_unit_ids = [] + current_unit_ids = old_unit_ids.copy() for unit_to_split, split_indices in unit_splits.items(): num_splits = len(split_indices) # select new_unit_ids greater that the max id, event greater than the numerical str ids if new_id_strategy == "append": if np.issubdtype(dtype, np.character): # dtype str - if all(p.isdigit() for p in old_unit_ids): + if all(p.isdigit() for p in current_unit_ids): # All str are digit : we can generate a max - m = max(int(p) for p in old_unit_ids) + 1 - new_unit_ids.append([str(m + i) for i in range(num_splits)]) + m = max(int(p) for p in current_unit_ids) + 1 + new_units_for_split = [str(m + i) for i in range(num_splits)] else: # we cannot automatically find new names - new_unit_ids.append([f"{unit_to_split}-split{i}" for i in range(num_splits)]) + new_units_for_split = [f"{unit_to_split}-split{i}" for i in range(num_splits)] else: # dtype int - new_unit_ids.append(list(max(old_unit_ids) + 1 + np.arange(num_splits, dtype=dtype))) - # we append the last unit id to the list of old unit ids so that the new unit ids - # will continue to increment - old_unit_ids = np.concatenate([old_unit_ids, new_unit_ids[-1]]) + new_units_for_split = list(max(current_unit_ids) + 1 + np.arange(num_splits, dtype=dtype)) + # we append the new split unit ids to continue to increment the max id + current_unit_ids = np.concatenate([current_unit_ids, new_units_for_split]) elif new_id_strategy == "split": # we made sure that dtype is not integer - new_unit_ids.append([f"{unit_to_split}-{i}" for i in np.arange(len(split_indices))]) + new_units_for_split = [f"{unit_to_split}-{i}" for i in np.arange(len(split_indices))] + new_unit_ids.append(new_units_for_split) return new_unit_ids diff --git a/src/spikeinterface/curation/curation_model.py b/src/spikeinterface/curation/curation_model.py index e32aa6e275..5eb025ace4 100644 --- a/src/spikeinterface/curation/curation_model.py +++ b/src/spikeinterface/curation/curation_model.py @@ -33,12 +33,11 @@ class Split(BaseModel): "If labels, the split is defined by a list of labels for each spike (`labels`). " ), ) - indices: Optional[Union[List[int], List[List[int]]]] = Field( + indices: Optional[List[List[int]]] = Field( default=None, description=( - "List of indices for the split. If a list of indices, the unit is splt in 2 (provided indices/others). " - "If a list of lists, the unit is split in multiple groups (one for each list of indices), plus an optional " - "extra if the spike train has more spikes than the sum of the indices in the lists." + "List of indices for the split. The unit is split in multiple groups (one for each list of indices), " + "plus an optional extra if the spike train has more spikes than the sum of the indices in the lists." ), ) labels: Optional[List[int]] = Field(default=None, description="List of labels for the split") From 40c922fae2182c12554cafea8c0efe9c2b092bab Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 12:08:29 +0200 Subject: [PATCH 156/157] remove comment --- src/spikeinterface/core/sortinganalyzer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index a123e70001..f8d11dd157 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -1059,8 +1059,6 @@ def _save_or_select_or_merge_or_split( unit_splits=split_units, new_unit_ids=split_new_unit_ids, ) - # TODO: sam/pierre would create a curation field / curation.json with the applied merges. - # What do you think? backend_options = {} if backend_options is None else backend_options From 0a3be661e205cfdcde09cc16cf38fa2b121f1133 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 18 Jul 2025 12:32:22 +0200 Subject: [PATCH 157/157] Update src/spikeinterface/postprocessing/template_similarity.py Co-authored-by: Garcia Samuel --- src/spikeinterface/postprocessing/template_similarity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/postprocessing/template_similarity.py b/src/spikeinterface/postprocessing/template_similarity.py index 833bbaf832..2771d382b0 100644 --- a/src/spikeinterface/postprocessing/template_similarity.py +++ b/src/spikeinterface/postprocessing/template_similarity.py @@ -171,7 +171,7 @@ def _split_extension_data(self, split_units, new_unit_ids, new_sorting_analyzer, old_ind2 = self.sorting_analyzer.sorting.id_to_index(unit_id2) s = self.data["similarity"][old_ind1, old_ind2] similarity[unit_ind1, unit_ind2] = s - similarity[unit_ind1, unit_ind2] = s + similarity[unit_ind2, unit_ind1] = s # insert new similarity both way for unit_ind, unit_id in enumerate(all_new_unit_ids):