diff --git a/.circleci/config.yml b/.circleci/config.yml index 96a6a06fa..aae6c58f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,13 +38,6 @@ references: username: hdmf password: $DOCKERHUB_PASSWORD - py36: &py36 - docker: - - image: circleci/python:3.6.13-buster - auth: - username: hdmf - password: $DOCKERHUB_PASSWORD - conda-image: &conda-image docker: - image: continuumio/miniconda3:4.9.2 @@ -161,14 +154,6 @@ jobs: - run: <<: *run-style-check - python36: - <<: *py36 - environment: - - TEST_TOX_ENV: "py36" - - BUILD_TOX_ENV: "build-py36" - - TEST_WHEELINSTALL_ENV: "wheelinstall" - <<: *ci-steps - python37: <<: *py37 environment: @@ -183,7 +168,6 @@ jobs: - TEST_TOX_ENV: "py38" - BUILD_TOX_ENV: "build-py38" - TEST_WHEELINSTALL_ENV: "wheelinstall" - - UPLOAD_WHEELS: "true" <<: *ci-steps python39: @@ -192,41 +176,33 @@ jobs: - TEST_TOX_ENV: "py39" - BUILD_TOX_ENV: "build-py39" - TEST_WHEELINSTALL_ENV: "wheelinstall" + - UPLOAD_WHEELS: "true" # upload distributions from only this job to pypi <<: *ci-steps - python38-upgrade-dev: - <<: *py38 + python39-upgrade-dev: + <<: *py39 environment: - - TEST_TOX_ENV: "py38-upgrade-dev" - - BUILD_TOX_ENV: "build-py38-upgrade-dev" + - TEST_TOX_ENV: "py39-upgrade-dev" + - BUILD_TOX_ENV: "build-py39-upgrade-dev" - TEST_WHEELINSTALL_ENV: "wheelinstall" <<: *ci-steps - python38-upgrade-dev-pre: - <<: *py38 + python39-upgrade-dev-pre: + <<: *py39 environment: - - TEST_TOX_ENV: "py38-upgrade-dev-pre" - - BUILD_TOX_ENV: "build-py38-upgrade-dev-pre" + - TEST_TOX_ENV: "py39-upgrade-dev-pre" + - BUILD_TOX_ENV: "build-py39-upgrade-dev-pre" - TEST_WHEELINSTALL_ENV: "wheelinstall" <<: *ci-steps - python36-min-req: - <<: *py36 + python37-min-req: + <<: *py37 environment: - - TEST_TOX_ENV: "py36-min-req" - - BUILD_TOX_ENV: "build-py36-min-req" + - TEST_TOX_ENV: "py37-min-req" + - BUILD_TOX_ENV: "build-py37-min-req" - TEST_WHEELINSTALL_ENV: "wheelinstall" <<: *ci-steps - miniconda36: - <<: *conda-image - environment: - - CONDA_PYTHON_VER: "3.6.*=*_cpython" # avoid using pypy compiler - - TEST_TOX_ENV: "py36" - - BUILD_TOX_ENV: "build-py36" - - TEST_WHEELINSTALL_ENV: "wheelinstall" - <<: *conda-steps - miniconda37: <<: *conda-image environment: @@ -254,12 +230,6 @@ jobs: - TEST_WHEELINSTALL_ENV: "wheelinstall" <<: *conda-steps - gallery36: - <<: *py36 - environment: - - TEST_TOX_ENV: "gallery-py36" - <<: *gallery-steps - gallery37: <<: *py37 environment: @@ -278,26 +248,26 @@ jobs: - TEST_TOX_ENV: "gallery-py39" <<: *gallery-steps - gallery38-upgrade-dev: - <<: *py38 + gallery39-upgrade-dev: + <<: *py39 environment: - - TEST_TOX_ENV: "gallery-py38-upgrade-dev" + - TEST_TOX_ENV: "gallery-py39-upgrade-dev" <<: *gallery-steps - gallery38-upgrade-dev-pre: - <<: *py38 + gallery39-upgrade-dev-pre: + <<: *py39 environment: - - TEST_TOX_ENV: "gallery-py38-upgrade-dev-pre" + - TEST_TOX_ENV: "gallery-py39-upgrade-dev-pre" <<: *gallery-steps - gallery36-min-req: - <<: *py36 + gallery37-min-req: + <<: *py37 environment: - - TEST_TOX_ENV: "gallery-py36-min-req" + - TEST_TOX_ENV: "gallery-py37-min-req" <<: *gallery-steps test-validation: - <<: *py38 + <<: *py39 steps: - checkout - run: git submodule sync @@ -309,13 +279,13 @@ jobs: command: | . ../venv/bin/activate pip install tox - tox -e validation-py38 + tox -e validation-py39 # Install is expected to be quick. Increase timeout in case there are some network issues. # While pip installing tox does not output by default. Circle thinks task is dead after 10 min. no_output_timeout: 30m deploy-dev: - <<: *py38 + <<: *py39 steps: - checkout - attach_workspace: @@ -336,7 +306,7 @@ jobs: --exit-success-if-missing-token deploy-release: - <<: *py38 + <<: *py39 steps: - attach_workspace: at: ./ @@ -366,27 +336,27 @@ workflows: jobs: - flake8: <<: *no_filters - - python38: + - python37-min-req: <<: *no_filters - - python36-min-req: + - python39: <<: *no_filters - - miniconda36: + - miniconda37: <<: *no_filters - - miniconda38: + - miniconda39: <<: *no_filters - - gallery38: + - gallery37-min-req: <<: *no_filters - - gallery36-min-req: + - gallery38: # TODO replace with gallery39 after allensdk support py39 <<: *no_filters - deploy-dev: requires: - flake8 - - python38 - - python36-min-req - - miniconda36 - - miniconda38 - - gallery38 - - gallery36-min-req + - python37-min-req + - python39 + - miniconda37 + - miniconda39 + - gallery37-min-req + - gallery38 # gallery39 filters: tags: ignore: @@ -400,12 +370,12 @@ workflows: - deploy-release: requires: - flake8 - - python38 - - python36-min-req - - miniconda36 - - miniconda38 - - gallery38 - - gallery36-min-req + - python37-min-req + - python39 + - miniconda37 + - miniconda39 + - gallery37-min-req + - gallery38 # gallery39 filters: tags: only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/ @@ -424,19 +394,17 @@ workflows: jobs: - flake8: <<: *no_filters - - python36: - <<: *no_filters - python37: <<: *no_filters + - python37-min-req: + <<: *no_filters - python38: <<: *no_filters - python39: <<: *no_filters - - python38-upgrade-dev: - <<: *no_filters - - python36-min-req: + - python39-upgrade-dev: <<: *no_filters - - miniconda36: + - python39-upgrade-dev-pre: <<: *no_filters - miniconda37: <<: *no_filters @@ -444,21 +412,15 @@ workflows: <<: *no_filters - miniconda39: <<: *no_filters - - gallery36: - <<: *no_filters - gallery37: <<: *no_filters + - gallery37-min-req: + <<: *no_filters - gallery38: <<: *no_filters - gallery39: <<: *no_filters - - gallery38-upgrade-dev: - <<: *no_filters - - gallery36-min-req: - <<: *no_filters - - python38-upgrade-dev-pre: - <<: *no_filters - - gallery38-upgrade-dev-pre: + - gallery39-upgrade-dev: <<: *no_filters - - test-validation: + - gallery39-upgrade-dev-pre: <<: *no_filters diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index ba3fdb027..14f7796b1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -20,7 +20,7 @@ assignees: '' Python Executable: Conda or Python - Python Version: Python 3.6, 3.7, or 3.8 + Python Version: Python 3.7, 3.8, or 3.9 Operating System: Windows, macOS or Linux HDMF Version: PyNWB Version: diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c0cfcc234..70eb5695c 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -15,7 +15,7 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] env: OS: ${{ matrix.os }} - PYTHON: '3.8' + PYTHON: '3.9' steps: - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.6.0 @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f0723196..51bdcf77f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,59 @@ # PyNWB Changelog -## PyNWB 1.6.0 (TBD, 2021) +## PyNWB 2.0.0 (Upcoming) + +### Breaking changes: +- ``SweepTable`` has been deprecated in favor of the new icephys metadata tables. Use of ``SweepTable`` + is still possible but no longer recommended. @oruebel (#1349 +- ``TimeSeries.__init__`` now requires the ``data`` argument because the 'data' dataset is required by the schema. + If a ``TimeSeries`` is read without a value for ``data``, it will be set to a default value. For most + ``TimeSeries``, this is a 1-dimensional empty array with dtype uint8. For ``ImageSeries`` and + ``DecompositionSeries``, this is a 3-dimensional empty array with dtype uint8. @rly (#1274) +- ``TimeSeries.__init__`` now requires the ``unit`` argument because the 'unit' attribute is required by the schema. + If a ``TimeSeries`` is read without a value for ``unit``, it will be set to a default value. For most + ``TimeSeries``, this is "unknown". For ``IndexSeries``, this is "N/A" according to the NWB 2.4.0 schema. @rly (#1274) -## Minor new features: +### New features: +- Added new intracellular electrophysiology hierarchical table structure from ndx-icephys-meta to NWB core. + This includes the new types ``TimeSeriesReferenceVectorData``, ``IntracellularRecordingsTable``, + ``SimultaneousRecordingsTable``, ``SequentialRecordingsTable``, ``RepetitionsTable`` and + ``ExperimentalConditionsTable`` as well as corresponding updates to ``NWBFile`` to support interaction + with the new tables. @oruebel (#1349) +- Added support for NWB 2.4.0. See [Release Notes](https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html) + for more details. @oruebel, @rly (#1349) +- Dropped Python 3.6 support, added Python 3.9 support. @rly (#1377) +- Updated requirements to allow compatibility with HDMF 3 and h5py 3. @rly (#1377) + +### Tutorial enhancements: +- Added new tutorial for intracellular electrophysiology to describe the use of the new metadata tables + and declared the previous tutoral using ``SweepTable`` as deprecated. @oruebel (#1349) +- Added new tutorial for querying intracellular electrophysiology metadata + (``docs/gallery/domain/plot_icephys_pandas.py``). @oruebel (#1349, #1383) +- Added thumbnails for tutorials to improve presentation of online docs. @oruebel (#1349) +- Used `sphinx.ext.extlinks` extension in docs to simplify linking to common targets. @oruebel (#1349) +- Created new section for advanced I/O tutorials and moved parallel I/O tutorial to its own file. @oruebel (#1349) + +### Minor new features: - Add RRID for citing PyNWB to the docs. @oruebel (#1372) +- Update CI and tests to handle deprecations in libraries. @rly (#1377) +- Add test utilities for icephys (``pynwb.testing.icephys_testutils``) to ease creation of test data + for tests and tutorials. @oruebel (#1349, #1383) -## Bug fix: -- Enforce electrode ID uniqueness during insertion into table. @CodyCBakerPhD (#1344) -- Fix integration tests with invalid test data that will be caught by future hdmf validator version. +### Bug fixes: +- Updated behavior of ``make clean`` command for docs to ensure tutorial files are cleaned up. @oruebel (#1349) +- Enforced electrode ID uniqueness during insertion into table. @CodyCBakerPhD (#1344) +- Fixed integration tests with invalid test data that will be caught by future hdmf validator version. @dsleiter, @rly (#1366, #1376) -- Fix build warnings in docs @oruebel (#1380) +- Fixed build warnings in docs. @oruebel (#1380) +- Fix intersphinx links in docs for numpy. @oruebel (#1386) +- Previously, the ``data`` argument was required in ``OpticalSeries.__init__`` even though ``external_file`` could + be provided in place of ``data``. ``OpticalSeries.__init__`` now makes ``data`` optional. However, this has the + side effect of moving the position of ``data`` to later in the argument list, which may break code that relies + on positional arguments for ``OpticalSeries.__init__``. @rly (#1274) ## PyNWB 1.5.1 (May 24, 2021) -## Bug fix: +### Bug fixes: - Raise minimum version of pandas from 0.23 to 1.0.5 to be compatible with numpy 1.20, and raise minimum version of HDMF to use the corresponding change in HDMF. @rly (#1363) - Update documentation and update structure of requirements files. @rly (#1363) @@ -56,7 +96,7 @@ - Add capability to add a row to a column after IO. - Add method `AbstractContainer.get_fields_conf`. - Add functionality for storing external resource references. - - Add method `hdmf.utils.get_docval_macro` to get a tuple of the current values for a docval_macro, e.g., 'array_data' + - Add method `hdmf.utils.get_docval_macro` to get a tuple of the current values for a docval_macro, e.g., 'array_data' and 'scalar_data'. - `DynamicTable` can be automatically generated using `get_class`. Now the HDMF API can read files with extensions that contain a DynamicTable without needing to import the extension first. diff --git a/azure-pipelines-nightly.yml b/azure-pipelines-nightly.yml index 552d1bcd3..c0a79b9d2 100644 --- a/azure-pipelines-nightly.yml +++ b/azure-pipelines-nightly.yml @@ -19,23 +19,21 @@ jobs: imageName: 'macos-10.15' pythonVersion: '3.9' testToxEnv: 'py39' - coverageToxEnv: '' buildToxEnv: 'build-py39' testWheelInstallEnv: 'wheelinstall' - macOS-py3.8-upgrade-dev-pre: + macOS-py3.9-upgrade-dev-pre: imageName: 'macos-10.15' - pythonVersion: '3.8' - testToxEnv: 'py38-upgrade-dev-pre' - coverageToxEnv: '' - buildToxEnv: 'build-py38-upgrade-dev-pre' + pythonVersion: '3.9' + testToxEnv: 'py39-upgrade-dev-pre' + buildToxEnv: 'build-py39-upgrade-dev-pre' testWheelInstallEnv: 'wheelinstall' - macOS-py3.8-upgrade-dev: + macOS-py3.9-upgrade-dev: imageName: 'macos-10.15' - pythonVersion: '3.8' - testToxEnv: 'py38-upgrade-dev' - buildToxEnv: 'build-py38-upgrade-dev' + pythonVersion: '3.9' + testToxEnv: 'py39-upgrade-dev' + buildToxEnv: 'build-py39-upgrade-dev' testWheelInstallEnv: 'wheelinstall' macOS-py3.8: @@ -52,41 +50,32 @@ jobs: buildToxEnv: 'build-py37' testWheelInstallEnv: 'wheelinstall' - macOS-py3.6: + macOS-py3.7-min-req: imageName: 'macos-10.15' - pythonVersion: '3.6' - testToxEnv: 'py36' - buildToxEnv: 'build-py36' - testWheelInstallEnv: 'wheelinstall' - - macOS-py3.6-min-req: - imageName: 'macos-10.15' - pythonVersion: '3.6' - testToxEnv: 'py36-min-req' - buildToxEnv: 'build-py36-min-req' + pythonVersion: '3.7' + testToxEnv: 'py37-min-req' + buildToxEnv: 'build-py37-min-req' testWheelInstallEnv: 'wheelinstall' Windows-py3.9: imageName: 'vs2017-win2016' pythonVersion: '3.9' testToxEnv: 'py39' - coverageToxEnv: '' buildToxEnv: 'build-py39' testWheelInstallEnv: 'wheelinstall' - Windows-py3.8-upgrade-dev-pre: + Windows-py3.9-upgrade-dev-pre: imageName: 'vs2017-win2016' - pythonVersion: '3.8' - testToxEnv: 'py38-upgrade-dev-pre' - coverageToxEnv: '' - buildToxEnv: 'build-py38-upgrade-dev-pre' + pythonVersion: '3.9' + testToxEnv: 'py39-upgrade-dev-pre' + buildToxEnv: 'build-py39-upgrade-dev-pre' testWheelInstallEnv: 'wheelinstall' - Windows-py3.8-upgrade-dev: + Windows-py3.9-upgrade-dev: imageName: 'vs2017-win2016' - pythonVersion: '3.8' - testToxEnv: 'py38-upgrade-dev' - buildToxEnv: 'build-py38-upgrade-dev' + pythonVersion: '3.9' + testToxEnv: 'py39-upgrade-dev' + buildToxEnv: 'build-py39-upgrade-dev' testWheelInstallEnv: 'wheelinstall' Windows-py3.8: @@ -103,18 +92,11 @@ jobs: buildToxEnv: 'build-py37' testWheelInstallEnv: 'wheelinstall' - Windows-py3.6: + Windows-py3.7-min-req: imageName: 'vs2017-win2016' - pythonVersion: '3.6' - testToxEnv: 'py36' - buildToxEnv: 'build-py36' - testWheelInstallEnv: 'wheelinstall' - - Windows-py3.6-min-req: - imageName: 'vs2017-win2016' - pythonVersion: '3.6' - testToxEnv: 'py36-min-req' - buildToxEnv: 'build-py36-min-req' + pythonVersion: '3.7' + testToxEnv: 'py37-min-req' + buildToxEnv: 'build-py37-min-req' testWheelInstallEnv: 'wheelinstall' pool: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b43eeb0a4..a720e78bd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,39 +8,38 @@ jobs: strategy: matrix: - macOS-py3.8: + macOS-py3.9: imageName: 'macos-10.15' - pythonVersion: '3.8' - testToxEnv: 'py38' - buildToxEnv: 'build-py38' + pythonVersion: '3.9' + testToxEnv: 'py39' + buildToxEnv: 'build-py39' testWheelInstallEnv: 'wheelinstall' - macOS-py3.6-min-req: + macOS-py3.7-min-req: imageName: 'macos-10.15' - pythonVersion: '3.6' - testToxEnv: 'py36-min-req' - buildToxEnv: 'build-py36-min-req' + pythonVersion: '3.7' + testToxEnv: 'py37-min-req' + buildToxEnv: 'build-py37-min-req' testWheelInstallEnv: 'wheelinstall' - Windows-py3.8: + Windows-py3.9: imageName: 'vs2017-win2016' - pythonVersion: '3.8' - testToxEnv: 'py38' - buildToxEnv: 'build-py38' + pythonVersion: '3.9' + testToxEnv: 'py39' + buildToxEnv: 'build-py39' testWheelInstallEnv: 'wheelinstall' - Windows-py3.6-min-req: + Windows-py3.7-min-req: imageName: 'vs2017-win2016' - pythonVersion: '3.6' - testToxEnv: 'py36-min-req' - buildToxEnv: 'build-py36-min-req' + pythonVersion: '3.7' + testToxEnv: 'py37-min-req' + buildToxEnv: 'build-py37-min-req' testWheelInstallEnv: 'wheelinstall' pool: vmImage: $(imageName) steps: - - checkout: self submodules: true diff --git a/docs/Makefile b/docs/Makefile index ece536eae..4cb0b9d41 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -45,7 +45,7 @@ help: @echo " apidoc to build RST from source code" clean: - -rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst + -rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst $(RSTDIR)/tutorials html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/gallery/advanced_io/README.txt b/docs/gallery/advanced_io/README.txt new file mode 100644 index 000000000..134df19b3 --- /dev/null +++ b/docs/gallery/advanced_io/README.txt @@ -0,0 +1,7 @@ + + +.. _general-tutorials: + + +Advanced I/O +------------ diff --git a/docs/gallery/general/advanced_hdf5_io.py b/docs/gallery/advanced_io/h5dataio.py similarity index 77% rename from docs/gallery/general/advanced_hdf5_io.py rename to docs/gallery/advanced_io/h5dataio.py index f2d72d406..f184e0719 100644 --- a/docs/gallery/general/advanced_hdf5_io.py +++ b/docs/gallery/advanced_io/h5dataio.py @@ -1,6 +1,6 @@ ''' -Advanced HDF5 I/O -===================== +Defining HDF5 Dataset I/O Settings (chunking, compression, etc.) +================================================================ The HDF5 storage backend supports a broad range of advanced dataset I/O options, such as, chunking and compression. Here we demonstrate how to use these features @@ -18,6 +18,7 @@ # Before we get started, lets create an NWBFile for testing so that we can add our data to it. # +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_h5dataio.png' from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile @@ -216,72 +217,6 @@ # will be ignored as the h5py.Dataset will either be linked to or copied as on write. # -#################### -# Parallel I/O using MPI -# ---------------------- -# -# The HDF5 storage backend supports parallel I/O using the Message Passing Interface (MPI). -# Using this feature requires that you install ``hdf5`` and ``h5py`` against an MPI driver, and you -# install ``mpi4py``. The basic installation of pynwb will not work. Setup can be tricky, and -# is outside the scope of this tutorial (for now), and the following assumes that you have -# HDF5 installed in a MPI configuration. Here we: -# -# 1. **Instantiate a dataset for parallel write**: We create TimeSeries with 4 timestamps that we -# will write in parallel -# -# 2. **Write to that file in parallel using MPI**: Here we assume 4 MPI ranks while each rank writes -# the data for a different timestamp. -# -# 3. **Read from the file in parallel using MPI**: Here each of the 4 MPI ranks reads one time -# step from the file -# -# .. code-block:: python -# -# from mpi4py import MPI -# import numpy as np -# from dateutil import tz -# from pynwb import NWBHDF5IO, NWBFile, TimeSeries -# from datetime import datetime -# from hdmf.data_utils import DataChunkIterator -# -# start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz('US/Pacific')) -# fname = 'test_parallel_pynwb.nwb' -# rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) -# -# # Create file on one rank. Here we only instantiate the dataset we want to -# # write in parallel but we do not write any data -# if rank == 0: -# nwbfile = NWBFile('aa', 'aa', start_time) -# data = DataChunkIterator(data=None, maxshape=(4,), dtype=np.dtype('int')) -# -# nwbfile.add_acquisition(TimeSeries('ts_name', description='desc', data=data, -# rate=100., unit='m')) -# with NWBHDF5IO(fname, 'w') as io: -# io.write(nwbfile) -# -# # write to dataset in parallel -# with NWBHDF5IO(fname, 'a', comm=MPI.COMM_WORLD) as io: -# nwbfile = io.read() -# print(rank) -# nwbfile.acquisition['ts_name'].data[rank] = rank -# -# # read from dataset in parallel -# with NWBHDF5IO(fname, 'r', comm=MPI.COMM_WORLD) as io: -# print(io.read().acquisition['ts_name'].data[rank]) - -#################### -# To specify details about chunking, compression and other HDF5-specific I/O options, -# we can wrap data via ``H5DataIO``, e.g, -# -# .. code-block:: python -# -# data = H5DataIO(DataChunkIterator(data=None, maxshape=(100000, 100), -# dtype=np.dtype('float')), -# chunks=(10, 10), maxshape=(None, None)) -# -# would initialize your dataset with a shape of (100000, 100) and maxshape of (None, None) -# and your own custom chunking of (10, 10). - #################### # Disclaimer # ---------------- diff --git a/docs/gallery/general/iterative_write.py b/docs/gallery/advanced_io/iterative_write.py similarity index 99% rename from docs/gallery/general/iterative_write.py rename to docs/gallery/advanced_io/iterative_write.py index e17a5e7b8..e9ea8a6c7 100644 --- a/docs/gallery/general/iterative_write.py +++ b/docs/gallery/advanced_io/iterative_write.py @@ -100,6 +100,7 @@ # simple helper function first to write a simple NWBFile containing a single timeseries to # avoid repetition of the same code and to allow us to focus on the important parts of this tutorial. +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_iterative_write.png' from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile, TimeSeries diff --git a/docs/gallery/general/linking_data.py b/docs/gallery/advanced_io/linking_data.py similarity index 99% rename from docs/gallery/general/linking_data.py rename to docs/gallery/advanced_io/linking_data.py index 7f9bcbc8f..6c2e005e8 100644 --- a/docs/gallery/general/linking_data.py +++ b/docs/gallery/advanced_io/linking_data.py @@ -56,7 +56,7 @@ # In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file. # We then show how we can integrate these files into a single NWBFile. - +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_linking_data.png' from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile diff --git a/docs/gallery/advanced_io/parallelio.py b/docs/gallery/advanced_io/parallelio.py new file mode 100644 index 000000000..a91e567d5 --- /dev/null +++ b/docs/gallery/advanced_io/parallelio.py @@ -0,0 +1,81 @@ +''' +Parallel I/O using MPI +====================== + +The HDF5 storage backend supports parallel I/O using the Message Passing Interface (MPI). +Using this feature requires that you install ``hdf5`` and ``h5py`` against an MPI driver, and you +install ``mpi4py``. The basic installation of pynwb will not work. Setup can be tricky, and +is outside the scope of this tutorial (for now), and the following assumes that you have +HDF5 installed in a MPI configuration. +''' + +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_parallelio.png' + +#################### +# Here we: +# +# 1. **Instantiate a dataset for parallel write**: We create TimeSeries with 4 timestamps that we +# will write in parallel +# +# 2. **Write to that file in parallel using MPI**: Here we assume 4 MPI ranks while each rank writes +# the data for a different timestamp. +# +# 3. **Read from the file in parallel using MPI**: Here each of the 4 MPI ranks reads one time +# step from the file +# +# .. code-block:: python +# +# from mpi4py import MPI +# import numpy as np +# from dateutil import tz +# from pynwb import NWBHDF5IO, NWBFile, TimeSeries +# from datetime import datetime +# from hdmf.data_utils import DataChunkIterator +# +# start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz('US/Pacific')) +# fname = 'test_parallel_pynwb.nwb' +# rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) +# +# # Create file on one rank. Here we only instantiate the dataset we want to +# # write in parallel but we do not write any data +# if rank == 0: +# nwbfile = NWBFile('aa', 'aa', start_time) +# data = DataChunkIterator(data=None, maxshape=(4,), dtype=np.dtype('int')) +# +# nwbfile.add_acquisition(TimeSeries('ts_name', description='desc', data=data, +# rate=100., unit='m')) +# with NWBHDF5IO(fname, 'w') as io: +# io.write(nwbfile) +# +# # write to dataset in parallel +# with NWBHDF5IO(fname, 'a', comm=MPI.COMM_WORLD) as io: +# nwbfile = io.read() +# print(rank) +# nwbfile.acquisition['ts_name'].data[rank] = rank +# +# # read from dataset in parallel +# with NWBHDF5IO(fname, 'r', comm=MPI.COMM_WORLD) as io: +# print(io.read().acquisition['ts_name'].data[rank]) + +#################### +# To specify details about chunking, compression and other HDF5-specific I/O options, +# we can wrap data via ``H5DataIO``, e.g, +# +# .. code-block:: python +# +# data = H5DataIO(DataChunkIterator(data=None, maxshape=(100000, 100), +# dtype=np.dtype('float')), +# chunks=(10, 10), maxshape=(None, None)) +# +# would initialize your dataset with a shape of (100000, 100) and maxshape of (None, None) +# and your own custom chunking of (10, 10). + +#################### +# Disclaimer +# ---------------- +# +# External links included in the tutorial are being provided as a convenience and for informational purposes only; +# they do not constitute an endorsement or an approval by the authors of any of the products, services or opinions of +# the corporation or organization or individual. The authors bear no responsibility for the accuracy, legality or +# content of the external site or for that of subsequent links. Contact the external site for answers to questions +# regarding its content. diff --git a/docs/gallery/domain/brain_observatory.py b/docs/gallery/domain/brain_observatory.py index 9be41de72..11e490251 100644 --- a/docs/gallery/domain/brain_observatory.py +++ b/docs/gallery/domain/brain_observatory.py @@ -16,6 +16,7 @@ # .. raw:: html # :url: https://gist.githubusercontent.com/nicain/82e6b3d8f9ff5b85ef01a582e41e2389/raw/ +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_allenbrainobservatory.png' from datetime import datetime from dateutil.tz import tzlocal diff --git a/docs/gallery/domain/ecephys.py b/docs/gallery/domain/ecephys.py index fd9bcf931..54f493ddf 100644 --- a/docs/gallery/domain/ecephys.py +++ b/docs/gallery/domain/ecephys.py @@ -5,13 +5,8 @@ Extracellular electrophysiology data ============================================ -The following examples will reference variables that may not be defined within the block they are used in. For -clarity, we define them here: ''' - -import numpy as np - ####################### # Creating and Writing NWB files # ------------------------------ @@ -19,6 +14,9 @@ # When creating a NWB file, the first step is to create the :py:class:`~pynwb.file.NWBFile`. The first # argument is the name of the NWB file, and the second argument is a brief description of the dataset. +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_ecephys.png' + +import numpy as np from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile diff --git a/docs/gallery/domain/icephys.py b/docs/gallery/domain/icephys.py index 9cad32b4a..e1c4907e5 100644 --- a/docs/gallery/domain/icephys.py +++ b/docs/gallery/domain/icephys.py @@ -2,11 +2,17 @@ ''' .. _icephys_tutorial: -Intracellular electrophysiology data -============================================ +Intracellular electrophysiology data using SweepTable +===================================================== -The following examples will reference variables that may not be defined within the block they are used in. For -clarity, we define them here: +The following tutorial describes storage of intracellular electrophysiology data in NWB using the +SweepTable to manage recordings. + +.. warning:: + The use of SweepTable has been deprecated as of PyNWB >v2.0 in favor of the new hierarchical + intracellular electrophysiology metadata tables to allow for a more complete description of + intracellular electrophysiology experiments. See the :doc:`Intracellular electrophysiology ` + tutorial for details. ''' ####################### @@ -16,6 +22,7 @@ # When creating a NWB file, the first step is to create the :py:class:`~pynwb.file.NWBFile`. The first # argument is is a brief description of the dataset. +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_icephys_sweeptable.png' from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile @@ -76,7 +83,7 @@ ccss = CurrentClampStimulusSeries( name="ccss", data=[1, 2, 3, 4, 5], starting_time=123.6, rate=10e3, electrode=elec, gain=0.02, sweep_number=0) -nwbfile.add_stimulus(ccss) +nwbfile.add_stimulus(ccss, use_sweep_table=True) ####################### # We now add another stimulus series but from a different sweep. TimeSeries @@ -87,7 +94,7 @@ vcss = VoltageClampStimulusSeries( name="vcss", data=[2, 3, 4, 5, 6], starting_time=234.5, rate=10e3, electrode=elec, gain=0.03, sweep_number=1) -nwbfile.add_stimulus(vcss) +nwbfile.add_stimulus(vcss, use_sweep_table=True) ####################### # Here, we will use :py:class:`~pynwb.icephys.CurrentClampSeries` to store current clamp @@ -102,7 +109,7 @@ electrode=elec, gain=0.02, bias_current=1e-12, bridge_balance=70e6, capacitance_compensation=1e-12, sweep_number=0) -nwbfile.add_acquisition(ccs) +nwbfile.add_acquisition(ccs, use_sweep_table=True) ####################### # And voltage clamp data from the second sweep using @@ -116,7 +123,7 @@ electrode=elec, gain=0.02, capacitance_slow=100e-12, resistance_comp_correction=70.0, sweep_number=1) -nwbfile.add_acquisition(vcs) +nwbfile.add_acquisition(vcs, use_sweep_table=True) #################### # .. _icephys_writing: @@ -183,7 +190,7 @@ # PatchClampSeries which belongs to a certain sweep number via # :py:meth:`~pynwb.icephys.SweepTable.get_series`. # -# The following call will return the voltage clamp data, two timeseries +# The following call will return the voltage clamp data of two timeseries # consisting of acquisition and stimulus, from sweep 1. series = nwbfile.sweep_table.get_series(1) diff --git a/docs/gallery/domain/ophys.py b/docs/gallery/domain/ophys.py index f59b20f92..623b52e51 100644 --- a/docs/gallery/domain/ophys.py +++ b/docs/gallery/domain/ophys.py @@ -17,6 +17,7 @@ clarity, we define them here: ''' +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_ophys.png' from datetime import datetime from dateutil.tz import tzlocal @@ -151,7 +152,13 @@ data = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] -rrs = fl.create_roi_response_series('my_rrs', data, rt_region, unit='lumens', timestamps=timestamps) +rrs = fl.create_roi_response_series( + name='my_rrs', + data=data, + rois=rt_region, + unit='lumens', + timestamps=timestamps +) #################### diff --git a/docs/gallery/domain/plot_icephys.py b/docs/gallery/domain/plot_icephys.py new file mode 100644 index 000000000..4e9c8f774 --- /dev/null +++ b/docs/gallery/domain/plot_icephys.py @@ -0,0 +1,703 @@ +# -*- coding: utf-8 -*- +''' +.. _icephys_tutorial_new: + +Intracellular electrophysiology +=============================== + +The following tutorial describes storage of intracellular electrophysiology data in NWB. +NWB supports storage of the time series describing the stimulus and response, information +about the electrode and device used, as well as metadata about the organization of the +experiment. + +.. note:: For a video tutorial on intracellular electrophysiology in NWB see also the + :incf_lesson:`Intracellular electrophysiology basics in NWB ` and + :incf_lesson:`Intracellular ephys metadata ` + tutorials as part of the :incf_collection:`NWB Course ` + at the INCF Training Space. + +.. figure:: ../../figures/plot_icephys_table_hierarchy.png + :width: 700 + :alt: Intracellular electrophysiology metadata table hierarchy + + Illustration of the hierarchy of metadata tables used to describe the organization of + intracellular electrophysiology experiments. + +Recordings of intracellular electrophysiology stimuli and responses are represented +with subclasses of :py:class:`~pynwb.icephys.PatchClampSeries` using the +:py:class:`~pynwb.icephys.IntracellularElectrode` and :py:class:`~pynwb.device.Device` +type to describe the electrode and device used. + +To describe the organization of intracellular experiments, the metadata is organized +hierarchically in a sequence of tables. All of the tables are so-called DynamicTables +enabling users to add columns for custom metadata. + +- :py:class:`~pynwb.icephys.IntracellularRecordingsTable` relates electrode, stimulus + and response pairs and describes metadata specific to individual recordings. +- :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` groups intracellular + recordings from the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` + together that were recorded simultaneously from different electrodes and/or cells + and describes metadata that is constant across the simultaneous recordings. + In practice a simultaneous recording is often also referred to as a sweep. +- :py:class:`~pynwb.icephys.SequentialRecordingsTable` groups simultaneously + recorded intracellular recordings from the + :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` together and describes + metadata that is constant across the simultaneous recordings. In practice a + sequential recording is often also referred to as a sweep sequence. A common + use of sequential recordings is to group together simultaneous recordings + where a sequence of stimuli of the same type with varying parameters + have been presented in a sequence (e.g., a sequence of square waveforms with + varying amplitude). +- :py:class:`~pynwb.icephys.RepetitionsTable` groups sequential recordings from + the :py:class:`~pynwb.icephys.SequentialRecordingsTable`. In practice a + repetition is often also referred to a run. A typical use of the + py:class:`~pynwb.icephys.RepetitionsTable` is to group sets of different stimuli + that are applied in sequence that may be repeated. +- :py:class:`~pynwb.icephys.ExperimentalConditionsTable` groups repetitions of + intracellular recording from the :py:class:`~pynwb.icephys.RepetitionsTable` + together that belong to the same experimental conditions. + +Storing data in hierarchical tables has the advantage that it allows us to avoid duplication of +metadata. E.g., for a single experiment we only need to describe the metadata that is constant +across an experimental condition as a single row in the :py:class:`~pynwb.icephys.ExperimentalConditionsTable` +without having to replicate the same information across all repetitions and sequential-, simultaneous-, and +individual intracellular recordings. For analysis, this means that we can easily focus on individual +aspects of an experiment while still being able to easily access information about information from +related tables. + +.. note:: All of the above mentioned metadata tables are optional and are created automatically + by the :py:class:`~pynwb.file.NWBFile` class the first time data is being + added to a table via the corresponding add functions. However, as tables at higher + levels of the hierarchy link to the other tables that are lower in the hierarchy, + we may only exclude tables from the top of the hierarchy. This means, for example, + a file containing a :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` then + must also always contain a corresponding :py:class:`~pynwb.icephys.IntracellularRecordingsTable`. +''' + +##################################################################### +# Imports used in the tutorial +# ------------------------------ + +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_icephys.png' +# Standard Python imports +from datetime import datetime +from dateutil.tz import tzlocal +import numpy as np +import pandas +# Set pandas rendering option to avoid very wide tables in the html docs +pandas.set_option("display.max_colwidth", 30) +pandas.set_option("display.max_rows", 10) + +# Import main NWB file class +from pynwb import NWBFile +# Import icephys TimeSeries types used +from pynwb.icephys import VoltageClampStimulusSeries, VoltageClampSeries +# Import I/O class used for reading and writing NWB files +from pynwb import NWBHDF5IO +# Import additional core datatypes used in the example +from pynwb.core import DynamicTable, VectorData + +##################################################################### +# A brief example +# --------------- +# +# The following brief example provides a quick overview of the main steps +# to create an NWBFile for intracelluar electrophysiology data. We then +# discuss the individual steps in more detail afterwards. +# +# .. note: +# To avoid collisions between this example script and the more +# detailed discussion we prefix all variables in the example +# script with ``ex_``. + +# Create an ICEphysFile +ex_nwbfile = NWBFile( + session_description='my first recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()) +) + +# Add a device +ex_device = ex_nwbfile.create_device(name='Heka ITC-1600') + +# Add an intracellular electrode +ex_electrode = ex_nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=ex_device +) + +# Create an ic-ephys stimulus +ex_stimulus = VoltageClampStimulusSeries( + name="stimulus", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=ex_electrode, + gain=0.02 +) + +# Create an ic-response +ex_response = VoltageClampSeries( + name='response', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=ex_electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0 +) + +# (A) Add an intracellular recording to the file +# NOTE: We can optionally define time-ranges for the stimulus/response via +# the corresponding optional _start_index and _index_count parameters. +# NOTE: It is allowed to add a recording with just a stimulus or a response +# NOTE: We can add custom columns to any of our tables in steps (A)-(E) +ex_ir_index = ex_nwbfile.add_intracellular_recording( + electrode=ex_electrode, + stimulus=ex_stimulus, + response=ex_response +) + +# (B) Add a list of sweeps to the simultaneous recordings table +ex_sweep_index = ex_nwbfile.add_icephys_simultaneous_recording(recordings=[ex_ir_index, ]) + +# (C) Add a list of simultaneous recordings table indices as a sequential recording +ex_sequence_index = ex_nwbfile.add_icephys_sequential_recording( + simultaneous_recordings=[ex_sweep_index, ], + stimulus_type='square' +) + +# (D) Add a list of sequential recordings table indices as a repetition +run_index = ex_nwbfile.add_icephys_repetition(sequential_recordings=[ex_sequence_index, ]) + +# (E) Add a list of repetition table indices as a experimental condition +ex_nwbfile.add_icephys_experimental_condition(repetitions=[run_index, ]) + +# Write our test file +ex_testpath = "ex_test_icephys_file.nwb" +with NWBHDF5IO(ex_testpath, 'w') as io: + io.write(ex_nwbfile) + +# Read the data back in +with NWBHDF5IO(ex_testpath, 'r') as io: + infile = io.read() + +# Optionally plot the organization of our example NWB file +try: + from hdmf_docutils.doctools.render import NXGraphHierarchyDescription, HierarchyDescription + import matplotlib.pyplot as plt + ex_file_hierarchy = HierarchyDescription.from_hdf5(ex_testpath) + ex_file_graph = NXGraphHierarchyDescription(ex_file_hierarchy) + ex_fig = ex_file_graph.draw(show_plot=False, figsize=(12, 16), label_offset=(0.0, 0.0065), label_font_size=10) + plt.show() +except ImportError: # ignore in case hdmf_docutils is not installed + pass + +##################################################################### +# Now that we have seen a brief example, we are going to start from the beginning and +# go through each of the steps in more detail in the following sections. + + +##################################################################### +# Creating an NWB file for Intracellular electrophysiology +# -------------------------------------------------------- +# +# When creating an NWB file, the first step is to create the :py:class:`~pynwb.file.NWBFile`. The first +# argument is a brief description of the dataset. + +# Create the file +nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + experimenter='Dr. Bilbo Baggins', + lab='Bag End Laboratory', + institution='University of Middle Earth at the Shire', + experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.', + session_id='LONELYMTN' +) + +##################################################################### +# Device metadata +# ^^^^^^^^^^^^^^^ +# +# Device metadata is represented by :py:class:`~pynwb.device.Device` objects. +# To create a device, you can use the :py:class:`~pynwb.file.NWBFile` instance method +# :py:meth:`~pynwb.file.NWBFile.create_device`. + +device = nwbfile.create_device(name='Heka ITC-1600') + +##################################################################### +# Electrode metadata +# ^^^^^^^^^^^^^^^^^^ +# +# Intracellular electrode metadata is represented by :py:class:`~pynwb.icephys.IntracellularElectrode` objects. +# To create an electrode group, you can use the :py:class:`~pynwb.file.NWBFile` instance method +# :py:meth:`~pynwb.file.NWBFile.create_icephys_electrode`. + +electrode = nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=device +) + +##################################################################### +# Stimulus and response data +# ^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Intracellular stimulus and response data are represented with subclasses of +# :py:class:`~pynwb.icephys.PatchClampSeries`. A stimulus is described by a +# time series representing voltage or current stimulation with a particular +# set of parameters. There are two classes for representing stimulus data: +# +# - :py:class:`~pynwb.icephys.VoltageClampStimulusSeries` +# - :py:class:`~pynwb.icephys.CurrentClampStimulusSeries` +# +# The response is then described by a time series representing voltage or current +# recorded from a single cell using a single intracellular electrode via one of +# the following classes: +# +# - :py:class:`~pynwb.icephys.VoltageClampSeries` +# - :py:class:`~pynwb.icephys.CurrentClampSeries` +# - :py:class:`~pynwb.icephys.IZeroClampSeries` +# +# Below we create a simple example stimulus/response recording data pair. + +# Create an example icephys stimulus. +stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=electrode, + gain=0.02, + sweep_number=np.uint64(15) +) + +# Create and icephys response +response = VoltageClampSeries( + name='vcs', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15) +) + +##################################################################### +# Adding an intracellular recording +# --------------------------------- +# +# As mentioned earlier, intracellular recordings are organized in the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable` which relates electrode, stimulus +# and response pairs and describes metadata specific to individual recordings. +# +# .. figure:: ../../figures/plot_icephys_intracellular_recordings_table.png +# :width: 700 +# :alt: IntracellularRecordingsTable +# +# Illustration of the structure of the IntracellularRecordingsTable +# +# We can add an intracellular recording to the file via :py:meth:`~pynwb.file.NWBFile.add_intracellular_recording`. +# The function will record the data in the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` +# and add the given electrode, stimulus, or response to the NWBFile object if necessary. +# Any time we add a row to one of our tables, the corresponding add function (here +# :py:meth:`~pynwb.file.NWBFile.add_intracellular_recording`) returns the integer index of the +# newly created row. The ``rowindex`` is used in subsequent tables that reference rows in our table. + +rowindex = nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + response=response, + id=10 +) + +##################################################################### +# .. note:: Since :py:meth:`~pynwb.file.NWBFile.add_intracellular_recording` can automatically add +# the objects to the NWBFile we do not need to separately call +# :py:meth:`~pynwb.file.NWBFile.add_stimulus` and :py:meth:`~pynwb.file.NWBFile.add_acquistion` +# to add our stimulus and response, but it is still fine to do so. +# +# .. note:: The ``id`` parameter in the call is optional and if the ``id`` is omitted then PyNWB will +# automatically number recordings in sequences (i.e., id is the same as the rowindex) +# +# .. note:: The IntracellularRecordigns, SimultaneousRecordings, SequentialRecordingsTable, +# RepetitionsTable and ExperimentalConditionsTable tables all enforce unique ids +# when adding rows. I.e., adding an intracellular recording with the same id twice +# results in a ValueError. +# + +##################################################################### +# .. note:: We may optionally also specify the relevant time range for a stimulus and/or response as part of +# the intracellular_recording. This is useful, e.g., in case where the recording of the stimulus +# and response do not align (e.g., in case the recording of the response started before +# the recording of the stimulus). + +rowindex2 = nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + stimulus_start_index=1, + stimulus_index_count=3, + response=response, + response_start_index=2, + response_index_count=3, + id=11 +) + +##################################################################### +# .. note:: A recording may optionally also consist of just an electrode and stimulus or electrode +# and response, but at least one of stimulus or response is required. If either stimulus or response +# is missing, then the stimulus and response are internally set to the same TimeSeries and the +# start_index and index_count for the missing parameter are set to -1. When retrieving +# data from the :py:class:`~pynwb.base.TimeSeriesReferenceVectorData`, the missing values +# will be represented via masked numpy arrays, i.e., as masked values in a +# ``numpy.ma.masked_array`` or as a ``np.ma.core.MaskedConstant``. + +rowindex3 = nwbfile.add_intracellular_recording( + electrode=electrode, + response=response, + id=12 +) + +##################################################################### +# .. warning:: For brevity we reused in the above example the same response and stimulus in +# all rows of the intracellular_recordings. While this is allowed, in most practical +# cases the stimulus and response will change between intracellular_recordings. + +##################################################################### +# Adding custom columns to the intracellular recordings table +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We can add a column to the main intracellular recordings table as follows. + +nwbfile.intracellular_recordings.add_column( + name='recording_tag', + data=['A1', 'A2', 'A3'], + description='String with a recording tag' +) + +##################################################################### +# The :py:class:`~pynwb.icephys.IntracellularRecordingsTable` table is not just a ``DynamicTable`` +# but an ``AlignedDynamicTable`. The ``AlignedDynamicTable` type is itself a ``DynamicTable`` +# that may contain an arbitrary number of additional ``DynamicTable``, each of which defines +# a "category". This is similar to a table with "sub-headings". In the case of the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable`, we have three predefined categories, +# i.e., electrodes, stimuli, and responses. We can also dynamically add new categories to +# the table. As each category corresponds to a DynamicTable, this means we have to create a +# new DynamicTable and add it to our table. + +# Create a new DynamicTable for our category that contains a location column of type VectorData +location_column = VectorData( + name='location', + data=['Mordor', 'Gondor', 'Rohan'], + description='Recording location in Middle Earth' +) + +lab_category = DynamicTable( + name='recording_lab_data', + description='category table for lab-specific recording metadata', + colnames=['location', ], + columns=[location_column, ] +) +# Add the table as a new category to our intracellular_recordings +nwbfile.intracellular_recordings.add_category(category=lab_category) +# Note, the name of the category is name of the table, i.e., 'recording_lab_data' + +##################################################################### +# .. note:: In an ``AlignedDynamicTable`` all category tables MUST align with the main table, +# i.e., all tables must have the same number of rows and rows are expected to +# correspond to each other by index + +##################################################################### +# We can also add custom columns to any of the subcategory tables, i.e., +# the electrodes, stimuli, and responses tables, and any custom subcategory tables. +# All we need to do is indicate the name of the category we want to add the column to. + +nwbfile.intracellular_recordings.add_column( + name='voltage_threshold', + data=[0.1, 0.12, 0.13], + description='Just an example column on the electrodes category table', + category='electrodes' +) + +##################################################################### +# Add a simultaneous recording +# --------------------------------- +# +# Before adding a simultaneous recording, we will take a brief discourse to illustrate +# how we can add custom columns to tables before and after we have populated the table with data +# +# Define a custom column for a simultaneous recording before populating the table +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Before we add a simultaneous recording, let's create a custom data column in our +# :py:class:`~pynwb.icephys.SimultaneousRecordingsTable`. We can create columns at the +# beginning (i.e., before we populate the table with rows/data) or we can add columns +# after we have already populated the table with rows. Here we will show the former. +# For this, we first need to get access to our table. + +print(nwbfile.icephys_simultaneous_recordings) + +##################################################################### +# The :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` is optional, and since we have +# not populated it with any data yet, we can see that the table does not actually exist yet. +# In order to make sure the table is being created we can use +# :py:meth:`~pynwb.file.NWBFile.get_icephys_simultaneous_recordings`, which ensures +# that the table is being created if it does not exist yet. + +icephys_simultaneous_recordings = nwbfile.get_icephys_simultaneous_recordings() +icephys_simultaneous_recordings.add_column( + name='simultaneous_recording_tag', + description='A custom tag for simultaneous_recordings' +) +print(icephys_simultaneous_recordings.colnames) + +##################################################################### +# As we can see, we now have succesfully created a new custom column. +# +# .. note:: The same process applies to all our other tables as well. We can use the +# corresponding :py:meth:`~pynwb.file.NWBFile.get_intracellular_recordings`, +# :py:meth:`~pynwb.file.NWBFile.get_icephys_sequential_recordings`, +# :py:meth:`~pynwb.file.NWBFile.get_icephys_repetitions`, and +# :py:meth:`~pynwb.file.NWBFile.get_icephys_conditions` functions instead. +# In general, we can always use the get functions instead of accessing the property +# of the file. +# + +##################################################################### +# Add a simultaneous recording +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Add a single simultaneous recording consisting of a set of intracellular recordings. +# Again, setting the id for a simultaneous recording is optional. The recordings +# argument of the :py:meth:`~pynwb.file.NWBFile.add_simultaneous_recording` function +# here is simply a list of ints with the indices of the corresponding rows in +# the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` +# +# .. note:: Since we created our custom ``simultaneous_recording_tag column`` earlier, +# we now also need to populate this custom field for every row we add to +# the :py:class:`~pynwb.icephys.SimultaneousRecordingsTable`. +# + +rowindex = nwbfile.add_icephys_simultaneous_recording( + recordings=[rowindex, rowindex2, rowindex3], + id=12, + simultaneous_recording_tag='LabTag1' +) + +##################################################################### +# .. note:: The ``recordings`` argument is the list of indices of the rows in the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable` that we want +# to reference. The indices are determined by the order in which we +# added the elements to the table. If we don't know the row indicies, +# but only the ids of the relevant intracellular recordings, then +# we can search for them as follows: + +temp_row_indices = (nwbfile.intracellular_recordings.id == [10, 11]) +print(temp_row_indices) + +##################################################################### +# .. note:: The same is true for our other tables as well, i.e., referencing is +# done always by indices of rows (NOT ids). If we only know ids then we +# can search for them in the same manner on the other tables as well, +# e.g,. via ``nwbfile.simultaneous_recordings.id == 15``. In the search +# we can use a list of integer ids or a single int. + +##################################################################### +# Define a custom column for a simultaneous recording after adding rows +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Depending on the lab workflow, it may be useful to add complete columns to a table +# after we have already populated the table with rows. We can do this the same way as +# before, only now we need to provide a data array to populate the values for +# the existing rows. E.g.: + +nwbfile.icephys_simultaneous_recordings.add_column( + name='simultaneous_recording_type', + description='Description of the type of simultaneous_recording', + data=['SimultaneousRecordingType1', ] +) + +##################################################################### +# Add a sequential recording +# -------------------------- +# +# Add a single sequential recording consisting of a set of simultaneous recordings. +# Again, setting the id for a sequential recording is optional. Also this table is +# optional and will be created automatically by NWBFile. The ``simultaneous_recordings`` +# argument of the :py:meth:`~pynwb.file.NWBFile.add_sequential_recording` function +# here is simply a list of ints with the indices of the corresponding rows in +# the :py:class:`~pynwb.icephys.SimultaneousRecordingsTable`. + +rowindex = nwbfile.add_icephys_sequential_recording( + simultaneous_recordings=[0], + stimulus_type='square', + id=15 +) + +##################################################################### +# Add a repetition +# ---------------- +# +# Add a single repetition consisting of a set of sequential recordings. Again, setting +# the id for a repetition is optional. Also this table is optional and will be created +# automatically by NWBFile. The ``sequential_recordings argument`` of the +# :py:meth:`~pynwb.file.NWBFile.add_sequential_recording` function here is simply +# a list of ints with the indices of the corresponding rows in +# the :py:class:`~pynwb.icephys.SequentialRecordingsTable`. + +rowindex = nwbfile.add_icephys_repetition(sequential_recordings=[0], id=17) + +##################################################################### +# Add an experimental condition +# ----------------------------- +# +# Add a single experimental condition consisting of a set of repetitions. Again, +# setting the id for a condition is optional. Also this table is optional and +# will be created automatically by NWBFile. The ``repetitions`` argument of +# the :py:meth:`~pynwb.file.NWBFile.add_icephys_condition` function again is +# simply a list of ints with the indices of the correspondingto rows in the +# :py:class:`~pynwb.icephys.RepetitionsTable`. + +rowindex = nwbfile.add_icephys_experimental_condition(repetitions=[0], id=19) + +##################################################################### +# As mentioned earlier, to add additonal columns to any of the tables, we can +# use the ``.add_column`` function on the corresponding table after they have +# been created. + +nwbfile.icephys_experimental_conditions.add_column( + name='tag', + data=np.arange(1), + description='integer tag for a experimental condition' +) + +##################################################################### +# When we add new items, then we now also need to set the values for the new column, e.g.: + +rowindex = nwbfile.add_icephys_experimental_condition( + repetitions=[0], + id=21, + tag=3 +) + +##################################################################### +# Read/write the NWBFile +# ----------------------------- +# + +# Write our test file +testpath = "test_icephys_file.nwb" +with NWBHDF5IO(testpath, 'w') as io: + io.write(nwbfile) + +# Read the data back in +with NWBHDF5IO(testpath, 'r') as io: + infile = io.read() + + +##################################################################### +# Accessing the tables +# ----------------------------- +# +# All of the icephys metadata tables are available as attributes on the NWBFile object. +# For display purposes, we convert the tables to pandas DataFrames to show their content. +# For a more in-depth discussion of how to access and use the tables, +# see the tutorial on :ref:`icephys_pandas_tutorial`. +pandas.set_option("display.max_columns", 6) # avoid oversize table in the html docs +nwbfile.intracellular_recordings.to_dataframe() + +##################################################################### +# + +# optionally we can ignore the id columns of the category subtables +pandas.set_option("display.max_columns", 5) # avoid oversize table in the html docs +nwbfile.intracellular_recordings.to_dataframe(ignore_category_ids=True) + +##################################################################### +# +nwbfile.icephys_simultaneous_recordings.to_dataframe() + +##################################################################### +# + +nwbfile.icephys_sequential_recordings.to_dataframe() + +##################################################################### +# + +nwbfile.icephys_repetitions.to_dataframe() + +##################################################################### +# + +nwbfile.icephys_experimental_conditions.to_dataframe() + + +##################################################################### +# Validate data +# ^^^^^^^^^^^^^ +# +# This section is for internal testing purposes only to validate that the roundtrip +# of the data (i.e., generate --> write --> read) produces the correct results. + +# Read the data back in +with NWBHDF5IO(testpath, 'r') as io: + infile = io.read() + + # assert intracellular_recordings + assert np.all(infile.intracellular_recordings.id[:] == nwbfile.intracellular_recordings.id[:]) + + # Assert that the ids and the VectorData, VectorIndex, and table target of the + # recordings column of the Sweeps table are correct + assert np.all(infile.icephys_simultaneous_recordings.id[:] == + nwbfile.icephys_simultaneous_recordings.id[:]) + assert np.all(infile.icephys_simultaneous_recordings['recordings'].target.data[:] == + nwbfile.icephys_simultaneous_recordings['recordings'].target.data[:]) + assert np.all(infile.icephys_simultaneous_recordings['recordings'].data[:] == + nwbfile.icephys_simultaneous_recordings['recordings'].data[:]) + assert (infile.icephys_simultaneous_recordings['recordings'].target.table.name == + nwbfile.icephys_simultaneous_recordings['recordings'].target.table.name) + + # Assert that the ids and the VectorData, VectorIndex, and table target of the simultaneous + # recordings column of the SweepSequences table are correct + assert np.all(infile.icephys_sequential_recordings.id[:] == + nwbfile.icephys_sequential_recordings.id[:]) + assert np.all(infile.icephys_sequential_recordings['simultaneous_recordings'].target.data[:] == + nwbfile.icephys_sequential_recordings['simultaneous_recordings'].target.data[:]) + assert np.all(infile.icephys_sequential_recordings['simultaneous_recordings'].data[:] == + nwbfile.icephys_sequential_recordings['simultaneous_recordings'].data[:]) + assert (infile.icephys_sequential_recordings['simultaneous_recordings'].target.table.name == + nwbfile.icephys_sequential_recordings['simultaneous_recordings'].target.table.name) + + # Assert that the ids and the VectorData, VectorIndex, and table target of the + # sequential_recordings column of the Repetitions table are correct + assert np.all(infile.icephys_repetitions.id[:] == nwbfile.icephys_repetitions.id[:]) + assert np.all(infile.icephys_repetitions['sequential_recordings'].target.data[:] == + nwbfile.icephys_repetitions['sequential_recordings'].target.data[:]) + assert np.all(infile.icephys_repetitions['sequential_recordings'] .data[:] == + nwbfile.icephys_repetitions['sequential_recordings'].data[:]) + assert (infile.icephys_repetitions['sequential_recordings'].target.table.name == + nwbfile.icephys_repetitions['sequential_recordings'].target.table.name) + + # Assert that the ids and the VectorData, VectorIndex, and table target of the + # repetitions column of the Conditions table are correct + assert np.all(infile.icephys_experimental_conditions.id[:] == + nwbfile.icephys_experimental_conditions.id[:]) + assert np.all(infile.icephys_experimental_conditions['repetitions'].target.data[:] == + nwbfile.icephys_experimental_conditions['repetitions'].target.data[:]) + assert np.all(infile.icephys_experimental_conditions['repetitions'] .data[:] == + nwbfile.icephys_experimental_conditions['repetitions'].data[:]) + assert (infile.icephys_experimental_conditions['repetitions'].target.table.name == + nwbfile.icephys_experimental_conditions['repetitions'].target.table.name) + assert np.all(infile.icephys_experimental_conditions['tag'][:] == + nwbfile.icephys_experimental_conditions['tag'][:]) diff --git a/docs/gallery/domain/plot_icephys_pandas.py b/docs/gallery/domain/plot_icephys_pandas.py new file mode 100644 index 000000000..39ee2ec0b --- /dev/null +++ b/docs/gallery/domain/plot_icephys_pandas.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +''' +.. _icephys_pandas_tutorial: + +Query intracellular electrophysiology metadata +============================================== + +This tutorial focuses on using pandas to query experiment metadata for +intracellular electrophysiology experiments using the metadata tables +from the :py:meth:`~pynwb.icephys` module. See the :ref:`icephys_tutorial_new` +tutorial for an introduction to the intracellular electrophysiology metadata +tables and how to create an NWBFile for intracellular electrophysiology data. +''' + +##################################################################### +# Imports used in the tutorial +# ------------------------------ + +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_icephys_pandas.png' +# Standard Python imports +import numpy as np +import pandas +# Set pandas rendering option to avoid very wide tables in the html docs +pandas.set_option("display.max_colwidth", 30) +pandas.set_option("display.max_rows", 10) +pandas.set_option("display.max_columns", 6) + +##################################################################### +# Example setup +# --------------- +# +# Generate a simple example NWBFile with dummy intracellular electrophysiology data. +# This example uses a utility function :py:meth:`~pynwb.testing.icephys_testutils.create_icephys_testfile` +# to create a dummy NWB file with random icephys data. + +from pynwb.testing.icephys_testutils import create_icephys_testfile +test_filename = "icephys_pandas_testfile.nwb" +nwbfile = create_icephys_testfile( + filename=test_filename, # Write the file to disk for testing + add_custom_columns=True, # Add a custom column to each metadata table + randomize_data=True, # Randomize the data in the simulus and response + with_missing_stimulus=True # Don't include the stimulus for row 0 and 10 +) + +##################################################################### +# Accessing the ICEphys metadata tables +# ------------------------------------- +# +# Get the parent metadata table +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# The intracellular electrophysiology metadata consists of a hierarchy of DynamicTables, i.e., +# :py:class:`~pynwb.icephys.ExperimentalConditionsTable` --> +# :py:class:`~pynwb.icephys.RepetitionsTable` --> +# :py:class:`~pynwb.icephys.SequentialRecordingsTable` --> +# :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` --> +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable`. +# However, in a given :py:class:`~pynwb.file.NWBFile`, not all tables may exist - a user may choose +# to exclude tables from the top of the hierarchy (e.g., a file may only contain +# :py:class:`~pynwb.icephys.SimultaneousRecordingsTable` and :py:class:`~pynwb.icephys.IntracellularRecordingsTable` +# while omitting all of the other tables that are higher in the hierarchy). +# To provide a consistent interface for users, PyNWB allows us to easily locate the table +# that defines the root of the table hierarchy via the function +# :py:meth:`~pynwb.file.NWBFile.get_icephys_meta_parent_table`. + +root_table = nwbfile.get_icephys_meta_parent_table() +print(root_table.neurodata_type) + +##################################################################### +# Getting a specific ICEphys metadata table +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We can retrieve any of the ICEphys metadata tables via the corresponding properties of NWBFile, i.e., +# :py:meth:`~pynwb.file.NWBFile.intracellular_recordings`, +# :py:meth:`~pynwb.file.NWBFile.icephys_simultaneous_recordings`, +# :py:meth:`~pynwb.file.NWBFile.icephys_sequential_recordings`, +# :py:meth:`~pynwb.file.NWBFile.icephys_repetitions`, +# :py:meth:`~pynwb.file.NWBFile.icephys_experimental_conditions`. +# The property will be ``None`` if the file does not contain the corresponding table. +# As such we can also easily check if a NWBFile contains a particular ICEphys metadata table via, e.g.: + +nwbfile.icephys_sequential_recordings is not None + +##################################################################### +# +# .. warning:: Always use the :py:class:`~pynwb.file.NWBFile` properties rather than the +# corresponding get methods if you only want to retrieve the ICEphys metadata tables. +# The get methods (e.g., :py:meth:`~pynwb.file.NWBFile.get_icephys_simultaneous_recordings`) +# are designed to always return a corresponding ICEphys metadata table for the file and will +# automatically add the missing table (and all required tables that are lower in the hierarchy) +# to the file. This behavior is to ease populating the ICEphys metadata tables when creating +# or updating an :py:class:`~pynwb.file.NWBFile`. +# + +##################################################################### +# Inspecting the table hierarchy +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# For any given table we can further check if and which columns are foreign +# :py:class:`~hdmf.common.table.DynamicTableRegion` columns pointing to other tables +# via the the :py:meth:`~hdmf.common.table.DynamicTable.has_foreign_columns` and +# :py:meth:`~hdmf.common.table.DynamicTable.get_foreign_columns`, respectively. +# + +print("Has Foreign Columns:", root_table.has_foreign_columns()) +print("Foreign Columns:", root_table.get_foreign_columns()) + +##################################################################### +# Using :py:meth:`~hdmf.common.table.DynamicTable.get_linked_tables` we can then also +# look at all links defined directly or indirectly from a given table to other tables. +# The result is a ``list`` of ``typing.NamedTuple`` objects containing, for each found link, the: +# +# * *"source_table"* :py:class:`~hdmf.common.table.DynamicTable` object, +# * *"source_column"* :py:class:`~hdmf.common.table.DynamicTableRegion` column from the source table, and +# * *"target_table"* :py:class:`~hdmf.common.table.DynamicTable` (which is the same as *source_column.table*). + +linked_tables = root_table.get_linked_tables() + +# Print the links +for i, link in enumerate(linked_tables): + print("%s (%s, %s) ----> %s" % (" " * i, + link.source_table.name, + link.source_column.name, + link.target_table.name)) + +##################################################################### +# Converting ICEphys metadata tables to pandas DataFrames +# ------------------------------------------------------- +# + +##################################################################### +# Using nested DataFrames +# ^^^^^^^^^^^^^^^^^^^^^^^ +# Using the :py:meth:`~hdmf.common.table.DynamicTable.to_dataframe` method we can easily convert tables +# to pandas `DataFrames `_. + +exp_cond_df = root_table.to_dataframe() +exp_cond_df + +##################################################################### +# By default, the method will resolve :py:class:`~hdmf.common.table.DynamicTableRegion` +# references and include the rows that are referenced in related tables as +# `DataFrame `_ objects, +# resulting in a hierarchically nested `DataFrame`_. For example, looking at a single cell of the +# ``repetitions`` column of our :py:class:`~pynwb.icephys.ExperimentalConditionsTable` table, +# we get the corresponding subset of repetitions from the py:class:`~pynwb.icephys.RepetitionsTable`. + +exp_cond_df.iloc[0]['repetitions'] + +##################################################################### +# In contrast to the other ICEphys metadata tables, the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable` does not contain any +# :py:class:`~hdmf.common.table.DynamicTableRegion` columns, but it is a +# :py:class:`~hdmf.common.alignedtable.AlignedDynamicTable` which contains sub-tables for +# ``electrodes``, ``stimuli``, and ``responses``. For convenience, the +# :py:meth:`~pynwb.icephys.IntracellularRecordingsTable.to_dataframe` of the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable` provides a few +# additonal optional parameters to optionally ignore the ids of the category tables +# (via ``ignore_category_ids=True``) or to convert the electrode, stimulus, and +# response references to ObjectIds. For example: +# + +nwbfile.intracellular_recordings.to_dataframe( + ignore_category_ids=True, + electrode_refs_as_objectids=True, + stimulus_refs_as_objectids=True, + response_refs_as_objectids=True +) + +##################################################################### +# Using indexed DataFrames +# ^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Depending on the particular analysis, we may be interested only in a particular table and do not +# want to recursively load and resolve all the linked tables. By setting ``index=True`` when +# converting the table :py:meth:`~hdmf.common.table.DynamicTable.to_dataframe` the +# :py:class:`~hdmf.common.table.DynamicTableRegion` links will be represented as +# lists of integers indicating the rows in the target table (without loading data from +# the referenced table). + +root_table.to_dataframe(index=True) + +##################################################################### +# To resolve links related to a set of rows, we can then simply use the corresponding +# :py:class:`~hdmf.common.table.DynamicTableRegion` column from our original table, e.g.: + +root_table['repetitions'][0] # Look-up the repetitions for the first experimental condition + +##################################################################### +# We can also naturally resolve links ourselves by looking up the relevant table and +# then accessing elements of the table directly. + +# All DynamicTableRegion columns in the ICEphys table are indexed so we first need to +# follow the ".target" to the VectorData and then look up the table via ".table" +target_table = root_table['repetitions'].target.table +target_table[[0, 1]] + +##################################################################### +# .. note:: We can also explicitly exclude the :py:class:`~hdmf.common.table.DynamicTableRegion` columns +# (or any other column) from the `DataFrame`_ using e.g., ``root_table.to_dataframe(exclude={'repetitions', })``. + +##################################################################### +# Using a single, hierarchical DataFrame +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# To gain a more direct overview of all metadata at once and avoid iterating across levels of nested +# DataFrames during analysis, it can be useful to flatten (or unnest) nested DataFrames, expanding the +# nested DataFrames by adding their columns to the main table, and expanding the corresponding rows in +# the parent table by duplicating the data from the existing columns across the new rows. +# For example, an experimental condition represented by a single row in the +# :py:class:`~pynwb.icephys.ExperimentalConditionsTable` containing 5 repetitions would be expanded +# to 5 rows, each containing a copy of the metadata from the experimental condition along with the +# metadata of one of the repetitions. Repeating this process recursively, a single row in the +# :py:class:`~pynwb.icephys.ExperimentalConditionsTable` will then ultimately expand to the total +# number of intracellular recordings from the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` +# that belong to the experimental conditions table. +# +# HDMF povides several convenience functions to help with this process. Using the +# :py:func:`~hdmf.common.hierarchicaltable.to_hierarchical_dataframe` method, we can transform +# our hierarchical table into a single pandas `DataFrame`_. +# To avoid duplication of data in the display, the hierarchy is represented as a pandas +# `MultiIndex `_ on +# the rows so that only the data from the last table in our hierarchy (i.e. here the +# :py:class:`~pynwb.icephys.IntracellularRecordingsTable`) is represented as columns. + +from hdmf.common.hierarchicaltable import to_hierarchical_dataframe +icephys_meta_df = to_hierarchical_dataframe(root_table) + +##################################################################### +# + +# To avoid a too wide display in the online docs we here only show 4 select rows of the +# table and transpose the table to show the large MultiIndex as columns instead of rows +icephys_meta_df.iloc[[0, 1, 18, 19]].transpose() + +##################################################################### +# Depending on the analysis, it can be useful to further process our `DataFrame`_. Using the standard +# `reset_index `_ +# function, we can turn the data from the `MultiIndex`_ to columns of the table itself, +# effectively denormalizing the display by repeating all data across rows. HDMF then also +# provides: 1) :py:func:`~hdmf.common.hierarchicaltable.drop_id_columns` to remove all "id" columns +# and 2) :py:func:`~hdmf.common.hierarchicaltable.flatten_column_index` to turn the +# `MultiIndex`_ on the columns of the table into a regular +# `Index `_ of tuples. +# +# .. note:: Dropping ``id`` columns is often useful for visualization purposes while for +# query and analysis it is often useful to maintain the ``id`` columns to facilitate +# lookups and correlation of information. + +from hdmf.common.hierarchicaltable import drop_id_columns, flatten_column_index +# Reset the index of the dataframe and turn the values into columns instead +icephys_meta_df.reset_index(inplace=True) +# Flatten the column-index, turning the pandas.MultiIndex into a pandas.Index of tuples +flatten_column_index(dataframe=icephys_meta_df, max_levels=2, inplace=True) +# Remove the id columns. By setting inplace=False allows us to visualize the result of this +# action while keeping the id columns in our main icephys_meta_df table +drop_id_columns(dataframe=icephys_meta_df, inplace=False) + +##################################################################### +# Useful additional data preparations +# ----------------------------------- +# +# Expanding TimeSeriesReference columns +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# For query purposes it can be useful to expand the stimulus and response columns to separate the +# ``(start, count, timeseries)`` values in separate columns. This is primarily useful if we want to +# perform queries on these components directly, otherwise it is usually best to keep the stimulus/response +# references around as `:py:class:`~pynwb.base.TimeSeriesReference`, which provides additional features +# to inspect and validate the references and load data. We, therefore, here keep the data in both forms +# in the table + +# Expand the ('stimuli', 'stimulus') to a DataFrame with 3 columns +stimulus_df = pandas.DataFrame( + icephys_meta_df[('stimuli', 'stimulus')].tolist(), + columns=[('stimuli', 'idx_start'), ('stimuli', 'count'), ('stimuli', 'timeseries')], + index=icephys_meta_df.index +) +# If we want to remove the original ('stimuli', 'stimulus') from the dataframe we can call +# icephys_meta_df.drop(labels=[('stimuli', 'stimulus'), ], axis=1, inplace=True) +# Add our expanded columns to the icephys_meta_df dataframe +icephys_meta_df = pandas.concat([icephys_meta_df, stimulus_df], axis=1) +# display the table for the HTML docs +icephys_meta_df + +##################################################################### +# We can then easily expand also the ``(responses, response)`` column in the same way + +response_df = pandas.DataFrame( + icephys_meta_df[('responses', 'response')].tolist(), + columns=[('responses', 'idx_start'), ('responses', 'count'), ('responses', 'timeseries')], + index=icephys_meta_df.index +) +icephys_meta_df = pandas.concat([icephys_meta_df, response_df], axis=1) + + +##################################################################### +# Adding Stimulus/Response Metadata +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# With all TimeSeries stimuli and responses listed in the table, we can easily iterate over the +# TimeSeries to expand our table with additional columns with information from the TimeSeries, e.g., +# the ``neurodata_type`` or ``name`` or any other properties we may wish to extract from our +# stimulus and response TimeSeries (e.g., ``rate``, ``starting_time``, ``gain`` etc.). +# Here we show a few examples. + +# Add a column with the name of the stimulus TimeSeries object. +# Note: We use getattr here to easily deal with missing values, i.e., cases where no stimulus is present +col = ('stimuli', 'name') +icephys_meta_df[col] = [getattr(s, 'name', None) + for s in icephys_meta_df[('stimuli', 'timeseries')]] + +# Often we can easily do this in bulk-fashion by specifing the collection of fields of interest +for field in ['neurodata_type', 'gain', 'rate', 'starting_time', 'object_id']: + col = ('stimuli', field) + icephys_meta_df[col] = [getattr(s, field, None) + for s in icephys_meta_df[('stimuli', 'timeseries')]] +icephys_meta_df + +##################################################################### +# Naturally we can again do the same also for our response columns +for field in ['name', 'neurodata_type', 'gain', 'rate', 'starting_time', 'object_id']: + col = ('responses', field) + icephys_meta_df[col] = [getattr(s, field, None) + for s in icephys_meta_df[('responses', 'timeseries')]] + +##################################################################### +# And we can use the same process to also gather additional metadata about the +# :py:class:`~pynwb.icephys.IntracellularElectrode`, :py:class:`~pynwb.device.Device` and others +for field in ['name', 'device', 'object_id']: + col = ('electrodes', field) + icephys_meta_df[col] = [getattr(s, field, None) + for s in icephys_meta_df[('electrodes', 'electrode')]] + +##################################################################### +# This basic approach allows us to easily collect all data needed for query in a convenient +# spreadsheet for display, query, and analysis. + +##################################################################### +# Performing common metadata queries +# ---------------------------------- +# +# With regard to the experiment metadata tables, many of the queries we identified based on +# feedback from the community follow the model of: *"Given X return Y"*, e.g.: +# +# * Given a particular stimulus return: +# * the corresponding response +# * the corresponding electrode +# * the stimulus type +# * all stimuli/responses recorded at the same time (i.e., during the same simultaneous recording) +# * all stimuli/responses recorded during the same sequential recording +# * Given a particular response return: +# * the corresponding stimulus +# * the corresponding electrode +# * all stimuli/responses recorded at the same time (i.e., during the same simultaneous recording) +# * all stimuli/responses recorded during the same sequential recording +# * Given an electrode return: +# * all responses (and stimuli) related to the electrode +# * all sequential recordings (a.k.a., sweeps) recorded with the electrode +# * Given a stimulus type return: +# * all related stimulus/response recordings +# * all the repetitions in which it is present +# * Given a stimulus type and a repetition return: +# * all the responses +# * Given a simultaneous recording (a.k.a., sweep) return: +# * the repetition/condition/sequential recording it belongs to +# * all other simultaneous recordings that are part of the same repetition +# * the experimental condition the simultaneous recording is part of +# * Given a repetition return: +# * the experimental condition the simultaneous recording is part of +# * all sequential- and/or simultaneous recordings within that repetition +# * Given an experimental condition return: +# * All corresponding repetitions or sequential/simultaneous/intracellular recordings +# * Get the list of all stimulus types +# +# More complex analytics will then commonly combine multiple such query constraints to further process +# the corresponding data, e.g., +# +# * Given a stimulus and a condition, return all simultaneous recordings (a.k.a., sweeps) across repetitions +# and average the responses +# +# Generally, many of the queries involve looking up a piece of information in on table (e.g., finding +# a stimulus type in :py:class:`~pynwb.icephys.SequentialRecordingsTable`) and then querying for +# related information in child tables (by following the :py:class:`~hdmf.common.table.DynamicTableRegion` links +# included in the corresponding rows) to look up more specific information (e.g., all recordings related to +# the stimulus type) or alternatively querying for related information in parent tables (by finding rows in the +# parent table that link to our rows) and then looking up more general information (e.g., information about the +# experimental condition). Using this approach, we can resolve the above queries using the individual +# :py:class:`~hdmf.common.table.DynamicTable` objects directly, while loading only the data that is +# absolutely necessary into memory. +# +# With the bulk data stored usually in some form of :py:class:`~pynwb.icephys.PatchClampSeries`, the +# ICEphys metadata tables will usually be comparatively small (in terms of total memory). Once we have created +# our integrated `DataFrame`_ as shown above, performing the queries described above becomes quite simple +# as all links between tables have already been resolved and all data has been expanded across all rows. +# In general, resolving queries on our "denormalized" table amounts to evaluating one or more conditions +# on one or more columns and then retrieving the rows that match our conditions form the table. +# +# Once we have all metadata in a single table, we can also easily sort the rows of our table based on +# a flexible set of conditions or even cluster rows to compute more advanced groupings of intracellular recordings. +# +# Below we show just a few simple examples: +# +# Given a response, get the stimulus +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +# Get a response 'vcs_9' from the file +response = nwbfile.get_acquisition('vcs_9') +# Return all data related to that response, including the stimulus as part of ('stimuli', 'stimulus') column +icephys_meta_df[icephys_meta_df[('responses', 'object_id')] == response.object_id] + + +##################################################################### +# Given a response load the associated data +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# References to timeseries are stored in the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` via +# :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` columns which return the references to the stimulus/response +# via :py:class:`~pynwb.base.TimeSeriesReference` objects. Using :py:class:`~pynwb.base.TimeSeriesReference` we can +# easily inspect the selected data. + +ref = icephys_meta_df[('responses', 'response')][0] # Get the TimeSeriesReference +_ = ref.isvalid() # Is the reference valid +_ = ref.idx_start # Get the start index +_ = ref.count # Get the count +_ = ref.timeseries.name # Get the timeseries +_ = ref.timestamps # Get the selected timestamps +ref_data = ref.data # Get the selected recorded response data values +# Print the data values just as an example +print("data = " + str(ref_data)) + +##################################################################### +# Get a list of all stimulus types +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +unique_stimulus_types = np.unique(icephys_meta_df[('sequential_recordings', 'stimulus_type')]) +print(unique_stimulus_types) + +##################################################################### +# Given a stimulus type, get all corresponding intracellular recordings +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +icephys_meta_df[icephys_meta_df[('sequential_recordings', 'stimulus_type')] == 'StimType_1'] diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index b03f3dd4f..33c23cf53 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -22,6 +22,7 @@ # # For example: +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_add_remove_containers.png' from pynwb import NWBFile, NWBHDF5IO, TimeSeries import datetime import numpy as np diff --git a/docs/gallery/general/extensions.py b/docs/gallery/general/extensions.py index c87d291c0..027d07644 100644 --- a/docs/gallery/general/extensions.py +++ b/docs/gallery/general/extensions.py @@ -30,6 +30,7 @@ # to this namespace. Finally, # it calls :py:meth:`~hdmf.spec.write.NamespaceBuilder.export` to save the extensions to disk for downstream use. +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_extensions.png' from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec ns_path = "mylab.namespace.yaml" diff --git a/docs/gallery/general/file.py b/docs/gallery/general/file.py index 982f54b3a..615898c0e 100644 --- a/docs/gallery/general/file.py +++ b/docs/gallery/general/file.py @@ -13,8 +13,8 @@ # The NWB file # ------------ # -# +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_file.png' from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile diff --git a/docs/gallery/general/object_id.py b/docs/gallery/general/object_id.py index f6b49a9dc..1c7c332b6 100644 --- a/docs/gallery/general/object_id.py +++ b/docs/gallery/general/object_id.py @@ -16,6 +16,7 @@ """ +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_objectid.png' from pynwb import NWBFile, TimeSeries from datetime import datetime from dateutil.tz import tzlocal diff --git a/docs/gallery/general/scratch.py b/docs/gallery/general/scratch.py index 2627a3cfe..306dbc65f 100644 --- a/docs/gallery/general/scratch.py +++ b/docs/gallery/general/scratch.py @@ -27,6 +27,7 @@ # To demonstrate linking and scratch space, lets assume we are starting with some acquired data. # +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_scratch.png' from pynwb import NWBFile, TimeSeries, NWBHDF5IO from datetime import datetime from dateutil.tz import tzlocal diff --git a/docs/make.bat b/docs/make.bat index b50d10f3a..1e2a19ff4 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -50,6 +50,7 @@ if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* del /q %RSTDIR%\%PKGNAME%*.rst + rmdir /q /s %RSTDIR%\tutorials goto end ) diff --git a/docs/source/conf.py b/docs/source/conf.py index 63923cff0..70c2afb4c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,6 +53,7 @@ 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', + 'sphinx.ext.extlinks', 'sphinx_gallery.gen_gallery' ] @@ -63,20 +64,26 @@ 'examples_dirs': ['../gallery'], # path where to save gallery generated examples 'gallery_dirs': ['tutorials'], - 'subsection_order': ExplicitOrder(['../gallery/general', '../gallery/domain']), + 'subsection_order': ExplicitOrder(['../gallery/general', '../gallery/domain', '../gallery/advanced_io']), 'backreferences_dir': 'gen_modules/backreferences', - 'min_reported_time': 5 + 'min_reported_time': 5, + 'remove_config_comments': True } intersphinx_mapping = { 'python': ('https://docs.python.org/3.8', None), - 'numpy': ('https://numpy.org/doc/stable/objects.inv', None), + 'numpy': ('https://numpy.org/doc/stable/', None), 'matplotlib': ('https://matplotlib.org', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), 'hdmf': ('https://hdmf.readthedocs.io/en/latest/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None) } +extlinks = {'incf_lesson': ('https://training.incf.org/lesson/%s', ''), + 'incf_collection': ('https://training.incf.org/collection/%s', ''), + 'nwb_extension': ('https://github.com/nwb-extensions/%s', ''), + 'pynwb': ('https://github.com/NeurodataWithoutBorders/pynwb/%s', '')} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/extensions_tutorial/2_create_extension_spec_walkthrough.rst b/docs/source/extensions_tutorial/2_create_extension_spec_walkthrough.rst index 9da0ee022..ad6fcf48e 100644 --- a/docs/source/extensions_tutorial/2_create_extension_spec_walkthrough.rst +++ b/docs/source/extensions_tutorial/2_create_extension_spec_walkthrough.rst @@ -5,10 +5,10 @@ Using ndx-template ~~~~~~~~~~~~~~~~~~ Extensions should be created in their own repository, not alongside data conversion code. This facilitates sharing and editing of the extension separately from the code that uses it. When starting a new extension, we highly -recommend using the `ndx-template `_ repository, which automatically -generates a repository with the appropriate directory structure. +recommend using the :nwb_extension:`ndx-template` repository, which automatically generates a repository with +the appropriate directory structure. -After you finish the instructions `here `_, +After you finish the instructions :nwb_extension:`here `, you should have a directory structure that looks like this .. code-block:: bash @@ -60,7 +60,7 @@ modifying this script to create your own NWB extension. Let's first walk through Creating a namespace ~~~~~~~~~~~~~~~~~~~~ NWB organizes types into namespaces. You must define a new namespace before creating any new types. After following -the instructions from `ndx-template `_, you should have a file +the instructions from the :nwb_extension:`ndx-template`, you should have a file ``ndx-my-ext/src/spec/create_extension_spec.py``. The beginning of this file should look like .. code-block:: python diff --git a/docs/source/figures/gallery_thumbnails.pptx b/docs/source/figures/gallery_thumbnails.pptx new file mode 100644 index 000000000..e16e5e0eb Binary files /dev/null and b/docs/source/figures/gallery_thumbnails.pptx differ diff --git a/docs/source/figures/gallery_thumbnails_add_remove_containers.png b/docs/source/figures/gallery_thumbnails_add_remove_containers.png new file mode 100644 index 000000000..fbe89b86e Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_add_remove_containers.png differ diff --git a/docs/source/figures/gallery_thumbnails_allenbrainobservatory.png b/docs/source/figures/gallery_thumbnails_allenbrainobservatory.png new file mode 100644 index 000000000..065aa14ed Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_allenbrainobservatory.png differ diff --git a/docs/source/figures/gallery_thumbnails_ecephys.png b/docs/source/figures/gallery_thumbnails_ecephys.png new file mode 100644 index 000000000..428a5c87e Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_ecephys.png differ diff --git a/docs/source/figures/gallery_thumbnails_extensions.png b/docs/source/figures/gallery_thumbnails_extensions.png new file mode 100644 index 000000000..4cca2a976 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_extensions.png differ diff --git a/docs/source/figures/gallery_thumbnails_file.png b/docs/source/figures/gallery_thumbnails_file.png new file mode 100644 index 000000000..56e3a826b Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_file.png differ diff --git a/docs/source/figures/gallery_thumbnails_h5dataio.png b/docs/source/figures/gallery_thumbnails_h5dataio.png new file mode 100644 index 000000000..034017799 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_h5dataio.png differ diff --git a/docs/source/figures/gallery_thumbnails_icephys.png b/docs/source/figures/gallery_thumbnails_icephys.png new file mode 100644 index 000000000..c70ab58c0 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_icephys.png differ diff --git a/docs/source/figures/gallery_thumbnails_icephys_pandas.png b/docs/source/figures/gallery_thumbnails_icephys_pandas.png new file mode 100644 index 000000000..c1e90d771 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_icephys_pandas.png differ diff --git a/docs/source/figures/gallery_thumbnails_icephys_sweeptable.png b/docs/source/figures/gallery_thumbnails_icephys_sweeptable.png new file mode 100644 index 000000000..c0d575c14 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_icephys_sweeptable.png differ diff --git a/docs/source/figures/gallery_thumbnails_iterative_write.png b/docs/source/figures/gallery_thumbnails_iterative_write.png new file mode 100644 index 000000000..a3a7a4a34 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_iterative_write.png differ diff --git a/docs/source/figures/gallery_thumbnails_linking_data.png b/docs/source/figures/gallery_thumbnails_linking_data.png new file mode 100644 index 000000000..dec6291fa Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_linking_data.png differ diff --git a/docs/source/figures/gallery_thumbnails_objectid.png b/docs/source/figures/gallery_thumbnails_objectid.png new file mode 100644 index 000000000..7d98a857c Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_objectid.png differ diff --git a/docs/source/figures/gallery_thumbnails_ophys.png b/docs/source/figures/gallery_thumbnails_ophys.png new file mode 100644 index 000000000..27ca6e477 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_ophys.png differ diff --git a/docs/source/figures/gallery_thumbnails_parallelio.png b/docs/source/figures/gallery_thumbnails_parallelio.png new file mode 100644 index 000000000..d620f939b Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_parallelio.png differ diff --git a/docs/source/figures/gallery_thumbnails_scratch.png b/docs/source/figures/gallery_thumbnails_scratch.png new file mode 100644 index 000000000..372304bd6 Binary files /dev/null and b/docs/source/figures/gallery_thumbnails_scratch.png differ diff --git a/docs/source/figures/plot_icephys_figures.pptx b/docs/source/figures/plot_icephys_figures.pptx new file mode 100644 index 000000000..d0340b10a Binary files /dev/null and b/docs/source/figures/plot_icephys_figures.pptx differ diff --git a/docs/source/figures/plot_icephys_intracellular_recordings_table.png b/docs/source/figures/plot_icephys_intracellular_recordings_table.png new file mode 100644 index 000000000..b3193e29b Binary files /dev/null and b/docs/source/figures/plot_icephys_intracellular_recordings_table.png differ diff --git a/docs/source/figures/plot_icephys_table_hierarchy.png b/docs/source/figures/plot_icephys_table_hierarchy.png new file mode 100644 index 000000000..f1360c03e Binary files /dev/null and b/docs/source/figures/plot_icephys_table_hierarchy.png differ diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index beb2b7542..3acbb94d1 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -6,7 +6,7 @@ Dependencies PyNWB has the following minimum requirements, which must be installed before you can get started using PyNWB. -#. Python 3.5, 3.6, or 3.7 +#. Python 3.7, 3.8, or 3.9 #. pip ------------ diff --git a/docs/source/make_a_release.rst b/docs/source/make_a_release.rst index 955a261ca..18d4fe075 100644 --- a/docs/source/make_a_release.rst +++ b/docs/source/make_a_release.rst @@ -122,7 +122,7 @@ PyPI: Step-by-step 8. Once the builds are completed, check that the distributions are available on `PyPI`_ and that - a new `GitHub release `_ was created. + a new :pynwb:`GitHub release ` was created. 9. Create a clean testing environment to test the installation diff --git a/requirements-dev.txt b/requirements-dev.txt index 563995cf9..27b099d30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,8 +2,8 @@ # compute coverage, and create test environments codecov==2.1.11 coverage==5.5 -flake8==3.9.1 +flake8==3.9.2 flake8-debugger==4.0.0 flake8-print==4.0.0 -importlib-metadata==4.0.1 -tox==3.23.0 +importlib-metadata==4.6.1 +tox==3.23.1 diff --git a/requirements-doc.txt b/requirements-doc.txt index 5f7ceff81..7ae6261cb 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -3,4 +3,4 @@ sphinx matplotlib sphinx_rtd_theme sphinx-gallery -allensdk # note that as of allensdk 2.10.0, python 3.8 is not supported +allensdk>=2.11.0 # python 3.8 is not supported in allensdk<2.11 diff --git a/requirements-min.txt b/requirements-min.txt index abf6943ef..f7adc6641 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB -h5py==2.9 # support for setting attrs to lists of utf-8 added in 2.9 -hdmf==2.5.6 +h5py==2.10 # support for selection of datasets with list of indices added in 2.10 +hdmf==3.1.1 numpy==1.16 pandas==1.0.5 python-dateutil==2.7 diff --git a/requirements.txt b/requirements.txt index f4a416850..31b8d102e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB -h5py==2.10.0 -hdmf==2.5.6 -numpy==1.19.3 -pandas==1.1.5 +h5py==3.3.0 +hdmf==3.1.1 +numpy==1.21.0 +pandas==1.3.0 python-dateutil==2.8.1 -setuptools==56.0.0 +setuptools==57.1.0 diff --git a/setup.py b/setup.py index 3047a2367..f6a05bee6 100755 --- a/setup.py +++ b/setup.py @@ -13,9 +13,9 @@ schema_dir = 'nwb-schema/core' reqs = [ - 'h5py>=2.9,<3', - 'hdmf>=2.5.6,<3', - 'numpy>=1.16,<1.21', + 'h5py>=2.9,<4', + 'hdmf>=3.1.1,<4', + 'numpy>=1.16,<1.22', 'pandas>=1.0.5,<2', 'python-dateutil>=2.7,<3', 'setuptools' @@ -40,9 +40,9 @@ 'package_data': {'pynwb': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]}, 'classifiers': [ "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "License :: OSI Approved :: BSD License", "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", diff --git a/src/pynwb/base.py b/src/pynwb/base.py index bc9fcd9df..9924bc7bf 100644 --- a/src/pynwb/base.py +++ b/src/pynwb/base.py @@ -1,9 +1,10 @@ from warnings import warn from collections.abc import Iterable +import numpy as np +from typing import NamedTuple -from hdmf.utils import docval, getargs, popargs, call_docval_func -from hdmf.common import DynamicTable - +from hdmf.utils import docval, getargs, popargs, call_docval_func, get_docval +from hdmf.common import DynamicTable, VectorData from . import register_class, CORE_NAMESPACE from .core import NWBDataInterface, MultiContainerInterface, NWBData @@ -97,12 +98,15 @@ class TimeSeries(NWBDataInterface): __time_unit = "seconds" + # values used when a TimeSeries is read and missing required fields + DEFAULT_DATA = np.ndarray(shape=(0, ), dtype=np.uint8) + DEFAULT_UNIT = 'unknown' + @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'}, # required {'name': 'data', 'type': ('array_data', 'data', 'TimeSeries'), 'doc': ('The data values. The first dimension must be time. ' - 'Can also store binary data, e.g., image frames'), - 'default': None}, - {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)', 'default': None}, + 'Can also store binary data, e.g., image frames')}, + {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)'}, {'name': 'resolution', 'type': (str, 'float'), 'doc': 'The smallest meaningful difference (in specified unit) between values in data', 'default': -1.0}, {'name': 'conversion', 'type': (str, 'float'), @@ -282,3 +286,227 @@ def __init__(self, **kwargs): super(Images, self).__init__(name, **kwargs) self.description = description self.images = images + + +class TimeSeriesReference(NamedTuple): + """ + Class used to represent data values of a :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` + This is a ``typing.NamedTuple`` type with predefined tuple components + :py:meth:`~pynwb.base.TimeSeriesReference.idx_start`, :py:meth:`~pynwb.base.TimeSeriesReference.count`, and + :py:meth:`~pynwb.base.TimeSeriesReference.timeseries`. + + :cvar idx_start: + :cvar count: + :cvar timeseries: + """ + idx_start: int + """Start index in time for the timeseries""" + + count: int + """Number of timesteps to be selected starting from :py:meth:`~pynwb.base.TimeSeriesReference.idx_start`""" + + timeseries: TimeSeries + """The :py:class:`~pynwb.base.TimeSeries` object the TimeSeriesReference applies to""" + + def check_types(self): + """ + Helper function to check correct types for :py:meth:`~pynwb.base.TimeSeriesReference.idx_start`, + :py:meth:`~pynwb.base.TimeSeriesReference.count`, and :py:meth:`~pynwb.base.TimeSeriesReference.timeseries`. + + This function is usually used in the try/except block to check for `TypeError` raised by the function. + + See also :py:meth:`~pynwb.base.TimeSeriesReference.isvalid` to check both validity of types and the + reference itself. + + :returns: True if successful. If unsuccessful `TypeError` is raised. + + :raises TypeError: If one of the fields does not match the expected type + """ + if not isinstance(self.idx_start, (int, np.integer)): + raise TypeError("idx_start must be an integer not %s" % str(type(self.idx_start))) + if not isinstance(self.count, (int, np.integer)): + raise TypeError("count must be an integer %s" % str(type(self.count))) + if not isinstance(self.timeseries, TimeSeries): + raise TypeError("timeseries must be of type TimeSeries. %s" % str(type(self.timeseries))) + return True + + def isvalid(self): + """ + Check whether the reference is valid. Setting :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and + :py:meth:`~pynwb.base.TimeSeriesReference.count` to -1 is used to indicate invalid references. This is + useful to allow for missing data in :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` + + :returns: True if the selection is valid. Returns False if both + :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and :py:meth:`~pynwb.base.TimeSeriesReference.count` + are negative. Raises `IndexError` in case the indices are bad. + + :raises IndexError: If the combination of :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and + :py:meth:`~pynwb.base.TimeSeriesReference.count` are not valid for the given timeseries. + + :raises TypeError: If one of the fields does not match the expected type + """ + # Check types first + self.check_types() + # Check for none-type selection + if self.idx_start < 0 and self.count < 0: + return False + num_samples = self.timeseries.num_samples + if num_samples is not None: + if self.idx_start >= num_samples or self.idx_start < 0: + raise IndexError("'idx_start' %i out of range for timeseries '%s'" % + (self.idx_start, self.timeseries.name)) + if self.count < 0: + raise IndexError("'count' %i invalid. 'count' must be positive" % self.count) + if (self.idx_start + self.count) > num_samples: + raise IndexError("'idx_start + count' out of range for timeseries '%s'" % self.timeseries.name) + return True + + @property + def timestamps(self): + """ + Get the floating point timestamp offsets in seconds from the timeseries that correspond to the array. + These are either loaded directly from the :py:meth:`~pynwb.base.TimeSeriesReference.timeseries` + timestamps or calculated from the starting time and sampling rate. + + + :raises IndexError: If the combination of :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and + :py:meth:`~pynwb.base.TimeSeriesReference.count` are not valid for the given timeseries. + + :raises TypeError: If one of the fields does not match the expected type + + :returns: Array with the timestamps. + """ + # isvalid will be False only if both idx_start and count are negative. Otherwise well get errors or be True. + if not self.isvalid(): + return None + # load the data from the timestamps + elif self.timeseries.timestamps is not None: + return self.timeseries.timestamps[self.idx_start: (self.idx_start + self.count)] + # construct the timestamps from the starting_time and rate + else: + start_time = self.timeseries.rate * self.idx_start + self.timeseries.starting_time + return np.arange(0, self.count) * self.timeseries.rate + start_time + + @property + def data(self): + """ + Get the selected data values. This is a convenience function to slice data from the + :py:meth:`~pynwb.base.TimeSeriesReference.timeseries` based on the given + :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and + :py:meth:`~pynwb.base.TimeSeriesReference.count` + + :raises IndexError: If the combination of :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` and + :py:meth:`~pynwb.base.TimeSeriesReference.count` are not valid for the given timeseries. + + :raises TypeError: If one of the fields does not match the expected type + + :returns: Result of ``self.timeseries.data[self.idx_start: (self.idx_start + self.count)]``. Returns + None in case the reference is invalid (i.e., if both :py:meth:`~pynwb.base.TimeSeriesReference.idx_start` + and :py:meth:`~pynwb.base.TimeSeriesReference.count` are negative. + """ + # isvalid will be False only if both idx_start and count are negative. Otherwise well get errors or be True. + if not self.isvalid(): + return None + # load the data from the timeseries + return self.timeseries.data[self.idx_start: (self.idx_start + self.count)] + + +@register_class('TimeSeriesReferenceVectorData', CORE_NAMESPACE) +class TimeSeriesReferenceVectorData(VectorData): + """ + Column storing references to a TimeSeries (rows). For each TimeSeries this VectorData + column stores the start_index and count to indicate the range in time to be selected + as well as an object reference to the TimeSeries. + + **Representing missing values** In practice we sometimes need to be able to represent missing values, + e.g., in the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` we have + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` to link to stimulus and + response recordings, but a user can specify either only one of them or both. Since there is no + ``None`` value for a complex types like ``(idx_start, count, TimeSeries)``, NWB defines + ``None`` as ``(-1, -1, TimeSeries)`` for storage, i.e., if the ``idx_start`` (and ``count``) is negative + then this indicates an invalid link (in practice both ``idx_start`` and ``count`` must always + either both be positive or both be negative). When selecting data via the + :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or + :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__` + functions, ``(-1, -1, TimeSeries)`` values are replaced by the corresponding + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE` tuple + to avoid exposing NWB storage internals to the user and simplifying the use of and checking + for missing values. **NOTE:** We can still inspect the raw data values by looking at ``self.data`` + directly instead. + + :cvar TIME_SERIES_REFERENCE_TUPLE: + :cvar TIME_SERIES_REFERENCE_NONE_TYPE: + """ + + TIME_SERIES_REFERENCE_TUPLE = TimeSeriesReference + """Return type when calling :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or + :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__`.""" + + TIME_SERIES_REFERENCE_NONE_TYPE = TIME_SERIES_REFERENCE_TUPLE(None, None, None) + """Tuple used to represent None values when calling :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or + :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__`. See also + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE`""" + + @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorData', 'default': 'timeseries'}, + {'name': 'description', 'type': str, 'doc': 'a description for this column', + 'default': "Column storing references to a TimeSeries (rows). For each TimeSeries this " + "VectorData column stores the start_index and count to indicate the range in time " + "to be selected as well as an object reference to the TimeSeries."}, + *get_docval(VectorData.__init__, 'data')) + def __init__(self, **kwargs): + call_docval_func(super().__init__, kwargs) + + @docval({'name': 'val', 'type': TIME_SERIES_REFERENCE_TUPLE, 'doc': 'the value to add to this column'}) + def add_row(self, **kwargs): + """Append a data value to this column.""" + val = getargs('val', kwargs) + val.check_types() + super().append(val) + + @docval({'name': 'arg', 'type': TIME_SERIES_REFERENCE_TUPLE, 'doc': 'the value to append to this column'}) + def append(self, **kwargs): + """Append a data value to this column.""" + arg = getargs('arg', kwargs) + arg.check_types() + super().append(arg) + + def get(self, key, **kwargs): + """ + Retrieve elements from this object. + + The function uses :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE` + to describe individual records in the dataset. This allows the code to avoid exposing internal + details of the schema to the user and simplifies handling of missing values by explictly + representing missing values via + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE` + rather than the internal representation used for storage of ``(-1, -1, TimeSeries)``. + + :param key: Selection of the elements + :param kwargs: Ignored + + :returns: :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE` if a single + element is being selected. Otherwise return a list of + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE` objects. + Missing values are represented by + :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE` + in which all values (i.e., idx_start, count, timeseries) are set to None. + """ + vals = super().get(key) + # we only selected one row. + if isinstance(key, (int, np.integer)): + # NOTE: If we never wrote the data to disk, then vals will be a single tuple. + # If the data is loaded from an h5py.Dataset then vals will be a single + # np.void object. I.e., an alternative check would be + # if isinstance(vals, tuple) or isinstance(vals, np.void): + # ... + if vals[0] < 0 or vals[1] < 0: + return self.TIME_SERIES_REFERENCE_NONE_TYPE + else: + return self.TIME_SERIES_REFERENCE_TUPLE(*vals) + else: # key selected multiple rows + # When loading from HDF5 we get an np.ndarray otherwise we get list-of-list. This + # makes the values consistent and tranforms the data to use our namedtuple type + re = [self.TIME_SERIES_REFERENCE_NONE_TYPE + if (v[0] < 0 or v[1] < 0) else self.TIME_SERIES_REFERENCE_TUPLE(*v) + for v in vals] + return re diff --git a/src/pynwb/behavior.py b/src/pynwb/behavior.py index f9756b2cc..431c81bbc 100644 --- a/src/pynwb/behavior.py +++ b/src/pynwb/behavior.py @@ -26,14 +26,16 @@ class SpatialSeries(TimeSeries): 'dimension represents different features, e.g., x, y position')}, {'name': 'reference_frame', 'type': str, # required 'doc': 'description defining what the zero-position is'}, + {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)', + 'default': 'meters'}, *get_docval(TimeSeries.__init__, 'conversion', 'resolution', 'timestamps', 'starting_time', 'rate', 'comments', 'description', 'control', 'control_description')) def __init__(self, **kwargs): """ Create a SpatialSeries TimeSeries dataset """ - name, data, reference_frame = popargs('name', 'data', 'reference_frame', kwargs) - super(SpatialSeries, self).__init__(name, data, 'meters', **kwargs) + name, data, reference_frame, unit = popargs('name', 'data', 'reference_frame', 'unit', kwargs) + super(SpatialSeries, self).__init__(name, data, unit, **kwargs) self.reference_frame = reference_frame diff --git a/src/pynwb/file.py b/src/pynwb/file.py index a31f1cda7..25bfdd86d 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -7,14 +7,16 @@ import numpy as np import pandas as pd -from hdmf.utils import docval, getargs, call_docval_func, get_docval +from hdmf.utils import docval, getargs, call_docval_func, get_docval, popargs from . import register_class, CORE_NAMESPACE from .base import TimeSeries, ProcessingModule from .device import Device from .epoch import TimeIntervals from .ecephys import ElectrodeGroup -from .icephys import IntracellularElectrode, SweepTable, PatchClampSeries +from .icephys import (IntracellularElectrode, SweepTable, PatchClampSeries, IntracellularRecordingsTable, + SimultaneousRecordingsTable, SequentialRecordingsTable, RepetitionsTable, + ExperimentalConditionsTable) from .ophys import ImagingPlane from .ogen import OptogeneticStimulusSite from .misc import Units @@ -217,8 +219,36 @@ class NWBFile(MultiContainerInterface): {'name': 'sweep_table', 'child': True, 'required_name': 'sweep_table'}, {'name': 'invalid_times', 'child': True, 'required_name': 'invalid_times'}, 'epoch_tags', - {'name': 'icephys_filtering', 'settable': False}) - # icephys_filtering is temporary. /intracellular_ephys/filtering dataset will be deprecated + # icephys_filtering is temporary. /intracellular_ephys/filtering dataset will be deprecated + {'name': 'icephys_filtering', 'settable': False}, + {'name': 'intracellular_recordings', 'child': True, + 'required_name': 'intracellular_recordings', + 'doc': 'IntracellularRecordingsTable table to group together a stimulus and response ' + 'from a single intracellular electrode and a single simultaneous recording.'}, + {'name': 'icephys_simultaneous_recordings', + 'child': True, + 'required_name': 'simultaneous_recordings', + 'doc': 'SimultaneousRecordingsTable table for grouping different intracellular recordings from' + 'the IntracellularRecordingsTable table together that were recorded simultaneously ' + 'from different electrodes'}, + {'name': 'icephys_sequential_recordings', + 'child': True, + 'required_name': 'sequential_recordings', + 'doc': 'A table for grouping different simultaneous intracellular recording from the ' + 'SimultaneousRecordingsTable table together. This is typically used to group ' + 'together simultaneous recordings where the a sequence of stimuli of the same ' + 'type with varying parameters have been presented in a sequence.'}, + {'name': 'icephys_repetitions', + 'child': True, + 'required_name': 'repetitions', + 'doc': 'A table for grouping different intracellular recording sequential recordings together.' + 'With each SweepSequence typically representing a particular type of stimulus, the ' + 'RepetitionsTable table is typically used to group sets of stimuli applied in sequence.'}, + {'name': 'icephys_experimental_conditions', + 'child': True, + 'required_name': 'experimental_conditions', + 'doc': 'A table for grouping different intracellular recording repetitions together that ' + 'belong to the same experimental experimental_conditions.'}) @docval({'name': 'session_description', 'type': str, 'doc': 'a description of the session where this data was generated'}, @@ -310,7 +340,19 @@ class NWBFile(MultiContainerInterface): {'name': 'scratch', 'type': (list, tuple), 'doc': 'scratch data', 'default': None}, {'name': 'icephys_electrodes', 'type': (list, tuple), - 'doc': 'IntracellularElectrodes that belong to this NWBFile.', 'default': None}) + 'doc': 'IntracellularElectrodes that belong to this NWBFile.', 'default': None}, + {'name': 'icephys_filtering', 'type': str, 'default': None, + 'doc': '[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used.'}, + {'name': 'intracellular_recordings', 'type': IntracellularRecordingsTable, 'default': None, + 'doc': 'the IntracellularRecordingsTable table that belongs to this NWBFile'}, + {'name': 'icephys_simultaneous_recordings', 'type': SimultaneousRecordingsTable, 'default': None, + 'doc': 'the SimultaneousRecordingsTable table that belongs to this NWBFile'}, + {'name': 'icephys_sequential_recordings', 'type': SequentialRecordingsTable, 'default': None, + 'doc': 'the SequentialRecordingsTable table that belongs to this NWBFile'}, + {'name': 'icephys_repetitions', 'type': RepetitionsTable, 'default': None, + 'doc': 'the RepetitionsTable table that belongs to this NWBFile'}, + {'name': 'icephys_experimental_conditions', 'type': ExperimentalConditionsTable, 'default': None, + 'doc': 'the ExperimentalConditionsTable table that belongs to this NWBFile'}) def __init__(self, **kwargs): kwargs['name'] = 'root' call_docval_func(super(NWBFile, self).__init__, kwargs) @@ -370,6 +412,12 @@ def __init__(self, **kwargs): 'surgery', 'virus', 'stimulus_notes', + 'icephys_filtering', # DEPRECATION warning will be raised in the setter when calling setattr in the loop + 'intracellular_recordings', + 'icephys_simultaneous_recordings', + 'icephys_sequential_recordings', + 'icephys_repetitions', + 'icephys_experimental_conditions' ] for attr in fieldnames: setattr(self, attr, kwargs.get(attr, None)) @@ -439,6 +487,17 @@ def ic_electrodes(self): warn("NWBFile.ic_electrodes has been replaced by NWBFile.icephys_electrodes.", DeprecationWarning) return self.icephys_electrodes + @property + def icephys_filtering(self): + return self.fields.get('icephys_filtering') + + @icephys_filtering.setter + def icephys_filtering(self, val): + if val is not None: + warn("Use of icephys_filtering is deprecated. Use the IntracellularElectrode.filtering field instead", + DeprecationWarning) + self.fields['icephys_filtering'] = val + def add_ic_electrode(self, *args, **kwargs): """ This method is deprecated and will be removed in future versions. Please @@ -676,20 +735,195 @@ def _update_sweep_table(self, nwbdata): self._check_sweep_table() self.sweep_table.add_entry(nwbdata) - @docval({'name': 'nwbdata', 'type': (NWBDataInterface, DynamicTable)}) - def add_acquisition(self, nwbdata): + @docval({'name': 'nwbdata', 'type': (NWBDataInterface, DynamicTable)}, + {'name': 'use_sweep_table', 'type': bool, 'default': False, 'doc': 'Use the deprecated SweepTable'}) + def add_acquisition(self, **kwargs): + nwbdata = popargs('nwbdata', kwargs) self._add_acquisition_internal(nwbdata) - self._update_sweep_table(nwbdata) - - @docval({'name': 'timeseries', 'type': TimeSeries}) - def add_stimulus(self, timeseries): + use_sweep_table = popargs('use_sweep_table', kwargs) + if use_sweep_table: + self._update_sweep_table(nwbdata) + + @docval({'name': 'timeseries', 'type': TimeSeries}, + {'name': 'use_sweep_table', 'type': bool, 'default': False, 'doc': 'Use the deprecated SweepTable'}) + def add_stimulus(self, **kwargs): + timeseries = popargs('timeseries', kwargs) self._add_stimulus_internal(timeseries) - self._update_sweep_table(timeseries) - - @docval({'name': 'timeseries', 'type': TimeSeries}) - def add_stimulus_template(self, timeseries): + use_sweep_table = popargs('use_sweep_table', kwargs) + if use_sweep_table: + self._update_sweep_table(timeseries) + + @docval({'name': 'timeseries', 'type': TimeSeries}, + {'name': 'use_sweep_table', 'type': bool, 'default': False, 'doc': 'Use the deprecated SweepTable'}) + def add_stimulus_template(self, **kwargs): + timeseries = popargs('timeseries', kwargs) self._add_stimulus_template_internal(timeseries) - self._update_sweep_table(timeseries) + use_sweep_table = popargs('use_sweep_table', kwargs) + if use_sweep_table: + self._update_sweep_table(timeseries) + + @docval(returns='The NWBFile.intracellular_recordings table', rtype=IntracellularRecordingsTable) + def get_intracellular_recordings(self): + """ + Get the NWBFile.intracellular_recordings table. + + In contrast to NWBFile.intracellular_recordings, this function will create the + IntracellularRecordingsTable table if not yet done, whereas NWBFile.intracellular_recordings + will return None if the table is currently not being used. + """ + if self.intracellular_recordings is None: + self.intracellular_recordings = IntracellularRecordingsTable() + return self.intracellular_recordings + + @docval(*get_docval(IntracellularRecordingsTable.add_recording), + returns='Integer index of the row that was added to IntracellularRecordingsTable', + rtype=int, + allow_extra=True) + def add_intracellular_recording(self, **kwargs): + """ + Add a intracellular recording to the intracellular_recordings table. If the + electrode, stimulus, and/or response do not exist yet in the NWBFile, then + they will be added to this NWBFile before adding them to the table. + + Note: For more complex organization of intracellular recordings you may also be + interested in the related SimultaneousRecordingsTable, SequentialRecordingsTable, + RepetitionsTable, and ExperimentalConditionsTable tables and the related functions + of NWBFile: add_icephys_simultaneous_recording, add_icephys_sequential_recording, + add_icephys_repetition, and add_icephys_experimental_condition. + """ + # Add the stimulus, response, and electrode to the file if they don't exist yet + stimulus, response, electrode = getargs('stimulus', 'response', 'electrode', kwargs) + if (stimulus is not None and + (stimulus.name not in self.stimulus and + stimulus.name not in self.stimulus_template)): + self.add_stimulus(stimulus, use_sweep_table=False) + if response is not None and response.name not in self.acquisition: + self.add_acquisition(response, use_sweep_table=False) + if electrode is not None and electrode.name not in self.icephys_electrodes: + self.add_icephys_electrode(electrode) + # make sure the intracellular recordings table exists and if not create it using get_intracellular_recordings + # Add the recoding to the intracellular_recordings table + return call_docval_func(self.get_intracellular_recordings().add_recording, kwargs) + + @docval(returns='The NWBFile.icephys_simultaneous_recordings table', rtype=SimultaneousRecordingsTable) + def get_icephys_simultaneous_recordings(self): + """ + Get the NWBFile.icephys_simultaneous_recordings table. + + In contrast to NWBFile.icephys_simultaneous_recordings, this function will create the + SimultaneousRecordingsTable table if not yet done, whereas NWBFile.icephys_simultaneous_recordings + will return None if the table is currently not being used. + """ + if self.icephys_simultaneous_recordings is None: + self.icephys_simultaneous_recordings = SimultaneousRecordingsTable(self.get_intracellular_recordings()) + return self.icephys_simultaneous_recordings + + @docval(*get_docval(SimultaneousRecordingsTable.add_simultaneous_recording), + returns='Integer index of the row that was added to SimultaneousRecordingsTable', + rtype=int, + allow_extra=True) + def add_icephys_simultaneous_recording(self, **kwargs): + """ + Add a new simultaneous recording to the icephys_simultaneous_recordings table + """ + return call_docval_func(self.get_icephys_simultaneous_recordings().add_simultaneous_recording, kwargs) + + @docval(returns='The NWBFile.icephys_sequential_recordings table', rtype=SequentialRecordingsTable) + def get_icephys_sequential_recordings(self): + """ + Get the NWBFile.icephys_sequential_recordings table. + + In contrast to NWBFile.icephys_sequential_recordings, this function will create the + IntracellularRecordingsTable table if not yet done, whereas NWBFile.icephys_sequential_recordings + will return None if the table is currently not being used. + """ + if self.icephys_sequential_recordings is None: + self.icephys_sequential_recordings = SequentialRecordingsTable(self.get_icephys_simultaneous_recordings()) + return self.icephys_sequential_recordings + + @docval(*get_docval(SequentialRecordingsTable.add_sequential_recording), + returns='Integer index of the row that was added to SequentialRecordingsTable', + rtype=int, + allow_extra=True) + def add_icephys_sequential_recording(self, **kwargs): + """ + Add a new sequential recording to the icephys_sequential_recordings table + """ + self.get_icephys_sequential_recordings() + return call_docval_func(self.icephys_sequential_recordings.add_sequential_recording, kwargs) + + @docval(returns='The NWBFile.icephys_repetitions table', rtype=RepetitionsTable) + def get_icephys_repetitions(self): + """ + Get the NWBFile.icephys_repetitions table. + + In contrast to NWBFile.icephys_repetitions, this function will create the + RepetitionsTable table if not yet done, whereas NWBFile.icephys_repetitions + will return None if the table is currently not being used. + """ + if self.icephys_repetitions is None: + self.icephys_repetitions = RepetitionsTable(self.get_icephys_sequential_recordings()) + return self.icephys_repetitions + + @docval(*get_docval(RepetitionsTable.add_repetition), + returns='Integer index of the row that was added to RepetitionsTable', + rtype=int, + allow_extra=True) + def add_icephys_repetition(self, **kwargs): + """ + Add a new repetition to the RepetitionsTable table + """ + return call_docval_func(self.get_icephys_repetitions().add_repetition, kwargs) + + @docval(returns='The NWBFile.icephys_experimental_conditions table', rtype=ExperimentalConditionsTable) + def get_icephys_experimental_conditions(self): + """ + Get the NWBFile.icephys_experimental_conditions table. + + In contrast to NWBFile.icephys_experimental_conditions, this function will create the + RepetitionsTable table if not yet done, whereas NWBFile.icephys_experimental_conditions + will return None if the table is currently not being used. + """ + if self.icephys_experimental_conditions is None: + self.icephys_experimental_conditions = ExperimentalConditionsTable(self.get_icephys_repetitions()) + return self.icephys_experimental_conditions + + @docval(*get_docval(ExperimentalConditionsTable.add_experimental_condition), + returns='Integer index of the row that was added to ExperimentalConditionsTable', + rtype=int, + allow_extra=True) + def add_icephys_experimental_condition(self, **kwargs): + """ + Add a new condition to the ExperimentalConditionsTable table + """ + return call_docval_func(self.get_icephys_experimental_conditions().add_experimental_condition, kwargs) + + def get_icephys_meta_parent_table(self): + """ + Get the top-most table in the intracellular ephys metadata table hierarchy that exists in this NWBFile. + + The intracellular ephys metadata consists of a hierarchy of DynamicTables, i.e., + experimental_conditions --> repetitions --> sequential_recordings --> + simultaneous_recordings --> intracellular_recordings etc. + In a given NWBFile not all tables may exist. This convenience functions returns the top-most + table that exists in this file. E.g., if the file contains only the simultaneous_recordings + and intracellular_recordings tables then the function would return the simultaneous_recordings table. + Similarly, if the file contains all tables then it will return the experimental_conditions table. + + :returns: DynamicTable object or None + """ + if self.icephys_experimental_conditions is not None: + return self.icephys_experimental_conditions + elif self.icephys_repetitions is not None: + return self.icephys_repetitions + elif self.icephys_sequential_recordings is not None: + return self.icephys_sequential_recordings + elif self.icephys_simultaneous_recordings is not None: + return self.icephys_simultaneous_recordings + elif self.intracellular_recordings is not None: + return self.intracellular_recordings + else: + return None @docval({'name': 'data', 'type': ('scalar_data', np.ndarray, list, tuple, pd.DataFrame, DynamicTable, NWBContainer, ScratchData), diff --git a/src/pynwb/icephys.py b/src/pynwb/icephys.py index bcceaa5ac..215e7d9d5 100644 --- a/src/pynwb/icephys.py +++ b/src/pynwb/icephys.py @@ -1,12 +1,28 @@ import warnings -from hdmf.common import DynamicTable -from hdmf.utils import docval, popargs, call_docval_func, get_docval +from hdmf.common import DynamicTable, AlignedDynamicTable +from hdmf.utils import docval, popargs, call_docval_func, get_docval, getargs from . import register_class, CORE_NAMESPACE -from .base import TimeSeries +from .base import TimeSeries, TimeSeriesReferenceVectorData from .core import NWBContainer from .device import Device +from copy import copy +import numpy as np + + +def ensure_unit(self, name, current_unit, unit, nwb_version): + """A helper to ensure correct unit used. + + Issues a warning with details if `current_unit` is to be ignored, and + `unit` to be used instead. + """ + if current_unit != unit: + warnings.warn( + "Unit '%s' for %s '%s' is ignored and will be set to '%s' " + "as per NWB %s." + % (current_unit, self.__class__.__name__, name, unit, nwb_version)) + return unit @register_class('IntracellularElectrode', CORE_NAMESPACE) @@ -278,6 +294,9 @@ class SweepTable(DynamicTable): 'default': "A sweep table groups different PatchClampSeries together."}, *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) def __init__(self, **kwargs): + warnings.warn("Use of SweepTable is deprecated. Use the IntracellularRecordingsTable " + "instead. See also the NWBFile.add_intracellular_recordings function.", + DeprecationWarning) call_docval_func(super().__init__, kwargs) @docval({'name': 'pcs', 'type': PatchClampSeries, @@ -319,15 +338,518 @@ def __get_row_ids(self, sweep_number): return [index for index, elem in enumerate(self['sweep_number'].data) if elem == sweep_number] -def ensure_unit(self, name, current_unit, unit, nwb_version): - """A helper to ensure correct unit used. +@register_class('IntracellularElectrodesTable', CORE_NAMESPACE) +class IntracellularElectrodesTable(DynamicTable): + """ + Table for storing intracellular electrode related metadata' + """ + __columns__ = ( + {'name': 'electrode', + 'description': 'Column for storing the reference to the intracellular electrode', + 'required': True, + 'index': False, + 'table': False}, + ) - Issues a warning with details if `current_unit` is to be ignored, and - `unit` to be used instead. + @docval(*get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + # Define defaultb name and description settings + kwargs['name'] = 'electrodes' + kwargs['description'] = ('Table for storing intracellular electrode related metadata') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + + +@register_class('IntracellularStimuliTable', CORE_NAMESPACE) +class IntracellularStimuliTable(DynamicTable): """ - if current_unit != unit: - warnings.warn( - "Unit '%s' for %s '%s' is ignored and will be set to '%s' " - "as per NWB %s." - % (current_unit, self.__class__.__name__, name, unit, nwb_version)) - return unit + Table for storing intracellular electrode related metadata' + """ + __columns__ = ( + {'name': 'stimulus', + 'description': 'Column storing the reference to the recorded stimulus for the recording (rows)', + 'required': True, + 'index': False, + 'table': False, + 'class': TimeSeriesReferenceVectorData}, + ) + + @docval(*get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + # Define defaultb name and description settings + kwargs['name'] = 'stimuli' + kwargs['description'] = ('Table for storing intracellular stimulus related metadata') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + + +@register_class('IntracellularResponsesTable', CORE_NAMESPACE) +class IntracellularResponsesTable(DynamicTable): + """ + Table for storing intracellular electrode related metadata' + """ + __columns__ = ( + {'name': 'response', + 'description': 'Column storing the reference to the recorded response for the recording (rows)', + 'required': True, + 'index': False, + 'table': False, + 'class': TimeSeriesReferenceVectorData}, + ) + + @docval(*get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + # Define defaultb name and description settings + kwargs['name'] = 'responses' + kwargs['description'] = ('Table for storing intracellular response related metadata') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + + +@register_class('IntracellularRecordingsTable', CORE_NAMESPACE) +class IntracellularRecordingsTable(AlignedDynamicTable): + """ + A table to group together a stimulus and response from a single electrode and + a single simultaneous_recording. Each row in the table represents a single recording consisting + typically of a stimulus and a corresponding response. + """ + @docval(*get_docval(AlignedDynamicTable.__init__, 'id', 'columns', 'colnames', 'category_tables', 'categories')) + def __init__(self, **kwargs): + kwargs['name'] = 'intracellular_recordings' + kwargs['description'] = ('A table to group together a stimulus and response from a single electrode ' + 'and a single simultaneous recording and for storing metadata about the ' + 'intracellular recording.') + in_category_tables = getargs('category_tables', kwargs) + if in_category_tables is None or len(in_category_tables) == 0: + kwargs['category_tables'] = [IntracellularElectrodesTable(), + IntracellularStimuliTable(), + IntracellularResponsesTable()] + kwargs['categories'] = None + else: + # Check if our required data tables are supplied, otherwise add them to the list + required_dynamic_table_given = [-1 for i in range(3)] # The first three are our required tables + for i, tab in enumerate(in_category_tables): + if isinstance(tab, IntracellularElectrodesTable): + required_dynamic_table_given[0] = i + elif isinstance(tab, IntracellularStimuliTable): + required_dynamic_table_given[1] = i + elif isinstance(tab, IntracellularResponsesTable): + required_dynamic_table_given[2] = i + # Check if the supplied tables contain data but not all required tables have been supplied + required_dynamic_table_missing = np.any(np.array(required_dynamic_table_given[0:3]) < 0) + if len(in_category_tables[0]) != 0 and required_dynamic_table_missing: + raise ValueError("IntracellularElectrodeTable, IntracellularStimuliTable, and " + "IntracellularResponsesTable are required when adding custom, non-empty " + "tables to IntracellularRecordingsTable as the missing data for the required " + "tables cannot be determined automatically") + # Compile the complete list of tables + dynamic_table_arg = copy(in_category_tables) + categories_arg = [] if getargs('categories', kwargs) is None else copy(getargs('categories', kwargs)) + if required_dynamic_table_missing: + if required_dynamic_table_given[2] < 0: + dynamic_table_arg.append(IntracellularResponsesTable) + if not dynamic_table_arg[-1].name in categories_arg: + categories_arg.insert(0, dynamic_table_arg[-1].name) + if required_dynamic_table_given[1] < 0: + dynamic_table_arg.append(IntracellularStimuliTable()) + if not dynamic_table_arg[-1].name in categories_arg: + categories_arg.insert(0, dynamic_table_arg[-1].name) + if required_dynamic_table_given[0] < 0: + dynamic_table_arg.append(IntracellularElectrodesTable()) + if not dynamic_table_arg[-1].name in categories_arg: + categories_arg.insert(0, dynamic_table_arg[-1].name) + kwargs['category_tables'] = dynamic_table_arg + kwargs['categories'] = categories_arg + + call_docval_func(super().__init__, kwargs) + + @docval({'name': 'electrode', 'type': IntracellularElectrode, 'doc': 'The intracellular electrode used'}, + {'name': 'stimulus_start_index', 'type': 'int', 'doc': 'Start index of the stimulus', 'default': None}, + {'name': 'stimulus_index_count', 'type': 'int', 'doc': 'Stop index of the stimulus', 'default': None}, + {'name': 'stimulus', 'type': TimeSeries, + 'doc': 'The TimeSeries (usually a PatchClampSeries) with the stimulus', + 'default': None}, + {'name': 'response_start_index', 'type': 'int', 'doc': 'Start index of the response', 'default': None}, + {'name': 'response_index_count', 'type': 'int', 'doc': 'Stop index of the response', 'default': None}, + {'name': 'response', 'type': TimeSeries, + 'doc': 'The TimeSeries (usually a PatchClampSeries) with the response', + 'default': None}, + {'name': 'electrode_metadata', 'type': dict, + 'doc': 'Additional electrode metadata to be stored in the electrodes table', 'default': None}, + {'name': 'stimulus_metadata', 'type': dict, + 'doc': 'Additional stimulus metadata to be stored in the stimuli table', 'default': None}, + {'name': 'response_metadata', 'type': dict, + 'doc': 'Additional resposnse metadata to be stored in the responses table', 'default': None}, + returns='Integer index of the row that was added to this table', + rtype=int, + allow_extra=True) + def add_recording(self, **kwargs): + """ + Add a single recording to the IntracellularRecordingsTable table. + + Typically, both stimulus and response are expected. However, in some cases only a stimulus + or a response may be recodred as part of a recording. In this case, None may be given + for either stimulus or response, but not both. Internally, this results in both stimulus + and response pointing to the same TimeSeries, while the start_index and index_count for + the invalid series will both be set to -1. + """ + # Get the input data + stimulus_start_index, stimulus_index_count, stimulus = popargs('stimulus_start_index', + 'stimulus_index_count', + 'stimulus', + kwargs) + response_start_index, response_index_count, response = popargs('response_start_index', + 'response_index_count', + 'response', + kwargs) + electrode = popargs('electrode', kwargs) + # Confirm that we have at least a valid stimulus or response + if stimulus is None and response is None: + raise ValueError("stimulus and response cannot both be None.") + + # Compute the start and stop index if necessary + stimulus_start_index, stimulus_index_count = self.__compute_index(stimulus_start_index, + stimulus_index_count, + stimulus, 'stimulus') + response_start_index, response_index_count = self.__compute_index(response_start_index, + response_index_count, + response, 'response') + # If either stimulus or response are None, then set them to the same TimeSeries to keep the I/O happy + response = response if response is not None else stimulus + stimulus = stimulus if stimulus is not None else response + + # Make sure the types are compatible + if ((response.neurodata_type.startswith("CurrentClamp") and + stimulus.neurodata_type.startswith("VoltageClamp")) or + (response.neurodata_type.startswith("VoltageClamp") and + stimulus.neurodata_type.startswith("CurrentClamp"))): + raise ValueError("Incompatible types given for 'stimulus' and 'response' parameters. " + "'stimulus' is of type %s and 'response' is of type %s." % + (stimulus.neurodata_type, response.neurodata_type)) + if response.neurodata_type == 'IZeroClampSeries': + if stimulus is not None: + raise ValueError("stimulus should usually be None for IZeroClampSeries response") + if isinstance(response, PatchClampSeries) and isinstance(stimulus, PatchClampSeries): + # # We could also check sweep_number, but since it is mostly relevant to the deprecated SweepTable + # # we don't really need to enforce it here + # if response.sweep_number != stimulus.sweep_number: + # warnings.warn("sweep_number are usually expected to be the same for PatchClampSeries type " + # "stimulus and response pairs in an intracellular recording.") + if response.electrode != stimulus.electrode: + raise ValueError("electrodes are usually expected to be the same for PatchClampSeries type " + "stimulus and response pairs in an intracellular recording.") + + # Compile the electrodes table data + electrodes = copy(popargs('electrode_metadata', kwargs)) + if electrodes is None: + electrodes = {} + electrodes['electrode'] = electrode + + # Compile the stimuli table data + stimuli = copy(popargs('stimulus_metadata', kwargs)) + if stimuli is None: + stimuli = {} + stimuli['stimulus'] = TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE( + stimulus_start_index, stimulus_index_count, stimulus) + + # Compile the responses table data + responses = copy(popargs('response_metadata', kwargs)) + if responses is None: + responses = {} + responses['response'] = TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE( + response_start_index, response_index_count, response) + + _ = super().add_row(enforce_unique_id=True, + electrodes=electrodes, + responses=responses, + stimuli=stimuli, + **kwargs) + return len(self) - 1 + + @staticmethod + def __compute_index(start_index, index_count, time_series, name): + """ + Internal helper function to compute the start_index and index_count + to use for the stimulus and response column + + :param start_index: The start_index provided by the user + :param index_count: The index count provided by the user + :param time_series: The timeseries object to reference. May be None. + :param name: Name of the table. Used only to enhance error reporting + + :raises IndexError: If index_count cannot be determined or start_index+index_count + are outside of the range of the timeseries. + + :returns: A tuple of integers with the start_index and index_count to use. + """ + # If times_series is not valid then return -1, -1 to indicate invalid times + if time_series is None: + return -1, -1 + # Since time_series is valid, negative or None start_index means the user did not specify a start_index + # so we now need to set it to 0 + if start_index is None or start_index < 0: + start_index = 0 + # If index_count has not been set yet (i.e., it is -1 or None) then attempt to set it to the + # full range of the timeseries starting from start_index + num_samples = time_series.num_samples + if index_count is None or index_count < 0: + index_count = (num_samples - start_index) if num_samples is not None else None + # Check that the start_index and index_count are valid and raise IndexError if they are invalid + if index_count is None: + raise IndexError("Invalid %s_index_count cannot be determined from %s data." % (name, name)) + if num_samples is not None: + if start_index >= num_samples: + raise IndexError("%s_start_index out of range" % name) + if (start_index + index_count) > num_samples: + raise IndexError("%s_start_index + %s_index_count out of range" % (name, name)) + # Return the values + return start_index, index_count + + @docval(*get_docval(AlignedDynamicTable.to_dataframe, 'ignore_category_ids'), + {'name': 'electrode_refs_as_objectids', 'type': bool, + 'doc': 'replace object references in the electrode column with object_ids', + 'default': False}, + {'name': 'stimulus_refs_as_objectids', 'type': bool, + 'doc': 'replace object references in the stimulus column with object_ids', + 'default': False}, + {'name': 'response_refs_as_objectids', 'type': bool, + 'doc': 'replace object references in the response column with object_ids', + 'default': False} + ) + def to_dataframe(self, **kwargs): + """Convert the collection of tables to a single pandas DataFrame""" + res = super().to_dataframe(ignore_category_ids=getargs('ignore_category_ids', kwargs)) + if getargs('electrode_refs_as_objectids', kwargs): + res[('electrodes', 'electrode')] = [e.object_id for e in res[('electrodes', 'electrode')]] + if getargs('stimulus_refs_as_objectids', kwargs): + res[('stimuli', 'stimulus')] = \ + [e if e[2] is None + else TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(e[0], e[1], e[2].object_id) + for e in res[('stimuli', 'stimulus')]] + if getargs('response_refs_as_objectids', kwargs): + res[('responses', 'response')] = \ + [e if e[2] is None else + TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(e[0], e[1], e[2].object_id) + for e in res[('responses', 'response')]] + return res + + +@register_class('SimultaneousRecordingsTable', CORE_NAMESPACE) +class SimultaneousRecordingsTable(DynamicTable): + """ + A table for grouping different intracellular recordings from the + IntracellularRecordingsTable table together that were recorded simultaneously + from different electrodes. + """ + + __columns__ = ( + {'name': 'recordings', + 'description': 'Column with references to one or more rows in the IntracellularRecordingsTable table', + 'required': True, + 'index': True, + 'table': True}, + ) + + @docval({'name': 'intracellular_recordings_table', + 'type': IntracellularRecordingsTable, + 'doc': 'the IntracellularRecordingsTable table that the recordings column indexes. May be None when ' + 'reading the Container from file as the table attribute is already populated in this case ' + 'but otherwise this is required.', + 'default': None}, + *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + intracellular_recordings_table = popargs('intracellular_recordings_table', kwargs) + # Define default name and description settings + kwargs['name'] = 'simultaneous_recordings' + kwargs['description'] = ('A table for grouping different intracellular recordings from the' + 'IntracellularRecordingsTable table together that were recorded simultaneously ' + 'from different electrodes.') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + if self['recordings'].target.table is None: + if intracellular_recordings_table is not None: + self['recordings'].target.table = intracellular_recordings_table + else: + raise ValueError("intracellular_recordings constructor argument required") + + @docval({'name': 'recordings', + 'type': 'array_data', + 'doc': 'the indices of the recordings belonging to this simultaneous recording'}, + returns='Integer index of the row that was added to this table', + rtype=int, + allow_extra=True) + def add_simultaneous_recording(self, **kwargs): + """ + Add a single simultaneous recording (i.e., one sweep, or one row) consisting of one or more + recordings and associated custom simultaneous recording metadata to the table. + """ + _ = super().add_row(enforce_unique_id=True, **kwargs) + return len(self.id) - 1 + + +@register_class('SequentialRecordingsTable', CORE_NAMESPACE) +class SequentialRecordingsTable(DynamicTable): + """ + A table for grouping different intracellular recording simultaneous_recordings from the + SimultaneousRecordingsTable table together. This is typically used to group together simultaneous_recordings + where the a sequence of stimuli of the same type with varying parameters + have been presented in a sequence. + """ + + __columns__ = ( + {'name': 'simultaneous_recordings', + 'description': 'Column with references to one or more rows in the SimultaneousRecordingsTable table', + 'required': True, + 'index': True, + 'table': True}, + {'name': 'stimulus_type', + 'description': 'Column storing the type of stimulus used for the sequential recording', + 'required': True, + 'index': False, + 'table': False} + ) + + @docval({'name': 'simultaneous_recordings_table', + 'type': SimultaneousRecordingsTable, + 'doc': 'the SimultaneousRecordingsTable table that the simultaneous_recordings ' + 'column indexes. May be None when reading the Container from file as the ' + 'table attribute is already populated in this case but otherwise this is required.', + 'default': None}, + *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + simultaneous_recordings_table = popargs('simultaneous_recordings_table', kwargs) + # Define defaultb name and description settings + kwargs['name'] = 'sequential_recordings' + kwargs['description'] = ('A table for grouping different intracellular recording simultaneous_recordings ' + 'from the SimultaneousRecordingsTable table together. This is typically used to ' + 'group together simultaneous_recordings where the a sequence of stimuli of the ' + 'same type with varying parameters have been presented in a sequence.') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + if self['simultaneous_recordings'].target.table is None: + if simultaneous_recordings_table is not None: + self['simultaneous_recordings'].target.table = simultaneous_recordings_table + else: + raise ValueError('simultaneous_recordings_table constructor argument required') + + @docval({'name': 'stimulus_type', + 'type': str, + 'doc': 'the type of stimulus used for the sequential recording'}, + {'name': 'simultaneous_recordings', + 'type': 'array_data', + 'doc': 'the indices of the simultaneous_recordings belonging to this sequential recording'}, + returns='Integer index of the row that was added to this table', + rtype=int, + allow_extra=True) + def add_sequential_recording(self, **kwargs): + """ + Add a sequential recording (i.e., one row) consisting of one or more simultaneous recordings + and associated custom sequential recording metadata to the table. + """ + _ = super().add_row(enforce_unique_id=True, **kwargs) + return len(self.id) - 1 + + +@register_class('RepetitionsTable', CORE_NAMESPACE) +class RepetitionsTable(DynamicTable): + """ + A table for grouping different intracellular recording sequential recordings together. + With each SweepSequence typically representing a particular type of stimulus, the + RepetitionsTable table is typically used to group sets of stimuli applied in sequence. + """ + + __columns__ = ( + {'name': 'sequential_recordings', + 'description': 'Column with references to one or more rows in the SequentialRecordingsTable table', + 'required': True, + 'index': True, + 'table': True}, + ) + + @docval({'name': 'sequential_recordings_table', + 'type': SequentialRecordingsTable, + 'doc': 'the SequentialRecordingsTable table that the sequential_recordings column indexes. May ' + 'be None when reading the Container from file as the table attribute is already populated ' + 'in this case but otherwise this is required.', + 'default': None}, + *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + sequential_recordings_table = popargs('sequential_recordings_table', kwargs) + # Define default name and description settings + kwargs['name'] = 'repetitions' + kwargs['description'] = ('A table for grouping different intracellular recording sequential recordings ' + 'together. With each SimultaneousRecording typically representing a particular type ' + 'of stimulus, the RepetitionsTable table is typically used to group sets ' + 'of stimuli applied in sequence.') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + if self['sequential_recordings'].target.table is None: + if sequential_recordings_table is not None: + self['sequential_recordings'].target.table = sequential_recordings_table + else: + raise ValueError('sequential_recordings_table constructor argument required') + + @docval({'name': 'sequential_recordings', + 'type': 'array_data', + 'doc': 'the indices of the sequential recordings belonging to this repetition', + 'default': None}, + returns='Integer index of the row that was added to this table', + rtype=int, + allow_extra=True) + def add_repetition(self, **kwargs): + """ + Add a repetition (i.e., one row) consisting of one or more sequential recordings + and associated custom repetition metadata to the table. + """ + _ = super().add_row(enforce_unique_id=True, **kwargs) + return len(self.id) - 1 + + +@register_class('ExperimentalConditionsTable', CORE_NAMESPACE) +class ExperimentalConditionsTable(DynamicTable): + """ + A table for grouping different intracellular recording repetitions together that + belong to the same experimental conditions. + """ + + __columns__ = ( + {'name': 'repetitions', + 'description': 'Column with references to one or more rows in the RepetitionsTable table', + 'required': True, + 'index': True, + 'table': True}, + ) + + @docval({'name': 'repetitions_table', + 'type': RepetitionsTable, + 'doc': 'the RepetitionsTable table that the repetitions column indexes', + 'default': None}, + *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames')) + def __init__(self, **kwargs): + repetitions_table = popargs('repetitions_table', kwargs) + # Define default name and description settings + kwargs['name'] = 'experimental_conditions' + kwargs['description'] = ('A table for grouping different intracellular recording repetitions together that ' + 'belong to the same experimental conditions.') + # Initialize the DynamicTable + call_docval_func(super().__init__, kwargs) + if self['repetitions'].target.table is None: + if repetitions_table is not None: + self['repetitions'].target.table = repetitions_table + else: + raise ValueError('repetitions_table constructor argument required') + + @docval({'name': 'repetitions', + 'type': 'array_data', + 'doc': 'the indices of the repetitions belonging to this condition', + 'default': None}, + returns='Integer index of the row that was added to this table', + rtype=int, + allow_extra=True) + def add_experimental_condition(self, **kwargs): + """ + Add a condition (i.e., one row) consisting of one or more repetitions of sequential recordings + and associated custom experimental_conditions metadata to the table. + """ + _ = super().add_row(enforce_unique_id=True, **kwargs) + return len(self.id) - 1 diff --git a/src/pynwb/image.py b/src/pynwb/image.py index 666177296..ba26113a3 100644 --- a/src/pynwb/image.py +++ b/src/pynwb/image.py @@ -1,7 +1,8 @@ import warnings +import numpy as np from collections.abc import Iterable -from hdmf.utils import docval, popargs, call_docval_func, get_docval +from hdmf.utils import docval, getargs, popargs, call_docval_func, get_docval from . import register_class, CORE_NAMESPACE from .base import TimeSeries, Image @@ -21,18 +22,27 @@ class ImageSeries(TimeSeries): 'format', 'device') + # value used when an ImageSeries is read and missing data + DEFAULT_DATA = np.ndarray(shape=(0, 0, 0), dtype=np.uint8) + # TODO: copy new docs from 2.4 schema + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4), 'doc': ('The data values. Can be 3D or 4D. The first dimension must be time (frame). The second and third ' - 'dimensions represent x and y. The optional fourth dimension represents z.'), + 'dimensions represent x and y. The optional fourth dimension represents z. Either data or ' + 'external_file must be specified (not None), but not both. If data is not specified, ' + 'data will be set to an empty 3D array.'), + 'default': None}, + {'name': 'unit', 'type': str, + 'doc': ('The unit of measurement of the image data, e.g., values between 0 and 255. Required when data ' + 'is specified. If unit (and data) are not specified, then unit will be set to "unknown".'), 'default': None}, - *get_docval(TimeSeries.__init__, 'unit'), {'name': 'format', 'type': str, 'doc': 'Format of image. Three types: 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.', 'default': None}, {'name': 'external_file', 'type': ('array_data', 'data'), 'doc': 'Path or URL to one or more external file(s). Field only present if format=external. ' - 'Either external_file or data must be specified, but not both.', 'default': None}, + 'Either external_file or data must be specified (not None), but not both.', 'default': None}, {'name': 'starting_frame', 'type': Iterable, 'doc': 'Each entry is the frame number in the corresponding external_file variable. ' 'This serves as an index to what frames each file contains. If external_file is not ' @@ -48,10 +58,22 @@ class ImageSeries(TimeSeries): def __init__(self, **kwargs): bits_per_pixel, dimension, external_file, starting_frame, format, device = popargs( 'bits_per_pixel', 'dimension', 'external_file', 'starting_frame', 'format', 'device', kwargs) - call_docval_func(super(ImageSeries, self).__init__, kwargs) - if external_file is None and self.data is None: + name, data, unit = getargs('name', 'data', 'unit', kwargs) + if data is not None and unit is None: + raise ValueError("Must supply 'unit' argument when supplying 'data' to %s '%s'." + % (self.__class__.__name__, name)) + if external_file is None and data is None: raise ValueError("Must supply either external_file or data to %s '%s'." - % (self.__class__.__name__, self.name)) + % (self.__class__.__name__, name)) + + # data and unit are required in TimeSeries, but allowed to be None here, so handle this specially + if data is None: + kwargs['data'] = ImageSeries.DEFAULT_DATA + if unit is None: + kwargs['unit'] = ImageSeries.DEFAULT_UNIT + + call_docval_func(super(ImageSeries, self).__init__, kwargs) + self.bits_per_pixel = bits_per_pixel self.dimension = dimension self.external_file = external_file @@ -76,7 +98,7 @@ def bits_per_pixel(self, val): @register_class('IndexSeries', CORE_NAMESPACE) class IndexSeries(TimeSeries): ''' - Stores indices to image frames stored in an ImageSeries. The purpose of the ImageIndexSeries is to allow + Stores indices to image frames stored in an ImageSeries. The purpose of the IndexSeries is to allow a static image stack to be stored somewhere, and the images in the stack to be referenced out-of-order. This can be for the display of individual images, or of movie segments (as a movie is simply a series of images). The data field stores the index of the frame in the referenced ImageSeries, and the timestamps @@ -85,10 +107,13 @@ class IndexSeries(TimeSeries): __nwbfields__ = ('indexed_timeseries',) + # # value used when an ImageSeries is read and missing data + # DEFAULT_UNIT = 'N/A' + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ), # required - 'doc': ('The data values. Must be 1D, where the first dimension must be time (frame)')}, - *get_docval(TimeSeries.__init__, 'unit'), + 'doc': ('The data values. Must be 1D, where the first dimension must be time (frame)')}, + *get_docval(TimeSeries.__init__, 'unit'), # required {'name': 'indexed_timeseries', 'type': TimeSeries, # required 'doc': 'HDF5 link to TimeSeries containing images that are indexed.'}, *get_docval(TimeSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', @@ -111,12 +136,11 @@ class ImageMaskSeries(ImageSeries): __nwbfields__ = ('masked_imageseries',) @docval(*get_docval(ImageSeries.__init__, 'name'), # required - *get_docval(ImageSeries.__init__, 'data', 'unit'), {'name': 'masked_imageseries', 'type': ImageSeries, # required 'doc': 'Link to ImageSeries that mask is applied to.'}, - *get_docval(ImageSeries.__init__, 'format', 'external_file', 'starting_frame', 'bits_per_pixel', - 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', - 'description', 'control', 'control_description'), + *get_docval(ImageSeries.__init__, 'data', 'unit', 'format', 'external_file', 'starting_frame', + 'bits_per_pixel', 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', + 'rate', 'comments', 'description', 'control', 'control_description'), {'name': 'device', 'type': Device, 'doc': ('Device used to capture the mask data. This field will likely not be needed. ' 'The device used to capture the masked ImageSeries data should be stored in the ImageSeries.'), @@ -141,19 +165,20 @@ class OpticalSeries(ImageSeries): 'field_of_view', 'orientation') - @docval(*get_docval(ImageSeries.__init__, 'name'), - {'name': 'data', 'type': ('array_data', 'data'), 'shape': ([None] * 3, [None, None, None, 3]), - 'doc': ('Images presented to subject, either grayscale or RGB. May be 3D or 4D. The first dimension must ' - 'be time (frame). The second and third dimensions represent x and y. The optional fourth ' - 'dimension must be length 3 and represents the RGB value for color images.')}, - *get_docval(ImageSeries.__init__, 'unit', 'format'), + @docval(*get_docval(ImageSeries.__init__, 'name'), # required {'name': 'distance', 'type': 'float', 'doc': 'Distance from camera/monitor to target/eye.'}, # required {'name': 'field_of_view', 'type': ('array_data', 'data', 'TimeSeries'), 'shape': ((2, ), (3, )), # required 'doc': 'Width, height and depth of image, or imaged area (meters).'}, {'name': 'orientation', 'type': str, # required 'doc': 'Description of image relative to some reference frame (e.g., which way is up). ' 'Must also specify frame of reference.'}, - *get_docval(ImageSeries.__init__, 'external_file', 'starting_frame', 'bits_per_pixel', + {'name': 'data', 'type': ('array_data', 'data'), 'shape': ([None] * 3, [None, None, None, 3]), + 'doc': ('Images presented to subject, either grayscale or RGB. May be 3D or 4D. The first dimension must ' + 'be time (frame). The second and third dimensions represent x and y. The optional fourth ' + 'dimension must be length 3 and represents the RGB value for color images. Either data or ' + 'external_file must be specified, but not both.'), + 'default': None}, + *get_docval(ImageSeries.__init__, 'unit', 'format', 'external_file', 'starting_frame', 'bits_per_pixel', 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', 'description', 'control', 'control_description', 'device')) def __init__(self, **kwargs): diff --git a/src/pynwb/io/base.py b/src/pynwb/io/base.py index b1aaa940b..7067ddd06 100644 --- a/src/pynwb/io/base.py +++ b/src/pynwb/io/base.py @@ -72,10 +72,47 @@ def timestamps_carg(self, builder, manager): # # NOTE: it is not available when data is externally linked # and we haven't explicitly read that file - if tstamps_builder.builder.parent is not None: - target = tstamps_builder.builder + target = tstamps_builder.builder + if target.parent is not None: return manager.construct(target.parent) else: - return tstamps_builder.builder.data + return target.data else: return tstamps_builder.data + + @NWBContainerMapper.constructor_arg("data") + def data_carg(self, builder, manager): + # handle case where a TimeSeries is read and missing data + timeseries_cls = manager.get_cls(builder) + data_builder = builder.get('data') + if data_builder is None: + return timeseries_cls.DEFAULT_DATA + if isinstance(data_builder, LinkBuilder): + # NOTE: parent is not available when data is externally linked + # and we haven't explicitly read that file + target = data_builder.builder + if target.parent is not None: + return manager.construct(target.parent) + else: + return target.data + return data_builder.data + + @NWBContainerMapper.constructor_arg("unit") + def unit_carg(self, builder, manager): + # handle case where a TimeSeries is read and missing unit + timeseries_cls = manager.get_cls(builder) + data_builder = builder.get('data') + if data_builder is None: + return timeseries_cls.DEFAULT_UNIT + if isinstance(data_builder, LinkBuilder): + # NOTE: parent is not available when data is externally linked + # and we haven't explicitly read that file + target = data_builder.builder + if target.parent is not None: + data_builder = manager.construct(target.parent) + else: + data_builder = target + unit_value = data_builder.attributes.get('unit') + if unit_value is None: + return timeseries_cls.DEFAULT_UNIT + return unit_value diff --git a/src/pynwb/io/file.py b/src/pynwb/io/file.py index 2c629ab7d..ccff9c5b6 100644 --- a/src/pynwb/io/file.py +++ b/src/pynwb/io/file.py @@ -47,12 +47,18 @@ def __init__(self, spec): general_spec = self.spec.get_group('general') self.unmap(general_spec) + # map icephys metadata structures and tables icephys_spec = general_spec.get_group('intracellular_ephys') self.unmap(icephys_spec) self.map_spec('icephys_electrodes', icephys_spec.get_neurodata_type('IntracellularElectrode')) self.map_spec('sweep_table', icephys_spec.get_neurodata_type('SweepTable')) + self.map_spec('intracellular_recordings', icephys_spec.get_neurodata_type('IntracellularRecordingsTable')) + self.map_spec('icephys_simultaneous_recordings', icephys_spec.get_neurodata_type('SimultaneousRecordingsTable')) + self.map_spec('icephys_sequential_recordings', icephys_spec.get_neurodata_type('SequentialRecordingsTable')) + self.map_spec('icephys_repetitions', icephys_spec.get_neurodata_type('RepetitionsTable')) + self.map_spec('icephys_experimental_conditions', icephys_spec.get_neurodata_type('ExperimentalConditionsTable')) - # 'filtering' will be deprecated. add this mapping in the meantime + # 'filtering' has been deprecated. add this mapping in the meantime icephys_filtering_spec = icephys_spec.get_dataset('filtering') self.unmap(icephys_filtering_spec) self.map_spec('icephys_filtering', icephys_filtering_spec) diff --git a/src/pynwb/io/icephys.py b/src/pynwb/io/icephys.py index 308f869fe..0ed1a2c3b 100644 --- a/src/pynwb/io/icephys.py +++ b/src/pynwb/io/icephys.py @@ -1,7 +1,8 @@ from .. import register_map -from pynwb.icephys import SweepTable, VoltageClampSeries +from pynwb.icephys import SweepTable, VoltageClampSeries, IntracellularRecordingsTable from hdmf.common.io.table import DynamicTableMap +from hdmf.common.io.alignedtable import AlignedDynamicTableMap from .base import TimeSeriesMap @@ -26,3 +27,24 @@ def __init__(self, spec): for field in fields_with_unit: field_spec = self.spec.get_dataset(field) self.map_spec('%s__unit' % field, field_spec.get_attribute('unit')) + + +@register_map(IntracellularRecordingsTable) +class IntracellularRecordingsTableMap(AlignedDynamicTableMap): + """ + Customize the mapping for AlignedDynamicTable + """ + def __init__(self, spec): + super().__init__(spec) + + @DynamicTableMap.object_attr('electrodes') + def electrodes(self, container, manager): + return container.category_tables.get('electrodes', None) + + @DynamicTableMap.object_attr('stimuli') + def stimuli(self, container, manager): + return container.category_tables.get('stimuli', None) + + @DynamicTableMap.object_attr('responses') + def responses(self, container, manager): + return container.category_tables.get('responses', None) diff --git a/src/pynwb/io/image.py b/src/pynwb/io/image.py index bb6b0212e..afa47c35c 100644 --- a/src/pynwb/io/image.py +++ b/src/pynwb/io/image.py @@ -1,5 +1,4 @@ from .. import register_map - from ..image import ImageSeries from .base import TimeSeriesMap @@ -8,6 +7,6 @@ class ImageSeriesMap(TimeSeriesMap): def __init__(self, spec): - super(ImageSeriesMap, self).__init__(spec) + super().__init__(spec) external_file_spec = self.spec.get_dataset('external_file') self.map_spec('starting_frame', external_file_spec.get_attribute('starting_frame')) diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 31303edf1..0e43dfd52 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -12,19 +12,16 @@ @register_class('AnnotationSeries', CORE_NAMESPACE) class AnnotationSeries(TimeSeries): - """ - Stores text-based records about the experiment. To use the - AnnotationSeries, add records individually through - add_annotation() and then call finalize(). Alternatively, if - all annotations are already stored in a list, use set_data() - and set_timestamps() + """Stores text-based records about the experiment. + To use the AnnotationSeries, add records individually through add_annotation(). Alternatively, if all annotations + are already stored in a list or numpy array, set the data and timestamps in the constructor. """ __nwbfields__ = () @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None,), - 'doc': 'The data values over time. Must be 1D.', + 'doc': 'The annotations over time. Must be 1D.', 'default': list()}, *get_docval(TimeSeries.__init__, 'timestamps', 'comments', 'description')) def __init__(self, **kwargs): @@ -34,9 +31,7 @@ def __init__(self, **kwargs): @docval({'name': 'time', 'type': 'float', 'doc': 'The time for the annotation'}, {'name': 'annotation', 'type': str, 'doc': 'the annotation'}) def add_annotation(self, **kwargs): - ''' - Add an annotation - ''' + """Add an annotation.""" time, annotation = getargs('time', 'annotation', kwargs) self.fields['timestamps'].append(time) self.fields['data'].append(annotation) @@ -255,6 +250,9 @@ class DecompositionSeries(TimeSeries): {'name': 'bands', 'doc': 'the bands that the signal is decomposed into', 'child': True}) + # value used when a DecompositionSeries is read and missing data + DEFAULT_DATA = np.ndarray(shape=(0, 0, 0), dtype=np.uint8) + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), # required 'doc': ('The data values. Must be 3D, where the first dimension must be time, the second dimension must ' diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 2fc379e09..884b90a22 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 2fc379e09f66ddbde6c06947fa1b94dae1189990 +Subproject commit 884b90a22135ca4f111d727f7fee1a8a52b58633 diff --git a/src/pynwb/ophys.py b/src/pynwb/ophys.py index 3844ddf09..97e3b2468 100644 --- a/src/pynwb/ophys.py +++ b/src/pynwb/ophys.py @@ -267,8 +267,8 @@ def pixel_to_image(pixel_mask): """Converts a 2D pixel_mask of a ROI into an image_mask.""" image_matrix = np.zeros(np.shape(pixel_mask)) npmask = np.asarray(pixel_mask) - x_coords = npmask[:, 0].astype(np.int) - y_coords = npmask[:, 1].astype(np.int) + x_coords = npmask[:, 0].astype(np.int32) + y_coords = npmask[:, 1].astype(np.int32) weights = npmask[:, -1] image_matrix[y_coords, x_coords] = weights return image_matrix diff --git a/src/pynwb/testing/__init__.py b/src/pynwb/testing/__init__.py index fa8eb5c7c..e017f4a16 100644 --- a/src/pynwb/testing/__init__.py +++ b/src/pynwb/testing/__init__.py @@ -1,5 +1,6 @@ from hdmf.testing import TestCase, H5RoundTripMixin from .testh5io import NWBH5IOMixin, AcquisitionH5IOMixin from .utils import remove_test_file +from .icephys_testutils import create_icephys_stimulus_and_response, create_icephys_testfile CORE_NAMESPACE = 'core' diff --git a/src/pynwb/testing/icephys_testutils.py b/src/pynwb/testing/icephys_testutils.py new file mode 100644 index 000000000..732f312e6 --- /dev/null +++ b/src/pynwb/testing/icephys_testutils.py @@ -0,0 +1,177 @@ +""" +Module with helper functions to facilitate testing +""" +import numpy as np +from datetime import datetime +from dateutil.tz import tzlocal +from pynwb.file import NWBFile +from pynwb.icephys import (VoltageClampStimulusSeries, VoltageClampSeries) +from pynwb import NWBHDF5IO + + +def create_icephys_stimulus_and_response(sweep_number, electrode, randomize_data): + """ + Internal helper function to construct a dummy stimulus and response pair representing an + intracellular recording: + + :param sweep_number: Integer sweep number of the recording + :type sweep_number: int + :param electrode: Intracellular electrode used + :type electrode: pynwb.icephys.IntracellularElectrode + :param randomize_data: Randomize data values in the stimulus and response + :type randomize_data: bool + + :returns: Tuple of VoltageClampStimulusSeries with the stimulus and VoltageClampSeries with the response. + """ + stimulus = VoltageClampStimulusSeries( + name="ccss_"+str(sweep_number), + data=[1, 2, 3, 4, 5] if not randomize_data else np.random.rand(10), + starting_time=123.6 if not randomize_data else (np.random.rand() * 100), + rate=10e3 if not randomize_data else int(np.random.rand()*10) * 1000 + 1000., + electrode=electrode, + gain=0.1 if not randomize_data else np.random.rand(), + sweep_number=sweep_number) + # Create and ic-response + response = VoltageClampSeries( + name='vcs_'+str(sweep_number), + data=[0.1, 0.2, 0.3, 0.4, 0.5] if not randomize_data else np.random.rand(10), + conversion=1e-12, + resolution=np.nan, + starting_time=123.6 if not randomize_data else (np.random.rand() * 100), + rate=20e3 if not randomize_data else int(np.random.rand() * 20) * 1000. + 1000., + electrode=electrode, + gain=0.02 if not randomize_data else np.random.rand(), + capacitance_slow=100e-12, + resistance_comp_correction=70.0 if not randomize_data else 70.0 + np.random.rand(), + sweep_number=sweep_number) + return stimulus, response + + +def create_icephys_testfile(filename=None, add_custom_columns=True, randomize_data=True, with_missing_stimulus=True): + """ + Create a small but relatively complex icephys test file that + we can use for testing of queries. + + :param filename: The name of the output file to be generated. If set to None then the file is not written + but only created in memory + :type filename: str, None + :param add_custom_colums: Add custom metadata columns to each table + :type add_custom_colums: bool + :param randomize_data: Randomize data values in the stimulus and response + :type randomize_data: bool + + :returns: ICEphysFile NWBFile object created for writing. NOTE: If filename is provided then + the file is written to disk, but the function does not read the file back. If + you want to use the file from disk then you will need to read it with NWBHDF5IO. + :rtype: ICEphysFile + """ + nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + experimenter='Dr. Bilbo Baggins', + lab='Bag End Laboratory', + institution='University of Middle Earth at the Shire', + experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.', + session_id='LONELYMTN') + # Add a device + device = nwbfile.create_device(name='Heka ITC-1600') + # Add an intracellular electrode + electrode0 = nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=device) + # Add an intracellular electrode + electrode1 = nwbfile.create_icephys_electrode( + name="elec1", + description='another mock intracellular electrode', + device=device) + # Add the intracelluar recordings + for sweep_number in range(20): + elec = (electrode0 if (sweep_number % 2 == 0) else electrode1) + stim, resp = create_icephys_stimulus_and_response(sweep_number=np.uint64(sweep_number), + electrode=elec, + randomize_data=randomize_data) + if with_missing_stimulus and sweep_number in [0, 10]: + stim = None + nwbfile.add_intracellular_recording(electrode=elec, + stimulus=stim, + response=resp, + id=sweep_number) + nwbfile.intracellular_recordings.add_column(name='recording_tags', + data=['A1', 'A2', + 'B1', 'B2', + 'C1', 'C2', 'C3', + 'D1', 'D2', 'D3', + 'A1', 'A2', + 'B1', 'B2', + 'C1', 'C2', 'C3', + 'D1', 'D2', 'D3'], + description='String with a set of recording tags') + # Add simultaneous_recordings + nwbfile.add_icephys_simultaneous_recording(recordings=[0, 1], id=np.int64(100)) + nwbfile.add_icephys_simultaneous_recording(recordings=[2, 3], id=np.int64(101)) + nwbfile.add_icephys_simultaneous_recording(recordings=[4, 5, 6], id=np.int64(102)) + nwbfile.add_icephys_simultaneous_recording(recordings=[7, 8, 9], id=np.int64(103)) + nwbfile.add_icephys_simultaneous_recording(recordings=[10, 11], id=np.int64(104)) + nwbfile.add_icephys_simultaneous_recording(recordings=[12, 13], id=np.int64(105)) + nwbfile.add_icephys_simultaneous_recording(recordings=[14, 15, 16], id=np.int64(106)) + nwbfile.add_icephys_simultaneous_recording(recordings=[17, 18, 19], id=np.int64(107)) + if add_custom_columns: + nwbfile.icephys_simultaneous_recordings.add_column( + name='tag', + data=np.arange(8), + description='some integer tag for a sweep') + + # Add sequential recordings + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[0, 1], + id=np.int64(1000), + stimulus_type="StimType_1") + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[2, ], + id=np.int64(1001), + stimulus_type="StimType_2") + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[3, ], + id=np.int64(1002), + stimulus_type="StimType_3") + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[4, 5], + id=np.int64(1003), + stimulus_type="StimType_1") + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[6, ], + id=np.int64(1004), + stimulus_type="StimType_2") + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[7, ], + id=np.int64(1005), + stimulus_type="StimType_3") + if add_custom_columns: + nwbfile.icephys_sequential_recordings.add_column( + name='type', + data=['T1', 'T2', 'T3', 'T1', 'T2', 'T3'], + description='type of the sequential recording') + + # Add repetitions + nwbfile.add_icephys_repetition(sequential_recordings=[0, ], id=np.int64(10000)) + nwbfile.add_icephys_repetition(sequential_recordings=[1, 2], id=np.int64(10001)) + nwbfile.add_icephys_repetition(sequential_recordings=[3, ], id=np.int64(10002)) + nwbfile.add_icephys_repetition(sequential_recordings=[4, 5], id=np.int64(10003)) + if add_custom_columns: + nwbfile.icephys_repetitions.add_column( + name='type', + data=['R1', 'R2', 'R1', 'R2'], + description='some repetition type indicator') + + # Add experimental_conditions + nwbfile.add_icephys_experimental_condition(repetitions=[0, 1], id=np.int64(100000)) + nwbfile.add_icephys_experimental_condition(repetitions=[2, 3], id=np.int64(100001)) + if add_custom_columns: + nwbfile.icephys_experimental_conditions.add_column( + name='temperature', + data=[32., 24.], + description='Temperatur in C') + + # Write our test file + if filename is not None: + with NWBHDF5IO(filename, 'w') as io: + io.write(nwbfile) + + # Return our in-memory NWBFile + return nwbfile diff --git a/src/pynwb/testing/make_test_files.py b/src/pynwb/testing/make_test_files.py index 161db19d0..6dcbf148d 100644 --- a/src/pynwb/testing/make_test_files.py +++ b/src/pynwb/testing/make_test_files.py @@ -1,10 +1,13 @@ -from pynwb import NWBFile, NWBHDF5IO, validate, __version__ +import numpy as np + from datetime import datetime +from pynwb import NWBFile, NWBHDF5IO, __version__, TimeSeries +from pynwb.image import ImageSeries # pynwb 1.0.2 should be installed with hdmf 1.0.3 # pynwb 1.0.3 should be installed with hdmf 1.0.5 # pynwb 1.1.0 should be installed with hdmf 1.2.0 -# pynwb 1.1.1+ should be installed with an appopriate version of hdmf +# pynwb 1.1.1+ should be installed with an appropriate version of hdmf def _write(test_name, nwbfile): @@ -13,12 +16,10 @@ def _write(test_name, nwbfile): with NWBHDF5IO(filename, 'w') as io: io.write(nwbfile) - with NWBHDF5IO(filename, 'r') as io: - validate(io) - nwbfile = io.read() + return filename -def make_nwbfile(): +def make_nwbfile_empty(): nwbfile = NWBFile(session_description='ADDME', identifier='ADDME', session_start_time=datetime.now().astimezone()) @@ -44,7 +45,83 @@ def make_nwbfile_str_pub(): _write(test_name, nwbfile) +def make_nwbfile_timeseries_no_data(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + ts = TimeSeries( + name='test_timeseries', + rate=1., + unit='unit', + ) + nwbfile.add_acquisition(ts) + + test_name = 'timeseries_no_data' + _write(test_name, nwbfile) + + +def make_nwbfile_timeseries_no_unit(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + ts = TimeSeries( + name='test_timeseries', + data=[0], + rate=1., + ) + nwbfile.add_acquisition(ts) + + test_name = 'timeseries_no_unit' + _write(test_name, nwbfile) + + +def make_nwbfile_imageseries_no_data(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + image_series = ImageSeries( + name='test_imageseries', + external_file=['external_file'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=[1., 2., 3.] + ) + + nwbfile.add_acquisition(image_series) + + test_name = 'imageseries_no_data' + _write(test_name, nwbfile) + + +def make_nwbfile_imageseries_no_unit(): + """Create a test file with an ImageSeries with data and no unit.""" + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + image_series = ImageSeries( + name='test_imageseries', + data=np.ones((3, 3, 3)), + external_file=['external_file'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=[1., 2., 3.] + ) + + nwbfile.add_acquisition(image_series) + + test_name = 'imageseries_no_unit' + _write(test_name, nwbfile) + + if __name__ == '__main__': - make_nwbfile() - make_nwbfile_str_experimenter() - make_nwbfile_str_pub() + + if __version__ == '1.1.2': + make_nwbfile_empty() + make_nwbfile_str_experimenter() + make_nwbfile_str_pub() + + if __version__ == '1.5.1': + make_nwbfile_timeseries_no_data() + make_nwbfile_timeseries_no_unit() + make_nwbfile_imageseries_no_data() + make_nwbfile_imageseries_no_unit() diff --git a/src/pynwb/testing/testh5io.py b/src/pynwb/testing/testh5io.py index eecc2da27..3627e3ad0 100644 --- a/src/pynwb/testing/testh5io.py +++ b/src/pynwb/testing/testh5io.py @@ -58,9 +58,7 @@ def setUpContainer(self): raise NotImplementedError('Cannot run test unless setUpContainer is implemented') def test_roundtrip(self): - """ - Test whether the test Container read from file has the same contents as the original test Container and - validate the file + """Test whether the read Container has the same contents as the original Container and validate the file. """ self.read_container = self.roundtripContainer() self.assertIsNotNone(str(self.container)) # added as a test to make sure printing works @@ -85,9 +83,7 @@ def test_roundtrip_export(self): self.assertContainerEqual(self.read_container, self.container, ignore_hdmf_attrs=True) def roundtripContainer(self, cache_spec=False): - """ - Add the test Container to an NWBFile, write it to file, read the file, and return the test Container from the - file + """Add the Container to an NWBFile, write it to file, read the file, and return the Container from the file. """ description = 'a file to test writing and reading a %s' % self.container_type identifier = 'TEST_%s' % self.container_type diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index b4c323f1d..f3f127b19 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -26,7 +26,7 @@ def _validate_helper(**kwargs): return (errors is not None and len(errors) > 0) -def main(): +def main(): # noqa: C901 ep = """ use --nspath to validate against an extension. If --ns is not specified, @@ -111,15 +111,22 @@ def main(): if args.ns: if args.ns in namespaces: namespaces = [args.ns] + elif args.cached_namespace and args.ns in ns_deps: # validating against a dependency + for k in ns_deps: + if args.ns in ns_deps[k]: + print(("The namespace '{}' is included by the namespace '{}'. Please validate against " + "that namespace instead.").format(args.ns, k), file=sys.stderr) + ret = 1 + continue else: - print("The namespace {} could not be found in {} as only {} is present.".format( + print("The namespace '{}' could not be found in {} as only {} is present.".format( args.ns, specloc, namespaces), file=sys.stderr) ret = 1 continue with NWBHDF5IO(path, mode='r', manager=manager) as io: for ns in namespaces: - print("Validating {} against {} using namespace {}.".format(path, specloc, ns)) + print("Validating {} against {} using namespace '{}'.".format(path, specloc, ns)) ret = ret or _validate_helper(io=io, namespace=ns) sys.exit(ret) diff --git a/tests/back_compat/1.5.1_imageseries_no_data.nwb b/tests/back_compat/1.5.1_imageseries_no_data.nwb new file mode 100644 index 000000000..b7ca1dceb Binary files /dev/null and b/tests/back_compat/1.5.1_imageseries_no_data.nwb differ diff --git a/tests/back_compat/1.5.1_imageseries_no_unit.nwb b/tests/back_compat/1.5.1_imageseries_no_unit.nwb new file mode 100644 index 000000000..b2784c2c6 Binary files /dev/null and b/tests/back_compat/1.5.1_imageseries_no_unit.nwb differ diff --git a/tests/back_compat/1.5.1_timeseries_no_data.nwb b/tests/back_compat/1.5.1_timeseries_no_data.nwb new file mode 100644 index 000000000..f5a8bbe25 Binary files /dev/null and b/tests/back_compat/1.5.1_timeseries_no_data.nwb differ diff --git a/tests/back_compat/1.5.1_timeseries_no_unit.nwb b/tests/back_compat/1.5.1_timeseries_no_unit.nwb new file mode 100644 index 000000000..dbf17442f Binary files /dev/null and b/tests/back_compat/1.5.1_timeseries_no_unit.nwb differ diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index 731895c7d..d3a813ead 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -1,16 +1,30 @@ +import numpy as np from pathlib import Path import warnings -from pynwb import NWBHDF5IO, validate +from pynwb import NWBHDF5IO, validate, TimeSeries +from pynwb.image import ImageSeries from pynwb.testing import TestCase class TestReadOldVersions(TestCase): + expected_errors = { + '1.0.2_str_experimenter.nwb': [("root/general/experimenter (general/experimenter): incorrect shape - expected " + "an array of shape '[None]', got non-array data 'one experimenter'")], + '1.0.3_str_experimenter.nwb': [("root/general/experimenter (general/experimenter): incorrect shape - expected " + "an array of shape '[None]', got non-array data 'one experimenter'")], + '1.0.2_str_pub.nwb': [("root/general/related_publications (general/related_publications): incorrect shape " + "- expected an array of shape '[None]', got non-array data 'one publication'")], + '1.0.3_str_pub.nwb': [("root/general/related_publications (general/related_publications): incorrect shape " + "- expected an array of shape '[None]', got non-array data 'one publication'")], + } + def test_read(self): - """ - Attempt to read and validate all NWB files in the same folder as this file. The folder should contain NWB files - from previous versions of NWB. See src/pynwb/testing/make_test_files.py for code to generate the NWB files. + """Test reading and validating all NWB files in the same folder as this file. + + This folder contains NWB files generated by previous versions of NWB using the script + src/pynwb/testing/make_test_files.py """ dir_path = Path(__file__).parent nwb_files = dir_path.glob('*.nwb') @@ -21,6 +35,35 @@ def test_read(self): io.read() if errors: for e in errors: - warnings.warn('%s: %s' % (f.name, e)) + if f.name in self.expected_errors and str(e) not in self.expected_errors[f.name]: + warnings.warn('%s: %s' % (f.name, e)) # TODO uncomment below when validation errors have been fixed # raise Exception('%d validation error(s). See warnings.' % len(errors)) + + def test_read_timeseries_no_data(self): + """Test that a TimeSeries written without data is read with data set to the default value.""" + f = Path(__file__).parent / '1.5.1_timeseries_no_data.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + np.testing.assert_array_equal(read_nwbfile.acquisition['test_timeseries'].data, TimeSeries.DEFAULT_DATA) + + def test_read_timeseries_no_unit(self): + """Test that an ImageSeries written without unit is read with unit set to the default value.""" + f = Path(__file__).parent / '1.5.1_timeseries_no_unit.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + self.assertEqual(read_nwbfile.acquisition['test_timeseries'].unit, TimeSeries.DEFAULT_UNIT) + + def test_read_imageseries_no_data(self): + """Test that an ImageSeries written without data is read with data set to the default value.""" + f = Path(__file__).parent / '1.5.1_imageseries_no_data.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].data, ImageSeries.DEFAULT_DATA) + + def test_read_imageseries_no_unit(self): + """Test that an ImageSeries written without unit is read with unit set to the default value.""" + f = Path(__file__).parent / '1.5.1_imageseries_no_unit.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + self.assertEqual(read_nwbfile.acquisition['test_imageseries'].unit, ImageSeries.DEFAULT_UNIT) diff --git a/tests/integration/hdf5/test_ecephys.py b/tests/integration/hdf5/test_ecephys.py index 0ea69c233..cc70ee9dc 100644 --- a/tests/integration/hdf5/test_ecephys.py +++ b/tests/integration/hdf5/test_ecephys.py @@ -150,6 +150,11 @@ def roundtripContainer(self, cache_spec=False): with self.assertWarnsWith(DeprecationWarning, 'use pynwb.misc.Units or NWBFile.units instead'): return super().roundtripContainer(cache_spec) + def roundtripExportContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the Clustering object from file + with self.assertWarnsWith(DeprecationWarning, 'use pynwb.misc.Units or NWBFile.units instead'): + return super().roundtripExportContainer(cache_spec) + class EventWaveformConstructor(AcquisitionH5IOMixin, TestCase): @@ -200,6 +205,11 @@ def roundtripContainer(self, cache_spec=False): with self.assertWarnsWith(DeprecationWarning, 'use pynwb.misc.Units or NWBFile.units instead'): return super().roundtripContainer(cache_spec) + def roundtripExportContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the Clustering object from file + with self.assertWarnsWith(DeprecationWarning, 'use pynwb.misc.Units or NWBFile.units instead'): + return super().roundtripExportContainer(cache_spec) + class FeatureExtractionConstructor(AcquisitionH5IOMixin, TestCase): diff --git a/tests/integration/hdf5/test_icephys.py b/tests/integration/hdf5/test_icephys.py index 564f0a342..490b06d68 100644 --- a/tests/integration/hdf5/test_icephys.py +++ b/tests/integration/hdf5/test_icephys.py @@ -6,6 +6,7 @@ VoltageClampSeries, IZeroClampSeries) from pynwb.device import Device from pynwb.testing import NWBH5IOMixin, AcquisitionH5IOMixin, TestCase +import warnings class TestIntracellularElectrode(NWBH5IOMixin, TestCase): @@ -128,7 +129,14 @@ def setUpContainer(self): self.pcs = PatchClampSeries(name="pcs", data=[1, 2, 3, 4, 5], unit='A', starting_time=123.6, rate=10e3, electrode=self.elec, gain=0.126, stimulus_description="gotcha ya!", sweep_number=np.uint(4711)) - return SweepTable(name='sweep_table') + # Create the SweepTable but ignore the DeprecationWarning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('ignore', DeprecationWarning) + sweeptable = SweepTable(name='sweep_table') + # Reissue any other warnings that may have occured + for i in w: + warnings.warn(i.message, i.category) + return sweeptable def addContainer(self, nwbfile): """ @@ -137,12 +145,26 @@ def addContainer(self, nwbfile): nwbfile.sweep_table = self.container nwbfile.add_device(self.device) nwbfile.add_icephys_electrode(self.elec) - nwbfile.add_acquisition(self.pcs) + nwbfile.add_acquisition(self.pcs, use_sweep_table=True) def getContainer(self, nwbfile): """ Return the test SweepTable from the given NWBFile """ return nwbfile.sweep_table + def roundtripContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the SweepTable object from file + with self.assertWarnsWith(DeprecationWarning, + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable instead. " + "See also the NWBFile.add_intracellular_recordings function."): + return super().roundtripContainer(cache_spec) + + def roundtripExportContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the SweepTable object from file + with self.assertWarnsWith(DeprecationWarning, + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable instead. " + "See also the NWBFile.add_intracellular_recordings function."): + return super().roundtripExportContainer(cache_spec) + def test_container(self): """ Test properties of the SweepTable read from file """ description = 'a file to test writing and reading a %s' % self.container_type @@ -178,7 +200,14 @@ def setUpContainer(self): starting_time=123.6, rate=10e3, electrode=self.elec, gain=0.126, stimulus_description="gotcha ya!", sweep_number=np.uint(4712)) - return SweepTable(name='sweep_table') + # Create the SweepTable but ignore the DeprecationWarning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('ignore', DeprecationWarning) + sweeptable = SweepTable(name='sweep_table') + # Reissue any other warnings that may have occured + for i in w: + warnings.warn(i.message, i.category) + return sweeptable def addContainer(self, nwbfile): """ @@ -188,14 +217,28 @@ def addContainer(self, nwbfile): nwbfile.add_device(self.device) nwbfile.add_icephys_electrode(self.elec) - nwbfile.add_acquisition(self.pcs1) - nwbfile.add_stimulus_template(self.pcs2a) - nwbfile.add_stimulus(self.pcs2b) + nwbfile.add_acquisition(self.pcs1, use_sweep_table=True) + nwbfile.add_stimulus_template(self.pcs2a, use_sweep_table=True) + nwbfile.add_stimulus(self.pcs2b, use_sweep_table=True) def getContainer(self, nwbfile): """ Return the test SweepTable from the given NWBFile """ return nwbfile.sweep_table + def roundtripContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the SweepTable object from file + with self.assertWarnsWith(DeprecationWarning, + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable instead. " + "See also the NWBFile.add_intracellular_recordings function."): + return super().roundtripContainer(cache_spec) + + def roundtripExportContainer(self, cache_spec=False): + # catch the DeprecationWarning raised when reading the SweepTable object from file + with self.assertWarnsWith(DeprecationWarning, + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable instead. " + "See also the NWBFile.add_intracellular_recordings function."): + return super().roundtripExportContainer(cache_spec) + def test_container(self): """ Test properties of the SweepTable read from file """ description = 'a file to test writing and reading a %s' % self.container_type diff --git a/tests/integration/hdf5/test_misc.py b/tests/integration/hdf5/test_misc.py index c4a371532..3bfa42a33 100644 --- a/tests/integration/hdf5/test_misc.py +++ b/tests/integration/hdf5/test_misc.py @@ -15,7 +15,7 @@ class TestUnitsIO(AcquisitionH5IOMixin, TestCase): def setUpContainer(self): """ Return the test Units to read/write """ ut = Units(name='UnitsTest', description='a simple table for testing Units') - ut.add_unit(spike_times=[0, 1, 2], obs_intervals=[[0, 1], [2, 3]], + ut.add_unit(spike_times=[0., 1., 2.], obs_intervals=[[0., 1.], [2., 3.]], waveform_mean=[1., 2., 3.], waveform_sd=[4., 5., 6.], waveforms=[ [ # elec 1 @@ -28,7 +28,7 @@ def setUpContainer(self): [1, 2, 3] ] ]) - ut.add_unit(spike_times=[3, 4, 5], obs_intervals=[[2, 5], [6, 7]], + ut.add_unit(spike_times=[3., 4., 5.], obs_intervals=[[2., 5.], [6., 7.]], waveform_mean=[1., 2., 3.], waveform_sd=[4., 5., 6.], waveforms=np.array([ [ # elec 1 @@ -56,19 +56,19 @@ def test_get_spike_times(self): """ Test whether the Units spike times read from file are what was written """ ut = self.roundtripContainer() received = ut.get_unit_spike_times(0) - self.assertTrue(np.array_equal(received, [0, 1, 2])) + self.assertTrue(np.array_equal(received, [0., 1., 2.])) received = ut.get_unit_spike_times(1) - self.assertTrue(np.array_equal(received, [3, 4, 5])) - self.assertTrue(np.array_equal(ut['spike_times'][:], [[0, 1, 2], [3, 4, 5]])) + self.assertTrue(np.array_equal(received, [3., 4., 5.])) + self.assertTrue(np.array_equal(ut['spike_times'][:], [[0., 1., 2.], [3., 4., 5.]])) def test_get_obs_intervals(self): """ Test whether the Units observation intervals read from file are what was written """ ut = self.roundtripContainer() received = ut.get_unit_obs_intervals(0) - self.assertTrue(np.array_equal(received, [[0, 1], [2, 3]])) + self.assertTrue(np.array_equal(received, [[0., 1.], [2., 3.]])) received = ut.get_unit_obs_intervals(1) - self.assertTrue(np.array_equal(received, [[2, 5], [6, 7]])) - self.assertTrue(np.array_equal(ut['obs_intervals'][:], [[[0, 1], [2, 3]], [[2, 5], [6, 7]]])) + self.assertTrue(np.array_equal(received, [[2., 5.], [6., 7.]])) + self.assertTrue(np.array_equal(ut['obs_intervals'][:], [[[0., 1.], [2., 3.]], [[2., 5.], [6., 7.]]])) class TestUnitsFileIO(NWBH5IOMixin, TestCase): diff --git a/tests/integration/hdf5/test_ophys.py b/tests/integration/hdf5/test_ophys.py index 6507efccf..e882cfaf8 100644 --- a/tests/integration/hdf5/test_ophys.py +++ b/tests/integration/hdf5/test_ophys.py @@ -175,9 +175,14 @@ def buildPlaneSegmentation(self): (7, 8, 2.0), (9, 10, 2.)] ts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] - self.image_series = ImageSeries(name='test_iS', dimension=[2], - external_file=['images.tiff'], - starting_frame=[1, 2, 3], format='tiff', timestamps=ts) + self.image_series = ImageSeries( + name='test_iS', + dimension=[2], + external_file=['images.tiff'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=ts + ) self.device = Device(name='dev1') self.optical_channel = OpticalChannel( @@ -239,9 +244,14 @@ class MaskIO(TestPlaneSegmentationIO, metaclass=ABCMeta): def buildPlaneSegmentationNoRois(self): """ Return an PlaneSegmentation and set related objects """ ts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] - self.image_series = ImageSeries(name='test_iS', dimension=[2], - external_file=['images.tiff'], - starting_frame=[1, 2, 3], format='tiff', timestamps=ts) + self.image_series = ImageSeries( + name='test_iS', + dimension=[2], + external_file=['images.tiff'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=ts + ) self.device = Device(name='dev1') self.optical_channel = OpticalChannel( name='test_optical_channel', diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index c15af44ec..dc0869f0d 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -1,6 +1,6 @@ import numpy as np -from pynwb.base import ProcessingModule, TimeSeries, Images, Image +from pynwb.base import ProcessingModule, TimeSeries, Images, Image, TimeSeriesReferenceVectorData, TimeSeriesReference from pynwb.testing import TestCase from hdmf.data_utils import DataChunkIterator from hdmf.backends.hdf5 import H5DataIO @@ -136,11 +136,6 @@ def test_bad_continuity_timeseries(self): 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], continuity='wrong') - def test_nodata(self): - ts1 = TimeSeries('test_ts1', starting_time=0.0, rate=0.1) - with self.assertWarns(UserWarning): - self.assertIs(ts1.num_samples, None) - def test_dataio_list_data(self): length = 100 data = list(range(length)) @@ -197,7 +192,7 @@ def test_no_time(self): def test_no_starting_time(self): # if no starting_time is given, 0.0 is assumed - ts1 = TimeSeries('test_ts1', rate=0.1) + ts1 = TimeSeries('test_ts1', data=[1, 2, 3], unit='unit', rate=0.1) self.assertEqual(ts1.starting_time, 0.0) def test_conflicting_time_args(self): @@ -221,3 +216,227 @@ def test_images(self): image = Image(name='test_image', data=np.ones((10, 10))) image2 = Image(name='test_image2', data=np.ones((10, 10))) Images(name='images_name', images=[image, image2]) + + +class TestTimeSeriesReferenceVectorData(TestCase): + + def test_init(self): + temp = TimeSeriesReferenceVectorData() + self.assertEqual(temp.name, 'timeseries') + self.assertEqual(temp.description, + "Column storing references to a TimeSeries (rows). For each TimeSeries this " + "VectorData column stores the start_index and count to indicate the range in time " + "to be selected as well as an object reference to the TimeSeries.") + self.assertListEqual(temp.data, []) + temp = TimeSeriesReferenceVectorData(name='test', description='test') + self.assertEqual(temp.name, 'test') + self.assertEqual(temp.description, 'test') + + def test_get_empty(self): + """Get data from an empty TimeSeriesReferenceVectorData""" + temp = TimeSeriesReferenceVectorData() + self.assertListEqual(temp[:], []) + with self.assertRaises(IndexError): + temp[0] + + def test_get_length1_valid_data(self): + """Get data from a TimeSeriesReferenceVectorData with one element and valid data""" + temp = TimeSeriesReferenceVectorData() + value = TimeSeriesReference(0, 5, TimeSeries(name='test', description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + temp.append(value) + self.assertTupleEqual(temp[0], value) + self.assertListEqual(temp[:], [TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*value), ]) + + def test_get_length1_invalid_data(self): + """Get data from a TimeSeriesReferenceVectorData with one element and invalid data""" + temp = TimeSeriesReferenceVectorData() + value = TimeSeriesReference(-1, -1, TimeSeries(name='test', description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + temp.append(value) + # test index slicing + re = temp[0] + self.assertTrue(isinstance(re, TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + self.assertTupleEqual(re, TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE) + # test array slicing and list slicing + selection = [slice(None), [0, ]] + for s in selection: + re = temp[s] + self.assertTrue(isinstance(re, list)) + self.assertTrue(len(re), 1) + self.assertTrue(isinstance(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE) + + def test_get_length5_valid_data(self): + """Get data from a TimeSeriesReferenceVectorData with 5 elements""" + temp = TimeSeriesReferenceVectorData() + num_values = 5 + values = [TimeSeriesReference(0, 5, TimeSeries(name='test'+str(i), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + for i in range(num_values)] + for v in values: + temp.append(v) + # Test single element selection + for i in range(num_values): + # test index slicing + re = temp[i] + self.assertTupleEqual(re, values[i]) + # test slicing + re = temp[i:i+1] + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*values[i])) + # Test multi element selection + re = temp[0:2] + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*values[0])) + self.assertTupleEqual(re[1], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*values[1])) + + def test_get_length5_with_invalid_data(self): + """Get data from a TimeSeriesReferenceVectorData with 5 elements""" + temp = TimeSeriesReferenceVectorData() + num_values = 5 + values = [TimeSeriesReference(0, 5, TimeSeries(name='test'+str(i+1), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + for i in range(num_values-2)] + values = ([TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, + rate=0.1)), ] + + values + + [TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(5), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)), ]) + for v in values: + temp.append(v) + # Test single element selection + for i in range(num_values): + # test index slicing + re = temp[i] + if i in [0, 4]: + self.assertTrue(isinstance(re, TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + self.assertTupleEqual(re, TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE) + else: + self.assertTupleEqual(re, values[i]) + # test slicing + re = temp[i:i+1] + if i in [0, 4]: + self.assertTrue(isinstance(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE) + else: + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*values[i])) + # Test multi element selection + re = temp[0:2] + self.assertTupleEqual(re[0], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE) + self.assertTupleEqual(re[1], TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*values[1])) + + def test_add_row_restricted_type(self): + v = TimeSeriesReferenceVectorData(name='a', description='a') + with self.assertRaisesWith(TypeError, "TimeSeriesReferenceVectorData.add_row: incorrect type for " + "'val' (got 'int', expected 'TimeSeriesReference')"): + v.add_row(1) + + def test_append_restricted_type(self): + v = TimeSeriesReferenceVectorData(name='a', description='a') + with self.assertRaisesWith(TypeError, "TimeSeriesReferenceVectorData.append: incorrect type for " + "'arg' (got 'float', expected 'TimeSeriesReference')"): + v.append(2.0) + + +class TestTimeSeriesReference(TestCase): + + def test_check_types(self): + # invalid selection but with correct types + tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + self.assertTrue(tsr.check_types()) + # invalid types, use float instead of int for both idx_start and count + tsr = TimeSeriesReference(1.0, 5.0, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(TypeError, "idx_start must be an integer not "): + tsr.check_types() + # invalid types, use float instead of int for idx_start only + tsr = TimeSeriesReference(1.0, 5, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(TypeError, "idx_start must be an integer not "): + tsr.check_types() + # invalid types, use float instead of int for count only + tsr = TimeSeriesReference(1, 5.0, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(TypeError, "count must be an integer "): + tsr.check_types() + # invalid type for TimeSeries but valid idx_start and count + tsr = TimeSeriesReference(1, 5, None) + with self.assertRaisesWith(TypeError, "timeseries must be of type TimeSeries. "): + tsr.check_types() + + def test_is_invalid(self): + tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + self.assertFalse(tsr.isvalid()) + + def test_is_valid(self): + tsr = TimeSeriesReference(0, 10, TimeSeries(name='test'+str(0), description='test', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + self.assertTrue(tsr.isvalid()) + + def test_is_valid_bad_index(self): + # Error: negative start_index but positive count + tsr = TimeSeriesReference(-1, 10, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'idx_start' -1 out of range for timeseries 'test0'"): + tsr.isvalid() + # Error: start_index too large + tsr = TimeSeriesReference(10, 0, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'idx_start' 10 out of range for timeseries 'test0'"): + tsr.isvalid() + # Error: positive start_index but negative count + tsr = TimeSeriesReference(0, -3, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'count' -3 invalid. 'count' must be positive"): + tsr.isvalid() + # Error: start_index + count too large + tsr = TimeSeriesReference(3, 10, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): + tsr.isvalid() + + def test_timestamps_property(self): + # Timestamps from starting_time and rate + tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + np.testing.assert_array_equal(tsr.timestamps, np.array([5.5, 5.6, 5.7, 5.8])) + # Timestamps from timestamps directly + tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', + timestamps=np.arange(10).astype(float))) + np.testing.assert_array_equal(tsr.timestamps, np.array([5., 6., 7., 8.])) + + def test_timestamps_property_invalid_reference(self): + # Timestamps from starting_time and rate + tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + self.assertIsNone(tsr.timestamps) + + def test_timestamps_property_bad_reference(self): + tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', + timestamps=np.arange(10).astype(float))) + with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): + tsr.timestamps + tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): + tsr.timestamps + + def test_data_property(self): + tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + np.testing.assert_array_equal(tsr.data, np.array([5., 6., 7., 8.])) + + def test_data_property_invalid_reference(self): + tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + self.assertIsNone(tsr.data) + + def test_data_property_bad_reference(self): + tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) + with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): + tsr.data diff --git a/tests/unit/test_behavior.py b/tests/unit/test_behavior.py index 0702c9aa1..058e63ec0 100644 --- a/tests/unit/test_behavior.py +++ b/tests/unit/test_behavior.py @@ -15,6 +15,13 @@ def test_init(self): self.assertEqual(sS.reference_frame, 'reference_frame') +class SpatialSeriesConstructorChangeableUnit(TestCase): + def test_init(self): + sS = SpatialSeries('test_sS', np.ones((2, 2)), 'reference_frame', 'degrees', + timestamps=[1., 2., 3.]) + self.assertEqual(sS.unit, 'degrees') + + class BehavioralEpochsConstructor(TestCase): def test_init(self): data = [0, 1, 0, 1] diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index ee599b00e..ef95a058e 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -41,9 +41,19 @@ class TestPrint(TestCase): def test_print_file(self): nwbfile = NWBFile(session_description='session_description', identifier='identifier', session_start_time=datetime.now(tzlocal())) - ts = TimeSeries('name', [1., 2., 3.] * 1000, timestamps=[1, 2, 3]) - ts2 = TimeSeries('name2', [1, 2, 3] * 1000, timestamps=[1, 2, 3]) - expected = """name pynwb.base.TimeSeries at 0x%d + ts1 = TimeSeries( + name='name1', + data=[1., 2., 3.] * 1000, + unit='unit', + timestamps=[1, 2, 3] + ) + ts2 = TimeSeries( + name='name2', + data=[1, 2, 3] * 1000, + unit='unit', + timestamps=[1, 2, 3] + ) + expected = """name1 pynwb.base.TimeSeries at 0x%d Fields: comments: no comments conversion: 1.0 @@ -53,16 +63,17 @@ def test_print_file(self): resolution: -1.0 timestamps: [1 2 3] timestamps_unit: seconds + unit: unit """ - expected %= id(ts) - self.assertEqual(str(ts), expected) - nwbfile.add_acquisition(ts) + expected %= id(ts1) + self.assertEqual(str(ts1), expected) + nwbfile.add_acquisition(ts1) nwbfile.add_acquisition(ts2) nwbfile.add_epoch(start_time=1.0, stop_time=10.0, tags=['tag1', 'tag2']) expected_re = r"""root pynwb\.file\.NWBFile at 0x\d+ Fields: acquisition: { - name , + name1 , name2 } epoch_tags: { diff --git a/tests/unit/test_epoch.py b/tests/unit/test_epoch.py index 774edc41f..b525356d1 100644 --- a/tests/unit/test_epoch.py +++ b/tests/unit/test_epoch.py @@ -11,9 +11,9 @@ class TimeIntervalsTest(TestCase): def test_init(self): - tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float) - ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) - ept = TimeIntervals('epochs', "TimeIntervals unittest") + tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64) + ts = TimeSeries(name="test_ts", data=list(range(len(tstamps))), unit='unit', timestamps=tstamps) + ept = TimeIntervals(name='epochs', description="TimeIntervals unittest") self.assertEqual(ept.name, 'epochs') ept.add_interval(10.0, 20.0, ["test", "unittest", "pynwb"], ts) row = ept[0] @@ -25,8 +25,8 @@ def test_init(self): def get_timeseries(self): return [ - TimeSeries(name='a', timestamps=np.linspace(0, 1, 11)), - TimeSeries(name='b', timestamps=np.linspace(0.1, 5, 13)), + TimeSeries(name='a', data=[1]*11, unit='unit', timestamps=np.linspace(0, 1, 11)), + TimeSeries(name='b', data=[1]*13, unit='unit', timestamps=np.linspace(0.1, 5, 13)), ] def get_dataframe(self): diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index d3afa8155..c68ff5589 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -130,7 +130,7 @@ def test_access_processing(self): def test_epoch_tags(self): tags1 = ['t1', 't2'] tags2 = ['t3', 't4'] - tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float) + tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64) ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) expected_tags = tags1 + tags2 self.nwbfile.add_epoch(0.0, 1.0, tags1, ts) diff --git a/tests/unit/test_icephys.py b/tests/unit/test_icephys.py index 428b9c6ef..bc5b43c98 100644 --- a/tests/unit/test_icephys.py +++ b/tests/unit/test_icephys.py @@ -1,7 +1,7 @@ import numpy as np -from pynwb.icephys import PatchClampSeries, CurrentClampSeries, IZeroClampSeries, CurrentClampStimulusSeries, \ - VoltageClampSeries, VoltageClampStimulusSeries, IntracellularElectrode +from pynwb.icephys import (PatchClampSeries, CurrentClampSeries, IZeroClampSeries, CurrentClampStimulusSeries, + VoltageClampSeries, VoltageClampStimulusSeries, IntracellularElectrode, SweepTable) from pynwb.device import Device from pynwb.testing import TestCase from pynwb.file import NWBFile # Needed to test icephys functionality defined on NWBFile @@ -29,6 +29,17 @@ class NWBFileICEphys(TestCase): def setUp(self): self.icephys_electrode = GetElectrode() + def test_sweep_table_depractation_warn(self): + msg = ("Use of SweepTable is deprecated. Use the IntracellularRecordingsTable " + "instead. See also the NWBFile.add_intracellular_recordings function.") + with self.assertWarnsWith(DeprecationWarning, msg): + _ = NWBFile( + session_description='NWBFile icephys test', + identifier='NWB123', # required + session_start_time=datetime(2017, 4, 3, 11, tzinfo=tzlocal()), + ic_electrodes=[self.icephys_electrode, ], + sweep_table=SweepTable()) + def test_ic_electrodes_parameter_deprecation(self): # Make sure we warn when using the ic_electrodes parameter on NWBFile msg = "Use of the ic_electrodes parameter is deprecated. Use the icephys_electrodes parameter instead" diff --git a/tests/unit/test_icephys_metadata_tables.py b/tests/unit/test_icephys_metadata_tables.py new file mode 100644 index 000000000..d63e1ffb8 --- /dev/null +++ b/tests/unit/test_icephys_metadata_tables.py @@ -0,0 +1,1411 @@ +""" +Module for testing of the intracellular electrophysiology experiment metadata +tables originally created as part of the ndx-icephys-meta extension. These +are tested in this separate module to avoid crowding of the main test_icephys +test module and to allow us to test the main experiment metadata structures +separately. +""" +import numpy as np +from datetime import datetime +from dateutil.tz import tzlocal +from pandas.testing import assert_frame_equal +from numpy.testing import assert_array_equal +import warnings +import h5py + + +from pynwb.testing import TestCase, remove_test_file, create_icephys_stimulus_and_response +from pynwb.file import NWBFile +from pynwb.icephys import (VoltageClampStimulusSeries, VoltageClampSeries, CurrentClampStimulusSeries, + IZeroClampSeries, IntracellularRecordingsTable, SimultaneousRecordingsTable, + SequentialRecordingsTable, RepetitionsTable, ExperimentalConditionsTable) +from pynwb.base import TimeSeriesReferenceVectorData +from pynwb import NWBHDF5IO +from hdmf.utils import docval, popargs + + +class ICEphysMetaTestBase(TestCase): + """ + Base helper class for setting up tests for the ndx-icephys-meta extension. + """ + + def setUp(self): + # Create an example nwbfile with a device, intracellular electrode, stimulus, and response + self.nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + experimenter='Dr. Bilbo Baggins', + lab='Bag End Laboratory', + institution='University of Middle Earth at the Shire', + experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.', + session_id='LONELYMTN' + ) + self.device = self.nwbfile.create_device(name='Heka ITC-1600') + self.electrode = self.nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=self.device + ) + self.stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=self.electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + self.nwbfile.add_stimulus(self.stimulus) + self.response = VoltageClampSeries( + name='vcs', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=self.electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15) + ) + self.nwbfile.add_acquisition(self.response) + self.path = 'test_icephys_meta_intracellularrecording.h5' + + def tearDown(self): + remove_test_file(self.path) + + @docval({'name': 'ir', + 'type': IntracellularRecordingsTable, + 'doc': 'Intracellular recording to be added to the file before write', + 'default': None}, + {'name': 'sw', + 'type': SimultaneousRecordingsTable, + 'doc': 'SimultaneousRecordingsTable table to be added to the file before write', + 'default': None}, + {'name': 'sws', + 'type': SequentialRecordingsTable, + 'doc': 'SequentialRecordingsTable table to be added to the file before write', + 'default': None}, + {'name': 'repetitions', + 'type': RepetitionsTable, + 'doc': 'RepetitionsTable table to be added to the file before write', + 'default': None}, + {'name': 'cond', + 'type': ExperimentalConditionsTable, + 'doc': 'ExperimentalConditionsTable table to be added to the file before write', + 'default': None}) + def write_test_helper(self, **kwargs): + """ + Internal helper function to roundtrip an ICEphys file with the given set of ICEphys tables + """ + ir, sw, sws, repetitions, cond = popargs('ir', 'sw', 'sws', 'repetitions', 'cond', kwargs) + + if ir is not None: + self.nwbfile.intracellular_recordings = ir + if sw is not None: + self.nwbfile.icephys_simultaneous_recordings = sw + if sws is not None: + self.nwbfile.icephys_sequential_recordings = sws + if repetitions is not None: + self.nwbfile.icephys_repetitions = repetitions + if cond is not None: + self.nwbfile.icephys_experimental_conditions = cond + + # Write our test file + with NWBHDF5IO(self.path, 'w') as io: + io.write(self.nwbfile) + + # Test that we can read the file + with NWBHDF5IO(self.path, 'r') as io: + infile = io.read() + if ir is not None: + in_ir = infile.intracellular_recordings + self.assertIsNotNone(in_ir) + to_dataframe_kwargs = dict(electrode_refs_as_objectids=True, + stimulus_refs_as_objectids=True, + response_refs_as_objectids=True) + assert_frame_equal(ir.to_dataframe(**to_dataframe_kwargs), in_ir.to_dataframe(**to_dataframe_kwargs)) + if sw is not None: + in_sw = infile.icephys_simultaneous_recordings + self.assertIsNotNone(in_sw) + self.assertListEqual(in_sw['recordings'].target.data[:].tolist(), sw['recordings'].target.data[:]) + self.assertEqual(in_sw['recordings'].target.table.object_id, sw['recordings'].target.table.object_id) + if sws is not None: + in_sws = infile.icephys_sequential_recordings + self.assertIsNotNone(in_sws) + self.assertListEqual(in_sws['simultaneous_recordings'].target.data[:].tolist(), + sws['simultaneous_recordings'].target.data[:]) + self.assertEqual(in_sws['simultaneous_recordings'].target.table.object_id, + sws['simultaneous_recordings'].target.table.object_id) + if repetitions is not None: + in_repetitions = infile.icephys_repetitions + self.assertIsNotNone(in_repetitions) + self.assertListEqual(in_repetitions['sequential_recordings'].target.data[:].tolist(), + repetitions['sequential_recordings'].target.data[:]) + self.assertEqual(in_repetitions['sequential_recordings'].target.table.object_id, + repetitions['sequential_recordings'].target.table.object_id) + if cond is not None: + in_cond = infile.icephys_experimental_conditions + self.assertIsNotNone(in_cond) + self.assertListEqual(in_cond['repetitions'].target.data[:].tolist(), + cond['repetitions'].target.data[:]) + self.assertEqual(in_cond['repetitions'].target.table.object_id, + cond['repetitions'].target.table.object_id) + + +class IntracellularElectrodesTableTests(TestCase): + """ + The IntracellularElectrodesTable is covered by the + IntracellularRecordingsTableTests as this table is part of that table. + """ + pass + + +class IntracellularStimuliTableTests(TestCase): + """ + The IntracellularStimuliTable is covered by the + IntracellularRecordingsTableTests as this table is part of that table. + """ + pass + + +class IntracellularResponsesTableTests(TestCase): + """ + The IntracellularResponsesTable is covered by the + IntracellularRecordingsTableTests as this table is part of that table. + """ + pass + + +class IntracellularRecordingsTableTests(ICEphysMetaTestBase): + """ + Class for testing the IntracellularRecordingsTable Container + """ + + def test_init(self): + ret = IntracellularRecordingsTable() + self.assertEqual(ret.name, 'intracellular_recordings') + + def test_add_row(self): + # Add a row to our IR table + ir = IntracellularRecordingsTable() + + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + # Test that we get the correct row index back + self.assertEqual(row_index, 0) + # Read our first (and only) row and assert that it is correct + res = ir[0] + # Confirm that slicing one row give the same result as converting the whole table, which has only one row + assert_frame_equal(ir.to_dataframe(), res) + # Check the row id + self.assertEqual(res.index[0], 10) + # Check electrodes + self.assertIs(res[('electrodes', 'electrode')].iloc[0], self.electrode) + # Check the stimulus + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], (0, 5, self.stimulus)) + # Check the response + self.assertTupleEqual(res[('responses', 'response')].iloc[0], (0, 5, self.response)) + # Test writing out ir table + self.write_test_helper(ir) + + def test_add_row_incompatible_types(self): + # Add a row that mixes CurrentClamp and VoltageClamp data + sweep_number = 15 + local_stimulus = CurrentClampStimulusSeries( + name="ccss_"+str(sweep_number), + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=self.electrode, + gain=0.1, + sweep_number=np.uint64(sweep_number) + ) + ir = IntracellularRecordingsTable() + with self.assertRaises(ValueError): + _ = ir.add_recording( + electrode=self.electrode, + stimulus=local_stimulus, + response=self.response, + id=np.int64(10) + ) + + def test_warn_if_IZeroClampSeries_with_stimulus(self): + local_response = IZeroClampSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=self.electrode, + gain=0.02, + sweep_number=np.uint64(100000) + ) + ir = IntracellularRecordingsTable() + with self.assertRaises(ValueError): + _ = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=local_response, + id=np.int64(10) + ) + + def test_inconsistent_PatchClampSeries(self): + local_electrode = self.nwbfile.create_icephys_electrode( + name="elec1", + description='a mock intracellular electrode', + device=self.device + ) + local_stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=local_electrode, + gain=0.02, + sweep_number=np.uint64(100000) + ) + ir = IntracellularRecordingsTable() + with self.assertRaises(ValueError): + _ = ir.add_recording( + electrode=self.electrode, + stimulus=local_stimulus, + response=self.response, + id=np.int64(10) + ) + + def test_add_row_no_response(self): + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=None, + id=np.int64(10) + ) + res = ir[0] + # Check the ID + self.assertEqual(row_index, 0) + self.assertEqual(res.index[0], 10) + # Check the row id + self.assertEqual(res.index[0], 10) + # Check electrodes + self.assertIs(res[('electrodes', 'electrode')].iloc[0], self.electrode) + # Check the stimulus + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], (0, 5, self.stimulus)) + # Check the response + self.assertTrue(isinstance(res[('responses', 'response')].iloc[0], + TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + # Test writing out ir table + self.write_test_helper(ir) + + def test_add_row_no_stimulus(self): + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=None, + response=self.response, + id=np.int64(10) + ) + res = ir[0] + # Check the ID + self.assertEqual(row_index, 0) + self.assertEqual(res.index[0], 10) + # Check the row id + self.assertEqual(res.index[0], 10) + # Check electrodes + self.assertIs(res[('electrodes', 'electrode')].iloc[0], self.electrode) + # Check the stimulus + self.assertTrue(isinstance(res[('stimuli', 'stimulus')].iloc[0], + TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + # Check the response + self.assertTupleEqual(res[('responses', 'response')].iloc[0], (0, 5, self.response)) + # Test writing out ir table + self.write_test_helper(ir) + + def test_add_row_check_start_index_and_index_count_are_fixed(self): + # Make sure -1 values are converted + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_start_index=-1, # assert this is fixed to 0 + stimulus_index_count=-1, # assert this is fixed to len(stimulus) + response=None, + response_start_index=0, # assert this is fixed to -1 + response_index_count=10, # assert this is fixed to -1 + id=np.int64(10) + ) + res = ir[0] + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], + (0, len(self.stimulus.data), self.stimulus)) + self.assertTrue(isinstance(res[('responses', 'response')].iloc[0], + TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE)) + # Make sure single -1 values are converted + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_start_index=2, + id=np.int64(10) + ) + res = ir[0] + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], + (2, len(self.stimulus.data)-2, self.stimulus)) + # Make sure single -1 values are converted + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_index_count=2, + id=np.int64(10) + ) + res = ir[0] + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], + (0, 2, self.stimulus)) + + def test_add_row_index_out_of_range(self): + # Stimulus/Response start_index to large + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_start_index=10, + response=self.response, + id=np.int64(10) + ) + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response_start_index=10, + response=self.response, + id=np.int64(10) + ) + # Stimulus/Reponse index count too large + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_index_count=10, + response=self.response, + id=np.int64(10) + ) + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response_index_count=10, + response=self.response, + id=np.int64(10) + ) + # Stimulus/Reponse start+count combination too large + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + stimulus_start_index=3, + stimulus_index_count=4, + response=self.response, + id=np.int64(10) + ) + with self.assertRaises(IndexError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response_start_index=3, + response_index_count=4, + response=self.response, + id=np.int64(10) + ) + + def test_add_row_no_stimulus_and_response(self): + with self.assertRaises(ValueError): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=None, + response=None + ) + + def test_add_column(self): + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + ir.add_column(name='test', description='test column', data=np.arange(1)) + self.assertTupleEqual(ir.colnames, ('test',)) + + def test_enforce_unique_id(self): + """ + Test to ensure that unique ids are enforced on RepetitionsTable table + """ + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + with self.assertRaises(ValueError): + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + + def test_basic_write(self): + """ + Populate, write, and read the SimultaneousRecordingsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + ir.add_column(name='test', description='test column', data=np.arange(1)) + self.write_test_helper(ir=ir) + + def test_to_dataframe(self): + # Add the intracelluar recordings + # Create a table setup for testing a number of conditions + electrode0 = self.electrode + electrode1 = self.nwbfile.create_icephys_electrode( + name="elec1", + description='another mock intracellular electrode', + device=self.device + ) + for sweep_number in range(20): + elec = (electrode0 if (sweep_number % 2 == 0) else electrode1) + stim, resp = create_icephys_stimulus_and_response(sweep_number=np.uint64(sweep_number), + electrode=elec, + randomize_data=False) + if sweep_number in [0, 10]: # include missing stimuli + stim = None + self.nwbfile.add_intracellular_recording(electrode=elec, + stimulus=stim, + response=resp, + id=sweep_number) + tags_data = ['A1', 'A2', 'B1', 'B2', 'C1', 'C2', 'C3', + 'D1', 'D2', 'D3', 'A1', 'A2', 'B1', 'B2', + 'C1', 'C2', 'C3', 'D1', 'D2', 'D3'] + self.nwbfile.intracellular_recordings.add_column(name='recording_tags', + data=tags_data, + description='String with a set of recording tags') + # Test normal conversion to a dataframe + df = self.nwbfile.intracellular_recordings.to_dataframe() + expected_cols = [('intracellular_recordings', 'recording_tags'), ('electrodes', 'id'), + ('electrodes', 'electrode'), ('stimuli', 'id'), ('stimuli', 'stimulus'), + ('responses', 'id'), ('responses', 'response')] + self.assertListEqual(df.columns.to_list(), expected_cols) + self.assertListEqual(df[('intracellular_recordings', 'recording_tags')].to_list(), tags_data) + # Test conversion with ignore category ids set + df = self.nwbfile.intracellular_recordings.to_dataframe(ignore_category_ids=True) + expected_cols_no_ids = [('intracellular_recordings', 'recording_tags'), + ('electrodes', 'electrode'), ('stimuli', 'stimulus'), + ('responses', 'response')] + self.assertListEqual(df.columns.to_list(), expected_cols_no_ids) + # Test conversion with stimulus_refs_as_objectids + df = self.nwbfile.intracellular_recordings.to_dataframe(stimulus_refs_as_objectids=True) + self.assertListEqual(df.columns.to_list(), expected_cols) + expects_stim_col = [e if e[2] is None else (e[0], e[1], e[2].object_id) + for e in self.nwbfile.intracellular_recordings[('stimuli', 'stimulus')][:]] + self.assertListEqual(df[('stimuli', 'stimulus')].tolist(), expects_stim_col) + # Test conversion with response_refs_as_objectids + df = self.nwbfile.intracellular_recordings.to_dataframe(response_refs_as_objectids=True) + self.assertListEqual(df.columns.to_list(), expected_cols) + expects_resp_col = [e if e[2] is None else (e[0], e[1], e[2].object_id) + for e in self.nwbfile.intracellular_recordings[('responses', 'response')][:]] + self.assertListEqual(df[('responses', 'response')].tolist(), expects_resp_col) + # Test conversion with all options enabled + df = self.nwbfile.intracellular_recordings.to_dataframe(ignore_category_ids=True, + stimulus_refs_as_objectids=True, + response_refs_as_objectids=True) + self.assertListEqual(df.columns.to_list(), expected_cols_no_ids) + self.assertListEqual(df[('stimuli', 'stimulus')].tolist(), expects_stim_col) + self.assertListEqual(df[('responses', 'response')].tolist(), expects_resp_col) + + def test_round_trip_container_no_data(self): + """Test read and write the container by itself""" + curr = IntracellularRecordingsTable() + with NWBHDF5IO(self.path, 'w') as io: + io.write(curr) + with NWBHDF5IO(self.path, 'r') as io: + incon = io.read() + self.assertListEqual(incon.categories, curr.categories) + for n in curr.categories: + # empty columns from file have dtype int64 or float64 but empty in-memory columns have dtype object + assert_frame_equal(incon[n], curr[n], check_dtype=False, check_index_type=False) + + def test_write_with_stimulus_template(self): + """ + Populate, write, and read the SimultaneousRecordingsTable container and other required containers + """ + local_nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + experimenter='Dr. Bilbo Baggins', + lab='Bag End Laboratory', + institution='University of Middle Earth at the Shire', + experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.', + session_id='LONELYMTN' + ) + # Add a device + local_device = local_nwbfile.create_device(name='Heka ITC-1600') + local_electrode = local_nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=local_device + ) + local_stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=local_electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + local_response = VoltageClampSeries( + name='vcs', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=local_electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15) + ) + local_nwbfile.add_stimulus_template(local_stimulus) + row_index = local_nwbfile.add_intracellular_recording( + electrode=local_electrode, + stimulus=local_stimulus, + response=local_response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + # Write our test file + with NWBHDF5IO(self.path, 'w') as io: + io.write(local_nwbfile) + + +class SimultaneousRecordingsTableTests(ICEphysMetaTestBase): + """ + Test class for testing the SimultaneousRecordingsTable Container class + """ + + def test_init(self): + """ + Test __init__ to make sure we can instantiate the SimultaneousRecordingsTable container + """ + ir = IntracellularRecordingsTable() + ret = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + self.assertIs(ret.recordings.table, ir) + self.assertEqual(ret.name, 'simultaneous_recordings') + + def test_missing_intracellular_recordings_on_init(self): + """ + Test that ValueError is raised when intracellular_recordings is missing. This is + allowed only on read where the intracellular_recordings table is already set + from the file. + """ + with self.assertRaises(ValueError): + _ = SimultaneousRecordingsTable() + + def test_add_simultaneous_recording(self): + """ + Populate, write, and read the SimultaneousRecordingsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + self.assertEqual(len(ir), 1) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + row_index = sw.add_simultaneous_recording(recordings=[row_index], id=100) + self.assertEqual(row_index, 0) + self.assertListEqual(sw.id[:], [100]) + self.assertListEqual(sw['recordings'].data, [1]) + self.assertListEqual(sw['recordings'].target.data[:], [0]) + + def test_basic_write(self): + """ + Populate, write, and read the SimultaneousRecordingsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + self.assertEqual(len(ir), 1) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + row_index = sw.add_simultaneous_recording(recordings=[row_index]) + self.assertEqual(row_index, 0) + self.write_test_helper(ir=ir, sw=sw) + + def test_enforce_unique_id(self): + """ + Test to ensure that unique ids are enforced on RepetitionsTable table + """ + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sw.add_simultaneous_recording(recordings=[0], id=np.int64(10)) + with self.assertRaises(ValueError): + sw.add_simultaneous_recording(recordings=[0], id=np.int64(10)) + + +class SequentialRecordingsTableTests(ICEphysMetaTestBase): + """ + Test class for testing the SequentialRecordingsTable Container class + """ + + def test_init(self): + """ + Test __init__ to make sure we can instantiate the SequentialRecordingsTable container + """ + ir = IntracellularRecordingsTable() + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + ret = SequentialRecordingsTable(simultaneous_recordings_table=sw) + self.assertIs(ret.simultaneous_recordings.table, sw) + self.assertEqual(ret.name, 'sequential_recordings') + + def test_missing_simultaneous_recordings_on_init(self): + """ + Test that ValueError is raised when simultaneous_recordings is missing. This is + allowed only on read where the simultaneous_recordings table is already set + from the file. + """ + with self.assertRaises(ValueError): + _ = SequentialRecordingsTable() + + def test_basic_write(self): + """ + Populate, write, and read the SequentialRecordingsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + row_index = sw.add_simultaneous_recording(recordings=[0]) + self.assertEqual(row_index, 0) + sws = SequentialRecordingsTable(sw) + row_index = sws.add_sequential_recording(simultaneous_recordings=[0, ], stimulus_type='MyStimStype') + self.assertEqual(row_index, 0) + self.write_test_helper(ir=ir, sw=sw, sws=sws) + + def test_enforce_unique_id(self): + """ + Test to ensure that unique ids are enforced on RepetitionsTable table + """ + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sw.add_simultaneous_recording(recordings=[0]) + sws = SequentialRecordingsTable(sw) + sws.add_sequential_recording(simultaneous_recordings=[0, ], id=np.int64(10), stimulus_type='MyStimStype') + with self.assertRaises(ValueError): + sws.add_sequential_recording(simultaneous_recordings=[0, ], id=np.int64(10), stimulus_type='MyStimStype') + + +class RepetitionsTableTests(ICEphysMetaTestBase): + """ + Test class for testing the RepetitionsTable Container class + """ + + def test_init(self): + """ + Test __init__ to make sure we can instantiate the RepetitionsTable container + """ + ir = IntracellularRecordingsTable() + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sws = SequentialRecordingsTable(simultaneous_recordings_table=sw) + ret = RepetitionsTable(sequential_recordings_table=sws) + self.assertIs(ret.sequential_recordings.table, sws) + self.assertEqual(ret.name, 'repetitions') + + def test_missing_sequential_recordings_on_init(self): + """ + Test that ValueError is raised when sequential_recordings is missing. This is + allowed only on read where the sequential_recordings table is already set + from the file. + """ + with self.assertRaises(ValueError): + _ = RepetitionsTable() + + def test_basic_write(self): + """ + Populate, write, and read the RepetitionsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + self.assertEqual(row_index, 0) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + row_index = sw.add_simultaneous_recording(recordings=[0]) + self.assertEqual(row_index, 0) + sws = SequentialRecordingsTable(sw) + row_index = sws.add_sequential_recording(simultaneous_recordings=[0, ], stimulus_type='MyStimStype') + self.assertEqual(row_index, 0) + repetitions = RepetitionsTable(sequential_recordings_table=sws) + repetitions.add_repetition(sequential_recordings=[0, ]) + self.write_test_helper(ir=ir, sw=sw, sws=sws, repetitions=repetitions) + + def test_enforce_unique_id(self): + """ + Test to ensure that unique ids are enforced on RepetitionsTable table + """ + ir = IntracellularRecordingsTable() + ir.add_recording( + electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10) + ) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sw.add_simultaneous_recording(recordings=[0]) + sws = SequentialRecordingsTable(sw) + sws.add_sequential_recording(simultaneous_recordings=[0, ], stimulus_type='MyStimStype') + repetitions = RepetitionsTable(sequential_recordings_table=sws) + repetitions.add_repetition(sequential_recordings=[0, ], id=np.int64(10)) + with self.assertRaises(ValueError): + repetitions.add_repetition(sequential_recordings=[0, ], id=np.int64(10)) + + +class ExperimentalConditionsTableTests(ICEphysMetaTestBase): + """ + Test class for testing the ExperimentalConditionsTable Container class + """ + + def test_init(self): + """ + Test __init__ to make sure we can instantiate the ExperimentalConditionsTable container + """ + ir = IntracellularRecordingsTable() + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sws = SequentialRecordingsTable(simultaneous_recordings_table=sw) + repetitions = RepetitionsTable(sequential_recordings_table=sws) + ret = ExperimentalConditionsTable(repetitions_table=repetitions) + self.assertIs(ret.repetitions.table, repetitions) + self.assertEqual(ret.name, 'experimental_conditions') + + def test_missing_repetitions_on_init(self): + """ + Test that ValueError is raised when repetitions is missing. This is + allowed only on read where the repetitions table is already set + from the file. + """ + with self.assertRaises(ValueError): + _ = ExperimentalConditionsTable() + + def test_basic_write(self): + """ + Populate, write, and read the ExperimentalConditionsTable container and other required containers + """ + ir = IntracellularRecordingsTable() + row_index = ir.add_recording(electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10)) + self.assertEqual(row_index, 0) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + row_index = sw.add_simultaneous_recording(recordings=[0]) + self.assertEqual(row_index, 0) + sws = SequentialRecordingsTable(sw) + row_index = sws.add_sequential_recording(simultaneous_recordings=[0, ], stimulus_type='MyStimStype') + self.assertEqual(row_index, 0) + repetitions = RepetitionsTable(sequential_recordings_table=sws) + row_index = repetitions.add_repetition(sequential_recordings=[0, ]) + self.assertEqual(row_index, 0) + cond = ExperimentalConditionsTable(repetitions_table=repetitions) + row_index = cond.add_experimental_condition(repetitions=[0, ]) + self.assertEqual(row_index, 0) + self.write_test_helper(ir=ir, sw=sw, sws=sws, repetitions=repetitions, cond=cond) + + def test_enforce_unique_id(self): + """ + Test to ensure that unique ids are enforced on RepetitionsTable table + """ + ir = IntracellularRecordingsTable() + ir.add_recording(electrode=self.electrode, + stimulus=self.stimulus, + response=self.response, + id=np.int64(10)) + sw = SimultaneousRecordingsTable(intracellular_recordings_table=ir) + sw.add_simultaneous_recording(recordings=[0]) + sws = SequentialRecordingsTable(sw) + sws.add_sequential_recording(simultaneous_recordings=[0, ], stimulus_type='MyStimStype') + repetitions = RepetitionsTable(sequential_recordings_table=sws) + repetitions.add_repetition(sequential_recordings=[0, ]) + cond = ExperimentalConditionsTable(repetitions_table=repetitions) + cond.add_experimental_condition(repetitions=[0, ], id=np.int64(10)) + with self.assertRaises(ValueError): + cond.add_experimental_condition(repetitions=[0, ], id=np.int64(10)) + + +class NWBFileTests(TestCase): + """ + Test class for testing the NWBFileTests Container class + """ + def setUp(self): + warnings.simplefilter("always") # Trigger all warnings + self.path = 'test_icephys_meta_intracellularrecording.h5' + + def tearDown(self): + remove_test_file(self.path) + + def __get_icephysfile(self): + """ + Create a dummy NWBFile instance + """ + icefile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()) + ) + return icefile + + def __add_device(self, icefile): + return icefile.create_device(name='Heka ITC-1600') + + def __add_electrode(self, icefile, device): + return icefile.create_icephys_electrode(name="elec0", + description='a mock intracellular electrode', + device=device) + + def __get_stimulus(self, electrode): + """ + Create a dummy VoltageClampStimulusSeries + """ + return VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + + def __get_response(self, electrode): + """ + Create a dummy VoltageClampSeries + """ + return VoltageClampSeries( + name='vcs', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15) + ) + + def test_deprecate_simultaneous_recordings_on_add_stimulus(self): + """ + Test that warnings are raised if the user tries to use a simultaneous_recordings table + """ + nwbfile = self.__get_icephysfile() + device = self.__add_device(nwbfile) + electrode = self.__add_electrode(nwbfile, device) + stimulus = self.__get_stimulus(electrode=electrode) + response = self.__get_response(electrode=electrode) + # Make sure we warn if sweeptable is added on add_stimulus + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") # Trigger all warnings + nwbfile.add_stimulus(stimulus, use_sweep_table=True) + self.assertEqual(len(w), 1) + assert issubclass(w[-1].category, DeprecationWarning) + # make sure we don't trigger the same deprecation warning twice + nwbfile.add_acquisition(response, use_sweep_table=True) + self.assertEqual(len(w), 1) + + def test_deprecate_sweeptable_on_add_stimulus_template(self): + """ + Make sure we warn when using the sweep-table + """ + nwbfile = self.__get_icephysfile() + local_electrode = nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=nwbfile.create_device(name='Heka ITC-1600') + ) + local_stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=local_electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + local_stimulus2 = VoltageClampStimulusSeries( + name="ccss2", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=local_electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + with warnings.catch_warnings(record=True) as w: + nwbfile.add_stimulus_template(local_stimulus, use_sweep_table=True) + self.assertEqual(len(w), 1) + assert issubclass(w[-1].category, DeprecationWarning) + self.assertEqual(str(w[-1].message), + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable " + "instead. See also the NWBFile.add_intracellular_recordings function.") + # make sure we don't trigger the same deprecation warning twice + nwbfile.add_stimulus_template(local_stimulus2, use_sweep_table=True) + self.assertEqual(len(w), 1) + + def test_deprecate_sweepstable_on_add_acquistion(self): + """ + Test that warnings are raised if the user tries to use a sweeps table + """ + nwbfile = self.__get_icephysfile() + device = self.__add_device(nwbfile) + electrode = self.__add_electrode(nwbfile, device) + stimulus = self.__get_stimulus(electrode=electrode) + response = self.__get_response(electrode=electrode) + # Make sure we warn if sweeptable is added on add_stimulus + with warnings.catch_warnings(record=True) as w: + nwbfile.add_acquisition(response, use_sweep_table=True) + self.assertEqual(len(w), 1) + assert issubclass(w[-1].category, DeprecationWarning) + self.assertEqual(str(w[-1].message), + "Use of SweepTable is deprecated. Use the IntracellularRecordingsTable " + "instead. See also the NWBFile.add_intracellular_recordings function.") + # make sure we don't trigger the same deprecation warning twice + nwbfile.add_stimulus(stimulus, use_sweep_table=True) + self.assertEqual(len(w), 1) + + def test_deprecate_sweepstable_on_init(self): + """ + Test that warnings are raised if the user tries to use a sweeps table + """ + from pynwb.icephys import SweepTable + with warnings.catch_warnings(record=True) as w: + nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + sweep_table=SweepTable() + ) + device = self.__add_device(nwbfile) + electrode = self.__add_electrode(nwbfile, device) + stimulus = self.__get_stimulus(electrode=electrode) + self.assertEqual(len(w), 1) + assert issubclass(w[-1].category, DeprecationWarning) + # make sure we don't trigger the same deprecation warning twice + nwbfile.add_stimulus(stimulus, use_sweep_table=True) + self.assertEqual(len(w), 1) + + def test_deprectation_icephys_filtering_on_init(self): + with warnings.catch_warnings(record=True) as w: + nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()), + icephys_filtering='test filtering' + ) + assert issubclass(w[-1].category, DeprecationWarning) + self.assertEqual(nwbfile.icephys_filtering, 'test filtering') + + def test_icephys_filtering_roundtrip(self): + # create the base file + nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()) + ) + # set the icephys_filtering attribute and make sure we get a deprectation warning + with warnings.catch_warnings(record=True) as w: + nwbfile.icephys_filtering = 'test filtering' + assert issubclass(w[-1].category, DeprecationWarning) + # write the test fil + with NWBHDF5IO(self.path, 'w') as io: + io.write(nwbfile) + # read the test file and confirm icephys_filtering has been written + with NWBHDF5IO(self.path, 'r') as io: + with warnings.catch_warnings(record=True) as w: + infile = io.read() + self.assertEqual(len(w), 1) # make sure a warning is being raised + assert issubclass(w[0].category, DeprecationWarning) # make sure it is a deprecation warning + self.assertEqual(infile.icephys_filtering, 'test filtering') # make sure the value is set + + def test_get_icephys_meta_parent_table(self): + """ + Create the table hierarchy step-by-step and check that as we add tables the get_icephys_meta_parent_table + returns the expected top table + """ + local_nwbfile = NWBFile( + session_description='my first synthetic recording', + identifier='EXAMPLE_ID', + session_start_time=datetime.now(tzlocal()) + ) + # Add a device + local_device = local_nwbfile.create_device(name='Heka ITC-1600') + local_electrode = local_nwbfile.create_icephys_electrode( + name="elec0", + description='a mock intracellular electrode', + device=local_device + ) + local_stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=local_electrode, + gain=0.02, + sweep_number=np.uint64(15) + ) + local_response = VoltageClampSeries( + name='vcs', + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=local_electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15) + ) + local_nwbfile.add_stimulus_template(local_stimulus) + # Check that none of the table exist yet + self.assertIsNone(local_nwbfile.get_icephys_meta_parent_table()) + # Add a recording and confirm that intracellular_recordings is the top table + _ = local_nwbfile.add_intracellular_recording( + electrode=local_electrode, + stimulus=local_stimulus, + response=local_response, + id=np.int64(10) + ) + self.assertIsInstance(local_nwbfile.get_icephys_meta_parent_table(), + IntracellularRecordingsTable) + # Add a sweep and check that the simultaneous_recordings table is the top table + _ = local_nwbfile.add_icephys_simultaneous_recording(recordings=[0]) + self.assertIsInstance(local_nwbfile.get_icephys_meta_parent_table(), + SimultaneousRecordingsTable) + # Add a sweep_sequence and check that it is now our top table + _ = local_nwbfile.add_icephys_sequential_recording(simultaneous_recordings=[0], stimulus_type="MyStimulusType") + self.assertIsInstance(local_nwbfile.get_icephys_meta_parent_table(), + SequentialRecordingsTable) + # Add a repetition and check that it is now our top table + _ = local_nwbfile.add_icephys_repetition(sequential_recordings=[0]) + self.assertIsInstance(local_nwbfile.get_icephys_meta_parent_table(), + RepetitionsTable) + # Add a condition and check that it is now our top table + _ = local_nwbfile.add_icephys_experimental_condition(repetitions=[0]) + self.assertIsInstance(local_nwbfile.get_icephys_meta_parent_table(), + ExperimentalConditionsTable) + + def test_add_icephys_meta_full_roundtrip(self): + """ + This test adds all data and then constructs step-by-step the full table structure + Returns: + + """ + #################################### + # Create our file and timeseries + ################################### + nwbfile = self.__get_icephysfile() + device = self.__add_device(nwbfile) + electrode = self.__add_electrode(nwbfile, device) + # Add the data using standard methods from NWBFile + stimulus = self.__get_stimulus(electrode=electrode) + nwbfile.add_stimulus(stimulus) + # Check that the deprecated sweep table has indeed not been created + self.assertIsNone(nwbfile.sweep_table) + response = self.__get_response(electrode=electrode) + nwbfile.add_acquisition(response) + # Check that the deprecated sweep table has indeed not been created + self.assertIsNone(nwbfile.sweep_table) + + ############################################# + # Test adding IntracellularRecordingsTable + ############################################# + # Check that our IntracellularRecordingsTable table does not yet exist + self.assertIsNone(nwbfile.intracellular_recordings) + # Add an intracellular recording + intracellular_recording_ids = [np.int64(10), np.int64(11)] + nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + response=response, + id=intracellular_recording_ids[0] + ) + nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + response=response, + id=intracellular_recording_ids[1] + ) + # Check that the table has been created + self.assertIsNotNone(nwbfile.intracellular_recordings) + # Check that the values in our row are correct + self.assertEqual(len(nwbfile.intracellular_recordings), 2) + res = nwbfile.intracellular_recordings[0] + # Check the ID + self.assertEqual(res.index[0], intracellular_recording_ids[0]) + # Check electrodes + self.assertIs(res[('electrodes', 'electrode')].iloc[0], electrode) + # Check the stimulus + self.assertTupleEqual(res[('stimuli', 'stimulus')].iloc[0], (0, 5, stimulus)) + # Check the response + self.assertTupleEqual(res[('responses', 'response')].iloc[0], (0, 5, response)) + + ############################################# + # Test adding SimultaneousRecordingsTable + ############################################# + # Confirm that our SimultaneousRecordingsTable table does not yet exist + self.assertIsNone(nwbfile.icephys_simultaneous_recordings) + # Add a sweep + simultaneous_recordings_id = np.int64(12) + recordings_indices = [0, 1] + nwbfile.add_icephys_simultaneous_recording(recordings=recordings_indices, id=simultaneous_recordings_id) + # Check that the SimultaneousRecordingsTable table has been added + self.assertIsNotNone(nwbfile.icephys_simultaneous_recordings) + # Check that the values for our icephys_simultaneous_recordings table are correct + self.assertListEqual(nwbfile.icephys_simultaneous_recordings.id[:], [simultaneous_recordings_id]) + self.assertListEqual(nwbfile.icephys_simultaneous_recordings['recordings'].data, [2]) + self.assertListEqual(nwbfile.icephys_simultaneous_recordings['recordings'].target.data[:], [0, 1]) + res = nwbfile.icephys_simultaneous_recordings[0] + # check the id value + self.assertEqual(res.index[0], simultaneous_recordings_id) + # Check that our simultaneous recording contains 2 IntracellularRecording + assert_array_equal(res.loc[simultaneous_recordings_id]['recordings'], + recordings_indices) + + ############################################# + # Test adding a SweepSequence + ############################################# + # Confirm that our SequentialRecordingsTable table does not yet exist + self.assertIsNone(nwbfile.icephys_sequential_recordings) + # Add a sweep + sequential_recording_id = np.int64(15) + simultaneous_recordings_indices = [0] + nwbfile.add_icephys_sequential_recording(simultaneous_recordings=simultaneous_recordings_indices, + id=sequential_recording_id, + stimulus_type="MyStimulusType") + # Check that the SimultaneousRecordingsTable table has been added + self.assertIsNotNone(nwbfile.icephys_sequential_recordings) + # Check that the values for our SimultaneousRecordingsTable table are correct + res = nwbfile.icephys_sequential_recordings[0] + # check the id value + self.assertEqual(res.index[0], sequential_recording_id) + # Check that our sequential recording containts 1 simultaneous recording + assert_array_equal(res.loc[sequential_recording_id]['simultaneous_recordings'], + simultaneous_recordings_indices) + + ############################################# + # Test adding a Run + ############################################# + # Confirm that our RepetitionsTable table does not yet exist + self.assertIsNone(nwbfile.icephys_repetitions) + # Add a repetition + sequential_recordings_indices = [0] + repetition_id = np.int64(17) + nwbfile.add_icephys_repetition(sequential_recordings=sequential_recordings_indices, id=repetition_id) + # Check that the SimultaneousRecordingsTable table has been added + self.assertIsNotNone(nwbfile.icephys_repetitions) + # Check that the values for our RepetitionsTable table are correct + res = nwbfile.icephys_repetitions[0] + # check the id value + self.assertEqual(res.index[0], repetition_id) + # Check that our repetition contains 1 SweepSequence + assert_array_equal(res.loc[repetition_id]['sequential_recordings'], + sequential_recordings_indices) + + ############################################# + # Test adding a Condition + ############################################# + # Confirm that our RepetitionsTable table does not yet exist + self.assertIsNone(nwbfile.icephys_experimental_conditions) + # Add a condition + repetitions_indices = [0] + experiment_id = np.int64(19) + nwbfile.add_icephys_experimental_condition(repetitions=repetitions_indices, id=experiment_id) + # Check that the ExperimentalConditionsTable table has been added + self.assertIsNotNone(nwbfile.icephys_experimental_conditions) + # Check that the values for our ExperimentalConditionsTable table are correct + res = nwbfile.icephys_experimental_conditions[0] + # check the id value + self.assertEqual(res.index[0], experiment_id) + # Check that our repetition contains 1 repetition + assert_array_equal(res.loc[experiment_id]['repetitions'], + repetitions_indices) + + ############################################# + # Test writing the file to disk + ############################################# + # Write our file to disk + # Write our test file + with NWBHDF5IO(self.path, 'w') as nwbio: + # # Uncomment the following lines to enable profiling for write + # import cProfile, pstats, io + # from pstats import SortKey + # pr = cProfile.Profile() + # pr.enable() + nwbio.write(nwbfile) + # pr.disable() + # s = io.StringIO() + # sortby = SortKey.CUMULATIVE + # ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + # ps.print_stats() + # print(s.getvalue()) + + ################################################################# + # Confirm that the low-level data has been written as expected + # using h5py to confirm all id values. We do this before we try + # to read the file back to confirm that data is correct on disk. + ################################################################# + with h5py.File(self.path, 'r') as io: + assert_array_equal(io['/general']['intracellular_ephys']['intracellular_recordings']['id'][:], + intracellular_recording_ids) + default_ids = [0, 1] + assert_array_equal(io['/general']['intracellular_ephys']['intracellular_recordings']['electrodes']['id'][:], + default_ids) + assert_array_equal(io['/general']['intracellular_ephys']['intracellular_recordings']['stimuli']['id'][:], + default_ids) + assert_array_equal(io['/general']['intracellular_ephys']['intracellular_recordings']['responses']['id'][:], + default_ids) + assert_array_equal(io['/general']['intracellular_ephys']['simultaneous_recordings']['id'][:], + [simultaneous_recordings_id, ]) + assert_array_equal(io['/general']['intracellular_ephys']['sequential_recordings']['id'][:], + [sequential_recording_id, ]) + assert_array_equal(io['/general']['intracellular_ephys']['repetitions']['id'][:], + [repetition_id, ]) + assert_array_equal(io['/general']['intracellular_ephys']['experimental_conditions']['id'][:], + [experiment_id, ]) + + ############################################# + # Test reading the file back from disk + ############################################# + with NWBHDF5IO(self.path, 'r') as nwbio: + # # Uncomment the following lines to enable profiling for read + # import cProfile, pstats, io + # from pstats import SortKey + # pr = cProfile.Profile() + # pr.enable() + infile = nwbio.read() + # pr.disable() + # s = io.StringIO() + # sortby = SortKey.CUMULATIVE + # ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + # ps.print_stats() + # print(s.getvalue()) + + ############################################################################ + # Test that the IntracellularRecordingsTable table has been written correctly + ############################################################################ + self.assertIsNotNone(infile.intracellular_recordings) + self.assertEqual(len(infile.intracellular_recordings), 2) + res = infile.intracellular_recordings[0] + # Check the ID + self.assertEqual(res.index[0], 10) + # Check the stimulus + self.assertEqual(res[('stimuli', 'stimulus')].iloc[0][0], 0) + self.assertEqual(res[('stimuli', 'stimulus')].iloc[0][1], 5) + self.assertEqual(res[('stimuli', 'stimulus')].iloc[0][2].object_id, stimulus.object_id) + # Check the response + self.assertEqual(res[('responses', 'response')].iloc[0][0], 0) + self.assertEqual(res[('responses', 'response')].iloc[0][1], 5) + self.assertEqual(res[('responses', 'response')].iloc[0][2].object_id, + nwbfile.get_acquisition('vcs').object_id) + # Check the Intracellular electrode + self.assertEqual(res[('electrodes', 'electrode')].iloc[0].object_id, electrode.object_id) + + ############################################################################ + # Test that the SimultaneousRecordingsTable table has been written correctly + ############################################################################ + self.assertIsNotNone(infile.icephys_simultaneous_recordings) + self.assertEqual(len(infile.icephys_simultaneous_recordings), 1) + res = infile.icephys_simultaneous_recordings[0] + # Check the ID and len of the intracellular_recordings column + self.assertEqual(res.index[0], simultaneous_recordings_id) + assert_array_equal(res.loc[simultaneous_recordings_id]['recordings'], recordings_indices) + + ############################################################################ + # Test that the SequentialRecordingsTable table has been written correctly + ############################################################################ + self.assertIsNotNone(infile.icephys_sequential_recordings) + self.assertEqual(len(infile.icephys_sequential_recordings), 1) + res = infile.icephys_sequential_recordings[0] + # Check the ID and len of the simultaneous_recordings column + self.assertEqual(res.index[0], sequential_recording_id) + assert_array_equal(res.loc[sequential_recording_id]['simultaneous_recordings'], + simultaneous_recordings_indices) + + ############################################################################ + # Test that the RepetitionsTable table has been written correctly + ############################################################################ + self.assertIsNotNone(infile.icephys_repetitions) + self.assertEqual(len(infile.icephys_repetitions), 1) + res = infile.icephys_repetitions[0] + # Check the ID and len of the simultaneous_recordings column + self.assertEqual(res.index[0], repetition_id) + assert_array_equal(res.loc[repetition_id]['sequential_recordings'], sequential_recordings_indices) + + ############################################################################ + # Test that the ExperimentalConditionsTable table has been written correctly + ############################################################################ + self.assertIsNotNone(infile.icephys_experimental_conditions) + self.assertEqual(len(infile.icephys_experimental_conditions), 1) + res = infile.icephys_experimental_conditions[0] + # Check the ID and len of the simultaneous_recordings column + self.assertEqual(res.index[0], experiment_id) + assert_array_equal(res.loc[experiment_id]['repetitions'], repetitions_indices) diff --git a/tests/unit/test_image.py b/tests/unit/test_image.py index c79a7b0d5..ef6f50cf7 100644 --- a/tests/unit/test_image.py +++ b/tests/unit/test_image.py @@ -55,15 +55,43 @@ def test_data_no_frame(self): ) self.assertIsNone(iS.starting_frame) + def test_data_no_unit(self): + msg = "Must supply 'unit' argument when supplying 'data' to ImageSeries 'test_iS'." + with self.assertRaisesWith(ValueError, msg): + ImageSeries( + name='test_iS', + data=np.ones((3, 3, 3)), + timestamps=list() + ) + + def test_external_file_no_unit(self): + iS = ImageSeries( + name='test_iS', + external_file=['external_file'], + timestamps=list() + ) + self.assertEqual(iS.unit, ImageSeries.DEFAULT_UNIT) + class IndexSeriesConstructor(TestCase): def test_init(self): - ts = TimeSeries('test_ts', list(), 'unit', timestamps=list()) - iS = IndexSeries('test_iS', list(), ts, unit='unit', timestamps=list()) + ts = TimeSeries( + name='test_ts', + data=[1, 2, 3], + unit='unit', + timestamps=[0.1, 0.2, 0.3] + ) + iS = IndexSeries( + name='test_iS', + data=[1, 2, 3], + unit='N/A', + indexed_timeseries=ts, + timestamps=[0.1, 0.2, 0.3] + ) self.assertEqual(iS.name, 'test_iS') - self.assertEqual(iS.unit, 'unit') - self.assertEqual(iS.indexed_timeseries, ts) + self.assertEqual(iS.unit, 'N/A') + self.assertIs(iS.indexed_timeseries, ts) class ImageMaskSeriesConstructor(TestCase): @@ -78,7 +106,7 @@ def test_init(self): format='tiff', timestamps=[1., 2.]) self.assertEqual(ims.name, 'test_ims') self.assertEqual(ims.unit, 'unit') - self.assertEqual(ims.masked_imageseries, iS) + self.assertIs(ims.masked_imageseries, iS) self.assertEqual(ims.external_file, ['external_file']) self.assertEqual(ims.starting_frame, [1, 2, 3]) self.assertEqual(ims.format, 'tiff') diff --git a/tests/unit/test_misc.py b/tests/unit/test_misc.py index 4412063ce..c8170d9fe 100644 --- a/tests/unit/test_misc.py +++ b/tests/unit/test_misc.py @@ -11,7 +11,7 @@ class AnnotationSeriesConstructor(TestCase): def test_init(self): - aS = AnnotationSeries('test_aS', timestamps=list()) + aS = AnnotationSeries('test_aS', data=[1, 2, 3], timestamps=list()) self.assertEqual(aS.name, 'test_aS') aS.add_annotation(2.0, 'comment') diff --git a/tests/unit/test_ogen.py b/tests/unit/test_ogen.py index 82f873de9..678f284dc 100644 --- a/tests/unit/test_ogen.py +++ b/tests/unit/test_ogen.py @@ -7,14 +7,25 @@ class OptogeneticSeriesConstructor(TestCase): def test_init(self): device = Device('name') - oS = OptogeneticStimulusSite('site1', device, 'description', 300., 'location') + oS = OptogeneticStimulusSite( + name='site1', + device=device, + description='description', + excitation_lambda=300., + location='location' + ) self.assertEqual(oS.name, 'site1') self.assertEqual(oS.device, device) self.assertEqual(oS.description, 'description') self.assertEqual(oS.excitation_lambda, 300.) self.assertEqual(oS.location, 'location') - iS = OptogeneticSeries('test_iS', list(), oS, timestamps=list()) + iS = OptogeneticSeries( + name='test_iS', + data=[1, 2, 3], + site=oS, + timestamps=[0.1, 0.2, 0.3] + ) self.assertEqual(iS.name, 'test_iS') self.assertEqual(iS.unit, 'watts') self.assertEqual(iS.site, oS) diff --git a/tests/unit/test_ophys.py b/tests/unit/test_ophys.py index a9dc9bc59..87e40a421 100644 --- a/tests/unit/test_ophys.py +++ b/tests/unit/test_ophys.py @@ -256,7 +256,7 @@ def test_init(self): format='tiff', timestamps=[1., 2.] ) - tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float) + tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64) ts = TimeSeries( name='xy_translation', data=list(range(len(tstamps))), @@ -280,10 +280,10 @@ def test_init(self): ts = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) self.assertEqual(ts.name, 'test_ts') self.assertEqual(ts.unit, 'unit') @@ -297,10 +297,10 @@ def test_init(self): rrs = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) dof = DfOverF(rrs) @@ -314,10 +314,10 @@ def test_init(self): ts = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) ff = Fluorescence(ts) diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py index cf946aa0d..d491fff47 100644 --- a/tests/validation/test_validate.py +++ b/tests/validation/test_validate.py @@ -39,7 +39,8 @@ def test_validate_file_no_cache_bad_ns(self): r"warnings.warn\(msg\)\s*" r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " r"Falling back to pynwb namespace information\.\s*" - r"The namespace 'notfound' could not be found in pynwb namespace information\.\s*" + r"The namespace 'notfound' could not be found in pynwb namespace information as only " + r"\['core'\] is present\.\s*" ) self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) @@ -63,7 +64,8 @@ def test_validate_file_cached_bad_ns(self): capture_output=True) stderr_regex = re.compile( - r"The namespace 'notfound' could not be found in cached namespace information\.\s*" + r"The namespace 'notfound' could not be found in cached namespace information as only " + r"\['core'\] is present\.\s*" ) self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) @@ -75,17 +77,11 @@ def test_validate_file_cached_hdmf_common(self): capture_output=True) stderr_regex = re.compile( - r".*ValueError: data type \'NWBFile\' not found in namespace hdmf-common.\s*", - re.DOTALL + r"The namespace 'hdmf-common' is included by the namespace 'core'\. Please validate against that " + r"namespace instead\.\s*", ) self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) - stdout_regex = re.compile( - r"Validating tests/back_compat/1.1.2_nwbfile.nwb against cached namespace information using namespace " - r"'hdmf-common'.\s*" - ) - self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) - def test_validate_file_cached_ignore(self): """Test that validating a file with cached spec against the core namespace succeeds.""" result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --no-cached-namespace", diff --git a/tox.ini b/tox.ini index a8792efbf..390204ad6 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py36, py37, py38, py39 +envlist = py37, py38, py39 [testenv] usedevelop = True @@ -22,32 +22,32 @@ commands = # Env to create coverage report locally [testenv:localcoverage] -basepython = python3.8 +basepython = python3.9 commands = python -m coverage run test.py --pynwb coverage html -d tests/coverage/htmlcov -# Test with python 3.8, pinned dev reqs, and upgraded run requirements -[testenv:py38-upgrade-dev] -basepython = python3.8 +# Test with python 3.9, pinned dev reqs, and upgraded run requirements +[testenv:py39-upgrade-dev] +basepython = python3.9 install_command = pip install -U -e . {opts} {packages} deps = -rrequirements-dev.txt commands = {[testenv]commands} -# Test with python 3.8, pinned dev reqs, and pre-release run requirements -[testenv:py38-upgrade-dev-pre] -basepython = python3.8 +# Test with python 3.9, pinned dev reqs, and pre-release run requirements +[testenv:py39-upgrade-dev-pre] +basepython = python3.9 install_command = pip install -U --pre -e . {opts} {packages} deps = -rrequirements-dev.txt commands = {[testenv]commands} -# Test with python 3.6, pinned dev reqs, and minimum run requirements -[testenv:py36-min-req] -basepython = python3.6 +# Test with python 3.7, pinned dev reqs, and minimum run requirements +[testenv:py37-min-req] +basepython = python3.7 deps = -rrequirements-dev.txt -rrequirements-min.txt @@ -59,10 +59,6 @@ commands = python setup.py sdist python setup.py bdist_wheel -[testenv:build-py36] -basepython = python3.6 -commands = {[testenv:build]commands} - [testenv:build-py37] basepython = python3.7 commands = {[testenv:build]commands} @@ -75,24 +71,24 @@ commands = {[testenv:build]commands} basepython = python3.9 commands = {[testenv:build]commands} -[testenv:build-py38-upgrade-dev] -basepython = python3.8 +[testenv:build-py39-upgrade-dev] +basepython = python3.9 install_command = pip install -U -e . {opts} {packages} deps = -rrequirements-dev.txt commands = {[testenv:build]commands} -[testenv:build-py38-upgrade-dev-pre] -basepython = python3.8 +[testenv:build-py39-upgrade-dev-pre] +basepython = python3.9 install_command = pip install -U --pre -e . {opts} {packages} deps = -rrequirements-dev.txt commands = {[testenv:build]commands} -[testenv:build-py36-min-req] -basepython = python3.6 +[testenv:build-py37-min-req] +basepython = python3.7 deps = -rrequirements-dev.txt -rrequirements-min.txt @@ -111,16 +107,15 @@ install_command = deps = -rrequirements-dev.txt -rrequirements.txt - -rrequirements-doc.txt commands = + pip install -r requirements-doc.txt + # installing allensdk may downgrade certain requirements so + # reinstall the repo with its requirements + pip install -r requirements-dev.txt -r requirements.txt + pip install -U -e . python test.py --example -[testenv:gallery-py36] -basepython = python3.6 -deps = {[testenv:gallery]deps} -commands = {[testenv:gallery]commands} - [testenv:gallery-py37] basepython = python3.7 deps = {[testenv:gallery]deps} @@ -136,38 +131,52 @@ basepython = python3.9 deps = {[testenv:gallery]deps} commands = {[testenv:gallery]commands} +# Test with python 3.9, pinned dev and doc reqs, and upgraded run requirements +[testenv:gallery-py39-upgrade-dev] +basepython = python3.9 -# Test with python 3.8, pinned dev and doc reqs, and upgraded run requirements -[testenv:gallery-py38-upgrade-dev] -basepython = python3.8 install_command = pip install -U -e . {opts} {packages} + deps = -rrequirements-dev.txt - -rrequirements-doc.txt -commands = {[testenv:gallery]commands} -# Test with python 3.8, pinned dev and doc reqs, and pre-release run requirements -[testenv:gallery-py38-upgrade-dev-pre] -basepython = python3.8 +commands = + pip install -r requirements-doc.txt + # installing allensdk may downgrade certain requirements so + # reinstall the repo with its requirements + pip install -r requirements-dev.txt -r requirements.txt + pip install -U -e . + python test.py --example + +# Test with python 3.9, pinned dev and doc reqs, and pre-release run requirements +[testenv:gallery-py39-upgrade-dev-pre] +basepython = python3.9 + install_command = pip install -U --pre -e . {opts} {packages} + deps = -rrequirements-dev.txt - -rrequirements-doc.txt -commands = {[testenv:gallery]commands} -# Test with python 3.6, pinned dev reqs, and minimum run requirements -[testenv:gallery-py36-min-req] -basepython = python3.6 +commands = + pip install -r requirements-doc.txt + # installing allensdk may downgrade certain requirements so + # reinstall the repo with its requirements + pip install -r requirements-dev.txt -r requirements.txt + pip install -U --pre -e . + python test.py --example + +# Test with python 3.7, pinned dev reqs, and minimum run requirements +[testenv:gallery-py37-min-req] +basepython = python3.7 deps = -rrequirements-dev.txt -rrequirements-min.txt - -rrequirements-doc.txt commands = {[testenv:gallery]commands} -[testenv:validation-py38] -basepython = python3.8 +[testenv:validation-py39] +basepython = python3.9 install_command = pip install -U {opts} {packages} deps =