Skip to content
This repository was archived by the owner on Dec 8, 2023. It is now read-only.

Commit af0f6d4

Browse files
kushalbakshikushalbakshi
authored
Merge pull request #91 from kabilar/main
Add GitHub Codespace environment and demo notebooks
2 parents d9e26ac + bdcc460 commit af0f6d4

33 files changed

+557
-3068
lines changed

‎.devcontainer/Dockerfile

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM mcr.microsoft.com/devcontainers/python:3.9-bullseye
2+
3+
COPY ./requirements.txt /tmp/
4+
RUN \
5+
# dev setup
6+
apt update && \
7+
apt-get install bash-completion graphviz default-mysql-client s3fs procps -y && \
8+
pip install --no-cache-dir --upgrade pip && \
9+
# workflow dependencies
10+
apt-get install gcc ffmpeg libsm6 libxext6 -y && \
11+
pip install --no-cache-dir -r /tmp/requirements.txt && \
12+
# clean up
13+
rm /tmp/requirements.txt && \
14+
apt-get clean
15+
16+
ENV DJ_HOST fakeservices.datajoint.io
17+
ENV DJ_USER root
18+
ENV DJ_PASS simple
19+
20+
ENV EPHYS_MODE acute
21+
ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-array-ephys/example_data
22+
ENV RAW_ROOT_DATA_DIR /workspaces/workflow-array-ephys/example_data/raw
23+
ENV PROCESSED_ROOT_DATA_DIR /workspaces/workflow-array-ephys/example_data/processed
24+
ENV DATABASE_PREFIX neuro_

‎.devcontainer/devcontainer.json

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// For format details, see https://aka.ms/devcontainer.json.
2+
{
3+
"name": "Remote Demo",
4+
"dockerComposeFile": "docker-compose.yaml",
5+
"service": "app",
6+
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
7+
// Use this environment variable if you need to bind mount your local source code into a new container.
8+
"remoteEnv": {
9+
"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
10+
},
11+
// https://containers.dev/features
12+
"features": {
13+
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
14+
"ghcr.io/devcontainers/features/git:1": {},
15+
},
16+
"onCreateCommand": "mkdir -p ${DJ_PUBLIC_S3_MOUNT_PATH} && pip install -e .",
17+
"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${DJ_PUBLIC_S3_MOUNT_PATH} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1 && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
18+
"hostRequirements": {
19+
"cpus": 4,
20+
"memory": "8gb"
21+
},
22+
"forwardPorts": [
23+
3306
24+
],
25+
"customizations": {
26+
"vscode": {
27+
"extensions": [
28+
"ms-python.python"
29+
]
30+
}
31+
}
32+
}

‎.devcontainer/docker-compose.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
version: "3"
2+
services:
3+
app:
4+
extends:
5+
file: ./local/docker-compose.yaml
6+
service: app
7+
environment:
8+
- DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-array-ephys-benchmark/v2
9+
devices:
10+
- /dev/fuse
11+
cap_add:
12+
- SYS_ADMIN
13+
security_opt:
14+
- apparmor:unconfined

‎.devcontainer/local/devcontainer.json

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// For format details, see https://aka.ms/devcontainer.json.
2+
{
3+
"name": "Local Demo",
4+
"dockerComposeFile": "docker-compose.yaml",
5+
"service": "app",
6+
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
7+
// Use this environment variable if you need to bind mount your local source code into a new container.
8+
"remoteEnv": {
9+
"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
10+
},
11+
// https://containers.dev/features
12+
"features": {
13+
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
14+
"ghcr.io/devcontainers/features/git:1": {},
15+
},
16+
"onCreateCommand": "pip install -e .",
17+
"postStartCommand": "docker volume prune -f && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
18+
"hostRequirements": {
19+
"cpus": 4,
20+
"memory": "8gb"
21+
},
22+
"forwardPorts": [
23+
3306
24+
],
25+
"customizations": {
26+
"vscode": {
27+
"extensions": [
28+
"ms-python.python"
29+
]
30+
}
31+
}
32+
}

‎.devcontainer/local/docker-compose.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
version: "3"
2+
services:
3+
app:
4+
cpus: 4
5+
mem_limit: 8g
6+
build:
7+
context: ../..
8+
dockerfile: ./.devcontainer/Dockerfile
9+
extra_hosts:
10+
- fakeservices.datajoint.io:127.0.0.1
11+
volumes:
12+
- ../../..:/workspaces:cached
13+
entrypoint: /usr/local/share/docker-init.sh
14+
command: tail -f /dev/null

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,5 @@ docker-compose.yml
108108

109109
# temporary figures
110110
temp_ephys_figures/
111+
112+
example_data

‎CHANGELOG.md

+4-3
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33
Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
44
[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
55

6-
## [0.2.7] - Unreleased
6+
## [0.3.0] - 2023-03-09
77

8-
+ Update - Revise pytest and docker structure to streamline testing
8+
+ Add - Demo notebooks and capabilities using GitHub Codespaces
99
+ Add - pre-commit, markdown lint, and spell check config files
10-
+ Bugfix - `element-array-ephys` version error when installing via requirements.txt
10+
+ Update - Revise pytest and docker structure to streamline testing
11+
+ Fix - `element-array-ephys` version error when installing via requirements.txt
1112

1213
## [0.2.6] - 2022-01-12
1314

‎README.md

+21-89
Original file line numberDiff line numberDiff line change
@@ -1,113 +1,45 @@
11
# DataJoint Workflow - Array Electrophysiology
22

3-
Workflow for extracellular array electrophysiology data acquired with a polytrode probe
3+
DataJoint Workflow for extracellular array electrophysiology combines multiple DataJoint Elements to process data acquired with a polytrode probe
44
(e.g. [Neuropixels](https://www.neuropixels.org), Neuralynx) using the
55
[SpikeGLX](https://github.com/billkarsh/SpikeGLX) or
66
[OpenEphys](https://open-ephys.org/gui) acquisition software and processed with
77
[MATLAB-based Kilosort](https://github.com/MouseLand/Kilosort) or [python-based
8-
Kilosort](https://github.com/MouseLand/pykilosort) spike sorting software.
8+
Kilosort](https://github.com/MouseLand/pykilosort) spike sorting software. DataJoint Elements collectively standardize and automate data collection and analysis for neuroscience experiments. Each Element is a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a fully functional pipeline.
99

10-
A complete electrophysiology workflow can be built using the DataJoint Elements.
10+
To get started, see below for an [interactive tutorial](#interactive-tutorial) on GitHub Codespaces. More information can be found at the
11+
[Element documentation page](https://datajoint.com/docs/elements/element-array-ephys).
1112

12-
+ [element-lab](https://github.com/datajoint/element-lab)
13-
+ [element-animal](https://github.com/datajoint/element-animal)
14-
+ [element-session](https://github.com/datajoint/element-session)
15-
+ [element-array-ephys](https://github.com/datajoint/element-array-ephys)
13+
## Experiment flowchart
1614

17-
This repository provides demonstrations for:
15+
![flowchart](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/diagram_flowchart.svg)
1816

19-
1. Set up a workflow using DataJoint Elements (see
20-
[workflow_array_ephys/pipeline.py](workflow_array_ephys/pipeline.py))
17+
## Data pipeline for acute experiment
2118

22-
2. Ingestion of data/metadata based on a predefined file structure, file naming
23-
convention, and directory lookup methods (see
24-
[workflow_array_ephys/paths.py](workflow_array_ephys/paths.py)).
19+
![datajoint](https://raw.githubusercontent.com/datajoint/workflow-array-ephys/main/images/attached_array_ephys_element.svg)
2520

26-
3. Ingestion of clustering results.
21+
## Interactive tutorial
2722

28-
4. Export of `no_curation` schema to NWB and DANDI (see
29-
[notebooks/09-NWB-export.ipynb](notebooks/09-NWB-export.ipynb)).
23+
The easiest way to learn about DataJoint Elements is to use the tutorial notebook within a [GitHub Codespace](https://docs.github.com/en/codespaces/overview). Please follow the steps below for the best experience:
3024

31-
See the [Element Array Electrophysiology documentation](https://elements.datajoint.org/description/array_ephys/)
32-
for the background information and development timeline.
25+
1. Fork this repository to your GitHub account.
3326

34-
For more information on the DataJoint Elements project, please visit
35-
<https://elements.datajoint.org>. This work is supported by the National Institutes of
36-
Health.
27+
2. Select the green `Code` button.
3728

38-
## Workflow architecture
29+
3. Within the dropdown menu, select the `Codespaces` tab.
3930

40-
The electrophysiology workflow presented here uses components from 4 DataJoint Elements
41-
([element-lab](https://github.com/datajoint/element-lab),
42-
[element-animal](https://github.com/datajoint/element-animal),
43-
[element-session](https://github.com/datajoint/element-session),
44-
[element-array-ephys](https://github.com/datajoint/element-array-ephys)) assembled
45-
together to form a fully functional workflow. Note that element-array-ephys offers three
46-
schema options, selected via the DataJoint config file, with
47-
`dj.config['custom']['ephys_mode']`
31+
4. Select the green `Create codespace on main` button.
4832

49-
+ `acute` probe insertion, with curated clustering
50-
+ `chronic` probe insertion, with curated clustering
51-
+ `no-curation`, acute probe insertion with kilosort triggered clustering and supported
52-
NWB export
53-
+ `precluster`, acute probe insertion with pre-processing steps prior to clustering and
54-
curated clustering
33+
5. The environment is ready when a Visual Studio Code window is rendered within your browser. This takes ~5 minutes the first time being launched, and ~1 minute if you revisit this Codespace.
5534

56-
![Ephys Diagram](https://raw.githubusercontent.com/datajoint/workflow-array-ephys/main/images/attached_array_ephys_element.svg)
35+
6. Navigate to the `notebooks` directory on the left panel and open the `tutorial.ipynb` Jupyter notebook. Execute the cells in this notebook to begin your walk through the tutorial.
5736

58-
Optionally, this can be used in conjunction with
59-
[element-event](https://github.com/datajoint/element-event)
60-
and [element-electrode-localization](https://github.com/datajoint/element-electrode-localization/).
37+
7. Once you are done, GitHub will automatically stop the Codespace after 30 minutes of inactivity or you can manually stop the Codespace.
6138

62-
![Diagram with trial schema](https://raw.githubusercontent.com/datajoint/workflow-array-ephys/main/images/attached_trial_analysis.svg)
39+
8. After stopping the Codespace, we recommend deleting the Codespace to save on storage costs, which are free for the first 15 GB-month.
6340

64-
![Diagram with localization schema](https://raw.githubusercontent.com/datajoint/workflow-array-ephys/main/images/attached_electrode_localization.svg)
41+
+ If you are new to GitHub and run into any errors, please contact us via email at support@datajoint.com. If you are experienced with GitHub, please create an issue on the upstream repository or issue a pull request with a thorough explanantion of the error and proposed solution.
6542

66-
## Installation instructions
43+
**Please Note:**
6744

68-
The installation instructions can be found at the
69-
[DataJoint Elements documentation](https://elements.datajoint.org/usage/install/).
70-
71-
## Interacting with the DataJoint workflow
72-
73-
Please refer to the workflow-specific
74-
[Jupyter notebooks](/notebooks)
75-
for an in-depth explanation of how to ...
76-
77-
1. Run the workflow ([03-process.ipynb](notebooks/03-process.ipynb)).
78-
79-
2. Explore the data ([05-explore.ipynb](notebooks/05-explore.ipynb)).
80-
81-
3. Examine trialized analyses, and establish downstream analyses
82-
([07-downstream-analysis.ipynb](notebooks/07-downstream-analysis.ipynb))
83-
84-
4. Locate probes within the
85-
[Common Coordinate Framework](https://www.sciencedirect.com/science/article/pii/S0092867420304025)
86-
([08-electrode-localization.ipynb](notebooks/08-electrode-localization.ipynb))
87-
88-
5. Export to NWB and DANDI ([09-NWB-export.ipynb](notebooks/09-NWB-export.ipynb))
89-
90-
See our YouTube tutorial for a walkthrough of the schemas and functions:
91-
[![YouTube tutorial](https://img.youtube.com/vi/KQlGYOBq7ow/0.jpg)](https://www.youtube.com/watch?v=KQlGYOBq7ow)
92-
93-
## Citation
94-
95-
If your work uses DataJoint and DataJoint Elements, please cite the respective Research
96-
Resource Identifiers (RRIDs) and manuscripts.
97-
98-
+ DataJoint for Python or MATLAB
99-
+ Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton
100-
RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or
101-
Python. bioRxiv. 2015 Jan 1:031658. doi: <https://doi.org/10.1101/031658>
102-
103-
+ DataJoint ([RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543)) - DataJoint
104-
for `<Select Python or MATLAB>` (version `<Enter version number>`)
105-
106-
+ DataJoint Elements
107-
+ Yatsenko D, Nguyen T, Shen S, Gunalan K, Turner CA, Guzman R, Sasaki M, Sitonic D,
108-
Reimer J, Walker EY, Tolias AS. DataJoint Elements: Data Workflows for
109-
Neurophysiology. bioRxiv. 2021 Jan 1. doi:
110-
<https://doi.org/10.1101/2021.03.30.437358>
111-
112-
+ DataJoint Elements ([RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)) -
113-
Element Array Electrophysiology (version `<Enter version number>`)
45+
+ GitHub Codespaces are limited to 120 core-hours per month and 15 GB-month for free users. Once you exceed this limit, you will have to wait for the hours to reset or pay to use Codespaces.

‎docker-compose.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# MYSQL_VER=8.0 docker compose up --build
2+
version: "2.4"
3+
services:
4+
db:
5+
restart: always
6+
image: datajoint/mysql:${MYSQL_VER}
7+
environment:
8+
- MYSQL_ROOT_PASSWORD=${DJ_PASS}
9+
ports:
10+
- "3306:3306"
11+
healthcheck:
12+
test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
13+
timeout: 15s
14+
retries: 10
15+
interval: 15s

0 commit comments

Comments
 (0)