diff --git a/docs/listofmaps.rst b/docs/listofmaps.rst index dc87a08..19830b8 100644 --- a/docs/listofmaps.rst +++ b/docs/listofmaps.rst @@ -37,29 +37,6 @@ zhu2018-typedensity-allenccfv3-region # region mapping file # source-zhu2018_regionmapping.csv -zhu2018-similarity-allenccfv3-region ------------------------------------- - -**Description**: Synaptic similarity - -**Format**: matrix - -**How to use** - -.. code:: python - - # get annotation - fetch_annotation(source='zhu2018', desc='similarity', space='allenccfv3', res='region') - - # file location - # $MOUSEMAPS_DATA/zhu2018 - - # file name - # source-zhu2018_desc-similarity_space-allenccfv3_res-region_matrix.csv - - # region mapping file - # source-zhu2018_regionmapping.csv - **References** ---- @@ -163,6 +140,52 @@ oh2014-pvalcontra-allenccfv3-region # region mapping file # source-oh2014_regionmapping.csv +oh2014-distipsi-allenccfv3-region +--------------------------------- + +**Description**: Distance (mm) for ipsilateral projections + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='oh2014', desc='distipsi', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/oh2014 + + # file name + # source-oh2014_desc-distipsi_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-oh2014_regionmapping.csv + +oh2014-distcontra-allenccfv3-region +----------------------------------- + +**Description**: Distance (mm) for contralateral projections + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='oh2014', desc='distcontra', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/oh2014 + + # file name + # source-oh2014_desc-distcontra_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-oh2014_regionmapping.csv + **References** ---- @@ -226,7 +249,7 @@ lein2006amba-sagittalenergy-allenccfv3-region # $MOUSEMAPS_DATA/lein2006amba # file name - # source-lein2006amba_desc-sagittalenergy_space-allenccfv3_res-region_tabular.csv + # source-lein2006amba_desc-sagittalenergy_space-allenccfv3_res-region_tabular.csv.gz # region mapping file # source-lein2006amba_regionmapping.csv @@ -249,7 +272,99 @@ lein2006amba-coronalenergy-allenccfv3-region # $MOUSEMAPS_DATA/lein2006amba # file name - # source-lein2006amba_desc-coronalenergy_space-allenccfv3_res-region_tabular.csv + # source-lein2006amba_desc-coronalenergy_space-allenccfv3_res-region_tabular.csv.gz + + # region mapping file + # source-lein2006amba_regionmapping.csv + +lein2006amba-sagittaldensity-allenccfv3-region +---------------------------------------------- + +**Description**: Expression density of sagittal slices + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='lein2006amba', desc='sagittaldensity', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/lein2006amba + + # file name + # source-lein2006amba_desc-sagittaldensity_space-allenccfv3_res-region_tabular.csv.gz + + # region mapping file + # source-lein2006amba_regionmapping.csv + +lein2006amba-coronaldensity-allenccfv3-region +--------------------------------------------- + +**Description**: Expression density of coronal slices + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='lein2006amba', desc='coronaldensity', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/lein2006amba + + # file name + # source-lein2006amba_desc-coronaldensity_space-allenccfv3_res-region_tabular.csv.gz + + # region mapping file + # source-lein2006amba_regionmapping.csv + +lein2006amba-sagittalintensity-allenccfv3-region +------------------------------------------------ + +**Description**: Expression intensity of sagittal slices + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='lein2006amba', desc='sagittalintensity', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/lein2006amba + + # file name + # source-lein2006amba_desc-sagittalintensity_space-allenccfv3_res-region_tabular.csv.gz + + # region mapping file + # source-lein2006amba_regionmapping.csv + +lein2006amba-coronalintensity-allenccfv3-region +----------------------------------------------- + +**Description**: Expression intensity of coronal slices + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='lein2006amba', desc='coronalintensity', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/lein2006amba + + # file name + # source-lein2006amba_desc-coronalintensity_space-allenccfv3_res-region_tabular.csv.gz # region mapping file # source-lein2006amba_regionmapping.csv @@ -403,6 +518,282 @@ yao2023abca-impsubsmean-allenccfv3-region # region mapping file # source-yao2023abca_substructure_regionmapping.csv +yao2023abca-divictclass-allenccfv3-region +----------------------------------------- + +**Description**: Cell type (class) at the division level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='divictclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-divictclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_division_regionmapping.csv + +yao2023abca-structclass-allenccfv3-region +----------------------------------------- + +**Description**: Cell type (class) at the structure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='structclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-structclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_structure_regionmapping.csv + +yao2023abca-subsctclass-allenccfv3-region +----------------------------------------- + +**Description**: Cell type (class) at the substructure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='subsctclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-subsctclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_substructure_regionmapping.csv + +yao2023abca-divictsubclass-allenccfv3-region +-------------------------------------------- + +**Description**: Cell type (subclass) at the division level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='divictsubclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-divictsubclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_division_regionmapping.csv + +yao2023abca-structsubclass-allenccfv3-region +-------------------------------------------- + +**Description**: Cell type (subclass) at the structure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='structsubclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-structsubclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_structure_regionmapping.csv + +yao2023abca-subsctsubclass-allenccfv3-region +-------------------------------------------- + +**Description**: Cell type (subclass) at the substructure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='subsctsubclass', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-subsctsubclass_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_substructure_regionmapping.csv + +yao2023abca-divictsupertype-allenccfv3-region +--------------------------------------------- + +**Description**: Cell type (supertype) at the division level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='divictsupertype', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-divictsupertype_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_division_regionmapping.csv + +yao2023abca-structsupertype-allenccfv3-region +--------------------------------------------- + +**Description**: Cell type (supertype) at the structure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='structsupertype', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-structsupertype_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_structure_regionmapping.csv + +yao2023abca-subsctsupertype-allenccfv3-region +--------------------------------------------- + +**Description**: Cell type (supertype) at the substructure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='subsctsupertype', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-subsctsupertype_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_substructure_regionmapping.csv + +yao2023abca-divictcluster-allenccfv3-region +------------------------------------------- + +**Description**: Cell type (cluster) at the division level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='divictcluster', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-divictcluster_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_division_regionmapping.csv + +yao2023abca-structcluster-allenccfv3-region +------------------------------------------- + +**Description**: Cell type (cluster) at the structure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='structcluster', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-structcluster_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_structure_regionmapping.csv + +yao2023abca-subsctcluster-allenccfv3-region +------------------------------------------- + +**Description**: Cell type (cluster) at the substructure level + +**Format**: tabular + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='yao2023abca', desc='subsctcluster', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/yao2023abca + + # file name + # source-yao2023abca_desc-subsctcluster_space-allenccfv3_res-region_tabular.csv + + # region mapping file + # source-yao2023abca_substructure_regionmapping.csv + **References** ---- @@ -483,4 +874,199 @@ zhang2023abca-subsmean-allenccfv3-region # region mapping file # source-zhang2023abca_substructure_regionmapping.csv +**References** + +---- + +hi-res connectome (knox2018) +============================ + +**Full description** + + High-resolution data-driven model of the mouse connectome + +knox2018-conndencontra-allenccfv3-region +---------------------------------------- + +**Description**: Connection density (contralateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='conndencontra', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-conndencontra_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-conndenipsi-allenccfv3-region +-------------------------------------- + +**Description**: Connection density (ipsilateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='conndenipsi', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-conndenipsi_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-connstrcontra-allenccfv3-region +---------------------------------------- + +**Description**: Connection strength (contralateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='connstrcontra', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-connstrcontra_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-connstripsi-allenccfv3-region +-------------------------------------- + +**Description**: Connection strength (ipsilateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='connstripsi', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-connstripsi_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-normconndencontra-allenccfv3-region +-------------------------------------------- + +**Description**: Normalized connection density (contralateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='normconndencontra', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-normconndencontra_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-normconndenipsi-allenccfv3-region +------------------------------------------ + +**Description**: Normalized connection density (ipsilateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='normconndenipsi', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-normconndenipsi_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-normconnstrcontra-allenccfv3-region +-------------------------------------------- + +**Description**: Normalized connection strength (contralateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='normconnstrcontra', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-normconnstrcontra_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + +knox2018-normconnstripsi-allenccfv3-region +------------------------------------------ + +**Description**: Normalized connection strength (ipsilateral) + +**Format**: matrix + +**How to use** + +.. code:: python + + # get annotation + fetch_annotation(source='knox2018', desc='normconnstripsi', space='allenccfv3', res='region') + + # file location + # $MOUSEMAPS_DATA/knox2018 + + # file name + # source-knox2018_desc-normconnstripsi_space-allenccfv3_res-region_matrix.csv + + # region mapping file + # source-knox2018_regionmapping.csv + **References** \ No newline at end of file diff --git a/neuromaps_mouse/datasets/atlases.py b/neuromaps_mouse/datasets/atlases.py index 4c828dd..fa86c1c 100644 --- a/neuromaps_mouse/datasets/atlases.py +++ b/neuromaps_mouse/datasets/atlases.py @@ -6,10 +6,12 @@ get_data_dir ) + def get_atlas_dir(data_dir=None): data_dir = get_data_dir(data_dir=data_dir) return data_dir / "atlases" + def fetch_allenccfv3(which=None, return_single=True, data_dir=None, verbose=1): data_dir = get_data_dir(data_dir=data_dir) atlas = MOUSEMAPS_ATLASES["allen-ccfv3"] diff --git a/neuromaps_mouse/datasets/data/annotations-meta.json b/neuromaps_mouse/datasets/data/annotations-meta.json index 5ad6c20..34830c5 100644 --- a/neuromaps_mouse/datasets/data/annotations-meta.json +++ b/neuromaps_mouse/datasets/data/annotations-meta.json @@ -5,8 +5,7 @@ "name": "synaptome", "description": "Architecture of the Mouse Brain Synaptome", "file_desc": { - "typedensity": "Synaptic type density", - "similarity": "Synaptic similarity" + "typedensity": "Synaptic type density" }, "comments": "Original source: ", "refs": [ @@ -45,7 +44,9 @@ "wipsi": "Weighted ipsilateral strength index", "pvalipsi": "P values for wipsi", "wcontra": "Weighted contralateral strength index", - "pvalcontra": "P values for wcontra" + "pvalcontra": "P values for wcontra", + "distipsi": "Distance (mm) for ipsilateral projections", + "distcontra": "Distance (mm) for contralateral projections" }, "comments": "Original source: Supplementary Table 3", "refs": [ @@ -148,7 +149,11 @@ "description": "Allen Mouse Brain Atlas", "file_desc": { "sagittalenergy": "Expression energy of sagittal slices", - "coronalenergy": "Expression energy of coronal slices" + "coronalenergy": "Expression energy of coronal slices", + "sagittaldensity": "Expression density of sagittal slices", + "coronaldensity": "Expression density of coronal slices", + "sagittalintensity": "Expression intensity of sagittal slices", + "coronalintensity": "Expression intensity of coronal slices" }, "comments": "", "warning": "This is a very large dataset, and may take a long time to download.", @@ -232,16 +237,16 @@ "impdivimean": "Average imputed regional gene expressions at the division level", "impstrumean": "Average imputed regional gene expressions at the structure level", "impsubsmean": "Average imputed regional gene expressions at the substructure level", - "divctclass": "Cell type (class) at the division level", + "divictclass": "Cell type (class) at the division level", "structclass": "Cell type (class) at the structure level", "subsctclass": "Cell type (class) at the substructure level", - "divctsubclass": "Cell type (subclass) at the division level", + "divictsubclass": "Cell type (subclass) at the division level", "structsubclass": "Cell type (subclass) at the structure level", "subsctsubclass": "Cell type (subclass) at the substructure level", - "divctsupertype": "Cell type (supertype) at the division level", + "divictsupertype": "Cell type (supertype) at the division level", "structsupertype": "Cell type (supertype) at the structure level", "subsctsupertype": "Cell type (supertype) at the substructure level", - "divctcluster": "Cell type (cluster) at the division level", + "divictcluster": "Cell type (cluster) at the division level", "structcluster": "Cell type (cluster) at the structure level", "subsctcluster": "Cell type (cluster) at the substructure level" }, diff --git a/neuromaps_mouse/datasets/utils.py b/neuromaps_mouse/datasets/utils.py index d5417e9..a0342b1 100644 --- a/neuromaps_mouse/datasets/utils.py +++ b/neuromaps_mouse/datasets/utils.py @@ -14,6 +14,7 @@ def get_data_dir(data_dir=None): + """Get the data directory.""" if data_dir is None: data_dir = os.environ.get( "MOUSEMAPS_DATA", str(Path.home() / "neuromaps-mouse-data") @@ -63,6 +64,7 @@ def _osfify_url(osf_file_id): def fetch_files(annotations, file_type="annotations", data_dir=None, verbose=1): + """Fetch files from OSF.""" targ_fname_list = [] for annot in annotations: if file_type in ["annotations", "annotations-meta"]: @@ -139,7 +141,7 @@ def _filter_annots_by_keys(keys_dict): return filtered -def _check_json(osfstorage_data): +def _check_json(osfstorage_data, overwrite=False): """ Check for errors in meta.json. @@ -151,9 +153,11 @@ def _check_json(osfstorage_data): """ # reload the datasets and meta json files from rich.console import Console + import importlib.resources console = Console() + # Load JSON files MOUSEMAPS_ATLASES = _load_resource_json("datasets/data/atlases.json")["atlases"] MOUSEMAPS_ANNOTS = _load_resource_json("datasets/data/annotations.json")[ "annotations" @@ -162,6 +166,11 @@ def _check_json(osfstorage_data): "annotations-meta" ] + # Track if any changes were made + atlases_updated = False + annots_updated = False + annots_meta_updated = False + console.print("ATLASES") for atlas_k, atlas_v in MOUSEMAPS_ATLASES.items(): console.print(f"{atlas_k} >") @@ -170,21 +179,35 @@ def _check_json(osfstorage_data): if file_v["checksum"] == osfstorage_data[file_v["fname"]]["md5"]: console.print(" [bold green]✓[/bold green] checksum") else: - console.print( - f" [bold red]x[/bold red] checksum local: {file_v['checksum']} remote: {osfstorage_data[file_v['fname']]['md5']}" - ) + if overwrite: + file_v["checksum"] = osfstorage_data[file_v["fname"]]["md5"] + atlases_updated = True + console.print(" [bold yellow]↻[/bold yellow] checksum updated") + else: + console.print( + f" [bold red]x[/bold red] " + f"checksum local: {file_v['checksum']} " + f"remote: {osfstorage_data[file_v['fname']]['md5']}" + ) if file_v["url"]["osf"] == osfstorage_data[file_v["fname"]]["guid"]: console.print(" [bold green]✓[/bold green] url") else: - console.print( - f" [bold red]x[/bold red] url local: {file_v['url']['osf']} remote: {osfstorage_data[file_v['fname']]['guid']}" - ) + if overwrite: + file_v["url"]["osf"] = osfstorage_data[file_v["fname"]]["guid"] + atlases_updated = True + console.print(" [bold yellow]↻[/bold yellow] url updated") + else: + console.print( + f" [bold red]x[/bold red] " + f"url local: {file_v['url']['osf']} " + f"remote: {osfstorage_data[file_v['fname']]['guid']}" + ) console.print("\nANNOTS_META") for annot_meta in MOUSEMAPS_ANNOTS_META: console.print(f"{annot_meta['source']} {annot_meta['name']} >") - console.print(" \[annot files] >") + console.print(r" \[annot files] >") for file_v in annot_meta["files"]: console.print(f" {'-'.join(file_v)} >") try: @@ -197,7 +220,7 @@ def _check_json(osfstorage_data): else: console.print(" [bold red]x[/bold red] json") - console.print(" \[aux files] >") + console.print(r" \[aux files] >") for aux_k, aux_v in annot_meta["aux_files"].items(): console.print(f" {aux_k} >") @@ -207,21 +230,40 @@ def _check_json(osfstorage_data): console.print(f" {file_v['fname']} >") if file_v["fname"] not in osfstorage_data: console.print( - f" [bold red]x[/bold red] {file_v['fname']} not found in osfstorage" + f" [bold red]x[/bold red] " + f"{file_v['fname']} not found in osfstorage" ) continue if file_v["checksum"] == osfstorage_data[file_v["fname"]]["md5"]: console.print(" [bold green]✓[/bold green] checksum") else: - console.print( - f" [bold red]x[/bold red] checksum local: {file_v['checksum']} remote: {osfstorage_data[file_v['fname']]['md5']}" - ) + if overwrite: + file_v["checksum"] = osfstorage_data[file_v["fname"]]["md5"] + annots_meta_updated = True + console.print( + " [bold yellow]↻[/bold yellow] checksum updated" + ) + else: + console.print( + f" [bold red]x[/bold red] " + f"checksum local: {file_v['checksum']} " + f"remote: {osfstorage_data[file_v['fname']]['md5']}" + ) if file_v["url"]["osf"] == osfstorage_data[file_v["fname"]]["guid"]: console.print(" [bold green]✓[/bold green] url") else: - console.print( - f" [bold red]x[/bold red] url local: {file_v['url']['osf']} remote: {osfstorage_data[file_v['fname']]['guid']}" - ) + if overwrite: + file_v["url"]["osf"] = osfstorage_data[file_v["fname"]]["guid"] + annots_meta_updated = True + console.print( + " [bold yellow]↻[/bold yellow] url updated" + ) + else: + console.print( + f" [bold red]x[/bold red] " + f"url local: {file_v['url']['osf']} " + f"remote: {osfstorage_data[file_v['fname']]['guid']}" + ) console.print("\nANNOTS") for annot in MOUSEMAPS_ANNOTS: @@ -230,20 +272,125 @@ def _check_json(osfstorage_data): ) console.print(f" {annotstr} >") if annot["fname"] not in osfstorage_data: - console.print(f" [bold red]x[/bold red] {annot['fname']} not found in osfstorage") + console.print( + f" [bold red]x[/bold red] " + f"{annot['fname']} not found in osfstorage" + ) continue if annot["checksum"] == osfstorage_data[annot["fname"]]["md5"]: console.print(" [bold green]✓[/bold green] checksum") else: - console.print( - f" [bold red]x[/bold red] checksum local: {annot['checksum']} remote: {osfstorage_data[annot['fname']]['md5']}" - ) + if overwrite: + annot["checksum"] = osfstorage_data[annot["fname"]]["md5"] + annots_updated = True + console.print(" [bold yellow]↻[/bold yellow] checksum updated") + else: + console.print( + f" [bold red]x[/bold red] " + f"checksum local: {annot['checksum']} " + f"remote: {osfstorage_data[annot['fname']]['md5']}" + ) if annot["url"]["osf"] == osfstorage_data[annot["fname"]]["guid"]: console.print(" [bold green]✓[/bold green] url") else: - console.print( - f" [bold red]x[/bold red] url local: {annot['url']['osf']} remote: {osfstorage_data[annot['fname']]['guid']}" + if overwrite: + annot["url"]["osf"] = osfstorage_data[annot["fname"]]["guid"] + annots_updated = True + console.print(" [bold yellow]↻[/bold yellow] url updated") + else: + console.print( + f" [bold red]x[/bold red] " + f"url local: {annot['url']['osf']} " + f"remote: {osfstorage_data[annot['fname']]['guid']}" + ) + + # Write updated JSON files if changes were made + if atlases_updated: + atlases_data = {"atlases": MOUSEMAPS_ATLASES} + if getattr(importlib.resources, "files", None) is not None: + atlases_file = ( + importlib.resources.files("neuromaps_mouse") + / "datasets/data/atlases.json" ) + else: + from pkg_resources import resource_filename + + atlases_file = resource_filename( + "neuromaps_mouse", "datasets/data/atlases.json" + ) + with open(atlases_file, "w") as f: + json.dump(atlases_data, f, indent=2) + console.print("\n[bold green]✓[/bold green] Updated atlases.json") + if annots_updated: + annots_data = {"annotations": MOUSEMAPS_ANNOTS} + if getattr(importlib.resources, "files", None) is not None: + annots_file = ( + importlib.resources.files("neuromaps_mouse") + / "datasets/data/annotations.json" + ) + else: + from pkg_resources import resource_filename + + annots_file = resource_filename( + "neuromaps_mouse", "datasets/data/annotations.json" + ) + with open(annots_file, "w") as f: + json.dump(annots_data, f, indent=2) + console.print("[bold green]✓[/bold green] Updated annotations.json") + if annots_meta_updated: + annots_meta_data = {"annotations-meta": MOUSEMAPS_ANNOTS_META} + if getattr(importlib.resources, "files", None) is not None: + annots_meta_file = ( + importlib.resources.files("neuromaps_mouse") + / "datasets/data/annotations-meta.json" + ) + else: + from pkg_resources import resource_filename + + annots_meta_file = resource_filename( + "neuromaps_mouse", "datasets/data/annotations-meta.json" + ) + with open(annots_meta_file, "w") as f: + json.dump(annots_meta_data, f, indent=2) + console.print("[bold green]✓[/bold green] Updated annotations-meta.json") + + # Check for files in OSF storage that are not referenced in JSON files + console.print("\nFILES IN OSF STORAGE NOT REFERENCED IN JSON") + + # Collect all filenames referenced in JSON files + json_files = set() + + # Atlas files + for atlas_v in MOUSEMAPS_ATLASES.values(): + for file_v in atlas_v["files"].values(): + json_files.add(file_v["fname"]) + + # Annotation files + for annot in MOUSEMAPS_ANNOTS: + json_files.add(annot["fname"]) + + # Annotation meta aux files + for annot_meta in MOUSEMAPS_ANNOTS_META: + for aux_v in annot_meta["aux_files"].values(): + if not isinstance(aux_v, list): + aux_v = [aux_v] + for file_v in aux_v: + json_files.add(file_v["fname"]) + + # Find files in OSF storage not in JSON + unreferenced_files = set(osfstorage_data.keys()) - json_files + + if unreferenced_files: + for fname in sorted(unreferenced_files): + console.print(f" [bold yellow]?[/bold yellow] {fname}") + console.print( + f"\n[bold yellow]Found {len(unreferenced_files)} " + f"unreferenced files in OSF storage[/bold yellow]" + ) + else: + console.print( + " [bold green]✓[/bold green] All OSF storage files are referenced in JSON" + ) def _check_osfstorage(): @@ -298,7 +445,9 @@ def _get_full_data(url): f" {file['attributes']['extra']['hashes']['md5']}" ) if not file["attributes"]["guid"]: - requests.get(f'https://osf.io/{OSF_NODEID}/files/osfstorage{file["attributes"]["path"]}') + requests.get( + f"https://osf.io/{OSF_NODEID}/files/osfstorage{file['attributes']['path']}" + ) osfstorage_data[file["attributes"]["name"]] = { "guid": file["attributes"]["guid"], "md5": file["attributes"]["extra"]["hashes"]["md5"], @@ -315,7 +464,9 @@ def _get_full_data(url): f" {file['attributes']['extra']['hashes']['md5']}" ) if not file["attributes"]["guid"]: - requests.get(f'https://osf.io/{OSF_NODEID}/files/osfstorage{file["attributes"]["path"]}') + requests.get( + f"https://osf.io/{OSF_NODEID}/files/osfstorage{file['attributes']['path']}" + ) osfstorage_data[file["attributes"]["name"]] = { "guid": file["attributes"]["guid"], "md5": file["attributes"]["extra"]["hashes"]["md5"], @@ -325,72 +476,16 @@ def _get_full_data(url): return osfstorage_data -# def _fill_meta_json_refs(bib_file, json_file, overwrite=False, use_defaults=False): -# """ -# Fill in citation information for references in a JSON file. - -# For internal use only. - -# Parameters -# ---------- -# bib_file : str -# Path to BibTeX file containing references -# json_file : str -# Path to JSON file containing references -# overwrite : bool, optional -# Whether to overwrite existing citation information. Default: False -# use_defaults : bool, optional -# Whether to use default paths for `bib_file` and `json_file`. Default: False - -# Returns -# ------- -# None -# """ -# if use_defaults: -# bib_file = \ -# importlib.resources.files("neuromaps") / "datasets/data/neuromaps.bib" -# json_file = \ -# importlib.resources.files("neuromaps") / "datasets/data/meta.json" - -# from pybtex import PybtexEngine -# engine = PybtexEngine() - -# def _get_citation(key): -# s = engine.format_from_file( -# filename=bib_file, style="unsrt", -# citations=[key], output_backend="plaintext" -# ) -# return s.strip("\n").replace("[1] ", "") - -# with open(json_file) as src: -# nm_meta = json.load(src) - -# for entry in nm_meta["annotations"]: -# for bib_category in ["primary", "secondary"]: -# for bib_item in entry["refs"][bib_category]: -# if bib_item["bibkey"] not in ["", None]: -# if bib_item["citation"] == "" or overwrite: -# bib_item["citation"] = _get_citation(bib_item["bibkey"]) - -# with open(json_file, "w") as dst: -# json.dump(nm_meta, dst, indent=4) - - def _gen_doc_listofmaps_rst(listofmaps_file): """ Generate a list of maps in reStructuredText format. For internal use only. - - Parameters - ---------- - listofmaps_file : str - Path to write the list of maps - - Returns - ------- - None """ + MOUSEMAPS_ANNOTS_META = _load_resource_json("datasets/data/annotations-meta.json")[ + "annotations-meta" + ] + output = [] output += [ diff --git a/neuromaps_mouse/plotting.py b/neuromaps_mouse/plotting.py index 296f259..f60217f 100644 --- a/neuromaps_mouse/plotting.py +++ b/neuromaps_mouse/plotting.py @@ -432,7 +432,7 @@ def plot_allenccfv3_lightbox( ) else: ax.autoscale() - + if view in ["sagittal", "coronal"]: ax.invert_yaxis() diff --git a/neuromaps_mouse/resampling.py b/neuromaps_mouse/resampling.py index de20a2e..88bfa50 100644 --- a/neuromaps_mouse/resampling.py +++ b/neuromaps_mouse/resampling.py @@ -1,4 +1,5 @@ """Functions for resampling and aligning structures.""" + from pathlib import Path import subprocess import pandas as pd @@ -91,16 +92,23 @@ def _get_nearest_descendant_region_allenccfv3( def align_structures_allenccfv3(acronyms_fixed, acronyms_moving, debug=False): - df_fixed = query_structure_graph_allenccfv3( - acronyms_fixed, in_col="acronym", out_col=["acronym", "id", "structure_id_path"], verbose=0 + acronyms_fixed, + in_col="acronym", + out_col=["acronym", "id", "structure_id_path"], + verbose=0, ) df_moving = query_structure_graph_allenccfv3( - acronyms_moving, in_col="acronym", out_col=["acronym", "id", "structure_id_path"], verbose=0 + acronyms_moving, + in_col="acronym", + out_col=["acronym", "id", "structure_id_path"], + verbose=0, ) df_moving["id_ancestor_fixed"] = _get_nearest_ancestor_region_allenccfv3( - df_moving["structure_id_path"].to_list(), df_fixed["id"].to_list(), include_self=True + df_moving["structure_id_path"].to_list(), + df_fixed["id"].to_list(), + include_self=True, ) df_moving["id_ancestor_fixed"] = df_moving["id_ancestor_fixed"].astype("Int64") @@ -113,7 +121,9 @@ def align_structures_allenccfv3(acronyms_fixed, acronyms_moving, debug=False): ) # also get descendant df_moving["id_descendant_fixed"] = _get_nearest_descendant_region_allenccfv3( - df_moving["id"].tolist(), df_fixed["structure_id_path"].tolist(), include_self=True + df_moving["id"].tolist(), + df_fixed["structure_id_path"].tolist(), + include_self=True, ) df_moving["id_descendant_fixed_acronym"] = df_moving.apply( lambda x: get_feature_allenccfv3( @@ -132,38 +142,49 @@ def match_structures_fuzzy_allenccfv3(): def visualize_structure_alignment_allenccfv3( - acronyms_fixed, acronyms_moving, save_path=Path("./"), save_name="graphviz" - ): + acronyms_fixed, acronyms_moving, save_path=Path("./"), save_name="graphviz" +): graphviz_path = shutil.which("dot") if graphviz_path is None: raise ValueError("Graphviz executable not found, please install graphviz") - + if not isinstance(save_path, Path): save_path = Path(save_path) struct_csv = pd.read_csv(fetch_allenccfv3(which="structure-graph-csv")) df_fixed = query_structure_graph_allenccfv3( - acronyms_fixed, in_col="acronym", out_col=["acronym", "id", "structure_id_path"], verbose=0 + acronyms_fixed, + in_col="acronym", + out_col=["acronym", "id", "structure_id_path"], + verbose=0, ) df_moving = query_structure_graph_allenccfv3( - acronyms_moving, in_col="acronym", out_col=["acronym", "id", "structure_id_path"], verbose=0 + acronyms_moving, + in_col="acronym", + out_col=["acronym", "id", "structure_id_path"], + verbose=0, ) all_regions = [ - _.strip("/").split("/") - for _ in df_fixed["structure_id_path"].tolist() + df_moving["structure_id_path"].tolist() + _.strip("/").split("/") + for _ in df_fixed["structure_id_path"].tolist() + + df_moving["structure_id_path"].tolist() ] - all_regions = list(map(int, list(set([r for regions in all_regions for r in regions])))) + all_regions = list( + map(int, list(set([r for regions in all_regions for r in regions]))) + ) struct_csv_filtered = struct_csv[struct_csv["id"].isin(all_regions)] - struct_csv_filtered["parent_structure_id"] = struct_csv_filtered["parent_structure_id"].astype("Int64") + struct_csv_filtered["parent_structure_id"] = struct_csv_filtered[ + "parent_structure_id" + ].astype("Int64") graphviz_script = [ "digraph G {", 'rankdir="LR";', 'node [shape=box, fontname="Arial", fontsize=12];', - 'edge [fontname="Arial", fontsize=10];' + 'edge [fontname="Arial", fontsize=10];', ] for i, row in struct_csv_filtered.iterrows(): @@ -177,14 +198,18 @@ def visualize_structure_alignment_allenccfv3( for i, row in struct_csv_filtered.iterrows(): if row["acronym"] == "root": continue - graphviz_script.append( - f" {row['parent_structure_id']} -> {row['id']}" - ) - graphviz_script += [ - "}" - ] + graphviz_script.append(f" {row['parent_structure_id']} -> {row['id']}") + graphviz_script += ["}"] with open(save_path / f"{save_name}.txt", "w", encoding="utf-8") as f: f.writelines("\n".join(graphviz_script)) - subprocess.run([graphviz_path, "-Tsvg", f"{save_path / save_name}.txt", "-o", f"{save_path / save_name}.svg"]) \ No newline at end of file + subprocess.run( + [ + graphviz_path, + "-Tsvg", + f"{save_path / save_name}.txt", + "-o", + f"{save_path / save_name}.svg", + ] + )