Post (#262)

* load users from dvid and write to hdf5

* find redundent synapses

* use KDTree to detect duplicated post synapses

* try python 3.8 as CI version. Because this is the version used and is working.

* fix neuroglancer visualization of probability maps of 4D array

* ignore code workspace

.github/workflows/python-app.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.9
+    - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: "3.9"
+        python-version: "3.8"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.gitignore

@@ -1,3 +1,5 @@
+*.code-workspace
+
 *.pyc
 *.chkpt
 *__pycache__/*

chunkflow/flow/flow.py

@@ -11,9 +11,7 @@
 import click
 import json
 
-from numpy.lib.arraysetops import isin
-
-from .lib import *
+from chunkflow.lib.flow import *
 
 from cloudvolume import CloudVolume
 from cloudvolume.lib import Vec
@@ -55,7 +53,7 @@
 @click.option('--mip', '-m',
               type=int, default=None, help='mip level of the dataset layer.')
 @click.option('--roi-start', '-s',
-              type=int, default=None, nargs=3, callback=default_none, 
+              type=tuple, default=None, nargs=3, callback=default_none, 
               help='(z y x), start of the chunks')
 @click.option('--roi-stop', '-r',
               type=int, nargs=3, default=None, callback=default_none,
@@ -81,14 +79,14 @@
     type=int, default=None, nargs=3, callback=default_none,
     help='force alignment of block size. Note that the alignment start from (0, 0, 0).')
 @click.option('--task-index-start', '-i',
-              type=int, default=None, help='starting index of task list.')
+              type=int, default=0, help='starting index of task list.')
 @click.option('--task-index-stop', '-p',
               type=int, default=None, help='stop index of task list.')
 @click.option('--disbatch/--no-disbatch', '-d',
               default=False, help='use disBatch environment variable or not')
 @generator
 def generate_tasks(
-        layer_path: str, mip: int, roi_start: tuple, roi_stop: tuple,roi_size, chunk_size, 
+        layer_path: str, mip: int, roi_start: Cartesian, roi_stop: tuple,roi_size, chunk_size, 
         grid_size: tuple, file_path: str, queue_name: str, respect_chunk_size: bool,
         aligned_block_size: tuple, task_index_start: tuple, 
         task_index_stop: tuple, disbatch: bool ):
@@ -137,7 +135,7 @@ def generate_tasks(
             if disbatch:
                 assert len(bboxes) == 1
                 bbox_index = disbatch_index
-            print(f'executing task {bbox_index} in {bbox_num} with bounding box: {bbox.to_filename()}')
+            print(f'executing task {bbox_index+task_index_start} in {bbox_num+task_index_start} with bounding box: {bbox.to_filename()}')
             task = get_initial_task()
             task['bbox'] = bbox
             task['bbox_index'] = bbox_index
@@ -199,15 +197,24 @@ def skip_all_zero(tasks, input_chunk_name: str, pre: str, post: str, adjust_size
 @main.command('skip-none')
 @click.option('--input-name', '-i',
     type=str, default=DEFAULT_CHUNK_NAME, help='input name')
+@click.option('--touch/--no-touch', default=True, help='touch an empty file or not')
+@click.option('--prefix', '-p', default=None, help='prefix of output file.')
+@click.option('--suffix', '-s', default=None, help='suffix of output file.')
 @operator
-def skip_none(tasks, input_name: str):
+def skip_none(tasks: dict, input_name: str, touch: bool, prefix: str, suffix: str):
     """If item is None, skip this task."""
     for task in tasks:
         if task is not None:
             data = task[input_name]
             if data is None:
                 # target task as None and task will be skipped
                 task = None
+                if touch:
+                    assert prefix is not None
+                    assert suffix is not None
+                    bbox = task['bbox']
+                    fname = f'{prefix}{bbox.to_filename()}{suffix}'
+                    Path(fname).touch()
         yield task
 
 
@@ -521,10 +528,12 @@ def create_chunk(tasks, name, size, dtype, all_zero, voxel_offset, voxel_size, o
     help='C order or Fortran order in the file. XYZ is Fortran order, ZYX is C order.')
 @click.option('--resolution', '-r', type=int, nargs=3, 
     default=None, callback=default_none, help='resolution of points.')
+@click.option('--remove-outside/--keep-all', default=True, 
+    help='remove synapses outside of the bounding box or not.')
 @click.option('--output-name', '-o', type=str, default='synapses', help='data name of the result.')
 @operator
 def load_synapses(tasks, name: str, file_path: str, path_suffix: str, c_order: bool, 
-        resolution: tuple, output_name: str):
+        resolution: tuple, remove_outside: bool, output_name: str):
     """Load synapses formated as JSON or HDF5."""
     for task in tasks:
         if task is not None:
@@ -545,12 +554,18 @@ def load_synapses(tasks, name: str, file_path: str, path_suffix: str, c_order: b
             if os.path.getsize(fname) == 0:
                 task[output_name] = None
             else:
-                task[output_name] = Synapses.from_file(
+                syns = Synapses.from_file(
                     fname, 
                     resolution = resolution,
                     c_order = c_order
                 )
-
+                if remove_outside:
+                    bbox = task['bbox']
+                    syns.remove_synapses_outside_bounding_box(bbox)
+                print(f'loaded synapses with {syns.pre_num} presynapses and {syns.post_num} post synapses.')
+                if syns.pre_num == 0:
+                    syns = None
+                task[output_name] = syns
             task['log']['timer'][name] = time() - start
         yield task
 
@@ -672,12 +687,16 @@ def write_tif(tasks, name, input_chunk_name, file_name):
               help = 'the offset of png images volume, could be negative.')
 @click.option('--voxel-size', '-x', type=int, nargs=3, default=None, callback=default_none,
               help='physical size of voxels. the unit is assumed to be nm.')
+@click.option('--digit-num', '-d', type=int, default=5,
+    help='the total number of digits with leading zero padding. For example, digit_num=3, the file name will be like 001.png')
 @click.option('--chunk-size', '-s',
               type=int, nargs=3, default=None, callback=default_none,
               help='cutout chunk size')
 @operator
-def read_pngs(tasks, path_prefix, output_chunk_name, cutout_offset,
-                volume_offset, voxel_size, chunk_size):
+def read_pngs(tasks: dict, path_prefix: str, 
+                output_chunk_name: str, cutout_offset: tuple,
+                volume_offset: tuple, voxel_size: tuple, 
+                digit_num: int, chunk_size: tuple):
     """Read a serials of png files."""
     for task in tasks:
         if task is not None:
@@ -690,6 +709,7 @@ def read_pngs(tasks, path_prefix, output_chunk_name, cutout_offset,
             task[output_chunk_name] = read_png_images(
                 path_prefix, bbox, 
                 volume_offset=volume_offset,
+                digit_num=digit_num,
                 voxel_size=voxel_size)
         yield task
 
@@ -753,8 +773,9 @@ def write_tif(tasks, name, input_chunk_name, file_name):
               default=None, help='transform data type.')
 @click.option('--voxel-offset', '-v', type=int, nargs=3, default=None,
               callback=default_none, help='voxel offset of the dataset in hdf5 file.')
-@click.option('--voxel-size', '-x', type=int, nargs=3, default=None,
-              callback=default_none, help='physical size of voxels. The unit is assumed to be nm.')
+@click.option('--voxel-size', '-x', type=int, nargs=3, 
+    default=None, callback=default_none, 
+    help='physical size of voxels. The unit is assumed to be nm.')
 @click.option('--cutout-start', '-t', type=int, nargs=3, callback=default_none,
               help='cutout voxel offset in the array')
 @click.option('--cutout-stop', '-p', type=int, nargs=3, callback=default_none,

chunkflow/flow/neuroglancer.py

@@ -202,7 +202,7 @@ def _append_probability_map_layer(self, viewer_state: ng.viewer_state.ViewerStat
                 data=chunk.array,
                 dimensions=dimensions,
                 # offset is in nm, not voxels
-                voxel_offset=(0, ) + chunk.voxel_offset,
+                voxel_offset=(0, *chunk.voxel_offset),
             ),
             shader=shader
         )

chunkflow/flow/read_pngs.py

@@ -2,8 +2,8 @@
 import logging
 
 import numpy as np
-from cloudvolume.lib import Bbox
 
+from chunkflow.lib.bounding_boxes import BoundingBox, Cartesian
 from chunkflow.chunk import Chunk
 
 from tqdm import tqdm
@@ -13,9 +13,10 @@
 Image.MAX_IMAGE_PIXELS = None 
 
 
-def read_png_images(path_prefix: str, bbox: Bbox, 
+def read_png_images(path_prefix: str, bbox: BoundingBox, 
                         volume_offset: tuple = (0, 0, 0),
                         voxel_size: tuple = (1, 1, 1),
+                        digit_num: int = 5,
                         dtype: np.dtype = np.uint8):
 
     chunk = Chunk.from_bbox(
@@ -27,7 +28,7 @@ def read_png_images(path_prefix: str, bbox: Bbox,
     assert len(volume_offset) == 3
 
     for z in tqdm( range(bbox.minpt[0], bbox.maxpt[0]) ):
-        file_name = '{}{:0>5d}.png'.format(path_prefix, z)
+        file_name = f'{path_prefix}{z:0>{digit_num}d}.png'
         file_name = path.expanduser(file_name)
         if path.exists(file_name):
             img = Image.open(file_name)

chunkflow/lib/bounding_boxes.py

@@ -340,7 +340,7 @@ def from_manual_setup(cls,
                     range(grid_size[1]),
                     range(grid_size[2])):
             chunk_start = roi_start + Vec(gz, gy, gx) * stride
-            bbox = Bbox.from_delta(chunk_start, chunk_size)
+            bbox = BoundingBox.from_delta(chunk_start, chunk_size)
             if not respect_chunk_size:
                 bbox.maxpt = np.minimum(bbox.maxpt, roi_stop)
             if not bounded or np.all(tuple(m < p for m, p in zip(bbox.maxpt, roi_stop))):
@@ -353,7 +353,7 @@ def from_manual_setup(cls,
     def from_array(cls, arr: np.ndarray):
         bboxes = []
         for idx in range(arr.shape(0)):
-            bbox = Bbox.from_vec(arr[idx, :])
+            bbox = BoundingBox.from_vec(arr[idx, :])
             bboxes.append(bbox)
 
         return cls(bboxes)

chunkflow/flow/lib.py → chunkflow/lib/flow.py

@@ -3,6 +3,18 @@
 from functools import update_wrapper, wraps
 import click
 
+from .bounding_boxes import Cartesian
+
+class CartesianParamType(click.ParamType):
+    name = 'Cartesian'
+    def __init__(self) -> None:
+        super().__init__()
+
+    def convert(self, value, param, ctx):
+        breakpoint()
+
+        return super().convert(value, param, ctx)
+
 # global dict to hold the operators and parameters
 state = {'operators': {}}
 DEFAULT_CHUNK_NAME = 'chunk'