Merge pull request #33 from kmpaul/sync_fix

Attempting to sync with dask

Author: Kevin Paul

.circleci/config.yml

@@ -8,6 +8,8 @@ workflows:
       - test-2.7-openmpi
       - test-3.6-mpich
       - test-3.6-openmpi
+      - test-3.7-mpich
+      - test-3.7-openmpi
 
 jobs:
 
@@ -130,4 +132,66 @@ jobs:
           paths:
             - "/opt/conda/envs/${ENV_NAME}/"
             - "/opt/conda/pkgs"
-  
+
+  test-3.7-mpich:
+    docker:
+      - image: continuumio/miniconda:latest
+
+    environment:
+      PYTHON: "3.7"
+      ENV_NAME: "dask-mpi-dev"
+      MPI: "mpich"
+
+    steps:
+      - checkout
+      - restore_cache:
+          key: deps-{{ .Branch }}-3.7-mpich-{{ checksum ".circleci/env-3.7-mpich.yml" }}
+      - run:
+          name: Install and activate conda environment
+          command: .circleci/install.sh
+      - run:
+          name: Running tests
+          command: |
+            source activate ${ENV_NAME}
+            pytest --junitxml=test-reports/junit.xml --cov=./ dask_mpi/tests/ --verbose
+      - save_cache:
+          key: deps-{{ .Branch }}-3.7-mpich-{{ checksum ".circleci/env-3.7-mpich.yml" }}
+          paths:
+            - "/opt/conda/envs/${ENV_NAME}/"
+            - "/opt/conda/pkgs"
+      - run:
+          name: Code Coverage
+          command: |
+            source activate ${ENV_NAME}
+
+            echo "[Uploading Coverage]"
+            codecov
+
+  test-3.7-openmpi:
+    docker:
+      - image: continuumio/miniconda:latest
+
+    environment:
+      PYTHON: "3.7"
+      ENV_NAME: "dask-mpi-dev"
+      MPI: "openmpi"
+      OMPI_MCA_rmaps_base_oversubscribe: "1"
+      ALLOW_RUN_AS_ROOT: "1"
+
+    steps:
+      - checkout
+      - restore_cache:
+          key: deps-{{ .Branch }}-3.7-openmpi-{{ checksum ".circleci/env-3.7-openmpi.yml" }}
+      - run:
+          name: Install and activate conda environment
+          command: .circleci/install.sh
+      - run:
+          name: Running tests
+          command: |
+            source activate ${ENV_NAME}
+            pytest --junitxml=test-reports/junit.xml --cov=./ dask_mpi/tests/ --verbose
+      - save_cache:
+          key: deps-{{ .Branch }}-3.7-openmpi-{{ checksum ".circleci/env-3.7-openmpi.yml" }}
+          paths:
+            - "/opt/conda/envs/${ENV_NAME}/"
+            - "/opt/conda/pkgs"

.circleci/env-2.7-mpich.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python=2.7
   - dask
-  - distributed=1.25.2
+  - distributed
   - mpich
   - mpi4py
   - pytest

.circleci/env-2.7-openmpi.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python=2.7
   - dask
-  - distributed=1.25.2
+  - distributed
   - openmpi
   - mpi4py
   - pytest

.circleci/env-3.6-mpich.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python=3.6
   - dask
-  - distributed=1.25.2
+  - distributed
   - mpich
   - mpi4py
   - pytest

.circleci/env-3.6-openmpi.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python=3.6
   - dask
-  - distributed=1.25.2
+  - distributed
   - openmpi
   - mpi4py
   - pytest
@@ -14,4 +14,3 @@ dependencies:
   - coverage
   - requests
   - codecov
-  

.circleci/env-3.7-mpich.yml

@@ -0,0 +1,17 @@
+name: dask-mpi-dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - dask
+  - distributed
+  - mpich
+  - mpi4py
+  - pytest
+  - pytest-icdiff
+  - pytest-cov
+  - coverage
+  - requests
+  - codecov
+

.circleci/env-3.7-openmpi.yml

@@ -0,0 +1,16 @@
+name: dask-mpi-dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - dask
+  - distributed
+  - openmpi
+  - mpi4py
+  - pytest
+  - pytest-icdiff
+  - pytest-cov
+  - coverage
+  - requests
+  - codecov

dask_mpi/cli.py

@@ -5,7 +5,12 @@
 from tornado.ioloop import IOLoop
 from distributed.cli.utils import check_python_3
 
-from dask_mpi.common import get_host_from_interface, create_scheduler, run_scheduler, create_and_run_worker
+from dask_mpi.common import (
+    get_host_from_interface,
+    create_scheduler,
+    run_scheduler,
+    create_and_run_worker
+)
 
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
@@ -47,9 +52,10 @@
 def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
          scheduler, bokeh, bokeh_port, bokeh_prefix, bokeh_worker_port,
          nanny, scheduler_port, nanny_port, worker_port):
+
     host = get_host_from_interface(interface)
 
-    loop = IOLoop()
+    loop = IOLoop.current()
 
     if rank == 0 and scheduler:
         scheduler_obj = create_scheduler(loop, host=host, scheduler_file=scheduler_file,

dask_mpi/core.py

@@ -5,10 +5,16 @@
 
 import dask
 from distributed import Client
+
 from tornado import gen
 from tornado.ioloop import IOLoop
 
-from .common import get_host_from_interface, create_scheduler, run_scheduler, create_and_run_worker
+from .common import (
+    get_host_from_interface,
+    create_scheduler,
+    run_scheduler,
+    create_and_run_worker
+)
 
 
 def initialize(interface=None, nthreads=1, local_directory='', memory_limit='auto', nanny=False,
@@ -42,12 +48,13 @@ def initialize(interface=None, nthreads=1, local_directory='', memory_limit='aut
     bokeh_worker_port : int
         Worker's Bokeh port for visual diagnostics
     """
+
     host = get_host_from_interface(interface)
 
     from mpi4py import MPI
     comm = MPI.COMM_WORLD
     rank = comm.Get_rank()
-    loop = IOLoop()
+    loop = IOLoop.current()
 
     if rank == 0:
         scheduler = create_scheduler(loop, host=host, bokeh=bokeh, bokeh_port=bokeh_port, bokeh_prefix=bokeh_prefix)

environment-dev.yml

@@ -1,7 +1,7 @@
 name: dask-mpi-dev
 dependencies:
   - dask
-  - distributed=1.25.2
+  - distributed
   - mpich
   - mpi4py
   - versioneer