Tiled & Databroker v2 support

.github/workflows/docs.yml

@@ -32,7 +32,7 @@ jobs:
           export DOCKER_BINARY="docker"
           echo "DOCKER_BINARY=${DOCKER_BINARY}" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -87,8 +87,12 @@ jobs:
           set -vxeuo pipefail
           make -C docs/ html
           status=$?
+
+          echo "::group::docker logs"
           echo "Sirepo ${DOCKER_BINARY} container id: ${SIREPO_DOCKER_CONTAINER_ID}"
           ${DOCKER_BINARY} logs ${SIREPO_DOCKER_CONTAINER_ID}
+          echo "::endgroup::"
+
           if [ $status -gt 0 ]; then
               exit $status
           fi

.github/workflows/pre-commit.yml

@@ -19,7 +19,7 @@ jobs:
         shell: bash -l {0}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
 

.github/workflows/publish-pypi.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/testing.yml

@@ -12,12 +12,17 @@ jobs:
     # pull requests are a duplicate of a branch push if within the same repo.
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
 
-    name: Test sirepo-bluesky with ${{ matrix.docker-binary }} and Python ${{ matrix.python-version }}
+    name: Test sirepo-bluesky with py${{ matrix.python-version }} and databroker ${{ matrix.databroker }} (${{ matrix.docker-binary }})
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        docker-binary: ["docker", "podman"]
+        databroker: ["v1", "v2"]
+        docker-binary: ["docker"]
+        include:
+          - python-version: "3.10"
+            docker-binary: "podman"
+            databroker: "v1"
       fail-fast: false
 
     defaults:
@@ -33,8 +38,16 @@ jobs:
           export DOCKER_BINARY=${{ matrix.docker-binary }}
           echo "DOCKER_BINARY=${DOCKER_BINARY}" >> $GITHUB_ENV
 
+          if [ "${{ matrix.databroker }}" == "v1" ]; then
+              EXTRA_INSTALL="databroker==1.2.5"
+          elif [ "${{ matrix.databroker }}" == "v2" ]; then
+              EXTRA_INSTALL="databroker[all]>=2.0.0b tiled[all]"
+          fi
+          export EXTRA_INSTALL
+          echo "EXTRA_INSTALL=${EXTRA_INSTALL}" >> $GITHUB_ENV
+
       - name: Checkout the code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start MongoDB
         uses: supercharge/[email protected]
@@ -63,24 +76,33 @@ jobs:
         run: |
           set -vxeo pipefail
           python -m pip install --upgrade pip wheel
-          python -m pip install -v .
+          python -m pip install -v . ${EXTRA_INSTALL}
           python -m pip install -r requirements-dev.txt
           python -m pip list
 
       - name: Copy databroker config file
         run: |
           set -vxeuo pipefail
-          mkdir -v -p ~/.config/databroker/
-          cp -v examples/local.yml ~/.config/databroker/
+          if [ "${{ matrix.databroker }}" == "v1" ]; then
+              mkdir -v -p ~/.config/databroker/
+              cp -v examples/local.yml ~/.config/databroker/
+          elif [ "${{ matrix.databroker }}" == "v2" ]; then
+              mkdir -v -p ~/.config/tiled/profiles/
+              cp -v examples/local-tiled.yml ~/.config/tiled/profiles/local.yml
+          fi
 
       - name: Test with pytest
         run: |
           set -vxuo pipefail
           pytest -s -vv
           status=$?
           ${DOCKER_BINARY} ps -a
+
+          echo "::group::docker logs"
           echo "Sirepo ${DOCKER_BINARY} container id: ${SIREPO_DOCKER_CONTAINER_ID}"
           ${DOCKER_BINARY} logs ${SIREPO_DOCKER_CONTAINER_ID}
+          echo "::endgroup::"
+
           if [ $status -gt 0 ]; then
               exit $status
           fi

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/ambv/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
         language_version: python3.10

docs/source/notebooks/madx.ipynb

@@ -30,11 +30,11 @@
     "\n",
     "import matplotlib.pyplot as plt\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.madx_flyer import MADXFlyer\n",
-    "from sirepo_bluesky.sirepo_ophyd import create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.madx.madx_flyer import MADXFlyer\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"madx\", \"00000002\")\n",
     "classes, objects = create_classes(\n",

docs/source/notebooks/shadow.ipynb

@@ -40,10 +40,10 @@
     "\n",
     "%run -i $prepare_re_env.__file__\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.sirepo_ophyd import BeamStatisticsReport, create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"shadow\", sim_id=\"00000002\")\n",
     "classes, objects = create_classes(connection=connection)\n",
@@ -110,10 +110,11 @@
     "\n",
     "%run -i $prepare_re_env.__file__\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.sirepo_ophyd import BeamStatisticsReport, create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.shadow.shadow_ophyd import BeamStatisticsReport\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"shadow\", sim_id=\"00000002\")\n",
     "\n",

docs/source/notebooks/srw.ipynb

@@ -40,10 +40,10 @@
     "\n",
     "%run -i $prepare_re_env.__file__\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.sirepo_ophyd import create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"srw\", sim_id=\"00000002\")\n",
     "classes, objects = create_classes(connection=connection)\n",
@@ -109,10 +109,10 @@
     "\n",
     "%run -i $prepare_re_env.__file__\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.sirepo_ophyd import create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"srw\", sim_id=\"00000002\")\n",
     "classes, objects = create_classes(connection=connection)\n",
@@ -162,9 +162,12 @@
     "    aspect=False,\n",
     ")\n",
     "\n",
+    "h_dims = 1e6 * np.array(\n",
+    "    w9.horizontal_extent_start.get(), w9.horizontal_extent_end.get()\n",
+    ")\n",
+    "v_dims = 1e6 * np.array(w9.vertical_extent_start.get(), w9.vertical_extent_start.get())\n",
+    "\n",
     "for ax, im in zip(grid, w9_image):\n",
-    "    h_dims = 1e6 * w9.horizontal_extent.get()\n",
-    "    v_dims = 1e6 * w9.vertical_extent.get()\n",
     "    ax.imshow(\n",
     "        im, interpolation=\"nearest\", aspect=\"auto\", extent=(*h_dims[:], *v_dims[:])\n",
     "    )"
@@ -192,10 +195,10 @@
     "\n",
     "%run -i $prepare_re_env.__file__\n",
     "\n",
-    "from sirepo_bluesky.sirepo_bluesky import SirepoBluesky\n",
-    "from sirepo_bluesky.sirepo_ophyd import create_classes\n",
+    "from sirepo_bluesky.common.sirepo_client import SirepoClient\n",
+    "from sirepo_bluesky.common.create_classes import create_classes\n",
     "\n",
-    "connection = SirepoBluesky(\"http://localhost:8000\")\n",
+    "connection = SirepoClient(\"http://localhost:8000\")\n",
     "\n",
     "data, schema = connection.auth(\"srw\", sim_id=\"00000003\")\n",
     "classes, objects = create_classes(\n",

examples/local-tiled.yml

@@ -0,0 +1,16 @@
+local:
+  direct:
+    authentication:
+      allow_anonymous_access: true
+    trees:
+      - tree: databroker.mongo_normalized:Tree.from_uri
+        path: /
+        args:
+          uri: mongodb://localhost:27017/datastore
+          asset_registry_uri: mongodb://localhost:27017/filestore
+          handler_registry:
+            srw: sirepo_bluesky.srw.srw_handler:SRWFileHandler
+            SIREPO_FLYER: sirepo_bluesky.srw.srw_handler:SRWFileHandler
+            SRW_HDF5: sirepo_bluesky.srw.srw_handler:SRWHDF5FileHandler
+            shadow: sirepo_bluesky.shadow.shadow_handler:ShadowFileHandler
+            madx: sirepo_bluesky.madx.madx_handler:MADXFileHandler

pytest.ini

@@ -1,3 +1,6 @@
 [pytest]
 markers =
     docker: marks tests as requiring Docker to be available/running (deselect with '-m "not docker"')
+    madx: sirepo/madx simulation code tests
+    shadow: sirepo/shadow simulation code tests
+    srw: sirepo/srw simulation code tests

requirements.txt

@@ -1,6 +1,7 @@
 area-detector-handlers
 bluesky
 databroker
+h5py
 inflection
 matplotlib
 numconv
@@ -10,6 +11,7 @@ peakutils
 pymongo
 pyqt5>=5.9
 requests
+scikit-image
 shadow3>=23.1.4
 srwpy>=4.0.0b1
 tfs-pandas

scripts/start_sirepo.sh

@@ -82,6 +82,8 @@ cmd_start="${docker_binary} run ${arg} --init ${remove_container} --name sirepo
     -p 8000:8000 \
     -v $SIREPO_SRDB_HOST_RO:/SIREPO_SRDB_ROOT:ro,z "
 
+# TODO: parametrize host port number.
+
 cmd_extra=""
 if [ ! -z "${SIREPO_SRDB_HOST}" -a ! -z "${SIREPO_SRDB_GUEST}" ]; then
     cmd_extra="-v ${SIREPO_SRDB_HOST}:${SIREPO_SRDB_GUEST}:rw,z "

sirepo_bluesky/__init__.py

@@ -1,26 +1,5 @@
-from ophyd import Signal
-
 from ._version import get_versions
 from .utils import prepare_re_env  # noqa: F401
 
 __version__ = get_versions()["version"]
 del get_versions
-
-
-class ExternalFileReference(Signal):
-    """
-    A pure software Signal that describe()s an image in an external file.
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def describe(self):
-        resource_document_data = super().describe()
-        resource_document_data[self.name].update(
-            dict(
-                external="FILESTORE:",
-                dtype="array",
-            )
-        )
-        return resource_document_data

sirepo_bluesky/common/__init__.py

@@ -0,0 +1,88 @@
+import logging
+from collections import deque
+
+from ophyd import Signal
+from ophyd.sim import NullStatus
+
+logger = logging.getLogger("sirepo-bluesky")
+# Note: the following handler could be created/added to the logger on the client side:
+# import sys
+# stream_handler = logging.StreamHandler(sys.stdout)
+# logger.addHandler(stream_handler)
+
+RESERVED_OPHYD_TO_SIREPO_ATTRS = {  # ophyd <-> sirepo
+    "position": "element_position",
+    "name": "element_name",
+    "class": "command_class",
+}
+RESERVED_SIREPO_TO_OPHYD_ATTRS = {v: k for k, v in RESERVED_OPHYD_TO_SIREPO_ATTRS.items()}
+
+
+class ExternalFileReference(Signal):
+    """
+    A pure software Signal that describe()s an image in an external file.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def describe(self):
+        resource_document_data = super().describe()
+        resource_document_data[self.name].update(
+            dict(
+                external="FILESTORE:",
+                dtype="array",
+            )
+        )
+        return resource_document_data
+
+
+class SirepoSignal(Signal):
+    def __init__(self, sirepo_dict, sirepo_param, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._sirepo_dict = sirepo_dict
+        self._sirepo_param = sirepo_param
+        if sirepo_param in RESERVED_SIREPO_TO_OPHYD_ATTRS:
+            self._sirepo_param = RESERVED_SIREPO_TO_OPHYD_ATTRS[sirepo_param]
+
+    def set(self, value, *, timeout=None, settle_time=None):
+        logger.debug(f"Setting value for {self.name} to {value}")
+        self._sirepo_dict[self._sirepo_param] = value
+        self._readback = value
+        return NullStatus()
+
+    def put(self, *args, **kwargs):
+        self.set(*args, **kwargs).wait()
+
+
+class ReadOnlyException(Exception):
+    ...
+
+
+class SirepoSignalRO(SirepoSignal):
+    def set(self, *args, **kwargs):
+        raise ReadOnlyException("Cannot set/put the read-only signal.")
+
+
+class BlueskyFlyer:
+    def __init__(self):
+        self.name = "bluesky_flyer"
+        self._asset_docs_cache = deque()
+        self._resource_uids = []
+        self._datum_counter = None
+        self._datum_ids = []
+
+    def kickoff(self):
+        return NullStatus()
+
+    def complete(self):
+        return NullStatus()
+
+    def collect(self):
+        ...
+
+    def collect_asset_docs(self):
+        items = list(self._asset_docs_cache)
+        self._asset_docs_cache.clear()
+        for item in items:
+            yield item