Update usage notebook

Author: constantinpape

example/bioimageio-core-usage.ipynb

@@ -23,9 +23,7 @@
     "# we use napari for visualising images, you can install it via `pip install napari` or`conda install napari`\n",
     "import napari\n",
     "import numpy as np\n",
-    "import xarray as xr\n",
-    "\n",
-    "from bioimageio.core.prediction_pipeline import create_prediction_pipeline"
+    "import xarray as xr"
    ]
   },
   {
@@ -170,8 +168,8 @@
    "source": [
     "## Prediction with the model\n",
     "\n",
-    "`bioimageio.core` implements functionality to run predictions with a model in bioimage.io format.\n",
-    "This includes functions to run prediction with numpy arrays (more precisely xarray DataArrays) and convenience functions to run predictions for inputs stored on disc."
+    "`bioimageio.core` implements functionality to run prediction with models in the `bioimage.io` format.\n",
+    "This includes functions to run prediction with `xarray.DataArrays` as input and convenience functions to run predictions for images stored on disc."
    ]
   },
   {
@@ -181,42 +179,62 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# load the example image for this model, which is stored in numpy file format\n",
+    "# Load the example image for this model, which is stored in numpy file format.\n",
     "input_image = np.load(model_resource.test_inputs[0])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07ff1e0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create an xarray.DataArray from the input image.\n",
+    "# DataArrays are like numpy arrays, but they have annotated axes.\n",
+    "# The axes are used to validate that the axes of the input image match the axes expected by a model.\n",
+    "input_array = xr.DataArray(input_image, dims=tuple(model_resource.inputs[0].axes))\n",
+    "# print the axis annotations ('dims') and the shape of the input array\n",
+    "print(input_array.dims)\n",
+    "print(input_array.shape)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "808e2ca7",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# define a function to run prediction on a numpy input\n",
-    "# \"devices\" can be used to run prediction on a gpu instead of the cpu\n",
-    "# \"weight_format\" to specify which weight format to use in case the model contains different weight formats\n",
-    "def predict_numpy(model, input_, devices=None, weight_format=None):\n",
-    "    # the prediction pipeline combines preprocessing, prediction and postprocessing.\n",
-    "    # it should always be used for prediction with a bioimageio model\n",
-    "    pred_pipeline = create_prediction_pipeline(\n",
-    "        bioimageio_model=model, devices=devices, weight_format=weight_format\n",
-    "    )\n",
+    "# Next, create a 'prediction_pipeline'. The prediction_pipeline is used to run prediction with a given model.\n",
+    "# This means it applies the preprocessing, runs inference with the model and applies the postprocessing.\n",
+    "\n",
+    "# The 'devices' argument can be used to specify which device(s) to use for inference with the model.\n",
+    "# Hence it can be used to specify whether to use the cpu, a single gpu or multiple gpus (not implemented yet).\n",
+    "# By default (devices=None) a gpu will be used if available and otherwise the cpu will be used.\n",
+    "devices = None\n",
     "\n",
-    "    # the prediction pipeline expects inputs as xarray.DataArrays.\n",
-    "    # these are similar to numpy arrays, but allow for named dimensions (the dims keyword argument)\n",
-    "    # in bioimage.io the dims have to agree with the input axes required by the model\n",
-    "    axes = tuple(model.inputs[0].axes)\n",
-    "    input_tensor = xr.DataArray(input_, dims=axes)\n",
-    "    \n",
-    "    # the prediction pipeline call expects the same number of inputs as the number of inputs required by the model\n",
-    "    # in the case here, the model just expects a single input. in the case of multiple inputs use\n",
-    "    # prediction = pred_pipeline(input1, input2, ...)\n",
-    "    # or, if you have the inputs in a list or tuple\n",
-    "    # prediction = pred_pipeline(*inputs)\n",
-    "    # the call returns a list of output tensors, corresponding to the output tensors of the model\n",
-    "    # (in this case, we just have a single output)\n",
-    "    prediction = pred_pipeline(input_tensor)[0]\n",
-    "    return prediction"
+    "# The 'weight_format' argument can be used to specify which weight format available in the model to use.\n",
+    "# By default (weight_format=None) the weight format with highest priority (as defined by bioimageio.core) will be used.\n",
+    "weight_format = None\n",
+    "\n",
+    "prediction_pipeline = bioimageio.core.create_prediction_pipeline(\n",
+    "    model_resource, devices=devices, weight_format=weight_format\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13c73742",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use the prediction pipeline to run prediction for the image we loaded before.\n",
+    "# The prediction pipeline always returns a tuple (even if the model only has a single output tensor).\n",
+    "# So we access the first element of the prediction to get the predicted tensor.\n",
+    "prediction = prediction_pipeline(input_array)[0]\n",
+    "show_images(input_image, prediction, names=[\"image\", \"prediction\"])  # show the prediction result"
    ]
   },
   {
@@ -226,9 +244,68 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# run prediction for the test input and show the result\n",
-    "prediction = predict_numpy(model_resource, input_image)\n",
-    "show_images(input_image, prediction, names=[\"image\", \"prediction\"])"
+    "# The prediction pipeline expects inputs to have a shape that fits the model exactly.\n",
+    "# So if the input does not fit the expected input shape the prediction will fail.\n",
+    "# E.g. if we crop the input to shape [1, 1, 250, 250] it will not work for our example model,\n",
+    "# which expects a spatial shape that is a multiple of 16\n",
+    "cropped_image = input_image[:, :, :250, :250]\n",
+    "cropped_array = xr.DataArray(cropped_image, dims=tuple(model_resource.inputs[0].axes))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f476af51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Applying the prediction pipeline to an image with the wrong shape will fail!\n",
+    "prediction_pipeline(cropped_array)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "580b0a36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Instead, we can use the function `predict_with_padding`, which will pad the image to a shape that fits the model.\n",
+    "prediction = bioimageio.core.predict_with_padding(prediction_pipeline, cropped_array)\n",
+    "show_images(cropped_image, prediction, names=[\"image\", \"prediction\"])  # show the prediction result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2d6472e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# There is also the function `predict_with_tiling`, which will run prediction for patches in a sliding window fashion.\n",
+    "# This is especially helpful for large inputs that do not fit into the model as a single input.\n",
+    "\n",
+    "# The `tiling` argument is used to specify the tile size and the `halo`, which is the part of the patch\n",
+    "# that is cropped in order to reduce boundary artifacts.\n",
+    "# Alternatively, `tiling` can also be set to `True`, than the tile size and halo will be deduced from the model config\n",
+    "# (this is also the default behavior when the `tiling` parameter is not passed).\n",
+    "tiling = {\"tile\": {\"x\": 128, \"y\": 128}, \"halo\": {\"x\": 16, \"y\": 16}}  # use a tile size of 128x128 and crop a halo of 16 pixels\n",
+    "\n",
+    "# if `verbose` is set to True a progress bar will be printed \n",
+    "prediction = bioimageio.core.predict_with_tiling(prediction_pipeline, cropped_array, tiling=tiling, verbose=True)\n",
+    "show_images(cropped_image, prediction, names=[\"image\", \"prediction\"]) "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ba91499",
+   "metadata": {},
+   "source": [
+    "### Convenience prediction functions\n",
+    "\n",
+    "`bioimageio.core` also contains a few convenience functions to directly predict images that are stored on disc:\n",
+    "- `predict_image` can be used to run prediction for a single image\n",
+    "- `predict_images` to run prediction for many images"
    ]
   },
   {
@@ -238,21 +315,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# the utility function `predict_image` can be used to run prediction with an image stored on disc\n",
+    "# The convenience function `predict_image` can be used to run prediction for an image stored on disc.\n",
     "from bioimageio.core.prediction import predict_image\n",
     "\n",
-    "# the filepath where the output should be stored, supports most common image formats as well as npy fileformat\n",
+    "# The filepath where the output should be stored; supports most common image formats as well as npy fileformat.\n",
     "outputs = [\"prediction.tif\"]\n",
     "predict_image(\n",
     "    model_resource, model_resource.test_inputs, outputs\n",
     ")\n",
     "\n",
-    "# the output tensor contains 2 channels, which is not supported by normal tif.\n",
-    "# thus, these 2 channels are stored as 2 separate images\n",
+    "# The output tensor contains 2 channels, which is not supported by normal tif.\n",
+    "# Thus, these 2 channels are stored as 2 separate images.\n",
     "fg_pred = imageio.imread(\"prediction-c0.tif\")\n",
     "bd_pred = imageio.imread(\"prediction-c1.tif\")\n",
-    "show_images(input_image, fg_pred, bd_pred,\n",
-    "            names=[\"image\", \"foreground-prediction\", \"boundary-prediction\"])"
+    "show_images(input_image, fg_pred, bd_pred, names=[\"image\", \"foreground-prediction\", \"boundary-prediction\"])"
    ]
   },
   {
@@ -262,20 +338,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# the utility function `predict_images` can be use to run prediction for a batch of images stored on disc\n",
-    "# note: this only works for models which have a single input and output!\n",
+    "# The convenience function `predict_images` can be use to run prediction for many images stored on disc\n",
+    "# Note: this only works for models which have a single input and output!\n",
     "from bioimageio.core.prediction import predict_images\n",
     "\n",
-    "# here, we use a subset of the dsb challenge data for prediction from the stardist (https://github.com/stardist/stardist)\n",
-    "# you can obtain it from: https://github.com/stardist/stardist/releases/download/0.1.0/dsb2018.zip\n",
+    "# Here we use a small subset of the dsb challenge data for prediction.\n",
+    "# The original data is available at https://github.com/stardist/stardist/releases/download/0.1.0/dsb2018.zip.\n",
+    "# We have added a few images to the repository so that the notebook runs out of the box.\n",
     "\n",
-    "# select all images in the \"test\" subfolder\n",
+    "# Get all paths to the images in the \"example-images\" folder.\n",
     "from glob import glob\n",
-    "folder = \"/home/pape/Downloads/dsb2018(1)/dsb2018/test\"\n",
-    "inputs = glob(os.path.join(folder, \"images\", \"*.tif\"))\n",
+    "inputs = glob(\"./example-images/*.png\")\n",
     "\n",
-    "# create an output folder and specify the output path for each image\n",
-    "output_folder = os.path.join(folder, \"predictions\")\n",
+    "# Create an output folder and specify the output path for each image.\n",
+    "output_folder = \"./predictions\"\n",
     "os.makedirs(output_folder, exist_ok=True)\n",
     "outputs = [os.path.join(output_folder, os.path.split(inp)[1]) for inp in inputs]\n",
     "\n",
@@ -289,20 +365,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# the model at hand can only predict images which have a xy-size that is\n",
-    "# a multiple of 16. To run with arbitrary size images, we pass the `padding`\n",
+    "# The model at hand can only predict images which have a spatial shape that is\n",
+    "# a multiple of 16. To run with images of other sizes we pass the `padding`\n",
     "# argument to `predict_images` and specify that the input is padded to the next bigger\n",
-    "# size that is divisible by 16 (mode: dynamic)\n",
-    "# as an alternative `\"mode\": \"fixed\"` will pad to a fixed shape, e.g.\n",
-    "# `{\"x\": 512, \"y\": 512, \"mode\": \"fixed\"}` will always pad to a size of 512x512\n",
-    "# the padding is cropped again after the prediction\n",
+    "# size that is divisible by 16 (mode: dynamic).\n",
+    "# As an alternative `\"mode\": \"fixed\"` will pad to a fixed shape, e.g.\n",
+    "# `{\"x\": 512, \"y\": 512, \"mode\": \"fixed\"}` will always pad to a size of 512x512.\n",
+    "# The padding is cropped again after the prediction to restore the input shape.\n",
     "padding = {\"x\": 16, \"y\": 16, \"mode\": \"dynamic\"}\n",
     "predict_images(\n",
     "    model_resource, inputs, outputs, padding=padding, verbose=True\n",
     ")\n",
     "\n",
     "# check the first input/output\n",
-    "show_images(inputs[0], outputs[0].replace(\".tif\", \"-c0.tif\"), outputs[0].replace(\".tif\", \"-c1.tif\"))"
+    "show_images(inputs[0], outputs[0].replace(\".png\", \"-c0.png\"), outputs[0].replace(\".png\", \"-c1.png\"))"
    ]
   },
   {
@@ -312,19 +388,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# instead of padding, we can also use tiling.\n",
-    "# here, we specify a tile size of 224 and a halo (= extension of tile on both sides)\n",
-    "# size of 16, which results in an effective tile shale of 256 = 224 + 2*16\n",
+    "# Instead of padding, we can also use tiling.\n",
+    "# Here, we specify a tile size of 256 and a halo (= what's cropped from the tile on either side) of 16.\n",
     "tiling = {\n",
-    "    \"tile\": {\"x\": 224, \"y\": 224},\n",
+    "    \"tile\": {\"x\": 256, \"y\": 256},\n",
     "    \"halo\": {\"x\": 16, \"y\": 16},\n",
     "}\n",
     "predict_images(\n",
     "    model_resource, inputs, outputs, tiling=tiling, verbose=True\n",
     ")\n",
     "\n",
-    "# check the first input/output\n",
-    "show_images(inputs[0], outputs[0].replace(\".tif\", \"-c0.tif\"), outputs[0].replace(\".tif\", \"-c1.tif\"))"
+    "# Check the first input output pair.\n",
+    "show_images(inputs[0], outputs[0].replace(\".png\", \"-c0.png\"), outputs[0].replace(\".png\", \"-c1.png\"))"
    ]
   },
   {
@@ -537,7 +612,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.10"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,