Skip to content

Virtual Remote Datasets #8657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Virtual Remote Datasets #8657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
11bcc9c
Load data via dataset ID
frcroth May 26, 2025
a964308
Update dataset cache when datasource is updated
frcroth May 26, 2025
9d3ad67
Merge branch 'master' into virtual-remote-datasets
frcroth Jun 2, 2025
63ddf7b
Include attachments in WK generated datasource
frcroth Jun 2, 2025
b9d05f3
In-Memory Cache
frcroth Jun 4, 2025
bec8480
Move byId routes to new controller
frcroth Jun 4, 2025
04bfd3b
remove todo
frcroth Jun 4, 2025
e64fd7b
Check early for WKW
frcroth Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions app/controllers/UserTokenController.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
answer <- accessRequest.resourceType match {
case AccessResourceType.datasource =>
handleDataSourceAccess(accessRequest.resourceId, accessRequest.mode, userBox)(sharingTokenAccessCtx)
case AccessResourceType.dataset =>
handleDataSetAccess(accessRequest.resourceId.directoryName, accessRequest.mode, userBox)(
sharingTokenAccessCtx)
case AccessResourceType.tracing =>
handleTracingAccess(accessRequest.resourceId.directoryName, accessRequest.mode, userBox, token)
case AccessResourceType.annotation =>
Expand Down Expand Up @@ -158,6 +161,34 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
}
}

def handleDataSetAccess(id: String, mode: AccessMode.Value, userBox: Box[User])(
implicit ctx: DBAccessContext): Fox[UserAccessAnswer] = {

def tryRead: Fox[UserAccessAnswer] =
for {
datasetId <- ObjectId.fromString(id)
datasetBox <- datasetDAO.findOne(datasetId).shiftBox
} yield
datasetBox match {
case Full(_) => UserAccessAnswer(granted = true)
case _ => UserAccessAnswer(granted = false, Some("No read access on dataset"))
}

def tryWrite: Fox[UserAccessAnswer] =
for {
datasetId <- ObjectId.fromString(id)
dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext) ?~> "dataset.notFound"
user <- userBox.toFox ?~> "auth.token.noUser"
isAllowed <- datasetService.isEditableBy(dataset, Some(user))
} yield UserAccessAnswer(isAllowed)

mode match {
case AccessMode.read => tryRead
case AccessMode.write => tryWrite
case _ => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
}
}

private def handleTracingAccess(tracingId: String,
mode: AccessMode,
userBox: Box[User],
Expand Down
11 changes: 11 additions & 0 deletions app/controllers/WKRemoteDataStoreController.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,17 @@ class WKRemoteDataStoreController @Inject()(
}
}

def getDataset(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
def getDataset(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
def getDataSource(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =

(as it does not return a Dataset object, which is what is stored in postgres, I’d rather follow the DataSource terminology in this context)

Action.async { implicit request =>
dataStoreService.validateAccess(name, key) { _ =>
for {
dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext)
dataSource <- datasetService.fullDataSourceFor(dataset)
} yield Ok(Json.toJson(dataSource))
}

}

def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
implicit request =>
dataStoreService.validateAccess(name, key) { _ =>
Expand Down
84 changes: 79 additions & 5 deletions app/models/dataset/Dataset.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
import com.scalableminds.util.objectid.ObjectId
import com.scalableminds.util.time.Instant
import com.scalableminds.util.tools.{Fox, JsonHelper}
import com.scalableminds.webknossos.datastore.dataformats.MagLocator
import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
import com.scalableminds.webknossos.datastore.helpers.DataSourceMagInfo
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize, datasource}
import com.scalableminds.webknossos.datastore.models.datasource.DatasetViewConfiguration.DatasetViewConfiguration
import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration
import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
Expand All @@ -17,10 +19,14 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
Category,
CoordinateTransformation,
CoordinateTransformationType,
DataFormat,
DatasetLayerAttachments => AttachmentWrapper,
DataSourceId,
ElementClass,
LayerAttachment,
LayerAttachmentDataFormat,
LayerAttachmentType,
SegmentationLayerLike,
ThinPlateSplineCorrespondences,
DataLayerLike => DataLayer
}
Expand All @@ -40,6 +46,7 @@ import slick.lifted.Rep
import slick.sql.SqlAction
import utils.sql.{SQLDAO, SimpleSQLDAO, SqlClient, SqlToken}

import java.net.URI
import scala.concurrent.ExecutionContext

case class Dataset(_id: ObjectId,
Expand Down Expand Up @@ -841,6 +848,30 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
magInfos <- rowsToMagInfos(rows)
} yield magInfos

def parseMagLocator(row: DatasetMagsRow): Fox[MagLocator] =
for {
mag <- parseMag(row.mag)
axisOrderParsed = row.axisorder match {
case Some(axisOrder) => JsonHelper.parseAs[AxisOrder](axisOrder).toOption
case None => None
}
} yield
MagLocator(
mag,
row.path,
None,
axisOrderParsed,
row.channelindex,
row.credentialid
)

def findAllByDatasetId(datasetId: ObjectId): Fox[Seq[(String, MagLocator)]] =
for {
// We assume non-WKW Datasets here (WKW Resolutions are not handled)
rows <- run(q"""SELECT * FROM webknossos.dataset_mags WHERE _dataset = $datasetId""".as[DatasetMagsRow])
mags <- Fox.combined(rows.map(parseMagLocator))
} yield rows.map(r => r.datalayername).zip(mags)

}

class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
Expand All @@ -855,7 +886,7 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
category <- Category.fromString(row.category).toFox ?~> "Could not parse Layer Category"
boundingBox <- BoundingBox
.fromSQL(parseArrayLiteral(row.boundingbox).map(_.toInt))
.toFox ?~> "Could not parse boundingbox"
.toFox ?~> "Could not parse bounding box"
elementClass <- ElementClass.fromString(row.elementclass).toFox ?~> "Could not parse Layer ElementClass"
mags <- datasetMagsDAO.findMagForLayer(datasetId, row.name) ?~> "Could not find mag for layer"
defaultViewConfigurationOpt <- Fox.runOptional(row.defaultviewconfiguration)(
Expand All @@ -867,6 +898,10 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
coordinateTransformationsOpt = if (coordinateTransformations.isEmpty) None else Some(coordinateTransformations)
additionalAxes <- datasetLayerAdditionalAxesDAO.findAllForDatasetAndDataLayerName(datasetId, row.name)
additionalAxesOpt = if (additionalAxes.isEmpty) None else Some(additionalAxes)
attachments <- datasetLayerAttachmentsDAO.findAllForDatasetAndDataLayerName(datasetId, row.name)
attachmentsOpt = if (attachments.isEmpty) None else Some(attachments)

dataFormat = row.dataformat.flatMap(df => DataFormat.fromString(df))
} yield {
category match {
case Category.segmentation =>
Expand All @@ -883,7 +918,10 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
defaultViewConfigurationOpt,
adminViewConfigurationOpt,
coordinateTransformationsOpt,
additionalAxesOpt
additionalAxesOpt,
numChannels = row.numchannels,
dataFormat = dataFormat,
attachments = attachmentsOpt
))
case Category.color =>
Fox.successful(
Expand All @@ -896,7 +934,10 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
defaultViewConfigurationOpt,
adminViewConfigurationOpt,
coordinateTransformationsOpt,
additionalAxesOpt
additionalAxesOpt,
numChannels = row.numchannels,
dataFormat = dataFormat,
attachments = attachmentsOpt
))
case _ => Fox.failure(s"Could not match dataset layer with category $category")
}
Expand All @@ -907,7 +948,7 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
def findAllForDataset(datasetId: ObjectId): Fox[List[DataLayer]] =
for {
rows <- run(q"""SELECT _dataset, name, category, elementClass, boundingBox, largestSegmentId, mappings,
defaultViewConfiguration, adminViewConfiguration
defaultViewConfiguration, adminViewConfiguration, numChannels, dataFormat
FROM webknossos.dataset_layers
WHERE _dataset = $datasetId
ORDER BY name""".as[DatasetLayersRow])
Expand Down Expand Up @@ -1015,6 +1056,39 @@ class DatasetLastUsedTimesDAO @Inject()(sqlClient: SqlClient)(implicit ec: Execu

class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
extends SimpleSQLDAO(sqlClient) {

def parseRow(row: DatasetLayerAttachmentsRow): Fox[LayerAttachment] =
for {
dataFormat <- LayerAttachmentDataFormat.fromString(row.dataformat).toFox ?~> "Could not parse data format"
uri = new URI(row.path)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
uri = new URI(row.path)
uri <- tryo(new URI(row.path))

} yield LayerAttachment(row.name, uri, dataFormat)

Comment on lines +1060 to +1065
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Add error handling for URI construction.

The URI constructor on line 1063 can throw URISyntaxException if the path is malformed. This should be handled gracefully to provide better error messages.

   def parseRow(row: DatasetLayerAttachmentsRow): Fox[LayerAttachment] =
     for {
       dataFormat <- LayerAttachmentDataFormat.fromString(row.dataformat).toFox ?~> "Could not parse data format"
-      uri = new URI(row.path)
+      uri <- Fox.successful(new URI(row.path)).recover {
+        case _: URISyntaxException => Failure(s"Invalid URI path for attachment: ${row.path}")
+      }
     } yield LayerAttachment(row.name, uri, dataFormat)

Committable suggestion skipped: line range outside the PR's diff.

🤖 Prompt for AI Agents 
In app/models/dataset/Dataset.scala around lines 1060 to 1065, the URI
constructor on line 1063 can throw a URISyntaxException if the path is
malformed, but currently there is no error handling for this. Wrap the URI
construction in a try-catch or equivalent error handling mechanism to catch
URISyntaxException and convert it into a meaningful error message within the Fox
monad context, ensuring the method returns a proper error instead of throwing an
exception.

def parseAttachments(rows: List[DatasetLayerAttachmentsRow]): Fox[AttachmentWrapper] =
for {
meshFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.mesh.toString))(parseRow)
agglomerateFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.agglomerate.toString))(
parseRow)
connectomeFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.connectome.toString))(parseRow)
segmentIndexFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.segmentIndex.toString))(
parseRow)
cumsumFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.cumsum.toString))(parseRow)
} yield
AttachmentWrapper(
agglomerates = agglomerateFiles,
connectomes = connectomeFiles,
segmentIndex = segmentIndexFiles.headOption,
meshes = meshFiles,
cumsum = cumsumFiles.headOption
)

def findAllForDatasetAndDataLayerName(datasetId: ObjectId, layerName: String): Fox[AttachmentWrapper] =
for {
rows <- run(q"""SELECT _dataset, layerName, name, path, type, dataFormat
FROM webknossos.dataset_layer_attachments
WHERE _dataset = $datasetId AND layerName = $layerName""".as[DatasetLayerAttachmentsRow])
attachments <- parseAttachments(rows.toList) ?~> "Could not parse attachments"
} yield attachments

def updateAttachments(datasetId: ObjectId, dataLayersOpt: Option[List[DataLayer]]): Fox[Unit] = {
def insertQuery(attachment: LayerAttachment, layerName: String, fileType: String) =
q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat)
Expand Down
Loading
Loading