scalableminds · frcroth · Apr 14, 2025 · Mar 10, 2025 · Mar 12, 2025 · Mar 12, 2025
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -18,17 +18,16 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - New config option `datastore.dataVaults.credentials` allows admins to set up global credentials for remote dataset loading. [#8509](https://github.com/scalableminds/webknossos/pull/8509)
 
 ### Changed
+- When deleting a dataset / layer, layers that are referenced in other datasets are moved there instead of being deleted. [#8437](https://github.com/scalableminds/webknossos/pull/8437/)
 - Added a parameter to the reserve manual upload route allowing to make the request fail if the name is already taken. Moreover, the new dataset's id and directory name are returned in the response. [#8476](https://github.com/scalableminds/webknossos/pull/8476)
 - The skeleton tool can no longer be activated if the skeleton layer is invisible. [#8501](https://github.com/scalableminds/webknossos/pull/8501)
 - Improved speed of mesh rendering and mouse interaction in 3D viewport. [#8106](https://github.com/scalableminds/webknossos/pull/8106)
-
 - Numbered docker image now use different and larger numbers. [#8147](https://github.com/scalableminds/webknossos/pull/8147)
 
 ### Fixed
 - Fixed the alignment of the button that allows restricting floodfill operations to a bounding box. [#8388](https://github.com/scalableminds/webknossos/pull/8388) 
-- Fixed that is was possible to trigger the find largest segment id job on layers which are not stored as segmentation layers on the server. [#8503](https://github.com/scalableminds/webknossos/pull/8503)
+- Fixed that it was possible to trigger the find largest segment id job on layers which are not stored as segmentation layers on the server. [#8503](https://github.com/scalableminds/webknossos/pull/8503)
 - Fixed a rare and subtle bug related to volume annotation and undo/redo. [#7506](https://github.com/scalableminds/webknossos/pull/7506)
-
 - Fixed a bug where segment statistics would sometimes be wrong in case of an on-disk segmentation fallback layer with segment index file. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
 - Fixed a bug where sometimes outdated segment statistics would be displayed. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
 - Fixed a bug where outbound zarr streaming would contain a typo in the zarr header dimension_separator field. [#8510](https://github.com/scalableminds/webknossos/pull/8510)

diff --git a/app/controllers/DatasetController.scala b/app/controllers/DatasetController.scala
@@ -71,12 +71,6 @@ object SegmentAnythingMaskParameters {
   implicit val jsonFormat: Format[SegmentAnythingMaskParameters] = Json.format[SegmentAnythingMaskParameters]
 }
 
-case class MagLinkInfo(mag: DatasetMagInfo, linkedMags: Seq[DatasetMagInfo])
-
-object MagLinkInfo {
-  implicit val jsonFormat: Format[MagLinkInfo] = Json.format[MagLinkInfo]
-}
-
 class DatasetController @Inject()(userService: UserService,
                                   userDAO: UserDAO,
                                   datasetService: DatasetService,

diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala
@@ -5,6 +5,7 @@ import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.controllers.JobExportProperties
+import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLinkInfo}
 import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
@@ -33,8 +34,8 @@ import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import security.{WebknossosBearerTokenAuthenticatorService, WkSilhouetteEnvironment}
 import telemetry.SlackNotificationService
 import utils.WkConf
-import scala.concurrent.duration.DurationInt
 
+import scala.concurrent.duration.DurationInt
 import javax.inject.Inject
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -47,6 +48,7 @@ class WKRemoteDataStoreController @Inject()(
     organizationDAO: OrganizationDAO,
     usedStorageService: UsedStorageService,
     datasetDAO: DatasetDAO,
+    datasetLayerDAO: DatasetLayerDAO,
     userDAO: UserDAO,
     folderDAO: FolderDAO,
     teamDAO: TeamDAO,
@@ -239,6 +241,23 @@ class WKRemoteDataStoreController @Inject()(
     }
   }
 
+  def getPaths(name: String, key: String, organizationId: String, directoryName: String): Action[AnyContent] =
+    Action.async { implicit request =>
+      dataStoreService.validateAccess(name, key) { _ =>
+        for {
+          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext)
+          dataset <- datasetDAO.findOneByDirectoryNameAndOrganization(directoryName, organization._id)(
+            GlobalAccessContext)
+          layers <- datasetLayerDAO.findAllForDataset(dataset._id)
+          magsAndLinkedMags <- Fox.serialCombined(layers)(l => datasetService.getPathsForDataLayer(dataset._id, l.name))
+          magLinkInfos = magsAndLinkedMags.map(_.map { case (mag, linkedMags) => MagLinkInfo(mag, linkedMags) })
+          layersAndMagLinkInfos = layers.zip(magLinkInfos).map {
+            case (layer, magLinkInfo) => LayerMagLinkInfo(layer.name, magLinkInfo)
+          }
+        } yield Ok(Json.toJson(layersAndMagLinkInfos))
+      }
+    }
+
   def deleteDataset(name: String, key: String): Action[JsValue] = Action.async(parse.json) { implicit request =>
     dataStoreService.validateAccess(name, key) { _ =>
       for {

diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala
@@ -5,6 +5,7 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
+import com.scalableminds.webknossos.datastore.helpers.DataSourceMagInfo
 import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
 import com.scalableminds.webknossos.datastore.models.datasource.DatasetViewConfiguration.DatasetViewConfiguration
 import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration
@@ -31,6 +32,7 @@ import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.json._
 import play.utils.UriEncoding
 import slick.dbio.DBIO
+import slick.jdbc.GetResult
 import slick.jdbc.PostgresProfile.api._
 import slick.jdbc.TransactionIsolation.Serializable
 import slick.lifted.Rep
@@ -709,22 +711,21 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
   }
 }
 
-case class DatasetMagInfo(datasetId: ObjectId,
-                          dataLayerName: String,
-                          mag: Vec3Int,
-                          path: Option[String],
-                          realPath: Option[String],
-                          hasLocalData: Boolean)
-
 case class MagWithPaths(layerName: String,
                         mag: Vec3Int,
                         path: Option[String],
                         realPath: Option[String],
                         hasLocalData: Boolean)
 
-object DatasetMagInfo {
-  implicit val jsonFormat: Format[DatasetMagInfo] = Json.format[DatasetMagInfo]
-}
+case class DataSourceMagRow(_dataset: ObjectId,
+                            dataLayerName: String,
+                            mag: String,
+                            path: Option[String],
+                            realPath: Option[String],
+                            hasLocalData: Boolean,
+                            _organization: String,
+                            directoryName: String)
+
 class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
     extends SQLDAO[MagWithPaths, DatasetMagsRow, DatasetMags](sqlClient) {
   protected val collection = DatasetMags
@@ -782,27 +783,45 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
       )
     } yield ()
 
-  def findPathsForDatasetAndDatalayer(datasetId: ObjectId, dataLayerName: String): Fox[List[DatasetMagInfo]] =
+  implicit def GetResultDataSourceMagRow: GetResult[DataSourceMagRow] =
+    GetResult(
+      r =>
+        DataSourceMagRow(ObjectId(r.nextString()),
+                         r.nextString(),
+                         r.nextString(),
+                         r.nextStringOption(),
+                         r.nextStringOption(),
+                         r.nextBoolean(),
+                         r.nextString(),
+                         r.nextString()))
+
+  private def rowsToMagInfos(rows: Vector[DataSourceMagRow]): Fox[List[DataSourceMagInfo]] =
     for {
-      rows <- run(q"""SELECT $columns
+      mags <- Fox.serialCombined(rows.toList)(r => parseMag(r.mag))
+      dataSources = rows.map(row => DataSourceId(row.directoryName, row._organization))
+      magInfos = rows.toList.zip(mags).zip(dataSources).map {
+        case ((row, mag), dataSource) =>
+          DataSourceMagInfo(dataSource, row.dataLayerName, mag, row.path, row.realPath, row.hasLocalData)
+      }
+    } yield magInfos
+
+  def findPathsForDatasetAndDatalayer(datasetId: ObjectId, dataLayerName: String): Fox[List[DataSourceMagInfo]] =
+    for {
+      rows <- run(q"""SELECT $columns, _organization, directoryName
             FROM webknossos.dataset_mags
+            INNER JOIN webknossos.datasets ON webknossos.dataset_mags._dataset = webknossos.datasets._id
             WHERE _dataset = $datasetId
-            AND dataLayerName = $dataLayerName""".as[DatasetMagsRow])
-      magInfos <- Fox.combined(rows.toList.map(parse))
-      datasetMagInfos = magInfos.map(magInfo =>
-        DatasetMagInfo(datasetId, magInfo.layerName, magInfo.mag, magInfo.path, magInfo.realPath, magInfo.hasLocalData))
-    } yield datasetMagInfos
+            AND dataLayerName = $dataLayerName""".as[DataSourceMagRow])
+      magInfos <- rowsToMagInfos(rows)
+    } yield magInfos
 
-  def findAllByRealPath(realPath: String): Fox[List[DatasetMagInfo]] =
+  def findAllByRealPath(realPath: String): Fox[List[DataSourceMagInfo]] =
     for {
-      rows <- run(q"""SELECT $columns
+      rows <- run(q"""SELECT $columns, _organization, directoryName
             FROM webknossos.dataset_mags
-            WHERE realPath = $realPath""".as[DatasetMagsRow])
-      mags <- Fox.serialCombined(rows.toList)(r => parseMag(r.mag))
-      magInfos = rows.toList.zip(mags).map {
-        case (row, mag) =>
-          DatasetMagInfo(ObjectId(row._Dataset), row.datalayername, mag, row.path, row.realpath, row.haslocaldata)
-      }
+            INNER JOIN webknossos.datasets ON webknossos.dataset_mags._dataset = webknossos.datasets._id
+            WHERE realPath = $realPath""".as[DataSourceMagRow])
+      magInfos <- rowsToMagInfos(rows)
     } yield magInfos
 
 }

diff --git a/app/models/dataset/DatasetService.scala b/app/models/dataset/DatasetService.scala
@@ -4,6 +4,7 @@ import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContex
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.helpers.DataSourceMagInfo
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{
   UnusableDataSource,
   InboxDataSourceLike => InboxDataSource
@@ -360,15 +361,24 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       _ <- Fox.serialCombined(pathInfos)(updateRealPath)
     } yield ()
 
-  def getPathsForDataLayer(datasetId: ObjectId, layerName: String): Fox[List[(DatasetMagInfo, List[DatasetMagInfo])]] =
+  /**
+    * Returns a list of tuples, where the first element is the magInfo and the second element is a list of all magInfos
+    * that share the same realPath but have a different dataSourceId. For each mag in the data layer there is one tuple.
+    * @param datasetId id of the dataset
+    * @param layerName name of the layer in the dataset
+    * @return
+    */
+  def getPathsForDataLayer(datasetId: ObjectId,
+                           layerName: String): Fox[List[(DataSourceMagInfo, List[DataSourceMagInfo])]] =
     for {
       magInfos <- datasetMagsDAO.findPathsForDatasetAndDatalayer(datasetId, layerName)
       magInfosAndLinkedMags <- Fox.serialCombined(magInfos)(magInfo =>
         magInfo.realPath match {
           case Some(realPath) =>
             for {
               pathInfos <- datasetMagsDAO.findAllByRealPath(realPath)
-            } yield (magInfo, pathInfos.filter(!_.equals(magInfo)))
+              filteredPathInfos = pathInfos.filter(_.dataSourceId != magInfo.dataSourceId)
+            } yield (magInfo, filteredPathInfos)
           case None => Fox.successful((magInfo, List()))
       })
     } yield magInfosAndLinkedMags

diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -110,6 +110,7 @@ GET           /datastores
 PUT           /datastores/:name/datasource                                                      controllers.WKRemoteDataStoreController.updateOne(name: String, key: String)
 PUT           /datastores/:name/datasources                                                     controllers.WKRemoteDataStoreController.updateAll(name: String, key: String)
 PUT           /datastores/:name/datasources/paths                                               controllers.WKRemoteDataStoreController.updatePaths(name: String, key: String)
+GET           /datastores/:name/datasources/:organizationId/:directoryName/paths                controllers.WKRemoteDataStoreController.getPaths(name: String, key: String, organizationId: String, directoryName: String)
 PATCH         /datastores/:name/status                                                          controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
 POST          /datastores/:name/reserveUpload                                                   controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String)
 GET           /datastores/:name/getUnfinishedUploadsForUser                                     controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String)

diff --git a/docs/datasets/settings.md b/docs/datasets/settings.md
@@ -71,4 +71,6 @@ You don't have to set complete *View Configurations* in either option, as WEBKNO
 
 Offers an option to delete a dataset and completely remove it from WEBKNOSSOS. Be careful, this cannot be undone!
 
+When other datasets reference layers from this dataset, WEBKNOSSOS will try to move these layers to the dataset they are referenced in, so that it can still be accessed.
+
 ![Dataset Editing: Delete Tab](../images/delete_tab.jpeg)
diff --git a/util/src/main/scala/com/scalableminds/util/mvc/Formatter.scala b/util/src/main/scala/com/scalableminds/util/mvc/Formatter.scala
@@ -88,8 +88,12 @@ trait Formatter {
 
     def formatStackTrace(failure: Failure) =
       failure.exception match {
-        case Full(exception) if includeStackTraces => s" Stack trace: ${TextUtils.stackTraceAsString(exception)} "
-        case _                                     => ""
+        case Full(exception) =>
+          if (includeStackTraces)
+            s" Stack trace: ${TextUtils.stackTraceAsString(exception)} "
+          else
+            exception.toString
+        case _ => ""
       }
 
     def formatNextChain(chainBox: Box[Failure]): String = chainBox match {

diff --git a/...tastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/...tastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -495,7 +495,7 @@ class DataSourceController @Inject()(
       val dataSourceId = DataSourceId(datasetDirectoryName, organizationId)
       accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataSource(dataSourceId)) {
         for {
-          _ <- binaryDataServiceHolder.binaryDataService.deleteOnDisk(
+          _ <- dataSourceService.deleteOnDisk(
             organizationId,
             datasetDirectoryName,
             reason = Some("the user wants to delete the dataset")) ?~> "dataset.delete.failed"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -71,4 +71,6 @@ You don't have to set complete View Configurations in either option, as WEBKNO

		Offers an option to delete a dataset and completely remove it from WEBKNOSSOS. Be careful, this cannot be undone!

		When other datasets reference layers from this dataset, WEBKNOSSOS will try to move these layers to the dataset they are referenced in, so that it can still be accessed.

		![Dataset Editing: Delete Tab](../images/delete_tab.jpeg)