Make manual reserveManualUpload route return datasetId and directoryName (#8476)

* Allow passing a manual dataset Id when triggering a manual upload

* Add param to enforce new dataset name on manual upload and return directory name f new dataset

* add changelog entry

* fix backend

* fix backend

* fix compilation and remove not needed reserved manual upload param

* make new requireUniqueName param optional

---------

Co-authored-by: Michael Büßemeyer <[email protected]>

Author: MichaelBuessemeyer

CHANGELOG.unreleased.md

@@ -20,6 +20,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - New config option `datastore.dataVaults.credentials` allows admins to set up global credentials for remote dataset loading. [#8509](https://github.com/scalableminds/webknossos/pull/8509)
 
 ### Changed
+- Added a parameter to the reserve manual upload route allowing to make the request fail if the name is already taken. Moreover, the new dataset's id and directory name are returned in the response. [#8476](https://github.com/scalableminds/webknossos/pull/8476)
 - The skeleton tool can no longer be activated if the skeleton layer is invisible. [#8501](https://github.com/scalableminds/webknossos/pull/8501)
 - Improved speed of mesh rendering and mouse interaction in 3D viewport. [#8106](https://github.com/scalableminds/webknossos/pull/8106)
 

app/controllers/WKRemoteDataStoreController.scala

@@ -83,7 +83,11 @@ class WKRemoteDataStoreController @Inject()(
           _ <- folderDAO.assertUpdateAccess(folderId)(AuthorizedAccessContext(user)) ?~> "folder.noWriteAccess"
           layersToLinkWithDatasetId <- Fox.serialCombined(uploadInfo.layersToLink.getOrElse(List.empty))(l =>
             validateLayerToLink(l, user)) ?~> "dataset.upload.invalidLinkedLayers"
-          dataset <- datasetService.createPreliminaryDataset(uploadInfo.name, uploadInfo.organization, dataStore) ?~> "dataset.name.alreadyTaken"
+          dataset <- datasetService.createPreliminaryDataset(
+            uploadInfo.name,
+            uploadInfo.organization,
+            dataStore,
+            uploadInfo.requireUniqueName.getOrElse(false)) ?~> "dataset.upload.creation.failed"
           _ <- datasetDAO.updateFolder(dataset._id, folderId)(GlobalAccessContext)
           _ <- datasetService.addInitialTeams(dataset, uploadInfo.initialTeams, user)(AuthorizedAccessContext(user))
           _ <- datasetService.addUploader(dataset, user._id)(AuthorizedAccessContext(user))

app/models/dataset/DatasetService.scala

@@ -65,12 +65,16 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       _ <- bool2Fox(!name.startsWith(".")) ?~> "dataset.layer.name.invalid.startsWithDot"
     } yield ()
 
-  def createPreliminaryDataset(datasetName: String, organizationId: String, dataStore: DataStore): Fox[Dataset] = {
+  def createPreliminaryDataset(datasetName: String,
+                               organizationId: String,
+                               dataStore: DataStore,
+                               requireUniqueName: Boolean): Fox[Dataset] = {
     val newDatasetId = ObjectId.generate
     for {
-      datasetDirectoryName <- datasetDAO
-        .doesDatasetDirectoryExistInOrganization(datasetName, organizationId)(GlobalAccessContext)
-        .map(if (_) s"$datasetName-${newDatasetId.toString}" else datasetName)
+      isDatasetNameAlreadyTaken <- datasetDAO.doesDatasetDirectoryExistInOrganization(datasetName, organizationId)(
+        GlobalAccessContext)
+      _ <- bool2Fox(!(isDatasetNameAlreadyTaken && requireUniqueName)) ?~> "dataset.name.alreadyTaken"
+      datasetDirectoryName = if (isDatasetNameAlreadyTaken) s"$datasetName-${newDatasetId.toString}" else datasetName
       unreportedDatasource = UnusableDataSource(DataSourceId(datasetDirectoryName, organizationId),
                                                 notYetUploadedStatus)
       newDataset <- createDataset(dataStore, newDatasetId, datasetName, unreportedDatasource)

conf/messages

@@ -113,6 +113,7 @@ dataset.upload.storageExceeded=Cannot upload dataset because the storage quota o
 dataset.upload.finishFailed=Failed to finalize dataset upload.
 dataset.upload.moveToTarget.failed=Failed to move uploaded dataset to target directory.
 dataset.upload.moreBytesThanReserved=The uploaded dataset contains more bytes than originally reserved. Please make sure to reserve the correct amount of bytes.
+dataset.upload.creation.failed=Failed to create dataset.
 dataset.explore.failed.readFile=Failed to read remote file
 dataset.explore.magDtypeMismatch=Element class must be the same for all mags of a layer. Got {0}
 dataset.explore.autoAdd.failed=Failed to automatically import the explored dataset.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -113,7 +113,7 @@ class DataSourceController @Inject()(
       accessTokenService.validateAccessFromTokenContext(
         UserAccessRequest.administrateDataSources(request.body.organization)) {
         for {
-          _ <- dsRemoteWebknossosClient.reserveDataSourceUpload(
+          reservedDatasetInfo <- dsRemoteWebknossosClient.reserveDataSourceUpload(
             ReserveUploadInformation(
               "aManualUpload",
               request.body.datasetName,
@@ -123,10 +123,14 @@ class DataSourceController @Inject()(
               None,
               None,
               request.body.initialTeamIds,
-              request.body.folderId
+              request.body.folderId,
+              Some(request.body.requireUniqueName)
             )
           ) ?~> "dataset.upload.validation.failed"
-        } yield Ok
+        } yield
+          Ok(
+            Json.obj("newDatasetId" -> reservedDatasetInfo.newDatasetId,
+                     "directoryName" -> reservedDatasetInfo.directoryName))
       }
     }
 
@@ -419,6 +423,7 @@ class DataSourceController @Inject()(
               layersToLink = None,
               initialTeams = List.empty,
               folderId = folderId,
+              requireUniqueName = Some(false),
             )
           ) ?~> "dataset.upload.validation.failed"
           datasourceId = DataSourceId(reservedAdditionalInfo.directoryName, organizationId)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/ComposeService.scala

@@ -63,7 +63,8 @@ class ComposeService @Inject()(dataSourceRepository: DataSourceRepository,
         None,
         None,
         List(),
-        Some(composeRequest.targetFolderId)
+        Some(composeRequest.targetFolderId),
+        requireUniqueName = Some(false)
       )
       reservedAdditionalInfo <- remoteWebknossosClient.reserveDataSourceUpload(reserveUploadInfo) ?~> "Failed to reserve upload."
       directory = uploadDirectory(composeRequest.organizationId, reservedAdditionalInfo.directoryName)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala

@@ -38,15 +38,18 @@ case class ReserveUploadInformation(
     totalFileSizeInBytes: Option[Long],
     layersToLink: Option[List[LinkedLayerIdentifier]],
     initialTeams: List[String], // team ids
-    folderId: Option[String])
+    folderId: Option[String],
+    requireUniqueName: Option[Boolean])
 object ReserveUploadInformation {
   implicit val reserveUploadInformation: OFormat[ReserveUploadInformation] = Json.format[ReserveUploadInformation]
 }
-case class ReserveManualUploadInformation(datasetName: String,
-                                          datasetDirectoryName: String,
-                                          organization: String,
-                                          initialTeamIds: List[String],
-                                          folderId: Option[String])
+case class ReserveManualUploadInformation(
+    datasetName: String,
+    organization: String,
+    initialTeamIds: List[String],
+    folderId: Option[String],
+    requireUniqueName: Boolean = false,
+)
 object ReserveManualUploadInformation {
   implicit val reserveUploadInformation: OFormat[ReserveManualUploadInformation] =
     Json.format[ReserveManualUploadInformation]