diff --git a/build.gradle.kts b/build.gradle.kts index ab00e574..96d28699 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import sciview.lwjglNatives @@ -17,8 +18,8 @@ plugins { } java { - sourceCompatibility = JavaVersion.VERSION_21 - targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 } repositories { @@ -44,7 +45,7 @@ dependencies { exclude("org.lwjgl") } - val sceneryVersion = "0.12.0" + val sceneryVersion = "1.0.0-beta-2" api("graphics.scenery:scenery:$sceneryVersion") { version { strictly(sceneryVersion) } exclude("org.biojava.thirdparty", "forester") @@ -130,6 +131,8 @@ dependencies { // OME implementation("ome:formats-bsd") implementation("ome:formats-gpl") + + implementation("org.mastodon:mastodon:1.0.0-beta-34") } //kapt { @@ -147,9 +150,9 @@ tasks { withType().all { val version = System.getProperty("java.version").substringBefore('.').toInt() val default = if (version == 1) "21" else "$version" - kotlinOptions { - jvmTarget = project.properties["jvmTarget"]?.toString() ?: default - freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget( project.properties["jvmTarget"]?.toString() ?: default)) + freeCompilerArgs.addAll(listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn")) } // sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default } diff --git a/gradle.properties b/gradle.properties index 93807ca5..e890f9e7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,8 +4,8 @@ org.gradle.configuration-cache=true #org.gradle.caching.debug=true jvmTarget=21 #useLocalScenery=true -kotlinVersion=1.9.23 -dokkaVersion=1.9.10 +kotlinVersion=2.2.10 +dokkaVersion=1.9.20 scijavaParentPOMVersion=40.0.0 version=0.4.1-SNAPSHOT diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 2b44d86f..0aa5d6c2 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -164,10 +164,10 @@ class SciView : SceneryBase, CalibratedRealInterval { * The primary camera/observer in the scene */ var camera: Camera? = null - set(value) { - field = value - setActiveObserver(field) - } + set(value) { + field = value + setActiveObserver(field) + } lateinit var controls: Controls val targetArcball: AnimatedCenteringBeforeArcBallControl @@ -1709,12 +1709,25 @@ class SciView : SceneryBase, CalibratedRealInterval { return renderer } + private var originalFOV = camera?.fov + /** * Enable VR rendering */ fun toggleVRRendering() { var renderer = renderer ?: return + // Save camera's original settings if we switch from 2D to VR + if (!vrActive) { + originalFOV = camera?.fov + } + + // If turning off VR, store the controls state before deactivating + if (vrActive) { + // We're about to turn off VR + controls.stashControls() + } + vrActive = !vrActive val cam = scene.activeObserver as? DetachedHeadCamera ?: return var ti: TrackerInput? = null @@ -1750,6 +1763,15 @@ class SciView : SceneryBase, CalibratedRealInterval { // Convert back to normal Camera logger.info("Shutting down VR") cam.tracker = null + + // Reset FOV to original value when turning off VR + originalFOV?.let { camera?.fov = it } + + // Restore controls after turning off VR + controls.restoreControls() + + // Reset input controls to ensure proper camera behavior + inputSetup() } // Enable push mode if VR is inactive, and the other way round @@ -1775,7 +1797,12 @@ class SciView : SceneryBase, CalibratedRealInterval { if (hub.has(SceneryElement.HMDInput)) { val hmd = hub.get(SceneryElement.HMDInput) as? OpenVRHMD hmd?.close() - // TODO hub.remove(hmd) + // Get the actual key that was used to store this element + val keyToRemove = hub.elements.entries.find { it.value == hmd }?.key + keyToRemove?.let { + hub.elements.remove(it) + logger.info("Removed ${it.name} from hub.") + } logger.debug("Closed HMD.") } } diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt index 7bb7ad35..2ab43b60 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt @@ -1,32 +1,34 @@ package sc.iview.commands.analysis -import graphics.scenery.DetachedHeadCamera -import graphics.scenery.Icosphere -import graphics.scenery.InstancedNode -import graphics.scenery.Mesh -import graphics.scenery.ShaderMaterial -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.controls.behaviours.AnalogInputWrapper +import graphics.scenery.controls.behaviours.ConfirmableClickBehaviour +import graphics.scenery.controls.behaviours.VRTouch import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.ui.* import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger import graphics.scenery.volumes.RAIVolume import graphics.scenery.volumes.Volume -import org.joml.Math -import org.joml.Matrix4f -import org.joml.Vector3f -import org.joml.Vector4f +import org.joml.* +import org.mastodon.mamut.model.Spot import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.ui.behaviour.DragBehaviour import sc.iview.SciView -import sc.iview.commands.analysis.ConfirmableClickBehaviour -import sc.iview.commands.analysis.SpineMetadata -import sc.iview.commands.analysis.TimepointObserver +import sc.iview.commands.analysis.HedgehogAnalysis.SpineGraphVertex +import sc.iview.controls.behaviours.MoveInstanceVR +import sc.iview.controls.behaviours.MultiButtonManager +import sc.iview.controls.behaviours.VR2HandNodeTransform +import sc.iview.controls.behaviours.VRGrabTheWorld import java.io.BufferedWriter import java.io.FileWriter import java.nio.file.Path +import java.util.ArrayList import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread @@ -34,12 +36,12 @@ import kotlin.concurrent.thread * Base class for different VR cell tracking purposes. It includes functionality to add spines and edgehogs, * as used by [EyeTracking], and registers controller bindings via [inputSetup]. It is possible to register observers * that listen to timepoint changes with [registerObserver]. - * @param [sciview] The [sc.iview.SciView] instance to use + * @param [sciview] The [SciView] instance to use */ open class CellTrackingBase( open var sciview: SciView ) { - val logger by lazyLogger() + val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) lateinit var sessionId: String lateinit var sessionDirectory: Path @@ -52,33 +54,181 @@ open class CellTrackingBase( val referenceTarget = Icosphere(0.004f, 2) - @Volatile var tracking = false - var playing = true + @Volatile var eyeTrackingActive = false + var playing = false var direction = PlaybackDirection.Backward - var volumesPerSecond = 1 + var volumesPerSecond = 6f var skipToNext = false var skipToPrevious = false var volumeScaleFactor = 1.0f - // determines whether the volume and hedgehogs should keep listening for updates or not + private lateinit var lightTetrahedron: List + + val volumeTimepointWidget = TextBoard() + + /** determines whether the volume and hedgehogs should keep listening for updates or not */ var cellTrackingActive: Boolean = false - open var linkCreationCallback: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null - open var finalTrackCallback: (() -> Unit)? = null + /** Takes a list of [SpineGraphVertex] and its positions to create the corresponding track in Mastodon. + * In the case of controller tracking, the points were already sent to Mastodon one by one via [singleLinkTrackedCallback] and the list is not needed. + * Set the first boolean to true if the coordinates are in world space. The bridge will convert them to Mastodon coords. + * The first Spot defines whether to start with an existing spot, so the lambda will use that as starting point. + * The second spot defines whether we want to merge into this spot. */ + var trackCreationCallback: (( + List>?, + radius: Float, + isWorldSpace: Boolean, + startSpot: Spot?, + mergeSpot: Spot? + ) -> Unit)? = null + + /** Passes the current time point, a position and a radius to the bridge to either create a new spot + * or to delete an existing spot if there is a spot selected. + * The deleteBranch flag indicates whether we want to delete the whole branch or just a spot. + * isVoxelCoords indicates whether the coordinates are in sciview or in mastodon space */ + var spotCreateDeleteCallback: (( + tp: Int, + sciviewPos: Vector3f, + radius: Float, + deleteBranch: Boolean, + isWorldSpace: Boolean + ) -> Unit)? = null + /** Select a spot based on the controller tip's position, current time point and a multiple of the radius + * in which a selection event is counted as valid. addOnly prevents deselection from clicking away. */ + var spotSelectCallback: ((sciviewPos: Vector3f, tp: Int, radiusFactor: Float, addOnly: Boolean) -> Pair)? = null + var spotMoveInitCallback: ((Vector3f) -> Unit)? = null + var spotMoveDragCallback: ((Vector3f) -> Unit)? = null + var spotMoveEndCallback: ((Vector3f) -> Unit)? = null + /** Links a selected spot to the closest spot to handle merge events. */ + var spotLinkCallback: (() -> Unit)? = null + /** Generates a single link between a new position and the previously annotated one. + * Sends the position data to the bridge for intermediary keeping. The integer is the timepoint. + * The Float contains the cursor's radius in sciview space. + * The boolean specifies whether the link preview should be rendered. */ + var singleLinkTrackedCallback: ((pos: Vector3f, tp: Int, radius: Float, preview: Boolean) -> Unit)? = null + var toggleTrackingPreviewCallback: ((Boolean) -> Unit)? = null + var rebuildGeometryCallback: (() -> Unit)? = null + + var stageSpotsCallback: (() -> Unit)? = null + var predictSpotsCallback: ((all: Boolean) -> Unit)? = null + var trainSpotsCallback: (() -> Unit)? = null + var neighborLinkingCallback: (() -> Unit)? = null + // TODO add train flow functionality + var trainFlowCallback: (() -> Unit)? = null + /** Reverts to the point previously saved by Mastodon's undo recorder. Also handles redo events if undo is set to false. */ + var mastodonUndoRedoCallback: ((undo: Boolean) -> Unit)? = null + /** Returns a list of spots currently selected in Mastodon. Used to determine whether to scale the cursor or the spots. */ + var getSelectionCallback: (() -> List)? = null + /** Adjusts the radii of spots, both in sciview and Mastodon. */ + var scaleSpotsCallback: ((radius: Float, update: Boolean) -> Unit)? = null + /** Toggle the visibility of spots in the scene. */ + var setSpotVisCallback: ((Boolean) -> Unit)? = null + /** Toggle the visibility of tracks in the scene. */ + var setTrackVisCallback: ((Boolean) -> Unit)? = null + /** Toggle the visiblity of the volume in the scene while maintaining visibility of spots and links as child elements. */ + var setVolumeVisCallback: ((Boolean) -> Unit)? = null + /** Merges overlapping spots in a given timepoint. */ + var mergeOverlapsCallback: ((Int) -> Unit)? = null - /** How to render the currently active hedgehog: all spines, only spines from the current time point, or none. */ enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + enum class PlaybackDirection { Forward, Backward } + + enum class ElephantMode { StageSpots, TrainAll, PredictTP, PredictAll, NNLinking } + var hedgehogVisibility = HedgehogVisibility.Hidden + var trackVisibility = true + var spotVisibility = true - enum class PlaybackDirection { - Forward, - Backward - } + var leftVRController: TrackedDevice? = null + var rightVRController: TrackedDevice? = null + + var cursor = CursorTool + var leftElephantColumn: Column? = null + var leftColumnPredict: Column? = null + var leftColumnLink: Column? = null + var leftUndoMenu: Column? = null + + var enableTrackingPreview = true + + val leftMenuList = mutableListOf() + var leftMenuIndex = 0 + + val grabButtonManager = MultiButtonManager() + val resetRotationBtnManager = MultiButtonManager() + + val mapper = CellTrackingButtonMapper private val observers = mutableListOf() + open fun run() { + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + // Try to load the correct button mapping corresponding to the controller layout + val isProfileLoaded = mapper.loadProfile(hmd.manufacturer) + if (!isProfileLoaded) { + throw IllegalStateException("Could not load profile, headset type unknown!") + } + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial { + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) + } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + shell.name = "Shell" + sciview.addNode(shell) + + lightTetrahedron = Light.createLightTetrahedron( + Vector3f(0.0f, 0.0f, 0.0f), + spread = 5.0f, + radius = 15.0f, + intensity = 5.0f + ) + lightTetrahedron.forEach { sciview.addNode(it) } + + val volumeNodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } + + val v = (volumeNodes.firstOrNull() as? Volume) + if(v == null) { + logger.warn("No volume found, bailing") + return + } else { + logger.info("found ${volumeNodes.size} volume nodes. Using the first one: ${volumeNodes.first()}") + volume = v + } + + logger.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + logger.info("onDeviceConnect called, cam=${sciview.camera}") + if (device.type == TrackedDeviceType.Controller) { + logger.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + when (device.role) { + TrackerRole.Invalid -> {} + TrackerRole.LeftHand -> leftVRController = device + TrackerRole.RightHand -> rightVRController = device + } + if (device.role == TrackerRole.RightHand) { + attachCursorAndTimepointWidget() + device.model?.name = "rightHand" + } else if (device.role == TrackerRole.LeftHand) { + device.model?.name = "leftHand" + setupElephantMenu() + setupGeneralMenu() + + logger.info("Set up navigation and editing controls.") + } + } + } + inputSetup() + + cellTrackingActive = true + launchUpdaterThread() + } + /** Registers a new observer that will get updated whenever the VR user triggers a timepoint update. */ fun registerObserver(observer: TimepointObserver) { observers.add(observer) @@ -89,36 +239,400 @@ open class CellTrackingBase( observers.remove(observer) } + /** Notifies all active observers of a change of timepoint. */ private fun notifyObservers(timepoint: Int) { observers.forEach { it.onTimePointChanged(timepoint) } } - /** Initialize new hedgehog, used for collecting spines (gaze rays) during eye tracking. */ + /** Attaches a column of [Gui3DElement]s to the left VR controller and adds the column to [leftMenuList]. */ + protected fun createWristMenuColumn( + vararg elements: Gui3DElement, + debug: Boolean = false, + name: String = "Menu" + ): Column { + val column = Column(*elements, centerVertically = true, centerHorizontally = true) + column.ifSpatial { + scale = Vector3f(0.05f) + position = Vector3f(0.05f, 0.05f, column.height / 20f + 0.1f) + rotation = Quaternionf().rotationXYZ(-1.57f, 1.57f, 0f) + } + leftVRController?.model?.let { + sciview.addNode(column, parent = it, activePublish = false) + if (debug) { + column.children.forEach { child -> + val bb = BoundingGrid() + bb.node = child + bb.gridColor = Vector3f(0.5f, 1f, 0.4f) + sciview.addNode(bb, parent = it) + } + } + } + column.name = name + column.pack() + leftMenuList.add(column) + return column + } + + var controllerTrackingActive = false + + /** Intermediate storage for a single track created with the controllers. + * Once tracking is finished, this track is sent to Mastodon. */ + var controllerTrackList = mutableListOf() + var startWithExistingSpot: Spot? = null + + /** This lambda is called every time the user performs a click with controller-based tracking. */ + val trackCellsWithController = ClickBehaviour { _, _ -> + if (!controllerTrackingActive) { + controllerTrackingActive = true + cursor.setTrackingColor() + // we dont want animation, because we track step by step + playing = false + // Assume the user didn't click on an existing spot to start the track. + startWithExistingSpot = null + } + // play the volume backwards, step by step, so cell split events can simply be turned into a merge event + if (volume.currentTimepoint > 0) { + val p = cursor.getPosition() + // did the user click on an existing cell and wants to merge the track into it? + val (selected, isValidSelection) = + spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, false) ?: (null to false) + // If this is the first spot we track, and its a valid existing spot, mark it as such + if (isValidSelection && controllerTrackList.size == 0) { + startWithExistingSpot = selected + logger.info("Set startWithExistingPost to $startWithExistingSpot") + } else { + controllerTrackList.add(p) + } + logger.debug("Tracked a new spot at position $p") + logger.debug("Do we want to merge? $isValidSelection. Selected spot is $selected") + // Create a placeholder link during tracking for immediate feedback + singleLinkTrackedCallback?.invoke(p, volume.currentTimepoint, cursor.radius, enableTrackingPreview) + + volume.goToTimepoint(volume.currentTimepoint - 1) + // If the user clicked a cell and its *not* the first in the track, we assume it is a merge event and end the tracking + if (isValidSelection && controllerTrackList.size > 1) { + endControllerTracking(selected) + } + // This will also redraw all geometry using Mastodon as source + notifyObservers(volume.currentTimepoint) + } else { + sciview.camera?.showMessage("Reached the first time point!", centered = true, distance = 2f, size = 0.2f) + // Let's head back to the last timepoint for starting a new track fast-like + volume.goToLastTimepoint() + endControllerTracking() + } + } + + /** Stops the current controller tracking process and sends the created track to Mastodon. */ + private fun endControllerTracking(mergeSpot: Spot? = null) { + if (controllerTrackingActive) { + logger.info("Ending controller tracking now and sending ${controllerTrackList.size} spots to Mastodon to chew on.") + controllerTrackingActive = false + // Radius can be 0 because the actual radii were already captured during tracking + trackCreationCallback?.invoke(null, 0f, true, startWithExistingSpot, mergeSpot) + controllerTrackList.clear() + cursor.resetColor() + } + } + + fun setupElephantMenu() { + val unpressedColor = Vector3f(0.81f, 0.81f, 1f) + val touchingColor = Vector3f(0.7f, 0.65f, 1f) + val pressedColor = Vector3f(0.54f, 0.44f, 0.96f) + val stageSpotsButton = Button( + "Stage all", + command = { updateElephantActions(ElephantMode.StageSpots) }, byTouch = true, depressDelay = 500, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + val trainAllButton = Button( + "Train All TPs", + command = { updateElephantActions(ElephantMode.TrainAll) }, byTouch = true, depressDelay = 500, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + val predictAllButton = Button( + "Predict All", + command = { updateElephantActions(ElephantMode.PredictAll) }, byTouch = true, depressDelay = 500, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + val predictTPButton = Button( + "Predict TP", + command = { updateElephantActions(ElephantMode.PredictTP) }, byTouch = true, depressDelay = 500, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + val linkingButton = Button( + "NN linking", + command = { updateElephantActions(ElephantMode.NNLinking) }, byTouch = true, depressDelay = 500, + color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) + + leftElephantColumn = + createWristMenuColumn(stageSpotsButton, name = "Stage Menu") + leftElephantColumn?.visible = false + leftColumnPredict = createWristMenuColumn(trainAllButton, predictTPButton, predictAllButton, name = "Train/Predict Menu") + leftColumnPredict?.visible = false + leftColumnLink = createWristMenuColumn(linkingButton, name = "Linking Menu") + leftColumnLink?.visible = false + } + + var lastButtonTime = System.currentTimeMillis() + + /** Ensure that only a single Elephant action is triggered at a time */ + private fun updateElephantActions(mode: ElephantMode) { + val buttonTime = System.currentTimeMillis() + + if ((buttonTime - lastButtonTime) > 1000) { + + thread { + when (mode) { + ElephantMode.StageSpots -> stageSpotsCallback?.invoke() + ElephantMode.TrainAll -> trainSpotsCallback?.invoke() + ElephantMode.PredictTP -> predictSpotsCallback?.invoke(false) + ElephantMode.PredictAll -> predictSpotsCallback?.invoke(true) + ElephantMode.NNLinking -> neighborLinkingCallback?.invoke() + } + + logger.info("We locked the buttons for ${(buttonTime-lastButtonTime)} ms ") + lastButtonTime = buttonTime + } + + } else { + sciview.camera?.showMessage("Have some patience!", duration = 1500, distance = 2f, size = 0.2f, centered = true) + } + + } + + fun setupGeneralMenu() { + + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + val color = Vector3f(0.8f) + val pressedColor = Vector3f(0.95f, 0.35f, 0.25f) + val touchingColor = Vector3f(0.7f, 0.55f, 0.55f) + + val undoButton = Button( + "Undo", + command = { mastodonUndoRedoCallback?.invoke(true) }, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val redoButton = Button( + "Redo", + command = {mastodonUndoRedoCallback?.invoke(false)}, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val toggleTrackingPreviewBtn = ToggleButton( + "Preview Off", "Preview On", command = { + enableTrackingPreview = !enableTrackingPreview + toggleTrackingPreviewCallback?.invoke(enableTrackingPreview) + }, byTouch = true, + color = color, + touchingColor = Vector3f(0.67f, 0.9f, 0.63f), + pressedColor = Vector3f(0.35f, 0.95f, 0.25f), + default = true + ) + val togglePlaybackDirBtn = ToggleButton( + textFalse = "BW", textTrue = "FW", command = { + direction = if (direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + }, byTouch = true, + color = Vector3f(0.52f, 0.87f, 0.86f), + touchingColor = color, + pressedColor = Vector3f(0.84f, 0.87f, 0.52f) + ) + val playSlowerBtn = Button( + "<", command = { + volumesPerSecond = maxOf(volumesPerSecond - 1f, 1f) + cam.showMessage( + "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", + distance = 1.2f, size = 0.2f, centered = true + ) + }, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val playFasterBtn = Button( + ">", command = { + volumesPerSecond = minOf(volumesPerSecond + 1f, 20f) + cam.showMessage( + "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", + distance = 1.2f, size = 0.2f, centered = true + ) + }, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val goToLastBtn = Button( + ">|", command = { + playing = false + volume.goToLastTimepoint() + notifyObservers(volume.currentTimepoint) + cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", + distance = 1.2f, size = 0.2f, centered = true) + }, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val goToFirstBtn = Button( + "|<", command = { + playing = false + volume.goToFirstTimepoint() + notifyObservers(volume.currentTimepoint) + cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", + distance = 1.2f, size = 0.2f, centered = true) + }, byTouch = true, depressDelay = 250, + color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + + val timeControlRow = Row(goToFirstBtn, playSlowerBtn, togglePlaybackDirBtn, playFasterBtn, goToLastBtn) + + leftUndoMenu = createWristMenuColumn(undoButton, redoButton, name = "Left Undo Menu") + leftUndoMenu?.visible = false + val previewMenu = createWristMenuColumn(toggleTrackingPreviewBtn, name = "Preview Menu") + previewMenu.visible = false + val timeMenu = createWristMenuColumn(timeControlRow, name = "Time Menu") + timeMenu.visible = false + + val toggleVolume = ToggleButton( + "Volume off", "Volume on", command = { + val state = volume.visible + setVolumeVisCallback?.invoke(!state) + }, byTouch = true, + color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true + ) + val toggleTracks = ToggleButton( + "Track off", "Track on", + command = { + trackVisibility = !trackVisibility + setTrackVisCallback?.invoke(trackVisibility) + }, + byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true + ) + val toggleSpots = ToggleButton( + "Spots off", "Spots on", + command = { + spotVisibility = !spotVisibility + setSpotVisCallback?.invoke(spotVisibility) + }, + byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true + ) + val toggleVisMenu = createWristMenuColumn(toggleVolume, toggleTracks, toggleSpots) + toggleVisMenu.visible = false + + val mergeButton = Button( + "Merge overlaps", command = { + mergeOverlapsCallback?.invoke(volume.currentTimepoint) + }, byTouch = true, depressDelay = 250, color = color, pressedColor = pressedColor, touchingColor = touchingColor + ) + val cleanupMenu = createWristMenuColumn(mergeButton) + cleanupMenu.visible = false + } + + + private fun cycleLeftMenus() { + leftMenuList.forEach { it.visible = false } + leftMenuIndex = (leftMenuIndex + 1) % leftMenuList.size + logger.debug("Cycling to ${leftMenuList[leftMenuIndex].name}") + leftMenuList[leftMenuIndex].visible = true + } + + fun addHedgehog() { logger.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) hedgehog.visible = false - hedgehog.setMaterial(ShaderMaterial.Companion.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) + hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.visible = false hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } hedgehogs.addChild(hedgehogInstanced) } + /** Attach a spherical cursor to the right controller. */ + private fun attachCursorAndTimepointWidget(debug: Boolean = false) { + // Only attach if not already attached + if (sciview.findNodes { it.name == "VR Cursor" }.isNotEmpty()) { + return + } + + volumeTimepointWidget.text = volume.currentTimepoint.toString() + volumeTimepointWidget.name = "Volume Timepoint Widget" + volumeTimepointWidget.fontColor = Vector4f(0.4f, 0.45f, 1f, 1f) + volumeTimepointWidget.spatial { + scale = Vector3f(0.07f) + position = Vector3f(-0.05f, -0.05f, 0.12f) + rotation = Quaternionf().rotationXYZ(-1.57f, -1.57f, 0f) + } + + rightVRController?.model?.let { + cursor.attachCursor(sciview, it) + sciview.addNode(volumeTimepointWidget, activePublish = false, parent = it) + } + } + + /** Object that represents the 3D cursor in form of a sphere. It needs to be attached to a VR controller via [attachCursor]. + * The current cursor position can be obtained with [getPosition]. The current radius is stored in [radius]. + * The tool can be scaled up and down with [scaleByFactor]. + * [resetColor], [setSelectColor] and [setTrackingColor] allow changing the cursor's color to reflect the currently active operation. */ + object CursorTool { + private val logger by lazyLogger() + var radius: Float = 0.007f + private set + val cursor = Sphere(radius) + private val initPos = Vector3f(-0.01f, -0.05f, -0.03f) + + fun getPosition() = cursor.spatial().worldPosition() + + fun attachCursor(sciview: SciView, parent: Node, debug: Boolean = false) { + cursor.name = "VR Cursor" + cursor.material { + diffuse = Vector3f(0.15f, 0.2f, 1f) + } + cursor.spatial().position = initPos + sciview.addNode(cursor, parent = parent) + + if (debug) { + val bb = BoundingGrid() + bb.node = cursor + bb.name = "Cursor BB" + bb.lineWidth = 2f + bb.gridColor = Vector3f(1f, 0.3f, 0.25f) + sciview.addNode(bb, parent = parent) + } + logger.info("Attached cursor to controller.") + } + + fun scaleByFactor(factor: Float) { + var clampedFac = 1f + // Only apply the factor if we are in the radius range 0.001f - 0.1f + if ((factor < 1f && radius > 0.001f) || (factor > 1f && radius < 0.15f)) { + clampedFac = factor + } + radius *= clampedFac + cursor.spatial().scale = Vector3f(radius/0.007f) + cursor.spatial().position = Vector3f(initPos) + Vector3f(initPos).normalize().times(radius - 0.007f) + } + + fun resetColor() { + cursor.material().diffuse = Vector3f(0.15f, 0.2f, 1f) + } + + fun setSelectColor() { + cursor.material().diffuse = Vector3f(1f, 0.25f, 0.25f) + } + + fun setTrackingColor() { + cursor.material().diffuse = Vector3f(0.65f, 1f, 0.22f) + } + + } + open fun inputSetup() { val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to (TrackerRole.LeftHand to OpenVRHMD.OpenVRButton.Up), - "move_back_fast" to (TrackerRole.LeftHand to OpenVRHMD.OpenVRButton.Down), - "move_left_fast" to (TrackerRole.LeftHand to OpenVRHMD.OpenVRButton.Left), - "move_right_fast" to (TrackerRole.LeftHand to OpenVRHMD.OpenVRButton.Right)).forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key.first, key.second) - + listOf( + "move_forward_fast", + "move_back_fast", + "move_left_fast", + "move_right_fast").forEach { name -> + handler.getBehaviour(name)?.let { behaviour -> + mapper.setKeyBindAndBehavior(hmd, name, behaviour) } } } @@ -154,24 +668,39 @@ open class CellTrackingBase( skipToPrevious = true } - val fasterOrScale = ClickBehaviour { _, _ -> - if (playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond + 1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s", distance = 1.2f, size = 0.2f, centered = true) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.spatial().scale = Vector3f(1.0f).mul(volumeScaleFactor) + class ScaleCursorOrSpotsBehavior(val factor: Float): DragBehaviour { + var isSelected = false + override fun init(p0: Int, p1: Int) { + // determine whether we selected spots or not + isSelected = getSelectionCallback?.invoke()?.isNotEmpty() ?: false } - } - val slowerOrScale = ClickBehaviour { _, _ -> - if (playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond - 1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s", distance = 2f, size = 0.2f, centered = true) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.spatial().scale = Vector3f(1.0f).mul(volumeScaleFactor) + override fun drag(p0: Int, p1: Int) { + if (isSelected) { + scaleSpotsCallback?.invoke(factor, false) + } else { + // Make cursor movement a little stronger than + cursor.scaleByFactor(factor * factor) + } } + + override fun end(p0: Int, p1: Int) { + scaleSpotsCallback?.invoke(factor, true) + } + } + + val scaleCursorOrSpotsUp = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(1.02f), sciview.currentScene) + + val scaleCursorOrSpotsDown = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(0.98f), sciview.currentScene) + + val faster = ClickBehaviour { _, _ -> + volumesPerSecond = maxOf(minOf(volumesPerSecond+0.2f, 20f), 1f) + cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 1.2f, size = 0.2f, centered = true) + } + + val slower = ClickBehaviour { _, _ -> + volumesPerSecond = maxOf(minOf(volumesPerSecond-0.2f, 20f), 1f) + cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 2f, size = 0.2f, centered = true) } val playPause = ClickBehaviour { _, _ -> @@ -183,17 +712,13 @@ open class CellTrackingBase( } } - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - val deleteLastHedgehog = ConfirmableClickBehaviour( armedAction = { timeout -> - cam.showMessage( - "Deleting last track, press again to confirm.", distance = 2f, size = 0.2f, + cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), duration = timeout.toInt(), - centered = true - ) + centered = true) }, confirmAction = { @@ -202,17 +727,14 @@ open class CellTrackingBase( volume.removeChild(lastTrack) } val hedgehogId = hedgehogIds.get() - val hedgehogFile = - sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.Companion.formatDateTime()}.csv") - .toFile() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) hedgehogFileWriter.newLine() hedgehogFileWriter.newLine() hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") hedgehogFileWriter.close() - cam.showMessage( - "Last track deleted.", distance = 2f, size = 0.2f, + cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), duration = 1000, @@ -220,40 +742,137 @@ open class CellTrackingBase( ) }) - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if (direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward + mapper.setKeyBindAndBehavior(hmd, "stepFwd", nextTimepoint) + mapper.setKeyBindAndBehavior(hmd, "stepBwd", prevTimepoint) + + mapper.setKeyBindAndBehavior(hmd, "playback", playPause) + mapper.setKeyBindAndBehavior(hmd, "radiusIncrease", scaleCursorOrSpotsUp) + mapper.setKeyBindAndBehavior(hmd, "radiusDecrease", scaleCursorOrSpotsDown) + + /** Local class that handles double assignment of the left A key which is used to cycle menus as well as + * reset the rotation when pressed while the [VR2HandNodeTransform] is active. */ + class CycleMenuAndLockAxisBehavior(val button: OpenVRHMD.OpenVRButton, val role: TrackerRole) + : DragBehaviour { + fun registerConfig() { + logger.debug("Setting up keybinds for CycleMenuAndLockAxisBehavior") + resetRotationBtnManager.registerButtonConfig(button, role) + } + override fun init(x: Int, y: Int) { + resetRotationBtnManager.pressButton(button, role) + if (!resetRotationBtnManager.isTwoHandedActive()) { + cycleLeftMenus() + } + } + override fun drag(x: Int, y: Int) {} + override fun end(x: Int, y: Int) { + resetRotationBtnManager.releaseButton(button, role) } - cam.showMessage("Playing: ${direction}", distance = 2f, centered = true) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 2f, duration = 1000) - dumpHedgehog() - addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("skip_to_next", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Right) - hmd.addKeyBinding("skip_to_prev", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Left) - hmd.addKeyBinding("faster_or_scale", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Up) - hmd.addKeyBinding("slower_or_scale", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Down) - hmd.addKeyBinding("play_pause", TrackerRole.LeftHand, OpenVRHMD.OpenVRButton.Menu) - hmd.addKeyBinding("toggle_hedgehog", TrackerRole.LeftHand, OpenVRHMD.OpenVRButton.Side) - hmd.addKeyBinding("delete_hedgehog", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Side) - hmd.addKeyBinding("playback_direction", TrackerRole.RightHand, OpenVRHMD.OpenVRButton.Menu) - hmd.addKeyBinding("cell_division", TrackerRole.LeftHand, OpenVRHMD.OpenVRButton.Trigger) + } + + val leftAButtonBehavior = CycleMenuAndLockAxisBehavior(OpenVRHMD.OpenVRButton.A, TrackerRole.LeftHand) + leftAButtonBehavior.let { + it.registerConfig() + mapper.setKeyBindAndBehavior(hmd, "cycleMenu", it) + } + + mapper.setKeyBindAndBehavior(hmd, "controllerTracking", trackCellsWithController) + + /** Several behaviors mapped per default to the right menu button. If controller tracking is active, + * end the tracking. If not, clicking will either create or delete a spot, depending on whether the user + * previously selected a spot. Holding the button for more than 0.5s deletes the whole connected branch. */ + class AddDeleteResetBehavior : DragBehaviour { + var start = System.currentTimeMillis() + var wasExecuted = false + override fun init(x: Int, y: Int) { + start = System.currentTimeMillis() + wasExecuted = false + } + override fun drag(x: Int, y: Int) { + if (System.currentTimeMillis() - start > 500 && !wasExecuted) { + val p = cursor.getPosition() + spotCreateDeleteCallback?.invoke(volume.currentTimepoint, p, cursor.radius, true, true) + wasExecuted = true + } + } + override fun end(x: Int, y: Int) { + if (controllerTrackingActive) { + endControllerTracking() + } else { + val p = cursor.getPosition() + logger.debug("Got cursor position: $p") + if (!wasExecuted) { + spotCreateDeleteCallback?.invoke(volume.currentTimepoint, p, cursor.radius, false, true) + } + } + } + } + + mapper.setKeyBindAndBehavior(hmd, "addDeleteReset", AddDeleteResetBehavior()) + + class DragSelectBehavior: DragBehaviour { + var time = System.currentTimeMillis() + override fun init(x: Int, y: Int) { + time = System.currentTimeMillis() + val p = cursor.getPosition() + cursor.setSelectColor() + spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, false) + } + override fun drag(x: Int, y: Int) { + // Only perform the selection method ten times a second + if (System.currentTimeMillis() - time > 100) { + val p = cursor.getPosition() + spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, true) + time = System.currentTimeMillis() + } + } + override fun end(x: Int, y: Int) { + cursor.resetColor() + } + } + + mapper.setKeyBindAndBehavior(hmd, "select", DragSelectBehavior()) + + // this behavior is needed for touching the menu buttons + VRTouch.createAndSet(sciview.currentScene, hmd, listOf(TrackerRole.RightHand), false, customTip = cursor.cursor) + + VRGrabTheWorld.createAndSet( + sciview.currentScene, + hmd, + listOf(OpenVRHMD.OpenVRButton.Side), + listOf(TrackerRole.LeftHand), + grabButtonManager, + 1.5f + ) + + VR2HandNodeTransform.createAndSet( + hmd, + OpenVRHMD.OpenVRButton.Side, + sciview.currentScene, + lockYaxis = false, + target = volume, + onStartCallback = { + setSpotVisCallback?.invoke(false) + setTrackVisCallback?.invoke(false) + }, + onEndCallback = { + rebuildGeometryCallback?.invoke() + // Only re-enable the spots or tracks if they were enabled in the first place + setSpotVisCallback?.invoke(spotVisibility) + setTrackVisCallback?.invoke(trackVisibility) + }, + resetRotationBtnManager = resetRotationBtnManager, + resetRotationButton = MultiButtonManager.ButtonConfig(leftAButtonBehavior.button, leftAButtonBehavior.role) + ) + + // drag behavior can stay enabled regardless of current tool mode + MoveInstanceVR.createAndSet( + sciview.currentScene, hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand), + grabButtonManager, + { cursor.getPosition() }, + spotMoveInitCallback, + spotMoveDragCallback, + spotMoveEndCallback, + ) hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand logger.info("Registered VR controller bindings.") @@ -287,8 +906,6 @@ open class CellTrackingBase( notifyObservers(oldTimepoint + 1) } } - val newTimepoint = volume.viewerState.currentTimepoint - if (hedgehogs.visible) { if (hedgehogVisibility == HedgehogVisibility.PerTimePoint) { @@ -309,13 +926,7 @@ open class CellTrackingBase( } } - if (tracking && oldTimepoint == (volume.timepointCount - 1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - sciview.camera!!.showMessage("Tracking deactivated.", distance = 1.2f, size = 0.2f) - dumpHedgehog() - } + updateLoopActions.forEach { it.invoke() } } Thread.sleep((1000.0f / volumesPerSecond).toLong()) @@ -324,13 +935,37 @@ open class CellTrackingBase( } } - /** Adds a single spine to the currently active hedgehog. */ + private val updateLoopActions: ArrayList<() -> Unit> = ArrayList() + + /** Allows hooking lambdas into the main update loop. This is needed for eye tracking related actions. */ + protected fun attachToLoop(action: () -> Unit) { + updateLoopActions.add(action) + } + + /** Samples a given [volume] from an [origin] point along a [direction]. + * @return a pair of lists, containing the samples and sample positions, respectively. */ + protected fun sampleRayThroughVolume(origin: Vector3f, direction: Vector3f, volume: Volume): Pair?, List?> { + val intersection = volume.spatial().intersectAABB(origin, direction.normalize(), ignoreChildren = true) + + if (intersection is MaybeIntersects.Intersection) { + val localEntry = (intersection.relativeEntry) + val localExit = (intersection.relativeExit) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + val volumeScale = (volume as RAIVolume).getVoxelScale() + return (samples?.map { it ?: 0.0f } to samplePos?.map { it?.mul(volumeScale) ?: Vector3f(0f) }) + } else { + logger.warn("Ray didn't intersect volume! Origin was $origin, direction was $direction.") + } + return (null to null) + } + open fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { val cam = sciview.camera as? DetachedHeadCamera ?: return val sphere = volume.boundingBox?.getBoundingSphere() ?: return val sphereDirection = sphere.origin.minus(center) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + val sphereDist = + Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius val p1 = center val temp = direction.mul(sphereDist + 2.0f * sphere.radius) @@ -382,90 +1017,75 @@ open class CellTrackingBase( } } - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog(){ - logger.info("dumping hedgehog...") - val lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.Companion.formatDateTime()}.csv").toFile() + protected fun writeHedgehogToFile(hedgehog: InstancedNode, hedgehogId: Int) { + val hedgehogFile = + sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + hedgehogFileWriter.write("Timepoint;Origin;Direction;LocalEntry;LocalExit;LocalDirection;HeadPosition;HeadOrientation;Position;Confidence;Samples\n") - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> + val spines = hedgehog.instances.mapNotNull { spine -> spine.metadata["spine"] as? SpineMetadata } spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + hedgehogFileWriter.write( + "${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};" + + "${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};" + + "${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";") + }\n" + ) } hedgehogFileWriter.close() + logger.info("Wrote hedgehog to file ${hedgehogFile.name}") + } - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - if(track == null) { - logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f( - 1.0f, - 0.0f, - 0.0f, - 1.0f - ) - ) - return + protected fun writeTrackToFile( + points: List>, + hedgehogId: Int + ) { + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") } - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - trackFileWriter.newLine() trackFileWriter.newLine() + val parentId = 0 trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - if (linkCreationCallback != null && finalTrackCallback != null) { - track.points.windowed(2, 1).forEach { pair -> - linkCreationCallback?.let { it(pair[0].second) } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - finalTrackCallback?.invoke() - } else { - track.points.windowed(2, 1).forEach { pair -> - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } + val volumeDimensions = volume.getDimensions() + points.windowed(2, 1).forEach { pair -> + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } + trackFileWriter.close() } /** * Stops the current tracking environment and restore the original state. - * This method needs to be overridden. + * This method should be overridden if functionality is extended, to make sure any extra objects are also deleted. */ open fun stop() { + logger.info("Objects in the scene: ${sciview.allSceneNodes.map { it.name }}") + cellTrackingActive = false + lightTetrahedron.forEach { sciview.deleteNode(it) } + // Try to find and delete possibly existing VR objects + listOf("Shell", "leftHand", "rightHand").forEach { + val n = sciview.find(it) + n?.let { sciview.deleteNode(n) } + } + sciview.deleteNode(rightVRController?.model) + sciview.deleteNode(leftVRController?.model) + + logger.info("Cleaned up basic VR objects. Objects left: ${sciview.allSceneNodes.map { it.name }}") + + sciview.toggleVRRendering() + logger.info("Shut down and disabled VR environment.") } } diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt new file mode 100644 index 00000000..1ade6cfb --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt @@ -0,0 +1,164 @@ +package sc.iview.commands.analysis + +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.OpenVRHMD.Manufacturer +import graphics.scenery.controls.OpenVRHMD.OpenVRButton +import graphics.scenery.utils.lazyLogger +import org.scijava.ui.behaviour.Behaviour +import kotlin.to + + +/** This input mapping manager provides several preconfigured profiles for different VR controller layouts. + * The active profile is stored in [currentProfile]. + * To change profile, call [loadProfile] with the new [Manufacturer] type. + * Note that for Quest-like layouts, the lower button always equals [OpenVRButton.A] + * and the upper button is always [OpenVRButton.Menu]. */ +object CellTrackingButtonMapper { + + var eyeTracking: ButtonConfig? = null + var controllerTracking: ButtonConfig? = null + var grabObserver: ButtonConfig? = null + var grabSpot: ButtonConfig? = null + var playback: ButtonConfig? = null + var cycleMenu: ButtonConfig? = null + var faster: ButtonConfig? = null + var slower: ButtonConfig? = null + var stepFwd: ButtonConfig? = null + var stepBwd: ButtonConfig? = null + var addDeleteReset: ButtonConfig? = null + var select: ButtonConfig? = null + var move_forward_fast: ButtonConfig? = null + var move_back_fast: ButtonConfig? = null + var move_left_fast: ButtonConfig? = null + var move_right_fast: ButtonConfig? = null + var radiusIncrease: ButtonConfig? = null + var radiusDecrease: ButtonConfig? = null + + private var currentProfile: Manufacturer = Manufacturer.Oculus + + val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) + + private val profiles = mapOf( + Manufacturer.HTC to mapOf( + "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), + "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), + "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), + "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), + "playback" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), + "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), + "faster" to null, + "slower" to null, + "radiusIncrease" to null, + "radiusDecrease" to null, + "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), + "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), + "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), + "select" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), + "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), + "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), + ), + + Manufacturer.Oculus to mapOf( + "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), + "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), + "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), + "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), + "playback" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.A), + "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), +// "faster" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), +// "slower" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), + "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), + "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), + "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), + "select" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.A), + "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), + "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), + "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), + "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), + "radiusIncrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), + "radiusDecrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), + ) + ) + + init { + loadProfile(Manufacturer.Oculus) + } + + /** Load the current profile's button mapping */ + fun loadProfile(p: Manufacturer): Boolean { + currentProfile = p + val profile = profiles[currentProfile] ?: return false + eyeTracking = profile["eyeTracking"] + controllerTracking = profile["controllerTracking"] + grabObserver = profile["grabObserver"] + grabSpot = profile["grabSpot"] + playback = profile["playback"] + cycleMenu = profile["cycleMenu"] + faster = profile["faster"] + slower = profile["slower"] + stepFwd = profile["stepFwd"] + stepBwd = profile["stepBwd"] + addDeleteReset = profile["addDeleteReset"] + select = profile["select"] + move_forward_fast = profile["move_forward_fast"] + move_back_fast = profile["move_back_fast"] + move_left_fast = profile["move_left_fast"] + move_right_fast = profile["move_right_fast"] + radiusIncrease = profile["radiusIncrease"] + radiusDecrease = profile["radiusDecrease"] + return true + } + + fun getCurrentMapping(): Map?{ + return profiles[currentProfile] + } + + fun getMapFromName(name: String): ButtonConfig? { + return when (name) { + "eyeTracking" -> eyeTracking + "controllerTracking" -> controllerTracking + "grabObserver" -> grabObserver + "grabSpot" -> grabSpot + "playback" -> playback + "cycleMenu" -> cycleMenu + "faster" -> faster + "slower" -> slower + "stepFwd" -> stepFwd + "stepBwd" -> stepBwd + "addDeleteReset" -> addDeleteReset + "select" -> select + "move_forward_fast" -> move_forward_fast + "move_back_fast" -> move_back_fast + "move_left_fast" -> move_left_fast + "move_right_fast" -> move_right_fast + "radiusIncrease" -> radiusIncrease + "radiusDecrease" -> radiusDecrease + else -> null + } + } + + /** Sets a keybinding and behavior for an [hmd], using the [name] string, a [behavior] + * and the keybinding if found in the current profile. */ + fun setKeyBindAndBehavior(hmd: OpenVRHMD, name: String, behavior: Behaviour) { + val config = getMapFromName(name) + if (config != null) { + hmd.addKeyBinding(name, config.r, config.b) + hmd.addBehaviour(name, behavior) + logger.debug("Added behavior $behavior to ${config.r}, ${config.b}.") + } else { + logger.warn("No valid button mapping found for key '$name' in current profile!") + } + } +} + + +/** Combines the [TrackerRole] ([r]) and the [OpenVRHMD.OpenVRButton] ([b]) into a single configuration. */ +data class ButtonConfig ( + /** The [TrackerRole] of this button configuration. */ + var r: TrackerRole, + /** The [OpenVRButton] of this button configuration. */ + var b: OpenVRButton +) \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/analysis/ConfirmableClickBehaviour.kt b/src/main/kotlin/sc/iview/commands/analysis/ConfirmableClickBehaviour.kt deleted file mode 100644 index df09ed4b..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/ConfirmableClickBehaviour.kt +++ /dev/null @@ -1,34 +0,0 @@ -package sc.iview.commands.analysis - -import org.scijava.ui.behaviour.ClickBehaviour -import kotlin.concurrent.thread - -/** - * [org.scijava.ui.behaviour.ClickBehaviour] that waits [timeout] for confirmation by re-executing the behaviour. - * Executes [armedAction] on first invocation, and [confirmAction] on second invocation, if - * it happens within [timeout]. - * - * @author Ulrik Guenther - */ -class ConfirmableClickBehaviour(val armedAction: (Long) -> Any, val confirmAction: (Long) -> Any, var timeout: Long = 3000): - ClickBehaviour { - /** Whether the action is armed at the moment. Action becomes disarmed after [timeout]. */ - private var armed: Boolean = false - - /** - * Action fired at position [x]/[y]. Parameters not used in VR actions. - */ - override fun click(x : Int, y : Int) { - if(!armed) { - armed = true - armedAction.invoke(timeout) - - thread { - Thread.sleep(timeout) - armed = false - } - } else { - confirmAction.invoke(timeout) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt index 0ad8f3bd..415d0a96 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt @@ -1,32 +1,20 @@ package sc.iview.commands.analysis -import graphics.scenery.BoundingGrid -import graphics.scenery.Box -import graphics.scenery.BufferUtils -import graphics.scenery.DetachedHeadCamera -import graphics.scenery.Icosphere -import graphics.scenery.Light -import graphics.scenery.Mesh -import graphics.scenery.PointLight -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.* import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole import graphics.scenery.controls.eyetracking.PupilEyeTracker import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import graphics.scenery.textures.Texture +import graphics.scenery.ui.Button +import graphics.scenery.ui.Column +import graphics.scenery.ui.ToggleButton import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Volume +import graphics.scenery.utils.extensions.* import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.Matrix4f -import org.joml.Vector2f -import org.joml.Vector3f -import org.joml.Vector3i -import org.joml.Vector4f +import org.apache.commons.math3.ml.clustering.Clusterable +import org.apache.commons.math3.ml.clustering.DBSCANClusterer +import org.joml.* import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView import java.awt.image.DataBufferByte @@ -36,16 +24,14 @@ import java.nio.file.Paths import javax.imageio.ImageIO import kotlin.concurrent.thread import kotlin.math.PI +import kotlin.time.TimeSource /** * Tracking class used for communicating with eye trackers, tracking cells with them in a sciview VR environment. * It calls the Hedgehog analysis on the eye tracking results and communicates the results to Mastodon via - * [linkCreationCallback], which is called on every spine graph vertex that is extracted, and - * [finalTrackCallback] which is called after all vertices of a track are iterated, giving Mastodon a chance to rebuild its tracks. + * [trackCreationCallback], which is called on every spine graph vertex that is extracted */ class EyeTracking( - override var linkCreationCallback: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null, - override var finalTrackCallback: (() -> Unit)? = null, sciview: SciView ): CellTrackingBase(sciview) { @@ -58,16 +44,19 @@ class EyeTracking( val confidenceThreshold = 0.60f - private lateinit var lightTetrahedron: List private lateinit var debugBoard: TextBoard - fun run() { + var leftEyeTrackColumn: Column? = null - sciview.toggleVRRendering() - cellTrackingActive = true - logger.info("VR mode has been toggled") - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.Companion.formatDateTime()}" + enum class TrackingType { Follow, Pick } + + private var currentTrackingType = TrackingType.Follow + + override fun run() { + // Do all the things for general VR startup before setting up the eye tracking environment + super.run() + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) referenceTarget.visible = false @@ -93,28 +82,6 @@ class EyeTracking( laser.name = "Laser" sciview.addNode(laser) - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) - } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - shell.name = "Shell" - sciview.addNode(shell) - - val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } - - val v = (volnodes.firstOrNull() as? Volume) - if(v == null) { - logger.warn("No volume found, bailing") - return - } else { - logger.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") - volume = v - } - volume.visible = false - val bb = BoundingGrid() bb.node = volume bb.visible = false @@ -155,9 +122,8 @@ class EyeTracking( Vector3i(image.width, image.height, 1), 3, UnsignedByteType(), - BufferUtils.Companion.allocateByteAndPut(data) - ) - } + BufferUtils.allocateByteAndPut(data) + ) } lastFrame = System.nanoTime() } @@ -171,176 +137,344 @@ class EyeTracking( debugBoard.visible = false sciview.camera?.addChild(debugBoard) - lightTetrahedron = Light.Companion.createLightTetrahedron( - Vector3f(0.0f, 0.0f, 0.0f), - spread = 5.0f, - radius = 15.0f, - intensity = 5.0f - ) - lightTetrahedron.forEach { sciview.addNode(it) } - - thread { - logger.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - logger.info("onDeviceConnect called, cam=${sciview.camera}") - if (device.type == TrackedDeviceType.Controller) { - logger.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + if (device.type == TrackedDeviceType.Controller) { + setupEyeTracking() + setupEyeTrackingMenu() } } - thread { - logger.info("started thread for inputSetup") - inputSetup() - setupCalibration() + + // Attach a behavior to the main loop that stops the eye tracking once we reached the first time point + // and analyzes the created track. + attachToLoop { + val newTimepoint = volume.viewerState.currentTimepoint + if (eyeTrackingActive && newTimepoint == 0) { + eyeTrackingActive = false + playing = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + logger.info("Deactivated eye tracking by reaching timepoint 0.") + sciview.camera!!.showMessage("Tracking deactivated.", distance = 2f, size = 0.2f, centered = true) + analyzeEyeTrack() + } } - launchUpdaterThread() } - private fun setupCalibration( - keybindingCalibration: Pair = (TrackerRole.RightHand to OpenVRHMD.OpenVRButton.Menu), - keybindingTracking: Pair = (TrackerRole.RightHand to OpenVRHMD.OpenVRButton.Trigger) - ) { - val startCalibration = ClickBehaviour { _, _ -> - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - pupilTracker.gazeConfidenceThreshold = confidenceThreshold - if (!pupilTracker.isCalibrated) { - logger.info("pupil is currently uncalibrated") - pupilTracker.onCalibrationInProgress = { - cam.showMessage( - "Crunching equations ...", - distance = 2f, - size = 0.2f, - messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), - duration = 15000, - centered = true - ) - } - pupilTracker.onCalibrationFailed = { - cam.showMessage( - "Calibration failed.", - distance = 2f, - size = 0.2f, - messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), - centered = true - ) - } + private fun setupEyeTracking() { + val cam = sciview.camera as? DetachedHeadCamera ?: return - pupilTracker.onCalibrationSuccess = { - cam.showMessage( - "Calibration succeeded!", - distance = 2f, - size = 0.2f, - messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), - centered = true - ) - - for (i in 0 until 20) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.0f, 1.0f, 0.0f) } - Thread.sleep(100) - referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f) } - Thread.sleep(30) - } + val toggleTracking = ClickBehaviour { _, _ -> + if (!pupilTracker.isCalibrated) { + logger.warn("Can't do eye tracking because eye trackers are not calibrated yet.") + return@ClickBehaviour + } + if (eyeTrackingActive) { + logger.info("deactivated tracking through user input.") + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) + if (currentTrackingType == TrackingType.Follow) { + analyzeEyeTrack() + } else { + analyzeGazeClusters() + } + playing = false + } else { + logger.info("activating tracking...") + playing = if (currentTrackingType == TrackingType.Follow) { + true + } else { + false + } + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) + } + eyeTrackingActive = !eyeTrackingActive + } - hmd.removeBehaviour("start_calibration") - hmd.removeKeyBinding("start_calibration") + mapper.setKeyBindAndBehavior(hmd, "eyeTracking", toggleTracking) + } - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - logger.info("deactivating tracking...") - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.", distance = 2f, size = 0.2f, centered = true) - dumpHedgehog() - } else { - logger.info("activating tracking...") - addHedgehog() - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.", distance = 2f, size = 0.2f, centered = true) - } - tracking = !tracking - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking.first, keybindingTracking.second) + private fun calibrateEyeTrackers(force: Boolean = false) { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated || force) { + logger.info("Calibrating pupil trackers...") - volume.visible = true - playing = true - } + volume.visible = false - pupilTracker.unsubscribeFrames() - sciview.deleteNode(sciview.find("eyeFrames")) + pupilTracker.onCalibrationInProgress = { + cam.showMessage( + "Crunching equations ...", + distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), + duration = 15000, centered = true + ) + } - logger.info("Starting eye tracker calibration") + pupilTracker.onCalibrationFailed = { cam.showMessage( - "Follow the white rabbit.", - distance = 2f, - size = 0.2f, - duration = 1500, + "Calibration failed.", + distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), centered = true ) - pupilTracker.calibrate( - cam, hmd, - generateReferenceData = true, - calibrationTarget = calibrationTarget + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage( + "Calibration succeeded!", + distance = 2f, size = 0.2f, + messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), + centered = true ) - pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { - - PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> - if (gaze.confidence > confidenceThreshold) { - val p = gaze.gazePoint() - referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.spatial().position = p - (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = - "${String.format("%.2f", p.x())}, ${ - String.format( - "%.2f", - p.y() - ) - }, ${String.format("%.2f", p.z())}" - - val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() - - if (tracking) { - addSpine( - headCenter, - direction, - volume, - gaze.confidence, - volume.viewerState.currentTimepoint - ) - } + for (i in 0 until 20) { + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f) } + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f) } + Thread.sleep(30) + } + + if (!pupilTracker.isCalibrated) { + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + } + + volume.visible = true + playing = false + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + logger.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) + + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + + PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.spatial().position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (eyeTrackingActive) { + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) } } } - logger.info("Calibration routine done.") + } + logger.info("Calibration routine done.") + } + } + } + + private fun setupEyeTrackingMenu() { + + val calibrateButton = Button("Calibrate", + command = { calibrateEyeTrackers() }, + byTouch = true, depressDelay = 500) + + val toggleHedgehogsBtn = ToggleButton( + "Hedgehogs Off", + "Hedgehogs On", + command = { + hedgehogVisibility = if (hedgehogVisibility == HedgehogVisibility.Hidden) { + HedgehogVisibility.PerTimePoint + } else { + HedgehogVisibility.Hidden + } + }, + byTouch = true + ) + + val toggleTrackTypeBtn = ToggleButton( + "Follow Cell", + "Count Cells", + command = { + currentTrackingType = if (currentTrackingType == TrackingType.Follow) { + TrackingType.Pick + } else { + TrackingType.Follow + } + }, + byTouch = true, + color = Vector3f(0.65f, 1f, 0.22f), + pressedColor = Vector3f(0.15f, 0.2f, 1f) + ) + + leftEyeTrackColumn = + createWristMenuColumn(toggleTrackTypeBtn, toggleHedgehogsBtn, calibrateButton, name = "Eye Tracking Menu") + leftEyeTrackColumn?.visible = false + } + + /** Writes the accumulated gazes (hedgehog) to a file, analyzes it, + * sends the track to Mastodon and writes the track to a file. */ + private fun analyzeEyeTrack() { + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + writeHedgehogToFile(lastHedgehog, hedgehogId) + + val spines = getSpinesFromHedgehog(lastHedgehog) + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { + logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + if (trackCreationCallback != null && rebuildGeometryCallback != null) { + trackCreationCallback?.invoke(track.points, cursor.radius,false, null, null) + rebuildGeometryCallback?.invoke() + } else { + logger.warn("Tried to send track data to Mastodon but couldn't find the callbacks!") + } + + writeTrackToFile(track.points, hedgehogId) + + } + + private fun Vector3f.toDoubleArray(): DoubleArray { + return this.toFloatArray().map { it.toDouble() }.toDoubleArray() + } + + fun DoubleArray.toVector3f(): Vector3f { + require(size == 3) { "DoubleArray must have exactly 3 elements" } + return Vector3f(this[0].toFloat(), this[1].toFloat(), this[2].toFloat()) + } + + private fun getSpinesFromHedgehog(hedgehog: InstancedNode): List { + return hedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + } + + /** Performs an analysis of collected gazes (hedgehogs) by first calculating the rotational distance between + * subsequent gazes, then discards all gazes larger than 0.3x median distance, clusters the remaining directions + * and samples the volume using the cluster centers as directions. It then extracts the first local minima and sends + * them as spots to Mastodon. */ + private fun analyzeGazeClusters() { + logger.info("Starting analysis of gaze clusters...") + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + writeHedgehogToFile(lastHedgehog, hedgehogId) + // Get spines from the most recent hedgehog + val spines = getSpinesFromHedgehog(lastHedgehog) + logger.info("Starting with ${spines.size} spines") + + // Calculate the distance from each direction to its neighbor + val speeds = spines.zipWithNext { a, b -> a.direction.distance(b.direction) } + logger.info("Min speed: ${speeds.min()}, max speed: ${speeds.max()}") + val medianSpeed = speeds.sorted()[speeds.size/2] + logger.info("Median speed: $medianSpeed") + + // Clean the list of spines by removing the ones that are too far from their neighbors + val cleanedSpines = spines.filterIndexed { index, _ -> speeds[index] < 0.3 * medianSpeed } + logger.info("After cleaning: ${cleanedSpines.size} spines remain") + + var start = TimeSource.Monotonic.markNow() + // Assuming ten times the median distance is a good clustering value... + val clustering = DBSCANClusterer((10 * medianSpeed).toDouble(), 3) + + // Create a map to efficiently find spine metadata by direction + val spineByDirection = cleanedSpines.associateBy { it.direction.toDoubleArray().contentHashCode() } + + val clusters = clustering.cluster(cleanedSpines.map { + Clusterable { + // On the fly conversion of a Vector3f to a double array + it.direction.toDoubleArray() + } + }) + logger.info("Clustering took ${TimeSource.Monotonic.markNow() - start}") + logger.info("We got ${clusters.size} clusters") + + // Extract the mean direction for each cluster, + // and find the corresponding start positions and average them too + val clusterCenters = clusters.map { cluster -> + var meanDir = Vector3f() + var meanPos = Vector3f() + + // Each "point" in the cluster is actually the ray direction + cluster.points.forEach { point -> + // Accumulate the directions + meanDir += point.point.toVector3f() + // Now grab the spine itself so we can also access its origin + val spine = spineByDirection[point.point.contentHashCode()] + if (spine != null) { + meanPos += spine.origin + } else { + logger.warn("Could not find spine for direction: ${point.point.contentToString()}") + } + } + // Calculate means by dividing by cluster size + meanDir /= cluster.points.size.toFloat() + meanPos /= cluster.points.size.toFloat() + + logger.debug("MeanDir for cluster is $meanDir") + logger.debug("MeanPos for cluster is $meanPos") + + (meanPos to meanDir) + } + + // We only need the analyzer to access the smoothing and maxima search functions + val analyzer = HedgehogAnalysis(cleanedSpines, Matrix4f(volume.spatial().world)) + + start = TimeSource.Monotonic.markNow() + val spots = clusterCenters.map { (origin, direction) -> + val (samples, samplePos) = sampleRayThroughVolume(origin, direction, volume) + var spotPos: Vector3f? = null + if (samples != null && samplePos != null) { + val smoothed = analyzer.gaussSmoothing(samples, 4) + val rayMax = smoothed.max() + // take the first local maximum that is at least 20% of the global maximum to prevent spot creation in noisy areas + analyzer.localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> + spotPos = samplePos[index] } } + spotPos + } + logger.info("Sampling volume and spot extraction took ${TimeSource.Monotonic.markNow() - start}") + spots.filterNotNull().forEach { spot -> + spotCreateDeleteCallback?.invoke(volume.currentTimepoint, spot, cursor.radius, false, false) } - hmd.addBehaviour("start_calibration", startCalibration) - hmd.addKeyBinding("start_calibration", keybindingCalibration.first, keybindingCalibration.second) } /** Toggles the VR rendering off, cleans up eyetracking-related scene objects and removes the light tetrahedron * that was created for the calibration routine. */ override fun stop() { pupilTracker.unsubscribeFrames() - cellTrackingActive = false logger.info("Stopped volume and hedgehog updater thread.") - lightTetrahedron.forEach { sciview.deleteNode(it) } - sciview.deleteNode(sciview.find("Shell")) - sciview.deleteNode(sciview.find("eyeFrames")) + val n = sciview.find("eyeFrames") + n?.let { sciview.deleteNode(it) } + // Delete definitely existing objects listOf(referenceTarget, calibrationTarget, laser, debugBoard, hedgehogs).forEach { sciview.deleteNode(it) } logger.info("Successfully cleaned up eye tracking environemt.") - sciview.toggleVRRendering() - logger.info("Shut down eye tracking environment and disabled VR.") + super.stop() } -} +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt index 5364adc4..4f5675c2 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt @@ -6,8 +6,8 @@ import org.joml.Quaternionf import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger import org.slf4j.LoggerFactory -import sc.iview.commands.analysis.SpineMetadata import java.io.File +import kotlin.collections.iterator import kotlin.math.sqrt /** @@ -58,7 +58,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix * From a [list] of Floats, return both the index of local maxima, and their value, * packaged nicely as a Pair */ - private fun localMaxima(list: List): List> { + fun localMaxima(list: List): List> { return list.windowed(3, 1).mapIndexed { index, l -> val left = l[0] val center = l[1] @@ -80,7 +80,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val worldPosition: Vector3f, val index: Int, val value: Float, - val metadata : SpineMetadata, + val metadata : SpineMetadata? = null, var previous: SpineGraphVertex? = null, var next: SpineGraphVertex? = null) { @@ -117,10 +117,31 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) + fun gaussSmoothing(samples: List, iterations: Int): List { + var smoothed = samples.toList() + val kernel = listOf(0.25f, 0.5f, 0.25f) + for (i in 0 until iterations) { + val newSmoothed = ArrayList(smoothed.size) + // Handle the first element + newSmoothed.add(smoothed[0] * 0.75f + smoothed[1] * 0.25f) + // Apply smoothing to the middle elements + for (j in 1 until smoothed.size - 1) { + val value = kernel[0] * smoothed[j-1] + kernel[1] * smoothed[j] + kernel[2] * smoothed[j+1] + newSmoothed.add(value) + } + // Handle the last element + newSmoothed.add(smoothed[smoothed.size - 2] * 0.25f + smoothed[smoothed.size - 1] * 0.75f) + + smoothed = newSmoothed + } + return smoothed + } + fun run(): Track? { - val startingThreshold = 0.002f - val localMaxThreshold = 0.001f + // Adapt thresholds based on data from the first spine + val startingThreshold = timepoints.entries.first().value.first.samples.min() * 2f + 0.002f + val localMaxThreshold = timepoints.entries.first().value.first.samples.max() * 0.2f val zscoreThreshold = 2.0f val removeTooFarThreshold = 5.0f @@ -129,47 +150,45 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } - //step1: find the startingPoint by using startingthreshold + //step1: find the startingPoint by using startingThreshold val startingPoint = timepoints.entries.firstOrNull { entry -> - entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } + entry.value.any { metadata -> metadata.samples.any { it > startingThreshold } } } ?: return null - logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold), localMayThreshold=$localMaxThreshold") // filter timepoints, remove all before the starting point timepoints.filter { it.key > startingPoint.key } - .forEach { timepoints.remove(it.key) } + .forEach { timepoints.remove(it.key) } - logger.info("${timepoints.size} timepoints left") + // Stop timepoints after reaching 0 + val result = mutableMapOf>() + var foundZero = false - fun gaussSmoothing(samples: List, iterations: Int): List { - var smoothed = samples.toList() - val kernel = listOf(0.25f, 0.5f, 0.25f) - for (i in 0 until iterations) { - val newSmoothed = ArrayList(smoothed.size) - // Handle the first element - newSmoothed.add(smoothed[0] * 0.75f + smoothed[1] * 0.25f) - // Apply smoothing to the middle elements - for (j in 1 until smoothed.size - 1) { - val value = kernel[0] * smoothed[j-1] + kernel[1] * smoothed[j] + kernel[2] * smoothed[j+1] - newSmoothed.add(value) - } - // Handle the last element - newSmoothed.add(smoothed[smoothed.size - 2] * 0.25f + smoothed[smoothed.size - 1] * 0.75f) - - smoothed = newSmoothed + for ((time, value) in timepoints) { + if (foundZero) { + break + } + result[time] = value + if (time == 0) { + foundZero = true } - return smoothed } + timepoints.clear() + timepoints.putAll(result) + + logger.info("${timepoints.size} timepoints left") - //step2: find the maxIndices along the spine + // step2: find the maxIndices along the spine // this will be a list of lists, where each entry in the first-level list // corresponds to a time point, which then contains a list of vertices within that timepoint. val candidates: List> = timepoints.map { tp -> val vs = tp.value.mapIndexedNotNull { i, spine -> + // First apply a subtle smoothing kernel to prevent many close/similar local maxima + val smoothedSamples = gaussSmoothing(spine.samples, 4) // determine local maxima (and their indices) along the spine, aka, actual things the user might have // seen when looking into the direction of the spine - val maxIndices = localMaxima(spine.samples.filterNotNull()) + val maxIndices = localMaxima(smoothedSamples) logger.debug("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata @@ -183,12 +202,11 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val position = spine.samplePosList[index.first] val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() SpineGraphVertex(tp.key, - position, - worldPosition, - index.first, - index.second, - spine) - + position, + worldPosition, + index.first, + index.second, + spine) } } else { null @@ -208,13 +226,13 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // calculate world-space distances between current point, and all candidate // vertices, sorting them by distance val vertices = vs - .filter { it.value > localMaxThreshold } - .map { vertex -> - val t = current.worldPosition - vertex.worldPosition - val distance = t.length() - VertexWithDistance(vertex, distance) - } - .sortedBy { it.distance } + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = t.length() + VertexWithDistance(vertex, distance) + } + .sortedBy { it.distance } val closest = vertices.firstOrNull() if(closest != null && closest.distance > 0) { @@ -237,53 +255,53 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) //step4: if some path is longer than multiple average length, it should be removed - while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { - shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() - shortestPath.windowed(3, 1, partialWindows = true).forEach { - // this reconnects the neighbors after the offending vertex has been removed - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - - } + // TODO Don't remove vertices along the path, as that doesn't translate well to Mastodon tracks. Find a different way? +// while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { +// shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() +// shortestPath.windowed(3, 1, partialWindows = true).forEach { +// // this reconnects the neighbors after the offending vertex has been removed +// it.getOrNull(0)?.next = it.getOrNull(1) +// it.getOrNull(1)?.previous = it.getOrNull(0) +// it.getOrNull(1)?.next = it.getOrNull(2) +// it.getOrNull(2)?.previous = it.getOrNull(1) +// } +// } // recalculate statistics after offending vertex removal avgPathLength = shortestPath.map { it.distance() }.average().toFloat() stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() //step5: remove some vertices according to zscoreThreshold - var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - while(remaining > 0) { - val outliers = shortestPath - .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - .map { - val idx = shortestPath.indexOf(it) - listOf(idx-1,idx,idx+1) - }.flatten() - - shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - } - - val afterCount = shortestPath.size - logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") +// var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") +// while(remaining > 0) { +// val outliers = shortestPath +// .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// .map { +// val idx = shortestPath.indexOf(it) +// listOf(idx-1,idx,idx+1) +// }.flatten() +// +// shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() +// remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// +// shortestPath.windowed(3, 1, partialWindows = true).forEach { +// it.getOrNull(0)?.next = it.getOrNull(1) +// it.getOrNull(1)?.previous = it.getOrNull(0) +// it.getOrNull(1)?.next = it.getOrNull(2) +// it.getOrNull(2)?.previous = it.getOrNull(1) +// } +// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") +// } + +// val afterCount = shortestPath.size +// logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") val singlePoints = shortestPath - .groupBy { it.timepoint } - .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } - .filter { - it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f - } + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata?.confidence ?: 0f } } + .filter { + (it.metadata?.direction?.dot(it.previous!!.metadata?.direction) ?: 0f) > 0.5f + } logger.info("Returning ${singlePoints.size} points") @@ -425,11 +443,13 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } } -fun main(filePath: String, args: Array) { +fun main(args: Array) { val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") // main should only be called for testing purposes - val file = File(filePath) - val analysis = HedgehogAnalysis.fromCSV(file) - val results = analysis.run() - logger.info("Results: \n$results") + if (args.isNotEmpty()) { + val file = File(args[0]) + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") + } } diff --git a/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt index 8417778c..dc7f1ebf 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt @@ -18,6 +18,6 @@ data class SpineMetadata( val headOrientation: Quaternionf, val position: Vector3f, val confidence: Float, - val samples: List, + val samples: List, val samplePosList: List = ArrayList() ) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/LoadCremiDatasetAndNeurons.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/LoadCremiDatasetAndNeurons.kt index 56987530..a3545482 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/LoadCremiDatasetAndNeurons.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/LoadCremiDatasetAndNeurons.kt @@ -125,7 +125,7 @@ class LoadCremiDatasetAndNeurons : Command { override fun run() { val task = sciview.taskManager.newTask("Cremi", "Loading dataset") val filter = FileFilter { file -> - val extension = file.name.substringAfterLast(".").toLowerCase() + val extension = file.name.substringAfterLast(".").lowercase() extension == "hdf5" || extension == "hdf" } diff --git a/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt b/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt new file mode 100644 index 00000000..049d75ba --- /dev/null +++ b/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt @@ -0,0 +1,85 @@ +package sc.iview.controls.behaviours + +import graphics.scenery.Scene +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import org.joml.Vector3f +import org.scijava.ui.behaviour.DragBehaviour +import kotlin.collections.forEach +import kotlin.let + +class MoveInstanceVR( + val buttonmanager: MultiButtonManager, + val button: OpenVRHMD.OpenVRButton, + val trackerRole: TrackerRole, + val getTipPosition: () -> Vector3f, + val spotMoveInitCallback: ((Vector3f) -> Unit)? = null, + val spotMoveDragCallback: ((Vector3f) -> Unit)? = null, + val spotMoveEndCallback: ((Vector3f) -> Unit)? = null, +): DragBehaviour { + + override fun init(x: Int, y: Int) { + buttonmanager.pressButton(button, trackerRole) + if (!buttonmanager.isTwoHandedActive()) { + spotMoveInitCallback?.invoke(getTipPosition()) + } + } + + override fun drag(x: Int, y: Int) { + // Only perform the single hand behavior when no other grab button is currently active + // to prevent simultaneous execution of behaviors + if (!buttonmanager.isTwoHandedActive()) { + spotMoveDragCallback?.invoke(getTipPosition()) + } + } + + override fun end(x: Int, y: Int) { + if (!buttonmanager.isTwoHandedActive()) { + spotMoveEndCallback?.invoke(getTipPosition()) + } + buttonmanager.releaseButton(button, trackerRole) + } + + companion object { + + /** + * Convenience method for adding grab behaviour + */ + fun createAndSet( + scene: Scene, + hmd: OpenVRHMD, + buttons: List, + controllerSide: List, + buttonmanager: MultiButtonManager, + getTipPosition: () -> Vector3f, + spotMoveInitCallback: ((Vector3f) -> Unit)? = null, + spotMoveDragCallback: ((Vector3f) -> Unit)? = null, + spotMoveEndCallback: ((Vector3f) -> Unit)? = null, + ) { + hmd.events.onDeviceConnect.add { _, device, _ -> + if (device.type == TrackedDeviceType.Controller) { + device.model?.let { controller -> + if (controllerSide.contains(device.role)) { + buttons.forEach { button -> + val name = "VRDrag:${hmd.trackingSystemName}:${device.role}:$button" + val grabBehaviour = MoveInstanceVR( + buttonmanager, + button, + device.role, + getTipPosition, + spotMoveInitCallback, + spotMoveDragCallback, + spotMoveEndCallback + ) + buttonmanager.registerButtonConfig(button, device.role) + hmd.addBehaviour(name, grabBehaviour) + hmd.addKeyBinding(name, device.role, button) + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt b/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt new file mode 100644 index 00000000..488c87aa --- /dev/null +++ b/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt @@ -0,0 +1,85 @@ +package sc.iview.controls.behaviours + +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerRole +import graphics.scenery.utils.lazyLogger +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean + +/** Keep track of which VR buttons are currently being pressed. This is useful if you want to assign the same button + * to different behaviors with different combinations. This class helps with managing the button states. + * Buttons to track first need to be registered with [registerButtonConfig]. Call [pressButton] and [releaseButton] + * in your behavior init/end methods. You can check if both hands are in use with [isTwoHandedActive] or if a specific + * button is currently pressed with [isButtonPressed]. */ +class MultiButtonManager { + data class ButtonConfig ( + val button: OpenVRHMD.OpenVRButton, + val trackerRole: TrackerRole + ) + + val logger by lazyLogger() + + /** List of registered buttons, stored as [ButtonConfig] and whether the button is pressed right now. */ + private val buttons = ConcurrentHashMap() + private val twoHandedActive = AtomicBoolean(false) + + init { + buttons.forEach { (config, value) -> + buttons[config] = false + } + } + + /** Add a new button configuration that the manager will keep track of. */ + fun registerButtonConfig(button: OpenVRHMD.OpenVRButton, trackerRole: TrackerRole) { + logger.debug("Registered new button config: $button, $trackerRole") + buttons[ButtonConfig(button, trackerRole)] = false + } + + /** Add a button to the list of pressed buttons. */ + fun pressButton(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { + val config = ButtonConfig(button, role) + if (!buttons.containsKey(config)) { return false } + buttons[config] = true + updateTwoHandedState() + return true + } + + /** Overload function that takes a button config instead of separate button and trackerrole inputs. */ + fun pressButton(buttonConfig: ButtonConfig): Boolean { + return pressButton(buttonConfig.button, buttonConfig.trackerRole) + } + + /** Remove a button from the list of pressed buttons. */ + fun releaseButton(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { + val config = ButtonConfig(button, role) + if (!buttons.containsKey(config)) { return false } + buttons[config] = false + updateTwoHandedState() + return true + } + + /** Overload function that takes a button config instead of separate button and trackerrole inputs. */ + fun releaseButton(buttonConfig: ButtonConfig): Boolean { + return releaseButton(buttonConfig.button, buttonConfig.trackerRole) + } + + private fun updateTwoHandedState() { + // Check if any buttons are pressed on both hands + val leftPressed = buttons.any { it.key.trackerRole == TrackerRole.LeftHand && it.value } + val rightPressed = buttons.any { it.key.trackerRole == TrackerRole.RightHand && it.value } + twoHandedActive.set(leftPressed && rightPressed) + } + + /** Returns true when the same button is currently pressed on both VR controllers. */ + fun isTwoHandedActive(): Boolean = twoHandedActive.get() + + /** Check if a button is currently being pressed. */ + fun isButtonPressed(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { + return buttons[ButtonConfig(button, role)] ?: false + } + + /** Retrieve a list of currently registered buttons. */ + fun getRegisteredButtons(): ConcurrentHashMap { + return buttons + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt b/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt new file mode 100644 index 00000000..2f3d1a03 --- /dev/null +++ b/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt @@ -0,0 +1,172 @@ +package sc.iview.controls.behaviours + +import graphics.scenery.Node +import graphics.scenery.Scene +import graphics.scenery.attribute.spatial.Spatial +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.VRScale +import graphics.scenery.controls.behaviours.VRTwoHandDragBehavior +import graphics.scenery.controls.behaviours.VRTwoHandDragOffhand +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.plus +import graphics.scenery.utils.extensions.times +import org.joml.Quaternionf +import org.joml.Vector3f +import sc.iview.controls.behaviours.VRGrabTheWorld.Companion.createAndSet +import java.util.concurrent.CompletableFuture + + +/** Transform a target node [target] by pressing the same buttons defined in [createAndSet] on both VR controllers. + * The fastest way to attach the behavior is by using [createAndSet]. + * [onEndCallback] is an optional lambda that is executed once the behavior ends. + * @author Jan Tiemann + * @author Samuel Pantze */ +class VR2HandNodeTransform( + name: String, + controller: Spatial, + offhand: VRTwoHandDragOffhand, + val scene: Scene, + val scaleLocked: Boolean = false, + val rotationLocked: Boolean = false, + val positionLocked: Boolean = false, + val lockYaxis: Boolean = true, + val target: Node, + private val onStartCallback: (() -> Unit)? = null, + private val onDragCallback: (() -> Unit)? = null, + private val onEndCallback: (() -> Unit)? = null, + private val resetRotationBtnManager: MultiButtonManager? = null, + private val resetRotationButton: MultiButtonManager.ButtonConfig? = null, +) : VRTwoHandDragBehavior(name, controller, offhand) { + + /** To trigger the [onStartCallback] regardless of which order of buttons was used. */ + private var startCallbackTriggered = false + + override fun init(x: Int, y: Int) { + super.init(x, y) + // Find the button that doesn't lock the y Axis and indicate that it is now pressed + val transformBtn = + resetRotationBtnManager?.getRegisteredButtons() + ?.filter { it.key != resetRotationButton }?.map { it.key }?.firstOrNull() + if (transformBtn != null) { + resetRotationBtnManager?.pressButton(transformBtn) + } + if (bothPressed) { + onStartCallback?.invoke() + startCallbackTriggered = true + } + } + + override fun dragDelta( + currentPositionMain: Vector3f, + currentPositionOff: Vector3f, + lastPositionMain: Vector3f, + lastPositionOff: Vector3f + ) { + + // Test whether we now press both buttons but the startCallback wasn't triggered yet. + if (bothPressed && !startCallbackTriggered) { + onStartCallback?.invoke() + startCallbackTriggered = true + } + + val scaleDelta = + VRScale.getScaleDelta(currentPositionMain, currentPositionOff, lastPositionMain, lastPositionOff) + + val currentDirection = (currentPositionMain - currentPositionOff).normalize() + val lastDirection = (lastPositionMain - lastPositionOff).normalize() + if (lockYaxis) { + lastDirection.y = 0f + currentDirection.y = 0f + } + + // Rotation implementation: https://discussions.unity.com/t/two-hand-grabbing-of-objects-in-virtual-reality/219972 + + target.let { + if (!rotationLocked) { + it.ifSpatial { + val rotationDelta = Quaternionf().rotationTo(lastDirection, currentDirection) + if (resetRotationBtnManager?.isTwoHandedActive() == true) { + // Reset the rotation when the reset button was pressed too + rotation = Quaternionf() + } else { + // Rotate node with respect to the world space delta + rotation = Quaternionf(rotationDelta).mul(Quaternionf(rotation)) + } + } + } + if (!scaleLocked) { + target.ifSpatial { + scale *= scaleDelta + } + } + if (!positionLocked) { + val positionDelta = + (currentPositionMain + currentPositionOff) / 2f - (lastPositionMain + lastPositionOff) / 2f + target.ifSpatial { + position.add(positionDelta) + } + } + } + onDragCallback?.invoke() + } + + override fun end(x: Int, y: Int) { + super.end(x, y) + onEndCallback?.invoke() + // Find the button that doesn't lock the y Axis and indicate that it is now released + val transformBtn = resetRotationBtnManager?.getRegisteredButtons()?.filter { it.key != resetRotationButton }?.map {it.key}?.firstOrNull() + if (transformBtn != null) { + resetRotationBtnManager?.releaseButton(transformBtn) + } + // Reset this flag for the next event + startCallbackTriggered = false + } + + companion object { + /** + * Convenience method for adding scale behaviour + */ + fun createAndSet( + hmd: OpenVRHMD, + button: OpenVRHMD.OpenVRButton, + scene: Scene, + scaleLocked: Boolean = false, + rotationLocked: Boolean = false, + positionLocked: Boolean = false, + lockYaxis: Boolean = true, + target: Node, + onStartCallback: (() -> Unit)? = null, + onDragCallback: (() -> Unit)? = null, + onEndCallback: (() -> Unit)? = null, + resetRotationBtnManager: MultiButtonManager? = null, + resetRotationButton: MultiButtonManager.ButtonConfig? = null, + ): CompletableFuture { + @Suppress("UNCHECKED_CAST") return createAndSet( + hmd, button + ) { controller: Spatial, offhand: VRTwoHandDragOffhand -> + // Assign the yLock button and the right grab button to the button manager to handle multi-button events + resetRotationButton?.let { + resetRotationBtnManager?.registerButtonConfig(it.button, it.trackerRole) + } + resetRotationBtnManager?.registerButtonConfig(button, TrackerRole.RightHand) + VR2HandNodeTransform( + "Scaling", + controller, + offhand, + scene, + scaleLocked, + rotationLocked, + positionLocked, + lockYaxis, + target, + onStartCallback, + onDragCallback, + onEndCallback, + resetRotationBtnManager, + resetRotationButton + ) + } as CompletableFuture + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt b/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt new file mode 100644 index 00000000..347f25bc --- /dev/null +++ b/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt @@ -0,0 +1,96 @@ +package sc.iview.controls.behaviours + +import graphics.scenery.Node +import graphics.scenery.Scene +import graphics.scenery.attribute.spatial.Spatial +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.plusAssign +import graphics.scenery.utils.extensions.times +import org.joml.Vector3f +import org.scijava.ui.behaviour.DragBehaviour + + +/** Move yourself (the scene camera) by pressing a VR button. + * The fastest way to attach the behavior is by using [createAndSet]. + * You can pass a [grabButtonmanager] to handle multiple assignments per button. + * @author Jan Tiemann + * @author Samuel Pantze */ +class VRGrabTheWorld ( + @Suppress("UNUSED_PARAMETER") name: String, + controllerHitbox: Node, + private val cam: Spatial, + private val grabButtonmanager: MultiButtonManager? = null, + val button: OpenVRHMD.OpenVRButton, + private val trackerRole: TrackerRole, + private val multiplier: Float +) : DragBehaviour { + + private var camDiff = Vector3f() + + private val controllerSpatial: Spatial = controllerHitbox.spatialOrNull() + ?: throw IllegalArgumentException("controller hitbox needs a spatial attribute") + + + override fun init(x: Int, y: Int) { + grabButtonmanager?.pressButton(button, trackerRole) + camDiff = controllerSpatial.worldPosition() - cam.position + } + + override fun drag(x: Int, y: Int) { + // Only drag when no other grab button is currently active + // to prevent simultaneous behaviors with two-handed gestures + if (grabButtonmanager?.isTwoHandedActive() != true) { + //grabbed world + val newCamDiff = controllerSpatial.worldPosition() - cam.position + val diffTranslation = camDiff - newCamDiff //reversed + cam.position += diffTranslation * multiplier + camDiff = newCamDiff + } + } + + override fun end(x: Int, y: Int) { + grabButtonmanager?.releaseButton(button, trackerRole) + } + + companion object { + + /** + * Convenience method for adding grab behaviour + */ + fun createAndSet( + scene: Scene, + hmd: OpenVRHMD, + buttons: List, + controllerSide: List, + buttonManager: MultiButtonManager? = null, + multiplier: Float = 1f + ) { + hmd.events.onDeviceConnect.add { _, device, _ -> + if (device.type == TrackedDeviceType.Controller) { + device.model?.let { controller -> + if (controllerSide.contains(device.role)) { + buttons.forEach { button -> + val name = "VRDrag:${hmd.trackingSystemName}:${device.role}:$button" + val grabBehaviour = VRGrabTheWorld( + name, + controller.children.first(), + scene.findObserver()!!.spatial(), + buttonManager, + button, + device.role, + multiplier + ) + buttonManager?.registerButtonConfig(button, device.role) + hmd.addBehaviour(name, grabBehaviour) + hmd.addKeyBinding(name, device.role, button) + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/node/Line3D.kt b/src/main/kotlin/sc/iview/node/Line3D.kt index 94b74e90..63cdb78e 100644 --- a/src/main/kotlin/sc/iview/node/Line3D.kt +++ b/src/main/kotlin/sc/iview/node/Line3D.kt @@ -147,7 +147,7 @@ class Line3D : Mesh { * geometry information into consideration if this Node implements [HasGeometry]. * In case a bounding box cannot be determined, the function will return null. */ - override fun generateBoundingBox(): OrientedBoundingBox? { + override fun generateBoundingBox(includeChildren: Boolean): OrientedBoundingBox? { var bb = OrientedBoundingBox(this, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f) for (n in children) {