feat: MediaStream Image Capture API (#32)

Capabilities and settings

Author: jkruse

.gitignore

@@ -41,3 +41,4 @@ WebApp.log
 AppHtml/Custom/
 dist/
 /.idea/
+/Programs/temp/

AppSrc/MediaStreamImageCaptureDemo.wo

@@ -1021,7 +1021,7 @@ Object oMediaStreamRecordingDemo is a cWebView
 
     Procedure OnShow
         Boolean bIsSupported
-        String sPermMic sPermCam
+        String sPermMic sPermCam sPermPanTiltZoom
 
         Forward Send OnShow
 
@@ -1034,6 +1034,8 @@ Object oMediaStreamRecordingDemo is a cWebView
             Send Log of oLogger (SFormat("Microphone permission is '%1'", sPermMic))
             WebGet psCameraPermission of oUserMediaStream to sPermCam
             Send Log of oLogger (SFormat("Camera permission is '%1'", sPermCam))
+            WebGet psPanTiltZoomPermission of oUserMediaStream to sPermPanTiltZoom
+            Send Log of oLogger (SFormat("Camera pan/tilt/zoom permission is '%1'", sPermPanTiltZoom))
 
             If ((sPermMic = "granted") and (sPermCam = "granted")) Begin
                 WebSet pbEnabled of oBtnRequestPermission to False

AppSrc/MediaStreamRecordingDemo.wo

@@ -306,6 +306,22 @@ Object oWebApp is a cWebApp
     Use BatteryStatusDemo.wo
     Use MediaStreamImageCaptureDemo.wo
     Set phoDefaultView to oDashboard
+    
+    Procedure End_Construct_Object
+        String sPath sLine
+        Boolean bTempExists
+        
+        Forward Send End_Construct_Object
+        
+        Get GetApplicationPath of ghoApplication to sPath
+        Move (sPath + "\temp") to sPath
+        File_Exist sPath bTempExists
+        If (not(bTempExists)) Begin
+            Make_Directory sPath
+        End
+        Send RegisterDownloadFolder of ghoWebResourceManager sPath
+        Set_Directory sPath
+    End_Procedure
 
 End_Object
 

AppSrc/WebApp.src

@@ -3,32 +3,105 @@
 Define MS_KIND_VIDEO for "videoinput"
 Define MS_KIND_AUDIO for "audioinput"
 
+Define MS_MODE_NONE for "none"
+Define MS_MODE_MANUAL for "manual"
+Define MS_MODE_SINGLE_SHOT for "single-shot"
+Define MS_MODE_CONTINUOUS for "continuous"
+
+Define MS_RESIZE_MODE_NONE for "none"
+Define MS_RESIZE_MODE_CROP_AND_SCALE for "crop-and-scale"
+
 Struct tInputDeviceInfo
     String deviceId
     String groupId
-    String kind // MS_KIND_VIDEO or MS_KIND_AUDIO
+    String kind // MS_KIND_*
     String label
 End_Struct
 
+Struct tMediaSettingsRange
+    Number min
+    Number max
+    Number step
+End_Struct
+
+Struct tMediaTrackCapabilities
+    String deviceId
+    String groupId
+    
+    // Audio
+    Boolean[]           autoGainControl
+    tMediaSettingsRange channelCount
+    Boolean[]           echoCancellation
+    tMediaSettingsRange latency
+    Boolean[]           noiseSuppression
+    tMediaSettingsRange sampleRate
+    tMediaSettingsRange sampleSize
+    Boolean[]           voiceIsolation
+    
+    // Video
+    tMediaSettingsRange aspectRatio
+    tMediaSettingsRange brightness
+    tMediaSettingsRange colorTemperature
+    tMediaSettingsRange contrast
+    tMediaSettingsRange exposureCompensation
+    String[]            exposureMode     // MS_MODE_*
+    tMediaSettingsRange exposureTime
+    String[]            facingMode
+    tMediaSettingsRange focusDistance
+    String[]            focusMode        // MS_MODE_*
+    tMediaSettingsRange frameRate
+    tMediaSettingsRange height
+    tMediaSettingsRange iso
+    tMediaSettingsRange pan
+    String[]            resizeMode       // MS_RESIZE_MODE_*
+    tMediaSettingsRange saturation
+    tMediaSettingsRange sharpness
+    tMediaSettingsRange tilt
+    Boolean             torch
+    String[]            whiteBalanceMode // MS_MODE_*
+    tMediaSettingsRange width
+    tMediaSettingsRange zoom
+End_Struct
+
 Struct tMediaTrackSettings
     String deviceId
     String groupId
 
+    // Audio
     String autoGainControl
     String channelCount
     String echoCancellation
     String latency
     String noiseSuppression
     String sampleRate
     String sampleSize
+    String voiceIsolation
 
+    // Video
     String aspectRatio
+    String brightness
+    String colorTemperature
+    String contrast
+    String exposureCompensation
+    String exposureMode
+    String exposureTime
     String facingMode
+    String focusDistance
+    String focusMode
     String frameRate
     String height
-    String width
+    String iso
+    String pan
     String resizeMode
+    String saturation
+    String sharpness
+    String tilt
+    String torch
+    String whiteBalanceMode
+    String width
+    String zoom
 
+    // Display
     String cursor
     String displaySurface
     String logicalSurface
@@ -37,6 +110,7 @@ End_Struct
 Struct tMediaStreamTrackInfo
     String kind
     String label
+    tMediaTrackCapabilities capabilities
     tMediaTrackSettings settings
 End_Struct
 
@@ -97,6 +171,8 @@ Class cUserMediaStreamAPI is a cMediaStream
         Property String psMicrophonePermission ""
         { WebProperty=Client DesignTime=False }
         Property String psCameraPermission ""
+        { WebProperty=Client DesignTime=False }
+        Property String psPanTiltZoomPermission ""
 
         // Set to True to include an audio stream
         { WebProperty=Client Category=Audio }
@@ -129,6 +205,9 @@ Class cUserMediaStreamAPI is a cMediaStream
         // Set to True to include a video stream
         { WebProperty=Client Category=Video }
         Property Boolean pbVideo False
+        // Set to True to enable pan/tilt/zoom features if available (will ask for extended permissions)
+        { WebProperty=Client Category=Video }
+        Property Boolean pbWithPanTiltZoom False
         // Set to a deviceId from OnEnumerateDevices to request a specific device
         { WebProperty=Client Category=Video }
         Property String psVideoDeviceId ""
@@ -160,6 +239,10 @@ Class cUserMediaStreamAPI is a cMediaStream
         { WebProperty=Client Category="Client events" }
         Property String psClientOnCameraPermissionChange ""
         { WebProperty=Client Category="Server events" }
+        Property Boolean pbServerOnPanTiltZoomPermissionChange False
+        { WebProperty=Client Category="Client events" }
+        Property String psClientOnPanTiltZoomPermissionChange ""
+        { WebProperty=Client Category="Server events" }
         Property Boolean pbServerOnEnumerateDevices False
         { WebProperty=Client Category="Client events" }
         Property String psClientOnEnumerateDevices ""
@@ -172,6 +255,7 @@ Class cUserMediaStreamAPI is a cMediaStream
 
         WebPublishProcedure OnMicrophonePermissionChange
         WebPublishProcedure OnCameraPermissionChange
+        WebPublishProcedure OnPanTiltZoomPermissionChange
         WebPublishProcedure OnEnumerateDevicesProxy
     End_Procedure
 
@@ -202,6 +286,10 @@ Class cUserMediaStreamAPI is a cMediaStream
     Procedure OnCameraPermissionChange
     End_Procedure
 
+    { MethodType=Event }
+    Procedure OnPanTiltZoomPermissionChange
+    End_Procedure
+    
     { MethodType=Event }
     Procedure OnEnumerateDevices tInputDeviceInfo[] stDevices
     End_Procedure

AppSrc/cMediaStreamAPI.pkg

@@ -1,6 +1,32 @@
 Use cWebObject.pkg
 Use cMediaStreamAPI.pkg
 
+Struct tMediaStreamPhotoCapabilities
+    String[] fillLightMode // Zero or more of "auto", "off", "flash"
+    tMediaSettingsRange imageHeight
+    tMediaSettingsRange imageWidth
+    String redEyeReduction // "never", "always" or "controllable"
+End_Struct
+
+Struct tMediaStreamPhotoSettings
+    String fillLightMode // numerous bugs concerning this, do not expect it to work
+    Integer imageHeight
+    Integer imageWidth
+    Boolean redEyeReduction
+End_Struct
+
+Struct tMediaStreamImageCaptureInfo
+    tMediaStreamPhotoCapabilities capabilities
+    tMediaStreamPhotoSettings settings
+    tMediaStreamTrackInfo track
+End_Struct
+
+Enum_List
+    Define MSIC_FILL_LIGHT_MODE_AUTO
+    Define MSIC_FILL_LIGHT_MODE_OFF
+    Define MSIC_FILL_LIGHT_MODE_FLASH
+End_Enum_List
+
 Class cMediaStreamImageCaptureAPI is a cWebObject
 
     Procedure Construct_Object
@@ -10,7 +36,8 @@ Class cMediaStreamImageCaptureAPI is a cWebObject
         Property Boolean pbIsSupported False
         { WebProperty=Client Visibility=Private }
         Property String psMediaStream
-        { WebProperty=Client Description="Set to True to include width and height in OnPhoto event" }
+        // Set to True to include width and height in OnPhoto event
+        { WebProperty=Client }
         Property Boolean pbWithDimension False
 
         { WebProperty=Client Category="Server events" }
@@ -47,17 +74,22 @@ Class cMediaStreamImageCaptureAPI is a cWebObject
 
     { Visibility=Private }
     Function SerializeObject Returns tWebObjectDef
-        Handle hoStream
+        Integer iCount i
+        Handle hoChild
         String sMediaStreamName
+        Boolean bFound
         tWebObjectDef tResult
 
-        If (Child_Count(Self) = 2) Begin
-            Get ChildByIndex 1 to hoStream
-            If (IsObjectOfClass(hoStream, RefClass(cMediaStream))) Begin
-                Get WebObjectName of hoStream to sMediaStreamName
+        Get Child_Count to iCount
+        For i from 0 to (iCount - 1)
+            Get ChildByIndex i to hoChild
+            If (IsObjectOfClass(hoChild, RefClass(cMediaStream))) Begin
+                Get WebObjectName of hoChild to sMediaStreamName
                 Set psMediaStream to sMediaStreamName
+                Move True to bFound
             End
-        End
+            If (bFound) Break
+        Loop
 
         Forward Get SerializeObject to tResult
 
@@ -74,9 +106,33 @@ Class cMediaStreamImageCaptureAPI is a cWebObject
         Send ClientAction "connect" aParams
     End_Procedure
 
+    // Change a setting on a video stream
+    // sName should be the name of a property from the Video section of tMediaTrackSettings
+    // Valid settings will be reported under stInfo.track.capabilities in OnConnect event
+    Procedure ApplyConstraint String sName String sValue
+        String[] aParams
+        Move sName to aParams[0]
+        Move sValue to aParams[1]
+        Send ClientAction "applyConstraint" aParams
+    End_Procedure
+    
     // Capture an image from the media stream, triggering OnPhoto with the result
-    Procedure TakePhoto
-        Send ClientAction "takePhoto"
+    // All arguments are optional
+    Procedure TakePhoto Integer eFillLightMode Integer iImageHeight Integer iImageWidth Boolean bRedEyeReduction
+        String[] aParams
+        If (num_arguments > 0) Begin
+            Move eFillLightMode to aParams[0]
+        End
+        If (num_arguments > 1) Begin
+            Move iImageHeight to aParams[1]
+        End
+        If (num_arguments > 2) Begin
+            Move iImageWidth to aParams[2]
+        End
+        If (num_arguments > 3) Begin
+            Move bRedEyeReduction to aParams[3]
+        End
+        Send ClientAction "takePhoto" aParams
     End_Procedure
 
     // Disconnect from the media stream and release any media device resources held
@@ -87,7 +143,7 @@ Class cMediaStreamImageCaptureAPI is a cWebObject
     { Visibility=Private }
     Procedure OnConnectProxy
         Handle hoJson
-        tMediaStreamTrackInfo stInfo
+        tMediaStreamImageCaptureInfo stInfo
 
         Get phoActionJsonData to hoJson
         If (hoJson <> C_WebUnresolvedObject) Begin
@@ -100,7 +156,7 @@ Class cMediaStreamImageCaptureAPI is a cWebObject
     End_Procedure
 
     { MethodType=Event }
-    Procedure OnConnect tMediaStreamTrackInfo stInfo
+    Procedure OnConnect tMediaStreamImageCaptureInfo stInfo
     End_Procedure
 
     { MethodType=Event }

AppSrc/cMediaStreamImageCaptureAPI.pkg

@@ -15,7 +15,7 @@
   "scripts": {
     "watch": "webpack --watch --config webpack.dev.js",
     "build": "webpack --config webpack.prod.js",
-    "dist": "rimraf dist && copyfiles -f \"%ProgramFiles%/DataFlex 24.0/Lib/WebAppMaster/Data/[Ww]eb[Aa]pp[Ss]erver[Pp]rops.*\" Data && npm run build && 7z a dist/WebAPIsDemo.zip * -r -x!.git -x!.github -x!.vscode -x!CssThemes -x!DfEngine -x!*.dep -x!*.prn -x!IdeSrc/* -x!node_modules -x!*.dbg -x!*.exe -x!*.log && 7z a dist/WebAPIsComponents.zip AppHtml/Custom/* AppSrc/*API.pkg LICENSE.txt"
+    "dist": "rimraf dist && copyfiles -f \"%ProgramFiles%/DataFlex 24.0/Lib/WebAppMaster/Data/[Ww]eb[Aa]pp[Ss]erver[Pp]rops.*\" Data && npm run build && 7z a dist/WebAPIsDemo.zip * -r -x!.git -x!.github -x!.vscode -x!CssThemes -x!DfEngine -x!*.dep -x!*.prn -x!IdeSrc/* -x!node_modules -x!temp -x!*.dbg -x!*.exe -x!*.log && 7z a dist/WebAPIsComponents.zip AppHtml/Custom/* AppSrc/*API.pkg LICENSE.txt"
   },
   "devDependencies": {
     "@babel/core": "^7.26.7",