Detect Face Landmarks

- Detect if the face is looking straight
- Refactor to components in videoCall
- Move components to components custom

src/app/(routes)/interview/video-call/videoCall.tsx

@@ -1,10 +1,12 @@
 import React, { useEffect, useRef, useState } from 'react'
-import { Button } from '@/components/ui/button'
-import { RefreshCcw, SquareIcon } from 'lucide-react'
 import { Configuration, NewSessionData, StreamingAvatarApi } from '@heygen/streaming-avatar'
 import * as faceapi from 'face-api.js'
 import { createFaceRepositoryAdapter } from '@/modules/face_proccesor/infrastructure/adapter/faceRepositoryAdapter'
 import { createFaceService } from '@/modules/face_proccesor/application/service/faceService'
+import UserCamera from '@/components/custom/interview/video-call/userCamera'
+import { isLookingStraight } from '@/lib/lookingUtils'
+import AvatarVideo from '@/components/custom/interview/video-call/avatarVideo'
+import { EndInterviewButton } from '@/components/custom/interview/video-call/endInterviewButtonProps'
 
 interface VideoCallProps {
   selectedAvatar: any;
@@ -17,10 +19,11 @@ interface VideoCallProps {
 
 type FaceDataState = {
   expressions: faceapi.FaceExpressions;
+  landmarks: faceapi.FaceLandmarks68;
 }
 
 const faceRepository = createFaceRepositoryAdapter()
-const faceService = createFaceService(faceRepository)
+const faceService    = createFaceService(faceRepository)
 
 export const VideoCall: React.FC<VideoCallProps> = ({
   selectedAvatar,
@@ -38,59 +41,60 @@ export const VideoCall: React.FC<VideoCallProps> = ({
   const [sessionData, setSessionData] = useState<NewSessionData | null>(null);
   const [faceData, setFaceData] = useState<FaceDataState[]>([]);
   const avatarApiRef = useRef<StreamingAvatarApi | null>(null);
+  const [isLookingToCamera, setIsLookingToCamera] = useState<boolean>(false);
 
   useEffect(() => {
-    initializeStreams();
+    initializeStreams()
     return () => {
-      stopStreams();
-    };
-  }, []);
+      stopStreams()
+    }
+  }, [])
 
   const initializeStreams = async () => {
     try {
       const stream = await navigator.mediaDevices.getUserMedia({
         video: { deviceId: { exact: cameraId } },
         audio: { deviceId: { exact: microphoneId } }
-      });
-      setUserStream(stream);
+      })
+      setUserStream(stream)
 
       avatarApiRef.current = new StreamingAvatarApi(
         new Configuration({
-          accessToken: streamingToken,
+          accessToken: streamingToken
         })
-      );
+      )
 
       const res = await avatarApiRef.current.createStartAvatar({
         newSessionRequest: {
           quality: 'high',
           avatarName: selectedAvatar.avatar_id,
-          voice: { voiceId: '077ab11b14f04ce0b49b5f6e5cc20979' },
-        },
-      });
+          voice: { voiceId: '077ab11b14f04ce0b49b5f6e5cc20979' }
+        }
+      })
 
-      setSessionData(res);
-      setAvatarStream(avatarApiRef.current.mediaStream);
+      setSessionData(res)
+      setAvatarStream(avatarApiRef.current.mediaStream)
     } catch (err) {
-      console.error('Error initializing streams:', err);
-      setCameraError('Failed to start video call. Please try again.');
+      console.error('Error initializing streams:', err)
+      setCameraError('Failed to start video call. Please try again.')
     }
-  };
+  }
 
   const stopStreams = () => {
     if (userStream) {
-      userStream.getTracks().forEach(track => track.stop());
+      userStream.getTracks().forEach(track => track.stop())
     }
     if (avatarApiRef.current && sessionData) {
       avatarApiRef.current.stopAvatar({
-        stopSessionRequest: { sessionId: sessionData.sessionId },
-      });
+        stopSessionRequest: { sessionId: sessionData.sessionId }
+      })
     }
-  };
+  }
 
   const retryCamera = () => {
-    setCameraError(null);
-    initializeStreams();
-  };
+    setCameraError(null)
+    initializeStreams()
+  }
 
   const loadModels = () => {
     Promise.all([
@@ -99,95 +103,59 @@ export const VideoCall: React.FC<VideoCallProps> = ({
       faceapi.nets.faceRecognitionNet.loadFromUri('/weights'),
       faceapi.nets.faceExpressionNet.loadFromUri('/weights')
     ]).then(() => {
-      detectFace()
-    });
+      detectFaceAndSaveFaceData()
+    })
   }
 
-  const detectFace = () => {
+  const detectFaceAndSaveFaceData = () => {
     setInterval(async () => {
       if (userVideoRef.current && isAllowSave) {
         const detections = await faceapi.detectAllFaces(userVideoRef.current,
           new faceapi.TinyFaceDetectorOptions())
-          .withFaceExpressions();
+          .withFaceLandmarks()
+          .withFaceExpressions()
         if (detections.length > 0) {
           const newFaceData = detections
-            .map(detection => ({
-              expressions: detection.expressions
-            }))
-          setFaceData((prevState) => [...prevState, ...newFaceData]);
+            .map(({ expressions, landmarks }) => {
+              setIsLookingToCamera(isLookingStraight(landmarks));
+              return {
+                expressions: expressions,
+                landmarks: landmarks
+              }
+            })
+          setFaceData((prevState) => [...prevState, ...newFaceData])
         }
       }
-    }, 300);
+    }, 300)
   }
 
   useEffect(() => {
     if (userStream && userVideoRef.current) {
-      userVideoRef.current.srcObject = userStream;
+      userVideoRef.current.srcObject = userStream
+      loadModels()
     }
-  }, [userStream]);
+  }, [userStream])
 
   useEffect(() => {
     if (avatarStream && avatarVideoRef.current) {
-      avatarVideoRef.current.srcObject = avatarStream;
-      loadModels();
+      avatarVideoRef.current.srcObject = avatarStream
     }
-  }, [avatarStream]);
+  }, [avatarStream])
 
   const handleEndInterview = async () => {
-    stopStreams();
-    onEndInterview();
+    stopStreams()
+    onEndInterview()
     if (isAllowSave) {
-      await faceService.save(faceData);
+      await faceService.save(faceData)
     }
-  };
+  }
 
   return (
     <div className="flex-grow relative bg-gray-200 rounded-lg overflow-hidden">
-      <div className="absolute top-4 left-4 z-10 md:w-1/4 w-1/3">
-        {cameraError ? (
-          <div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert">
-            <strong className="font-bold">Camera Error:</strong>
-            <span className="block sm:inline"> {cameraError}</span>
-            <button
-              className="absolute top-0 right-0 px-4 py-3"
-              onClick={retryCamera}
-            >
-              <RefreshCcw className="h-5 w-5 text-red-500"/>
-            </button>
-          </div>
-        ) : (
-          <video
-            ref={userVideoRef}
-            className="w-full rounded-lg"
-            autoPlay
-            playsInline
-            muted
-          />
-        )}
-      </div>
-      <div className="w-full h-full flex items-center justify-center">
-        {avatarStream ? (
-          <video
-            ref={avatarVideoRef}
-            className="w-full h-full object-cover"
-            autoPlay
-            playsInline
-          >
-            Your browser does not support the video tag.
-          </video>
-        ) : (
-          <p>Waiting for avatar stream...</p>
-        )}
-      </div>
-      <div className="absolute bottom-4 right-4 space-x-2">
-        <Button
-          className="bg-black text-white hover:bg-gray-800"
-          onClick={handleEndInterview}
-        >
-          <SquareIcon className="size-4 mr-2"/>
-          End Interview
-        </Button>
-      </div>
+      <UserCamera cameraError={cameraError} onClick={retryCamera} ref={userVideoRef}
+                  isLookingToCamera={isLookingToCamera} />
+      <AvatarVideo avatarStream={avatarStream} ref={avatarVideoRef} />
+      <EndInterviewButton onClick={handleEndInterview} />
     </div>
-  );
-};
+  )
+}

src/app/layout.tsx

@@ -1,7 +1,7 @@
 import type { Metadata } from "next";
 import { Inter } from "next/font/google";
 import "/styles/vendor/globals.css"
-import SessionProviderWrapper from '@/app/auth/components/sessionPrividerWrapper';
+import SessionProviderWrapper from '@/components/custom/auth/sessionPrividerWrapper';
 
 const inter = Inter({ subsets: ["latin"] });
 

src/app/page.tsx

@@ -1,7 +1,7 @@
 import { getServerSession } from 'next-auth';
 import { authOptions } from '@/app/api/auth/[...nextauth]/route';
-import Login from '@/app/auth/components/login';
-import Logout from '@/app/auth/components/logout';
+import Login from '@/components/custom/auth/login';
+import Logout from '@/components/custom/auth/logout';
 import Link from 'next/link';
 
 export default async function Home() {

src/components/custom/interview/video-call/avatarVideo.tsx

@@ -0,0 +1,26 @@
+import { forwardRef } from 'react'
+
+type AvatarVideoProps = {
+  avatarStream: MediaStream | null;
+}
+
+const AvatarVideo = forwardRef<HTMLVideoElement, AvatarVideoProps>((props, ref) => {
+  return <div className="w-full h-full flex items-center justify-center">
+    {props.avatarStream ? (
+      <video
+        ref={ref}
+        className="w-full h-full object-cover"
+        autoPlay
+        playsInline
+      >
+        Your browser does not support the video tag.
+      </video>
+    ) : (
+      <p>Waiting for avatar stream...</p>
+    )}
+  </div>
+})
+
+AvatarVideo.displayName = 'AvatarVideo'
+
+export default AvatarVideo

src/components/custom/interview/video-call/endInterviewButtonProps.tsx

@@ -0,0 +1,18 @@
+import { Button } from '@/components/ui/button'
+import { SquareIcon } from 'lucide-react'
+
+type EndInterviewButtonProps = {
+  onClick: () => Promise<void>;
+}
+
+export const EndInterviewButton = (props: EndInterviewButtonProps) => {
+  return <div className="absolute bottom-4 right-4 space-x-2">
+    <Button
+      className="bg-black text-white hover:bg-gray-800"
+      onClick={props.onClick}
+    >
+      <SquareIcon className="size-4 mr-2" />
+      End Interview
+    </Button>
+  </div>
+}

src/components/custom/interview/video-call/userCamera.tsx

@@ -0,0 +1,39 @@
+import { forwardRef } from 'react'
+import { RefreshCcw } from 'lucide-react'
+
+type UserCameraProps = {
+  cameraError: string | null;
+  onClick: () => void;
+  isLookingToCamera: boolean;
+}
+
+const UserCamera = forwardRef<HTMLVideoElement, UserCameraProps>((props, ref) => {
+  return <div className="absolute top-4 left-4 z-10 md:w-1/4 w-1/3">
+    {props.cameraError ? (
+      <div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert">
+        <strong className="font-bold">Camera Error:</strong>
+        <span className="block sm:inline"> {props.cameraError}</span>
+        <button
+          className="absolute top-0 right-0 px-4 py-3"
+          onClick={props.onClick}
+        >
+          <RefreshCcw className="h-5 w-5 text-red-500" />
+        </button>
+      </div>
+    ) : (
+      <div className={props.isLookingToCamera ? '' : 'border-2 border-red-500'}>
+        <video
+          ref={ref}
+          className="w-full rounded-lg"
+          autoPlay
+          playsInline
+          muted
+        />
+      </div>
+    )}
+  </div>
+});
+
+UserCamera.displayName = 'UserCamera'
+
+export default UserCamera;

src/lib/lookingUtils.ts

@@ -0,0 +1,24 @@
+import * as faceapi from 'face-api.js'
+
+export const isLookingStraight = (landmarks: faceapi.FaceLandmarks68) => {
+  const leftEye  = landmarks.getLeftEye()
+  const rightEye = landmarks.getRightEye()
+  const nose     = landmarks.getNose()
+
+  const eyeYDifference = Math.abs(leftEye[0].y - rightEye[0].y)
+
+  const averageEyeX = (leftEye[0].x + rightEye[0].x) / 2
+
+  const noseXDeviation = Math.abs(nose[0].x - averageEyeX)
+
+  const eyeYThreshold  = 20
+  const noseXThreshold = 35
+
+  const eyesAligned  = eyeYDifference < eyeYThreshold
+  const noseCentered = noseXDeviation < noseXThreshold
+
+  console.log(eyesAligned);
+  console.log(noseCentered);
+
+  return eyesAligned && noseCentered
+}