playground flow

src/constants.ts

@@ -35,6 +35,7 @@ export const ApiPath = {
   scene: 'scene',
   scenes: 'scenes',
   timeline: 'timeline',
+  frame: 'frame',
 } as const;
 
 export const ResponseStatus = {

src/core/image.ts

@@ -1,9 +1,7 @@
 import { ApiPath } from '@/constants';
-import type { ImageBase, IImage } from '@/interfaces/core';
+import type { FrameBase, ImageBase, IImage } from '@/interfaces/core';
 import { HttpClient } from '@/utils/httpClient';
 
-const { image } = ApiPath;
-
 /**
  * The base Image class
  * @remarks
@@ -30,8 +28,59 @@ export class Image implements IImage {
    */
   public delete = async () => {
     return await this.#vhttp.delete<Record<string, never>>([
-      image,
+      ApiPath.image,
       this.meta.id,
     ]);
   };
-}
+}
+
+export class Frame extends Image {
+  public videoId: string;
+  public sceneId: string;
+  public frameTime: number;
+  public description: string;
+  #vhttp: HttpClient;
+
+  constructor(http: HttpClient, data: FrameBase) {
+    super(http, {
+      id: data.id,
+      collectionId: undefined,
+      url: data.url,
+    });
+
+    this.videoId = data.videoId;
+    this.sceneId = data.sceneId;
+    this.frameTime = data.frameTime;
+    this.description = data.description;
+    this.#vhttp = http;
+  }
+
+  public getRequestData(): object {
+    return {
+      id: this.meta.id,
+      videoId: this.videoId,
+      sceneId: this.sceneId,
+      url: this.meta.url || '',
+      frameTime: this.frameTime,
+      description: this.description,
+    };
+  }
+
+  public async describe(prompt?: string, modelName?: string): Promise<string> {
+    const response = await this.#vhttp.post<{ description: string }, object>(
+      [
+        ApiPath.video,
+        this.videoId,
+        ApiPath.frame,
+        this.meta.id,
+        ApiPath.describe,
+      ],
+      {
+        prompt,
+        model_name: modelName,
+      }
+    );
+    this.description = response.data.description;
+    return this.description;
+  }
+}

src/core/scene.ts

@@ -1,11 +1,89 @@
-export class Scene {
-  response: string;
+import { HttpClient } from '@/utils/httpClient';
+import { ApiPath } from '@/constants';
+import type { Frame } from './image';
+
+export interface SceneBase {
+  id: string;
+  videoId: string;
   start: number;
   end: number;
+  description?: string;
+  frames: Frame[];
+}
+
+export interface SceneCollectionBase {
+  id: string;
+  videoId: string;
+  config: object;
+  scenes: Scene[];
+}
+
+export class Scene {
+  public id: string;
+  public videoId: string;
+  public start: number;
+  public end: number;
+  public frames: Frame[];
+  public description: string | undefined;
+  #vhttp: HttpClient;
+
+  constructor(http: HttpClient, data: SceneBase) {
+    this.id = data.id;
+    this.videoId = data.videoId;
+    this.start = data.start;
+    this.end = data.end;
+    this.frames = data.frames || [];
+    this.description = data?.description;
+    this.#vhttp = http;
+  }
 
-  constructor(response: string, start: number, end: number) {
-    this.response = response;
-    this.start = start;
-    this.end = end;
+  public async describe(prompt?: string, modelName?: string): Promise<string> {
+    const response = await this.#vhttp.post<{ description: string }, object>(
+      [ApiPath.video, this.videoId, ApiPath.scene, this.id, ApiPath.describe],
+      {
+        data: {
+          prompt,
+          model_name: modelName,
+        },
+      }
+    );
+    this.description = response.data.description;
+    return this.description;
   }
-}
+
+  public getRequestData(): object {
+    return {
+      id: this.id,
+      videoId: this.videoId,
+      start: this.start,
+      end: this.end,
+      frames: this.frames.map(frame => frame.getRequestData()),
+      description: this.description,
+    };
+  }
+}
+
+export class SceneCollection {
+  public id: string;
+  public videoId: string;
+  public config: object;
+  public scenes: Scene[];
+  #vhttp: HttpClient;
+
+  constructor(http: HttpClient, data: SceneCollectionBase) {
+    this.#vhttp = http;
+    this.id = data.id;
+    this.videoId = data.videoId;
+    this.config = data.config;
+    this.scenes = data.scenes;
+  }
+
+  public delete = async () => {
+    return await this.#vhttp.delete<Record<string, never>>([
+      ApiPath.video,
+      this.videoId,
+      ApiPath.scenes,
+      this.id,
+    ]);
+  };
+}

src/core/video.ts

@@ -1,6 +1,8 @@
 import { fromSnakeToCamel } from './../utils/index';
 import { ApiPath, Workflows } from '@/constants';
 import type { IVideo, VideoBase } from '@/interfaces/core';
+import { Frame } from '@/core/image';
+import { Scene, SceneCollection } from '@/core/scene';
 import {
   ListSceneIndex,
   IndexScenesResponse,
@@ -15,9 +17,18 @@ import {
   IndexTypeValues,
   SceneExtractionType,
 } from '@/core/config';
-import { IndexJob, TranscriptJob, SceneIndexJob } from '@/utils/job';
+import {
+  IndexJob,
+  TranscriptJob,
+  SceneIndexJob,
+  ExtractScenesJob,
+} from '@/utils/job';
 import { SearchFactory } from './search';
-import { IndexSceneConfig, SubtitleStyleProps } from '@/types/config';
+import {
+  ExtractSceneConfig,
+  IndexSceneConfig,
+  SubtitleStyleProps,
+} from '@/types/config';
 import { SearchType, IndexType } from '@/types/search';
 import { SceneIndexRecords, SceneIndexes } from '@/types';
 
@@ -149,6 +160,73 @@ export class Video implements IVideo {
     return indexJob;
   };
 
+  public _formatSceneCollectionData = (
+    sceneCollectionData: any
+  ): SceneCollection => {
+    const scenes: Scene[] = [];
+
+    for (const sceneData of sceneCollectionData.scenes) {
+      const frames: Frame[] = [];
+      for (const frameData of sceneData.frames) {
+        frames.push(
+          new Frame(this.#vhttp, {
+            id: frameData.frameId,
+            videoId: this.meta.id,
+            sceneId: sceneData.sceneId,
+            url: frameData.url,
+            frameTime: frameData.frameTime,
+            description: frameData.description,
+          })
+        );
+      }
+      scenes.push(
+        new Scene(this.#vhttp, {
+          id: sceneData.sceneId,
+          videoId: this.meta.id,
+          start: sceneData.start,
+          end: sceneData.end,
+          frames: frames,
+        })
+      );
+    }
+    return new SceneCollection(this.#vhttp, {
+      id: sceneCollectionData.sceneCollectionId,
+      videoId: this.meta.id,
+      scenes: scenes,
+      config: sceneCollectionData.config,
+    });
+  };
+
+  public extractScenes = async (config: Partial<ExtractSceneConfig> = {}) => {
+    const defaultConfig = {
+      extractionType: SceneExtractionType.shotBased,
+      extractionConfig: {},
+      force: false,
+    };
+    const extractScenePayload = fromCamelToSnake(
+      Object.assign({}, defaultConfig, config)
+    );
+    const extractScenesJob = new ExtractScenesJob(
+      this.#vhttp,
+      this.meta.id,
+      extractScenePayload
+    );
+    return new Promise<SceneCollection>((resolve, reject) => {
+      extractScenesJob.on('success', data => {
+        resolve(this._formatSceneCollectionData(data));
+      });
+      extractScenesJob.on('error', err => {
+        reject(err);
+      });
+      extractScenesJob
+        .start()
+        .then(() => {})
+        .catch(err => {
+          reject(err);
+        });
+    });
+  };
+
   /**
    * Indexs the video with scenes
    * @returns an awaited boolean signifying whether the process
@@ -159,9 +237,14 @@ export class Video implements IVideo {
       extractionType: SceneExtractionType.shotBased,
       extractionConfig: {},
     };
+    if (config.scenes) {
+      //@ts-ignore
+      config.scenes = config.scenes.map(scene => scene.getRequestData());
+    }
     const indexScenesPayload = fromCamelToSnake(
       Object.assign({}, defaultConfig, config)
     );
+    console.log('this is payload', indexScenesPayload);
     const res = await this.#vhttp.post<IndexScenesResponse, object>(
       [video, this.meta.id, index, scene],
       indexScenesPayload

src/interfaces/core.ts

@@ -1,6 +1,6 @@
 import { SearchResult } from '@/core/search/searchResult';
 import { Video } from '@/core/video';
-import type { SearchType, IndexType } from '@/types/search';
+import type { SearchType } from '@/types/search';
 import type { FileUploadConfig, URLUploadConfig } from '@/types/collection';
 import type { StreamableURL, Timeline, Transcript } from '@/types/video';
 import { IndexJob, TranscriptJob, UploadJob } from '@/utils/job';
@@ -82,9 +82,19 @@ export interface IAudio {
  * Base type for all Image objects
  */
 export interface ImageBase {
-  collectionId: string;
+  collectionId?: string;
   id: string;
-  name: string;
+  name?: string;
+  url?: string;
+}
+
+export interface FrameBase {
+  id: string;
+  videoId: string;
+  sceneId: string;
+  frameTime: number;
+  description: string;
+  url: string;
 }
 
 /**

src/types/config.ts

@@ -1,5 +1,6 @@
 import { IndexType } from '@/types/search';
 import { SubtitleBorderStyle, SubtitleAlignment } from '@/core/config';
+import { Scene } from '@/core/scene';
 
 export type SubtitleStyleProps = {
   fontName: string;
@@ -94,6 +95,7 @@ export type IndexSceneConfig = {
   modelConfig?: object;
   prompt?: string | null;
   callbackUrl?: string | null;
+  scenes?: Scene[];
 };
 
 export type IndexConfig = {