From 60f74e109f8db73c1a33de101844328bf88fa444 Mon Sep 17 00:00:00 2001 From: 0xrohitgarg Date: Thu, 23 Jan 2025 10:50:12 +0530 Subject: [PATCH] playground flow --- src/constants.ts | 1 + src/core/image.ts | 59 ++++++++++++++++++++++++--- src/core/scene.ts | 92 ++++++++++++++++++++++++++++++++++++++---- src/core/video.ts | 87 ++++++++++++++++++++++++++++++++++++++- src/interfaces/core.ts | 16 ++++++-- src/types/config.ts | 2 + src/utils/job.ts | 51 ++++++++++++++++++++++- 7 files changed, 290 insertions(+), 18 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index ab54d08..a54fed9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -35,6 +35,7 @@ export const ApiPath = { scene: 'scene', scenes: 'scenes', timeline: 'timeline', + frame: 'frame', } as const; export const ResponseStatus = { diff --git a/src/core/image.ts b/src/core/image.ts index 7cfe32d..5bab378 100644 --- a/src/core/image.ts +++ b/src/core/image.ts @@ -1,9 +1,7 @@ import { ApiPath } from '@/constants'; -import type { ImageBase, IImage } from '@/interfaces/core'; +import type { FrameBase, ImageBase, IImage } from '@/interfaces/core'; import { HttpClient } from '@/utils/httpClient'; -const { image } = ApiPath; - /** * The base Image class * @remarks @@ -30,8 +28,59 @@ export class Image implements IImage { */ public delete = async () => { return await this.#vhttp.delete>([ - image, + ApiPath.image, this.meta.id, ]); }; -} \ No newline at end of file +} + +export class Frame extends Image { + public videoId: string; + public sceneId: string; + public frameTime: number; + public description: string; + #vhttp: HttpClient; + + constructor(http: HttpClient, data: FrameBase) { + super(http, { + id: data.id, + collectionId: undefined, + url: data.url, + }); + + this.videoId = data.videoId; + this.sceneId = data.sceneId; + this.frameTime = data.frameTime; + this.description = data.description; + this.#vhttp = http; + } + + public getRequestData(): object { + return { + id: this.meta.id, + videoId: this.videoId, + sceneId: this.sceneId, + url: this.meta.url || '', + frameTime: this.frameTime, + description: this.description, + }; + } + + public async describe(prompt?: string, modelName?: string): Promise { + const response = await this.#vhttp.post<{ description: string }, object>( + [ + ApiPath.video, + this.videoId, + ApiPath.frame, + this.meta.id, + ApiPath.describe, + ], + { + prompt, + model_name: modelName, + } + ); + this.description = response.data.description; + return this.description; + } +} diff --git a/src/core/scene.ts b/src/core/scene.ts index 54c8747..48b4839 100644 --- a/src/core/scene.ts +++ b/src/core/scene.ts @@ -1,11 +1,89 @@ -export class Scene { - response: string; +import { HttpClient } from '@/utils/httpClient'; +import { ApiPath } from '@/constants'; +import type { Frame } from './image'; + +export interface SceneBase { + id: string; + videoId: string; start: number; end: number; + description?: string; + frames: Frame[]; +} + +export interface SceneCollectionBase { + id: string; + videoId: string; + config: object; + scenes: Scene[]; +} + +export class Scene { + public id: string; + public videoId: string; + public start: number; + public end: number; + public frames: Frame[]; + public description: string | undefined; + #vhttp: HttpClient; + + constructor(http: HttpClient, data: SceneBase) { + this.id = data.id; + this.videoId = data.videoId; + this.start = data.start; + this.end = data.end; + this.frames = data.frames || []; + this.description = data?.description; + this.#vhttp = http; + } - constructor(response: string, start: number, end: number) { - this.response = response; - this.start = start; - this.end = end; + public async describe(prompt?: string, modelName?: string): Promise { + const response = await this.#vhttp.post<{ description: string }, object>( + [ApiPath.video, this.videoId, ApiPath.scene, this.id, ApiPath.describe], + { + data: { + prompt, + model_name: modelName, + }, + } + ); + this.description = response.data.description; + return this.description; } -} \ No newline at end of file + + public getRequestData(): object { + return { + id: this.id, + videoId: this.videoId, + start: this.start, + end: this.end, + frames: this.frames.map(frame => frame.getRequestData()), + description: this.description, + }; + } +} + +export class SceneCollection { + public id: string; + public videoId: string; + public config: object; + public scenes: Scene[]; + #vhttp: HttpClient; + + constructor(http: HttpClient, data: SceneCollectionBase) { + this.#vhttp = http; + this.id = data.id; + this.videoId = data.videoId; + this.config = data.config; + this.scenes = data.scenes; + } + + public delete = async () => { + return await this.#vhttp.delete>([ + ApiPath.video, + this.videoId, + ApiPath.scenes, + this.id, + ]); + }; +} diff --git a/src/core/video.ts b/src/core/video.ts index 1fdd227..8865cb4 100644 --- a/src/core/video.ts +++ b/src/core/video.ts @@ -1,6 +1,8 @@ import { fromSnakeToCamel } from './../utils/index'; import { ApiPath, Workflows } from '@/constants'; import type { IVideo, VideoBase } from '@/interfaces/core'; +import { Frame } from '@/core/image'; +import { Scene, SceneCollection } from '@/core/scene'; import { ListSceneIndex, IndexScenesResponse, @@ -15,9 +17,18 @@ import { IndexTypeValues, SceneExtractionType, } from '@/core/config'; -import { IndexJob, TranscriptJob, SceneIndexJob } from '@/utils/job'; +import { + IndexJob, + TranscriptJob, + SceneIndexJob, + ExtractScenesJob, +} from '@/utils/job'; import { SearchFactory } from './search'; -import { IndexSceneConfig, SubtitleStyleProps } from '@/types/config'; +import { + ExtractSceneConfig, + IndexSceneConfig, + SubtitleStyleProps, +} from '@/types/config'; import { SearchType, IndexType } from '@/types/search'; import { SceneIndexRecords, SceneIndexes } from '@/types'; @@ -149,6 +160,73 @@ export class Video implements IVideo { return indexJob; }; + public _formatSceneCollectionData = ( + sceneCollectionData: any + ): SceneCollection => { + const scenes: Scene[] = []; + + for (const sceneData of sceneCollectionData.scenes) { + const frames: Frame[] = []; + for (const frameData of sceneData.frames) { + frames.push( + new Frame(this.#vhttp, { + id: frameData.frameId, + videoId: this.meta.id, + sceneId: sceneData.sceneId, + url: frameData.url, + frameTime: frameData.frameTime, + description: frameData.description, + }) + ); + } + scenes.push( + new Scene(this.#vhttp, { + id: sceneData.sceneId, + videoId: this.meta.id, + start: sceneData.start, + end: sceneData.end, + frames: frames, + }) + ); + } + return new SceneCollection(this.#vhttp, { + id: sceneCollectionData.sceneCollectionId, + videoId: this.meta.id, + scenes: scenes, + config: sceneCollectionData.config, + }); + }; + + public extractScenes = async (config: Partial = {}) => { + const defaultConfig = { + extractionType: SceneExtractionType.shotBased, + extractionConfig: {}, + force: false, + }; + const extractScenePayload = fromCamelToSnake( + Object.assign({}, defaultConfig, config) + ); + const extractScenesJob = new ExtractScenesJob( + this.#vhttp, + this.meta.id, + extractScenePayload + ); + return new Promise((resolve, reject) => { + extractScenesJob.on('success', data => { + resolve(this._formatSceneCollectionData(data)); + }); + extractScenesJob.on('error', err => { + reject(err); + }); + extractScenesJob + .start() + .then(() => {}) + .catch(err => { + reject(err); + }); + }); + }; + /** * Indexs the video with scenes * @returns an awaited boolean signifying whether the process @@ -159,9 +237,14 @@ export class Video implements IVideo { extractionType: SceneExtractionType.shotBased, extractionConfig: {}, }; + if (config.scenes) { + //@ts-ignore + config.scenes = config.scenes.map(scene => scene.getRequestData()); + } const indexScenesPayload = fromCamelToSnake( Object.assign({}, defaultConfig, config) ); + console.log('this is payload', indexScenesPayload); const res = await this.#vhttp.post( [video, this.meta.id, index, scene], indexScenesPayload diff --git a/src/interfaces/core.ts b/src/interfaces/core.ts index b95f60f..ecd93fc 100644 --- a/src/interfaces/core.ts +++ b/src/interfaces/core.ts @@ -1,6 +1,6 @@ import { SearchResult } from '@/core/search/searchResult'; import { Video } from '@/core/video'; -import type { SearchType, IndexType } from '@/types/search'; +import type { SearchType } from '@/types/search'; import type { FileUploadConfig, URLUploadConfig } from '@/types/collection'; import type { StreamableURL, Timeline, Transcript } from '@/types/video'; import { IndexJob, TranscriptJob, UploadJob } from '@/utils/job'; @@ -82,9 +82,19 @@ export interface IAudio { * Base type for all Image objects */ export interface ImageBase { - collectionId: string; + collectionId?: string; id: string; - name: string; + name?: string; + url?: string; +} + +export interface FrameBase { + id: string; + videoId: string; + sceneId: string; + frameTime: number; + description: string; + url: string; } /** diff --git a/src/types/config.ts b/src/types/config.ts index cd0ed07..945a755 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1,5 +1,6 @@ import { IndexType } from '@/types/search'; import { SubtitleBorderStyle, SubtitleAlignment } from '@/core/config'; +import { Scene } from '@/core/scene'; export type SubtitleStyleProps = { fontName: string; @@ -94,6 +95,7 @@ export type IndexSceneConfig = { modelConfig?: object; prompt?: string | null; callbackUrl?: string | null; + scenes?: Scene[]; }; export type IndexConfig = { diff --git a/src/utils/job.ts b/src/utils/job.ts index 4ebe327..72ab9af 100644 --- a/src/utils/job.ts +++ b/src/utils/job.ts @@ -5,7 +5,7 @@ import { Image } from '@/core/image'; import type { AudioBase, ImageBase, VideoBase } from '@/interfaces/core'; import type { SyncUploadConfig } from '@/types/collection'; import type { MediaBase, SceneIndexRecords } from '@/types/index'; -import type { IndexConfig } from '@/types/config'; +import type { ExtractSceneConfig, IndexConfig } from '@/types/config'; import type { IndexType } from '@/types/search'; import type { NoDataResponse, @@ -357,3 +357,52 @@ export class SceneIndexJob extends Job< return data.sceneIndexRecords; }; } + +export class ExtractScenesJob extends Job { + videoId: string; + config: Partial; + constructor( + http: HttpClient, + videoId: string, + config: Partial + ) { + super(http); + this.videoId = videoId; + this.config = config; + this.jobTitle = 'Extract Scenes Job'; + } + + /** + * Fetches the callbackURL from the server and initiates a backoff + */ + public start = async () => { + try { + const res = await this.vhttp.post< + SyncJobResponse, + Partial + >([video, this.videoId, scenes], this.config); + if (res.status === processing) { + void this._initiateBackoff(res.data.output_url); + } else { + // @ts-ignore + this._handleSuccess(res.data); + } + } catch (err) { + this._handleError(err); + } + }; + + /** + * Initializes a new video object with the returned data + * @param data - Media data returned from the API and converted to camelCase + * @returns a new Video object + */ + //@ts-ignore + protected beforeSuccess = (data: object) => { + //@ts-ignore + if (data?.sceneCollection) { + //@ts-ignore + return data.sceneCollection; + } + }; +}