Skip to content

Commit cea40b3

Browse files
P3il4P3il4
authored
support video inputs in .chat (#2740)
* support video inputs in .chat * update docs for .chat
1 parent 7fc9a97 commit cea40b3

7 files changed

Lines changed: 82 additions & 27 deletions

File tree

‎src/backend/src/services/ai/utils/OpenAIUtil.js‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ export const process_input_messages = async (messages) => {
1313
const content = msg.content;
1414

1515
for ( const o of content ) {
16-
if ( ! o['image_url'] ) continue;
17-
if ( o.type ) continue;
18-
o.type = 'image_url';
16+
if ( o['image_url'] && !o.type ) {
17+
o.type = 'image_url';
18+
}
19+
if ( o['video_url'] && !o.type ) {
20+
o.type = 'video_url';
21+
}
1922
}
2023

2124
// coerce tool calls
@@ -93,9 +96,12 @@ export const process_input_messages_responses_api = async (messages) => {
9396
const content = msg.content;
9497

9598
for ( const o of content ) {
96-
if ( ! o['image_url'] ) continue;
97-
if ( o.type ) continue;
98-
o.type = 'image_url';
99+
if ( o['image_url'] && !o.type ) {
100+
o.type = 'image_url';
101+
}
102+
if ( o['video_url'] && !o.type ) {
103+
o.type = 'video_url';
104+
}
99105
}
100106

101107
// coerce tool calls

‎src/docs/src/AI/chat.md‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
title: puter.ai.chat()
3-
description: Chat with AI models, analyze images, and perform function calls using 500+ models from OpenAI, Anthropic, Google, and more.
3+
description: Chat with AI models, analyze images and videos, and perform function calls using 500+ models from OpenAI, Anthropic, Google, and more.
44
platforms: [websites, apps, nodejs, workers]
55
---
66

@@ -12,8 +12,8 @@ Given a prompt returns the completion that best matches the prompt.
1212
puter.ai.chat(prompt)
1313
puter.ai.chat(prompt, options = {})
1414
puter.ai.chat(prompt, testMode = false, options = {})
15-
puter.ai.chat(prompt, image, testMode = false, options = {})
16-
puter.ai.chat(prompt, [imageURLArray], testMode = false, options = {})
15+
puter.ai.chat(prompt, media, testMode = false, options = {})
16+
puter.ai.chat(prompt, [mediaURLArray], testMode = false, options = {})
1717
puter.ai.chat([messages], testMode = false, options = {})
1818
```
1919

@@ -39,13 +39,13 @@ An object containing the following properties:
3939

4040
A boolean indicating whether you want to use the test API. Defaults to `false`. This is useful for testing your code without using up API credits.
4141

42-
#### `image` (String | File)
42+
#### `media` (String | File)
4343

44-
A string containing the URL or Puter path of the image, or a `File` object containing the image you want to provide as context for the completion.
44+
A string containing the URL or Puter path of an image or video, or a `File` object containing the media you want to provide as context for the completion.
4545

46-
#### `imageURLArray` (Array)
46+
#### `mediaURLArray` (Array)
4747

48-
An array of strings containing the URLs of images you want to provide as context for the completion.
48+
An array of strings containing the URLs of images or videos you want to provide as context for the completion.
4949

5050
#### `messages` (Array)
5151

@@ -216,6 +216,23 @@ You can find the implementation in our [prompt caching example](/playground/ai-c
216216
</html>
217217
```
218218

219+
<strong class="example-title">Video Analysis</strong>
220+
221+
```html;ai-video-analysis
222+
<html>
223+
<body>
224+
<script src="https://js.puter.com/v2/"></script>
225+
<script>
226+
puter.ai
227+
.chat(`What do you see?`, `https://assets.puter.site/puppy.mp4`, {
228+
model: "reka/reka-edge",
229+
})
230+
.then(puter.print);
231+
</script>
232+
</body>
233+
</html>
234+
```
235+
219236
<strong class="example-title">Stream the response</strong>
220237

221238
```html;ai-chat-stream

‎src/docs/src/examples.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@ const examples = [
5555
slug: 'ai-gpt-vision',
5656
source: '/playground/examples/ai-gpt-vision.html',
5757
},
58+
{
59+
title: 'Video Analysis',
60+
description: 'Analyze videos with AI using Puter.js. Run and modify this video analysis example instantly in your browser.',
61+
slug: 'ai-video-analysis',
62+
source: '/playground/examples/ai-video-analysis.html',
63+
},
5864
{
5965
title: 'Stream the response',
6066
description: 'Stream AI chat responses in real-time with Puter.js. Run and experiment with this streaming example in the playground.',

‎src/docs/src/playground/examples/ai-video-analysis.html‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<html>
2+
<body>
3+
<script src="https://js.puter.com/v2/"></script>
4+
<script>
5+
// Loading ...
6+
puter.print(`Loading...`);
7+
8+
// Video analysis with Reka Edge
9+
puter.ai
10+
.chat(`What do you see?`, `https://assets.puter.site/puppy.mp4`, {
11+
model: "reka/reka-edge",
12+
})
13+
.then(puter.print);
14+
</script>
15+
</body>
16+
</html>

‎src/puter-js/src/lib/utils.js‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,15 @@ function arrayBufferToDataUri (arrayBuffer) {
632632
});
633633
}
634634

635+
const VIDEO_EXTENSIONS = ['mp4', 'webm', 'mov', 'mpeg', 'avi', 'mkv', 'm4v', 'ogv'];
636+
637+
const isVideoInput = (url) => {
638+
if ( typeof url !== 'string' ) return false;
639+
if ( url.startsWith('data:video/') ) return true;
640+
const ext = url.split('?')[0].split('#')[0].split('.').pop()?.toLowerCase();
641+
return VIDEO_EXTENSIONS.includes(ext);
642+
};
643+
635644
export {
636-
arrayBufferToDataUri, blob_to_url, blobToDataUri, driverCall, handle_error, handle_resp, initXhr, make_driver_method, parseResponse, setupXhrEventHandlers, uuidv4,
645+
arrayBufferToDataUri, blob_to_url, blobToDataUri, driverCall, handle_error, handle_resp, initXhr, isVideoInput, make_driver_method, parseResponse, setupXhrEventHandlers, uuidv4,
637646
};

‎src/puter-js/src/modules/AI.js‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -647,36 +647,37 @@ class AI {
647647
requestParams = { messages: [{ content: args[0] }] };
648648
}
649649

650-
// ai.chat(prompt, imageURL/File)
651-
// ai.chat(prompt, imageURL/File, testMode)
650+
// ai.chat(prompt, mediaURL/File)
651+
// ai.chat(prompt, mediaURL/File, testMode)
652652
else if ( typeof args[0] === 'string' && (typeof args[1] === 'string' || args[1] instanceof File) ) {
653-
// if imageURL is a File, transform it to a data URI
653+
// if mediaURL is a File, transform it to a data URI
654654
if ( args[1] instanceof File ) {
655655
args[1] = await utils.blobToDataUri(args[1]);
656656
}
657657

658-
// parse args[1] as an image_url object
658+
const mediaBlock = utils.isVideoInput(args[1])
659+
? { video_url: { url: args[1] } }
660+
: { image_url: { url: args[1] } };
661+
659662
requestParams = {
660663
vision: true,
661664
messages: [
662665
{
663666
content: [
664667
args[0],
665-
{
666-
image_url: {
667-
url: args[1],
668-
},
669-
},
668+
mediaBlock,
670669
],
671670
},
672671
],
673672
};
674673
}
675-
// chat(prompt, [imageURLs])
674+
// chat(prompt, [mediaURLs])
676675
else if ( typeof args[0] === 'string' && Array.isArray(args[1]) ) {
677-
// parse args[1] as an array of image_url objects
678676
for ( let i = 0; i < args[1].length; i++ ) {
679-
args[1][i] = { image_url: { url: args[1][i] } };
677+
const url = args[1][i];
678+
args[1][i] = utils.isVideoInput(url)
679+
? { video_url: { url } }
680+
: { image_url: { url } };
680681
}
681682
requestParams = {
682683
vision: true,

‎src/puter-js/types/modules/ai.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
export type AIMessageContent = string | { image_url?: { url: string } } | Record<string, unknown>;
1+
export type AIMessageContent = string | { image_url?: { url: string } } | { video_url?: { url: string } } | Record<string, unknown>;
22

33
export interface ChatMessage {
44
role?: string;

0 commit comments

Comments
 (0)