diff --git a/package.json b/package.json index 79e5f1d..c4dd79a 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,6 @@ "typescript": "4.7.4" }, "dependencies": { - "js-tiktoken": "^1.0.10" + "js-tiktoken": "^1.0.12" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0633ee..fd10b6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,8 +6,8 @@ settings: dependencies: js-tiktoken: - specifier: ^1.0.10 - version: 1.0.10 + specifier: ^1.0.12 + version: 1.0.12 devDependencies: '@jest/globals': @@ -2599,8 +2599,8 @@ packages: - ts-node dev: true - /js-tiktoken@1.0.10: - resolution: {integrity: sha512-ZoSxbGjvGyMT13x6ACo9ebhDha/0FHdKA+OsQcMOWcm1Zs7r90Rhk5lhERLzji+3rA7EKpXCgwXcM5fF3DMpdA==} + /js-tiktoken@1.0.12: + resolution: {integrity: sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==} dependencies: base64-js: 1.5.1 dev: false @@ -3457,7 +3457,7 @@ packages: prepare: true requiresBuild: true dependencies: - '@codemirror/state': 6.2.0 + '@codemirror/state': 6.4.1 '@codemirror/view': 6.26.0 '@lezer/common': 1.2.1 '@lezer/highlight': 1.2.0 diff --git a/src/noteGenerator.ts b/src/noteGenerator.ts index d9a0784..f2b3c76 100644 --- a/src/noteGenerator.ts +++ b/src/noteGenerator.ts @@ -125,8 +125,19 @@ export function noteGenerator( const inputLimit = getTokenLimit(settings) let shouldContinue = true + if (!nodeText) { + return shouldContinue + } - if (nodeText) { + if (nodeText.startsWith('data:image')) { + messages.unshift({ + content: [{ + 'type': 'image_url', + 'image_url': { 'url': nodeText } + }], + role: 'user' + }) + } else { if (isSystemPromptNode(nodeText)) return true let nodeTokens = encoding.encode(nodeText) diff --git a/src/obsidian/fileUtil.ts b/src/obsidian/fileUtil.ts index e33351e..67b5e47 100644 --- a/src/obsidian/fileUtil.ts +++ b/src/obsidian/fileUtil.ts @@ -40,10 +40,14 @@ export async function readNodeContent(node: CanvasNode) { case 'file': const file = app.vault.getAbstractFileByPath(nodeData.file) if (file instanceof TFile) { - const body = await app.vault.read(file) + const ext = file.extension if (node.subpath) { return await readFileContent(app, file, nodeData.subpath) + } else if (ext === 'png' || ext === 'jpg' || ext === 'jpeg') { + const fileBuffer = Buffer.from(await app.vault.adapter.readBinary(file.path)) + return `data:image/${ext};base64,${fileBuffer.toString('base64')}` } else { + const body = await app.vault.read(file) return `## ${file.basename}\n${body}` } } else { diff --git a/src/openai/chatGPT-types.ts b/src/openai/chatGPT-types.ts index 921b60c..c359a46 100644 --- a/src/openai/chatGPT-types.ts +++ b/src/openai/chatGPT-types.ts @@ -42,7 +42,7 @@ export namespace openai { * @type {string} * @memberof ChatCompletionRequestMessage */ - content: string + content: string | [{}] /** * The name of the user in a multi-user chat * @type {string} diff --git a/src/openai/chatGPT.ts b/src/openai/chatGPT.ts index cac88dd..44ba0a6 100644 --- a/src/openai/chatGPT.ts +++ b/src/openai/chatGPT.ts @@ -20,6 +20,10 @@ export const CHAT_MODELS = { name: 'gpt-3.5-turbo-1106', tokenLimit: 16385 }, + GPT_4o: { + name: 'gpt-4o', + tokenLimit: 128000 + }, GPT_4: { name: 'gpt-4', tokenLimit: 8192