Merge remote-tracking branch 'origin/master'

package.json

@@ -30,6 +30,6 @@
 		"typescript": "4.7.4"
 	},
 	"dependencies": {
-		"js-tiktoken": "^1.0.10"
+		"js-tiktoken": "^1.0.12"
 	}
 }

src/noteGenerator.ts

@@ -125,8 +125,19 @@ export function noteGenerator(
 			const inputLimit = getTokenLimit(settings)
 
 			let shouldContinue = true
+			if (!nodeText) {
+				return shouldContinue
+			}
 
-			if (nodeText) {
+			if (nodeText.startsWith('data:image')) {
+				messages.unshift({
+					content: [{
+						'type': 'image_url',
+						'image_url': { 'url': nodeText }
+					}],
+					role: 'user'
+				})
+			} else {
 				if (isSystemPromptNode(nodeText)) return true
 
 				let nodeTokens = encoding.encode(nodeText)

src/obsidian/fileUtil.ts

@@ -40,10 +40,14 @@ export async function readNodeContent(node: CanvasNode) {
 		case 'file':
 			const file = app.vault.getAbstractFileByPath(nodeData.file)
 			if (file instanceof TFile) {
-				const body = await app.vault.read(file)
+				const ext = file.extension
 				if (node.subpath) {
 					return await readFileContent(app, file, nodeData.subpath)
+				} else if (ext === 'png' || ext === 'jpg' || ext === 'jpeg') {
+					const fileBuffer = Buffer.from(await app.vault.adapter.readBinary(file.path))
+					return `data:image/${ext};base64,${fileBuffer.toString('base64')}`
 				} else {
+					const body = await app.vault.read(file)
 					return `## ${file.basename}\n${body}`
 				}
 			} else {

src/openai/chatGPT-types.ts

@@ -42,7 +42,7 @@ export namespace openai {
 		 * @type {string}
 		 * @memberof ChatCompletionRequestMessage
 		 */
-		content: string
+		content: string | [{}]
 		/**
 		 * The name of the user in a multi-user chat
 		 * @type {string}

src/openai/chatGPT.ts

@@ -20,6 +20,10 @@ export const CHAT_MODELS = {
 		name: 'gpt-3.5-turbo-1106',
 		tokenLimit: 16385
 	},
+	GPT_4o: {
+		name: 'gpt-4o',
+		tokenLimit: 128000
+	},
 	GPT_4: {
 		name: 'gpt-4',
 		tokenLimit: 8192