From c3b15ba534510545cba80d2ea75ab0a4289819cb Mon Sep 17 00:00:00 2001 From: Dron Hazra Date: Thu, 5 Jan 2023 22:39:27 -0500 Subject: [PATCH 1/5] added cohere embeddings --- package-lock.json | 220 +++++++++++++++++++++++-------- package.json | 2 + pages/[username]/[updateUrl].tsx | 29 +++- utils/requests.ts | 77 ++++++++++- 4 files changed, 266 insertions(+), 62 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5e6ba02..e7fd43b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,10 @@ "@types/mongoose": "^5.10.3", "@types/node": "^14.14.9", "axios": "^0.21.0", + "cohere-ai": "^5.0.2", "date-fns": "^2.16.1", "html-react-parser": "^0.14.2", + "mathjs": "^11.5.0", "mongoose": "^5.10.16", "next": "^12.1.6", "next-auth": "^4.3.4", @@ -46,11 +48,14 @@ } }, "node_modules/@babel/runtime": { - "version": "7.12.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.12.5.tgz", - "integrity": "sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==", + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.7.tgz", + "integrity": "sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ==", "dependencies": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.11" + }, + "engines": { + "node": ">=6.9.0" } }, "node_modules/@csstools/convert-colors": { @@ -103,17 +108,6 @@ "hoist-non-react-statics": "^3.3.1" } }, - "node_modules/@emotion/react/node_modules/@babel/runtime": { - "version": "7.14.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.14.6.tgz", - "integrity": "sha512-/PCB2uJ7oM44tz8YhC4Z/6PeOKXp4K588f+5M3clr1M4zbqztlo0XEfJ2LEzj/FgwfgGcIdl8n7YYjTCI0BYwg==", - "dependencies": { - "regenerator-runtime": "^0.13.4" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@emotion/serialize": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.0.2.tgz", @@ -981,6 +975,11 @@ "typo-js": "*" } }, + "node_modules/cohere-ai": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/cohere-ai/-/cohere-ai-5.0.2.tgz", + "integrity": "sha512-Svt8VC20/GgwCBF2kHYZI3JZkfqEoG6wCbTT6tohNK8x/aBFyMxlBUYEF0gRGXH1055vQpBjj5ewHF8LpnSSOA==" + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -1010,6 +1009,18 @@ "node": ">=0.1.90" } }, + "node_modules/complex.js": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.1.1.tgz", + "integrity": "sha512-8njCHOTtFFLtegk6zQo0kkVX1rngygb/KQI6z1qZxlFI3scluC+LVTCFbrkWjBv4vvLlbQ9t88IPMC6k95VTTg==", + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://www.patreon.com/infusion" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1191,6 +1202,11 @@ "node": ">=0.10.0" } }, + "node_modules/decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" + }, "node_modules/defined": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", @@ -1284,6 +1300,11 @@ "node": ">=6" } }, + "node_modules/escape-latex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==" + }, "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -1366,6 +1387,18 @@ "node": ">=4.0" } }, + "node_modules/fraction.js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", + "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://www.patreon.com/infusion" + } + }, "node_modules/fs-extra": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", @@ -1661,6 +1694,11 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "node_modules/javascript-natural-sort": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", + "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==" + }, "node_modules/jose": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/jose/-/jose-4.8.1.tgz", @@ -1796,6 +1834,28 @@ "node": ">= 8.16.2" } }, + "node_modules/mathjs": { + "version": "11.5.0", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-11.5.0.tgz", + "integrity": "sha512-vJ/+SqWtxjW6/aeDRt8xL3TlOVKqwN15BIyTGVqGbIWuiqgY4SxZ0yLuna82YH9CB757iFP7uJ4m3KvVBX7Qcg==", + "dependencies": { + "@babel/runtime": "^7.20.6", + "complex.js": "^2.1.1", + "decimal.js": "^10.4.3", + "escape-latex": "^1.2.0", + "fraction.js": "^4.2.0", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^4.1.0" + }, + "bin": { + "mathjs": "bin/cli.js" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/memoize-one": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", @@ -2068,17 +2128,6 @@ } } }, - "node_modules/next-auth/node_modules/@babel/runtime": { - "version": "7.17.9", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", - "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "dependencies": { - "regenerator-runtime": "^0.13.4" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/next-response-helpers": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/next-response-helpers/-/next-response-helpers-0.2.0.tgz", @@ -3525,9 +3574,9 @@ "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" }, "node_modules/regenerator-runtime": { - "version": "0.13.7", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==" }, "node_modules/regexp-clone": { "version": "1.0.0", @@ -3649,6 +3698,11 @@ "loose-envify": "^1.1.0" } }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -4096,6 +4150,11 @@ "node": ">=0.8" } }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -4112,6 +4171,14 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" }, + "node_modules/typed-function": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.1.0.tgz", + "integrity": "sha512-DGwUl6cioBW5gw2L+6SMupGwH/kZOqivy17E4nsh1JI9fKF87orMmlQx3KISQPmg3sfnOUGlwVkroosvgddrlg==", + "engines": { + "node": ">= 14" + } + }, "node_modules/typeorm": { "version": "0.2.39", "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.39.tgz", @@ -4507,11 +4574,11 @@ }, "dependencies": { "@babel/runtime": { - "version": "7.12.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.12.5.tgz", - "integrity": "sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==", + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.7.tgz", + "integrity": "sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ==", "requires": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.11" } }, "@csstools/convert-colors": { @@ -4561,16 +4628,6 @@ "@emotion/utils": "^1.0.0", "@emotion/weak-memoize": "^0.2.5", "hoist-non-react-statics": "^3.3.1" - }, - "dependencies": { - "@babel/runtime": { - "version": "7.14.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.14.6.tgz", - "integrity": "sha512-/PCB2uJ7oM44tz8YhC4Z/6PeOKXp4K588f+5M3clr1M4zbqztlo0XEfJ2LEzj/FgwfgGcIdl8n7YYjTCI0BYwg==", - "requires": { - "regenerator-runtime": "^0.13.4" - } - } } }, "@emotion/serialize": { @@ -5212,6 +5269,11 @@ "typo-js": "*" } }, + "cohere-ai": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/cohere-ai/-/cohere-ai-5.0.2.tgz", + "integrity": "sha512-Svt8VC20/GgwCBF2kHYZI3JZkfqEoG6wCbTT6tohNK8x/aBFyMxlBUYEF0gRGXH1055vQpBjj5ewHF8LpnSSOA==" + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -5238,6 +5300,11 @@ "dev": true, "optional": true }, + "complex.js": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.1.1.tgz", + "integrity": "sha512-8njCHOTtFFLtegk6zQo0kkVX1rngygb/KQI6z1qZxlFI3scluC+LVTCFbrkWjBv4vvLlbQ9t88IPMC6k95VTTg==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -5371,6 +5438,11 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" + }, "defined": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", @@ -5446,6 +5518,11 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "escape-latex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -5512,6 +5589,11 @@ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" }, + "fraction.js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", + "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==" + }, "fs-extra": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", @@ -5765,6 +5847,11 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "javascript-natural-sort": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", + "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==" + }, "jose": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/jose/-/jose-4.8.1.tgz", @@ -5878,6 +5965,22 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-1.2.5.tgz", "integrity": "sha512-2AlqgYnVPOc9WDyWu7S5DJaEZsfk6dNh/neatQ3IHUW4QLutM/VPSH9lG7bif+XjFWc9K9XR3QvR+fXuECmfdA==" }, + "mathjs": { + "version": "11.5.0", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-11.5.0.tgz", + "integrity": "sha512-vJ/+SqWtxjW6/aeDRt8xL3TlOVKqwN15BIyTGVqGbIWuiqgY4SxZ0yLuna82YH9CB757iFP7uJ4m3KvVBX7Qcg==", + "requires": { + "@babel/runtime": "^7.20.6", + "complex.js": "^2.1.1", + "decimal.js": "^10.4.3", + "escape-latex": "^1.2.0", + "fraction.js": "^4.2.0", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^4.1.0" + } + }, "memoize-one": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", @@ -6075,16 +6178,6 @@ "preact": "^10.6.3", "preact-render-to-string": "^5.1.19", "uuid": "^8.3.2" - }, - "dependencies": { - "@babel/runtime": { - "version": "7.17.9", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", - "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "requires": { - "regenerator-runtime": "^0.13.4" - } - } } }, "next-response-helpers": { @@ -7260,9 +7353,9 @@ "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" }, "regenerator-runtime": { - "version": "0.13.7", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==" }, "regexp-clone": { "version": "1.0.0", @@ -7350,6 +7443,11 @@ "loose-envify": "^1.1.0" } }, + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -7692,6 +7790,11 @@ "thenify": ">= 3.1.0 < 4" } }, + "tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -7705,6 +7808,11 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" }, + "typed-function": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.1.0.tgz", + "integrity": "sha512-DGwUl6cioBW5gw2L+6SMupGwH/kZOqivy17E4nsh1JI9fKF87orMmlQx3KISQPmg3sfnOUGlwVkroosvgddrlg==" + }, "typeorm": { "version": "0.2.39", "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.2.39.tgz", diff --git a/package.json b/package.json index 6c77924..69f538f 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,10 @@ "@types/mongoose": "^5.10.3", "@types/node": "^14.14.9", "axios": "^0.21.0", + "cohere-ai": "^5.0.2", "date-fns": "^2.16.1", "html-react-parser": "^0.14.2", + "mathjs": "^11.5.0", "mongoose": "^5.10.16", "next": "^12.1.6", "next-auth": "^4.3.4", diff --git a/pages/[username]/[updateUrl].tsx b/pages/[username]/[updateUrl].tsx index 12fc6c0..670ed32 100644 --- a/pages/[username]/[updateUrl].tsx +++ b/pages/[username]/[updateUrl].tsx @@ -1,6 +1,6 @@ import {GetServerSideProps} from "next"; import {getSession} from "next-auth/react"; -import {getCurrUserRequest, getUpdateRequest} from "../../utils/requests"; +import {getCurrUserRequest, getTopThree, getUpdateRequest} from "../../utils/requests"; import {format} from "date-fns"; import {cleanForJSON, dateOnly, fetcher} from "../../utils/utils"; import Link from "next/link"; @@ -22,7 +22,7 @@ import {notificationModel} from "../../models/models"; import {getMentionsAndBodySegments} from "../../components/UpdateCommentItem"; import { DeleteModal } from "../../components/Modal"; -export default function UpdatePage(props: { data: {user: User, updates: (Update & {mentionedUsersArr: User[]})[]}, updateUrl: string, userData: User }) { +export default function UpdatePage(props: { data: {user: User, updates: (Update & {mentionedUsersArr: User[]})[]}, updateUrl: string, userData: User, topThree: Update[] }) { const router = useRouter(); const [data, setData] = useState<{user: User, updates: (Update & {mentionedUsersArr: User[]})[]}>(props.data); const [userData, setUserData] = useState(props.userData); @@ -217,6 +217,24 @@ export default function UpdatePage(props: { data: {user: User, updates: (Update {Parser(markdownConverter.makeHtml(bodyToParse))}
+
Similar
+ {props.topThree.map(update => ( +
+ + + +
{update.published ? "" : "DRAFT: "}{format(dateOnly(update.date), "MMMM d, yyyy")}
+
{update.title}
+ + +
+ ))} )} @@ -250,10 +268,11 @@ export default function UpdatePage(props: { data: {user: User, updates: (Update } export const getServerSideProps: GetServerSideProps = async (context) => { - if (Array.isArray(context.params.username) || Array.isArray(context.params.updateUrl) || context.params.username.substr(0, 1) !== "@") return { notFound: true }; - const username: string = context.params.username.substr(1); + if (Array.isArray(context.params.username) || Array.isArray(context.params.updateUrl) || context.params.username.substring(0, 1) !== "@") return { notFound: true }; + const username: string = context.params.username.substring(1); const updateUrl: string = context.params.updateUrl; const data = await getUpdateRequest(username, updateUrl); + const topThree = await getTopThree(username, updateUrl); if (!data) return { notFound: true }; @@ -274,5 +293,5 @@ export const getServerSideProps: GetServerSideProps = async (context) => { if (userData) await notificationModel.updateMany({userId: userData._id, updateId: data.updates.find(d => d.url === encodeURIComponent(updateUrl))._id}, {read: true}); - return { props: { data: cleanForJSON(data), updateUrl: updateUrl, userData: cleanForJSON(userData), key: updateUrl }}; + return { props: { data: cleanForJSON(data), updateUrl: updateUrl, userData: cleanForJSON(userData), key: updateUrl, topThree: cleanForJSON(topThree) }}; }; \ No newline at end of file diff --git a/utils/requests.ts b/utils/requests.ts index ab9a70a..e3b83c4 100644 --- a/utils/requests.ts +++ b/utils/requests.ts @@ -2,8 +2,11 @@ import * as mongoose from "mongoose"; import {updateModel, userModel} from "../models/models"; import short from "short-uuid"; import axios from "axios"; +import cohere from "cohere-ai" import {getSession} from "next-auth/react"; -import { SortBy } from "./types"; +import { SortBy, Update } from "./types"; +import { NextApiRequest } from "next"; +import { dot, norm } from "mathjs"; export async function getUpdateRequest(username: string, url: string) { await mongoose.connect(process.env.MONGODB_URL, { @@ -204,4 +207,76 @@ export async function createAccount(user) { private: false, truePrivate: false, }); +} + + +async function getUserUpdates (username: string) { + await mongoose.connect(process.env.MONGODB_URL, { + useNewUrlParser: true, + useUnifiedTopology: true, + useFindAndModify: false, + }); + let user = await userModel.findOne({ "urlName": username }) + if (user === null) return null; + + const updates: Update[] = await updateModel.find({ + userId : user._id, + published: true + }) + return updates +} + +export async function generateUserEmbeddings (username: string) { + cohere.init(process.env.COHERE_KEY) + const updates = await getUserUpdates(username); + const response = await cohere.embed({ + texts: updates.map(update => update.body), + truncate: "RIGHT" + }); + console.log(response) + return updates.map((update, index) => { + return { + update: update, + embeddings: response.body.embeddings[index] + } + }) +} + + + +async function getUpdate (url: string) { + await mongoose.connect(process.env.MONGODB_URL, { + useNewUrlParser: true, + useUnifiedTopology: true, + useFindAndModify: false, + }); + + const update: Update = await updateModel.findOne({ + url: url + }); + return update +} + + +export async function getTopThree (username: string, url: string) { + cohere.init(process.env.COHERE_KEY) + const update = await getUpdate(url); + const updateEmbedding = (await cohere.embed({ + texts: [update.body], + truncate: "RIGHT" + })).body.embeddings[0]; + const embeddings = (await generateUserEmbeddings(username)); + + const similarities = embeddings.map(embedding => { + return { + update: embedding.update, + similarity: dot(updateEmbedding, embedding.embeddings) / ((norm(updateEmbedding) as number) * (norm(embedding.embeddings) as number)) + } + }) + + similarities.sort((a, b) => b.similarity - a.similarity); + console.log(similarities.map(update => ({similarity: update.similarity, title: update.update.title}))); + const finalUpdates = similarities.slice(1, 4).map(similarity => similarity.update); + + return finalUpdates; } \ No newline at end of file From a24e29576e4e7489915a559e77def787e66e295c Mon Sep 17 00:00:00 2001 From: Dron Hazra Date: Sun, 8 Jan 2023 10:38:33 -0500 Subject: [PATCH 2/5] embeddings stored in database now --- models/models.ts | 1 + utils/requests.ts | 73 +++++++++++++++++++++++++++++------------------ utils/types.ts | 1 + 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/models/models.ts b/models/models.ts index 3c25399..2e19670 100644 --- a/models/models.ts +++ b/models/models.ts @@ -46,6 +46,7 @@ const updateSchema: Schema = new Schema({ comments: [commentSchema], mentionedUsers: [mongoose.Schema.Types.ObjectId], published: {type: Boolean, required: true}, + embedding: [mongoose.Schema.Types.Number] }, { timestamps: true, }); diff --git a/utils/requests.ts b/utils/requests.ts index e3b83c4..3153826 100644 --- a/utils/requests.ts +++ b/utils/requests.ts @@ -226,20 +226,14 @@ async function getUserUpdates (username: string) { return updates } -export async function generateUserEmbeddings (username: string) { +export async function generateUserEmbeddings (updates: Update[]) { cohere.init(process.env.COHERE_KEY) - const updates = await getUserUpdates(username); const response = await cohere.embed({ texts: updates.map(update => update.body), truncate: "RIGHT" }); console.log(response) - return updates.map((update, index) => { - return { - update: update, - embeddings: response.body.embeddings[index] - } - }) + return response.body.embeddings } @@ -257,26 +251,49 @@ async function getUpdate (url: string) { return update } +async function getUserEmbeddings(username: string) { + const updates = await getUserUpdates(username); + if (updates[0].embedding === undefined || updates[0].embedding.length === 0) { + console.log("CACHE MISS") + const embeddings = await generateUserEmbeddings(updates); + updates.forEach(async (update, index) => { + update['embedding'] = embeddings[index]; + await update.save(); //ive been casting these as Updates oops + }); + return updates; + } + return updates; +} -export async function getTopThree (username: string, url: string) { - cohere.init(process.env.COHERE_KEY) - const update = await getUpdate(url); - const updateEmbedding = (await cohere.embed({ - texts: [update.body], - truncate: "RIGHT" - })).body.embeddings[0]; - const embeddings = (await generateUserEmbeddings(username)); - - const similarities = embeddings.map(embedding => { - return { - update: embedding.update, - similarity: dot(updateEmbedding, embedding.embeddings) / ((norm(updateEmbedding) as number) * (norm(embedding.embeddings) as number)) - } - }) - - similarities.sort((a, b) => b.similarity - a.similarity); - console.log(similarities.map(update => ({similarity: update.similarity, title: update.update.title}))); - const finalUpdates = similarities.slice(1, 4).map(similarity => similarity.update); +export function semanticSimilarity(a: Update, b: Update) { + return ( + dot(a.embedding, b.embedding) / + ((norm(a.embedding) as number) * (norm(b.embedding) as number)) + ); +} - return finalUpdates; +export async function getTopThree(username: string, url: string) { + console.log('GETTING TOP THREE'); + cohere.init(process.env.COHERE_KEY); + const update = await getUpdate(url); + console.log('GOT UPDATE'); + const hasNoEmbedding = (update: Update) => + update.embedding === undefined || update.embedding.length === 0; + if (hasNoEmbedding(update)) { + console.log('update has no embedding'); + update['embedding'] = ( + await cohere.embed({ + texts: [update.body], + truncate: 'RIGHT', + }) + ).body.embeddings[0]; + } + const updatesWithEmbeddings = await getUserEmbeddings(username); + console.log('GOT EMBEDDINGS'); + + updatesWithEmbeddings + .map((x): [Update, number] => [x, semanticSimilarity(x, update)]) + .sort((a, b) => b[1] - a[1]); + + return updatesWithEmbeddings.slice(1, 4); } \ No newline at end of file diff --git a/utils/types.ts b/utils/types.ts index c150e47..adc2f60 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -37,6 +37,7 @@ export interface Update { updatedAt: string, // date string mentionedUsers: string[], // ids published: boolean, + embedding?: number[] } export interface PrivateAggregation { From 92768684742f3e2532c94444396b1237ae1fa3dc Mon Sep 17 00:00:00 2001 From: Dron Hazra Date: Sun, 8 Jan 2023 18:43:17 -0500 Subject: [PATCH 3/5] cached top three semantically similar! --- components/TopThree.tsx | 41 +++++++ models/models.ts | 6 ++ package.json | 1 - pages/[username]/[updateUrl].tsx | 25 +---- pages/api/get-top-three.ts | 18 ++++ utils/requests.ts | 179 +++++++++++++++++++++---------- 6 files changed, 194 insertions(+), 76 deletions(-) create mode 100644 components/TopThree.tsx create mode 100644 pages/api/get-top-three.ts diff --git a/components/TopThree.tsx b/components/TopThree.tsx new file mode 100644 index 0000000..f2ff1dd --- /dev/null +++ b/components/TopThree.tsx @@ -0,0 +1,41 @@ +import { format } from 'date-fns'; +import Link from 'next/link'; +import useSWR from 'swr'; +import { Update, User } from '../utils/types'; +import { dateOnly } from '../utils/utils'; + +export default function TopThree(props: { + user: User; + update: Update; +}) { + const { data: top } = useSWR( + `/api/get-top-three?userId=${props.update.userId}&updateId=${props.update._id}` + ); + if (top=== undefined) { + return; + } + console.log(top) + return ( + <> + {top.data.map((update, index) => ( + + ))} + + ); +} diff --git a/models/models.ts b/models/models.ts index 2e19670..0e2961c 100644 --- a/models/models.ts +++ b/models/models.ts @@ -51,6 +51,11 @@ const updateSchema: Schema = new Schema({ timestamps: true, }); +const embeddingSchema: Schema = new Schema({ + updateId: mongoose.Schema.Types.ObjectId, + embedding: [mongoose.Schema.Types.Number] +}) + const userSchema: Schema = new Schema({ ...authorObj, private: {type: Boolean, required: true}, @@ -84,6 +89,7 @@ const likeSchema: Schema = new Schema({ export const userModel = (!!mongoose.models && mongoose.models.user) || mongoose.model("user", userSchema); export const updateModel = (!!mongoose.models && mongoose.models.update) || mongoose.model("update", updateSchema); +export const embeddingModel = (!!mongoose.models && mongoose.models.embedding) || mongoose.model("embedding", embeddingSchema); export const commentModel = (!!mongoose.models && mongoose.models.comment) || mongoose.model("comment", commentSchema); export const notificationModel: Model = (!!mongoose.models && mongoose.models.notification) || mongoose.model("notification", notificationSchema); export const likeModel: Model = (!!mongoose.models && mongoose.models.like) || mongoose.model("like", likeSchema); \ No newline at end of file diff --git a/package.json b/package.json index 69f538f..ecc5546 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "cohere-ai": "^5.0.2", "date-fns": "^2.16.1", "html-react-parser": "^0.14.2", - "mathjs": "^11.5.0", "mongoose": "^5.10.16", "next": "^12.1.6", "next-auth": "^4.3.4", diff --git a/pages/[username]/[updateUrl].tsx b/pages/[username]/[updateUrl].tsx index 670ed32..13c6a29 100644 --- a/pages/[username]/[updateUrl].tsx +++ b/pages/[username]/[updateUrl].tsx @@ -20,9 +20,10 @@ import useSWR, {responseInterface} from "swr"; import {FiHeart} from "react-icons/fi"; import {notificationModel} from "../../models/models"; import {getMentionsAndBodySegments} from "../../components/UpdateCommentItem"; +import TopThree from "../../components/TopThree"; import { DeleteModal } from "../../components/Modal"; -export default function UpdatePage(props: { data: {user: User, updates: (Update & {mentionedUsersArr: User[]})[]}, updateUrl: string, userData: User, topThree: Update[] }) { +export default function UpdatePage(props: { data: {user: User, updates: (Update & {mentionedUsersArr: User[]})[]}, updateUrl: string, userData: User }) { const router = useRouter(); const [data, setData] = useState<{user: User, updates: (Update & {mentionedUsersArr: User[]})[]}>(props.data); const [userData, setUserData] = useState(props.userData); @@ -218,23 +219,7 @@ export default function UpdatePage(props: { data: {user: User, updates: (Update
Similar
- {props.topThree.map(update => ( - - ))} + )} @@ -272,7 +257,7 @@ export const getServerSideProps: GetServerSideProps = async (context) => { const username: string = context.params.username.substring(1); const updateUrl: string = context.params.updateUrl; const data = await getUpdateRequest(username, updateUrl); - const topThree = await getTopThree(username, updateUrl); + // const topThree = await getTopThree(username, updateUrl); if (!data) return { notFound: true }; @@ -293,5 +278,5 @@ export const getServerSideProps: GetServerSideProps = async (context) => { if (userData) await notificationModel.updateMany({userId: userData._id, updateId: data.updates.find(d => d.url === encodeURIComponent(updateUrl))._id}, {read: true}); - return { props: { data: cleanForJSON(data), updateUrl: updateUrl, userData: cleanForJSON(userData), key: updateUrl, topThree: cleanForJSON(topThree) }}; + return { props: { data: cleanForJSON(data), updateUrl: updateUrl, userData: cleanForJSON(userData), key: updateUrl}}; }; \ No newline at end of file diff --git a/pages/api/get-top-three.ts b/pages/api/get-top-three.ts new file mode 100644 index 0000000..720ca6a --- /dev/null +++ b/pages/api/get-top-three.ts @@ -0,0 +1,18 @@ +import {getSession} from "next-auth/react"; +import {NextApiRequest, NextApiResponse} from "next"; +import {getCurrUserRequest, getTopThree} from "../../utils/requests"; + +export default async function getTopThreeHandler(req: NextApiRequest, res: NextApiResponse) { + if (req.method !== "GET") return res.status(405); + try { + const topThree = await getTopThree(req.query.userId as string, req.query.updateId as string); + + res.status(200).json({data: topThree.map(update => ({ + date: update.date, + title: update.title, + url: update.url + }))}); + } catch (e) { + res.status(500).json({error: e}); + } +} \ No newline at end of file diff --git a/utils/requests.ts b/utils/requests.ts index 3153826..6805ec5 100644 --- a/utils/requests.ts +++ b/utils/requests.ts @@ -1,12 +1,12 @@ import * as mongoose from "mongoose"; -import {updateModel, userModel} from "../models/models"; +import {embeddingModel, updateModel, userModel} from "../models/models"; import short from "short-uuid"; import axios from "axios"; import cohere from "cohere-ai" import {getSession} from "next-auth/react"; import { SortBy, Update } from "./types"; import { NextApiRequest } from "next"; -import { dot, norm } from "mathjs"; +// import { dot, norm } from "mathjs"; export async function getUpdateRequest(username: string, url: string) { await mongoose.connect(process.env.MONGODB_URL, { @@ -210,21 +210,21 @@ export async function createAccount(user) { } -async function getUserUpdates (username: string) { - await mongoose.connect(process.env.MONGODB_URL, { - useNewUrlParser: true, - useUnifiedTopology: true, - useFindAndModify: false, - }); - let user = await userModel.findOne({ "urlName": username }) - if (user === null) return null; +// async function getUserUpdates (userId: string) { +// await mongoose.connect(process.env.MONGODB_URL, { +// useNewUrlParser: true, +// useUnifiedTopology: true, +// useFindAndModify: false, +// }); +// let user = await userModel.findOne({ "urlName": username }) +// if (user === null) return null; - const updates: Update[] = await updateModel.find({ - userId : user._id, - published: true - }) - return updates -} +// const updates: Update[] = await updateModel.find({ +// userId : user._id, +// published: true +// }) +// return updates +// } export async function generateUserEmbeddings (updates: Update[]) { cohere.init(process.env.COHERE_KEY) @@ -232,10 +232,17 @@ export async function generateUserEmbeddings (updates: Update[]) { texts: updates.map(update => update.body), truncate: "RIGHT" }); - console.log(response) return response.body.embeddings } +export async function generateEmbedding(text:string) { + cohere.init(process.env.COHERE_KEY) + const response = await cohere.embed({ + texts: [text], + truncate: "RIGHT" + }) + return response.body.embeddings[0] +} async function getUpdate (url: string) { @@ -251,49 +258,111 @@ async function getUpdate (url: string) { return update } -async function getUserEmbeddings(username: string) { - const updates = await getUserUpdates(username); - if (updates[0].embedding === undefined || updates[0].embedding.length === 0) { - console.log("CACHE MISS") - const embeddings = await generateUserEmbeddings(updates); - updates.forEach(async (update, index) => { - update['embedding'] = embeddings[index]; - await update.save(); //ive been casting these as Updates oops - }); - return updates; +async function getUserEmbeddings(userId: string) { + await mongoose.connect(process.env.MONGODB_URL, { + useNewUrlParser: true, + useUnifiedTopology: true, + useFindAndModify: false, + }); + const updatesWithEmbeddings: (Update & { embedding: number[] })[] = + await updateModel.aggregate([ + { $match: { userId: mongoose.Types.ObjectId(userId) } }, + { + $lookup: { + from: 'embeddings', + localField: '_id', + foreignField: 'updateId', + as: 'fromEmbeddings', + }, + }, + { $match: { fromEmbeddings: { $ne: [] } } }, + { + $replaceRoot: { + newRoot: { + $mergeObjects: [{ $arrayElemAt: ['$fromEmbeddings', 0] }, '$$ROOT'], + }, + }, + }, + { $project: { fromEmbeddings: 0 } }, + ]); + const updatesWithoutEmbeddings = await updateModel.aggregate([ + { $match: { userId: mongoose.Types.ObjectId(userId) } }, + { + $lookup: { + from: 'embeddings', + localField: '_id', + foreignField: 'updateId', + as: 'fromEmbeddings', + }, + }, + { $match: { fromEmbeddings: { $eq: [] } } }, + { $project: { fromEmbeddings: 0 } }, + ]); + console.log(updatesWithoutEmbeddings.map(update => update.title)); + if (updatesWithoutEmbeddings.length != 0) { + console.log('cache misses: recomputing embeddings'); + const embeddings = await generateUserEmbeddings(updatesWithoutEmbeddings); + console.log(embeddings[0]); + console.log('computed embeddings from cohere'); + await embeddingModel.insertMany( + embeddings.map((embedding, index) => ({ + updateId: mongoose.Types.ObjectId(updatesWithoutEmbeddings[index]._id), + embedding: embedding, + })), + error => { + throw new Error(error); + } + ); + + updatesWithEmbeddings.push( + ...updatesWithoutEmbeddings.map((update, index) => ({ + embedding: embeddings[index], + ...update, + })) + ); } - return updates; + // console.log(updatesWithEmbeddings) + return updatesWithEmbeddings; } -export function semanticSimilarity(a: Update, b: Update) { - return ( - dot(a.embedding, b.embedding) / - ((norm(a.embedding) as number) * (norm(b.embedding) as number)) - ); + +export function semanticSimilarity(a: number[], b: number[]) { + const dot = (a: Float64Array, b: Float64Array) => + a.reduce((prev, curr, index) => prev + curr * b[index]); + + const norm = (x: Float64Array) => + Math.sqrt(x.reduce((prev, curr) => prev + curr * curr)); + + const aEmbed = new Float64Array(a); + const bEmbed = new Float64Array(b); + return dot(aEmbed, bEmbed) / (norm(aEmbed) * norm(bEmbed)); } -export async function getTopThree(username: string, url: string) { +export async function getTopThree(userId: string, updateId: string) { console.log('GETTING TOP THREE'); cohere.init(process.env.COHERE_KEY); - const update = await getUpdate(url); - console.log('GOT UPDATE'); - const hasNoEmbedding = (update: Update) => - update.embedding === undefined || update.embedding.length === 0; - if (hasNoEmbedding(update)) { - console.log('update has no embedding'); - update['embedding'] = ( - await cohere.embed({ - texts: [update.body], - truncate: 'RIGHT', - }) - ).body.embeddings[0]; - } - const updatesWithEmbeddings = await getUserEmbeddings(username); - console.log('GOT EMBEDDINGS'); - - updatesWithEmbeddings - .map((x): [Update, number] => [x, semanticSimilarity(x, update)]) - .sort((a, b) => b[1] - a[1]); - - return updatesWithEmbeddings.slice(1, 4); + await mongoose.connect(process.env.MONGODB_URL, { + useNewUrlParser: true, + useUnifiedTopology: true, + useFindAndModify: false, + }); + + const update = await updateModel.findById(updateId); + const updatesWithEmbeddings = await getUserEmbeddings(userId); + const embedding = await embeddingModel.findOne({ + updateId: update._id, + }); + console.log('got embeddings'); + + console.time('computing similarity'); + const similarities = updatesWithEmbeddings.map((x): [Update, number] => [ + x, + semanticSimilarity(x.embedding, embedding.embedding), + ]); + console.timeEnd('computing similarity'); + console.time('sorting'); + const sortedSimilarities = similarities.sort((a, b) => b[1] - a[1]); + console.timeEnd('sorting'); + // console.log(sortedSimilarities.map(x => `title: ${x[0].title} similarity: ${x[1]}`)) + return sortedSimilarities.map(x => x[0]).slice(1, 4); } \ No newline at end of file From 1cb1c849c9f1da65f9c17e1ff779b44dd90a5c0e Mon Sep 17 00:00:00 2001 From: Dron Hazra Date: Sun, 8 Jan 2023 19:01:19 -0500 Subject: [PATCH 4/5] generate embedding on creation/update of update --- pages/api/update.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pages/api/update.ts b/pages/api/update.ts index 8bd7c3e..d8f56ef 100644 --- a/pages/api/update.ts +++ b/pages/api/update.ts @@ -1,11 +1,12 @@ import {getSession} from "next-auth/react"; import {NextApiRequest, NextApiResponse} from "next"; import mongoose from "mongoose"; -import {userModel, updateModel, notificationModel} from "../../models/models"; +import {userModel, updateModel, notificationModel, embeddingModel} from "../../models/models"; import short from "short-uuid"; import {isEqual} from "date-fns"; import {dateOnly} from "../../utils/utils"; import {Update, User} from "../../utils/types"; +import { generateEmbedding } from "../../utils/requests"; function generateUrlName(title: string, date: string) { let url: string = date; @@ -61,7 +62,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const update = await updateModel.findOne({ "_id": req.body.id }); if (update === null) return res.status(500).json({message: "No update found for given username and ID"}); if (update.userId.toString() !== thisUser._id.toString()) return res.status(403).send("Unauthed"); - + const embedding = generateEmbedding(req.body.body); + await embeddingModel.findOneAndUpdate( + { updateId: req.body.id }, + { + updateId: req.body.id, + embedding: embedding + }, + { upsert: true } + ) switch(req.body.requestType) { case "saveDraft": { update["title"] = req.body.title; From 5b58cb30e318c4c4825713e062d8879ac57d646f Mon Sep 17 00:00:00 2001 From: Dron Hazra Date: Fri, 20 Jan 2023 14:45:44 +0000 Subject: [PATCH 5/5] silenced null db error --- utils/requests.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/requests.ts b/utils/requests.ts index 6805ec5..f74995b 100644 --- a/utils/requests.ts +++ b/utils/requests.ts @@ -298,20 +298,17 @@ async function getUserEmbeddings(userId: string) { { $match: { fromEmbeddings: { $eq: [] } } }, { $project: { fromEmbeddings: 0 } }, ]); + console.log("computed embeddings for: ") console.log(updatesWithoutEmbeddings.map(update => update.title)); if (updatesWithoutEmbeddings.length != 0) { console.log('cache misses: recomputing embeddings'); const embeddings = await generateUserEmbeddings(updatesWithoutEmbeddings); - console.log(embeddings[0]); - console.log('computed embeddings from cohere'); + console.log(`computed ${embeddings.length} embeddings from cohere`); await embeddingModel.insertMany( embeddings.map((embedding, index) => ({ updateId: mongoose.Types.ObjectId(updatesWithoutEmbeddings[index]._id), embedding: embedding, - })), - error => { - throw new Error(error); - } + })) ); updatesWithEmbeddings.push(