Merge branch 'refactor' into release/v1.1.0-alpha.3

Author: t83714

package.json

@@ -22,7 +22,7 @@
     "prebuild": "rimraf dist",
     "build": "tsc",
     "watch": "tsc -w",
-    "start": "npm run build && fastify start -T 60000 -l info dist/app.js",
+    "start": "npm run build && fastify start -T 60000 -l info dist/app.js -- --workerTaskTimeout 10000",
     "download-default-model": "node ./dist/loadDefaultModel.js",
     "dev": "npm run build && concurrently -k -p \"[{name}]\" -n \"TypeScript,App\" -c \"yellow.bold,cyan.bold\" \"npm:watch\" \"npm:dev:start\"",
     "dev:start": "fastify start --ignore-watch=.ts$ -w -l info -P dist/app.js",

src/libs/EmbeddingEncoder.ts

@@ -209,7 +209,7 @@ class EmbeddingEncoder {
 
         // Run tokenization
         const model_inputs = this.tokenizer(texts, {
-            padding: true,
+            padding: typeof texts !== "string",
             truncation: true,
             max_length:
                 typeof max_length !== "undefined" && max_length > 0
@@ -334,6 +334,8 @@ class EmbeddingEncoder {
                 `Model \`${model}\` is not supported. Supported models: ${this.supportModels.join(", ")}`
             );
         }
+        const modelOpts = this.getModelByName(model);
+        const { max_length } = modelOpts;
         opts = {
             ...opts,
             ...(typeof opts.padding !== "boolean" ? { padding: true } : {}),
@@ -343,8 +345,12 @@ class EmbeddingEncoder {
         };
 
         return this.tokenizer(texts, {
-            padding: true,
-            truncation: true
+            padding: typeof texts !== "string",
+            truncation: true,
+            max_length:
+                typeof max_length !== "undefined" && max_length > 0
+                    ? max_length
+                    : DEFAULT_MAX_LENGTH
         });
     }
 }