feat: Support JS local tokenizer

api-extractor.tokenizer-node.json

@@ -0,0 +1,26 @@
+/**
+ * API Extractor configuration for the tokenizer/node subpackage.
+ * This configuration extends the one in api-extractor.json and changes
+ * input and output paths for the tokenizer/node subpath export.
+ */
+{
+  "extends": "./api-extractor.json",
+
+  "mainEntryPointFilePath": "<projectFolder>/dist/src/tokenizer/node.d.ts",
+
+  "apiReport": {
+    "reportFileName": "<unscopedPackageName>-tokenizer-node"
+  },
+
+  "docModel": {
+    "apiJsonFilePath": "<projectFolder>/temp/tokenizer-node/<unscopedPackageName>.api.json"
+  },
+
+  "dtsRollup": {
+    "untrimmedFilePath": "<projectFolder>/dist/tokenizer/node.d.ts"
+  },
+
+  "tsdocMetadata": {
+    "tsdocMetadataFilePath": "<projectFolder>/dist/tokenizer/node-tsdoc-metadata.json"
+  }
+}

api-report/genai-node.api.md

@@ -410,6 +410,11 @@ export class ComputeTokensResponse {
     tokensInfo?: TokensInfo[];
 }
 
+// @public
+export interface ComputeTokensResult {
+    tokensInfo?: TokensInfo[];
+}
+
 // @public
 export interface Content {
     parts?: Part[];
@@ -499,6 +504,11 @@ export class CountTokensResponse {
     totalTokens?: number;
 }
 
+// @public
+export interface CountTokensResult {
+    totalTokens?: number;
+}
+
 // @public
 export interface CreateAuthTokenConfig {
     abortSignal?: AbortSignal;

api-report/genai-tokenizer-node.api.md

@@ -0,0 +1,35 @@
+## API Report File for "@google/genai"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+// @public
+export interface ComputeTokensResult {
+    tokensInfo?: TokensInfo[];
+}
+
+// @public
+export interface CountTokensResult {
+    totalTokens?: number;
+}
+
+// @public
+export class LocalTokenizer {
+    constructor(modelName: string);
+    computeTokens(contents: ContentListUnion): Promise<ComputeTokensResult>;
+    // Warning: (ae-forgotten-export) The symbol "ContentListUnion" needs to be exported by the entry point node.d.ts
+    // Warning: (ae-forgotten-export) The symbol "CountTokensConfig" needs to be exported by the entry point node.d.ts
+    countTokens(contents: ContentListUnion, config?: CountTokensConfig): Promise<CountTokensResult>;
+}
+
+// @public
+export interface TokensInfo {
+    role?: string;
+    tokenIds?: string[];
+    tokens?: string[];
+}
+
+// (No @packageDocumentation comment for this package)
+
+```

api-report/genai-tokenizer.api.md

@@ -0,0 +1,38 @@
+## API Report File for "@google/genai"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+// @public
+export interface ComputeTokensResult {
+    tokensInfo?: TokensInfo[];
+}
+
+// @public
+export interface CountTokensResult {
+    totalTokens?: number;
+}
+
+// Warning: (ae-forgotten-export) The symbol "ILocalTokenizer" needs to be exported by the entry point index.d.ts
+//
+// @public
+export class LocalTokenizer implements ILocalTokenizer {
+    // Warning: (ae-forgotten-export) The symbol "TokenizerPlatform" needs to be exported by the entry point index.d.ts
+    constructor(modelName: string, platform: TokenizerPlatform);
+    computeTokens(contents: ContentListUnion): Promise<ComputeTokensResult>;
+    // Warning: (ae-forgotten-export) The symbol "ContentListUnion" needs to be exported by the entry point index.d.ts
+    // Warning: (ae-forgotten-export) The symbol "CountTokensConfig" needs to be exported by the entry point index.d.ts
+    countTokens(contents: ContentListUnion, config?: CountTokensConfig): Promise<CountTokensResult>;
+}
+
+// @public
+export interface TokensInfo {
+    role?: string;
+    tokenIds?: string[];
+    tokens?: string[];
+}
+
+// (No @packageDocumentation comment for this package)
+
+```

api-report/genai-web.api.md

@@ -410,6 +410,11 @@ export class ComputeTokensResponse {
     tokensInfo?: TokensInfo[];
 }
 
+// @public
+export interface ComputeTokensResult {
+    tokensInfo?: TokensInfo[];
+}
+
 // @public
 export interface Content {
     parts?: Part[];
@@ -499,6 +504,11 @@ export class CountTokensResponse {
     totalTokens?: number;
 }
 
+// @public
+export interface CountTokensResult {
+    totalTokens?: number;
+}
+
 // @public
 export interface CreateAuthTokenConfig {
     abortSignal?: AbortSignal;

api-report/genai.api.md

@@ -410,6 +410,11 @@ export class ComputeTokensResponse {
     tokensInfo?: TokensInfo[];
 }
 
+// @public
+export interface ComputeTokensResult {
+    tokensInfo?: TokensInfo[];
+}
+
 // @public
 export interface Content {
     parts?: Part[];
@@ -499,6 +504,11 @@ export class CountTokensResponse {
     totalTokens?: number;
 }
 
+// @public
+export interface CountTokensResult {
+    totalTokens?: number;
+}
+
 // @public
 export interface CreateAuthTokenConfig {
     abortSignal?: AbortSignal;