Merge branch 'huggingface:main' into main

.github/workflows/lint-and-test.yml

@@ -35,7 +35,7 @@ jobs:
       - run: |
           pnpm install --frozen-lockfile --filter .
           pnpm install --frozen-lockfile --filter ...[${{ steps.since.outputs.SINCE }}]...
-          pnpm --filter [${{ steps.since.outputs.SINCE }}]... build
+          pnpm --filter ...[${{ steps.since.outputs.SINCE }}]... build
       - name: "Checking lint errors"
         run: |
           pnpm --filter ...[${{ steps.since.outputs.SINCE }}] lint:check
@@ -50,12 +50,12 @@ jobs:
       - name: Test
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test
         env:
-          HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
       - name: Test in browser
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test:browser
         env:
-          HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
       - name: E2E - start mock npm registry
         run: |
@@ -68,7 +68,7 @@ jobs:
         working-directory: e2e
         run: |
           sleep 3
-          pnpm i --filter inference --filter hub --frozen-lockfile
+          pnpm i --filter inference... --filter hub... --frozen-lockfile
           pnpm --filter inference --filter hub publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install
@@ -86,7 +86,7 @@ jobs:
           pnpm i --ignore-workspace --registry http://localhost:4874/
           pnpm start
         env:
-          token: ${{ secrets.HF_ACCESS_TOKEN }}
+          token: ${{ secrets.HF_TOKEN }}
 
       - name: E2E test - svelte app build
         working-directory: e2e/svelte

.github/workflows/upload-pr-documentation.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
     with:
       package_name: huggingface.js
     secrets:

.github/workflows/widgets-publish.yml

@@ -37,43 +37,43 @@ jobs:
           node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
           git commit . -m "🔖 @huggingface/widgets $BUMPED_VERSION"
           git tag "widgets-v$BUMPED_VERSION"
-      - run: |
-          # Clean up the workspace
-          rm -rf ../tasks/node_modules ../tasks/dist
 
-          # Calculate checksum of the cleaned package
-          echo "Calculating checksum of local @huggingface/tasks package..."
-          LOCAL_CHECKSUM=$(cd ../tasks && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
-          echo "Local package checksum: $LOCAL_CHECKSUM"
+      - name: Make sure that the latest version of @huggingface/tasks is consistent with the local version
+        run: |
+          LOCAL_TASKS_VERSION=$(node -p "require('./package.json').version")
+          REMOTE_TASKS_VERSION=$(npm view @huggingface/tasks version)
 
-          # Fetch the latest published version of @huggingface/tasks from npm
-          echo "Fetching latest published version of @huggingface/tasks..."
-          LATEST_PUBLISHED_VERSION=$(npm view @huggingface/tasks version)
-          if [ -z "$LATEST_PUBLISHED_VERSION" ]; then
-            echo "Error: Unable to fetch the latest version of @huggingface/tasks"
+          # If the versions are different, error 
+          if [ "$LOCAL_TASKS_VERSION" != "$REMOTE_TASKS_VERSION" ]; then
+            echo "Error: The local @huggingface/tasks package version ($LOCAL_TASKS_VERSION) differs from the remote version ($REMOTE_TASKS_VERSION). Release halted."
             exit 1
           fi
 
-          # Download the package tarball from npm and calculate its checksum
-          npm pack @huggingface/tasks@$LATEST_PUBLISHED_VERSION
-          if [ ! -f "huggingface-tasks-$LATEST_PUBLISHED_VERSION.tgz" ]; then
-            echo "Error: Failed to download @huggingface/tasks@$LATEST_PUBLISHED_VERSION"
-            exit 1
-          fi
+          npm pack @huggingface/tasks
+          mv huggingface-tasks-$LOCAL_TASKS_VERSION.tgz tasks-local.tgz
 
-          # Extract and remove dist and node_modules directories before checksum calculation
-          mkdir -p /tmp/tasks
-          tar -xf huggingface-tasks-$LATEST_PUBLISHED_VERSION.tgz -C /tmp/tasks
-          rm -rf /tmp/tasks/dist /tmp/tasks/node_modules
+          npm pack @huggingface/tasks@$REMOTE_TASKS_VERSION
+          mv huggingface-tasks-$REMOTE_TASKS_VERSION.tgz tasks-remote.tgz
 
-          REMOTE_CHECKSUM=$(cd /tmp/tasks && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          # Compute checksum of local tar. We need to extract both tar since the remote compression might be different
+          tar -xf tasks-local.tgz
+          LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Local package checksum: $LOCAL_CHECKSUM"
+
+          rm -Rf package
+
+          tar -xf tasks-remote.tgz
+          REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
           echo "Remote package checksum: $REMOTE_CHECKSUM"
 
+          rm -Rf package
+
           if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
             echo "Checksum Verification Failed: The local @huggingface/tasks package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
             exit 1
           fi
           echo "Checksum Verification Successful: The local and remote @huggingface/tasks packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
+        working-directory: packages/tasks
 
       - run: pnpm publish --no-git-checks .
         env:

CODEOWNERS

@@ -1,19 +1,23 @@
-# This is a comment.
-
 # Ownership for the Inference Package
+
 /packages/inference/ @vvmnnnkv @radames
 
 # Ownership for the Tasks Package
-/packages/tasks/ @osanseviero @merveenoyan @pcuenca
+
+/packages/tasks/ @osanseviero @SBrandeis @gary149 @Wauplin
 
 # Ownership for the Widgets Package
-/packages/widgets/ @mishig25
+
+/packages/widgets/ @mishig25 @SBrandeis
 
 # Ownership for the Hub Package
+
 /packages/hub/ @coyotte508
 
 # Ownership for the Agents Package
+
 /packages/agents/ @nsarrazin
 
 # Ownership for the Jinja Package
-/packages/jinja/ @xenova
+
+/packages/jinja/ @xenova

README.md

@@ -76,7 +76,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 ```html
 <script type="module">
     import { HfInference } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm';
-    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected].2/+esm";
+    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected].3/+esm";
 </script>
 ```
 
@@ -105,9 +105,9 @@ Get your HF access token in your [account settings](https://huggingface.co/setti
 ```ts
 import { HfInference } from "@huggingface/inference";
 
-const HF_ACCESS_TOKEN = "hf_...";
+const HF_TOKEN = "hf_...";
 
-const inference = new HfInference(HF_ACCESS_TOKEN);
+const inference = new HfInference(HF_TOKEN);
 
 // You can also omit "model" to use the recommended model for the task
 await inference.translation({
@@ -137,11 +137,11 @@ const { generated_text } = await gpt2.textGeneration({inputs: 'The answer to the
 ```ts
 import {HfAgent, LLMFromHub, defaultTools} from '@huggingface/agents';
 
-const HF_ACCESS_TOKEN = "hf_...";
+const HF_TOKEN = "hf_...";
 
 const agent = new HfAgent(
-  HF_ACCESS_TOKEN,
-  LLMFromHub(HF_ACCESS_TOKEN),
+  HF_TOKEN,
+  LLMFromHub(HF_TOKEN),
   [...defaultTools]
 );
 
@@ -162,16 +162,16 @@ console.log(messages);
 ```ts
 import { createRepo, uploadFile, deleteFiles } from "@huggingface/hub";
 
-const HF_ACCESS_TOKEN = "hf_...";
+const HF_TOKEN = "hf_...";
 
 await createRepo({
   repo: "my-user/nlp-model", // or {type: "model", name: "my-user/nlp-test"},
-  credentials: {accessToken: HF_ACCESS_TOKEN}
+  credentials: {accessToken: HF_TOKEN}
 });
 
 await uploadFile({
   repo: "my-user/nlp-model",
-  credentials: {accessToken: HF_ACCESS_TOKEN},
+  credentials: {accessToken: HF_TOKEN},
   // Can work with native File in browsers
   file: {
     path: "pytorch_model.bin",
@@ -181,7 +181,7 @@ await uploadFile({
 
 await deleteFiles({
   repo: {type: "space", name: "my-user/my-space"}, // or "spaces/my-user/my-space"
-  credentials: {accessToken: HF_ACCESS_TOKEN},
+  credentials: {accessToken: HF_TOKEN},
   paths: ["README.md", ".gitattributes"]
 });
 ```

package.json

@@ -10,7 +10,7 @@
 	"devDependencies": {
 		"@typescript-eslint/eslint-plugin": "^5.54.0",
 		"@typescript-eslint/parser": "^5.54.0",
-		"@vitest/browser": "^0.29.7",
+		"@vitest/browser": "^0.34.6",
 		"eslint": "^8.35.0",
 		"eslint-config-prettier": "^9.0.0",
 		"eslint-plugin-prettier": "^4.2.1",
@@ -20,9 +20,9 @@
 		"semver": "^7.5.0",
 		"ts-node": "^10.9.1",
 		"tsup": "^6.7.0",
-		"typescript": "^5.0.2",
-		"vite": "4.1.4",
-		"vitest": "^0.29.4",
+		"typescript": "^5.0.4",
+		"vite": "^5.0.2",
+		"vitest": "^0.34.6",
 		"webdriverio": "^8.6.7"
 	}
 }

packages/agents/README.md

@@ -131,7 +131,7 @@ const uppercaseTool: Tool = {
 };
 
 // pass it in the agent
-const agent = new HfAgent(process.env.HF_ACCESS_TOKEN, 
+const agent = new HfAgent(process.env.HF_TOKEN,
                 LLMFromHub("hf_...", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"),
                 [uppercaseTool, ...defaultTools]);
 ```

packages/agents/package.json

@@ -33,7 +33,7 @@
 		"prepublishOnly": "pnpm run build",
 		"build": "tsup",
 		"test": "vitest run",
-		"test:browser": "vitest run --browser.name=chrome --browser.headless --config ./vitest-browser.config.ts",
+		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
 		"check": "tsc"
 	},
 	"files": [
@@ -54,8 +54,7 @@
 	"license": "MIT",
 	"devDependencies": {
 		"@types/node": "^18.13.0",
-		"type-fest": "^3.9.0",
-		"typescript": "^5.0.4"
+		"type-fest": "^3.9.0"
 	},
 	"dependencies": {
 		"@huggingface/inference": "^2.6.1"

packages/agents/test/HfAgent.spec.ts

@@ -1,20 +1,23 @@
 import { describe, expect, it } from "vitest";
 import { HfAgent, defaultTools, LLMFromHub, LLMFromEndpoint } from "../src";
+import type { Data } from "../src/types";
+import type { HfInference } from "@huggingface/inference";
 
-if (!process.env.HF_ACCESS_TOKEN) {
-	console.warn("Set HF_ACCESS_TOKEN in the env to run the tests for better rate limits");
+const env = import.meta.env;
+if (!env.HF_TOKEN) {
+	console.warn("Set HF_TOKEN in the env to run the tests for better rate limits");
 }
 
 describe("HfAgent", () => {
 	it("You can create an agent from the hub", async () => {
-		const llm = LLMFromHub(process.env.HF_ACCESS_TOKEN, "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5");
-		const agent = new HfAgent(process.env.HF_ACCESS_TOKEN, llm);
+		const llm = LLMFromHub(env.HF_TOKEN, "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5");
+		const agent = new HfAgent(env.HF_TOKEN, llm);
 		expect(agent).toBeDefined();
 	});
 
 	it("You can create an agent from an endpoint", async () => {
-		const llm = LLMFromEndpoint(process.env.HF_ACCESS_TOKEN ?? "", "endpoint");
-		const agent = new HfAgent(process.env.HF_ACCESS_TOKEN, llm);
+		const llm = LLMFromEndpoint(env.HF_TOKEN ?? "", "endpoint");
+		const agent = new HfAgent(env.HF_TOKEN, llm);
 		expect(agent).toBeDefined();
 	});
 
@@ -29,7 +32,8 @@ describe("HfAgent", () => {
 					tools: ["uppercase"],
 				},
 			],
-			call: async (input) => {
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			call: async (input: Promise<Data>, inference: HfInference): Promise<Data> => {
 				const data = await input;
 				if (typeof data !== "string") {
 					throw new Error("Input must be a string");
@@ -38,7 +42,7 @@ describe("HfAgent", () => {
 			},
 		};
 
-		const agent = new HfAgent(process.env.HF_ACCESS_TOKEN, undefined, [uppercaseTool, ...defaultTools]);
+		const agent = new HfAgent(env.HF_TOKEN, undefined, [uppercaseTool, ...defaultTools]);
 		const code = `
 async function generate() {
 	const output = uppercase("hello friends");
@@ -57,7 +61,7 @@ async function generate() {
 	message(output);
 }`;
 
-		const agent = new HfAgent(process.env.HF_ACCESS_TOKEN);
+		const agent = new HfAgent(env.HF_TOKEN);
 
 		await agent.evaluateCode(code).then((output) => {
 			expect(output.length).toBeGreaterThan(0);
@@ -71,7 +75,7 @@ async function generate() {
 	toolThatDoesntExist(aaa);
 }`;
 
-		const hf = new HfAgent(process.env.HF_ACCESS_TOKEN);
+		const hf = new HfAgent(env.HF_TOKEN);
 
 		await hf.evaluateCode(code).then((output) => {
 			expect(output.length).toBeGreaterThan(0);

packages/agents/test/vitest.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

packages/agents/tsconfig.json

@@ -2,7 +2,7 @@
 	"compilerOptions": {
 		"allowSyntheticDefaultImports": true,
 		"lib": ["ES2022", "DOM"],
-		"module": "CommonJS",
+		"module": "ESNext",
 		"moduleResolution": "node",
 		"target": "ES2022",
 		"forceConsistentCasingInFileNames": true,
@@ -14,6 +14,6 @@
 		"outDir": "./dist",
 		"declaration": true
 	},
-	"include": ["src", "index.ts", "../shared/src"],
+	"include": ["src", "test", "index.ts", "../shared/src"],
 	"exclude": ["dist"]
 }

packages/agents/vitest-browser.config.ts → packages/agents/vitest-browser.config.mts

@@ -1,6 +1,7 @@
 import { configDefaults, defineConfig } from "vitest/config";
 
 export default defineConfig({
+	envPrefix: ["HF_"],
 	test: {
 		exclude: [...configDefaults.exclude],
 	},

packages/doc-internal/package.json

@@ -26,8 +26,7 @@
 	"devDependencies": {
 		"@types/node": "^18.14.5",
 		"typedoc": "^0.23.26",
-		"typedoc-plugin-markdown": "^3.14.0",
-		"typescript": "^4.9.5"
+		"typedoc-plugin-markdown": "^3.14.0"
 	},
 	"dependencies": {
 		"glob": "^9.2.1",