Build embeddings

.github/workflows/accelerate_doc.yml

@@ -1,3 +1,4 @@
+
 name: Accelerate doc build
 
 on: [pull_request]

.github/workflows/build_embeddings.yml

@@ -0,0 +1,146 @@
+name: Daily Build Embeddings
+
+env:
+  DIFFUSERS_SLOW_IMPORT: yes
+
+on:
+  schedule:
+    - cron: "5 7 * * *" # every day at 07:05
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  matrix-job:
+    runs-on: ubuntu-latest
+    container: huggingface/transformers-doc-builder
+    strategy:
+      max-parallel: 1 # run sequentially
+      matrix:
+        include:
+          - repo_id: huggingface/tokenizers
+            doc_folder: docs/source-doc-builder
+            package_path: bindings/python
+          - repo_id: huggingface/diffusers
+            doc_folder: docs/source/en
+          - repo_id: huggingface/accelerate
+            doc_folder: docs/source
+          - repo_id: huggingface/huggingface_hub
+            doc_folder: docs/source/en
+          - repo_id: huggingface/transformers
+            doc_folder: docs/source/en
+          - repo_id: huggingface/hub-docs
+            doc_folder: docs/hub
+            package_name: hub
+            is_not_python_module: true
+          - repo_id: huggingface/huggingface.js
+            doc_folder: docs
+            is_not_python_module: true
+            pre_command: npm install -g corepack@latest && corepack enable && cd huggingface.js && pnpm install && pnpm -r build && pnpm --filter doc-internal start
+          - repo_id: huggingface/transformers.js
+            doc_folder: docs/source
+            is_not_python_module: true
+          - repo_id: huggingface/smolagents
+            doc_folder: docs/source/en
+          - repo_id: huggingface/peft
+            doc_folder: docs/source
+          - repo_id: huggingface/trl
+            doc_folder: docs/source
+          - repo_id: bitsandbytes-foundation/bitsandbytes
+            doc_folder: docs/source
+          - repo_id: huggingface/lerobot
+            doc_folder: docs/source
+          - repo_id: huggingface/pytorch-image-models
+            doc_folder: hfdocs/source
+            package_name: timm
+          - repo_id: huggingface/hub-docs
+            doc_folder: docs/inference-providers
+            package_name: inference-providers
+            is_not_python_module: true
+          - repo_id: huggingface/safetensors
+            doc_folder: docs/source
+            package_path: bindings/python
+          - repo_id: huggingface/hf-endpoints-documentation
+            doc_folder: docs/source
+            package_name: inference-endpoints
+            is_not_python_module: true
+          - repo_id: huggingface/dataset-viewer
+            doc_folder: docs/source
+            package_name: dataset-viewer
+            is_not_python_module: true
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    timeout-minutes: 360  # Set timeout to 6 hours
+    steps:
+      - name: Setup REPO_NAME
+        shell: bash
+        run: |
+          current_path=$(pwd)
+          repo_id="${{ matrix.repo_id }}"
+          repo_name="${repo_id#*/}"
+          echo "REPO_NAME=${repo_name}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.repo_id }}
+          path: ${{ github.workspace }}/${{ env.REPO_NAME }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install libgl1
+        run: apt-get install -y libgl1
+
+      - name: Export PIP_OR_UV ('pip' or 'uv pip')
+        run: |
+          if [ -z "${{ job.container }}" ]
+          then
+            echo "PIP_OR_UV=uv pip" >> $GITHUB_ENV
+          else
+            echo "PIP_OR_UV=pip" >> $GITHUB_ENV
+          fi
+
+      - name: Setup environment
+        shell: bash
+        run: |
+          if [[ "${{ matrix.is_not_python_module }}" != "true" ]]; then
+            current_path=$(pwd)
+            cd ${{ env.REPO_NAME }}
+            if [[ -n "${{ matrix.package_path }}" ]]; then
+              cd ${{ matrix.package_path }}
+              $PIP_OR_UV install .[dev]
+              cd $current_path
+            else
+              $PIP_OR_UV install .[dev]
+              cd $current_path
+            fi
+          fi
+
+          rm -rf doc-builder
+          rm -rf .git
+          git clone https://github.com/huggingface/doc-builder.git
+          cd doc-builder
+          git fetch
+          git checkout main
+          $PIP_OR_UV install .
+
+      - name: Run pre-command
+        shell: bash
+        run: |
+          if [ ! -z "${{ matrix.pre_command }}" ]
+          then
+            bash -c "${{ matrix.pre_command }}"
+          fi
+
+      - name: Build embeddings
+        shell: bash
+        run: |
+          echo Building docs for ${{ matrix.package_name || env.REPO_NAME }}
+          FLAGS=""
+          if [[ "${{ matrix.is_not_python_module }}" == "true" ]]; then
+            FLAGS="--not_python_module"
+          fi
+          doc-builder embeddings ${{ matrix.package_name || env.REPO_NAME }} ${{ env.REPO_NAME }}/${{ matrix.doc_folder }} --hf_ie_name docs-embed-bge-base-en-v1-5 --hf_ie_namespace huggingface --hf_ie_token ${{ secrets.HF_IE_TOKEN }} --meilisearch_key ${{ secrets.MEILISEARCH_KEY }} $FLAGS
+

setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-install_requires = ["black", "GitPython", "tqdm", "pyyaml", "packaging", "nbformat", "huggingface_hub", "pillow"]
+install_requires = ["black", "GitPython", "tqdm", "pyyaml", "packaging", "nbformat", "huggingface_hub @ git+https://github.com/huggingface/huggingface_hub.git@feature_extraction_working", "pillow", "meilisearch"]
 
 extras = {}
 

src/doc_builder/__init__.py

@@ -19,8 +19,9 @@
 
 __version__ = "0.6.0.dev0"
 
-from .autodoc import autodoc
+from .autodoc import autodoc_svelte
 from .build_doc import build_doc
+from .build_embeddings import build_embeddings, clean_meilisearch
 from .convert_rst_to_mdx import convert_rst_docstring_to_mdx, convert_rst_to_mdx
 from .style_doc import style_doc_files
 from .utils import update_versions_file