Build embeddings

.github/workflows/accelerate_doc.yml

@@ -1,6 +1,6 @@
 name: Accelerate doc build
 
-on: [pull_request]
+# on: [pull_request]
 
 jobs:
   integration_doc_build:

.github/workflows/build_embeddings.yml

@@ -0,0 +1,101 @@
+name: Daily Build Embeddings
+
+env:
+  DIFFUSERS_SLOW_IMPORT: yes
+
+on:
+  push:
+  schedule:
+    - cron: "5 7 * * *" # every day at 07:05
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  matrix-job:
+    runs-on: ubuntu-latest
+    container: huggingface/transformers-doc-builder
+    strategy:
+      max-parallel: 1 # run the matrix jobs sequentially
+      matrix:
+        include:
+          - repo_id: huggingface/diffusers
+            doc_folder: docs/source/en
+          - repo_id: huggingface/accelerate
+            doc_folder: docs/source
+          - repo_id: huggingface/huggingface_hub
+            doc_folder: docs/source/en
+          - repo_id: huggingface/transformers
+            doc_folder: docs/source/en
+          - repo_id: huggingface/hub-docs
+            doc_folder: docs/hub
+            package_name: hub
+            is_not_python_module: true
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    timeout-minutes: 360  # Set timeout to 6 hours
+    steps:
+      - name: Setup REPO_NAME
+        shell: bash
+        run: |
+          current_path=$(pwd)
+          repo_id="${{ matrix.repo_id }}"
+          repo_name="${repo_id#*/}"
+          echo "REPO_NAME=${repo_name}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.repo_id }}
+          path: ${{ github.workspace }}/${{ env.REPO_NAME }}
+
+      - name: Install libgl1
+        run: apt-get install -y libgl1
+
+      - name: Setup environment
+        shell: bash
+        run: |
+          if [[ "${{ matrix.is_not_python_module }}" != "true" ]]; then
+            current_path=$(pwd)
+            cd ${{ env.REPO_NAME }}
+            pip install .[dev]
+            cd $current_path
+          fi
+
+          rm -rf doc-builder
+          rm -rf .git
+          git clone https://github.com/huggingface/doc-builder.git
+          cd doc-builder
+          git fetch
+          git checkout build_embeddings
+          pip install .
+
+      - name: Build embeddings
+        shell: bash
+        run: |
+          echo Building docs for ${{ matrix.package_name || env.REPO_NAME }}
+          FLAGS=""
+          if [[ "${{ matrix.is_not_python_module }}" == "true" ]]; then
+            FLAGS="--not_python_module"
+          fi
+          doc-builder embeddings ${{ matrix.package_name || env.REPO_NAME }} ${{ env.REPO_NAME }}/${{ matrix.doc_folder }} --hf_ie_name docs-embed-bge-base-en-v1-5 --hf_ie_namespace huggingface --hf_ie_token ${{ secrets.HF_IE_TOKEN }} --meilisearch_key ${{ secrets.MEILISEARCH_KEY }} $FLAGS
+
+
+  cleanup-job:
+    needs: matrix-job
+    runs-on: ubuntu-latest
+    if: always() # This ensures that the cleanup job runs regardless of the result of matrix-job
+    steps:
+      - name: Checkout doc-builder
+        uses: actions/checkout@v2
+
+      - name: Install doc-builder
+        run: pip install .[dev]
+
+      - name: Success Cleanup
+        if: success() # Runs if all matrix jobs succeeded
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }} --swap
+
+      - name: Failure Cleanup
+        if: failure() # Runs if any matrix job failed
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }}

setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-install_requires = ["black", "GitPython", "tqdm", "pyyaml", "packaging", "nbformat", "huggingface_hub", "pillow"]
+install_requires = ["black", "GitPython", "tqdm", "pyyaml", "packaging", "nbformat", "huggingface_hub @ git+https://github.com/huggingface/huggingface_hub.git", "pillow", "meilisearch"]
 
 extras = {}
 

src/doc_builder/__init__.py

@@ -19,8 +19,9 @@
 
 __version__ = "0.6.0.dev0"
 
-from .autodoc import autodoc
+from .autodoc import autodoc_svelte
 from .build_doc import build_doc
+from .build_embeddings import build_embeddings, clean_meilisearch
 from .convert_rst_to_mdx import convert_rst_docstring_to_mdx, convert_rst_to_mdx
 from .style_doc import style_doc_files
 from .utils import update_versions_file