better meilisearch handling

Author: mishig25

.github/workflows/build_embeddings.yml

@@ -75,12 +75,18 @@ jobs:
   cleanup-job:
     needs: matrix-job
     runs-on: ubuntu-latest
+    if: always() # This ensures that the cleanup job runs regardless of the result of matrix-job
     steps:
       - name: Checkout doc-builder
         uses: actions/checkout@v2
-        
+      
       - name: Install doc-builder
         run: pip install .[dev]
 
-      - name: Clean meilisearch
-        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }}
+      - name: Success Cleanup
+        if: success() # Runs if all matrix jobs succeeded
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }} --swap
+
+      - name: Failure Cleanup
+        if: failure() # Runs if any matrix job failed
+        run: doc-builder meilisearch-clean --meilisearch_key ${{ secrets.MEILISEARCH_KEY }}

src/doc_builder/build_embeddings.py

@@ -473,12 +473,13 @@ def build_embeddings(
         add_embeddings_to_db(client, MEILI_INDEX_TEMP, chunk_embeddings)
 
 
-def clean_meilisearch(meilisearch_key: str):
+def clean_meilisearch(meilisearch_key: str, swap: bool):
     """
     Swap & delete temp index.
     """
     client = meilisearch.Client("https://edge.meilisearch.com", meilisearch_key)
-    swap_indexes(client, MEILI_INDEX, MEILI_INDEX_TEMP)
+    if swap:
+        swap_indexes(client, MEILI_INDEX, MEILI_INDEX_TEMP)
     delete_embedding_db(client, MEILI_INDEX_TEMP)
     create_embedding_db(client, MEILI_INDEX_TEMP)
     update_db_settings(client, MEILI_INDEX_TEMP)

src/doc_builder/commands/embeddings.py

@@ -127,7 +127,10 @@ def embeddings_command_parser(subparsers=None):
             "Doc Builder meilisearch clean command. Swap & delete the temp index."
         )
     parser_meilisearch_clean.add_argument("--meilisearch_key", type=str, help="Meilisearch key.", required=True)
+    parser_meilisearch_clean.add_argument(
+        "--swap", action="store_true", help="Whether to swap temp index with prod index."
+    )
     if subparsers is not None:
-        parser_meilisearch_clean.set_defaults(func=lambda args: clean_meilisearch(args.meilisearch_key))
+        parser_meilisearch_clean.set_defaults(func=lambda args: clean_meilisearch(args.meilisearch_key, args.swap))
 
     return parser