feat: adding workflow file to run notebooks

.github/workflows/run_cookbooks.yml

@@ -0,0 +1,76 @@
+name: Run Haystack Cookbooks
+
+on:
+  pull_request:
+  schedule:
+    - cron: '0 0 */14 * *'  # Every 14 days at midnight UTC
+  workflow_dispatch:
+
+
+jobs:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generator.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - id: generator
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Get cookbooks
+          pip install requests
+          NOTEBOOKS=$(python ./scripts/generate_matrix.py --include-main)
+          echo "matrix={\"include\":$NOTEBOOKS}" >> "$GITHUB_OUTPUT"
+
+  run-notebooks:
+    runs-on: ubuntu-latest
+    needs: generate-matrix
+    container: deepset/haystack:${{ matrix.haystack_version }}
+
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
+
+    env:
+      HAYSTACK_TELEMETRY_ENABLED: "False"
+      # HF_API_TOKEN: ${{ secrets.HF_API_KEY }}
+      # Note: HF_API_TOKEN needs to be unset for notebooks using sentence-transformers
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Dump matrix content
+        run: echo '${{ toJSON(matrix) }}'
+
+      - name: Install common dependencies
+        run: |
+          apt-get update && apt-get install -y \
+              build-essential \
+              gcc \
+              libsndfile1 \
+              ffmpeg
+
+          pip install nbconvert ipython
+
+      - name: Install dependencies
+        if: toJSON(matrix.dependencies) != '[]'
+        run: |
+          pip install "${{ join(matrix.dependencies, '" "')}}"
+
+      - name: Convert notebook to Python
+        run: |
+          jupyter nbconvert --to python --RegexRemovePreprocessor.patterns '%%bash' ./notebooks/${{ matrix.notebook }}.ipynb
+
+      - name: Run the converted notebook
+        run: |
+          python ./notebooks/${{ matrix.notebook }}.py

index.toml

@@ -283,6 +283,7 @@ topics = ["Function Calling", "Agents"]
 title = "Extracting Metadata with an LLM"
 notebook = "metadata_extraction_with_llm_metadata_extractor.ipynb"
 topics = ["Metadata"]
+dependencies = ["sentence-transformers>=4.1.0"]
 
 [[cookbook]]
 title = "Building an Interactive Feedback Review Agent with Azure AI Search and Haystack"
@@ -309,6 +310,7 @@ title = "DevOps Support Agent with Human in the Loop"
 notebook = "agent_with_human_in_the_loop.ipynb"
 new = true
 topics = ["Function Calling", "Agents"]
+dependencies = ["requests"]
 
 [[cookbook]]
 title = "Introduction to Multimodal Text Generation"

notebooks/auto_merging_retriever.ipynb

@@ -24,17 +24,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
     },
     "id": "LaJsFx4P1o_l",
     "outputId": "a5b29fa2-6d74-4ccf-e732-77c8a4f68491"
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[2mUsing Python 3.12.6 environment at: /Users/dsbatista/haystack-cookbook/.venv\u001b[0m\n",
+      "\u001b[2mAudited \u001b[1m1 package\u001b[0m \u001b[2min 23ms\u001b[0m\u001b[0m\n"
+     ]
+    }
+   ],
    "source": [
-    "!pip install haystack-ai"
+    "%%bash\n",
+    "\n",
+    "pip install haystack-ai"
    ]
   },
   {
@@ -52,35 +63,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "cpMYVx1VY7Z7",
-    "outputId": "521dbe20-c6dc-4897-c4d7-764b6b82cea1"
-   },
+   "execution_count": 2,
+   "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "--2024-09-06 09:41:04--  https://raw.githubusercontent.com/amankharwal/Website-data/master/bbc-news-data.csv\n",
-      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n",
-      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
-      "HTTP request sent, awaiting response... 200 OK\n",
-      "Length: 5080260 (4.8M) [text/plain]\n",
-      "Saving to: ‘bbc-news-data.csv’\n",
-      "\n",
-      "bbc-news-data.csv   100%[===================>]   4.84M  --.-KB/s    in 0.09s   \n",
-      "\n",
-      "2024-09-06 09:41:05 (56.4 MB/s) - ‘bbc-news-data.csv’ saved [5080260/5080260]\n",
-      "\n"
-     ]
+     "data": {
+      "text/plain": [
+       "('bbc-news-data.csv', <http.client.HTTPMessage at 0x103bd8260>)"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
-    "!wget https://raw.githubusercontent.com/amankharwal/Website-data/master/bbc-news-data.csv"
+    "import urllib.request\n",
+    "urllib.request.urlretrieve('https://raw.githubusercontent.com/amankharwal/Website-data/master/bbc-news-data.csv', 'bbc-news-data.csv')"
    ]
   },
   {

notebooks/llama3_rag.ipynb

@@ -309,26 +309,21 @@
       },
       "outputs": [],
       "source": [
-        "from haystack.components.builders import PromptBuilder\n",
-        "\n",
-        "prompt_template = \"\"\"\n",
-        "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n",
-        "\n",
+        "from haystack.components.builders import ChatPromptBuilder\n",
+        "from haystack.dataclasses import ChatMessage\n",
         "\n",
+        "template = [ChatMessage.from_user(\"\"\"\n",
         "Using the information contained in the context, give a comprehensive answer to the question.\n",
         "If the answer cannot be deduced from the context, do not give an answer.\n",
         "\n",
         "Context:\n",
         "  {% for doc in documents %}\n",
         "  {{ doc.content }} URL:{{ doc.meta['url'] }}\n",
         "  {% endfor %};\n",
-        "  Question: {{query}}<|eot_id|>\n",
-        "\n",
-        "<|start_header_id|>assistant<|end_header_id|>\n",
+        "  Question: {{query}}\n",
         "\n",
-        "\n",
-        "\"\"\"\n",
-        "prompt_builder = PromptBuilder(template=prompt_template)"
+        "\"\"\")]\n",
+        "prompt_builder = ChatPromptBuilder(template=template)"
       ]
     },
     {
@@ -337,7 +332,7 @@
         "id": "pbvNtRzxPSOe"
       },
       "source": [
-        "Here, we use the [`HuggingFaceLocalGenerator`](https://docs.haystack.deepset.ai/docs/huggingfacelocalgenerator), loading the model in Colab with 4-bit quantization."
+        "Here, we use the [`HuggingFaceLocalChatGenerator`](https://docs.haystack.deepset.ai/docs/huggingfacelocalchatgenerator), loading the model in Colab with 4-bit quantization."
       ]
     },
     {
@@ -664,9 +659,9 @@
       ],
       "source": [
         "import torch\n",
-        "from haystack.components.generators import HuggingFaceLocalGenerator\n",
+        "from haystack.components.generators.chat import HuggingFaceLocalChatGenerator\n",
         "\n",
-        "generator = HuggingFaceLocalGenerator(\n",
+        "generator = HuggingFaceLocalChatGenerator(\n",
         "    model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
         "    huggingface_pipeline_kwargs={\"device_map\":\"auto\",\n",
         "                                  \"model_kwargs\":{\"load_in_4bit\":True,\n",
@@ -688,27 +683,7 @@
         "id": "lx6PNcm-I1zF",
         "outputId": "363cf752-2b84-48d2-b8bc-d9f83542ff96"
       },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "<haystack.core.pipeline.pipeline.Pipeline object at 0x7fcda58f5300>\n",
-              "🚅 Components\n",
-              "  - text_embedder: SentenceTransformersTextEmbedder\n",
-              "  - retriever: InMemoryEmbeddingRetriever\n",
-              "  - prompt_builder: PromptBuilder\n",
-              "  - generator: HuggingFaceLocalGenerator\n",
-              "🛤️ Connections\n",
-              "  - text_embedder.embedding -> retriever.query_embedding (List[float])\n",
-              "  - retriever.documents -> prompt_builder.documents (List[Document])\n",
-              "  - prompt_builder.prompt -> generator.prompt (str)"
-            ]
-          },
-          "execution_count": 10,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
       "source": [
         "from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever\n",
         "\n",
@@ -722,7 +697,7 @@
         "        prefix=\"Represent this sentence for searching relevant passages: \",  # as explained in the model card (https://huggingface.co/Snowflake/snowflake-arctic-embed-l#using-huggingface-transformers), queries should be prefixed\n",
         "    ))\n",
         "query_pipeline.add_component(\"retriever\", InMemoryEmbeddingRetriever(document_store=document_store, top_k=5))\n",
-        "query_pipeline.add_component(\"prompt_builder\", PromptBuilder(template=prompt_template))\n",
+        "query_pipeline.add_component(\"prompt_builder\", ChatPromptBuilder(template=template))\n",
         "query_pipeline.add_component(\"generator\", generator)\n",
         "\n",
         "# connect the components\n",
@@ -756,7 +731,7 @@
         "    }\n",
         "  )\n",
         "\n",
-        "  answer = results[\"generator\"][\"replies\"][0]\n",
+        "  answer = results[\"generator\"][\"replies\"][0].text\n",
         "  rich.print(answer)"
       ]
     },
@@ -1148,8 +1123,8 @@
         "\n",
         "To use Llama 3 models in Haystack, you also have **other options**:\n",
         "- [LlamaCppGenerator](https://docs.haystack.deepset.ai/docs/llamacppgenerator) and [OllamaGenerator](https://docs.haystack.deepset.ai/docs/ollamagenerator): using the GGUF quantized format, these solutions are ideal to run LLMs on standard machines (even without GPUs).\n",
-        "- [HuggingFaceAPIGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapigenerator), which allows you to query a local TGI container or a (paid) HF Inference Endpoint. TGI is a toolkit for efficiently deploying and serving LLMs in production.\n",
-        "- [vLLM via OpenAIGenerator](https://haystack.deepset.ai/integrations/vllm): high-throughput and memory-efficient inference and serving engine for LLMs.\n",
+        "- [HuggingFaceAPIChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapichatgenerator), which allows you to query a the Hugging Face API, a local TGI container or a (paid) HF Inference Endpoint. TGI is a toolkit for efficiently deploying and serving LLMs in production.\n",
+        "- [vLLM via OpenAIChatGenerator](https://haystack.deepset.ai/integrations/vllm): high-throughput and memory-efficient inference and serving engine for LLMs.\n",
         "\n"
       ]
     },