Add per-version testing restrictions (#246)

* Add per-version testing restrictions

* updated testing documentation

* replace eland pin on 8.12 with an exemption list

Author: miguelgrinberg

.github/workflows/tests.yml

@@ -49,9 +49,9 @@ jobs:
         run: make install-nbtest
       - name: Warm up
         continue-on-error: true
-        run: sleep 30 && PATCH_ES=1 ELASTIC_CLOUD_ID=foo ELASTIC_API_KEY=bar bin/nbtest notebooks/search/00-quick-start.ipynb
+        run: sleep 30 && PATCH_ES=1 ELASTIC_CLOUD_ID=foo ELASTIC_API_KEY=bar ES_STACK=${{ matrix.es_stack }} bin/nbtest notebooks/search/00-quick-start.ipynb
       - name: Run tests
-        run: PATCH_ES=1 FORCE_COLOR=1 make -s test
+        run: PATCH_ES=1 FORCE_COLOR=1 ES_STACK=${{ matrix.es_stack }} make -s test
       - name: Show installed Python dependencies
         if: always()
         run: .venv/bin/pip freeze

CONTRIBUTING.md

@@ -88,7 +88,13 @@ To run all supported notebooks under this tool, run the following from the top-l
 make test
 ```
 
-To add a new notebook to our automated testing, you will need to modify the `Makefile` in the directory where your notebook is located. If there is no Makefile in your directory, you can use the one in the `notebooks/search` directory as a model to create one, and then add a reference to it in the top-level `Makefile`.
+If you want to test under a previous release of Elasticsearch, you can define the `ES_STACK` variable with the target version so that notebooks that are not intended for that or previous versions are skipped. For example, here is how to only run tests for the 8.12 release of Elasticsearch:
+
+```bash
+ES_STACK=8.12 make test
+```
+
+Any notebooks that are added in subdirectories under `notebooks` are automatically picked up for testing. Notebooks that are known to not be suitable for automatic testing should be added to the list of exempt tests in the `bin/find_notebooks_to_test.sh` script. If the notebook should never be tested, add it to the `EXEMPT_NOTEBOOKS` list. If the notebook should be tested only under some versions of Elasticsearch, then add it to the appropriate versioned exemption list. For example, the `EXEMPT_NOTEBOOKS__8_12` list should include notebooks that are not be tested under releases 8.12 and older. New versioned lists are automatically recognized and can be added as needed.
 
 ## Contributing to example applications 💻
 

bin/find-notebooks-to-test.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
-# add any notebooks that are currently not testable to the exempt list
+
 EXEMPT_NOTEBOOKS=(
+    # Add any notebooks that are currently not testable to the exempt list
     "notebooks/esql/esql-getting-started.ipynb"
     "notebooks/search/07-inference.ipynb"
     "notebooks/search/08-learning-to-rank.ipynb"
@@ -25,9 +26,46 @@ EXEMPT_NOTEBOOKS=(
     "notebooks/enterprise-search/app-search-engine-exporter.ipynb"
 )
 
+# Per-version testing exceptions
+# use variables named EXEMPT_NOTEBOOKS__{major}_[minor} to list notebooks that
+# cannot run on that stack version or older
+# Examples:
+# EXEMPT_NOTEBOOKS__8 for notebooks that must be skipped on all versions 8.x and older
+# EXEMPT_NOTEBOOKS__8_12 for notebooks that must skipped on versions 8.12 and older
+
+EXEMPT_NOTEBOOKS__8_12=(
+    # Add any notebooks that must be skipped on versions 8.12 or older here
+    "notebooks/document-chunking/with-index-pipelines.ipynb"
+    "notebooks/document-chunking/with-langchain-splitters.ipynb"
+    "notebooks/integrations/hugging-face/loading-model-from-hugging-face.ipynb"
+    "notebooks/langchain/langchain-using-own-model.ipynb"
+)
+
+# this function parses a version given as M[.N[.P]] or M[_N[_P]] into a numeric form
+function parse_version { echo "$@" | awk -F'[._]' '{ printf("%02d%02d\n", $1, $2); }'; }
+
+# this is the version CI is running
+ci_version=$(parse_version ${ES_STACK:-99.99})
+
 ALL_NOTEBOOKS=$(find notebooks -name "*.ipynb" | grep -v "_nbtest" | grep -v ".ipynb_checkpoints" | sort)
 for notebook in $ALL_NOTEBOOKS; do
-    if [[ ! "${EXEMPT_NOTEBOOKS[@]}" =~ $notebook ]]; then
+    skip=
+    # check the master exception list
+    if [[ "${EXEMPT_NOTEBOOKS[@]}" =~ $notebook ]]; then
+        skip=yes
+    else
+        # check the versioned exception lists
+        for exempt_key in ${!EXEMPT_NOTEBOOKS__*}; do
+            exempt_version=$(parse_version ${exempt_key/EXEMPT_NOTEBOOKS__/})
+            if [ $exempt_version -ge $ci_version ]; then
+                exempt_notebooks=$(eval 'echo ${'${exempt_key}'[@]}')
+                if [[ "${exempt_notebooks[@]}" =~ $notebook ]]; then
+                    skip=yes
+                fi
+            fi
+        done
+    fi
+    if [[ "$skip" == "" ]]; then
         echo $notebook
     fi
 done

notebooks/document-chunking/with-index-pipelines.ipynb

@@ -54,7 +54,7 @@
    },
    "outputs": [],
    "source": [
-    "!python3 -m pip install -qU elasticsearch eland==8.12.1"
+    "!python3 -m pip install -qU elasticsearch eland"
    ]
   },
   {
@@ -84,6 +84,7 @@
    "source": [
     "from elasticsearch import Elasticsearch\n",
     "from getpass import getpass\n",
+    "import time\n",
     "\n",
     "# https://www.elastic.co/search-labs/tutorials/install-elasticsearch/elastic-cloud#finding-your-cloud-id\n",
     "ELASTIC_CLOUD_ID = getpass(\"Elastic Cloud ID: \")\n",

notebooks/document-chunking/with-langchain-splitters.ipynb

@@ -31,7 +31,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!python3 -m pip install -qU langchain langchain-elasticsearch elasticsearch eland==8.12.1 jq"
+    "!python3 -m pip install -qU langchain langchain-community langchain-elasticsearch elasticsearch eland jq"
    ]
   },
   {

notebooks/integrations/hugging-face/loading-model-from-hugging-face.ipynb

@@ -47,7 +47,7 @@
    },
    "outputs": [],
    "source": [
-    "!python3 -m pip install sentence-transformers eland==8.12.1 elasticsearch transformers"
+    "!python3 -m pip install sentence-transformers eland elasticsearch transformers"
    ]
   },
   {

notebooks/langchain/langchain-using-own-model.ipynb

@@ -30,7 +30,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!python3 -m pip install -qU langchain langchain-elasticsearch tiktoken  sentence-transformers eland==8.12.1 transformers\n",
+    "!python3 -m pip install -qU langchain langchain-elasticsearch tiktoken  sentence-transformers eland transformers\n",
     "\n",
     "from getpass import getpass\n",
     "from langchain_elasticsearch import ElasticsearchStore\n",