Skip to content

Commit e826b12

Browse files
tim-schillingtim-schilling
authored
Update elasticsearch testing for v8. (#741)
* Update elasticsearch testing for v8. Transport was moved to elastic_transport. Check there first and fallback to elasticsearch if it's not there. Updates the development docs to contain instructions on running elasticsearch v8. Closes #725 * Rework github tests to use elasticsearch 7 for py2.7 and 3.5. * Disable https and basic auth for elasticsearch in CI.
1 parent 3c4d08d commit e826b12

File tree

6 files changed

+178
-75
lines changed

6 files changed

+178
-75
lines changed

‎.github/workflows/main.yml‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,69 @@ on:
1111
- cron: '44 2 * * *'
1212

1313
jobs:
14-
tests-ubuntu-20:
15-
name: Python ${{ matrix.python-version }} Ubuntu 20
16-
runs-on: ubuntu-20.04
14+
tests-ubuntu-latest:
15+
name: Python ${{ matrix.python-version }} Ubuntu Latest
16+
runs-on: ubuntu-latest
1717

1818
strategy:
1919
fail-fast: false
2020
matrix:
2121
python-version:
22-
- "2.7"
23-
- "3.5"
2422
- "3.6"
2523
- "3.7"
2624
- "3.8"
2725
- "3.9"
2826
- "3.10"
2927

28+
services:
29+
elasticsearch:
30+
image: elasticsearch:8.2.3
31+
ports:
32+
- 9200:9200
33+
env:
34+
discovery.type: single-node
35+
xpack.security.enabled: false
36+
37+
mongodb:
38+
image: mongo:4
39+
ports:
40+
- 27017:27017
41+
42+
redis:
43+
image: redis:6
44+
ports:
45+
- 6379:6379
46+
47+
env:
48+
ELASTICSEARCH_URL: http://localhost:9200/
49+
MONGODB_URL: mongodb://localhost:27017/
50+
REDIS_URL: redis://localhost:6379/0
51+
52+
steps:
53+
- uses: actions/checkout@v2
54+
- uses: actions/setup-python@v2
55+
with:
56+
python-version: ${{ matrix.python-version }}
57+
- name: Upgrade packaging tools
58+
run: python -m pip install --upgrade pip setuptools virtualenv
59+
- name: Install dependencies
60+
run: python -m pip install --upgrade tox
61+
- name: Run tox targets for ${{ matrix.python-version }}
62+
run: |
63+
ENV_PREFIX=$(tr -C -d "0-9" <<< "${{ matrix.python-version }}" | cut -c -3)
64+
TOXENV=$(tox --listenvs | grep "^py$ENV_PREFIX" | tr '\n' ',') python -m tox
65+
66+
tests-ubuntu-20:
67+
name: Python ${{ matrix.python-version }} Ubuntu 20
68+
runs-on: ubuntu-20.04
69+
70+
strategy:
71+
fail-fast: false
72+
matrix:
73+
python-version:
74+
- "2.7"
75+
- "3.5"
76+
3077
services:
3178
elasticsearch:
3279
image: elasticsearch:7.10.1

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
### Added
66
- Added Django 4.1 to the testing matrix.
7+
- Added support for Elasticsearch v8+.
8+
([Issue 725](https://github.com/scoutapp/scout_apm_python/issues/725))
79

810
### Fixed
911
- Catch SIGQUIT and SIGTERM errors when running core agent. They will now

‎DEVELOPMENT.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ by environment variables. These are:
7070

7171
* `ELASTICSEARCH_URL` - point to a running Elasticsearch instance, e.g.
7272
"http://localhost:9200/" . You can start it with:
73-
`docker run --detach --name elasticsearch --publish 9200:9200 -e "discovery.type=single-node" elasticsearch:7.10.1` .
73+
`docker run --detach --name elasticsearch --publish 9200:9200 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:8.2.3` .
7474
* `MONGODB_URL` - point to a running MongoDB instance e.g.
7575
"mongodb://localhost:27017/" . You can start it with:
7676
`docker run --detach --name mongo --publish 27017:27017 mongo:4.0` .

‎src/scout_apm/instruments/elasticsearch.py‎

Lines changed: 86 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,79 +4,104 @@
44
import logging
55
from collections import namedtuple
66

7+
import elasticsearch
78
import wrapt
89

910
from scout_apm.compat import get_pos_args, unwrap_decorators
1011
from scout_apm.core.tracked_request import TrackedRequest
1112

1213
try:
13-
from elasticsearch import Elasticsearch, Transport
14+
from elasticsearch import Elasticsearch
1415
except ImportError: # pragma: no cover
1516
Elasticsearch = None
16-
Transport = None
17+
18+
try:
19+
# Transport was moved to elastic_transport as of v8.0.0
20+
from elastic_transport import Transport
21+
except ImportError: # pragma: no cover
22+
try:
23+
from elasticsearch import Transport
24+
except ImportError: # pragma: no cover
25+
Transport = None
1726

1827
logger = logging.getLogger(__name__)
1928

2029

2130
def ensure_installed():
2231
logger.debug("Instrumenting elasticsearch.")
2332

24-
if Elasticsearch is None:
25-
logger.debug(
26-
"Couldn't import elasticsearch.Elasticsearch - probably not installed."
27-
)
33+
if Elasticsearch is None or Transport is None:
34+
if Elasticsearch is None:
35+
logger.debug(
36+
"Couldn't import elasticsearch.Elasticsearch "
37+
"- probably not installed."
38+
)
39+
if Transport is None:
40+
logger.debug(
41+
"Couldn't import elasticsearch.Transport or "
42+
"elastic_transport.Transport - probably not installed."
43+
)
2844
else:
2945
ensure_client_instrumented()
3046
ensure_transport_instrumented()
3147

3248

3349
ClientMethod = namedtuple("ClientMethod", ["name", "takes_index_argument"])
3450

35-
CLIENT_METHODS = [
36-
ClientMethod("bulk", True),
37-
ClientMethod("clear_scroll", False),
38-
ClientMethod("close", False),
39-
ClientMethod("close_point_in_time", False),
40-
ClientMethod("count", True),
41-
ClientMethod("create", True),
42-
ClientMethod("delete", True),
43-
ClientMethod("delete_by_query", True),
44-
ClientMethod("delete_by_query_rethrottle", False),
45-
ClientMethod("delete_script", False),
46-
ClientMethod("exists", True),
47-
ClientMethod("exists_source", True),
48-
ClientMethod("explain", True),
49-
ClientMethod("field_caps", True),
50-
ClientMethod("get", True),
51-
ClientMethod("get_script", False),
52-
ClientMethod("get_script_context", False),
53-
ClientMethod("get_script_languages", False),
54-
ClientMethod("get_source", True),
55-
ClientMethod("index", True),
56-
ClientMethod("info", False),
57-
ClientMethod("mget", True),
58-
ClientMethod("msearch", True),
59-
ClientMethod("msearch_template", True),
60-
ClientMethod("mtermvectors", True),
61-
ClientMethod("open_point_in_time", True),
62-
ClientMethod("ping", False),
63-
ClientMethod("put_script", False),
64-
ClientMethod("rank_eval", True),
65-
ClientMethod("reindex", False),
66-
ClientMethod("reindex_rethrottle", False),
67-
ClientMethod("render_search_template", False),
68-
ClientMethod("scripts_painless_execute", False),
69-
ClientMethod("scroll", False),
70-
ClientMethod("search", True),
71-
ClientMethod("search_mvt", True),
72-
ClientMethod("search_shards", True),
73-
ClientMethod("search_template", True),
74-
ClientMethod("termvectors", True),
75-
ClientMethod("terms_enum", True),
76-
ClientMethod("update", True),
77-
ClientMethod("update_by_query", True),
78-
ClientMethod("update_by_query_rethrottle", False),
79-
]
51+
VERSIONED_CLIENT_METHODS = {
52+
"v7": [
53+
ClientMethod("bulk", True),
54+
ClientMethod("clear_scroll", False),
55+
ClientMethod("close", False),
56+
ClientMethod("close_point_in_time", False),
57+
ClientMethod("count", True),
58+
ClientMethod("create", True),
59+
ClientMethod("delete", True),
60+
ClientMethod("delete_by_query", True),
61+
ClientMethod("delete_by_query_rethrottle", False),
62+
ClientMethod("delete_script", False),
63+
ClientMethod("exists", True),
64+
ClientMethod("exists_source", True),
65+
ClientMethod("explain", True),
66+
ClientMethod("field_caps", True),
67+
ClientMethod("get", True),
68+
ClientMethod("get_script", False),
69+
ClientMethod("get_script_context", False),
70+
ClientMethod("get_script_languages", False),
71+
ClientMethod("get_source", True),
72+
ClientMethod("index", True),
73+
ClientMethod("info", False),
74+
ClientMethod("mget", True),
75+
ClientMethod("msearch", True),
76+
ClientMethod("msearch_template", True),
77+
ClientMethod("mtermvectors", True),
78+
ClientMethod("open_point_in_time", True),
79+
ClientMethod("ping", False),
80+
ClientMethod("put_script", False),
81+
ClientMethod("rank_eval", True),
82+
ClientMethod("reindex", False),
83+
ClientMethod("reindex_rethrottle", False),
84+
ClientMethod("render_search_template", False),
85+
ClientMethod("scripts_painless_execute", False),
86+
ClientMethod("scroll", False),
87+
ClientMethod("search", True),
88+
ClientMethod("search_mvt", True),
89+
ClientMethod("search_shards", True),
90+
ClientMethod("search_template", True),
91+
ClientMethod("termvectors", True),
92+
ClientMethod("terms_enum", True),
93+
ClientMethod("update", True),
94+
ClientMethod("update_by_query", True),
95+
ClientMethod("update_by_query_rethrottle", False),
96+
],
97+
"v8": [
98+
ClientMethod("knn_search", True),
99+
],
100+
}
101+
102+
CLIENT_METHODS = VERSIONED_CLIENT_METHODS["v7"][:]
103+
if elasticsearch.VERSION > (8, 0, 0):
104+
CLIENT_METHODS += VERSIONED_CLIENT_METHODS["v8"]
80105

81106

82107
have_patched_client = False
@@ -185,8 +210,11 @@ def _sanitize_name(name):
185210
"bench",
186211
"bulk",
187212
"count",
213+
"delete_by_query",
214+
"execute",
188215
"exists",
189216
"explain",
217+
"field_caps",
190218
"field_stats",
191219
"health",
192220
"mget",
@@ -195,14 +223,21 @@ def _sanitize_name(name):
195223
"msearch",
196224
"mtermvectors",
197225
"percolate",
226+
"pit",
198227
"query",
228+
"rank_eval",
229+
"reindex",
230+
"script_context",
231+
"script_language",
199232
"scroll",
200233
"search_shards",
201234
"source",
202235
"suggest",
203236
"template",
204237
"termvectors",
238+
"terms_enum",
205239
"update",
240+
"update_by_query",
206241
"search",
207242
)
208243
if op in known_names:

0 commit comments

Comments
 (0)