Skip to content

Commit 75dd021

Browse files
ahsimbahsimb
committed
Merge remote-tracking branch 'origin/main'
2 parents d41ce67 + 296d0ac commit 75dd021

12 files changed

+119
-61
lines changed

‎doc/changes/changelog.md

+2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# 📝 Changes
22

33
* [unreleased](unreleased.md)
4+
* [0.6.0](changes_0.6.0.md)
45
* [0.5.0](changes_0.5.0.md)
56
* [0.4.0](changes_0.4.0.md)
67
* [0.3.1](changes_0.3.1.md)
@@ -13,6 +14,7 @@
1314
hidden:
1415
---
1516
unreleased
17+
changes_0.6.0
1618
changes_0.5.0
1719
changes_0.4.0
1820
changes_0.3.1

‎doc/changes/changes_0.6.0.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# 0.6.0 - 2024-09-23
2+
3+
This release allows using the `extract_validator` for SLCs, even when lacking the permissions to create a database schema.
4+
5+
## Features
6+
7+
* #41: Make `temp_schema optional` for `wait_for_completion`.

‎doc/user_guide/user-guide.md

+30-29
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,36 @@ an option in the command line, without providing its value. In this case, the co
3232
interactively. For long values, such as the SaaS account id, it is more practical to copy/paste the value from
3333
another source.
3434

35-
| Option name | On-Prem | SaaS | Comment |
36-
|:-----------------------------|:-------:|:----:|:--------------------------------------------------|
37-
| dsn | [x] | | i.e. <db_host:db_port> |
38-
| db-user | [x] | | |
39-
| db-pass | [x] | | Env. [DB_PASSWORD] |
40-
| bucketfs-name | [x] | | |
41-
| bucketfs-host | [x] | | |
42-
| bucketfs-port | [x] | | |
43-
| bucketfs-user | [x] | | |
44-
| bucketfs-password | [x] | | Env. [BUCKETFS_PASSWORD] |
45-
| bucketfs-use-https | [x] | | Optional boolean, defaults to False |
46-
| bucket | [x] | | |
47-
| saas-url | | [x] | Env. [SAAS_HOST] |
48-
| saas-account-id | | [x] | Env. [SAAS_ACCOUNT_ID] |
49-
| saas-database-id | | [x] | Optional, Env. [SAAS_DATABASE_ID] |
50-
| saas-database-name | | [x] | Optional, provide if the database_id is unknown |
51-
| saas-token | | [x] | Env. [SAAS_TOKEN] |
52-
| path-in-bucket | [x] | [x] | |
53-
| language-alias | [x] | [x] | |
54-
| version | [x] | [x] | Optional, provide for downloading SLC from GitHub |
55-
| container-file | [x] | [x] | Optional, provide for uploading SLC file |
56-
| ssl-cert-path | [x] | [x] | Optional |
57-
| [no_]use-ssl-cert-validation | [x] | [x] | Optional boolean, defaults to True |
58-
| ssl-client-cert-path | [x] | | Optional |
59-
| ssl-client-private-key | [x] | | Optional |
60-
| [no_]upload-container | [x] | [x] | Optional boolean, defaults to True |
61-
| [no_]alter-system | [x] | [x] | Optional boolean, defaults to True |
62-
| [dis]allow-override | [x] | [x] | Optional boolean, defaults to False |
63-
| [no_]wait_for_completion | [x] | [x] | Optional boolean, defaults to True |
35+
| Option name | On-Prem | SaaS | Comment |
36+
|:-----------------------------|:-------:|:----:|:--------------------------------------------------------|
37+
| dsn | [x] | | i.e. <db_host:db_port> |
38+
| db-user | [x] | | |
39+
| db-pass | [x] | | Env. [DB_PASSWORD] |
40+
| bucketfs-name | [x] | | |
41+
| bucketfs-host | [x] | | |
42+
| bucketfs-port | [x] | | |
43+
| bucketfs-user | [x] | | |
44+
| bucketfs-password | [x] | | Env. [BUCKETFS_PASSWORD] |
45+
| bucketfs-use-https | [x] | | Optional boolean, defaults to False |
46+
| bucket | [x] | | |
47+
| saas-url | | [x] | Env. [SAAS_HOST] |
48+
| saas-account-id | | [x] | Env. [SAAS_ACCOUNT_ID] |
49+
| saas-database-id | | [x] | Optional, Env. [SAAS_DATABASE_ID] |
50+
| saas-database-name | | [x] | Optional, provide if the database_id is unknown |
51+
| saas-token | | [x] | Env. [SAAS_TOKEN] |
52+
| path-in-bucket | [x] | [x] | |
53+
| language-alias | [x] | [x] | |
54+
| schema | [x] | [x] | Required if the user has no permission to create a database schema |
55+
| version | [x] | [x] | Optional, provide for downloading SLC from GitHub |
56+
| container-file | [x] | [x] | Optional, provide for uploading SLC file |
57+
| ssl-cert-path | [x] | [x] | Optional |
58+
| [no_]use-ssl-cert-validation | [x] | [x] | Optional boolean, defaults to True |
59+
| ssl-client-cert-path | [x] | | Optional |
60+
| ssl-client-private-key | [x] | | Optional |
61+
| [no_]upload-container | [x] | [x] | Optional boolean, defaults to True |
62+
| [no_]alter-system | [x] | [x] | Optional boolean, defaults to True |
63+
| [dis]allow-override | [x] | [x] | Optional boolean, defaults to False |
64+
| [no_]wait_for_completion | [x] | [x] | Optional boolean, defaults to True |
6465

6566
### Container selection
6667

‎exasol/python_extension_common/deployment/language_container_deployer.py

+24-4
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import exasol.bucketfs as bfs # type: ignore
1212
from exasol.saas.client.api_access import (get_connection_params, get_database_id) # type: ignore
1313

14-
from exasol.python_extension_common.deployment.temp_schema import temp_schema
14+
from exasol.python_extension_common.deployment.temp_schema import temp_schema, get_schema
1515
from exasol.python_extension_common.deployment.extract_validator import ExtractValidator
1616

1717

@@ -162,6 +162,9 @@ def run(self, container_file: Optional[Path] = None,
162162
allow_override - If True the activation of a language container with the same alias will be
163163
overriden, otherwise a RuntimeException will be thrown.
164164
wait_for_completion - If True will wait until the language container becomes operational.
165+
For this to work either of the two conditions should be met.
166+
The pyexasol connection should have an open schema, or
167+
The calling user should have a permission to create schema.
165168
"""
166169

167170
if not bucket_file_path:
@@ -182,9 +185,7 @@ def run(self, container_file: Optional[Path] = None,
182185
# Optionally wait until the container is extracted on all nodes of the
183186
# database cluster.
184187
if container_file and wait_for_completion:
185-
with temp_schema(self._pyexasol_conn) as schema:
186-
self._extract_validator.verify_all_nodes(
187-
schema, self._language_alias, self._upload_path(bucket_file_path))
188+
self._wait_container_upload_completion(bucket_file_path)
188189

189190
if not alter_system:
190191
message = dedent(f"""
@@ -248,6 +249,23 @@ def generate_activation_command(self, bucket_file_path: str,
248249
f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';"
249250
return alter_command
250251

252+
def _wait_container_upload_completion(self, bucket_file_path: str):
253+
"""
254+
The function waits till the container is fully uploaded and operational on all nodes.
255+
It creates and then subsequently deletes a simple UDF that checks for the presence of
256+
a certain file. This UDF is created in the current schema of the pyexasol connection
257+
if one is open. Otherwise, a temporary schema will be created.
258+
"""
259+
upload_path = self._upload_path(bucket_file_path)
260+
schema = get_schema(self._pyexasol_conn)
261+
if schema:
262+
self._extract_validator.verify_all_nodes(
263+
schema, self._language_alias, upload_path)
264+
else:
265+
with temp_schema(self._pyexasol_conn) as schema:
266+
self._extract_validator.verify_all_nodes(
267+
schema, self._language_alias, upload_path)
268+
251269
def _update_previous_language_settings(self, alter_type: LanguageActivationLevel,
252270
allow_override: bool,
253271
path_in_udf: PurePosixPath) -> str:
@@ -299,6 +317,7 @@ def _check_if_requested_language_alias_already_exists(
299317
@classmethod
300318
def create(cls,
301319
language_alias: str, dsn: Optional[str] = None,
320+
schema: str = '',
302321
db_user: Optional[str] = None, db_password: Optional[str] = None,
303322
bucketfs_host: Optional[str] = None, bucketfs_port: Optional[int] = None,
304323
bucketfs_name: Optional[str] = None, bucket: Optional[str] = None,
@@ -360,6 +379,7 @@ def create(cls,
360379
ssl_client_certificate, ssl_private_key)
361380

362381
pyexasol_conn = pyexasol.connect(**connection_params,
382+
schema=schema,
363383
encryption=True,
364384
websocket_sslopt=websocket_sslopt)
365385

‎exasol/python_extension_common/deployment/language_container_deployer_cli.py

+3
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ def secret_callback(ctx: click.Context, param: click.Option, value: Any):
144144
prompt='DB password', prompt_required=False,
145145
hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
146146
@click.option('--language-alias', type=str, default="PYTHON3_EXT")
147+
@click.option('--schema', type=str, default="")
147148
@click.option('--ssl-cert-path', type=str, default="")
148149
@click.option('--ssl-client-cert-path', type=str, default="")
149150
@click.option('--ssl-client-private-key', type=str, default="")
@@ -171,6 +172,7 @@ def language_container_deployer_main(
171172
db_user: str,
172173
db_pass: str,
173174
language_alias: str,
175+
schema: str,
174176
ssl_cert_path: str,
175177
ssl_client_cert_path: str,
176178
ssl_client_private_key: str,
@@ -200,6 +202,7 @@ def language_container_deployer_main(
200202
db_user=db_user,
201203
db_password=db_pass,
202204
language_alias=language_alias,
205+
schema=schema,
203206
ssl_trusted_ca=ssl_cert_path,
204207
ssl_client_certificate=ssl_client_cert_path,
205208
ssl_private_key=ssl_client_private_key,

‎exasol/python_extension_common/deployment/temp_schema.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,18 @@ def _create_random_schema(conn: pyexasol.ExaConnection, schema_name_length: int)
2222
return schema
2323

2424

25-
def _get_schema(conn: pyexasol.ExaConnection) -> str | None:
25+
def get_schema(conn: pyexasol.ExaConnection) -> str | None:
2626
return conn.execute(f"SELECT CURRENT_SCHEMA;").fetchval()
2727

2828

29-
def _set_schema(conn: pyexasol.ExaConnection, schema: str | None):
29+
def set_schema(conn: pyexasol.ExaConnection, schema: str | None):
3030
if schema:
3131
conn.execute(f'OPEN SCHEMA "{schema}";')
3232
else:
3333
conn.execute("CLOSE SCHEMA;")
3434

3535

36-
def _delete_schema(conn: pyexasol.ExaConnection, schema: str) -> None:
36+
def delete_schema(conn: pyexasol.ExaConnection, schema: str) -> None:
3737
sql = f'DROP SCHEMA IF EXISTS "{schema}" CASCADE;'
3838
conn.execute(query=sql)
3939

@@ -50,11 +50,11 @@ def temp_schema(conn: pyexasol.ExaConnection,
5050
conn - pyexasol connection.
5151
schema_name_length - Number of characters in the temporary schema name.
5252
"""
53-
current_schema = _get_schema(conn)
53+
current_schema = get_schema(conn)
5454
schema = ''
5555
try:
5656
schema = _create_random_schema(conn, schema_name_length)
5757
yield schema
5858
finally:
59-
_delete_schema(conn, schema)
60-
_set_schema(conn, current_schema)
59+
delete_schema(conn, schema)
60+
set_schema(conn, current_schema)

‎poetry.lock

+13-13
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "exasol-python-extension-common"
3-
version = "0.5.0"
3+
version = "0.6.0"
44
description = "A collection of common utilities for Exasol extensions."
55
packages = [ {include = "exasol"}, ]
66
authors = ["Mikhail Beck <[email protected]>"]

‎test/integration/conftest.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from exasol.python_extension_common.deployment.language_container_deployer_cli import (
1313
language_container_deployer_main, slc_parameter_formatters, CustomizableParameters)
1414
from test.utils.revert_language_settings import revert_language_settings
15-
from test.utils.db_utils import create_schema
15+
from test.utils.db_utils import create_schema, open_schema
1616

1717

1818
SLC_NAME = "template-Exasol-all-python-3.10_release.tar.gz"
@@ -84,12 +84,14 @@ def deployer_factory(
8484
db_schema,
8585
language_alias):
8686
@contextmanager
87-
def create_deployer(create_test_schema: bool = False):
87+
def create_deployer(create_test_schema: bool = False, open_test_schema: bool = True):
8888
with ExitStack() as stack:
8989
pyexasol_connection = stack.enter_context(pyexasol.connect(**backend_aware_database_params))
9090
bucketfs_path = bfs.path.build_path(**backend_aware_bucketfs_params)
9191
stack.enter_context(revert_language_settings(pyexasol_connection))
9292
if create_test_schema:
93-
create_schema(pyexasol_connection, db_schema)
93+
create_schema(pyexasol_connection, db_schema, open_test_schema)
94+
elif open_test_schema:
95+
open_schema(pyexasol_connection, db_schema)
9496
yield LanguageContainerDeployer(pyexasol_connection, language_alias, bucketfs_path)
9597
return create_deployer

‎test/integration/test_language_container_deployer.py

+18-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
LanguageContainerDeployer,
1010
LanguageActivationLevel,
1111
)
12+
from exasol.python_extension_common.deployment.temp_schema import get_schema
1213
from test.utils.db_utils import assert_udf_running
1314

1415

@@ -19,8 +20,24 @@ def test_container_file(deployer_factory,
1920
"""
2021
Tests the deployment of a container in one call, including the activation at the System level.
2122
"""
22-
with deployer_factory(create_test_schema=True) as deployer:
23+
with deployer_factory(create_test_schema=True, open_test_schema=False) as deployer:
2324
deployer.run(container_file=Path(container_path), alter_system=True, allow_override=True)
25+
# Make sure the deployer's connection doesn't have an open schema.
26+
assert not get_schema(deployer.pyexasol_connection)
27+
assert_udf_running(deployer.pyexasol_connection, language_alias, db_schema)
28+
29+
30+
def test_container_file_no_temp_schema(deployer_factory,
31+
db_schema,
32+
language_alias,
33+
container_path):
34+
"""
35+
Tests the deployment of a container without creating a temporary schema for the upload completion test.
36+
"""
37+
with deployer_factory(create_test_schema=True, open_test_schema=True) as deployer:
38+
deployer.run(container_file=Path(container_path), alter_system=True, allow_override=True)
39+
# Make sure the deployer's connection has an open schema.
40+
assert get_schema(deployer.pyexasol_connection)
2441
assert_udf_running(deployer.pyexasol_connection, language_alias, db_schema)
2542

2643

‎test/utils/db_utils.py

+9-3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,15 @@
33
from pyexasol import ExaConnection
44

55

6-
def create_schema(pyexasol_connection: ExaConnection, schema: str):
7-
pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;")
8-
pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};")
6+
def create_schema(pyexasol_connection: ExaConnection, schema: str, open_test_schema: bool = True):
7+
pyexasol_connection.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE;')
8+
pyexasol_connection.execute(f'CREATE SCHEMA "{schema}";')
9+
if not open_test_schema:
10+
pyexasol_connection.execute("CLOSE SCHEMA;")
11+
12+
13+
def open_schema(pyexasol_connection: ExaConnection, schema: str):
14+
pyexasol_connection.execute(f'OPEN SCHEMA "{schema}";')
915

1016

1117
def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str, schema: str):

‎version.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
# Do not edit this file manually!
66
# If you need to change the version, do so in the project.toml, e.g. by using `poetry version X.Y.Z`.
77
MAJOR = 0
8-
MINOR = 5
8+
MINOR = 6
99
PATCH = 0
1010
VERSION = f"{MAJOR}.{MINOR}.{PATCH}"

0 commit comments

Comments
 (0)