From 1967d0dba00c35aa31bfb016cfe28b6661e6f0ec Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Mon, 19 Sep 2022 14:21:30 +0100 Subject: [PATCH 01/16] models action from v1 to v2 --- examples/delete_model.py | 3 +-- examples/install_model.py | 2 +- railib/api.py | 38 ++++++++++++++++++++------------------ 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/examples/delete_model.py b/examples/delete_model.py index c50186c..3d18012 100644 --- a/examples/delete_model.py +++ b/examples/delete_model.py @@ -24,8 +24,7 @@ def run(database: str, engine: str, model: str, profile: str): cfg = config.read(profile=profile) ctx = api.Context(**cfg) rsp = api.delete_model(ctx, database, engine, model) - print(json.dumps(rsp, indent=2)) - + print(rsp) if __name__ == "__main__": p = ArgumentParser() diff --git a/examples/install_model.py b/examples/install_model.py index 8d6937e..274c129 100644 --- a/examples/install_model.py +++ b/examples/install_model.py @@ -33,7 +33,7 @@ def run(database: str, engine: str, fname: str, profile: str): cfg = config.read(profile=profile) ctx = api.Context(**cfg) rsp = api.install_model(ctx, database, engine, models) - print(json.dumps(rsp, indent=2)) + print(rsp) if __name__ == "__main__": diff --git a/railib/api.py b/railib/api.py index 3574401..9c4e106 100644 --- a/railib/api.py +++ b/railib/api.py @@ -658,15 +658,17 @@ def _model(name: str, model: str) -> dict: # Returns full list of models. -def _list_models(ctx: Context, database: str, engine: str) -> dict: - tx = Transaction(database, engine, mode=Mode.OPEN) - rsp = tx.run(ctx, _list_action()) - actions = rsp["actions"] - assert len(actions) == 1 - action = actions[0] - models = action["result"]["sources"] - return models +def _list_models(ctx: Context, database: str, engine: str) -> List: + models = [] + response = exec(ctx, database, engine, "def output:__models__ = rel:catalog:model") + for result in response.results: + if '/:output/:__models__' in result['relationId']: + table = result['table'].to_pydict() + models.extend([{'name': table['v1'][i], 'value': table['v2'][i]} for i in range(1, len(table['v1']))]) + if len(response.problems) > 0: + print(response.problems) + return models def create_database(ctx: Context, database: str, source: str = None) -> dict: data = {"name": database, "source_name": source} @@ -674,12 +676,11 @@ def create_database(ctx: Context, database: str, source: str = None) -> dict: rsp = rest.put(ctx, url, data) return json.loads(rsp.read()) +def delete_model(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: + return exec(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) -def delete_model(ctx: Context, database: str, engine: str, model: str) -> dict: - tx = Transaction(database, engine, mode=Mode.OPEN, readonly=False) - actions = [_delete_model_action(model)] - return tx.run(ctx, *actions) - +def delete_model_async(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: + return exec_async(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) # Returns the named model def get_model(ctx: Context, database: str, engine: str, name: str) -> str: @@ -689,12 +690,13 @@ def get_model(ctx: Context, database: str, engine: str, name: str) -> str: return model["value"] raise Exception(f"model '{name}' not found") +def install_model(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: + queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] + return exec(ctx, database, engine, '\n'.join(queries), readonly=False) -def install_model(ctx: Context, database: str, engine: str, models: dict) -> dict: - tx = Transaction(database, engine, mode=Mode.OPEN, readonly=False) - actions = [_install_model_action(name, model) for name, model in models.items()] - return tx.run(ctx, *actions) - +def install_model_async(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: + queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] + return exec_async(ctx, database, engine, '\n'.join(queries), readonly=False) def list_edbs(ctx: Context, database: str, engine: str) -> list: tx = Transaction(database, engine, mode=Mode.OPEN) From ead5b03324a4aeb22b08e467115393f62635a5f1 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 20 Sep 2022 05:15:45 +0100 Subject: [PATCH 02/16] adding integration tests --- tests/integration.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration.py b/tests/integration.py index 3e34791..8eba4e8 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -90,6 +90,26 @@ def tearDown(self): api.delete_engine(ctx, engine) api.delete_database(ctx, dbname) +class TestModels(unittest.TestCase): + def setUp(self): + create_engine_wait(ctx, engine) + api.create_database(ctx, dbname) + + def test_models(self): + models = api.list_models(ctx, dbname, engine) + self.assertTrue(len(models) > 0) + + resp = api.install_model(ctx, dbname, engine, {"test_model": "def foo=:bar"}) + self.assertEqual(resp.transaction["state"], "COMPLETED") + + models = api.list_models(ctx, dbname, engine) + self.assertTrue("test_model" in models) + + resp = api.delete_model(ctx, dbname, engine, "test_model") + self.assertEqual(resp.transaction["state"], "COMPLETED") + + models = api.list_models(ctx, dbname, engine) + self.assertFalse("test_model" in models) if __name__ == '__main__': unittest.main() From 806fde3ef7f3a9794072612e53d6613efe34e267 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 20 Sep 2022 05:19:31 +0100 Subject: [PATCH 03/16] fix linter --- examples/delete_model.py | 2 +- examples/install_model.py | 1 - railib/api.py | 7 +++++++ tests/integration.py | 6 ++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/delete_model.py b/examples/delete_model.py index 3d18012..b2c6daa 100644 --- a/examples/delete_model.py +++ b/examples/delete_model.py @@ -15,7 +15,6 @@ """Delete the given Rel model from the given database.""" from argparse import ArgumentParser -import json from urllib.request import HTTPError from railib import api, config, show @@ -26,6 +25,7 @@ def run(database: str, engine: str, model: str, profile: str): rsp = api.delete_model(ctx, database, engine, model) print(rsp) + if __name__ == "__main__": p = ArgumentParser() p.add_argument("database", type=str, help="database name") diff --git a/examples/install_model.py b/examples/install_model.py index 274c129..9918f9a 100644 --- a/examples/install_model.py +++ b/examples/install_model.py @@ -15,7 +15,6 @@ """Install the given Rel model in the given database""" from argparse import ArgumentParser -import json from os import path from urllib.request import HTTPError from railib import api, config, show diff --git a/railib/api.py b/railib/api.py index 9c4e106..8734a56 100644 --- a/railib/api.py +++ b/railib/api.py @@ -670,18 +670,22 @@ def _list_models(ctx: Context, database: str, engine: str) -> List: return models + def create_database(ctx: Context, database: str, source: str = None) -> dict: data = {"name": database, "source_name": source} url = _mkurl(ctx, PATH_DATABASE) rsp = rest.put(ctx, url, data) return json.loads(rsp.read()) + def delete_model(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: return exec(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) + def delete_model_async(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: return exec_async(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) + # Returns the named model def get_model(ctx: Context, database: str, engine: str, name: str) -> str: models = _list_models(ctx, database, engine) @@ -690,14 +694,17 @@ def get_model(ctx: Context, database: str, engine: str, name: str) -> str: return model["value"] raise Exception(f"model '{name}' not found") + def install_model(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] return exec(ctx, database, engine, '\n'.join(queries), readonly=False) + def install_model_async(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] return exec_async(ctx, database, engine, '\n'.join(queries), readonly=False) + def list_edbs(ctx: Context, database: str, engine: str) -> list: tx = Transaction(database, engine, mode=Mode.OPEN) rsp = tx.run(ctx, _list_edb_action()) diff --git a/tests/integration.py b/tests/integration.py index 8eba4e8..738a491 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -90,6 +90,7 @@ def tearDown(self): api.delete_engine(ctx, engine) api.delete_database(ctx, dbname) + class TestModels(unittest.TestCase): def setUp(self): create_engine_wait(ctx, engine) @@ -111,5 +112,10 @@ def test_models(self): models = api.list_models(ctx, dbname, engine) self.assertFalse("test_model" in models) + def tearDown(self): + api.delete_engine(ctx, engine) + api.delete_database(ctx, dbname) + + if __name__ == '__main__': unittest.main() From e048fd6a7d1e4880cdac6c4ba72d5dd997ab5d47 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 20 Sep 2022 05:31:16 +0100 Subject: [PATCH 04/16] fix tests --- tests/integration.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index 738a491..85ba401 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -86,16 +86,6 @@ def test_v2_exec(self): 1, 8, 27, 64, 125], 'v4': [ 1, 16, 81, 256, 625]}, rsp.results[0]["table"].to_pydict()) - def tearDown(self): - api.delete_engine(ctx, engine) - api.delete_database(ctx, dbname) - - -class TestModels(unittest.TestCase): - def setUp(self): - create_engine_wait(ctx, engine) - api.create_database(ctx, dbname) - def test_models(self): models = api.list_models(ctx, dbname, engine) self.assertTrue(len(models) > 0) From e081c04f2a3b50a0954aedb7ba9a32496b2e2e8b Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 21 Sep 2022 02:28:55 +0100 Subject: [PATCH 05/16] fix integration tests --- tests/integration.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index 85ba401..eba13ae 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -49,15 +49,13 @@ def create_engine_wait(ctx: api.Context, engine: str): ctx = api.Context(**cfg) -suffix = uuid.uuid4() -engine = f"python-sdk-{suffix}" -dbname = f"python-sdk-{suffix}" - - class TestTransactionAsync(unittest.TestCase): def setUp(self): - create_engine_wait(ctx, engine) - api.create_database(ctx, dbname) + self.suffix = uuid.uuid4() + self.engine = f"python-sdk-{self.suffix}" + self.dbname = f"python-sdk-{self.suffix}" + create_engine_wait(ctx, self.engine) + api.create_database(ctx, self.dbname) def test_v2_exec(self): cmd = "x, x^2, x^3, x^4 from x in {1; 2; 3; 4; 5}" @@ -86,26 +84,38 @@ def test_v2_exec(self): 1, 8, 27, 64, 125], 'v4': [ 1, 16, 81, 256, 625]}, rsp.results[0]["table"].to_pydict()) + def tearDown(self): + api.delete_engine(ctx, self.engine) + api.delete_database(ctx, self.dbname) + + +class TestModels(unittest.TestCase): + def setUp(self): + self.suffix = uuid.uuid4() + self.engine = f"python-sdk-{self.suffix}" + self.dbname = f"python-sdk-{self.suffix}" + create_engine_wait(ctx, self.engine) + api.create_database(ctx, self.dbname) + def test_models(self): - models = api.list_models(ctx, dbname, engine) + models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue(len(models) > 0) - resp = api.install_model(ctx, dbname, engine, {"test_model": "def foo=:bar"}) + resp = api.install_model(ctx, self.dbname, self.engine, {"test_model": "def foo=:bar"}) self.assertEqual(resp.transaction["state"], "COMPLETED") - models = api.list_models(ctx, dbname, engine) + models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue("test_model" in models) - resp = api.delete_model(ctx, dbname, engine, "test_model") + resp = api.delete_model(ctx, self.dbname, self.engine, "test_model") self.assertEqual(resp.transaction["state"], "COMPLETED") - models = api.list_models(ctx, dbname, engine) + models = api.list_models(ctx, self.dbname, self.engine) self.assertFalse("test_model" in models) def tearDown(self): - api.delete_engine(ctx, engine) - api.delete_database(ctx, dbname) - + api.delete_engine(ctx, self.engine) + api.delete_database(ctx, self.dbname) if __name__ == '__main__': unittest.main() From a87fe47486e19115ea08ce7d0c6bfe87a432633e Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 21 Sep 2022 10:27:25 +0100 Subject: [PATCH 06/16] fix linter --- tests/integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration.py b/tests/integration.py index eba13ae..32625e6 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -49,6 +49,7 @@ def create_engine_wait(ctx: api.Context, engine: str): ctx = api.Context(**cfg) + class TestTransactionAsync(unittest.TestCase): def setUp(self): self.suffix = uuid.uuid4() @@ -117,5 +118,6 @@ def tearDown(self): api.delete_engine(ctx, self.engine) api.delete_database(ctx, self.dbname) + if __name__ == '__main__': unittest.main() From cb85a1ae061e1cd9a667081b0e0deb764f4c9f45 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 21 Sep 2022 12:28:40 +0100 Subject: [PATCH 07/16] cleanup --- tests/integration.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index 32625e6..79bf51e 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -60,7 +60,7 @@ def setUp(self): def test_v2_exec(self): cmd = "x, x^2, x^3, x^4 from x in {1; 2; 3; 4; 5}" - rsp = api.exec(ctx, "hnr-db", "hnr-engine", cmd) + rsp = api.exec(ctx, self.dbname, self.engine, cmd) # transaction self.assertEqual("COMPLETED", rsp.transaction["state"]) @@ -79,24 +79,12 @@ def test_v2_exec(self): # results self.assertEqual( { - 'v1': [ - 1, 2, 3, 4, 5], 'v2': [ - 1, 4, 9, 16, 25], 'v3': [ - 1, 8, 27, 64, 125], 'v4': [ - 1, 16, 81, 256, 625]}, rsp.results[0]["table"].to_pydict()) - - def tearDown(self): - api.delete_engine(ctx, self.engine) - api.delete_database(ctx, self.dbname) - - -class TestModels(unittest.TestCase): - def setUp(self): - self.suffix = uuid.uuid4() - self.engine = f"python-sdk-{self.suffix}" - self.dbname = f"python-sdk-{self.suffix}" - create_engine_wait(ctx, self.engine) - api.create_database(ctx, self.dbname) + 'v1': [1, 2, 3, 4, 5], + 'v2': [1, 4, 9, 16, 25], + 'v3': [1, 8, 27, 64, 125], + 'v4': [1, 16, 81, 256, 625] + }, + rsp.results[0]["table"].to_pydict()) def test_models(self): models = api.list_models(ctx, self.dbname, self.engine) From 5abc0f9a43aeee10e8d5eff638579e9272324018 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Mon, 26 Sep 2022 15:17:24 +0100 Subject: [PATCH 08/16] cleanup && refactoring --- .../{delete_model.py => delete_models.py} | 2 +- .../{install_model.py => install_models.py} | 2 +- examples/run-all | 4 +- railib/api.py | 60 ++++++++++--------- tests/integration.py | 4 +- 5 files changed, 37 insertions(+), 35 deletions(-) rename examples/{delete_model.py => delete_models.py} (95%) rename examples/{install_model.py => install_models.py} (96%) diff --git a/examples/delete_model.py b/examples/delete_models.py similarity index 95% rename from examples/delete_model.py rename to examples/delete_models.py index b2c6daa..60af688 100644 --- a/examples/delete_model.py +++ b/examples/delete_models.py @@ -22,7 +22,7 @@ def run(database: str, engine: str, model: str, profile: str): cfg = config.read(profile=profile) ctx = api.Context(**cfg) - rsp = api.delete_model(ctx, database, engine, model) + rsp = api.delete_models(ctx, database, engine, model) print(rsp) diff --git a/examples/install_model.py b/examples/install_models.py similarity index 96% rename from examples/install_model.py rename to examples/install_models.py index 9918f9a..c447288 100644 --- a/examples/install_model.py +++ b/examples/install_models.py @@ -31,7 +31,7 @@ def run(database: str, engine: str, fname: str, profile: str): models[_sansext(fname)] = fp.read() # model name => model cfg = config.read(profile=profile) ctx = api.Context(**cfg) - rsp = api.install_model(ctx, database, engine, models) + rsp = api.install_models(ctx, database, engine, models) print(rsp) diff --git a/examples/run-all b/examples/run-all index 07810dd..7fc9a39 100755 --- a/examples/run-all +++ b/examples/run-all @@ -36,7 +36,7 @@ python3 ./show_results.py $DATABASE $ENGINE python3 ./show_problems.py $DATABASE $ENGINE # load model -python3 ./install_model.py $DATABASE $ENGINE hello.rel +python3 ./install_models.py $DATABASE $ENGINE hello.rel python3 ./get_model.py $DATABASE $ENGINE hello python3 ./list_models.py $DATABASE $ENGINE python3 ./delete_model.py $DATABASE $ENGINE hello @@ -60,7 +60,7 @@ python3 ./list_edbs.py $DATABASE $ENGINE python3 ./delete_database.py $DATABASE python3 ./create_database.py $DATABASE python3 ./load_json.py $DATABASE $ENGINE sample.json -r sample_json -python3 ./install_model.py $DATABASE $ENGINE hello.rel +python3 ./install_models.py $DATABASE $ENGINE hello.rel python3 ./clone_database.py $DATABASE_CLONE $DATABASE python3 ./get_database.py $DATABASE_CLONE python3 ./list_databases.py diff --git a/railib/api.py b/railib/api.py index 8734a56..204b92b 100644 --- a/railib/api.py +++ b/railib/api.py @@ -19,9 +19,13 @@ import time import re import io +import sys +import random + from enum import Enum, unique from typing import List, Union from requests_toolbelt import multipart + from . import rest from .pb.message_pb2 import MetadataInfo @@ -114,7 +118,7 @@ class Permission(str, Enum): "create_oauth_client", "delete_database", "delete_engine", - "delete_model", + "delete_models", "disable_user", "enable_user", "delete_oauth_client", @@ -598,18 +602,6 @@ def run(self, ctx: Context, command: str, language: str, inputs: dict = None) -> raise Exception("invalid response type") -def _delete_model_action(name: str) -> dict: - return {"type": "ModifyWorkspaceAction", "delete_source": [name]} - - -def _install_model_action(name: str, model: str) -> dict: - return {"type": "InstallAction", "sources": [_model(name, model)]} - - -def _list_action(): - return {"type": "ListSourceAction"} - - def _list_edb_action(): return {"type": "ListEdbAction"} @@ -660,13 +652,12 @@ def _model(name: str, model: str) -> dict: # Returns full list of models. def _list_models(ctx: Context, database: str, engine: str) -> List: models = [] - response = exec(ctx, database, engine, "def output:__models__ = rel:catalog:model") - for result in response.results: - if '/:output/:__models__' in result['relationId']: + out_name = f'model{random.randint(0, sys.maxsize)}' + resp = exec(ctx, database, engine, f'def output:{out_name} = rel:catalog:model') + for result in resp.results: + if '/:output/:{out_name}' in result['relationId']: table = result['table'].to_pydict() models.extend([{'name': table['v1'][i], 'value': table['v2'][i]} for i in range(1, len(table['v1']))]) - if len(response.problems) > 0: - print(response.problems) return models @@ -678,29 +669,40 @@ def create_database(ctx: Context, database: str, source: str = None) -> dict: return json.loads(rsp.read()) -def delete_model(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: - return exec(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) +def delete_models(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: + queries = [ + f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' + for model_name in models + ] + return exec(ctx, database, engine, '\n'.join(queries), readonly=False) -def delete_model_async(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: - return exec_async(ctx, database, engine, f'def delete:rel:catalog:model["{model}"] = rel:catalog:model["{model}"]', readonly=False) +def delete_model_async(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: + queries = [ + f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' + for model_name in models + ] + return exec_async(ctx, database, engine, '\n'.join(queries), readonly=False) # Returns the named model def get_model(ctx: Context, database: str, engine: str, name: str) -> str: - models = _list_models(ctx, database, engine) - for model in models: - if model["name"] == name: - return model["value"] + out_name = f'model{random.randint(0, sys.maxsize)}' + cmd = f'def output:{out_name} = rel:catalog:model["{name}"]' + resp = exec(ctx, database, engine, cmd) + for result in resp.results: + if f'/:output/:{out_name}' in result['relationId']: + table = result['table'].to_pydict() + return table['v1'][0] raise Exception(f"model '{name}' not found") -def install_model(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: +def install_models(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] return exec(ctx, database, engine, '\n'.join(queries), readonly=False) -def install_model_async(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: +def install_models_async(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] return exec_async(ctx, database, engine, '\n'.join(queries), readonly=False) @@ -888,6 +890,6 @@ def exec_async( get_compute = get_engine # deprecated, use get_engine list_computes = list_engines # deprecated, use list_engines list_edb = list_edbs # deprecated, use list_edbs -delete_source = delete_model # deprecated, use delete_model +delete_source = delete_models # deprecated, use delete_model get_source = get_model # deprecated, use get_model list_sources = list_models # deprecated, use list_models diff --git a/tests/integration.py b/tests/integration.py index 79bf51e..fcc2b24 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -90,13 +90,13 @@ def test_models(self): models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue(len(models) > 0) - resp = api.install_model(ctx, self.dbname, self.engine, {"test_model": "def foo=:bar"}) + resp = api.install_models(ctx, self.dbname, self.engine, {"test_model": "def foo=:bar"}) self.assertEqual(resp.transaction["state"], "COMPLETED") models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue("test_model" in models) - resp = api.delete_model(ctx, self.dbname, self.engine, "test_model") + resp = api.delete_models(ctx, self.dbname, self.engine, ["test_model"]) self.assertEqual(resp.transaction["state"], "COMPLETED") models = api.list_models(ctx, self.dbname, self.engine) From 35b49978b64f53c139c719dc61945636b9d5c9f4 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Mon, 26 Sep 2022 15:22:34 +0100 Subject: [PATCH 09/16] fix --- railib/api.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/railib/api.py b/railib/api.py index 204b92b..8ff7333 100644 --- a/railib/api.py +++ b/railib/api.py @@ -650,14 +650,14 @@ def _model(name: str, model: str) -> dict: # Returns full list of models. -def _list_models(ctx: Context, database: str, engine: str) -> List: +def list_models(ctx: Context, database: str, engine: str) -> List: models = [] out_name = f'model{random.randint(0, sys.maxsize)}' resp = exec(ctx, database, engine, f'def output:{out_name} = rel:catalog:model') for result in resp.results: - if '/:output/:{out_name}' in result['relationId']: + if f'/:output/:{out_name}' in result['relationId']: table = result['table'].to_pydict() - models.extend([{'name': table['v1'][i], 'value': table['v2'][i]} for i in range(1, len(table['v1']))]) + models.extend([table['v1'][i] for i in range(1, len(table['v1']))]) return models @@ -717,12 +717,6 @@ def list_edbs(ctx: Context, database: str, engine: str) -> list: return rels -# Returns a list of models installed in the given database. -def list_models(ctx: Context, database: str, engine: str) -> list: - models = _list_models(ctx, database, engine) - return [model["name"] for model in models] - - # Generate a rel literal relation for the given dict. def _gen_literal_dict(items: dict) -> str: result = [] From a6cf17d1148bd8b8fc5a3122af8ace2ff639a828 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Mon, 26 Sep 2022 15:23:36 +0100 Subject: [PATCH 10/16] enhance list models query --- railib/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/railib/api.py b/railib/api.py index 8ff7333..ac78e77 100644 --- a/railib/api.py +++ b/railib/api.py @@ -653,7 +653,7 @@ def _model(name: str, model: str) -> dict: def list_models(ctx: Context, database: str, engine: str) -> List: models = [] out_name = f'model{random.randint(0, sys.maxsize)}' - resp = exec(ctx, database, engine, f'def output:{out_name} = rel:catalog:model') + resp = exec(ctx, database, engine, f'def output:{out_name}[name] = rel:catalog:model(name, _)') for result in resp.results: if f'/:output/:{out_name}' in result['relationId']: table = result['table'].to_pydict() From 747a058a4796d8206873c4943193a827305598ce Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 5 Oct 2022 00:48:35 +0100 Subject: [PATCH 11/16] updates --- examples/delete_models.py | 2 +- railib/api.py | 45 ++++++++++++++++++++++++++++----------- requirements.txt | 2 +- setup.py | 2 +- tests/integration.py | 6 +++++- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/examples/delete_models.py b/examples/delete_models.py index 60af688..1d4616d 100644 --- a/examples/delete_models.py +++ b/examples/delete_models.py @@ -22,7 +22,7 @@ def run(database: str, engine: str, model: str, profile: str): cfg = config.read(profile=profile) ctx = api.Context(**cfg) - rsp = api.delete_models(ctx, database, engine, model) + rsp = api.delete_models(ctx, database, engine, [model]) print(rsp) diff --git a/railib/api.py b/railib/api.py index ac78e77..f932c5e 100644 --- a/railib/api.py +++ b/railib/api.py @@ -649,6 +649,13 @@ def _model(name: str, model: str) -> dict: } +def create_database(ctx: Context, database: str, source: str = None) -> dict: + data = {"name": database, "source_name": source} + url = _mkurl(ctx, PATH_DATABASE) + rsp = rest.put(ctx, url, data) + return json.loads(rsp.read()) + + # Returns full list of models. def list_models(ctx: Context, database: str, engine: str) -> List: models = [] @@ -662,13 +669,6 @@ def list_models(ctx: Context, database: str, engine: str) -> List: return models -def create_database(ctx: Context, database: str, source: str = None) -> dict: - data = {"name": database, "source_name": source} - url = _mkurl(ctx, PATH_DATABASE) - rsp = rest.put(ctx, url, data) - return json.loads(rsp.read()) - - def delete_models(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: queries = [ f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' @@ -677,7 +677,7 @@ def delete_models(ctx: Context, database: str, engine: str, models: List[str]) - return exec(ctx, database, engine, '\n'.join(queries), readonly=False) -def delete_model_async(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: +def delete_models_async(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: queries = [ f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' for model_name in models @@ -698,13 +698,34 @@ def get_model(ctx: Context, database: str, engine: str, name: str) -> str: def install_models(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: - queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] - return exec(ctx, database, engine, '\n'.join(queries), readonly=False) + queries = [] + queries_inputs = {} + randint = random.randint(0, sys.maxsize) + index = 0 + for name, value in models.items(): + input_name = f'input_{randint}_{index}' + queries.append(f'def delete:rel:catalog:model["{name}"] = rel:catalog:model["{name}"]') + queries.append(f'def insert:rel:catalog:model["{name}"] = {input_name}') + + queries_inputs[input_name] = value + index += 1 + + return exec(ctx, database, engine, '\n'.join(queries), inputs=queries_inputs, readonly=False) def install_models_async(ctx: Context, database: str, engine: str, models: dict) -> TransactionAsyncResponse: - queries = [f'def insert:rel:catalog:model["{name}"] = """{models[name]}"""' for name in models.keys()] - return exec_async(ctx, database, engine, '\n'.join(queries), readonly=False) + queries = [] + queries_inputs = {} + randint = random.randint(0, sys.maxsize) + index = 0 + for name, value in models.items(): + input_name = f'input_{randint}_{index}' + queries.append(f'def delete:rel:catalog:model["{name}"] = rel:catalog:model["{name}"]') + queries.append(f'def insert:rel:catalog:model["{name}"] = {input_name}') + + queries_inputs[input_name] = value + index += 1 + return exec_async(ctx, database, engine, '\n'.join(queries), inputs=queries_inputs, readonly=False) def list_edbs(ctx: Context, database: str, engine: str) -> list: diff --git a/requirements.txt b/requirements.txt index 744088b..788c556 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ ed25519==1.5 grpcio-tools==1.47.0 -protobuf==3.20.1 +protobuf==3.20.2 pyarrow==6.0.1 requests-toolbelt==0.9.1 diff --git a/setup.py b/setup.py index 0b10bde..e428a84 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "ed25519==1.5", "pyarrow>=6.0.1", "requests-toolbelt==0.9.1", - "protobuf==3.20.1"], + "protobuf==3.20.2"], license="http://www.apache.org/licenses/LICENSE-2.0", long_description="Enables access to the RelationalAI REST APIs from Python", long_description_content_type="text/markdown", diff --git a/tests/integration.py b/tests/integration.py index fcc2b24..ee322b3 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -90,12 +90,16 @@ def test_models(self): models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue(len(models) > 0) - resp = api.install_models(ctx, self.dbname, self.engine, {"test_model": "def foo=:bar"}) + models = {"test_model": "def foo=:bar"} + resp = api.install_models(ctx, self.dbname, self.engine, models) self.assertEqual(resp.transaction["state"], "COMPLETED") models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue("test_model" in models) + value = api.get_model(ctx, self.dbname, self.engine, "test_model") + self.assertEqual(models["test_model"], value) + resp = api.delete_models(ctx, self.dbname, self.engine, ["test_model"]) self.assertEqual(resp.transaction["state"], "COMPLETED") From e9d254fc438b97ed0c133acb7c68f83e62521363 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 5 Oct 2022 00:56:04 +0100 Subject: [PATCH 12/16] fix --- tests/integration.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index ee322b3..e4eb443 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -90,21 +90,21 @@ def test_models(self): models = api.list_models(ctx, self.dbname, self.engine) self.assertTrue(len(models) > 0) - models = {"test_model": "def foo=:bar"} + models = {'test_model': 'def foo=:bar'} resp = api.install_models(ctx, self.dbname, self.engine, models) - self.assertEqual(resp.transaction["state"], "COMPLETED") + self.assertEqual(resp.transaction['state'], 'COMPLETED') models = api.list_models(ctx, self.dbname, self.engine) - self.assertTrue("test_model" in models) + self.assertTrue('test_model' in models) - value = api.get_model(ctx, self.dbname, self.engine, "test_model") - self.assertEqual(models["test_model"], value) + value = api.get_model(ctx, self.dbname, self.engine, 'test_model') + self.assertEqual(models['test_model'], value) - resp = api.delete_models(ctx, self.dbname, self.engine, ["test_model"]) - self.assertEqual(resp.transaction["state"], "COMPLETED") + resp = api.delete_models(ctx, self.dbname, self.engine, ['test_model']) + self.assertEqual(resp.transaction['state'], 'COMPLETED') models = api.list_models(ctx, self.dbname, self.engine) - self.assertFalse("test_model" in models) + self.assertFalse('test_model' in models) def tearDown(self): api.delete_engine(ctx, self.engine) From 19a397cae25b7a4fb09d88a0180ffb0822c2092b Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Wed, 5 Oct 2022 01:00:50 +0100 Subject: [PATCH 13/16] fix --- tests/integration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index e4eb443..2bf32da 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -94,12 +94,12 @@ def test_models(self): resp = api.install_models(ctx, self.dbname, self.engine, models) self.assertEqual(resp.transaction['state'], 'COMPLETED') - models = api.list_models(ctx, self.dbname, self.engine) - self.assertTrue('test_model' in models) - value = api.get_model(ctx, self.dbname, self.engine, 'test_model') self.assertEqual(models['test_model'], value) + models = api.list_models(ctx, self.dbname, self.engine) + self.assertTrue('test_model' in models) + resp = api.delete_models(ctx, self.dbname, self.engine, ['test_model']) self.assertEqual(resp.transaction['state'], 'COMPLETED') From 6e25efbd13d2e0330dafd44f4e72f354dbd2ebbe Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 22 Nov 2022 01:12:18 +0100 Subject: [PATCH 14/16] fix linter --- railib/api.py | 2 +- test/test_integration.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/railib/api.py b/railib/api.py index 3658c02..81aa7fe 100644 --- a/railib/api.py +++ b/railib/api.py @@ -695,7 +695,7 @@ def _model(name: str, model: str) -> dict: } -def create_database(ctx: Context, database: str, source: str = None) -> dict: +def create_database(ctx: Context, database: str, source: str = None, **kwargs) -> dict: data = {"name": database, "source_name": source} url = _mkurl(ctx, PATH_DATABASE) rsp = rest.put(ctx, url, data, **kwargs) diff --git a/test/test_integration.py b/test/test_integration.py index 193b6ac..c95959a 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -39,6 +39,7 @@ engine = f"python-sdk-{suffix}" dbname = f"python-sdk-{suffix}" + class TestTransactionAsync(unittest.TestCase): def setUp(self): rsp = api.create_engine_wait(ctx, engine, headers=custom_headers) From fcaf17c29e1b5cd2d4909dc2e0dc65d05c2b9fe3 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 22 Nov 2022 01:24:37 +0100 Subject: [PATCH 15/16] fix failing test --- test/test_integration.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_integration.py b/test/test_integration.py index c95959a..9161592 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -76,28 +76,28 @@ def test_v2_exec(self): rsp.results[0]["table"].to_pydict()) def test_models(self): - models = api.list_models(ctx, self.dbname, self.engine) + models = api.list_models(ctx, dbname, engine) self.assertTrue(len(models) > 0) models = {'test_model': 'def foo=:bar'} - resp = api.install_models(ctx, self.dbname, self.engine, models) + resp = api.install_models(ctx, dbname, engine, models) self.assertEqual(resp.transaction['state'], 'COMPLETED') - value = api.get_model(ctx, self.dbname, self.engine, 'test_model') + value = api.get_model(ctx, dbname, engine, 'test_model') self.assertEqual(models['test_model'], value) - models = api.list_models(ctx, self.dbname, self.engine) + models = api.list_models(ctx, dbname, engine) self.assertTrue('test_model' in models) - resp = api.delete_models(ctx, self.dbname, self.engine, ['test_model']) + resp = api.delete_models(ctx, dbname, engine, ['test_model']) self.assertEqual(resp.transaction['state'], 'COMPLETED') - models = api.list_models(ctx, self.dbname, self.engine) + models = api.list_models(ctx, dbname, engine) self.assertFalse('test_model' in models) def tearDown(self): - api.delete_engine(ctx, self.engine) - api.delete_database(ctx, self.dbname) + api.delete_engine(ctx, engine) + api.delete_database(ctx, dbname) if __name__ == '__main__': From 92893426890c189687230bcc273ba0bde2502024 Mon Sep 17 00:00:00 2001 From: "helmi.nour" Date: Tue, 22 Nov 2022 01:33:37 +0100 Subject: [PATCH 16/16] cleanup --- railib/api.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/railib/api.py b/railib/api.py index 81aa7fe..1cf4e8b 100644 --- a/railib/api.py +++ b/railib/api.py @@ -715,6 +715,10 @@ def list_models(ctx: Context, database: str, engine: str) -> List: return models +def delete_model(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: + return delete_models(ctx, database, engine, [model]) + + def delete_models(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: queries = [ f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' @@ -723,6 +727,10 @@ def delete_models(ctx: Context, database: str, engine: str, models: List[str]) - return exec(ctx, database, engine, '\n'.join(queries), readonly=False) +def delete_model_async(ctx: Context, database: str, engine: str, model: str) -> TransactionAsyncResponse: + return delete_models_async(ctx, database, engine, [model]) + + def delete_models_async(ctx: Context, database: str, engine: str, models: List[str]) -> TransactionAsyncResponse: queries = [ f'def delete:rel:catalog:model["{model_name}"] = rel:catalog:model["{model_name}"]' @@ -956,6 +964,6 @@ def exec_async( get_compute = get_engine # deprecated, use get_engine list_computes = list_engines # deprecated, use list_engines list_edb = list_edbs # deprecated, use list_edbs -delete_source = delete_models # deprecated, use delete_model +delete_source = delete_model # deprecated, use delete_model get_source = get_model # deprecated, use get_model list_sources = list_models # deprecated, use list_models