From db56bb82d5ae377062c1ca59cd91346e3359d800 Mon Sep 17 00:00:00 2001 From: Johannes Coym Date: Wed, 13 Mar 2019 16:38:10 +0100 Subject: [PATCH 1/2] Add Python interface --- .gitignore | 3 + Doxyfile | 2 +- python/benchmark.py | 31 +++ python/hello_world.py | 36 +++ python/julea/__init__.py | 26 ++ python/julea/benchmark/__init__.py | 20 ++ .../benchmark/bench_distributed_object.py | 199 +++++++++++++++ python/julea/benchmark/bench_item.py | 235 ++++++++++++++++++ python/julea/benchmark/bench_kv.py | 116 +++++++++ python/julea/benchmark/bench_object.py | 185 ++++++++++++++ python/julea/item/__init__.py | 20 ++ python/julea/item/collection.py | 61 +++++ python/julea/item/item.py | 135 ++++++++++ python/julea/kv/__init__.py | 18 ++ python/julea/kv/kv.py | 78 ++++++ python/julea/lib/__init__.py | 15 ++ python/julea/lib/batch.py | 62 +++++ python/julea/lib/common.py | 54 ++++ python/julea/lib/distribution.py | 62 +++++ python/julea/object/__init__.py | 20 ++ python/julea/object/distributed_object.py | 116 +++++++++ python/julea/object/object.py | 112 +++++++++ python/julea/test/__init__.py | 20 ++ python/julea/test/test_distributed_object.py | 86 +++++++ python/julea/test/test_item.py | 88 +++++++ python/julea/test/test_kv.py | 54 ++++ python/julea/test/test_object.py | 82 ++++++ python/test.py | 30 +++ 28 files changed, 1965 insertions(+), 1 deletion(-) create mode 100644 python/benchmark.py create mode 100644 python/hello_world.py create mode 100644 python/julea/__init__.py create mode 100644 python/julea/benchmark/__init__.py create mode 100644 python/julea/benchmark/bench_distributed_object.py create mode 100644 python/julea/benchmark/bench_item.py create mode 100644 python/julea/benchmark/bench_kv.py create mode 100644 python/julea/benchmark/bench_object.py create mode 100644 python/julea/item/__init__.py create mode 100644 python/julea/item/collection.py create mode 100644 python/julea/item/item.py create mode 100644 python/julea/kv/__init__.py create mode 100644 python/julea/kv/kv.py create mode 100644 python/julea/lib/__init__.py create mode 100644 python/julea/lib/batch.py create mode 100644 python/julea/lib/common.py create mode 100644 python/julea/lib/distribution.py create mode 100644 python/julea/object/__init__.py create mode 100644 python/julea/object/distributed_object.py create mode 100644 python/julea/object/object.py create mode 100644 python/julea/test/__init__.py create mode 100644 python/julea/test/test_distributed_object.py create mode 100644 python/julea/test/test_item.py create mode 100644 python/julea/test/test_kv.py create mode 100644 python/julea/test/test_object.py create mode 100644 python/test.py diff --git a/.gitignore b/.gitignore index 9216c84bb..8b0043130 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ waf-extras/__pycache__/ .lock-waf* .waf-*/ .waf3-*/ + +# Python +python/**/__pycache__/ diff --git a/Doxyfile b/Doxyfile index 70dfde65f..9550ce554 100644 --- a/Doxyfile +++ b/Doxyfile @@ -112,7 +112,7 @@ WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- -INPUT = include lib +INPUT = include lib python INPUT_ENCODING = UTF-8 FILE_PATTERNS = *.c \ *.cc \ diff --git a/python/benchmark.py b/python/benchmark.py new file mode 100644 index 000000000..f6964b04c --- /dev/null +++ b/python/benchmark.py @@ -0,0 +1,31 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.benchmark import ( + distributed_object_bench, item_bench, kv_bench, object_bench) +from julea import juleadir +import subprocess + +setupfile = juleadir.joinpath('scripts', 'setup.sh') + +subprocess.run([setupfile, "start"]) + +kv_bench() +distributed_object_bench() +object_bench() +item_bench() + +subprocess.run([setupfile, "stop"]) diff --git a/python/hello_world.py b/python/hello_world.py new file mode 100644 index 000000000..f5e745b5b --- /dev/null +++ b/python/hello_world.py @@ -0,0 +1,36 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT + + +def main(): + object = JObject("hello", "world") + hello_world = "Hello World!" + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + object.create(batch) + bw = object.write(hello_world.encode('utf-8'), 0, batch) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + out = object.read(bw.value, 0, batch) + + print("Object contains: {buffer} ({length} bytes)".format( + buffer=out[0].raw.decode('utf-8'), length=out[1].value)) + + +if __name__ == "__main__": + main() diff --git a/python/julea/__init__.py b/python/julea/__init__.py new file mode 100644 index 000000000..18b4f3a37 --- /dev/null +++ b/python/julea/__init__.py @@ -0,0 +1,26 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +import pathlib + +juleadir = pathlib.Path(__file__).parent.parent.parent +libdir = juleadir.joinpath('build', 'lib') + +JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so')) +JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so')) +JULEA_KV = ctypes.CDLL(libdir.joinpath('libjulea-kv.so')) +JULEA_OBJECT = ctypes.CDLL(libdir.joinpath('libjulea-object.so')) diff --git a/python/julea/benchmark/__init__.py b/python/julea/benchmark/__init__.py new file mode 100644 index 000000000..18f0c3312 --- /dev/null +++ b/python/julea/benchmark/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .bench_distributed_object import distributed_object_bench +from .bench_item import item_bench +from .bench_kv import kv_bench +from .bench_object import object_bench diff --git a/python/julea/benchmark/bench_distributed_object.py b/python/julea/benchmark/bench_distributed_object.py new file mode 100644 index 000000000..56caba96c --- /dev/null +++ b/python/julea/benchmark/bench_distributed_object.py @@ -0,0 +1,199 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2017-2019 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import (JDistributedObject, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + if use_batch: + n = 100000 + else: + n = 1000 + + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.create(batch) + o.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/distributed-object/create-batch", end - start, n + else: + return "/object/distributed-object/create", end - start, n + + +def bench_delete(use_batch): + n = 10000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/distributed-object/delete-batch", end - start, n + else: + return "/object/distributed-object/delete", end - start, n + + +def bench_status(use_batch): + n = 1000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytearray(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + o.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + o.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/distributed-object/status-batch", end - start, n + else: + return "/object/distributed-object/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 25000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + + for i in range(0, n): + o.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.read(block_size, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + name = "/object/distributed-object/read-batch" + return name, end - start, n, n * block_size + else: + name = "/object/distributed-object/read" + return name, end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 25000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + name = "/object/distributed-object/write-batch" + return name, end - start, n, n * block_size + else: + name = "/object/distributed-object/write" + return name, end - start, n, n * block_size + + +def distributed_object_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/benchmark/bench_item.py b/python/julea/benchmark/bench_item.py new file mode 100644 index 000000000..09a32db78 --- /dev/null +++ b/python/julea/benchmark/bench_item.py @@ -0,0 +1,235 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2010-2019 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..item import (JItem, JCollection, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + if use_batch: + n = 100000 + else: + n = 1000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, batch, False, None) + item.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + if use_batch: + return "/item/item/create-batch", end - start, n + else: + return "/item/item/create", end - start, n + + +def bench_delete(use_batch): + n = 10000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + JItem(collection, name, batch, False, None) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as get_batch: + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, get_batch, True) + get_batch.execute() + + item.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + if use_batch: + return "/item/item/delete-batch", end - start, n + else: + return "/item/item/delete", end - start, n + + +def bench_delete_batch_without_get(): + n = 10000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, batch, False, None) + + item.delete(batch) + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + return "/item/item/delete-batch-without-get", end - start, n + + +def bench_status(use_batch): + n = 1000 + dummy = bytearray(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + batch.execute() + item = JItem(collection, "benchmark", batch, False, None) + item.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + item.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + return "/item/item/status-batch", end - start, n + else: + return "/item/item/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 25000 + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + item = JItem(collection, "benchmark", batch, False, None) + + for i in range(0, n): + item.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + # Unused variable required, otherwise glibc crashes somehow + out = item.read(block_size, # pylint: disable=unused-variable + i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + name = "/item/item/read-batch" + return name, end - start, n, n * block_size + else: + name = "/item/item/read" + return name, end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 25000 + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + batch.execute() + item = JItem(collection, "benchmark", batch, False, None) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + item.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + name = "/item/item/write-batch" + return name, end - start, n, n * block_size + else: + name = "/item/item/write" + return name, end - start, n, n * block_size + + +def item_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + delete_batch_without_get = bench_delete_batch_without_get() + print(format_benchmark(delete_batch_without_get)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/benchmark/bench_kv.py b/python/julea/benchmark/bench_kv.py new file mode 100644 index 000000000..cd8465535 --- /dev/null +++ b/python/julea/benchmark/bench_kv.py @@ -0,0 +1,116 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2017-2019 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import format_benchmark +import time + + +def bench_put(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put("empty".encode('utf-8'), batch) + k.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/put-batch", end - start, n + else: + return "/kv/kv/put", end - start, n + + +def bench_get(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put(name.encode('utf-8'), batch) + k.delete(delete_batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.get(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/get-batch", end - start, n + else: + return "/kv/kv/get", end - start, n + + +def bench_delete(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put("empty".encode('utf-8'), batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/delete-batch", end - start, n + else: + return "/kv/kv/delete", end - start, n + + +def kv_bench(): + create = bench_put(False) + print(format_benchmark(create)) + create_batch = bench_put(True) + print(format_benchmark(create_batch)) + + read = bench_get(False) + print(format_benchmark(read)) + read_batch = bench_get(True) + print(format_benchmark(read_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) diff --git a/python/julea/benchmark/bench_object.py b/python/julea/benchmark/bench_object.py new file mode 100644 index 000000000..716b9b1eb --- /dev/null +++ b/python/julea/benchmark/bench_object.py @@ -0,0 +1,185 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2017-2019 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + n = 100000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.create(batch) + o.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/object/create-batch", end - start, n + else: + return "/object/object/create", end - start, n + + +def bench_delete(use_batch): + n = 100000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/object/delete-batch", end - start, n + else: + return "/object/object/delete", end - start, n + + +def bench_status(use_batch): + n = 200000 + dummy = bytearray(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + o.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + o.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/status-batch", end - start, n + else: + return "/object/object/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 200000 + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + + for i in range(0, n): + o.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.read(block_size, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/read-batch", end - start, n, n * block_size + else: + return "/object/object/read", end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 200000 + dummy = bytearray(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/write-batch", end - start, n, n * block_size + else: + return "/object/object/write", end - start, n, n * block_size + + +def object_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/item/__init__.py b/python/julea/item/__init__.py new file mode 100644 index 000000000..145fd0df5 --- /dev/null +++ b/python/julea/item/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .collection import JCollection +from .item import JItem +from ..lib.batch import * +from ..lib.distribution import * diff --git a/python/julea/item/collection.py b/python/julea/item/collection.py new file mode 100644 index 000000000..8d4119ac3 --- /dev/null +++ b/python/julea/item/collection.py @@ -0,0 +1,61 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_ITEM +from ..lib.common import my_void_p + +JULEA_ITEM.j_collection_create.restype = my_void_p + + +class JCollection: + def __init__(self, name, batch, get): + """Initializes and creates a collection + + Args: + name: The name of the collection + batch: A batch + get: A boolean, False when the collection should be created, + True if get should be called + """ + enc_name = name.encode('utf-8') + if get: + self.collection = my_void_p + JULEA_ITEM.j_collection_get( + ctypes.byref(self.collection), enc_name, batch.get_pointer()) + else: + self.collection = JULEA_ITEM.j_collection_create( + enc_name, batch.get_pointer()) + + def __del__(self): + """Dereferences a collection when it is deleted in Python""" + JULEA_ITEM.j_collection_unref(self.collection) + + def delete(self, batch): + """Deletes a collection + + Args: + batch: A batch + """ + JULEA_ITEM.j_collection_delete(self.collection, batch.get_pointer()) + + def get_pointer(self): + """Gives a pointer to a collection + + Returns: + collection: A pointer to the collection + """ + return self.collection diff --git a/python/julea/item/item.py b/python/julea/item/item.py new file mode 100644 index 000000000..dd2cbe94f --- /dev/null +++ b/python/julea/item/item.py @@ -0,0 +1,135 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_ITEM +from ..lib.common import my_void_p + +JULEA_ITEM.j_item_create.restype = my_void_p +JULEA_ITEM.j_item_get_size.restype = ctypes.c_ulonglong +JULEA_ITEM.j_item_get_modification_time.restype = ctypes.c_longlong + +JULEA_ITEM.j_item_get.argtypes = [my_void_p, ctypes.POINTER(my_void_p), + ctypes.c_char_p, my_void_p] + + +class JItem: + def __init__(self, collection, name, batch, get, distribution=None): + """Initializes and creates an item + + Args: + collection: The collection of the item + name: The name of the item + batch: A batch + get: A boolean, False when the item should be created, + True if get should be called + distribution: The distribution of the item(only for create) + """ + enc_name = name.encode('utf-8') + if get: + self.item = my_void_p(0) + JULEA_ITEM.j_item_get(collection.get_pointer(), + ctypes.byref(self.item), enc_name, + batch.get_pointer()) + else: + if distribution is None: + self.item = JULEA_ITEM.j_item_create( + collection.get_pointer(), enc_name, + distribution, batch.get_pointer()) + else: + self.item = JULEA_ITEM.j_item_create( + collection.get_pointer(), enc_name, + distribution.get_pointer(), batch.get_pointer()) + + def __del__(self): + """Dereferences an item when it is deleted in Python""" + JULEA_ITEM.j_item_unref(self.item) + + def delete(self, batch): + """Deletes an item from the server + + Args: + batch: A batch + """ + JULEA_ITEM.j_item_delete(self.item, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an item + + Args: + batch: A batch + """ + JULEA_ITEM.j_item_get_status(self.item, batch.get_pointer()) + + def get_size(self): + """Gets the size of an item + Requires status to be called earlier + + Returns: + size: The size of the item + """ + size = JULEA_ITEM.j_item_get_size(self.item) + return size + + def get_modification_time(self): + """Gets the modification time of an item + Requires status to be called earlier + + Returns: + modtime: The size of the item + """ + modtime = JULEA_ITEM.j_item_get_modification_time(self.item) + return modtime + + def write(self, value, offset, batch): + """Writes an item + + Args: + value: A bytearray holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.create_string_buffer(length.value) + value2.raw = value + bytes_written = ctypes.c_ulonglong(0) + JULEA_ITEM.j_item_write( + self.item, ctypes.byref(value2), length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an item + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.raw) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.create_string_buffer(length) + bytes_read = ctypes.c_ulonglong(0) + JULEA_ITEM.j_item_read( + self.item, ctypes.byref(value), length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/kv/__init__.py b/python/julea/kv/__init__.py new file mode 100644 index 000000000..90ead6fd4 --- /dev/null +++ b/python/julea/kv/__init__.py @@ -0,0 +1,18 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .kv import JKV +from ..lib.batch import * diff --git a/python/julea/kv/kv.py b/python/julea/kv/kv.py new file mode 100644 index 000000000..18baf1a01 --- /dev/null +++ b/python/julea/kv/kv.py @@ -0,0 +1,78 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_KV +from ..lib.common import my_void_p + +JULEA_KV.j_kv_new.restype = my_void_p + + +class JKV: + + def __init__(self, namespace, key): + """Initializes the Key-Value pair and + gets the new KV pair from the C Interface + + Args: + namespace: The namespace of the KV pair + key: The name of the KV pair + """ + enc_namespace = namespace.encode('utf-8') + enc_key = key.encode('utf-8') + self.kv = JULEA_KV.j_kv_new(enc_namespace, enc_key) + + def __del__(self): + """Dereferences a KV pair when it is deleted in Python""" + JULEA_KV.j_kv_unref(self.kv) + + def delete(self, batch): + """Deletes a Key-Value pair from the server + + Args: + batch: A batch + """ + JULEA_KV.j_kv_delete(self.kv, batch.get_pointer()) + + def put(self, value, batch): + """Writes a Key-Value pair + + Args: + value: A bytearray of data to be written in the KV pair + batch: A batch + """ + length = ctypes.c_ulong(len(value)) + value2 = ctypes.create_string_buffer(length.value) + value2.raw = value + JULEA_KV.j_kv_put(self.kv, ctypes.byref(value2), + length, None, batch.get_pointer()) + + def get(self, batch): + """Reads a Key-Value pair + + Args: + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.value[:length.value]) + length: The length of the data + (Access with length.value) + """ + value = ctypes.c_char_p() + length = ctypes.c_ulong(0) + JULEA_KV.j_kv_get(self.kv, ctypes.byref(value), + ctypes.byref(length), batch.get_pointer()) + return value, length diff --git a/python/julea/lib/__init__.py b/python/julea/lib/__init__.py new file mode 100644 index 000000000..5e9613478 --- /dev/null +++ b/python/julea/lib/__init__.py @@ -0,0 +1,15 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . diff --git a/python/julea/lib/batch.py b/python/julea/lib/batch.py new file mode 100644 index 000000000..3c03c2dac --- /dev/null +++ b/python/julea/lib/batch.py @@ -0,0 +1,62 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA +from .common import my_void_p + +(J_SEMANTICS_TEMPLATE_DEFAULT, J_SEMANTICS_TEMPLATE_POSIX, + J_SEMANTICS_TEMPLATE_TEMPORARY_LOCAL) = (0, 1, 2) + +JULEA.j_batch_new_for_template.restype = my_void_p + + +class JBatch: + def __init__(self, template): + """Initializes a batch + + Args: + template: The template defined in JSemanticsTemplate (C Interface) + """ + self.batch = JULEA.j_batch_new_for_template(template) + + def __del__(self): + """Dereferences a batch when it is deleted in Python""" + JULEA.j_batch_unref(self.batch) + + def __enter__(self): + """Returns itself when a with-statement is entered + + Returns: + self: The batch itself + """ + return self + + def __exit__(self, type, value, traceback): + """Executes the Batch when the with-statement is left""" + self.execute() + + def execute(self): + """Executes the Batch""" + ret = JULEA.j_batch_execute(self.batch) + if ret == 1: + return True + else: + return False + + def get_pointer(self): + """Returns the C Pointer to the batch""" + return self.batch diff --git a/python/julea/lib/common.py b/python/julea/lib/common.py new file mode 100644 index 000000000..48b6e6cf6 --- /dev/null +++ b/python/julea/lib/common.py @@ -0,0 +1,54 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes + + +# Workaround so Python doesn't convert the pointer to a Python datatype +class my_void_p(ctypes.c_void_p): + pass + + +def format_benchmark(input): + name = input[0] + ':' + ops = round(input[2] / input[1]) + if len(input) == 3: + output = "{name: <{width}}{time} seconds ({ops}/s)" + return output.format(name=name, width=51, time=round(input[1], 3), + ops=ops) + elif len(input) == 4: + rate = input[3] / input[1] + unit = "B" + if rate >= 1024: + rate = rate / 1024 + unit = "KB" + if rate >= 1024: + rate = rate / 1024 + unit = "MB" + if rate >= 1024: + rate = rate / 1024 + unit = "GB" + if rate >= 1024: + rate = rate / 1024 + unit = "TB" + rate = round(rate, 1) + output = "{name: <{width}}{time} seconds ({ops}/s) ({rate} {unit}/s)" + return output.format(name=name, width=51, time=round(input[1], 3), + ops=ops, rate=rate, unit=unit) + + +def print_test(name): + print(" {name: <{width}}".format(name=name, width=69), end='', flush=True) diff --git a/python/julea/lib/distribution.py b/python/julea/lib/distribution.py new file mode 100644 index 000000000..8c6c199dd --- /dev/null +++ b/python/julea/lib/distribution.py @@ -0,0 +1,62 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA +from .common import my_void_p + +(J_DISTRIBUTION_ROUND_ROBIN, J_DISTRIBUTION_SINGLE_SERVER, + J_DISTRIBUTION_WEIGHTED) = (0, 1, 2) + +JULEA.j_distribution_new.restype = my_void_p +JULEA.j_distribution_new_from_bson.restype = my_void_p +JULEA.j_distribution_serialize.restype = my_void_p + + +class JDistribution: + + def __init__(self, input): + """Creates a new distribution + + Args: + input: The type of the distribution or a pointer to a + serialized distribution + """ + if isinstance(input, int): + self.distribution = JULEA.j_distribution_new(input) + else: + self.distribution = JULEA.j_distribution_new_from_bson(input) + + def __del__(self): + """Dereferences a distribution when it is deleted in Python""" + JULEA.j_distribution_unref(self.distribution) + + def serialize(self): + """Serializes the distribution into a BSON. + + Returns: + b: A pointer to the BSON object in C. + """ + b = JULEA.j_distribution_serialize(self.distribution) + return b + + def get_pointer(self): + """Gets the pointer to the distribution + + Returns: + distribution: A pointer to the distribution + """ + return self.distribution diff --git a/python/julea/object/__init__.py b/python/julea/object/__init__.py new file mode 100644 index 000000000..b4acb894a --- /dev/null +++ b/python/julea/object/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .distributed_object import JDistributedObject +from .object import JObject +from ..lib.batch import * +from ..lib.distribution import * diff --git a/python/julea/object/distributed_object.py b/python/julea/object/distributed_object.py new file mode 100644 index 000000000..1f9668cc8 --- /dev/null +++ b/python/julea/object/distributed_object.py @@ -0,0 +1,116 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_OBJECT +from ..lib.common import my_void_p + +JULEA_OBJECT.j_distributed_object_new.restype = my_void_p + + +class JDistributedObject: + + def __init__(self, namespace, name, distribution): + """Initializes the objects and gets the new object from the C Interface + + Args: + namespace: The namespace of the object + name: The name of the object + """ + enc_namespace = namespace.encode('utf-8') + enc_name = name.encode('utf-8') + dist = distribution.get_pointer() + self.object = JULEA_OBJECT.j_distributed_object_new( + enc_namespace, enc_name, dist) + + def __del__(self): + """Dereferences an object when it is deleted in Python""" + JULEA_OBJECT.j_distributed_object_unref(self.object) + + def create(self, batch): + """Creates an object + + Args: + batch: A batch + """ + JULEA_OBJECT.j_distributed_object_create( + self.object, batch.get_pointer()) + + def delete(self, batch): + """Deletes an object from the server + + Args: + batch: A batch + """ + JULEA_OBJECT.j_distributed_object_delete( + self.object, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an object + + Args: + batch: A batch + Returns: + modification_time: The modification time of the object + (Access data with modification_time.value) + size: The size of the object (Access data with size.value) + """ + modification_time = ctypes.c_longlong(0) + size = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_status( + self.object, ctypes.byref(modification_time), + ctypes.byref(size), batch.get_pointer()) + return modification_time, size + + def write(self, value, offset, batch): + """Writes an object + + Args: + value: A bytearray holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.create_string_buffer(length.value) + value2.raw = value + bytes_written = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_write( + self.object, ctypes.byref(value2), length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an object + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.raw) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.create_string_buffer(length) + bytes_read = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_read( + self.object, ctypes.byref(value), length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/object/object.py b/python/julea/object/object.py new file mode 100644 index 000000000..14fd71988 --- /dev/null +++ b/python/julea/object/object.py @@ -0,0 +1,112 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_OBJECT +from ..lib.common import my_void_p + +JULEA_OBJECT.j_object_new.restype = my_void_p + + +class JObject: + + def __init__(self, namespace, name): + """Initializes the object and gets the new object from the C Interface + + Args: + namespace: The namespace of the object + name: The name of the object + """ + enc_namespace = namespace.encode('utf-8') + enc_name = name.encode('utf-8') + self.object = JULEA_OBJECT.j_object_new(enc_namespace, enc_name) + + def __del__(self): + """Dereferences an object when it is deleted in Python""" + JULEA_OBJECT.j_object_unref(self.object) + + def create(self, batch): + """Creates an object + + Args: + batch: A batch + """ + JULEA_OBJECT.j_object_create(self.object, batch.get_pointer()) + + def delete(self, batch): + """Deletes an object from the server + + Args: + batch: A batch + """ + JULEA_OBJECT.j_object_delete(self.object, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an object + + Args: + batch: A batch + Returns: + modification_time: The modification time of the object + (Access data with modification_time.value) + size: The size of the object (Access data with size.value) + """ + modification_time = ctypes.c_longlong(0) + size = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_status( + self.object, ctypes.byref(modification_time), + ctypes.byref(size), batch.get_pointer()) + return modification_time, size + + def write(self, value, offset, batch): + """Writes an object + + Args: + value: A bytearray holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.create_string_buffer(length.value) + value2.raw = value + bytes_written = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_write( + self.object, ctypes.byref(value2), length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an object + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.raw) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.create_string_buffer(length) + bytes_read = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_read( + self.object, ctypes.byref(value), length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/test/__init__.py b/python/julea/test/__init__.py new file mode 100644 index 000000000..96b97c814 --- /dev/null +++ b/python/julea/test/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .test_distributed_object import distributed_object_test +from .test_item import item_test +from .test_kv import kv_test +from .test_object import object_test diff --git a/python/julea/test/test_distributed_object.py b/python/julea/test/test_distributed_object.py new file mode 100644 index 000000000..3c432d250 --- /dev/null +++ b/python/julea/test/test_distributed_object.py @@ -0,0 +1,86 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import (JDistributedObject, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/object/distributed-object/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_read_write(): + print_test("/object/distributed-object/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + bw = o.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = o.read(5, 0, batch) + batch.execute() + assert out[0].raw.decode('utf-8') == "dummy" + assert out[1].value == 5 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_status(): + print_test("/object/distributed-object/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + dummy = bytearray(42) + o.write(dummy, 0, batch) + batch.execute() + + modtime, size = o.status(batch) + batch.execute() + + assert modtime.value != 0 + assert size.value == 42 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def distributed_object_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/julea/test/test_item.py b/python/julea/test/test_item.py new file mode 100644 index 000000000..6d501078f --- /dev/null +++ b/python/julea/test/test_item.py @@ -0,0 +1,88 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..item import (JItem, JCollection, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/item/item/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def test_read_write(): + print_test("/item/item/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + bw = i.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = i.read(5, 0, batch) + batch.execute() + assert out[0].raw.decode('utf-8') == "dummy" + assert out[1].value == 5 + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def test_status(): + print_test("/item/item/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + dummy = bytearray(42) + i.write(dummy, 0, batch) + batch.execute() + + i.status(batch) + batch.execute() + size = i.get_size() + modtime = i.get_modification_time() + + assert modtime != 0 + assert size == 42 + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def item_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/julea/test/test_kv.py b/python/julea/test/test_kv.py new file mode 100644 index 000000000..0aa5678ed --- /dev/null +++ b/python/julea/test/test_kv.py @@ -0,0 +1,54 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import print_test + + +def test_put_delete(): + print_test("/kv/kv/put_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + k = JKV("python", "test") + k.put("dummy".encode('utf-8'), batch) + batch.execute() + + k.delete(batch) + batch.execute() + del k + print("OK") + + +def test_put_get(): + print_test("/kv/kv/put_get") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + k = JKV("python", "test") + k.put("dummy".encode('utf-8'), batch) + batch.execute() + + out = k.get(batch) + batch.execute() + assert out[0].value[:out[1].value].decode('utf-8') == "dummy" + assert out[1].value == 5 + + k.delete(batch) + batch.execute() + del k + print("OK") + + +def kv_test(): + test_put_delete() + test_put_get() diff --git a/python/julea/test/test_object.py b/python/julea/test/test_object.py new file mode 100644 index 000000000..773513095 --- /dev/null +++ b/python/julea/test/test_object.py @@ -0,0 +1,82 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/object/object/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_read_write(): + print_test("/object/object/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + bw = o.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = o.read(5, 0, batch) + batch.execute() + assert out[0].raw.decode('utf-8') == "dummy" + assert out[1].value == 5 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_status(): + print_test("/object/object/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + dummy = bytearray(42) + o.write(dummy, 0, batch) + batch.execute() + + modtime, size = o.status(batch) + batch.execute() + + assert modtime.value != 0 + assert size.value == 42 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def object_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/test.py b/python/test.py new file mode 100644 index 000000000..dbc670828 --- /dev/null +++ b/python/test.py @@ -0,0 +1,30 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.test import distributed_object_test, item_test, kv_test, object_test +from julea import juleadir +import subprocess + +setupfile = juleadir.joinpath('scripts', 'setup.sh') + +subprocess.run([setupfile, "start"]) + +object_test() +distributed_object_test() +item_test() +kv_test() + +subprocess.run([setupfile, "stop"]) From 44e13b48441c0ed573daf65ab62bb47ebe240e9e Mon Sep 17 00:00:00 2001 From: Johannes Coym Date: Sun, 20 Dec 2020 16:56:48 +0100 Subject: [PATCH 2/2] Adjust python module for meson build and fix bug with >16 bytes --- python/benchmark.py | 2 +- python/hello_world.py | 2 +- python/julea/__init__.py | 4 ++-- python/julea/benchmark/__init__.py | 2 +- .../julea/benchmark/bench_distributed_object.py | 10 +++++----- python/julea/benchmark/bench_item.py | 10 +++++----- python/julea/benchmark/bench_kv.py | 4 ++-- python/julea/benchmark/bench_object.py | 10 +++++----- python/julea/item/__init__.py | 2 +- python/julea/item/collection.py | 2 +- python/julea/item/item.py | 15 +++++++-------- python/julea/kv/__init__.py | 2 +- python/julea/kv/kv.py | 9 ++++----- python/julea/lib/__init__.py | 2 +- python/julea/lib/batch.py | 2 +- python/julea/lib/common.py | 2 +- python/julea/lib/distribution.py | 2 +- python/julea/object/__init__.py | 2 +- python/julea/object/distributed_object.py | 15 +++++++-------- python/julea/object/object.py | 15 +++++++-------- python/julea/test/__init__.py | 2 +- python/julea/test/test_distributed_object.py | 6 +++--- python/julea/test/test_item.py | 6 +++--- python/julea/test/test_kv.py | 2 +- python/julea/test/test_object.py | 6 +++--- python/test.py | 2 +- 26 files changed, 67 insertions(+), 71 deletions(-) diff --git a/python/benchmark.py b/python/benchmark.py index f6964b04c..4ad8faf83 100644 --- a/python/benchmark.py +++ b/python/benchmark.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/hello_world.py b/python/hello_world.py index f5e745b5b..92ba4a735 100644 --- a/python/hello_world.py +++ b/python/hello_world.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/__init__.py b/python/julea/__init__.py index 18b4f3a37..b6d08320a 100644 --- a/python/julea/__init__.py +++ b/python/julea/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -18,7 +18,7 @@ import pathlib juleadir = pathlib.Path(__file__).parent.parent.parent -libdir = juleadir.joinpath('build', 'lib') +libdir = juleadir.joinpath('bld') JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so')) JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so')) diff --git a/python/julea/benchmark/__init__.py b/python/julea/benchmark/__init__.py index 18f0c3312..606804bd0 100644 --- a/python/julea/benchmark/__init__.py +++ b/python/julea/benchmark/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/benchmark/bench_distributed_object.py b/python/julea/benchmark/bench_distributed_object.py index 56caba96c..ed26c45c9 100644 --- a/python/julea/benchmark/bench_distributed_object.py +++ b/python/julea/benchmark/bench_distributed_object.py @@ -1,6 +1,6 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym -# Copyright (C) 2017-2019 Michael Kuhn +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -80,7 +80,7 @@ def bench_delete(use_batch): def bench_status(use_batch): n = 1000 distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) - dummy = bytearray(1) + dummy = bytes(1) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JDistributedObject("python", "benchmark", distribution) @@ -110,7 +110,7 @@ def bench_status(use_batch): def bench_read(use_batch, block_size): n = 25000 distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JDistributedObject("python", "benchmark", distribution) @@ -144,7 +144,7 @@ def bench_read(use_batch, block_size): def bench_write(use_batch, block_size): n = 25000 distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JDistributedObject("python", "benchmark", distribution) diff --git a/python/julea/benchmark/bench_item.py b/python/julea/benchmark/bench_item.py index 09a32db78..a36eb4dfb 100644 --- a/python/julea/benchmark/bench_item.py +++ b/python/julea/benchmark/bench_item.py @@ -1,6 +1,6 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym -# Copyright (C) 2010-2019 Michael Kuhn +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2010-2020 Michael Kuhn # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -112,7 +112,7 @@ def bench_delete_batch_without_get(): def bench_status(use_batch): n = 1000 - dummy = bytearray(1) + dummy = bytes(1) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: collection = JCollection("benchmark", batch, False) @@ -142,7 +142,7 @@ def bench_status(use_batch): def bench_read(use_batch, block_size): n = 25000 - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: collection = JCollection("benchmark", batch, False) @@ -177,7 +177,7 @@ def bench_read(use_batch, block_size): def bench_write(use_batch, block_size): n = 25000 - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: collection = JCollection("benchmark", batch, False) diff --git a/python/julea/benchmark/bench_kv.py b/python/julea/benchmark/bench_kv.py index cd8465535..051438c49 100644 --- a/python/julea/benchmark/bench_kv.py +++ b/python/julea/benchmark/bench_kv.py @@ -1,6 +1,6 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym -# Copyright (C) 2017-2019 Michael Kuhn +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/benchmark/bench_object.py b/python/julea/benchmark/bench_object.py index 716b9b1eb..a3ef09423 100644 --- a/python/julea/benchmark/bench_object.py +++ b/python/julea/benchmark/bench_object.py @@ -1,6 +1,6 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym -# Copyright (C) 2017-2019 Michael Kuhn +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -72,7 +72,7 @@ def bench_delete(use_batch): def bench_status(use_batch): n = 200000 - dummy = bytearray(1) + dummy = bytes(1) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JObject("python", "benchmark") @@ -101,7 +101,7 @@ def bench_status(use_batch): def bench_read(use_batch, block_size): n = 200000 - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JObject("python", "benchmark") @@ -132,7 +132,7 @@ def bench_read(use_batch, block_size): def bench_write(use_batch, block_size): n = 200000 - dummy = bytearray(block_size) + dummy = bytes(block_size) with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: o = JObject("python", "benchmark") diff --git a/python/julea/item/__init__.py b/python/julea/item/__init__.py index 145fd0df5..7286d3dfd 100644 --- a/python/julea/item/__init__.py +++ b/python/julea/item/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/item/collection.py b/python/julea/item/collection.py index 8d4119ac3..4c024944d 100644 --- a/python/julea/item/collection.py +++ b/python/julea/item/collection.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/item/item.py b/python/julea/item/item.py index dd2cbe94f..d72b661e2 100644 --- a/python/julea/item/item.py +++ b/python/julea/item/item.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -98,7 +98,7 @@ def write(self, value, offset, batch): """Writes an item Args: - value: A bytearray holding the data to write + value: bytes holding the data to write offset: The offset where the data is written batch: A batch Returns: @@ -106,11 +106,10 @@ def write(self, value, offset, batch): (Access with bytes_written.value) """ length = ctypes.c_ulonglong(len(value)) - value2 = ctypes.create_string_buffer(length.value) - value2.raw = value + value2 = ctypes.c_char_p(value) bytes_written = ctypes.c_ulonglong(0) JULEA_ITEM.j_item_write( - self.item, ctypes.byref(value2), length, offset, + self.item, value2, length, offset, ctypes.byref(bytes_written), batch.get_pointer()) return bytes_written @@ -123,13 +122,13 @@ def read(self, length, offset, batch): batch: A batch Returns: value: The ctype with the data that was read - (Access data with value.raw) + (Access data with value.value) bytes_read: A c_ulonglong with the bytes read (Access with bytes_read.value) """ - value = ctypes.create_string_buffer(length) + value = ctypes.c_char_p(bytes(length)) bytes_read = ctypes.c_ulonglong(0) JULEA_ITEM.j_item_read( - self.item, ctypes.byref(value), length, offset, + self.item, value, length, offset, ctypes.byref(bytes_read), batch.get_pointer()) return value, bytes_read diff --git a/python/julea/kv/__init__.py b/python/julea/kv/__init__.py index 90ead6fd4..6b266fe1a 100644 --- a/python/julea/kv/__init__.py +++ b/python/julea/kv/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/kv/kv.py b/python/julea/kv/kv.py index 18baf1a01..98e311bc4 100644 --- a/python/julea/kv/kv.py +++ b/python/julea/kv/kv.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -51,13 +51,12 @@ def put(self, value, batch): """Writes a Key-Value pair Args: - value: A bytearray of data to be written in the KV pair + value: bytes of data to be written in the KV pair batch: A batch """ length = ctypes.c_ulong(len(value)) - value2 = ctypes.create_string_buffer(length.value) - value2.raw = value - JULEA_KV.j_kv_put(self.kv, ctypes.byref(value2), + value2 = ctypes.c_char_p(value) + JULEA_KV.j_kv_put(self.kv, value2, length, None, batch.get_pointer()) def get(self, batch): diff --git a/python/julea/lib/__init__.py b/python/julea/lib/__init__.py index 5e9613478..85c7d6777 100644 --- a/python/julea/lib/__init__.py +++ b/python/julea/lib/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/lib/batch.py b/python/julea/lib/batch.py index 3c03c2dac..eab95efe9 100644 --- a/python/julea/lib/batch.py +++ b/python/julea/lib/batch.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/lib/common.py b/python/julea/lib/common.py index 48b6e6cf6..13512937a 100644 --- a/python/julea/lib/common.py +++ b/python/julea/lib/common.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/lib/distribution.py b/python/julea/lib/distribution.py index 8c6c199dd..ca205732d 100644 --- a/python/julea/lib/distribution.py +++ b/python/julea/lib/distribution.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/object/__init__.py b/python/julea/object/__init__.py index b4acb894a..63e7b6da1 100644 --- a/python/julea/object/__init__.py +++ b/python/julea/object/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/object/distributed_object.py b/python/julea/object/distributed_object.py index 1f9668cc8..381f97f71 100644 --- a/python/julea/object/distributed_object.py +++ b/python/julea/object/distributed_object.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -79,7 +79,7 @@ def write(self, value, offset, batch): """Writes an object Args: - value: A bytearray holding the data to write + value: bytes holding the data to write offset: The offset where the data is written batch: A batch Returns: @@ -87,11 +87,10 @@ def write(self, value, offset, batch): (Access with bytes_written.value) """ length = ctypes.c_ulonglong(len(value)) - value2 = ctypes.create_string_buffer(length.value) - value2.raw = value + value2 = ctypes.c_char_p(value) bytes_written = ctypes.c_ulonglong(0) JULEA_OBJECT.j_distributed_object_write( - self.object, ctypes.byref(value2), length, offset, + self.object, value2, length, offset, ctypes.byref(bytes_written), batch.get_pointer()) return bytes_written @@ -104,13 +103,13 @@ def read(self, length, offset, batch): batch: A batch Returns: value: The ctype with the data that was read - (Access data with value.raw) + (Access data with value.value) bytes_read: A c_ulonglong with the bytes read (Access with bytes_read.value) """ - value = ctypes.create_string_buffer(length) + value = ctypes.c_char_p(bytes(length)) bytes_read = ctypes.c_ulonglong(0) JULEA_OBJECT.j_distributed_object_read( - self.object, ctypes.byref(value), length, offset, + self.object, value, length, offset, ctypes.byref(bytes_read), batch.get_pointer()) return value, bytes_read diff --git a/python/julea/object/object.py b/python/julea/object/object.py index 14fd71988..f9ed975e3 100644 --- a/python/julea/object/object.py +++ b/python/julea/object/object.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -75,7 +75,7 @@ def write(self, value, offset, batch): """Writes an object Args: - value: A bytearray holding the data to write + value: bytes holding the data to write offset: The offset where the data is written batch: A batch Returns: @@ -83,11 +83,10 @@ def write(self, value, offset, batch): (Access with bytes_written.value) """ length = ctypes.c_ulonglong(len(value)) - value2 = ctypes.create_string_buffer(length.value) - value2.raw = value + value2 = ctypes.c_char_p(value) bytes_written = ctypes.c_ulonglong(0) JULEA_OBJECT.j_object_write( - self.object, ctypes.byref(value2), length, offset, + self.object, value2, length, offset, ctypes.byref(bytes_written), batch.get_pointer()) return bytes_written @@ -100,13 +99,13 @@ def read(self, length, offset, batch): batch: A batch Returns: value: The ctype with the data that was read - (Access data with value.raw) + (Access data with value.value) bytes_read: A c_ulonglong with the bytes read (Access with bytes_read.value) """ - value = ctypes.create_string_buffer(length) + value = ctypes.c_char_p(bytes(length)) bytes_read = ctypes.c_ulonglong(0) JULEA_OBJECT.j_object_read( - self.object, ctypes.byref(value), length, offset, + self.object, value, length, offset, ctypes.byref(bytes_read), batch.get_pointer()) return value, bytes_read diff --git a/python/julea/test/__init__.py b/python/julea/test/__init__.py index 96b97c814..9efa356eb 100644 --- a/python/julea/test/__init__.py +++ b/python/julea/test/__init__.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/test/test_distributed_object.py b/python/julea/test/test_distributed_object.py index 3c432d250..0da8b90cf 100644 --- a/python/julea/test/test_distributed_object.py +++ b/python/julea/test/test_distributed_object.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -47,7 +47,7 @@ def test_read_write(): out = o.read(5, 0, batch) batch.execute() - assert out[0].raw.decode('utf-8') == "dummy" + assert out[0].value.decode('utf-8') == "dummy" assert out[1].value == 5 o.delete(batch) @@ -64,7 +64,7 @@ def test_status(): o.create(batch) batch.execute() - dummy = bytearray(42) + dummy = bytes(42) o.write(dummy, 0, batch) batch.execute() diff --git a/python/julea/test/test_item.py b/python/julea/test/test_item.py index 6d501078f..2a6463b9d 100644 --- a/python/julea/test/test_item.py +++ b/python/julea/test/test_item.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -47,7 +47,7 @@ def test_read_write(): out = i.read(5, 0, batch) batch.execute() - assert out[0].raw.decode('utf-8') == "dummy" + assert out[0].value.decode('utf-8') == "dummy" assert out[1].value == 5 i.delete(batch) @@ -64,7 +64,7 @@ def test_status(): i = JItem(c, "test-item", batch, False, d) batch.execute() - dummy = bytearray(42) + dummy = bytes(42) i.write(dummy, 0, batch) batch.execute() diff --git a/python/julea/test/test_kv.py b/python/julea/test/test_kv.py index 0aa5678ed..2821d8f56 100644 --- a/python/julea/test/test_kv.py +++ b/python/julea/test/test_kv.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by diff --git a/python/julea/test/test_object.py b/python/julea/test/test_object.py index 773513095..485c3758d 100644 --- a/python/julea/test/test_object.py +++ b/python/julea/test/test_object.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by @@ -44,7 +44,7 @@ def test_read_write(): out = o.read(5, 0, batch) batch.execute() - assert out[0].raw.decode('utf-8') == "dummy" + assert out[0].value.decode('utf-8') == "dummy" assert out[1].value == 5 o.delete(batch) @@ -60,7 +60,7 @@ def test_status(): o.create(batch) batch.execute() - dummy = bytearray(42) + dummy = bytes(42) o.write(dummy, 0, batch) batch.execute() diff --git a/python/test.py b/python/test.py index dbc670828..d8e444b43 100644 --- a/python/test.py +++ b/python/test.py @@ -1,5 +1,5 @@ # JULEA - Flexible storage framework -# Copyright (C) 2019 Johannes Coym +# Copyright (C) 2019-2020 Johannes Coym # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by