From b322d0bb8e92f951a5dc45bbead7cf7aebed11a6 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 7 Sep 2023 17:26:16 -0600 Subject: [PATCH 001/445] setup the path correctly --- tests/config.nims | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/config.nims diff --git a/tests/config.nims b/tests/config.nims new file mode 100644 index 00000000..0f840a15 --- /dev/null +++ b/tests/config.nims @@ -0,0 +1 @@ +--path:".." From 030dc9e16eb9846ea8ab1d09be9f86048feea5b9 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 7 Sep 2023 17:26:34 -0600 Subject: [PATCH 002/445] fix conflicting testing symbols --- tests/datastore/sql/testsqliteds.nim | 2 +- tests/datastore/testfsds.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 49910eb6..c629eb0c 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -3,7 +3,7 @@ import std/os import std/sequtils from std/algorithm import sort, reversed -import pkg/asynctest/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index c2c048f1..26336772 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -3,7 +3,7 @@ import std/sequtils import std/os from std/algorithm import sort, reversed -import pkg/asynctest/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils From 542fdf268c0b34061480de2eaef604f6285c5273 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 15:19:52 -0700 Subject: [PATCH 003/445] initial data buffer --- .gitignore | 1 + datastore/databuffer.nim | 74 +++++++++++++++++++++++++ tests/datastore/testsharedbuffer.nim | 82 ++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 datastore/databuffer.nim create mode 100644 tests/datastore/testsharedbuffer.nim diff --git a/.gitignore b/.gitignore index 1a87a994..38068420 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ coverage datastore.nims nimcache TODO +nim.cfg diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim new file mode 100644 index 00000000..14d7d217 --- /dev/null +++ b/datastore/databuffer.nim @@ -0,0 +1,74 @@ + + + +import std/os +import std/locks +import std/atomics + +import events + + +type + DataBuffer* = object + cnt: ptr int + buf: ptr UncheckedArray[byte] + size: int + +proc `$`*(data: DataBuffer): string = + if data.buf.isNil: + result = "nil" + else: + let sz = min(16, data.size) + result = newString(sz + 2) + copyMem(addr result[1], data.buf, sz) + result[0] = '<' + result[^1] = '>' + +proc `=destroy`*(x: var DataBuffer) = + ## copy pointer implementation + if x.buf != nil and x.cnt != nil: + let res = atomicSubFetch(x.cnt, 1, ATOMIC_ACQUIRE) + if res == 0: + when isMainModule: + echo "buffer: FREE: ", repr x.buf.pointer, " ", x.cnt[] + deallocShared(x.buf) + deallocShared(x.cnt) + else: + when isMainModule: + echo "buffer: decr: ", repr x.buf.pointer, " ", x.cnt[] + +proc `=copy`*(a: var DataBuffer; b: DataBuffer) = + ## copy pointer implementation + + # do nothing for self-assignments: + if a.buf == b.buf: return + `=destroy`(a) + discard atomicAddFetch(b.cnt, 1, ATOMIC_RELAXED) + a.size = b.size + a.buf = b.buf + a.cnt = b.cnt + when isMainModule: + echo "buffer: Copy: repr: ", b.cnt[], + " ", repr a.buf.pointer, + " ", repr b.buf.pointer + +proc incrAtomicCount*(a: DataBuffer) = + let res = atomicAddFetch(a.cnt, 1, ATOMIC_RELAXED) +proc getAtomicCount*(a: DataBuffer): int = + atomicLoad(a.cnt, addr result, ATOMIC_RELAXED) + +proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = + let cnt = cast[ptr int](allocShared0(sizeof(result.cnt))) + cnt[] = 1 + DataBuffer( + cnt: cnt, + buf: cast[typeof(result.buf)](allocShared0(size)), + size: size, + ) + +proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuffer = + ## allocate new buffer and copies indata from openArray + ## + result = DataBuffer.new(data.len) + if data.len() > 0: + copyMem(result.buf, unsafeAddr data[0], data.len) diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testsharedbuffer.nim new file mode 100644 index 00000000..499438ac --- /dev/null +++ b/tests/datastore/testsharedbuffer.nim @@ -0,0 +1,82 @@ +import std/options +import std/sequtils +import std/algorithm +import std/locks +import std/os + +import pkg/unittest2 +import pkg/questionable +import pkg/questionable/results + +include ../../datastore/databuffer + +type + AtomicFreed* = ptr int + +proc newFreedValue*(val = 0): ptr int = + result = cast[ptr int](alloc0(sizeof(int))) + result[] = val + +proc getFreedValue*(x: ptr int): int = + atomicLoad(x, addr result, ATOMIC_ACQUIRE) + +proc incrFreedValue*(x: ptr int): int = + atomicAddFetch(x, 1, ATOMIC_ACQUIRE) + +proc decrFreedValue*(x: ptr int): int = + atomicSubFetch(x, 1, ATOMIC_ACQUIRE) + + +var + shareVal: DataBuffer + lock: Lock + cond: Cond + +var threads: array[2,Thread[int]] + +proc thread1(val: int) {.thread.} = + echo "thread1" + {.cast(gcsafe).}: + for i in 0.. Date: Thu, 24 Aug 2023 15:25:15 -0700 Subject: [PATCH 004/445] add helpers --- datastore/databuffer.nim | 8 +++++++- tests/datastore/testsharedbuffer.nim | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 14d7d217..1ee5c5f6 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -54,9 +54,15 @@ proc `=copy`*(a: var DataBuffer; b: DataBuffer) = proc incrAtomicCount*(a: DataBuffer) = let res = atomicAddFetch(a.cnt, 1, ATOMIC_RELAXED) -proc getAtomicCount*(a: DataBuffer): int = +proc unsafeGetAtomicCount*(a: DataBuffer): int = atomicLoad(a.cnt, addr result, ATOMIC_RELAXED) +proc len*(a: DataBuffer): int = a.size + +proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = + result = newSeq[T](a.len) + copyMem(addr result[0], unsafeAddr a.buf[0], a.len) + proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = let cnt = cast[ptr int](allocShared0(sizeof(result.cnt))) cnt[] = 1 diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testsharedbuffer.nim index 499438ac..4a0f062e 100644 --- a/tests/datastore/testsharedbuffer.nim +++ b/tests/datastore/testsharedbuffer.nim @@ -43,8 +43,8 @@ proc thread1(val: int) {.thread.} = var myBytes = DataBuffer.new(@"hello world") myBytes2 = myBytes - echo "thread1: sending: ", myBytes, " cnt: ", myBytes.getAtomicCount() - echo "mybytes2: ", myBytes2, " cnt: ", myBytes2.getAtomicCount() + echo "thread1: sending: ", myBytes, " cnt: ", myBytes.unsafeGetAtomicCount() + echo "mybytes2: ", myBytes2, " cnt: ", myBytes2.unsafeGetAtomicCount() shareVal = myBytes echo "thread1: sent, left over: ", $myBytes @@ -60,7 +60,8 @@ proc thread2(val: int) {.thread.} = wait(cond, lock) echo "thread2: receiving " let msg: DataBuffer = shareVal - echo "thread2: received: ", msg, " cnt: ", msg.getAtomicCount() + echo "thread2: received: ", msg, " cnt: ", msg.unsafeGetAtomicCount() + check msg.toSeq(char) == @"hello world" # os.sleep(100) proc runBasicTest() = From da0eafe16e746f43506e832a062abce32bd0f69d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 15:51:25 -0700 Subject: [PATCH 005/445] setup shared datastore --- datastore/databuffer.nim | 3 + datastore/sharedds.nim | 137 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 datastore/sharedds.nim diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 1ee5c5f6..6f7543e8 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -13,6 +13,9 @@ type cnt: ptr int buf: ptr UncheckedArray[byte] size: int + + KeyBuffer* = DataBuffer + ValueBuffer* = DataBuffer proc `$`*(data: DataBuffer): string = if data.buf.isNil: diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim new file mode 100644 index 00000000..dbe74df7 --- /dev/null +++ b/datastore/sharedds.nim @@ -0,0 +1,137 @@ +import std/tables + +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/upraises + +import ./key +import ./query +import ./datastore + +export key, query + +push: {.upraises: [].} + +type + MountedStore* = object + store*: Datastore + key*: Key + + MountedDatastore* = ref object of Datastore + stores*: Table[Key, MountedStore] + +method mount*(self: MountedDatastore, key: Key, store: Datastore): ?!void {.base.} = + ## Mount a store on a namespace - namespaces are only `/` + ## + + if key in self.stores: + return failure("Key already has store mounted!") + + self.stores.add(key, MountedStore(store: store, key: key)) + + return success() + +func findStore*(self: MountedDatastore, key: Key): ?!MountedStore = + ## Find a store mounted under a particular key + ## + + for (k, v) in self.stores.pairs: + var + mounted = key + + while mounted.len > 0: + if ?k.path == ?mounted.path: + return success v + + if mounted.parent.isErr: + break + + mounted = mounted.parent.get + + failure newException(DatastoreKeyNotFound, "No datastore found for key") + +proc dispatch( + self: MountedDatastore, + key: Key): ?!tuple[store: MountedStore, relative: Key] = + ## Helper to retrieve the store and corresponding relative key + ## + + let + mounted = ?self.findStore(key) + + return success (store: mounted, relative: ?key.relative(mounted.key)) + +method has*( + self: MountedDatastore, + key: Key): Future[?!bool] {.async.} = + + without mounted =? self.dispatch(key): + return failure "No mounted datastore found" + + return (await mounted.store.store.has(mounted.relative)) + +method delete*( + self: MountedDatastore, + key: Key): Future[?!void] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return (await mounted.store.store.delete(mounted.relative)) + +method delete*( + self: MountedDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err + + return success() + +method get*( + self: MountedDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return await mounted.store.store.get(mounted.relative) + +method put*( + self: MountedDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return (await mounted.store.store.put(mounted.relative, data)) + +method put*( + self: MountedDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + return success() + +method close*(self: MountedDatastore): Future[?!void] {.async.} = + for s in self.stores.values: + discard await s.store.close() + + # TODO: how to handle failed close? + return success() + +func new*( + T: type MountedDatastore, + stores: Table[Key, Datastore] = initTable[Key, Datastore]()): ?!T = + + var self = T() + for (k, v) in stores.pairs: + self.stores[?k.path] = MountedStore(store: v, key: k) + + success self From 57450fe25afc2c8336f1980a144d7b741cdaaf47 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 15:57:15 -0700 Subject: [PATCH 006/445] make blank sharedds --- datastore/sharedds.nim | 134 ++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 83 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index dbe74df7..62a06a9d 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -14,104 +14,68 @@ export key, query push: {.upraises: [].} type - MountedStore* = object - store*: Datastore - key*: Key - MountedDatastore* = ref object of Datastore - stores*: Table[Key, MountedStore] - -method mount*(self: MountedDatastore, key: Key, store: Datastore): ?!void {.base.} = - ## Mount a store on a namespace - namespaces are only `/` - ## - - if key in self.stores: - return failure("Key already has store mounted!") - - self.stores.add(key, MountedStore(store: store, key: key)) - - return success() - -func findStore*(self: MountedDatastore, key: Key): ?!MountedStore = - ## Find a store mounted under a particular key - ## - - for (k, v) in self.stores.pairs: - var - mounted = key - - while mounted.len > 0: - if ?k.path == ?mounted.path: - return success v - - if mounted.parent.isErr: - break - - mounted = mounted.parent.get - - failure newException(DatastoreKeyNotFound, "No datastore found for key") - -proc dispatch( - self: MountedDatastore, - key: Key): ?!tuple[store: MountedStore, relative: Key] = - ## Helper to retrieve the store and corresponding relative key - ## - - let - mounted = ?self.findStore(key) - - return success (store: mounted, relative: ?key.relative(mounted.key)) + SharedDatastore* = ref object of Datastore + # stores*: Table[Key, SharedDatastore] method has*( - self: MountedDatastore, - key: Key): Future[?!bool] {.async.} = - - without mounted =? self.dispatch(key): - return failure "No mounted datastore found" + self: SharedDatastore, + key: Key +): Future[?!bool] {.async.} = - return (await mounted.store.store.has(mounted.relative)) + # without mounted =? self.dispatch(key): + # return failure "No mounted datastore found" + # return (await mounted.store.store.has(mounted.relative)) + return success(true) method delete*( - self: MountedDatastore, - key: Key): Future[?!void] {.async.} = + self: SharedDatastore, + key: Key +): Future[?!void] {.async.} = - without mounted =? self.dispatch(key), error: - return failure(error) - - return (await mounted.store.store.delete(mounted.relative)) + # without mounted =? self.dispatch(key), error: + # return failure(error) + # return (await mounted.store.store.delete(mounted.relative)) + return success() method delete*( - self: MountedDatastore, - keys: seq[Key]): Future[?!void] {.async.} = + self: SharedDatastore, + keys: seq[Key] +): Future[?!void] {.async.} = - for key in keys: - if err =? (await self.delete(key)).errorOption: - return failure err + # for key in keys: + # if err =? (await self.delete(key)).errorOption: + # return failure err return success() method get*( - self: MountedDatastore, - key: Key): Future[?!seq[byte]] {.async.} = + self: SharedDatastore, + key: Key +): Future[?!seq[byte]] {.async.} = - without mounted =? self.dispatch(key), error: - return failure(error) + # without mounted =? self.dispatch(key), error: + # return failure(error) - return await mounted.store.store.get(mounted.relative) + # return await mounted.store.store.get(mounted.relative) + return success(newSeq[byte]()) method put*( - self: MountedDatastore, + self: SharedDatastore, key: Key, - data: seq[byte]): Future[?!void] {.async.} = + data: seq[byte] +): Future[?!void] {.async.} = - without mounted =? self.dispatch(key), error: - return failure(error) + # without mounted =? self.dispatch(key), error: + # return failure(error) - return (await mounted.store.store.put(mounted.relative, data)) + # return (await mounted.store.store.put(mounted.relative, data)) + return success() method put*( - self: MountedDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = + self: SharedDatastore, + batch: seq[BatchEntry] +): Future[?!void] {.async.} = for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: @@ -119,19 +83,23 @@ method put*( return success() -method close*(self: MountedDatastore): Future[?!void] {.async.} = - for s in self.stores.values: - discard await s.store.close() +method close*( + self: SharedDatastore +): Future[?!void] {.async.} = + + # for s in self.stores.values: + # discard await s.store.close() # TODO: how to handle failed close? return success() -func new*( - T: type MountedDatastore, - stores: Table[Key, Datastore] = initTable[Key, Datastore]()): ?!T = +func new*[S: ref Datastore]( + T: typedesc[SharedDatastore], + storeTp: typedesc[S] +): ?!SharedDatastore = var self = T() - for (k, v) in stores.pairs: - self.stores[?k.path] = MountedStore(store: v, key: k) + # for (k, v) in stores.pairs: + # self.stores[?k.path] = MountedStore(store: v, key: k) success self From e592aaf74e31fd1e3b9021133c9e12b3f9e4eda1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 16:02:13 -0700 Subject: [PATCH 007/445] updating items --- datastore/threadbackend.nim | 105 ++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 datastore/threadbackend.nim diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim new file mode 100644 index 00000000..57a245d7 --- /dev/null +++ b/datastore/threadbackend.nim @@ -0,0 +1,105 @@ +import std/tables + +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/upraises + +import ./key +import ./query +import ./datastore +import ./databuffer + +export key, query + +push: {.upraises: [].} + +type + ThreadDatastore* = ref object of Datastore + # stores*: Table[KeyBuffer, ThreadDatastore] + +proc has*( + self: ThreadDatastore, + key: KeyBuffer +): Future[?!bool] {.async.} = + + # without mounted =? self.dispatch(key): + # return failure "No mounted datastore found" + # return (await mounted.store.store.has(mounted.relative)) + return success(true) + +proc delete*( + self: ThreadDatastore, + key: KeyBuffer +): Future[?!void] {.async.} = + + # without mounted =? self.dispatch(key), error: + # return failure(error) + # return (await mounted.store.store.delete(mounted.relative)) + return success() + +proc delete*( + self: ThreadDatastore, + keys: seq[KeyBuffer] +): Future[?!void] {.async.} = + + # for key in keys: + # if err =? (await self.delete(key)).errorOption: + # return failure err + + return success() + +proc get*( + self: ThreadDatastore, + key: KeyBuffer +): Future[?!DataBuffer] {.async.} = + + # without mounted =? self.dispatch(key), error: + # return failure(error) + + # return await mounted.store.store.get(mounted.relative) + return success(DataBuffer.new()) + +proc put*( + self: ThreadDatastore, + key: KeyBuffer, + data: DataBuffer +): Future[?!void] {.async.} = + + # without mounted =? self.dispatch(key), error: + # return failure(error) + + # return (await mounted.store.store.put(mounted.relative, data)) + return success() + +proc put*( + self: ThreadDatastore, + batch: seq[BatchEntry] +): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + return success() + +proc close*( + self: ThreadDatastore +): Future[?!void] {.async.} = + + # for s in self.stores.values: + # discard await s.store.close() + + # TODO: how to handle failed close? + return success() + +func new*[S: ref Datastore]( + T: typedesc[ThreadDatastore], + storeTp: typedesc[S] +): ?!ThreadDatastore = + + var self = T() + # for (k, v) in stores.pairs: + # self.stores[?k.path] = MountedStore(store: v, key: k) + + success self From edc93216583cea1b38c4991e9572ef64817ba12c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 16:18:31 -0700 Subject: [PATCH 008/445] updates --- datastore.nimble | 1 + datastore/threadbackend.nim | 40 ++++++++++++++++++++++++++++--------- tests/exampletaskpool.nim | 34 +++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 tests/exampletaskpool.nim diff --git a/datastore.nimble b/datastore.nimble index 6252c6d1..e14bb515 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -13,6 +13,7 @@ requires "nim >= 1.2.0", "sqlite3_abi", "stew", "unittest2", + "patty", "upraises >= 0.1.0 & < 0.2.0" task coverage, "generates code coverage report": diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 57a245d7..27fe9c39 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -4,6 +4,8 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/upraises +import pkg/taskpools +import pkg/patty import ./key import ./query @@ -14,9 +16,29 @@ export key, query push: {.upraises: [].} +variant Shape: + Circle(r: float) + Rectangle(w: float, h: float) + type + DatastoreBackend* {.pure.} = enum + FileSystem + SQlite + ThreadDatastore* = ref object of Datastore - # stores*: Table[KeyBuffer, ThreadDatastore] + tp: Taskpool + +var backendDatastore {.threadvar.}: ref Datastore + +proc startupDatastore(backend: DatastoreBackend): bool = + + case backend: + of FileSystem: + backendDatastore = FSDatastore.new( + root: string, + depth = 2, + caseSensitive = true, + ignoreProtected = false proc has*( self: ThreadDatastore, @@ -86,20 +108,20 @@ proc put*( proc close*( self: ThreadDatastore ): Future[?!void] {.async.} = - - # for s in self.stores.values: - # discard await s.store.close() - - # TODO: how to handle failed close? + self.tp.shutdown() return success() func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], - storeTp: typedesc[S] + backend: DatastoreBackend, ): ?!ThreadDatastore = var self = T() - # for (k, v) in stores.pairs: - # self.stores[?k.path] = MountedStore(store: v, key: k) + self.tp = Taskpool.new(num_threads = 1) ##\ + ## Default to one thread, multiple threads \ + ## will require more work + + let pending = self.tp.spawn startupDatastore(backend) + sync pending success self diff --git a/tests/exampletaskpool.nim b/tests/exampletaskpool.nim new file mode 100644 index 00000000..c4102351 --- /dev/null +++ b/tests/exampletaskpool.nim @@ -0,0 +1,34 @@ + + +import + std/[strutils, math, cpuinfo], + taskpools + +# From https://github.com/nim-lang/Nim/blob/v1.6.2/tests/parallel/tpi.nim +# Leibniz Formula https://en.wikipedia.org/wiki/Leibniz_formula_for_%CF%80 +proc term(k: int): float = + if k mod 2 == 1: + -4'f / float(2*k + 1) + else: + 4'f / float(2*k + 1) + +proc piApprox(tp: Taskpool, n: int): float = + var pendingFuts = newSeq[FlowVar[float]](n) + for k in 0 ..< pendingFuts.len: + pendingFuts[k] = tp.spawn term(k) # Schedule a task on the threadpool a return a handle to retrieve the result. + for k in 0 ..< pendingFuts.len: + result += sync pendingFuts[k] # Block until the result is available. + +proc main() = + var n = 1_000_000 + var nthreads = countProcessors() + + var tp = Taskpool.new(num_threads = nthreads) # Default to the number of hardware threads. + + echo formatFloat(tp.piApprox(n)) + + tp.syncAll() # Block until all pending tasks are processed (implied in tp.shutdown()) + tp.shutdown() + +# Compile with nim c -r -d:release --threads:on --outdir:build example.nim +main() \ No newline at end of file From d39c0965ffdc3f469ad5a03a3600d1440e2b6332 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 16:29:25 -0700 Subject: [PATCH 009/445] setting up thread backend --- datastore/databuffer.nim | 14 ++++++++---- datastore/threadbackend.nim | 45 +++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 6f7543e8..9be717a7 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -16,6 +16,7 @@ type KeyBuffer* = DataBuffer ValueBuffer* = DataBuffer + StringBuffer* = DataBuffer proc `$`*(data: DataBuffer): string = if data.buf.isNil: @@ -62,10 +63,6 @@ proc unsafeGetAtomicCount*(a: DataBuffer): int = proc len*(a: DataBuffer): int = a.size -proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = - result = newSeq[T](a.len) - copyMem(addr result[0], unsafeAddr a.buf[0], a.len) - proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = let cnt = cast[ptr int](allocShared0(sizeof(result.cnt))) cnt[] = 1 @@ -81,3 +78,12 @@ proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuf result = DataBuffer.new(data.len) if data.len() > 0: copyMem(result.buf, unsafeAddr data[0], data.len) + +proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = + result = newSeq[T](a.len) + copyMem(addr result[0], unsafeAddr a.buf[0], a.len) + +proc toString*(data: StringBuffer): string = + result = newString(data.len()) + if data.len() > 0: + copyMem(addr result[0], unsafeAddr data.buf[0], data.len) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 27fe9c39..3bc5c919 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -5,40 +5,51 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises import pkg/taskpools -import pkg/patty import ./key import ./query import ./datastore import ./databuffer +import fsds + export key, query push: {.upraises: [].} -variant Shape: - Circle(r: float) - Rectangle(w: float, h: float) - type - DatastoreBackend* {.pure.} = enum - FileSystem - SQlite + ThreadBackendKind* {.pure.} = enum + FSBackend + SQliteBackend + + ThreadBackend* = object + case kind*: ThreadBackendKind + of FSBackend: + root: StringBuffer + depth: int + caseSensitive: bool + ignoreProtected: bool + of SQliteBackend: + discard ThreadDatastore* = ref object of Datastore tp: Taskpool var backendDatastore {.threadvar.}: ref Datastore -proc startupDatastore(backend: DatastoreBackend): bool = - - case backend: - of FileSystem: - backendDatastore = FSDatastore.new( - root: string, - depth = 2, - caseSensitive = true, - ignoreProtected = false +proc startupDatastore(backend: ThreadBackend): bool = + ## starts up a FS instance on a give thread + case backend.kind: + of FSBackend: + let res = FSDatastore.new( + root = backend.root.toString(), + depth = backend.depth, + caseSensitive = backend.caseSensitive, + ignoreProtected = backend.ignoreProtected) + if res.isOk: + backendDatastore = res.get() + else: + discard proc has*( self: ThreadDatastore, From d09ef9b21a3508cf2891465fe3503e879de66cc4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 16:32:57 -0700 Subject: [PATCH 010/445] plumbing --- datastore/threadbackend.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 3bc5c919..a4c1bfbd 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -35,7 +35,7 @@ type ThreadDatastore* = ref object of Datastore tp: Taskpool -var backendDatastore {.threadvar.}: ref Datastore +var backendDatastore {.threadvar.}: Datastore proc startupDatastore(backend: ThreadBackend): bool = ## starts up a FS instance on a give thread @@ -47,7 +47,7 @@ proc startupDatastore(backend: ThreadBackend): bool = caseSensitive = backend.caseSensitive, ignoreProtected = backend.ignoreProtected) if res.isOk: - backendDatastore = res.get() + backendDatastore = res.get() else: discard @@ -124,7 +124,7 @@ proc close*( func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], - backend: DatastoreBackend, + backend: ThreadBackend, ): ?!ThreadDatastore = var self = T() From 055edb4a41a47b52068540d5c9781d8d7a1f0abf Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 17:33:32 -0700 Subject: [PATCH 011/445] plumbing values --- datastore/databuffer.nim | 12 +++++- datastore/sharedds.nim | 23 ++++------ datastore/threadbackend.nim | 84 +++++++++---------------------------- 3 files changed, 40 insertions(+), 79 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 9be717a7..dd745fa9 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -17,6 +17,8 @@ type KeyBuffer* = DataBuffer ValueBuffer* = DataBuffer StringBuffer* = DataBuffer + CatchableErrorBuffer* = object + msg: StringBuffer proc `$`*(data: DataBuffer): string = if data.buf.isNil: @@ -83,7 +85,15 @@ proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = result = newSeq[T](a.len) copyMem(addr result[0], unsafeAddr a.buf[0], a.len) -proc toString*(data: StringBuffer): string = +proc toString*(data: DataBuffer): string = result = newString(data.len()) if data.len() > 0: copyMem(addr result[0], unsafeAddr data.buf[0], data.len) + +proc toCatchable*(data: CatchableErrorBuffer): ref CatchableError = + result = (ref CatchableError)(msg: data.msg.toString()) + +proc toBuffer*(err: ref Exception): CatchableErrorBuffer = + return CatchableErrorBuffer( + msg: StringBuffer.new(err.msg) + ) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 62a06a9d..a374a11e 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -1,6 +1,7 @@ import std/tables import pkg/chronos +import pkg/chronos/threadsync import pkg/questionable import pkg/questionable/results import pkg/upraises @@ -66,22 +67,18 @@ method put*( data: seq[byte] ): Future[?!void] {.async.} = - # without mounted =? self.dispatch(key), error: - # return failure(error) - - # return (await mounted.store.store.put(mounted.relative, data)) - return success() + let signal = ThreadSignalPtr.new() + if signal.isErr: + return failure("error creating signal") + else: + await wait(signal.get()) + return success() method put*( self: SharedDatastore, batch: seq[BatchEntry] ): Future[?!void] {.async.} = - - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err - - return success() + raiseAssert("Not implemented!") method close*( self: SharedDatastore @@ -98,8 +95,6 @@ func new*[S: ref Datastore]( storeTp: typedesc[S] ): ?!SharedDatastore = - var self = T() - # for (k, v) in stores.pairs: - # self.stores[?k.path] = MountedStore(store: v, key: k) + var self = SharedDatastore() success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index a4c1bfbd..69704439 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -1,8 +1,6 @@ -import std/tables -import pkg/chronos -import pkg/questionable -import pkg/questionable/results +import pkg/chronos/threadsync +import stew/results import pkg/upraises import pkg/taskpools @@ -32,7 +30,7 @@ type of SQliteBackend: discard - ThreadDatastore* = ref object of Datastore + ThreadDatastore* = ptr object tp: Taskpool var backendDatastore {.threadvar.}: Datastore @@ -51,81 +49,39 @@ proc startupDatastore(backend: ThreadBackend): bool = else: discard -proc has*( - self: ThreadDatastore, - key: KeyBuffer -): Future[?!bool] {.async.} = - - # without mounted =? self.dispatch(key): - # return failure "No mounted datastore found" - # return (await mounted.store.store.has(mounted.relative)) - return success(true) - -proc delete*( - self: ThreadDatastore, - key: KeyBuffer -): Future[?!void] {.async.} = - - # without mounted =? self.dispatch(key), error: - # return failure(error) - # return (await mounted.store.store.delete(mounted.relative)) - return success() - -proc delete*( - self: ThreadDatastore, - keys: seq[KeyBuffer] -): Future[?!void] {.async.} = - - # for key in keys: - # if err =? (await self.delete(key)).errorOption: - # return failure err - - return success() - proc get*( self: ThreadDatastore, + signal: ThreadSignalPtr, key: KeyBuffer -): Future[?!DataBuffer] {.async.} = - - # without mounted =? self.dispatch(key), error: - # return failure(error) +): Result[DataBuffer, CatchableErrorBuffer] = - # return await mounted.store.store.get(mounted.relative) - return success(DataBuffer.new()) + return ok(DataBuffer.new()) proc put*( self: ThreadDatastore, + signal: ThreadSignalPtr, key: KeyBuffer, data: DataBuffer -): Future[?!void] {.async.} = +): Result[void, CatchableErrorBuffer] = - # without mounted =? self.dispatch(key), error: - # return failure(error) + return ok() - # return (await mounted.store.store.put(mounted.relative, data)) - return success() - -proc put*( - self: ThreadDatastore, - batch: seq[BatchEntry] -): Future[?!void] {.async.} = - - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err - - return success() proc close*( - self: ThreadDatastore -): Future[?!void] {.async.} = - self.tp.shutdown() - return success() + self: ThreadDatastore, + signal: ThreadSignalPtr, +): Result[void, CatchableErrorBuffer] = + try: + self[].tp.shutdown() + return ok() + except Exception as exc: + return err(exc.toBuffer()) func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], + signal: ThreadSignalPtr, backend: ThreadBackend, -): ?!ThreadDatastore = +): Result[ThreadDatastore, CatchableErrorBuffer] = var self = T() self.tp = Taskpool.new(num_threads = 1) ##\ @@ -135,4 +91,4 @@ func new*[S: ref Datastore]( let pending = self.tp.spawn startupDatastore(backend) sync pending - success self + ok self From 5ef4196e5139fa69d29405eb1e354a84b4da8ddb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 17:59:43 -0700 Subject: [PATCH 012/445] plumbing values --- datastore.nimble | 1 + datastore/databuffer.nim | 1 - datastore/sharedds.nim | 14 +++++--------- datastore/threadbackend.nim | 16 ++++++++++++---- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/datastore.nimble b/datastore.nimble index e14bb515..93ec6af4 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -14,6 +14,7 @@ requires "nim >= 1.2.0", "stew", "unittest2", "patty", + "threading", "upraises >= 0.1.0 & < 0.2.0" task coverage, "generates code coverage report": diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index dd745fa9..1367b40e 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -7,7 +7,6 @@ import std/atomics import events - type DataBuffer* = object cnt: ptr int diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index a374a11e..512836a5 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -67,12 +67,11 @@ method put*( data: seq[byte] ): Future[?!void] {.async.} = - let signal = ThreadSignalPtr.new() - if signal.isErr: - return failure("error creating signal") - else: - await wait(signal.get()) - return success() + let signal = ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") + + await wait(signal) + return success() method put*( self: SharedDatastore, @@ -84,9 +83,6 @@ method close*( self: SharedDatastore ): Future[?!void] {.async.} = - # for s in self.stores.values: - # discard await s.store.close() - # TODO: how to handle failed close? return success() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 69704439..9e4d7f2c 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -8,6 +8,7 @@ import ./key import ./query import ./datastore import ./databuffer +import threading/smartptrs import fsds @@ -16,6 +17,11 @@ export key, query push: {.upraises: [].} type + + ThreadResult*[T: DataBuffer | void] = Result[T, CatchableErrorBuffer] + + TResult*[T] = UniquePtr[ThreadResult[T]] + ThreadBackendKind* {.pure.} = enum FSBackend SQliteBackend @@ -35,6 +41,9 @@ type var backendDatastore {.threadvar.}: Datastore +proc new*[T](tp: typedesc[TResult[T]]): TResult[T] = + newUniquePtr(ThreadResult[T]) + proc startupDatastore(backend: ThreadBackend): bool = ## starts up a FS instance on a give thread case backend.kind: @@ -61,11 +70,10 @@ proc put*( self: ThreadDatastore, signal: ThreadSignalPtr, key: KeyBuffer, - data: DataBuffer -): Result[void, CatchableErrorBuffer] = - - return ok() + data: DataBuffer, +): TResult[void] = + return TResult[void].new() proc close*( self: ThreadDatastore, From 3bde00d111a142811f5cbd6487a33318df4f61ee Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:06:02 -0700 Subject: [PATCH 013/445] plumbing values --- datastore/sharedds.nim | 18 ------------------ datastore/threadbackend.nim | 6 +++--- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 512836a5..79cafb81 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -23,42 +23,24 @@ method has*( self: SharedDatastore, key: Key ): Future[?!bool] {.async.} = - - # without mounted =? self.dispatch(key): - # return failure "No mounted datastore found" - # return (await mounted.store.store.has(mounted.relative)) return success(true) method delete*( self: SharedDatastore, key: Key ): Future[?!void] {.async.} = - - # without mounted =? self.dispatch(key), error: - # return failure(error) - # return (await mounted.store.store.delete(mounted.relative)) return success() method delete*( self: SharedDatastore, keys: seq[Key] ): Future[?!void] {.async.} = - - # for key in keys: - # if err =? (await self.delete(key)).errorOption: - # return failure err - return success() method get*( self: SharedDatastore, key: Key ): Future[?!seq[byte]] {.async.} = - - # without mounted =? self.dispatch(key), error: - # return failure(error) - - # return await mounted.store.store.get(mounted.relative) return success(newSeq[byte]()) method put*( diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 9e4d7f2c..019d8b6c 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -78,18 +78,18 @@ proc put*( proc close*( self: ThreadDatastore, signal: ThreadSignalPtr, -): Result[void, CatchableErrorBuffer] = +): TResult[void] = try: self[].tp.shutdown() return ok() except Exception as exc: - return err(exc.toBuffer()) + return TResult[void].new() func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], signal: ThreadSignalPtr, backend: ThreadBackend, -): Result[ThreadDatastore, CatchableErrorBuffer] = +): TResult[ThreadDatastore] = var self = T() self.tp = Taskpool.new(num_threads = 1) ##\ From a0d60cc9be796bd7dd102e9e38a5143920454646 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:28:30 -0700 Subject: [PATCH 014/445] reworking results and tasks --- datastore/sharedds.nim | 15 ++++++-- datastore/threadbackend.nim | 72 +++++++++++++++++++++++-------------- 2 files changed, 59 insertions(+), 28 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 79cafb81..a4ad5646 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -5,10 +5,12 @@ import pkg/chronos/threadsync import pkg/questionable import pkg/questionable/results import pkg/upraises +import pkg/taskpools import ./key import ./query import ./datastore +import ./threadbackend export key, query @@ -18,6 +20,11 @@ type SharedDatastore* = ref object of Datastore # stores*: Table[Key, SharedDatastore] + tds: ThreadDatastorePtr + +template newSignal(): auto = + ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") method has*( self: SharedDatastore, @@ -70,9 +77,13 @@ method close*( func new*[S: ref Datastore]( T: typedesc[SharedDatastore], - storeTp: typedesc[S] + backend: ThreadBackend, ): ?!SharedDatastore = - var self = SharedDatastore() + var + self = SharedDatastore() + signal = newSignal() + res = TResult[ThreadDatastore].new() + self.tds = ThreadDatastore.new(signal, backend, res) success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 019d8b6c..f64716d1 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -12,13 +12,17 @@ import threading/smartptrs import fsds -export key, query +export key, query, smartptrs, databuffer push: {.upraises: [].} type - ThreadResult*[T: DataBuffer | void] = Result[T, CatchableErrorBuffer] + ThreadResult*[T: DataBuffer | void] = object + ready*: bool + success*: bool + val*: T + err*: CatchableErrorBuffer TResult*[T] = UniquePtr[ThreadResult[T]] @@ -36,27 +40,43 @@ type of SQliteBackend: discard - ThreadDatastore* = ptr object - tp: Taskpool + ThreadDatastore* = object + taskpool: Taskpool + backendDatastore: Datastore -var backendDatastore {.threadvar.}: Datastore + ThreadDatastorePtr* = SharedPtr[ThreadDatastore] proc new*[T](tp: typedesc[TResult[T]]): TResult[T] = newUniquePtr(ThreadResult[T]) -proc startupDatastore(backend: ThreadBackend): bool = +proc startupDatastore( + signal: ThreadSignalPtr, + backend: ThreadBackend, + res: TResult[ThreadDatastorePtr], +): bool = ## starts up a FS instance on a give thread case backend.kind: of FSBackend: - let res = FSDatastore.new( + let ds = FSDatastore.new( root = backend.root.toString(), depth = backend.depth, caseSensitive = backend.caseSensitive, - ignoreProtected = backend.ignoreProtected) - if res.isOk: - backendDatastore = res.get() + ignoreProtected = backend.ignoreProtected + ) + if ds.isOk: + let tds = newSharedPtr(ThreadDatastore) + tds[].backendDatastore = ds.get() + + res[].ready = true + res[].success = true + res[].val = tds + else: + res[].ready = true + res[].success = false else: discard + + signal.fireSync().get() proc get*( self: ThreadDatastore, @@ -66,37 +86,37 @@ proc get*( return ok(DataBuffer.new()) -proc put*( +proc putWorker*( self: ThreadDatastore, signal: ThreadSignalPtr, key: KeyBuffer, data: DataBuffer, -): TResult[void] = + res: TResult[void] +) = - return TResult[void].new() + discard -proc close*( - self: ThreadDatastore, - signal: ThreadSignalPtr, -): TResult[void] = - try: - self[].tp.shutdown() - return ok() - except Exception as exc: - return TResult[void].new() +# proc close*( +# self: ThreadDatastore, +# signal: ThreadSignalPtr, +# ): TResult[void] = +# try: +# self[].tp.shutdown() +# return ok() +# except Exception as exc: +# return TResult[void].new() func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], signal: ThreadSignalPtr, backend: ThreadBackend, -): TResult[ThreadDatastore] = + res: TResult[ThreadDatastore] +) = var self = T() self.tp = Taskpool.new(num_threads = 1) ##\ ## Default to one thread, multiple threads \ ## will require more work - let pending = self.tp.spawn startupDatastore(backend) - sync pending + let pending = self.tp.spawn startupDatastore(signal, backend) - ok self From 972ee22efdbc014353ce1d50d7d8b211bd37ae1c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:33:20 -0700 Subject: [PATCH 015/445] more changes --- datastore/threadbackend.nim | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f64716d1..328e0fa8 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -18,9 +18,14 @@ push: {.upraises: [].} type + ThreadResultKind* {.pure.} = enum + NotReady + Success + Error + ThreadResult*[T: DataBuffer | void] = object - ready*: bool - success*: bool + state*: ThreadResultKind + signal*: ThreadSignalPtr val*: T err*: CatchableErrorBuffer @@ -52,7 +57,7 @@ proc new*[T](tp: typedesc[TResult[T]]): TResult[T] = proc startupDatastore( signal: ThreadSignalPtr, backend: ThreadBackend, - res: TResult[ThreadDatastorePtr], + ret: TResult[ThreadDatastorePtr], ): bool = ## starts up a FS instance on a give thread case backend.kind: @@ -67,33 +72,30 @@ proc startupDatastore( let tds = newSharedPtr(ThreadDatastore) tds[].backendDatastore = ds.get() - res[].ready = true - res[].success = true - res[].val = tds + ret[].val = tds + ret[].state = Success else: - res[].ready = true - res[].success = false + ret[].state = Error else: discard - signal.fireSync().get() + ret[].signal.fireSync().get() -proc get*( - self: ThreadDatastore, - signal: ThreadSignalPtr, - key: KeyBuffer -): Result[DataBuffer, CatchableErrorBuffer] = +proc getTask*( + self: ThreadDatastorePtr, + key: KeyBuffer, + ret: TResult[DataBuffer] +) = - return ok(DataBuffer.new()) + # return ok(DataBuffer.new()) + discard -proc putWorker*( - self: ThreadDatastore, - signal: ThreadSignalPtr, +proc putTask*( + self: ThreadDatastorePtr, key: KeyBuffer, data: DataBuffer, - res: TResult[void] + ret: TResult[void] ) = - discard # proc close*( @@ -110,7 +112,7 @@ func new*[S: ref Datastore]( T: typedesc[ThreadDatastore], signal: ThreadSignalPtr, backend: ThreadBackend, - res: TResult[ThreadDatastore] + ret: TResult[ThreadDatastore] ) = var self = T() From ec4b44212574d1f4d1fd210ee8947d4628b9e50a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:35:56 -0700 Subject: [PATCH 016/445] more changes --- datastore/sharedds.nim | 5 +++-- datastore/threadbackend.nim | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index a4ad5646..c33fa2ea 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -82,8 +82,9 @@ func new*[S: ref Datastore]( var self = SharedDatastore() - signal = newSignal() res = TResult[ThreadDatastore].new() - self.tds = ThreadDatastore.new(signal, backend, res) + + res[].signal = newSignal() + self.tds = ThreadDatastore.new(backend, res) success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 328e0fa8..efc2ee1f 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -86,7 +86,6 @@ proc getTask*( key: KeyBuffer, ret: TResult[DataBuffer] ) = - # return ok(DataBuffer.new()) discard @@ -108,7 +107,7 @@ proc putTask*( # except Exception as exc: # return TResult[void].new() -func new*[S: ref Datastore]( +func new*( T: typedesc[ThreadDatastore], signal: ThreadSignalPtr, backend: ThreadBackend, From 368c25172e20e6ee9028591a72b61ca4c154a29d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:48:56 -0700 Subject: [PATCH 017/445] more changes --- datastore/sharedds.nim | 9 +++++---- datastore/threadbackend.nim | 34 +++++++++++++++++----------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index c33fa2ea..b720be72 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -75,16 +75,17 @@ method close*( # TODO: how to handle failed close? return success() -func new*[S: ref Datastore]( - T: typedesc[SharedDatastore], +func newSharedDataStore*( + # T: typedesc[SharedDatastore], backend: ThreadBackend, ): ?!SharedDatastore = var self = SharedDatastore() - res = TResult[ThreadDatastore].new() + res = newThreadResult(ThreadDatastore) res[].signal = newSignal() - self.tds = ThreadDatastore.new(backend, res) + res.createThreadDatastore(backend) + await wait(res[].signal) success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index efc2ee1f..fc552099 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -26,8 +26,8 @@ type ThreadResult*[T: DataBuffer | void] = object state*: ThreadResultKind signal*: ThreadSignalPtr - val*: T - err*: CatchableErrorBuffer + value*: T + error*: CatchableErrorBuffer TResult*[T] = UniquePtr[ThreadResult[T]] @@ -45,20 +45,19 @@ type of SQliteBackend: discard - ThreadDatastore* = object - taskpool: Taskpool + ThreadDatastore = object + tp: Taskpool backendDatastore: Datastore ThreadDatastorePtr* = SharedPtr[ThreadDatastore] -proc new*[T](tp: typedesc[TResult[T]]): TResult[T] = +proc newThreadResult*[T](tp: typedesc[TResult[T]]): TResult[T] = newUniquePtr(ThreadResult[T]) proc startupDatastore( - signal: ThreadSignalPtr, - backend: ThreadBackend, ret: TResult[ThreadDatastorePtr], -): bool = + backend: ThreadBackend, +) = ## starts up a FS instance on a give thread case backend.kind: of FSBackend: @@ -72,14 +71,14 @@ proc startupDatastore( let tds = newSharedPtr(ThreadDatastore) tds[].backendDatastore = ds.get() - ret[].val = tds + ret[].value = tds ret[].state = Success else: ret[].state = Error else: discard - ret[].signal.fireSync().get() + discard ret[].signal.fireSync().get() proc getTask*( self: ThreadDatastorePtr, @@ -107,17 +106,18 @@ proc putTask*( # except Exception as exc: # return TResult[void].new() -func new*( - T: typedesc[ThreadDatastore], - signal: ThreadSignalPtr, +func createThreadDatastore*( + ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, - ret: TResult[ThreadDatastore] ) = - var self = T() - self.tp = Taskpool.new(num_threads = 1) ##\ + try: + ret[].value[].tp = Taskpool.new(num_threads = 1) ##\ ## Default to one thread, multiple threads \ ## will require more work + ret[].value[].tp.spawn startupDatastore(ret, backend) - let pending = self.tp.spawn startupDatastore(signal, backend) + except Exception as exc: + ret[].state = Error + ret[].error = exc.toBuffer() From 28097881f80a5fae8f855a6a6096cbeafcb73312 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:53:06 -0700 Subject: [PATCH 018/445] getting somewhere now --- datastore/threadbackend.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index fc552099..fcc16771 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -57,7 +57,7 @@ proc newThreadResult*[T](tp: typedesc[TResult[T]]): TResult[T] = proc startupDatastore( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, -) = +) {.raises: [].} = ## starts up a FS instance on a give thread case backend.kind: of FSBackend: @@ -68,10 +68,7 @@ proc startupDatastore( ignoreProtected = backend.ignoreProtected ) if ds.isOk: - let tds = newSharedPtr(ThreadDatastore) - tds[].backendDatastore = ds.get() - - ret[].value = tds + ret[].value[].backendDatastore = ds.get() ret[].state = Success else: ret[].state = Error @@ -106,7 +103,7 @@ proc putTask*( # except Exception as exc: # return TResult[void].new() -func createThreadDatastore*( +proc createThreadDatastore*( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, ) = From 8644cbc9dc3d37d795bad53ad6e888f528186f3a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 18:57:06 -0700 Subject: [PATCH 019/445] fixes --- datastore/sharedds.nim | 4 ++-- datastore/threadbackend.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index b720be72..573e7e30 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -75,14 +75,14 @@ method close*( # TODO: how to handle failed close? return success() -func newSharedDataStore*( +proc newSharedDataStore*( # T: typedesc[SharedDatastore], backend: ThreadBackend, ): ?!SharedDatastore = var self = SharedDatastore() - res = newThreadResult(ThreadDatastore) + res = newThreadResult(ThreadDatastorePtr) res[].signal = newSignal() res.createThreadDatastore(backend) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index fcc16771..f9438a0c 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -51,7 +51,7 @@ type ThreadDatastorePtr* = SharedPtr[ThreadDatastore] -proc newThreadResult*[T](tp: typedesc[TResult[T]]): TResult[T] = +proc newThreadResult*[T](tp: typedesc[T]): UniquePtr[ThreadResult[T]] = newUniquePtr(ThreadResult[T]) proc startupDatastore( From 567e0d700351a13e8b5c603cf9cf6820f0653635 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:02:46 -0700 Subject: [PATCH 020/445] fixes --- datastore/sharedds.nim | 9 +++------ datastore/threadbackend.nim | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 573e7e30..1bc22e08 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -22,10 +22,6 @@ type # stores*: Table[Key, SharedDatastore] tds: ThreadDatastorePtr -template newSignal(): auto = - ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") - method has*( self: SharedDatastore, key: Key @@ -78,13 +74,14 @@ method close*( proc newSharedDataStore*( # T: typedesc[SharedDatastore], backend: ThreadBackend, -): ?!SharedDatastore = +): Future[?!SharedDatastore] {.async.} = var self = SharedDatastore() res = newThreadResult(ThreadDatastorePtr) - res[].signal = newSignal() + res[].signal = ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") res.createThreadDatastore(backend) await wait(res[].signal) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f9438a0c..fb289ea1 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -23,7 +23,7 @@ type Success Error - ThreadResult*[T: DataBuffer | void] = object + ThreadResult*[T: DataBuffer | void | ThreadDatastorePtr] = object state*: ThreadResultKind signal*: ThreadSignalPtr value*: T From 321a4b67498d139ec3bfe4c8778d787452592ba3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:05:12 -0700 Subject: [PATCH 021/445] cleanup --- datastore/sharedds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 1bc22e08..e68c3fa8 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -79,10 +79,11 @@ proc newSharedDataStore*( var self = SharedDatastore() res = newThreadResult(ThreadDatastorePtr) - + res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") res.createThreadDatastore(backend) await wait(res[].signal) + res[].signal.close() success self From 567f84e4605372fa20e93b98e95f288197628b83 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:10:08 -0700 Subject: [PATCH 022/445] cleanup --- datastore/sharedds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index e68c3fa8..c2587f91 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -82,8 +82,9 @@ proc newSharedDataStore*( res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") - res.createThreadDatastore(backend) + res.createThreadDatastore(backend) await wait(res[].signal) res[].signal.close() + success self From b48e45c03f33e2096e6c85cbd7d4db349319b6df Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:50:58 -0700 Subject: [PATCH 023/445] adding testing --- datastore/sharedds.nim | 14 +++++++------- datastore/threadbackend.nim | 22 +++++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index c2587f91..8c334555 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -12,7 +12,7 @@ import ./query import ./datastore import ./threadbackend -export key, query +export key, query, ThreadBackend push: {.upraises: [].} @@ -78,13 +78,13 @@ proc newSharedDataStore*( var self = SharedDatastore() - res = newThreadResult(ThreadDatastorePtr) + # res = newThreadResult(ThreadDatastorePtr) - res[].signal = ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") + # res[].signal = ThreadSignalPtr.new().valueOr: + # return failure newException(DatastoreError, "error creating signal") - res.createThreadDatastore(backend) - await wait(res[].signal) - res[].signal.close() + # res.createThreadDatastore(backend) + # await wait(res[].signal) + # res[].signal.close() success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index fb289ea1..6f5c0dc9 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -29,21 +29,24 @@ type value*: T error*: CatchableErrorBuffer - TResult*[T] = UniquePtr[ThreadResult[T]] + TResult*[T] = SharedPtr[ThreadResult[T]] ThreadBackendKind* {.pure.} = enum FSBackend SQliteBackend + TestBackend ThreadBackend* = object case kind*: ThreadBackendKind of FSBackend: - root: StringBuffer - depth: int - caseSensitive: bool - ignoreProtected: bool + root*: StringBuffer + depth*: int + caseSensitive*: bool + ignoreProtected*: bool of SQliteBackend: discard + of TestBackend: + count*: int ThreadDatastore = object tp: Taskpool @@ -51,8 +54,8 @@ type ThreadDatastorePtr* = SharedPtr[ThreadDatastore] -proc newThreadResult*[T](tp: typedesc[T]): UniquePtr[ThreadResult[T]] = - newUniquePtr(ThreadResult[T]) +proc newThreadResult*[T](tp: typedesc[T]): SharedPtr[ThreadResult[T]] = + newSharedPtr(ThreadResult[T]) proc startupDatastore( ret: TResult[ThreadDatastorePtr], @@ -72,6 +75,11 @@ proc startupDatastore( ret[].state = Success else: ret[].state = Error + ret[].value[].backendDatastore = ds.get() + ret[].state = Success + of TestBackend: + ret[].value[].backendDatastore = nil + ret[].state = Success else: discard From 0daf1683e807fd97361e9e9b9a30bab6031ea8b7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:51:04 -0700 Subject: [PATCH 024/445] adding testing --- tests/datastore/testsharedds.nim | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/datastore/testsharedds.nim diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim new file mode 100644 index 00000000..99167ff5 --- /dev/null +++ b/tests/datastore/testsharedds.nim @@ -0,0 +1,147 @@ +import std/options +import std/sequtils +import std/os +from std/algorithm import sort, reversed + +import pkg/asynctest/unittest2 +import pkg/chronos +import pkg/stew/results +import pkg/stew/byteutils + +import pkg/datastore/sharedds + +import ./dscommontests +import ./querycommontests + +suite "Test Basic SharedDatastore": + + test "check create": + + let backend = ThreadBackend( + kind: TestBackend, + ) + let sds = newSharedDataStore(backend) + +# suite "Test Basic FSDatastore": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes + +# var +# fsStore: FSDatastore + +# setupAll: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) + +# fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + +# teardownAll: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) + +# basicStoreTests(fsStore, key, bytes, otherBytes) + +# suite "Test Misc FSDatastore": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath +# bytes = "some bytes".toBytes + +# setup: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) + +# teardown: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) + +# test "Test validDepth()": +# let +# fs = FSDatastore.new(root = "/", depth = 3).tryGet() +# invalid = Key.init("/a/b/c/d").tryGet() +# valid = Key.init("/a/b/c").tryGet() + +# check: +# not fs.validDepth(invalid) +# fs.validDepth(valid) + +# test "Test invalid key (path) depth": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/b/c/d").tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + +# test "Test valid key (path) depth": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/b/c").tryGet() + +# check: +# (await fs.put(key, bytes)).isOk +# (await fs.get(key)).isOk +# (await fs.delete(key)).isOk +# (await fs.has(key)).isOk + +# test "Test key cannot write outside of root": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/../../c").tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + +# test "Test key cannot convert to invalid path": +# let +# fs = FSDatastore.new(root = basePathAbs).tryGet() + +# for c in invalidFilenameChars: +# if c == ':': continue +# if c == '/': continue + +# let +# key = Key.init("/" & c).tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + +# suite "Test Query": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath + +# var +# ds: FSDatastore + +# setup: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) + +# ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() + +# teardown: + +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) + +# queryTests(ds, false) From d5fe24c37287db76e1277e2a56c503c643ed1459 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:55:44 -0700 Subject: [PATCH 025/445] adding testing --- datastore/sharedds.nim | 5 ++++- datastore/threadbackend.nim | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 8c334555..4c84379b 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -11,6 +11,7 @@ import ./key import ./query import ./datastore import ./threadbackend +import threading/smartptrs export key, query, ThreadBackend @@ -78,7 +79,9 @@ proc newSharedDataStore*( var self = SharedDatastore() - # res = newThreadResult(ThreadDatastorePtr) + res = newThreadResult(ThreadDatastorePtr) + + let buf = DataBuffer.new() # res[].signal = ThreadSignalPtr.new().valueOr: # return failure newException(DatastoreError, "error creating signal") diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 6f5c0dc9..bdcbf223 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -54,7 +54,7 @@ type ThreadDatastorePtr* = SharedPtr[ThreadDatastore] -proc newThreadResult*[T](tp: typedesc[T]): SharedPtr[ThreadResult[T]] = +proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) proc startupDatastore( From 2e9695ea86de2adeca4ae002ef1057fd99ed725b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:56:36 -0700 Subject: [PATCH 026/445] adding testing --- datastore/sharedds.nim | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 4c84379b..0f0619b9 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -81,13 +81,11 @@ proc newSharedDataStore*( self = SharedDatastore() res = newThreadResult(ThreadDatastorePtr) - let buf = DataBuffer.new() - - # res[].signal = ThreadSignalPtr.new().valueOr: - # return failure newException(DatastoreError, "error creating signal") + res[].signal = ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") - # res.createThreadDatastore(backend) - # await wait(res[].signal) - # res[].signal.close() + res.createThreadDatastore(backend) + await wait(res[].signal) + res[].signal.close() success self From 464896bf88a5437b689d031b2a6373108404879a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 19:59:37 -0700 Subject: [PATCH 027/445] adding test backend --- datastore/threadbackend.nim | 5 ++++- tests/datastore/testsharedds.nim | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index bdcbf223..f7886f35 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -78,6 +78,7 @@ proc startupDatastore( ret[].value[].backendDatastore = ds.get() ret[].state = Success of TestBackend: + echo "startupDatastore: TestBackend" ret[].value[].backendDatastore = nil ret[].state = Success else: @@ -117,10 +118,12 @@ proc createThreadDatastore*( ) = try: - ret[].value[].tp = Taskpool.new(num_threads = 1) ##\ + echo "createThreadDatastore: start" + ret[].value[].tp = Taskpool.new(num_threads = 2) ##\ ## Default to one thread, multiple threads \ ## will require more work ret[].value[].tp.spawn startupDatastore(ret, backend) + echo "createThreadDatastore: done" except Exception as exc: ret[].state = Error diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 99167ff5..54b6ae06 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -20,7 +20,8 @@ suite "Test Basic SharedDatastore": let backend = ThreadBackend( kind: TestBackend, ) - let sds = newSharedDataStore(backend) + let sds = await newSharedDataStore(backend) + echo "sds: ", repr sds # suite "Test Basic FSDatastore": # let From 6c5059ecdf774dfe92c116bf286b9036e3dabb0e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 20:07:03 -0700 Subject: [PATCH 028/445] fixing tp setup --- datastore/sharedds.nim | 11 ++++++----- datastore/threadbackend.nim | 26 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 0f0619b9..e7eba277 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -79,13 +79,14 @@ proc newSharedDataStore*( var self = SharedDatastore() - res = newThreadResult(ThreadDatastorePtr) + # res = newThreadResult(ThreadDatastorePtr) + res = cast[ThreadDatastorePtr](allocShared0(sizeof(ThreadDatastorePtr))) - res[].signal = ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") + # res[].signal = ThreadSignalPtr.new().valueOr: + # return failure newException(DatastoreError, "error creating signal") res.createThreadDatastore(backend) - await wait(res[].signal) - res[].signal.close() + # await wait(res[].signal) + # res[].signal.close() success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f7886f35..680cbeb6 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -52,16 +52,20 @@ type tp: Taskpool backendDatastore: Datastore - ThreadDatastorePtr* = SharedPtr[ThreadDatastore] + ThreadDatastorePtr* = ptr ThreadDatastore proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) proc startupDatastore( - ret: TResult[ThreadDatastorePtr], - backend: ThreadBackend, -) {.raises: [].} = + # ret: TResult[ThreadDatastorePtr], + # backend: ThreadBackend, + ) {.raises: [].} = ## starts up a FS instance on a give thread + var + ret: TResult[ThreadDatastorePtr] + backend: ThreadBackend + case backend.kind: of FSBackend: let ds = FSDatastore.new( @@ -113,19 +117,23 @@ proc putTask*( # return TResult[void].new() proc createThreadDatastore*( - ret: TResult[ThreadDatastorePtr], + ret: var ThreadDatastorePtr, backend: ThreadBackend, ) = try: echo "createThreadDatastore: start" - ret[].value[].tp = Taskpool.new(num_threads = 2) ##\ + # var tp = Taskpool.new(num_threads = 2) + ret[].tp = Taskpool.new(num_threads = 2) ##\ ## Default to one thread, multiple threads \ ## will require more work - ret[].value[].tp.spawn startupDatastore(ret, backend) + echo "ret.tp: ", ret[].tp.repr + ret[].tp.spawn startupDatastore() + # ret[].value[].tp.spawn startupDatastore(ret, backend) echo "createThreadDatastore: done" except Exception as exc: - ret[].state = Error - ret[].error = exc.toBuffer() + # ret[].state = Error + # ret[].error = exc.toBuffer() + discard From 0790ccacef838272af9251aad9ac99495f9ce4bc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 20:15:22 -0700 Subject: [PATCH 029/445] setup sharedptr --- datastore.nimble | 2 +- datastore/sharedds.nim | 9 +++++---- datastore/threadbackend.nim | 12 ++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/datastore.nimble b/datastore.nimble index 93ec6af4..ddc7bed9 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -13,7 +13,7 @@ requires "nim >= 1.2.0", "sqlite3_abi", "stew", "unittest2", - "patty", + "pretty", "threading", "upraises >= 0.1.0 & < 0.2.0" diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index e7eba277..667c5454 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -79,12 +79,13 @@ proc newSharedDataStore*( var self = SharedDatastore() - # res = newThreadResult(ThreadDatastorePtr) - res = cast[ThreadDatastorePtr](allocShared0(sizeof(ThreadDatastorePtr))) + res = newThreadResult(ThreadDatastorePtr) - # res[].signal = ThreadSignalPtr.new().valueOr: - # return failure newException(DatastoreError, "error creating signal") + res[].value = newSharedPtr(ThreadDatastore) + res[].signal = ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") + echo "sds:res: ", res.repr res.createThreadDatastore(backend) # await wait(res[].signal) # res[].signal.close() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 680cbeb6..27e93144 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -48,11 +48,11 @@ type of TestBackend: count*: int - ThreadDatastore = object + ThreadDatastore* = object tp: Taskpool backendDatastore: Datastore - ThreadDatastorePtr* = ptr ThreadDatastore + ThreadDatastorePtr* = SharedPtr[ThreadDatastore] proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) @@ -117,18 +117,18 @@ proc putTask*( # return TResult[void].new() proc createThreadDatastore*( - ret: var ThreadDatastorePtr, + ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, ) = try: echo "createThreadDatastore: start" # var tp = Taskpool.new(num_threads = 2) - ret[].tp = Taskpool.new(num_threads = 2) ##\ + ret[].value[].tp = Taskpool.new(num_threads = 2) ##\ ## Default to one thread, multiple threads \ ## will require more work - echo "ret.tp: ", ret[].tp.repr - ret[].tp.spawn startupDatastore() + echo "ret.tp: ", ret[].repr + ret[].value[].tp.spawn startupDatastore() # ret[].value[].tp.spawn startupDatastore(ret, backend) echo "createThreadDatastore: done" From a9e90766cccf771895745123c8503b08d083fe0d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 20:28:22 -0700 Subject: [PATCH 030/445] updates --- datastore/sharedds.nim | 6 +++--- datastore/threadbackend.nim | 32 ++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 667c5454..941fe148 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -81,13 +81,13 @@ proc newSharedDataStore*( self = SharedDatastore() res = newThreadResult(ThreadDatastorePtr) - res[].value = newSharedPtr(ThreadDatastore) + res[].value = newUniquePtr(ThreadDatastore) res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") echo "sds:res: ", res.repr res.createThreadDatastore(backend) - # await wait(res[].signal) - # res[].signal.close() + await wait(res[].signal) + res[].signal.close() success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 27e93144..f3915fef 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -32,9 +32,9 @@ type TResult*[T] = SharedPtr[ThreadResult[T]] ThreadBackendKind* {.pure.} = enum + TestBackend FSBackend SQliteBackend - TestBackend ThreadBackend* = object case kind*: ThreadBackendKind @@ -52,20 +52,26 @@ type tp: Taskpool backendDatastore: Datastore - ThreadDatastorePtr* = SharedPtr[ThreadDatastore] + ThreadDatastorePtr* = UniquePtr[ThreadDatastore] proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) proc startupDatastore( - # ret: TResult[ThreadDatastorePtr], - # backend: ThreadBackend, - ) {.raises: [].} = + ret: TResult[ThreadDatastorePtr], + backend: ThreadBackend, + count: int, +) {.raises: [].} = ## starts up a FS instance on a give thread var ret: TResult[ThreadDatastorePtr] backend: ThreadBackend + echo "\n\nstartupDatastore: ret:\n", ret.repr + echo "\nstartupDatastore: backend:\n", backend.repr + echo "\nstartupDatastore: count:\n", count.repr + + echo "" case backend.kind: of FSBackend: let ds = FSDatastore.new( @@ -88,6 +94,7 @@ proc startupDatastore( else: discard + echo "startupDatastore: signal" discard ret[].signal.fireSync().get() proc getTask*( @@ -123,17 +130,14 @@ proc createThreadDatastore*( try: echo "createThreadDatastore: start" - # var tp = Taskpool.new(num_threads = 2) - ret[].value[].tp = Taskpool.new(num_threads = 2) ##\ - ## Default to one thread, multiple threads \ - ## will require more work - echo "ret.tp: ", ret[].repr - ret[].value[].tp.spawn startupDatastore() - # ret[].value[].tp.spawn startupDatastore(ret, backend) + ret[].value[].tp = Taskpool.new(num_threads = 2) + echo "\n\ncreateThreadDatastore:tp:\n", ret[].repr + ret[].value[].tp.spawn startupDatastore(ret, backend, 22) echo "createThreadDatastore: done" + ret[].state = Success except Exception as exc: - # ret[].state = Error - # ret[].error = exc.toBuffer() + ret[].state = Error + ret[].error = exc.toBuffer() discard From ab850d41144d1efd26cd651fdff133c149a6cb86 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 20:44:50 -0700 Subject: [PATCH 031/445] updates --- datastore/threadbackend.nim | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f3915fef..89385e7a 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -50,17 +50,24 @@ type ThreadDatastore* = object tp: Taskpool - backendDatastore: Datastore + backendDatastore: ThreadBackendKind ThreadDatastorePtr* = UniquePtr[ThreadDatastore] + Test* = object + count*: int + + TestPtr* = SharedPtr[Test] + +var fsBackend {.threadvar.}: FSDatastore + proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) proc startupDatastore( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, - count: int, + count: TestPtr, ) {.raises: [].} = ## starts up a FS instance on a give thread var @@ -81,15 +88,13 @@ proc startupDatastore( ignoreProtected = backend.ignoreProtected ) if ds.isOk: - ret[].value[].backendDatastore = ds.get() + fsBackend = ds.get() ret[].state = Success else: ret[].state = Error - ret[].value[].backendDatastore = ds.get() - ret[].state = Success + ret[].error = newException(DatastoreError, "error creating signal").toBuffer() of TestBackend: echo "startupDatastore: TestBackend" - ret[].value[].backendDatastore = nil ret[].state = Success else: discard @@ -113,16 +118,6 @@ proc putTask*( ) = discard -# proc close*( -# self: ThreadDatastore, -# signal: ThreadSignalPtr, -# ): TResult[void] = -# try: -# self[].tp.shutdown() -# return ok() -# except Exception as exc: -# return TResult[void].new() - proc createThreadDatastore*( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, @@ -132,7 +127,8 @@ proc createThreadDatastore*( echo "createThreadDatastore: start" ret[].value[].tp = Taskpool.new(num_threads = 2) echo "\n\ncreateThreadDatastore:tp:\n", ret[].repr - ret[].value[].tp.spawn startupDatastore(ret, backend, 22) + ret[].value[].tp.spawn startupDatastore( + ret, backend, newSharedPtr(Test(count: 22))) echo "createThreadDatastore: done" ret[].state = Success From a3cff9861b4c91c8a9494ff35a36057eded1ac99 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 21:07:07 -0700 Subject: [PATCH 032/445] updates --- datastore/sharedds.nim | 2 +- datastore/threadbackend.nim | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 941fe148..213c4376 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -81,7 +81,7 @@ proc newSharedDataStore*( self = SharedDatastore() res = newThreadResult(ThreadDatastorePtr) - res[].value = newUniquePtr(ThreadDatastore) + res[].value = newSharedPtr(ThreadDatastore) res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 89385e7a..760322bf 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -32,6 +32,7 @@ type TResult*[T] = SharedPtr[ThreadResult[T]] ThreadBackendKind* {.pure.} = enum + NoBackend TestBackend FSBackend SQliteBackend @@ -47,20 +48,20 @@ type discard of TestBackend: count*: int + of NoBackend: + discard ThreadDatastore* = object tp: Taskpool backendDatastore: ThreadBackendKind - ThreadDatastorePtr* = UniquePtr[ThreadDatastore] + ThreadDatastorePtr* = SharedPtr[ThreadDatastore] Test* = object - count*: int + count*: ThreadBackendKind TestPtr* = SharedPtr[Test] -var fsBackend {.threadvar.}: FSDatastore - proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) @@ -70,9 +71,6 @@ proc startupDatastore( count: TestPtr, ) {.raises: [].} = ## starts up a FS instance on a give thread - var - ret: TResult[ThreadDatastorePtr] - backend: ThreadBackend echo "\n\nstartupDatastore: ret:\n", ret.repr echo "\nstartupDatastore: backend:\n", backend.repr @@ -88,13 +86,15 @@ proc startupDatastore( ignoreProtected = backend.ignoreProtected ) if ds.isOk: - fsBackend = ds.get() + # ret[].value[].backendDatastore = ds.get() ret[].state = Success else: ret[].state = Error - ret[].error = newException(DatastoreError, "error creating signal").toBuffer() + # ret[].value[].backendDatastore = ds.get() + ret[].state = Success of TestBackend: echo "startupDatastore: TestBackend" + ret[].value[].backendDatastore = TestBackend ret[].state = Success else: discard @@ -119,16 +119,17 @@ proc putTask*( discard proc createThreadDatastore*( - ret: TResult[ThreadDatastorePtr], + ret: var TResult[ThreadDatastorePtr], backend: ThreadBackend, ) = try: echo "createThreadDatastore: start" ret[].value[].tp = Taskpool.new(num_threads = 2) - echo "\n\ncreateThreadDatastore:tp:\n", ret[].repr + # echo "\n\ncreateThreadDatastore:tp:\n", ret[].repr + echo "\n\ncreateThreadDatastore:value:\n", ret[].value.repr ret[].value[].tp.spawn startupDatastore( - ret, backend, newSharedPtr(Test(count: 22))) + ret, backend, newSharedPtr(Test(count: FSBackend))) echo "createThreadDatastore: done" ret[].state = Success From 8ab5da01c68ea370b08d268e316375e729841e19 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 21:18:22 -0700 Subject: [PATCH 033/445] create backend --- datastore/sharedds.nim | 4 +++- datastore/threadbackend.nim | 13 ++++++++----- tests/datastore/testsharedds.nim | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 213c4376..83504ce6 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -85,9 +85,11 @@ proc newSharedDataStore*( res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") - echo "sds:res: ", res.repr res.createThreadDatastore(backend) await wait(res[].signal) res[].signal.close() + echo "\nnewSharedDataStore:state: ", res[].state.repr + echo "\nnewSharedDataStore:value: ", res[].value[].backend.repr + success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 760322bf..96f7229c 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -53,7 +53,7 @@ type ThreadDatastore* = object tp: Taskpool - backendDatastore: ThreadBackendKind + backend*: ThreadBackendKind ThreadDatastorePtr* = SharedPtr[ThreadDatastore] @@ -62,6 +62,10 @@ type TestPtr* = SharedPtr[Test] +var + fsDatastore {.threadvar.}: FSDatastore ##\ + ## TODO: figure out a better way to capture this? + proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = newSharedPtr(ThreadResult[T]) @@ -86,7 +90,7 @@ proc startupDatastore( ignoreProtected = backend.ignoreProtected ) if ds.isOk: - # ret[].value[].backendDatastore = ds.get() + fsDatastore = ds.get() ret[].state = Success else: ret[].state = Error @@ -94,13 +98,12 @@ proc startupDatastore( ret[].state = Success of TestBackend: echo "startupDatastore: TestBackend" - ret[].value[].backendDatastore = TestBackend + ret[].value[].backend = TestBackend ret[].state = Success else: discard - echo "startupDatastore: signal" - discard ret[].signal.fireSync().get() + echo "startupDatastore: signal", ret[].signal.fireSync().get() proc getTask*( self: ThreadDatastorePtr, diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 54b6ae06..285dbf54 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -21,7 +21,7 @@ suite "Test Basic SharedDatastore": kind: TestBackend, ) let sds = await newSharedDataStore(backend) - echo "sds: ", repr sds + # echo "sds: ", repr sds # suite "Test Basic FSDatastore": # let From 3da60e021bd7694100269e315c56c03b3f1ec1b4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 21:19:59 -0700 Subject: [PATCH 034/445] add threadid --- datastore/sharedds.nim | 1 + datastore/threadbackend.nim | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 83504ce6..eb4eeeae 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -85,6 +85,7 @@ proc newSharedDataStore*( res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") + echo "\nnewDataStore: threadId:", getThreadId() res.createThreadDatastore(backend) await wait(res[].signal) res[].signal.close() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 96f7229c..acc36c5a 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -75,7 +75,9 @@ proc startupDatastore( count: TestPtr, ) {.raises: [].} = ## starts up a FS instance on a give thread - echo "\n\nstartupDatastore: ret:\n", ret.repr + echo "\n" + echo "\nstartupDatastore: threadId:", getThreadId() + echo "\nstartupDatastore: ret:\n", ret.repr echo "\nstartupDatastore: backend:\n", backend.repr echo "\nstartupDatastore: count:\n", count.repr From f66531de0665095c27682db7f77080f495a220b3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 21:55:53 -0700 Subject: [PATCH 035/445] passing put args --- datastore/databuffer.nim | 2 +- datastore/sharedds.nim | 14 +++++++++++-- datastore/threadbackend.nim | 34 ++++++++++++++++++++++++++++---- tests/datastore/testsharedds.nim | 11 ++++++++++- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 1367b40e..d1e78aac 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -34,7 +34,7 @@ proc `=destroy`*(x: var DataBuffer) = if x.buf != nil and x.cnt != nil: let res = atomicSubFetch(x.cnt, 1, ATOMIC_ACQUIRE) if res == 0: - when isMainModule: + when isMainModule or true: echo "buffer: FREE: ", repr x.buf.pointer, " ", x.cnt[] deallocShared(x.buf) deallocShared(x.cnt) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index eb4eeeae..e071d86c 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -13,6 +13,8 @@ import ./datastore import ./threadbackend import threading/smartptrs +import pretty + export key, query, ThreadBackend push: {.upraises: [].} @@ -53,10 +55,16 @@ method put*( data: seq[byte] ): Future[?!void] {.async.} = - let signal = ThreadSignalPtr.new().valueOr: + var res = newThreadResult(void) + res[].signal = ThreadSignalPtr.new().valueOr: return failure newException(DatastoreError, "error creating signal") - await wait(signal) + put(res, self.tds, key, data) + await wait(res[].signal) + res[].signal.close() + + echo "\nSharedDataStore:put:value: ", res[].repr + return success() method put*( @@ -93,4 +101,6 @@ proc newSharedDataStore*( echo "\nnewSharedDataStore:state: ", res[].state.repr echo "\nnewSharedDataStore:value: ", res[].value[].backend.repr + self.tds = res[].value + success self diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index acc36c5a..7fc38413 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -12,6 +12,8 @@ import threading/smartptrs import fsds +import pretty + export key, query, smartptrs, databuffer push: {.upraises: [].} @@ -52,7 +54,7 @@ type discard ThreadDatastore* = object - tp: Taskpool + tp*: Taskpool backend*: ThreadBackendKind ThreadDatastorePtr* = SharedPtr[ThreadDatastore] @@ -116,12 +118,36 @@ proc getTask*( discard proc putTask*( - self: ThreadDatastorePtr, + ret: TResult[void], + backend: ThreadBackendKind, key: KeyBuffer, data: DataBuffer, - ret: TResult[void] ) = - discard + print "\nthrbackend: putTask: ", ret[] + print "\nthrbackend: putTask:key: ", key + print "\nthrbackend: putTask:data: ", data + + print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + +import os + +proc put*( + ret: TResult[void], + tds: ThreadDatastorePtr, + key: Key, + data: seq[byte] +): TResult[void] = + echo "thrfrontend:put: " + + let bkey = StringBuffer.new(key.id()) + let bval = DataBuffer.new(data) + print "bkey: ", bkey + print "bval: ", bval + + tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) + os.sleep(500) + print "res:bkey: ", bkey + print "res:bval: ", bval proc createThreadDatastore*( ret: var TResult[ThreadDatastorePtr], diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 285dbf54..85fda1fd 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -17,12 +17,21 @@ suite "Test Basic SharedDatastore": test "check create": + var sds: SharedDatastore + let backend = ThreadBackend( kind: TestBackend, ) - let sds = await newSharedDataStore(backend) + let res = await newSharedDataStore(backend) + check res.isOk() + sds = res.get() # echo "sds: ", repr sds + echo "\n\n=== put ===" + let key1 = Key.init("/a").tryGet + let res1 = await sds.put(key1, "value for 1".toBytes()) + echo "res1: ", res1.repr + # suite "Test Basic FSDatastore": # let # path = currentSourcePath() # get this file's name From 0b2e016883dbc83579bd1aef6bc94a2d200fa1e2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 22:14:21 -0700 Subject: [PATCH 036/445] adding sharedds get --- datastore/sharedds.nim | 13 ++++++++++++- datastore/threadbackend.nim | 31 +++++++++++++++++++++---------- tests/datastore/testsharedds.nim | 8 ++++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index e071d86c..a48ad5d2 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -47,7 +47,18 @@ method get*( self: SharedDatastore, key: Key ): Future[?!seq[byte]] {.async.} = - return success(newSeq[byte]()) + + var res = newThreadResult(DataBuffer) + res[].signal = ThreadSignalPtr.new().valueOr: + return failure newException(DatastoreError, "error creating signal") + + get(res, self.tds, key) + await wait(res[].signal) + res[].signal.close() + + echo "\nSharedDataStore:put:value: ", res[].repr + let data = res[].value.toSeq(byte) + return success(data) method put*( self: SharedDatastore, diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 7fc38413..2332764c 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -98,7 +98,6 @@ proc startupDatastore( ret[].state = Success else: ret[].state = Error - # ret[].value[].backendDatastore = ds.get() ret[].state = Success of TestBackend: echo "startupDatastore: TestBackend" @@ -110,12 +109,30 @@ proc startupDatastore( echo "startupDatastore: signal", ret[].signal.fireSync().get() proc getTask*( - self: ThreadDatastorePtr, + ret: TResult[DataBuffer], + backend: ThreadBackendKind, key: KeyBuffer, - ret: TResult[DataBuffer] ) = # return ok(DataBuffer.new()) - discard + print "\nthrbackend: getTask: ", ret[] + print "\nthrbackend: getTask:key: ", key + let data = DataBuffer.new("hello world!") + print "\nthrbackend: getTask:data: ", data + ret[].state = Success + ret[].value = data + + print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + +proc get*( + ret: TResult[DataBuffer], + tds: ThreadDatastorePtr, + key: Key, +) = + echo "thrfrontend:put: " + let bkey = StringBuffer.new(key.id()) + print "bkey: ", bkey + + tds[].tp.spawn getTask(ret, tds[].backend, bkey) proc putTask*( ret: TResult[void], @@ -129,8 +146,6 @@ proc putTask*( print "thrbackend: putTask: fire", ret[].signal.fireSync().get() -import os - proc put*( ret: TResult[void], tds: ThreadDatastorePtr, @@ -138,16 +153,12 @@ proc put*( data: seq[byte] ): TResult[void] = echo "thrfrontend:put: " - let bkey = StringBuffer.new(key.id()) let bval = DataBuffer.new(data) print "bkey: ", bkey print "bval: ", bval tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) - os.sleep(500) - print "res:bkey: ", bkey - print "res:bval: ", bval proc createThreadDatastore*( ret: var TResult[ThreadDatastorePtr], diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 85fda1fd..2010744d 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -32,6 +32,14 @@ suite "Test Basic SharedDatastore": let res1 = await sds.put(key1, "value for 1".toBytes()) echo "res1: ", res1.repr + echo "\n\n=== get ===" + let res2 = await sds.get(key1) + check res2.get() == "hello world!".toBytes() + var val = "" + for c in res2.get(): + val &= char(c) + echo "res2: ", $val + # suite "Test Basic FSDatastore": # let # path = currentSourcePath() # get this file's name From 6c133a255123d0ece96d4fb4a4f0b82d9f792156 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 24 Aug 2023 22:20:49 -0700 Subject: [PATCH 037/445] cleanup --- tests/datastore/testsharedds.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 2010744d..4974f6d2 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -13,6 +13,8 @@ import pkg/datastore/sharedds import ./dscommontests import ./querycommontests +import pretty + suite "Test Basic SharedDatastore": test "check create": @@ -38,7 +40,7 @@ suite "Test Basic SharedDatastore": var val = "" for c in res2.get(): val &= char(c) - echo "res2: ", $val + print "get res2: ", $val # suite "Test Basic FSDatastore": # let From 86a70855390f4c1b0aefb808d7333d196ddc8854 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 14:34:11 -0700 Subject: [PATCH 038/445] switching up --- datastore/sharedds.nim | 16 ++++++++-------- datastore/threadbackend.nim | 15 ++++++++++++--- tests/datastore/testsharedds.nim | 4 +++- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index a48ad5d2..8cc0d0ab 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -6,6 +6,7 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises import pkg/taskpools +import pkg/stew/results import ./key import ./query @@ -48,9 +49,8 @@ method get*( key: Key ): Future[?!seq[byte]] {.async.} = - var res = newThreadResult(DataBuffer) - res[].signal = ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") + var res = ?newThreadResult(DataBuffer) + echo "res: ", res get(res, self.tds, key) await wait(res[].signal) @@ -66,18 +66,18 @@ method put*( data: seq[byte] ): Future[?!void] {.async.} = - var res = newThreadResult(void) - res[].signal = ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") + var res = ?newThreadResult(void) + echo "res: ", res put(res, self.tds, key, data) await wait(res[].signal) + echo "closing signal" res[].signal.close() echo "\nSharedDataStore:put:value: ", res[].repr - return success() + method put*( self: SharedDatastore, batch: seq[BatchEntry] @@ -98,7 +98,7 @@ proc newSharedDataStore*( var self = SharedDatastore() - res = newThreadResult(ThreadDatastorePtr) + res = ?newThreadResult(ThreadDatastorePtr) res[].value = newSharedPtr(ThreadDatastore) res[].signal = ThreadSignalPtr.new().valueOr: diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 2332764c..acc24700 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -68,8 +68,14 @@ var fsDatastore {.threadvar.}: FSDatastore ##\ ## TODO: figure out a better way to capture this? -proc newThreadResult*[T](tp: typedesc[T]): TResult[T] = - newSharedPtr(ThreadResult[T]) +proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError] = + let res = newSharedPtr(ThreadResult[T]) + let signal = ThreadSignalPtr.new() + if signal.isErr: + return err((ref CatchableError)(msg: signal.error())) + else: + res[].signal = signal.get() + ok res proc startupDatastore( ret: TResult[ThreadDatastorePtr], @@ -134,6 +140,8 @@ proc get*( tds[].tp.spawn getTask(ret, tds[].backend, bkey) +import os + proc putTask*( ret: TResult[void], backend: ThreadBackendKind, @@ -144,6 +152,7 @@ proc putTask*( print "\nthrbackend: putTask:key: ", key print "\nthrbackend: putTask:data: ", data + os.sleep(200) print "thrbackend: putTask: fire", ret[].signal.fireSync().get() proc put*( @@ -151,7 +160,7 @@ proc put*( tds: ThreadDatastorePtr, key: Key, data: seq[byte] -): TResult[void] = +) = echo "thrfrontend:put: " let bkey = StringBuffer.new(key.id()) let bval = DataBuffer.new(data) diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 4974f6d2..1f870567 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -31,7 +31,9 @@ suite "Test Basic SharedDatastore": echo "\n\n=== put ===" let key1 = Key.init("/a").tryGet - let res1 = await sds.put(key1, "value for 1".toBytes()) + # let res1 = await sds.put(key1, "value for 1".toBytes()) + let res1 = sds.put(key1, "value for 1".toBytes()) + res1.cancel() echo "res1: ", res1.repr echo "\n\n=== get ===" From aa01665c16d74de6b193334b4e9cbe3b07a59779 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 14:46:29 -0700 Subject: [PATCH 039/445] switching up errors --- datastore/sharedds.nim | 31 +++++++++++++++++-------------- datastore/threadbackend.nim | 6 +++++- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 8cc0d0ab..2ab30ff7 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -49,15 +49,15 @@ method get*( key: Key ): Future[?!seq[byte]] {.async.} = - var res = ?newThreadResult(DataBuffer) - echo "res: ", res + without ret =? newThreadResult(DataBuffer), err: + return failure(err) - get(res, self.tds, key) - await wait(res[].signal) - res[].signal.close() + get(ret, self.tds, key) + await wait(ret[].signal) + ret[].signal.close() - echo "\nSharedDataStore:put:value: ", res[].repr - let data = res[].value.toSeq(byte) + echo "\nSharedDataStore:put:value: ", ret[].repr + let data = ret[].value.toSeq(byte) return success(data) method put*( @@ -66,15 +66,16 @@ method put*( data: seq[byte] ): Future[?!void] {.async.} = - var res = ?newThreadResult(void) + without ret =? newThreadResult(void), err: + return failure(err) - echo "res: ", res - put(res, self.tds, key, data) - await wait(res[].signal) + echo "res: ", ret + put(ret, self.tds, key, data) + await wait(ret[].signal) echo "closing signal" - res[].signal.close() + ret[].signal.close() - echo "\nSharedDataStore:put:value: ", res[].repr + echo "\nSharedDataStore:put:value: ", ret[].repr return success() @@ -98,7 +99,9 @@ proc newSharedDataStore*( var self = SharedDatastore() - res = ?newThreadResult(ThreadDatastorePtr) + + without res =? newThreadResult(ThreadDatastorePtr), err: + return failure(err) res[].value = newSharedPtr(ThreadDatastore) res[].signal = ThreadSignalPtr.new().valueOr: diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index acc24700..e3ef0649 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -68,6 +68,10 @@ var fsDatastore {.threadvar.}: FSDatastore ##\ ## TODO: figure out a better way to capture this? +# proc `=destroy`*[T](x: var ThreadResult[T]) = +# when T isnot void: +# x.value.`=destroy` + proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError] = let res = newSharedPtr(ThreadResult[T]) let signal = ThreadSignalPtr.new() @@ -170,7 +174,7 @@ proc put*( tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) proc createThreadDatastore*( - ret: var TResult[ThreadDatastorePtr], + ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, ) = From 35c4466de0bc0a111548e90709d910443e30fff5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 14:48:49 -0700 Subject: [PATCH 040/445] switching up errors --- datastore/sharedds.nim | 2 -- datastore/threadbackend.nim | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 2ab30ff7..9820debd 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -104,8 +104,6 @@ proc newSharedDataStore*( return failure(err) res[].value = newSharedPtr(ThreadDatastore) - res[].signal = ThreadSignalPtr.new().valueOr: - return failure newException(DatastoreError, "error creating signal") echo "\nnewDataStore: threadId:", getThreadId() res.createThreadDatastore(backend) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index e3ef0649..9bde2b78 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -68,9 +68,9 @@ var fsDatastore {.threadvar.}: FSDatastore ##\ ## TODO: figure out a better way to capture this? -# proc `=destroy`*[T](x: var ThreadResult[T]) = -# when T isnot void: -# x.value.`=destroy` +proc `=destroy`*[T](x: var ThreadResult[T]) = + when T isnot void: + x.value.`=destroy` proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError] = let res = newSharedPtr(ThreadResult[T]) From 727f8b6c2ab15e649f75e0b6735850413f10b0a5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 15:00:18 -0700 Subject: [PATCH 041/445] properly close signal even on cancel --- datastore/sharedds.nim | 33 +++++++++++++++++++------------- datastore/threadbackend.nim | 4 ---- tests/datastore/testsharedds.nim | 12 ++++++++---- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 9820debd..cef050fb 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -52,9 +52,12 @@ method get*( without ret =? newThreadResult(DataBuffer), err: return failure(err) - get(ret, self.tds, key) - await wait(ret[].signal) - ret[].signal.close() + try: + get(ret, self.tds, key) + await wait(ret[].signal) + finally: + echo "closing signal" + ret[].signal.close() echo "\nSharedDataStore:put:value: ", ret[].repr let data = ret[].value.toSeq(byte) @@ -70,10 +73,12 @@ method put*( return failure(err) echo "res: ", ret - put(ret, self.tds, key, data) - await wait(ret[].signal) - echo "closing signal" - ret[].signal.close() + try: + put(ret, self.tds, key, data) + await wait(ret[].signal) + finally: + echo "closing signal" + ret[].signal.close() echo "\nSharedDataStore:put:value: ", ret[].repr return success() @@ -103,12 +108,14 @@ proc newSharedDataStore*( without res =? newThreadResult(ThreadDatastorePtr), err: return failure(err) - res[].value = newSharedPtr(ThreadDatastore) - - echo "\nnewDataStore: threadId:", getThreadId() - res.createThreadDatastore(backend) - await wait(res[].signal) - res[].signal.close() + try: + res[].value = newSharedPtr(ThreadDatastore) + echo "\nnewDataStore: threadId:", getThreadId() + res.createThreadDatastore(backend) + await wait(res[].signal) + finally: + echo "closing signal" + res[].signal.close() echo "\nnewSharedDataStore:state: ", res[].state.repr echo "\nnewSharedDataStore:value: ", res[].value[].backend.repr diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 9bde2b78..f91d7cec 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -68,10 +68,6 @@ var fsDatastore {.threadvar.}: FSDatastore ##\ ## TODO: figure out a better way to capture this? -proc `=destroy`*[T](x: var ThreadResult[T]) = - when T isnot void: - x.value.`=destroy` - proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError] = let res = newSharedPtr(ThreadResult[T]) let signal = ThreadSignalPtr.new() diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 1f870567..5c47f438 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -31,10 +31,8 @@ suite "Test Basic SharedDatastore": echo "\n\n=== put ===" let key1 = Key.init("/a").tryGet - # let res1 = await sds.put(key1, "value for 1".toBytes()) - let res1 = sds.put(key1, "value for 1".toBytes()) - res1.cancel() - echo "res1: ", res1.repr + let res1 = await sds.put(key1, "value for 1".toBytes()) + print "res1: ", res1 echo "\n\n=== get ===" let res2 = await sds.get(key1) @@ -44,6 +42,12 @@ suite "Test Basic SharedDatastore": val &= char(c) print "get res2: ", $val + echo "\n\n=== put cancel ===" + # let res1 = await sds.put(key1, "value for 1".toBytes()) + let res3 = sds.put(key1, "value for 1".toBytes()) + res3.cancel() + # print "res3: ", res3 + # suite "Test Basic FSDatastore": # let # path = currentSourcePath() # get this file's name From 0d4e6f203a2052b5ffd0de78466cf2b9caef8e15 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 15:08:19 -0700 Subject: [PATCH 042/445] switch to smartptrs --- datastore/databuffer.nim | 73 +++++++++------------------------------- 1 file changed, 16 insertions(+), 57 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index d1e78aac..3ce46516 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -1,93 +1,52 @@ -import std/os -import std/locks -import std/atomics +# import std/atomics +import threading/smartptrs -import events type - DataBuffer* = object - cnt: ptr int + DataBufferHolder* = object buf: ptr UncheckedArray[byte] size: int + DataBuffer* = SharedPtr[DataBufferHolder] KeyBuffer* = DataBuffer ValueBuffer* = DataBuffer StringBuffer* = DataBuffer CatchableErrorBuffer* = object msg: StringBuffer -proc `$`*(data: DataBuffer): string = - if data.buf.isNil: - result = "nil" - else: - let sz = min(16, data.size) - result = newString(sz + 2) - copyMem(addr result[1], data.buf, sz) - result[0] = '<' - result[^1] = '>' - -proc `=destroy`*(x: var DataBuffer) = - ## copy pointer implementation - if x.buf != nil and x.cnt != nil: - let res = atomicSubFetch(x.cnt, 1, ATOMIC_ACQUIRE) - if res == 0: - when isMainModule or true: - echo "buffer: FREE: ", repr x.buf.pointer, " ", x.cnt[] - deallocShared(x.buf) - deallocShared(x.cnt) - else: - when isMainModule: - echo "buffer: decr: ", repr x.buf.pointer, " ", x.cnt[] - -proc `=copy`*(a: var DataBuffer; b: DataBuffer) = +proc `=destroy`*(x: var DataBufferHolder) = ## copy pointer implementation + if x.buf != nil: + when isMainModule or true: + echo "buffer: FREE: ", repr x.buf.pointer + deallocShared(x.buf) - # do nothing for self-assignments: - if a.buf == b.buf: return - `=destroy`(a) - discard atomicAddFetch(b.cnt, 1, ATOMIC_RELAXED) - a.size = b.size - a.buf = b.buf - a.cnt = b.cnt - when isMainModule: - echo "buffer: Copy: repr: ", b.cnt[], - " ", repr a.buf.pointer, - " ", repr b.buf.pointer - -proc incrAtomicCount*(a: DataBuffer) = - let res = atomicAddFetch(a.cnt, 1, ATOMIC_RELAXED) -proc unsafeGetAtomicCount*(a: DataBuffer): int = - atomicLoad(a.cnt, addr result, ATOMIC_RELAXED) - -proc len*(a: DataBuffer): int = a.size +proc len*(a: DataBuffer): int = a[].size proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = - let cnt = cast[ptr int](allocShared0(sizeof(result.cnt))) - cnt[] = 1 - DataBuffer( - cnt: cnt, - buf: cast[typeof(result.buf)](allocShared0(size)), + newSharedPtr(DataBufferHolder( + buf: cast[typeof(result[].buf)](allocShared0(size)), size: size, - ) + )) proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuffer = ## allocate new buffer and copies indata from openArray ## result = DataBuffer.new(data.len) if data.len() > 0: - copyMem(result.buf, unsafeAddr data[0], data.len) + copyMem(result[].buf, unsafeAddr data[0], data.len) proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = result = newSeq[T](a.len) - copyMem(addr result[0], unsafeAddr a.buf[0], a.len) + copyMem(addr result[0], unsafeAddr a[].buf[0], a.len) proc toString*(data: DataBuffer): string = result = newString(data.len()) if data.len() > 0: - copyMem(addr result[0], unsafeAddr data.buf[0], data.len) + copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) proc toCatchable*(data: CatchableErrorBuffer): ref CatchableError = result = (ref CatchableError)(msg: data.msg.toString()) From 768de3bc53f6f98535ad7bf60f949397c0dd1cff Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 15:14:53 -0700 Subject: [PATCH 043/445] cleanup --- datastore/databuffer.nim | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 3ce46516..1a5e5461 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -1,16 +1,17 @@ - - - # import std/atomics import threading/smartptrs - type DataBufferHolder* = object buf: ptr UncheckedArray[byte] size: int - DataBuffer* = SharedPtr[DataBufferHolder] + DataBuffer* = SharedPtr[DataBufferHolder] ##\ + ## A fixed length data buffer using a SharedPtr. + ## It is thread safe even with `refc` since + ## it doesn't use string or seq types internally. + ## + KeyBuffer* = DataBuffer ValueBuffer* = DataBuffer StringBuffer* = DataBuffer @@ -27,6 +28,7 @@ proc `=destroy`*(x: var DataBufferHolder) = proc len*(a: DataBuffer): int = a[].size proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = + ## allocate new buffer with given size newSharedPtr(DataBufferHolder( buf: cast[typeof(result[].buf)](allocShared0(size)), size: size, @@ -40,18 +42,22 @@ proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuf copyMem(result[].buf, unsafeAddr data[0], data.len) proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = + ## convert buffer to a seq type using copy and either a byte or char result = newSeq[T](a.len) copyMem(addr result[0], unsafeAddr a[].buf[0], a.len) proc toString*(data: DataBuffer): string = + ## convert buffer to string type using copy result = newString(data.len()) if data.len() > 0: copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) -proc toCatchable*(data: CatchableErrorBuffer): ref CatchableError = - result = (ref CatchableError)(msg: data.msg.toString()) +proc toCatchable*(err: CatchableErrorBuffer): ref CatchableError = + ## convert back to a ref CatchableError + result = (ref CatchableError)(msg: err.msg.toString()) proc toBuffer*(err: ref Exception): CatchableErrorBuffer = + ## convert exception to an object with StringBuffer return CatchableErrorBuffer( msg: StringBuffer.new(err.msg) ) From 89f43f5e971b08e716330c37b0c61cd1f2b7c31a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 25 Aug 2023 15:21:12 -0700 Subject: [PATCH 044/445] more cleanup --- datastore/sharedds.nim | 6 +++--- datastore/threadbackend.nim | 11 ++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index cef050fb..8c2d2a7b 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -59,7 +59,7 @@ method get*( echo "closing signal" ret[].signal.close() - echo "\nSharedDataStore:put:value: ", ret[].repr + print "\nSharedDataStore:put:value: ", ret[] let data = ret[].value.toSeq(byte) return success(data) @@ -117,8 +117,8 @@ proc newSharedDataStore*( echo "closing signal" res[].signal.close() - echo "\nnewSharedDataStore:state: ", res[].state.repr - echo "\nnewSharedDataStore:value: ", res[].value[].backend.repr + print "\nnewSharedDataStore:state: ", res[].state + print "\nnewSharedDataStore:value: ", res[].value[].backend self.tds = res[].value diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f91d7cec..32bdec54 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -80,15 +80,11 @@ proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError proc startupDatastore( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, - count: TestPtr, ) {.raises: [].} = ## starts up a FS instance on a give thread echo "\n" echo "\nstartupDatastore: threadId:", getThreadId() - echo "\nstartupDatastore: ret:\n", ret.repr - - echo "\nstartupDatastore: backend:\n", backend.repr - echo "\nstartupDatastore: count:\n", count.repr + print "\nstartupDatastore: backend:\n", backend echo "" case backend.kind: @@ -177,10 +173,7 @@ proc createThreadDatastore*( try: echo "createThreadDatastore: start" ret[].value[].tp = Taskpool.new(num_threads = 2) - # echo "\n\ncreateThreadDatastore:tp:\n", ret[].repr - echo "\n\ncreateThreadDatastore:value:\n", ret[].value.repr - ret[].value[].tp.spawn startupDatastore( - ret, backend, newSharedPtr(Test(count: FSBackend))) + ret[].value[].tp.spawn startupDatastore(ret, backend) echo "createThreadDatastore: done" ret[].state = Success From b90eff2e128e261b281d078ea0580d58620efbf0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 17:28:14 -0700 Subject: [PATCH 045/445] cleanup --- datastore/sharedds.nim | 2 -- datastore/threadbackend.nim | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 8c2d2a7b..92d82e5e 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -83,7 +83,6 @@ method put*( echo "\nSharedDataStore:put:value: ", ret[].repr return success() - method put*( self: SharedDatastore, batch: seq[BatchEntry] @@ -93,7 +92,6 @@ method put*( method close*( self: SharedDatastore ): Future[?!void] {.async.} = - # TODO: how to handle failed close? return success() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 32bdec54..75e1d94d 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -108,7 +108,7 @@ proc startupDatastore( else: discard - echo "startupDatastore: signal", ret[].signal.fireSync().get() + print "startupDatastore: signal", ret[].signal.fireSync() proc getTask*( ret: TResult[DataBuffer], @@ -123,7 +123,7 @@ proc getTask*( ret[].state = Success ret[].value = data - print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + print "thrbackend: putTask: fire", ret[].signal.fireSync() proc get*( ret: TResult[DataBuffer], From a359edbda9747e49dc62e7dbf25bde59a331baff Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 17:29:38 -0700 Subject: [PATCH 046/445] cleanup --- datastore/threadbackend.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 75e1d94d..48695fa4 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -68,7 +68,9 @@ var fsDatastore {.threadvar.}: FSDatastore ##\ ## TODO: figure out a better way to capture this? -proc newThreadResult*[T](tp: typedesc[T]): Result[TResult[T], ref CatchableError] = +proc newThreadResult*[T]( + tp: typedesc[T] +): Result[TResult[T], ref CatchableError] = let res = newSharedPtr(ThreadResult[T]) let signal = ThreadSignalPtr.new() if signal.isErr: From d60bd35d736d846ba0f544b58fe797f066eb82d0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 17:51:48 -0700 Subject: [PATCH 047/445] rework ds --- datastore/sharedds.nim | 10 ++++- datastore/threadbackend.nim | 75 +++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 92d82e5e..7f5afc9f 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -93,7 +93,15 @@ method close*( self: SharedDatastore ): Future[?!void] {.async.} = # TODO: how to handle failed close? - return success() + echo "ThreadDatastore: FREE: " + result = success() + + without res =? self.tds[].ds.close(), err: + result = failure(err) + + if self.tds[].tp != nil: + ## this can block... how to handle? maybe just leak? + self.tds[].tp.shutdown() proc newSharedDataStore*( # T: typedesc[SharedDatastore], diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 48695fa4..2f019231 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -56,6 +56,7 @@ type ThreadDatastore* = object tp*: Taskpool backend*: ThreadBackendKind + ds*: Datastore ThreadDatastorePtr* = SharedPtr[ThreadDatastore] @@ -64,10 +65,6 @@ type TestPtr* = SharedPtr[Test] -var - fsDatastore {.threadvar.}: FSDatastore ##\ - ## TODO: figure out a better way to capture this? - proc newThreadResult*[T]( tp: typedesc[T] ): Result[TResult[T], ref CatchableError] = @@ -79,39 +76,6 @@ proc newThreadResult*[T]( res[].signal = signal.get() ok res -proc startupDatastore( - ret: TResult[ThreadDatastorePtr], - backend: ThreadBackend, -) {.raises: [].} = - ## starts up a FS instance on a give thread - echo "\n" - echo "\nstartupDatastore: threadId:", getThreadId() - print "\nstartupDatastore: backend:\n", backend - - echo "" - case backend.kind: - of FSBackend: - let ds = FSDatastore.new( - root = backend.root.toString(), - depth = backend.depth, - caseSensitive = backend.caseSensitive, - ignoreProtected = backend.ignoreProtected - ) - if ds.isOk: - fsDatastore = ds.get() - ret[].state = Success - else: - ret[].state = Error - ret[].state = Success - of TestBackend: - echo "startupDatastore: TestBackend" - ret[].value[].backend = TestBackend - ret[].state = Success - else: - discard - - print "startupDatastore: signal", ret[].signal.fireSync() - proc getTask*( ret: TResult[DataBuffer], backend: ThreadBackendKind, @@ -167,6 +131,8 @@ proc put*( tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) +proc startupDatastore(ret: TResult[ThreadDatastorePtr], backend: ThreadBackend,) {.raises: [].} + proc createThreadDatastore*( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, @@ -184,3 +150,38 @@ proc createThreadDatastore*( ret[].error = exc.toBuffer() discard + +proc startupDatastore( + ret: TResult[ThreadDatastorePtr], + backend: ThreadBackend, +) {.raises: [].} = + ## starts up a FS instance on a give thread + echo "\n" + echo "\nstartupDatastore: threadId:", getThreadId() + print "\nstartupDatastore: backend:\n", backend + + echo "" + case backend.kind: + of FSBackend: + let ds = FSDatastore.new( + root = backend.root.toString(), + depth = backend.depth, + caseSensitive = backend.caseSensitive, + ignoreProtected = backend.ignoreProtected + ) + if ds.isOk: + let ds = ds.get() + GC_ref(ds) + ret[].value[].ds = ds + ret[].state = Success + else: + ret[].state = Error + ret[].state = Success + of TestBackend: + echo "startupDatastore: TestBackend" + ret[].value[].backend = TestBackend + ret[].state = Success + else: + discard + + # print "startupDatastore: signal", ret[].signal.fireSync() From 88e26f94ab9da040d887bd048c316988fd5ee666 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:00:42 -0700 Subject: [PATCH 048/445] rework ds --- datastore/sharedds.nim | 14 +++++++------ datastore/threadbackend.nim | 39 ------------------------------------- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 7f5afc9f..769e964b 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -7,12 +7,13 @@ import pkg/questionable/results import pkg/upraises import pkg/taskpools import pkg/stew/results +import pkg/threading/smartptrs import ./key import ./query import ./datastore import ./threadbackend -import threading/smartptrs +import ./fsds import pretty @@ -98,14 +99,14 @@ method close*( without res =? self.tds[].ds.close(), err: result = failure(err) + # GC_unref(self.tds[].ds) ## TODO: is this needed? if self.tds[].tp != nil: ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() -proc newSharedDataStore*( - # T: typedesc[SharedDatastore], - backend: ThreadBackend, +proc newSharedDataStore*[T: Datastore]( + ds: Datastore, ): Future[?!SharedDatastore] {.async.} = var @@ -113,11 +114,12 @@ proc newSharedDataStore*( without res =? newThreadResult(ThreadDatastorePtr), err: return failure(err) - + try: res[].value = newSharedPtr(ThreadDatastore) echo "\nnewDataStore: threadId:", getThreadId() - res.createThreadDatastore(backend) + # GC_ref(ds) + res[].value[].ds = ds await wait(res[].signal) finally: echo "closing signal" diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 2f019231..6c062314 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -10,8 +10,6 @@ import ./datastore import ./databuffer import threading/smartptrs -import fsds - import pretty export key, query, smartptrs, databuffer @@ -131,8 +129,6 @@ proc put*( tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) -proc startupDatastore(ret: TResult[ThreadDatastorePtr], backend: ThreadBackend,) {.raises: [].} - proc createThreadDatastore*( ret: TResult[ThreadDatastorePtr], backend: ThreadBackend, @@ -150,38 +146,3 @@ proc createThreadDatastore*( ret[].error = exc.toBuffer() discard - -proc startupDatastore( - ret: TResult[ThreadDatastorePtr], - backend: ThreadBackend, -) {.raises: [].} = - ## starts up a FS instance on a give thread - echo "\n" - echo "\nstartupDatastore: threadId:", getThreadId() - print "\nstartupDatastore: backend:\n", backend - - echo "" - case backend.kind: - of FSBackend: - let ds = FSDatastore.new( - root = backend.root.toString(), - depth = backend.depth, - caseSensitive = backend.caseSensitive, - ignoreProtected = backend.ignoreProtected - ) - if ds.isOk: - let ds = ds.get() - GC_ref(ds) - ret[].value[].ds = ds - ret[].state = Success - else: - ret[].state = Error - ret[].state = Success - of TestBackend: - echo "startupDatastore: TestBackend" - ret[].value[].backend = TestBackend - ret[].state = Success - else: - discard - - # print "startupDatastore: signal", ret[].signal.fireSync() From d70e0df285187f4f9a151c43acef209f14b38164 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:04:59 -0700 Subject: [PATCH 049/445] rework ds --- datastore/sharedds.nim | 1 + datastore/threadbackend.nim | 14 +++----------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 769e964b..f9d689b4 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -120,6 +120,7 @@ proc newSharedDataStore*[T: Datastore]( echo "\nnewDataStore: threadId:", getThreadId() # GC_ref(ds) res[].value[].ds = ds + res.createThreadDatastore() await wait(res[].signal) finally: echo "closing signal" diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 6c062314..ba91ad8f 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -129,20 +129,12 @@ proc put*( tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) -proc createThreadDatastore*( - ret: TResult[ThreadDatastorePtr], - backend: ThreadBackend, -) = - +proc createThreadDatastore*(ret: ThreadDatastorePtr): Result[void, ref CatchableError] = try: echo "createThreadDatastore: start" - ret[].value[].tp = Taskpool.new(num_threads = 2) - ret[].value[].tp.spawn startupDatastore(ret, backend) + ret[].tp = Taskpool.new(num_threads = 2) echo "createThreadDatastore: done" - ret[].state = Success except Exception as exc: - ret[].state = Error - ret[].error = exc.toBuffer() - discard + return err((ref DatastoreError)(msg: exc.msg)) From 904d65b98aa12bfc3e90a2c8286ac8aff9e6ba8b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:22:53 -0700 Subject: [PATCH 050/445] add memory (test) ds --- datastore/memoryds.nim | 96 +++++++++++++++++++++ tests/datastore/testmemoryds.nim | 138 +++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 datastore/memoryds.nim create mode 100644 tests/datastore/testmemoryds.nim diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim new file mode 100644 index 00000000..c4d735e9 --- /dev/null +++ b/datastore/memoryds.nim @@ -0,0 +1,96 @@ +import std/tables + +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/upraises + +import ./key +import ./query +import ./datastore + +export key, query + +push: {.upraises: [].} + +type + MemoryStore* = object + store*: Datastore + key*: Key + + MemoryDatastore* = ref object of Datastore + stores*: Table[Key, MemoryStore] + +method has*( + self: MemoryDatastore, + key: Key): Future[?!bool] {.async.} = + + without mounted =? self.dispatch(key): + return failure "No mounted datastore found" + + return (await mounted.store.store.has(mounted.relative)) + +method delete*( + self: MemoryDatastore, + key: Key): Future[?!void] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return (await mounted.store.store.delete(mounted.relative)) + +method delete*( + self: MemoryDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err + + return success() + +method get*( + self: MemoryDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return await mounted.store.store.get(mounted.relative) + +method put*( + self: MemoryDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + + without mounted =? self.dispatch(key), error: + return failure(error) + + return (await mounted.store.store.put(mounted.relative, data)) + +method put*( + self: MemoryDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + return success() + +method close*(self: MemoryDatastore): Future[?!void] {.async.} = + for s in self.stores.values: + discard await s.store.close() + + # TODO: how to handle failed close? + return success() + +func new*( + T: type MemoryDatastore, + stores: Table[Key, Datastore] = initTable[Key, Datastore]()): ?!T = + + var self = T() + for (k, v) in stores.pairs: + self.stores[?k.path] = MemoryStore(store: v, key: k) + + success self diff --git a/tests/datastore/testmemoryds.nim b/tests/datastore/testmemoryds.nim new file mode 100644 index 00000000..626d98da --- /dev/null +++ b/tests/datastore/testmemoryds.nim @@ -0,0 +1,138 @@ +import std/options +import std/sequtils +import std/os +from std/algorithm import sort, reversed + +import pkg/asynctest/unittest2 +import pkg/chronos +import pkg/stew/results +import pkg/stew/byteutils + +import pkg/datastore/fsds + +import ./dscommontests +import ./querycommontests + +suite "Test Basic MemoryDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + var + fsStore: MemoryDatastore + + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + fsStore = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() + + teardownAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + basicStoreTests(fsStore, key, bytes, otherBytes) + +suite "Test Misc MemoryDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + bytes = "some bytes".toBytes + + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + teardown: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + test "Test validDepth()": + let + fs = MemoryDatastore.new(root = "/", depth = 3).tryGet() + invalid = Key.init("/a/b/c/d").tryGet() + valid = Key.init("/a/b/c").tryGet() + + check: + not fs.validDepth(invalid) + fs.validDepth(valid) + + test "Test invalid key (path) depth": + let + fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/b/c/d").tryGet() + + check: + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr + + test "Test valid key (path) depth": + let + fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/b/c").tryGet() + + check: + (await fs.put(key, bytes)).isOk + (await fs.get(key)).isOk + (await fs.delete(key)).isOk + (await fs.has(key)).isOk + + test "Test key cannot write outside of root": + let + fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/../../c").tryGet() + + check: + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr + + test "Test key cannot convert to invalid path": + let + fs = MemoryDatastore.new(root = basePathAbs).tryGet() + + for c in invalidFilenameChars: + if c == ':': continue + if c == '/': continue + + let + key = Key.init("/" & c).tryGet() + + check: + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr + +suite "Test Query": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + + var + ds: MemoryDatastore + + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + ds = MemoryDatastore.new(root = basePathAbs, depth = 5).tryGet() + + teardown: + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + queryTests(ds, false) From 6399534ded61bea706fa1c07149b74233d2352de Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:37:14 -0700 Subject: [PATCH 051/445] add memory (test) ds --- datastore/databuffer.nim | 5 +++++ datastore/memoryds.nim | 47 ++++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 1a5e5461..de35b06c 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -61,3 +61,8 @@ proc toBuffer*(err: ref Exception): CatchableErrorBuffer = return CatchableErrorBuffer( msg: StringBuffer.new(err.msg) ) + +import ./key + +proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = + KeyBuffer.new(key.id()) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index c4d735e9..4a4a50c4 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -8,36 +8,34 @@ import pkg/upraises import ./key import ./query import ./datastore +import ./databuffer export key, query push: {.upraises: [].} type - MemoryStore* = object - store*: Datastore - key*: Key MemoryDatastore* = ref object of Datastore - stores*: Table[Key, MemoryStore] + store*: Table[KeyBuffer, ValueBuffer] method has*( - self: MemoryDatastore, - key: Key): Future[?!bool] {.async.} = - - without mounted =? self.dispatch(key): - return failure "No mounted datastore found" + self: MemoryDatastore, + key: Key +): Future[?!bool] {.async.} = - return (await mounted.store.store.has(mounted.relative)) + let dk = KeyBuffer.new(key) + return success self.store.hasKey(dk) method delete*( - self: MemoryDatastore, - key: Key): Future[?!void] {.async.} = - - without mounted =? self.dispatch(key), error: - return failure(error) + self: MemoryDatastore, + key: Key +): Future[?!void] {.async.} = - return (await mounted.store.store.delete(mounted.relative)) + let dk = KeyBuffer.new(key) + var val: ValueBuffer + discard self.store.pop(dk, val) + return success() method delete*( self: MemoryDatastore, @@ -50,13 +48,16 @@ method delete*( return success() method get*( - self: MemoryDatastore, - key: Key): Future[?!seq[byte]] {.async.} = - - without mounted =? self.dispatch(key), error: - return failure(error) - - return await mounted.store.store.get(mounted.relative) + self: MemoryDatastore, + key: Key +): Future[?!seq[byte]] {.async.} = + + let dk = KeyBuffer.new(key) + if self.store.hasKey(dk): + let res = self.store[dk] + return success res.toSeq(byte) + else: + return failure (ref DatastoreError)(msg: "no such key") method put*( self: MemoryDatastore, From 9ee931b92d6ff83a1ba34805c5d4e875a5c157a8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:40:33 -0700 Subject: [PATCH 052/445] add memory (test) ds --- datastore/databuffer.nim | 2 ++ datastore/memoryds.nim | 30 +++++++++++------------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index de35b06c..7bd18d57 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -66,3 +66,5 @@ import ./key proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = KeyBuffer.new(key.id()) +proc new*(tp: typedesc[ValueBuffer], data: seq[byte]): KeyBuffer = + DataBuffer.new(data) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 4a4a50c4..9db571b7 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -60,14 +60,15 @@ method get*( return failure (ref DatastoreError)(msg: "no such key") method put*( - self: MemoryDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async.} = - - without mounted =? self.dispatch(key), error: - return failure(error) + self: MemoryDatastore, + key: Key, + data: seq[byte] +): Future[?!void] {.async.} = - return (await mounted.store.store.put(mounted.relative, data)) + let dk = KeyBuffer.new(key) + let dv = ValueBuffer.new(key) + self.store[dk] = dv + return success() method put*( self: MemoryDatastore, @@ -80,18 +81,9 @@ method put*( return success() method close*(self: MemoryDatastore): Future[?!void] {.async.} = - for s in self.stores.values: - discard await s.store.close() - - # TODO: how to handle failed close? + self.store.clear() return success() -func new*( - T: type MemoryDatastore, - stores: Table[Key, Datastore] = initTable[Key, Datastore]()): ?!T = - - var self = T() - for (k, v) in stores.pairs: - self.stores[?k.path] = MemoryStore(store: v, key: k) - +func new*(tp: typedesc[MemoryDatastore]): ?!MemoryDatastore = + var self = default(tp) success self From de406a5b11a08e1a34f4f8d8eb5b26661c263639 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 18:43:42 -0700 Subject: [PATCH 053/445] add memory (test) ds --- datastore/memoryds.nim | 4 +- tests/datastore/testmemoryds.nim | 80 ++------------------------------ 2 files changed, 7 insertions(+), 77 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 9db571b7..56cfef82 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -84,6 +84,6 @@ method close*(self: MemoryDatastore): Future[?!void] {.async.} = self.store.clear() return success() -func new*(tp: typedesc[MemoryDatastore]): ?!MemoryDatastore = +func new*(tp: typedesc[MemoryDatastore]): MemoryDatastore = var self = default(tp) - success self + return self diff --git a/tests/datastore/testmemoryds.nim b/tests/datastore/testmemoryds.nim index 626d98da..fedb497a 100644 --- a/tests/datastore/testmemoryds.nim +++ b/tests/datastore/testmemoryds.nim @@ -8,35 +8,24 @@ import pkg/chronos import pkg/stew/results import pkg/stew/byteutils -import pkg/datastore/fsds +import pkg/datastore/memoryds import ./dscommontests import ./querycommontests suite "Test Basic MemoryDatastore": let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes var - fsStore: MemoryDatastore + memStore: MemoryDatastore setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - fsStore = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() - - teardownAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + memStore = MemoryDatastore.new() - basicStoreTests(fsStore, key, bytes, otherBytes) + basicStoreTests(memStore, key, bytes, otherBytes) suite "Test Misc MemoryDatastore": let @@ -54,65 +43,6 @@ suite "Test Misc MemoryDatastore": removeDir(basePathAbs) require(not dirExists(basePathAbs)) - test "Test validDepth()": - let - fs = MemoryDatastore.new(root = "/", depth = 3).tryGet() - invalid = Key.init("/a/b/c/d").tryGet() - valid = Key.init("/a/b/c").tryGet() - - check: - not fs.validDepth(invalid) - fs.validDepth(valid) - - test "Test invalid key (path) depth": - let - fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/b/c/d").tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr - - test "Test valid key (path) depth": - let - fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/b/c").tryGet() - - check: - (await fs.put(key, bytes)).isOk - (await fs.get(key)).isOk - (await fs.delete(key)).isOk - (await fs.has(key)).isOk - - test "Test key cannot write outside of root": - let - fs = MemoryDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/../../c").tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr - - test "Test key cannot convert to invalid path": - let - fs = MemoryDatastore.new(root = basePathAbs).tryGet() - - for c in invalidFilenameChars: - if c == ':': continue - if c == '/': continue - - let - key = Key.init("/" & c).tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr suite "Test Query": let @@ -128,7 +58,7 @@ suite "Test Query": require(not dirExists(basePathAbs)) createDir(basePathAbs) - ds = MemoryDatastore.new(root = basePathAbs, depth = 5).tryGet() + ds = MemoryDatastore.new() teardown: From 9b29a00a9200a90a59373198f5c23b90875ac28d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:22:16 -0700 Subject: [PATCH 054/445] add memory (test) ds --- datastore/databuffer.nim | 5 +++++ datastore/memoryds.nim | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 7bd18d57..2bafb951 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -27,6 +27,8 @@ proc `=destroy`*(x: var DataBufferHolder) = proc len*(a: DataBuffer): int = a[].size +proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) + proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = ## allocate new buffer with given size newSharedPtr(DataBufferHolder( @@ -63,8 +65,11 @@ proc toBuffer*(err: ref Exception): CatchableErrorBuffer = ) import ./key +import stew/results proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = KeyBuffer.new(key.id()) +proc toKey*(kb: KeyBuffer): Result[Key, ref CatchableError] = + Key.init(kb.toString()) proc new*(tp: typedesc[ValueBuffer], data: seq[byte]): KeyBuffer = DataBuffer.new(data) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 56cfef82..437866d7 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -1,4 +1,6 @@ import std/tables +import std/sequtils +import std/strutils import pkg/chronos import pkg/questionable @@ -80,6 +82,44 @@ method put*( return success() +proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer = + return iterator(): KeyBuffer {.closure.} = + let keys = self.store.keys().toSeq() + for key in keys: + if key.toString().startsWith(queryKey): + yield key + +method query*( + self: MemoryDatastore, + query: Query, +): Future[?!QueryIter] {.async.} = + + let + queryKey = query.key.id() + walker = keyIterator(self, queryKey) + + var + iter = QueryIter.new() + + proc next(): Future[?!QueryResponse] {.async.} = + let + kb = walker() + + if finished(walker): + iter.finished = true + return success (Key.none, EmptyBytes) + + let + key = kb.toKey().expect("should not fail") + var ds: ValueBuffer + if query.value: + ds = self.store[kb] + let data = if ds.isNil: EmptyBytes else: ds.toSeq(byte) + + return success (key.some, data) + + iter.next = next + return success iter method close*(self: MemoryDatastore): Future[?!void] {.async.} = self.store.clear() return success() From 170d50dd9296e65f5a97af8e8ae262705ddb458d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:23:49 -0700 Subject: [PATCH 055/445] add memory (test) ds --- datastore/memoryds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 437866d7..0ba37127 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -82,7 +82,7 @@ method put*( return success() -proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer = +proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer {.gcsafe.} = return iterator(): KeyBuffer {.closure.} = let keys = self.store.keys().toSeq() for key in keys: From 985697b1bb5f2d8fc8dbf66b4f42ad978c0548ba Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:25:33 -0700 Subject: [PATCH 056/445] add memory (test) ds --- datastore/memoryds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 0ba37127..e300feea 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -125,5 +125,5 @@ method close*(self: MemoryDatastore): Future[?!void] {.async.} = return success() func new*(tp: typedesc[MemoryDatastore]): MemoryDatastore = - var self = default(tp) + var self = tp() return self From a7f6794f7efeb90d3ae321fadebf64e44e9e6dae Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:41:37 -0700 Subject: [PATCH 057/445] add memory (test) ds --- datastore/databuffer.nim | 13 +++++++++++++ datastore/memoryds.nim | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 2bafb951..44fe2d3d 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -1,5 +1,8 @@ # import std/atomics import threading/smartptrs +import std/hashes + +export hashes type DataBufferHolder* = object @@ -29,6 +32,16 @@ proc len*(a: DataBuffer): int = a[].size proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) +proc hash*(a: DataBuffer): Hash = + a[].buf.toOpenArray(0, a[].size-1).hash() + +proc `==`*(a, b: DataBuffer): bool = + if a.isNil and b.isNil: return true + elif a.isNil or b.isNil: return false + elif a[].size != b[].size: return false + elif a[].buf == b[].buf: return true + else: a.hash() == b.hash() + proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = ## allocate new buffer with given size newSharedPtr(DataBufferHolder( diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index e300feea..dbc6421b 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -49,12 +49,18 @@ method delete*( return success() +import pretty + method get*( self: MemoryDatastore, key: Key ): Future[?!seq[byte]] {.async.} = let dk = KeyBuffer.new(key) + echo "getting: ", key + for k, v in self.store.pairs(): + print "get: ", k.toString() + if self.store.hasKey(dk): let res = self.store[dk] return success res.toSeq(byte) From 23423e52a9c95e00fac8a60b835626ed8caa8cbb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:44:41 -0700 Subject: [PATCH 058/445] add memory (test) ds --- datastore/memoryds.nim | 2 +- tests/datastore/testsharedbuffer.nim | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index dbc6421b..e8a1f53b 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -59,7 +59,7 @@ method get*( let dk = KeyBuffer.new(key) echo "getting: ", key for k, v in self.store.pairs(): - print "get: ", k.toString() + print "get: ", k.toString(), " v: ", v.toString().repr if self.store.hasKey(dk): let res = self.store[dk] diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testsharedbuffer.nim index 4a0f062e..38138b83 100644 --- a/tests/datastore/testsharedbuffer.nim +++ b/tests/datastore/testsharedbuffer.nim @@ -43,8 +43,8 @@ proc thread1(val: int) {.thread.} = var myBytes = DataBuffer.new(@"hello world") myBytes2 = myBytes - echo "thread1: sending: ", myBytes, " cnt: ", myBytes.unsafeGetAtomicCount() - echo "mybytes2: ", myBytes2, " cnt: ", myBytes2.unsafeGetAtomicCount() + echo "thread1: sending: ", myBytes + echo "mybytes2: ", myBytes2 shareVal = myBytes echo "thread1: sent, left over: ", $myBytes @@ -60,7 +60,7 @@ proc thread2(val: int) {.thread.} = wait(cond, lock) echo "thread2: receiving " let msg: DataBuffer = shareVal - echo "thread2: received: ", msg, " cnt: ", msg.unsafeGetAtomicCount() + echo "thread2: received: ", msg check msg.toSeq(char) == @"hello world" # os.sleep(100) From 344ba6a19b9787b87b215bb0d28351790ba24934 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:51:34 -0700 Subject: [PATCH 059/445] basic datstore checks --- tests/datastore/testsharedbuffer.nim | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testsharedbuffer.nim index 38138b83..0d59e7b8 100644 --- a/tests/datastore/testsharedbuffer.nim +++ b/tests/datastore/testsharedbuffer.nim @@ -78,6 +78,30 @@ proc runBasicTest() = suite "Share buffer test": - test "basic test": + setup: + let k1 {.used.} = Key.init("/a/b").get() + let k2 {.used.} = Key.init("/a").get() + let a {.used.} = KeyBuffer.new(k1) + let b {.used.} = KeyBuffer.new(Key.init("/a/b").get) + let c {.used.} = KeyBuffer.new(k2) + + test "creation": + let db = DataBuffer.new("abc") + check db[].size == 3 + check db[].buf[0].char == 'a' + check db[].buf[1].char == 'b' + check db[].buf[2].char == 'c' + test "equality": + check a == b + test "toString": + check a.toString() == "/a/b" + test "key conversion": + check a.toKey().get() == k1 + test "hash": + check a.hash() == b.hash() + test "hashes differ": + check a.hash() != c.hash() + + test "basic thread test": runBasicTest() From a11a681a6f655145e35f5556b7b6f89c940b6e5b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:55:05 -0700 Subject: [PATCH 060/445] basic datstore checks --- tests/datastore/testsharedbuffer.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testsharedbuffer.nim index 0d59e7b8..1d48b109 100644 --- a/tests/datastore/testsharedbuffer.nim +++ b/tests/datastore/testsharedbuffer.nim @@ -3,7 +3,7 @@ import std/sequtils import std/algorithm import std/locks import std/os - +import pkg/stew/byteutils import pkg/unittest2 import pkg/questionable import pkg/questionable/results @@ -95,12 +95,16 @@ suite "Share buffer test": check a == b test "toString": check a.toString() == "/a/b" - test "key conversion": - check a.toKey().get() == k1 test "hash": check a.hash() == b.hash() test "hashes differ": check a.hash() != c.hash() + test "key conversion": + check a.toKey().get() == k1 + test "seq conversion": + check a.toSeq(char) == @"/a/b" + test "seq conversion": + check a.toSeq(byte) == "/a/b".toBytes test "basic thread test": runBasicTest() From 8445920232b336441c44cd5224c3c70499b87f5a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 19:57:58 -0700 Subject: [PATCH 061/445] passing basic ds tests --- datastore/databuffer.nim | 2 -- datastore/memoryds.nim | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 44fe2d3d..10810687 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -84,5 +84,3 @@ proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = KeyBuffer.new(key.id()) proc toKey*(kb: KeyBuffer): Result[Key, ref CatchableError] = Key.init(kb.toString()) -proc new*(tp: typedesc[ValueBuffer], data: seq[byte]): KeyBuffer = - DataBuffer.new(data) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index e8a1f53b..3d3461e5 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -62,8 +62,9 @@ method get*( print "get: ", k.toString(), " v: ", v.toString().repr if self.store.hasKey(dk): - let res = self.store[dk] - return success res.toSeq(byte) + let res = self.store[dk].toSeq(byte) + print "get:res: ", res + return success res else: return failure (ref DatastoreError)(msg: "no such key") @@ -74,7 +75,7 @@ method put*( ): Future[?!void] {.async.} = let dk = KeyBuffer.new(key) - let dv = ValueBuffer.new(key) + let dv = ValueBuffer.new(data) self.store[dk] = dv return success() From 84986c77c35f46f9cc46115b8dd5e1006e2b5d79 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:32:28 -0700 Subject: [PATCH 062/445] sorting keys - fix --- datastore/databuffer.nim | 4 ++-- datastore/memoryds.nim | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 10810687..fef4c784 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -24,8 +24,8 @@ type proc `=destroy`*(x: var DataBufferHolder) = ## copy pointer implementation if x.buf != nil: - when isMainModule or true: - echo "buffer: FREE: ", repr x.buf.pointer + # when isMainModule or true: + # echo "buffer: FREE: ", repr x.buf.pointer deallocShared(x.buf) proc len*(a: DataBuffer): int = a[].size diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 3d3461e5..dccf2fcb 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -1,6 +1,7 @@ import std/tables import std/sequtils import std/strutils +import std/algorithm import pkg/chronos import pkg/questionable @@ -91,11 +92,14 @@ method put*( proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer {.gcsafe.} = return iterator(): KeyBuffer {.closure.} = - let keys = self.store.keys().toSeq() + var keys = self.store.keys().toSeq() + keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) for key in keys: if key.toString().startsWith(queryKey): yield key +import pretty + method query*( self: MemoryDatastore, query: Query, @@ -108,10 +112,14 @@ method query*( var iter = QueryIter.new() + echo "queryKey: ", queryKey + proc next(): Future[?!QueryResponse] {.async.} = let kb = walker() + print "query: ", kb.toString + if finished(walker): iter.finished = true return success (Key.none, EmptyBytes) From 18e96b6b8512cb24c18a3d86f570d263b751ea10 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:33:52 -0700 Subject: [PATCH 063/445] cleanup --- datastore/memoryds.nim | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index dccf2fcb..1c83aed1 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -108,24 +108,17 @@ method query*( let queryKey = query.key.id() walker = keyIterator(self, queryKey) - var iter = QueryIter.new() - echo "queryKey: ", queryKey - proc next(): Future[?!QueryResponse] {.async.} = - let - kb = walker() - - print "query: ", kb.toString + let kb = walker() if finished(walker): iter.finished = true return success (Key.none, EmptyBytes) - let - key = kb.toKey().expect("should not fail") + let key = kb.toKey().expect("should not fail") var ds: ValueBuffer if query.value: ds = self.store[kb] @@ -135,6 +128,7 @@ method query*( iter.next = next return success iter + method close*(self: MemoryDatastore): Future[?!void] {.async.} = self.store.clear() return success() From f3188777276105fdb0308b6cf6ece617c83bd788 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:35:00 -0700 Subject: [PATCH 064/445] cleanup --- datastore/memoryds.nim | 9 --------- 1 file changed, 9 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 1c83aed1..dd3192fd 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -50,21 +50,14 @@ method delete*( return success() -import pretty - method get*( self: MemoryDatastore, key: Key ): Future[?!seq[byte]] {.async.} = let dk = KeyBuffer.new(key) - echo "getting: ", key - for k, v in self.store.pairs(): - print "get: ", k.toString(), " v: ", v.toString().repr - if self.store.hasKey(dk): let res = self.store[dk].toSeq(byte) - print "get:res: ", res return success res else: return failure (ref DatastoreError)(msg: "no such key") @@ -98,8 +91,6 @@ proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer { if key.toString().startsWith(queryKey): yield key -import pretty - method query*( self: MemoryDatastore, query: Query, From b27593e8cf6d661870d319cd16dbbf17ae711675 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:45:55 -0700 Subject: [PATCH 065/445] cleanup --- datastore/sharedds.nim | 7 +++++-- datastore/threadbackend.nim | 10 ---------- tests/datastore/testsharedds.nim | 6 ++++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index f9d689b4..68e0f4ce 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -105,7 +105,7 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() -proc newSharedDataStore*[T: Datastore]( +proc newSharedDataStore*( ds: Datastore, ): Future[?!SharedDatastore] {.async.} = @@ -120,7 +120,10 @@ proc newSharedDataStore*[T: Datastore]( echo "\nnewDataStore: threadId:", getThreadId() # GC_ref(ds) res[].value[].ds = ds - res.createThreadDatastore() + try: + res[].value[].tp = Taskpool.new(num_threads = 2) + except Exception as exc: + return err((ref DatastoreError)(msg: exc.msg)) await wait(res[].signal) finally: echo "closing signal" diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index ba91ad8f..69e6e31e 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -128,13 +128,3 @@ proc put*( print "bval: ", bval tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) - -proc createThreadDatastore*(ret: ThreadDatastorePtr): Result[void, ref CatchableError] = - try: - echo "createThreadDatastore: start" - ret[].tp = Taskpool.new(num_threads = 2) - echo "createThreadDatastore: done" - - except Exception as exc: - return err((ref DatastoreError)(msg: exc.msg)) - diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 5c47f438..81ef17eb 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -1,13 +1,14 @@ import std/options import std/sequtils import std/os -from std/algorithm import sort, reversed +import std/algorithm import pkg/asynctest/unittest2 import pkg/chronos import pkg/stew/results import pkg/stew/byteutils +import pkg/datastore/memoryds import pkg/datastore/sharedds import ./dscommontests @@ -24,7 +25,8 @@ suite "Test Basic SharedDatastore": let backend = ThreadBackend( kind: TestBackend, ) - let res = await newSharedDataStore(backend) + let mem = MemoryDatastore.new() + let res = await newSharedDataStore(mem) check res.isOk() sds = res.get() # echo "sds: ", repr sds From 51683be0e536ab0613892d6d079600703d9106d7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:48:42 -0700 Subject: [PATCH 066/445] cleanup --- datastore/sharedds.nim | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index 68e0f4ce..c97f8392 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -109,29 +109,17 @@ proc newSharedDataStore*( ds: Datastore, ): Future[?!SharedDatastore] {.async.} = - var - self = SharedDatastore() - - without res =? newThreadResult(ThreadDatastorePtr), err: - return failure(err) + var self = SharedDatastore() + let value = newSharedPtr(ThreadDatastore) + echo "\nnewDataStore: threadId:", getThreadId() + # GC_ref(ds) + value[].ds = ds try: - res[].value = newSharedPtr(ThreadDatastore) - echo "\nnewDataStore: threadId:", getThreadId() - # GC_ref(ds) - res[].value[].ds = ds - try: - res[].value[].tp = Taskpool.new(num_threads = 2) - except Exception as exc: - return err((ref DatastoreError)(msg: exc.msg)) - await wait(res[].signal) - finally: - echo "closing signal" - res[].signal.close() - - print "\nnewSharedDataStore:state: ", res[].state - print "\nnewSharedDataStore:value: ", res[].value[].backend + value[].tp = Taskpool.new(num_threads = 2) + except Exception as exc: + return err((ref DatastoreError)(msg: exc.msg)) - self.tds = res[].value + self.tds = value success self From 03830cef8f77c7dcd5c019eb6067ff108082baec Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:56:29 -0700 Subject: [PATCH 067/445] cleanup --- datastore/sharedds.nim | 3 +-- datastore/threadbackend.nim | 32 ++---------------------------- tests/datastore/testsharedds.nim | 34 +++++++++++--------------------- 3 files changed, 14 insertions(+), 55 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index c97f8392..e20b22b3 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -17,12 +17,11 @@ import ./fsds import pretty -export key, query, ThreadBackend +export key, query push: {.upraises: [].} type - SharedDatastore* = ref object of Datastore # stores*: Table[Key, SharedDatastore] tds: ThreadDatastorePtr diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 69e6e31e..0167608b 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -31,38 +31,12 @@ type TResult*[T] = SharedPtr[ThreadResult[T]] - ThreadBackendKind* {.pure.} = enum - NoBackend - TestBackend - FSBackend - SQliteBackend - - ThreadBackend* = object - case kind*: ThreadBackendKind - of FSBackend: - root*: StringBuffer - depth*: int - caseSensitive*: bool - ignoreProtected*: bool - of SQliteBackend: - discard - of TestBackend: - count*: int - of NoBackend: - discard - ThreadDatastore* = object tp*: Taskpool - backend*: ThreadBackendKind ds*: Datastore ThreadDatastorePtr* = SharedPtr[ThreadDatastore] - Test* = object - count*: ThreadBackendKind - - TestPtr* = SharedPtr[Test] - proc newThreadResult*[T]( tp: typedesc[T] ): Result[TResult[T], ref CatchableError] = @@ -76,7 +50,6 @@ proc newThreadResult*[T]( proc getTask*( ret: TResult[DataBuffer], - backend: ThreadBackendKind, key: KeyBuffer, ) = # return ok(DataBuffer.new()) @@ -98,13 +71,12 @@ proc get*( let bkey = StringBuffer.new(key.id()) print "bkey: ", bkey - tds[].tp.spawn getTask(ret, tds[].backend, bkey) + tds[].tp.spawn getTask(ret, bkey) import os proc putTask*( ret: TResult[void], - backend: ThreadBackendKind, key: KeyBuffer, data: DataBuffer, ) = @@ -127,4 +99,4 @@ proc put*( print "bkey: ", bkey print "bval: ", bval - tds[].tp.spawn putTask(ret, tds[].backend, bkey, bval) + tds[].tp.spawn putTask(ret, bkey, bval) diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 81ef17eb..7ae3ca47 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -22,9 +22,6 @@ suite "Test Basic SharedDatastore": var sds: SharedDatastore - let backend = ThreadBackend( - kind: TestBackend, - ) let mem = MemoryDatastore.new() let res = await newSharedDataStore(mem) check res.isOk() @@ -50,30 +47,21 @@ suite "Test Basic SharedDatastore": res3.cancel() # print "res3: ", res3 -# suite "Test Basic FSDatastore": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes - -# var -# fsStore: FSDatastore +suite "Test Basic FSDatastore": -# setupAll: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) + var + memStore: MemoryDatastore + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes -# fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + setupAll: + memStore = MemoryDatastore.new() -# teardownAll: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) + teardownAll: + (await memStore.close()).get() -# basicStoreTests(fsStore, key, bytes, otherBytes) + basicStoreTests(memStore, key, bytes, otherBytes) # suite "Test Misc FSDatastore": # let From a68ce94c8dac3e385bd051ed1f92c646b1ccca99 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 21:59:31 -0700 Subject: [PATCH 068/445] test setup --- datastore/sharedds.nim | 2 +- tests/datastore/testsharedds.nim | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datastore/sharedds.nim b/datastore/sharedds.nim index e20b22b3..41fcdf90 100644 --- a/datastore/sharedds.nim +++ b/datastore/sharedds.nim @@ -106,7 +106,7 @@ method close*( proc newSharedDataStore*( ds: Datastore, -): Future[?!SharedDatastore] {.async.} = +): ?!SharedDatastore = var self = SharedDatastore() diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 7ae3ca47..0fcf9d71 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -23,7 +23,7 @@ suite "Test Basic SharedDatastore": var sds: SharedDatastore let mem = MemoryDatastore.new() - let res = await newSharedDataStore(mem) + let res = newSharedDataStore(mem) check res.isOk() sds = res.get() # echo "sds: ", repr sds @@ -51,12 +51,14 @@ suite "Test Basic FSDatastore": var memStore: MemoryDatastore + ds: SharedDatastore key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes setupAll: memStore = MemoryDatastore.new() + ds = newSharedDataStore(memStore).expect("should work") teardownAll: (await memStore.close()).get() From 6364c8316d8e48803507e92c0cbc5fea7b0ddf3b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 22:18:08 -0700 Subject: [PATCH 069/445] test setup --- datastore/memoryds.nim | 1 + datastore/threadbackend.nim | 62 ++++++++++++++++++++------------ tests/datastore/testsharedds.nim | 2 +- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index dd3192fd..1910c439 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -70,6 +70,7 @@ method put*( let dk = KeyBuffer.new(key) let dv = ValueBuffer.new(data) + echo "MemoryDatastore:put: ", key, " => ", data.repr self.store[dk] = dv return success() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 0167608b..d0fef14f 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -1,5 +1,7 @@ - +import pkg/chronos import pkg/chronos/threadsync +import pkg/questionable +import pkg/questionable/results import stew/results import pkg/upraises import pkg/taskpools @@ -50,42 +52,58 @@ proc newThreadResult*[T]( proc getTask*( ret: TResult[DataBuffer], - key: KeyBuffer, + tds: ThreadDatastorePtr, + kb: KeyBuffer, ) = - # return ok(DataBuffer.new()) - print "\nthrbackend: getTask: ", ret[] - print "\nthrbackend: getTask:key: ", key - let data = DataBuffer.new("hello world!") - print "\nthrbackend: getTask:data: ", data - ret[].state = Success - ret[].value = data + without key =? kb.toKey(), err: + ret[].state = Error + try: + let res = waitFor tds[].ds.get(key) + if res.isErr: + ret[].state = Error + ret[].error = res.error().toBuffer() + else: + let db = DataBuffer.new res.get() + ret[].state = Success + ret[].value = db + + discard ret[].signal.fireSync() + except CatchableError as err: + ret[].state = Error + ret[].error = err.toBuffer() - print "thrbackend: putTask: fire", ret[].signal.fireSync() proc get*( ret: TResult[DataBuffer], tds: ThreadDatastorePtr, key: Key, ) = - echo "thrfrontend:put: " let bkey = StringBuffer.new(key.id()) - print "bkey: ", bkey - - tds[].tp.spawn getTask(ret, bkey) + tds[].tp.spawn getTask(ret, tds, bkey) import os proc putTask*( ret: TResult[void], - key: KeyBuffer, - data: DataBuffer, + tds: ThreadDatastorePtr, + kb: KeyBuffer, + db: DataBuffer, ) = - print "\nthrbackend: putTask: ", ret[] - print "\nthrbackend: putTask:key: ", key - print "\nthrbackend: putTask:data: ", data - os.sleep(200) - print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + without key =? kb.toKey(), err: + ret[].state = Error + + let data = db.toSeq(byte) + let res = (waitFor tds[].ds.put(key, data)).catch + # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + if res.isErr: + ret[].state = Error + ret[].error = res.error().toBuffer() + else: + ret[].state = Success + + discard ret[].signal.fireSync() + proc put*( ret: TResult[void], @@ -99,4 +117,4 @@ proc put*( print "bkey: ", bkey print "bval: ", bval - tds[].tp.spawn putTask(ret, bkey, bval) + tds[].tp.spawn putTask(ret, tds, bkey, bval) diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index 0fcf9d71..fa777f03 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -63,7 +63,7 @@ suite "Test Basic FSDatastore": teardownAll: (await memStore.close()).get() - basicStoreTests(memStore, key, bytes, otherBytes) + basicStoreTests(ds, key, bytes, otherBytes) # suite "Test Misc FSDatastore": # let From 40cc03e3c34ccb56d255b229331d1542d0933968 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Aug 2023 22:26:00 -0700 Subject: [PATCH 070/445] updates --- tests/datastore/testsharedds.nim | 39 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testsharedds.nim index fa777f03..a0888d2b 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testsharedds.nim @@ -17,37 +17,40 @@ import ./querycommontests import pretty suite "Test Basic SharedDatastore": + var + sds: SharedDatastore + mem: MemoryDatastore + res: SharedDatastore + key1: Key + data: seq[byte] - test "check create": - - var sds: SharedDatastore - - let mem = MemoryDatastore.new() - let res = newSharedDataStore(mem) - check res.isOk() - sds = res.get() - # echo "sds: ", repr sds + setupAll: + mem = MemoryDatastore.new() + sds = newSharedDataStore(mem).expect("should work") + key1 = Key.init("/a").tryGet + data = "value for 1".toBytes() + test "check put": echo "\n\n=== put ===" - let key1 = Key.init("/a").tryGet - let res1 = await sds.put(key1, "value for 1".toBytes()) + let res1 = await sds.put(key1, data) print "res1: ", res1 + test "check get": echo "\n\n=== get ===" let res2 = await sds.get(key1) - check res2.get() == "hello world!".toBytes() + check res2.get() == data var val = "" for c in res2.get(): val &= char(c) print "get res2: ", $val - echo "\n\n=== put cancel ===" - # let res1 = await sds.put(key1, "value for 1".toBytes()) - let res3 = sds.put(key1, "value for 1".toBytes()) - res3.cancel() - # print "res3: ", res3 + # echo "\n\n=== put cancel ===" + # # let res1 = await sds.put(key1, "value for 1".toBytes()) + # let res3 = sds.put(key1, "value for 1".toBytes()) + # res3.cancel() + # # print "res3: ", res3 -suite "Test Basic FSDatastore": +suite "Test Basic SharedDatastore": var memStore: MemoryDatastore From 22c9d8556a048ab410a288570974d8a6ee6c5a73 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 12:50:36 -0700 Subject: [PATCH 071/445] rename sharedds to threadproxyds --- datastore/{sharedds.nim => threadproxyds.nim} | 22 +++++++++---------- ...testsharedds.nim => testthreadproxyds.nim} | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) rename datastore/{sharedds.nim => threadproxyds.nim} (85%) rename tests/datastore/{testsharedds.nim => testthreadproxyds.nim} (95%) diff --git a/datastore/sharedds.nim b/datastore/threadproxyds.nim similarity index 85% rename from datastore/sharedds.nim rename to datastore/threadproxyds.nim index 41fcdf90..111008f7 100644 --- a/datastore/sharedds.nim +++ b/datastore/threadproxyds.nim @@ -22,30 +22,30 @@ export key, query push: {.upraises: [].} type - SharedDatastore* = ref object of Datastore - # stores*: Table[Key, SharedDatastore] + ThreadProxyDatastore* = ref object of Datastore + # stores*: Table[Key, ThreadProxyDatastore] tds: ThreadDatastorePtr method has*( - self: SharedDatastore, + self: ThreadProxyDatastore, key: Key ): Future[?!bool] {.async.} = return success(true) method delete*( - self: SharedDatastore, + self: ThreadProxyDatastore, key: Key ): Future[?!void] {.async.} = return success() method delete*( - self: SharedDatastore, + self: ThreadProxyDatastore, keys: seq[Key] ): Future[?!void] {.async.} = return success() method get*( - self: SharedDatastore, + self: ThreadProxyDatastore, key: Key ): Future[?!seq[byte]] {.async.} = @@ -64,7 +64,7 @@ method get*( return success(data) method put*( - self: SharedDatastore, + self: ThreadProxyDatastore, key: Key, data: seq[byte] ): Future[?!void] {.async.} = @@ -84,13 +84,13 @@ method put*( return success() method put*( - self: SharedDatastore, + self: ThreadProxyDatastore, batch: seq[BatchEntry] ): Future[?!void] {.async.} = raiseAssert("Not implemented!") method close*( - self: SharedDatastore + self: ThreadProxyDatastore ): Future[?!void] {.async.} = # TODO: how to handle failed close? echo "ThreadDatastore: FREE: " @@ -106,9 +106,9 @@ method close*( proc newSharedDataStore*( ds: Datastore, -): ?!SharedDatastore = +): ?!ThreadProxyDatastore = - var self = SharedDatastore() + var self = ThreadProxyDatastore() let value = newSharedPtr(ThreadDatastore) echo "\nnewDataStore: threadId:", getThreadId() diff --git a/tests/datastore/testsharedds.nim b/tests/datastore/testthreadproxyds.nim similarity index 95% rename from tests/datastore/testsharedds.nim rename to tests/datastore/testthreadproxyds.nim index a0888d2b..893ba1ca 100644 --- a/tests/datastore/testsharedds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -9,18 +9,18 @@ import pkg/stew/results import pkg/stew/byteutils import pkg/datastore/memoryds -import pkg/datastore/sharedds +import pkg/datastore/threadproxyds import ./dscommontests import ./querycommontests import pretty -suite "Test Basic SharedDatastore": +suite "Test Basic ThreadProxyDatastore": var - sds: SharedDatastore + sds: ThreadProxyDatastore mem: MemoryDatastore - res: SharedDatastore + res: ThreadProxyDatastore key1: Key data: seq[byte] @@ -50,11 +50,11 @@ suite "Test Basic SharedDatastore": # res3.cancel() # # print "res3: ", res3 -suite "Test Basic SharedDatastore": +suite "Test Basic ThreadProxyDatastore": var memStore: MemoryDatastore - ds: SharedDatastore + ds: ThreadProxyDatastore key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes From 2cded134dd8743d3a7865b7107bb9488547cce87 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 12:55:38 -0700 Subject: [PATCH 072/445] impl delete --- datastore/threadbackend.nim | 28 +++++++++++++++++++++++++++- datastore/threadproxyds.nim | 15 ++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index d0fef14f..9eb528b6 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -81,7 +81,6 @@ proc get*( let bkey = StringBuffer.new(key.id()) tds[].tp.spawn getTask(ret, tds, bkey) -import os proc putTask*( ret: TResult[void], @@ -118,3 +117,30 @@ proc put*( print "bval: ", bval tds[].tp.spawn putTask(ret, tds, bkey, bval) + +proc deleteTask*( + ret: TResult[void], + tds: ThreadDatastorePtr, + kb: KeyBuffer, +) = + + without key =? kb.toKey(), err: + ret[].state = Error + + let res = (waitFor tds[].ds.delete(key)).catch + # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() + if res.isErr: + ret[].state = Error + ret[].error = res.error().toBuffer() + else: + ret[].state = Success + + discard ret[].signal.fireSync() + +proc delete*( + ret: TResult[void], + tds: ThreadDatastorePtr, + key: Key, +) = + let bkey = StringBuffer.new(key.id()) + tds[].tp.spawn deleteTask(ret, tds, bkey) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 111008f7..7ddbd8a1 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -23,7 +23,6 @@ push: {.upraises: [].} type ThreadProxyDatastore* = ref object of Datastore - # stores*: Table[Key, ThreadProxyDatastore] tds: ThreadDatastorePtr method has*( @@ -36,8 +35,22 @@ method delete*( self: ThreadProxyDatastore, key: Key ): Future[?!void] {.async.} = + + without ret =? newThreadResult(void), err: + return failure(err) + + echo "res: ", ret + try: + delete(ret, self.tds, key) + await wait(ret[].signal) + finally: + echo "closing signal" + ret[].signal.close() + + echo "\nSharedDataStore:put:value: ", ret[].repr return success() + method delete*( self: ThreadProxyDatastore, keys: seq[Key] From 9fa6c6e6617d2757e421009e65d1e3089be2cc38 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 12:57:31 -0700 Subject: [PATCH 073/445] impl has --- datastore/threadbackend.nim | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 9eb528b6..61f0e88b 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -50,6 +50,34 @@ proc newThreadResult*[T]( res[].signal = signal.get() ok res +proc hasTask*( + ret: TResult[bool], + tds: ThreadDatastorePtr, + kb: KeyBuffer, +) = + without key =? kb.toKey(), err: + ret[].state = Error + try: + let res = waitFor tds[].ds.has(key) + if res.isErr: + ret[].state = Error + ret[].error = res.error().toBuffer() + else: + ret[].state = Success + ret[].value = res.get() + discard ret[].signal.fireSync() + except CatchableError as err: + ret[].state = Error + ret[].error = err.toBuffer() + +proc has*( + ret: TResult[bool], + tds: ThreadDatastorePtr, + key: Key, +) = + let bkey = StringBuffer.new(key.id()) + tds[].tp.spawn hasTask(ret, tds, bkey) + proc getTask*( ret: TResult[DataBuffer], tds: ThreadDatastorePtr, @@ -72,7 +100,6 @@ proc getTask*( ret[].state = Error ret[].error = err.toBuffer() - proc get*( ret: TResult[DataBuffer], tds: ThreadDatastorePtr, @@ -103,7 +130,6 @@ proc putTask*( discard ret[].signal.fireSync() - proc put*( ret: TResult[void], tds: ThreadDatastorePtr, @@ -118,6 +144,7 @@ proc put*( tds[].tp.spawn putTask(ret, tds, bkey, bval) + proc deleteTask*( ret: TResult[void], tds: ThreadDatastorePtr, From db666f9441476098739046a0e921ea1ec30cd0d5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 13:00:01 -0700 Subject: [PATCH 074/445] impl has --- datastore/threadproxyds.nim | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 7ddbd8a1..bd49de19 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -29,7 +29,19 @@ method has*( self: ThreadProxyDatastore, key: Key ): Future[?!bool] {.async.} = - return success(true) + + without ret =? newThreadResult(bool), err: + return failure(err) + + try: + has(ret, self.tds, key) + await wait(ret[].signal) + finally: + echo "closing signal" + ret[].signal.close() + + echo "\nSharedDataStore:has:value: ", ret[].repr + return success(ret[].value) method delete*( self: ThreadProxyDatastore, @@ -39,7 +51,6 @@ method delete*( without ret =? newThreadResult(void), err: return failure(err) - echo "res: ", ret try: delete(ret, self.tds, key) await wait(ret[].signal) @@ -50,7 +61,6 @@ method delete*( echo "\nSharedDataStore:put:value: ", ret[].repr return success() - method delete*( self: ThreadProxyDatastore, keys: seq[Key] From c21532b731672aec292ca27178d5d646a0ae3afd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 13:00:53 -0700 Subject: [PATCH 075/445] impl has --- datastore/threadbackend.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 61f0e88b..f495cf7e 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -25,7 +25,7 @@ type Success Error - ThreadResult*[T: DataBuffer | void | ThreadDatastorePtr] = object + ThreadResult*[T: DataBuffer | void | bool | ThreadDatastorePtr] = object state*: ThreadResultKind signal*: ThreadSignalPtr value*: T From 11267c842fedc64df1d745aedfe34fdf0e498fe8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 13:08:55 -0700 Subject: [PATCH 076/445] first impl of batch ops --- datastore/threadproxyds.nim | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index bd49de19..73396420 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -65,6 +65,11 @@ method delete*( self: ThreadProxyDatastore, keys: seq[Key] ): Future[?!void] {.async.} = + + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err + return success() method get*( @@ -110,7 +115,12 @@ method put*( self: ThreadProxyDatastore, batch: seq[BatchEntry] ): Future[?!void] {.async.} = - raiseAssert("Not implemented!") + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + return success() method close*( self: ThreadProxyDatastore From a2c9a7efc358bb9e0059fe6aed736cc513d12f11 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 13:14:33 -0700 Subject: [PATCH 077/445] various cleanup --- datastore/memoryds.nim | 1 - datastore/threadbackend.nim | 3 --- datastore/threadproxyds.nim | 16 ++++++---------- .../{testsharedbuffer.nim => testdatabuffer.nim} | 0 4 files changed, 6 insertions(+), 14 deletions(-) rename tests/datastore/{testsharedbuffer.nim => testdatabuffer.nim} (100%) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 1910c439..dd3192fd 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -70,7 +70,6 @@ method put*( let dk = KeyBuffer.new(key) let dv = ValueBuffer.new(data) - echo "MemoryDatastore:put: ", key, " => ", data.repr self.store[dk] = dv return success() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f495cf7e..476ebfe3 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -136,11 +136,8 @@ proc put*( key: Key, data: seq[byte] ) = - echo "thrfrontend:put: " let bkey = StringBuffer.new(key.id()) let bval = DataBuffer.new(data) - print "bkey: ", bkey - print "bval: ", bval tds[].tp.spawn putTask(ret, tds, bkey, bval) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 73396420..a67c9abc 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -37,10 +37,10 @@ method has*( has(ret, self.tds, key) await wait(ret[].signal) finally: - echo "closing signal" + # echo "closing signal" ret[].signal.close() - echo "\nSharedDataStore:has:value: ", ret[].repr + # echo "\nSharedDataStore:has:value: ", ret[].repr return success(ret[].value) method delete*( @@ -55,10 +55,10 @@ method delete*( delete(ret, self.tds, key) await wait(ret[].signal) finally: - echo "closing signal" + # echo "closing signal" ret[].signal.close() - echo "\nSharedDataStore:put:value: ", ret[].repr + # echo "\nSharedDataStore:put:value: ", ret[].repr return success() method delete*( @@ -84,10 +84,9 @@ method get*( get(ret, self.tds, key) await wait(ret[].signal) finally: - echo "closing signal" ret[].signal.close() - print "\nSharedDataStore:put:value: ", ret[] + # print "\nSharedDataStore:put:value: ", ret[] let data = ret[].value.toSeq(byte) return success(data) @@ -100,15 +99,13 @@ method put*( without ret =? newThreadResult(void), err: return failure(err) - echo "res: ", ret try: put(ret, self.tds, key, data) await wait(ret[].signal) finally: - echo "closing signal" ret[].signal.close() - echo "\nSharedDataStore:put:value: ", ret[].repr + # echo "\nSharedDataStore:put:value: ", ret[].repr return success() method put*( @@ -126,7 +123,6 @@ method close*( self: ThreadProxyDatastore ): Future[?!void] {.async.} = # TODO: how to handle failed close? - echo "ThreadDatastore: FREE: " result = success() without res =? self.tds[].ds.close(), err: diff --git a/tests/datastore/testsharedbuffer.nim b/tests/datastore/testdatabuffer.nim similarity index 100% rename from tests/datastore/testsharedbuffer.nim rename to tests/datastore/testdatabuffer.nim From 1e6fc450f5392dac364c879a89676502472796e8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 13:14:59 -0700 Subject: [PATCH 078/445] don't test 1.2 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9b89081..a1b3c2fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: cache_nonce: [ 1 ] - nim_version: [ 1.2.18, 1.6.6 ] + nim_version: [ 1.6.6 ] platform: - { icon: 🐧, From 485f2ecaed416c1cf551d1cf59859f7ca6e5a449 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:04:51 -0700 Subject: [PATCH 079/445] cleanup --- tests/datastore/testthreadproxyds.nim | 76 --------------------------- 1 file changed, 76 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 893ba1ca..8eec311a 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -68,82 +68,6 @@ suite "Test Basic ThreadProxyDatastore": basicStoreTests(ds, key, bytes, otherBytes) -# suite "Test Misc FSDatastore": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath -# bytes = "some bytes".toBytes - -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) - -# teardown: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) - -# test "Test validDepth()": -# let -# fs = FSDatastore.new(root = "/", depth = 3).tryGet() -# invalid = Key.init("/a/b/c/d").tryGet() -# valid = Key.init("/a/b/c").tryGet() - -# check: -# not fs.validDepth(invalid) -# fs.validDepth(valid) - -# test "Test invalid key (path) depth": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/b/c/d").tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr - -# test "Test valid key (path) depth": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/b/c").tryGet() - -# check: -# (await fs.put(key, bytes)).isOk -# (await fs.get(key)).isOk -# (await fs.delete(key)).isOk -# (await fs.has(key)).isOk - -# test "Test key cannot write outside of root": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/../../c").tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr - -# test "Test key cannot convert to invalid path": -# let -# fs = FSDatastore.new(root = basePathAbs).tryGet() - -# for c in invalidFilenameChars: -# if c == ':': continue -# if c == '/': continue - -# let -# key = Key.init("/" & c).tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr - # suite "Test Query": # let # path = currentSourcePath() # get this file's name From 9e0b976f388c0767374a0ef4f3d62d3049fa9a99 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:14:55 -0700 Subject: [PATCH 080/445] change TResult to case type --- datastore/threadbackend.nim | 55 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 476ebfe3..47c77295 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -26,10 +26,14 @@ type Error ThreadResult*[T: DataBuffer | void | bool | ThreadDatastorePtr] = object - state*: ThreadResultKind signal*: ThreadSignalPtr - value*: T - error*: CatchableErrorBuffer + case state*: ThreadResultKind + of ThreadResultKind.NotReady: + discard + of ThreadResultKind.Success: + value*: T + of ThreadResultKind.Error: + error*: CatchableErrorBuffer TResult*[T] = SharedPtr[ThreadResult[T]] @@ -50,25 +54,37 @@ proc newThreadResult*[T]( res[].signal = signal.get() ok res +proc success*[T](ret: TResult[T], value: T) = + {.cast(uncheckedAssign).}: + ret[].state = ThreadResultKind.Success + ret[].value = value + +proc success*[T: void](ret: TResult[T]) = + {.cast(uncheckedAssign).}: + ret[].state = ThreadResultKind.Success + +proc failure*[T](ret: TResult[T], err: ref CatchableError) = + {.cast(uncheckedAssign).}: + ret[].state = ThreadResultKind.Error + ret[].error = err.toBuffer() + proc hasTask*( ret: TResult[bool], tds: ThreadDatastorePtr, kb: KeyBuffer, ) = without key =? kb.toKey(), err: - ret[].state = Error + ret.failure(err) + try: let res = waitFor tds[].ds.has(key) if res.isErr: - ret[].state = Error - ret[].error = res.error().toBuffer() + ret.failure(res.error()) else: - ret[].state = Success - ret[].value = res.get() + ret.success(res.get()) discard ret[].signal.fireSync() except CatchableError as err: - ret[].state = Error - ret[].error = err.toBuffer() + ret.failure(err) proc has*( ret: TResult[bool], @@ -88,17 +104,14 @@ proc getTask*( try: let res = waitFor tds[].ds.get(key) if res.isErr: - ret[].state = Error - ret[].error = res.error().toBuffer() + ret.failure(res.error()) else: let db = DataBuffer.new res.get() - ret[].state = Success - ret[].value = db + ret.success(db) discard ret[].signal.fireSync() except CatchableError as err: - ret[].state = Error - ret[].error = err.toBuffer() + ret.failure(err) proc get*( ret: TResult[DataBuffer], @@ -123,10 +136,9 @@ proc putTask*( let res = (waitFor tds[].ds.put(key, data)).catch # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() if res.isErr: - ret[].state = Error - ret[].error = res.error().toBuffer() + ret.failure(res.error()) else: - ret[].state = Success + ret.success() discard ret[].signal.fireSync() @@ -154,10 +166,9 @@ proc deleteTask*( let res = (waitFor tds[].ds.delete(key)).catch # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() if res.isErr: - ret[].state = Error - ret[].error = res.error().toBuffer() + ret.failure(res.error()) else: - ret[].state = Success + ret.success() discard ret[].signal.fireSync() From 5f13765a490080400fe9c93ee093395cf4a9c298 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:40:44 -0700 Subject: [PATCH 081/445] switch to result type --- datastore/threadbackend.nim | 44 ++++++++++++--------------- datastore/threadproxyds.nim | 8 ++--- tests/datastore/testthreadproxyds.nim | 1 + 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 47c77295..395d54b4 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -20,20 +20,9 @@ push: {.upraises: [].} type - ThreadResultKind* {.pure.} = enum - NotReady - Success - Error - ThreadResult*[T: DataBuffer | void | bool | ThreadDatastorePtr] = object signal*: ThreadSignalPtr - case state*: ThreadResultKind - of ThreadResultKind.NotReady: - discard - of ThreadResultKind.Success: - value*: T - of ThreadResultKind.Error: - error*: CatchableErrorBuffer + results*: Result[T, CatchableErrorBuffer] TResult*[T] = SharedPtr[ThreadResult[T]] @@ -55,18 +44,25 @@ proc newThreadResult*[T]( ok res proc success*[T](ret: TResult[T], value: T) = - {.cast(uncheckedAssign).}: - ret[].state = ThreadResultKind.Success - ret[].value = value + ret[].results.ok(value) proc success*[T: void](ret: TResult[T]) = - {.cast(uncheckedAssign).}: - ret[].state = ThreadResultKind.Success + ret[].results.ok() + +proc failure*[T](ret: TResult[T], exc: ref CatchableError) = + ret[].results.err(exc.toBuffer()) -proc failure*[T](ret: TResult[T], err: ref CatchableError) = - {.cast(uncheckedAssign).}: - ret[].state = ThreadResultKind.Error - ret[].error = err.toBuffer() +proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableError] = + if ret[].results.isOk(): + when S is seq[byte]: + result.ok(ret[].results.get().toSeq(byte)) + elif S is string: + result.ok(ret[].results.get().toString()) + else: + result.ok(ret[].results.get()) + else: + let exc: ref CatchableError = ret[].results.error().toCatchable() + result.err(exc) proc hasTask*( ret: TResult[bool], @@ -100,7 +96,7 @@ proc getTask*( kb: KeyBuffer, ) = without key =? kb.toKey(), err: - ret[].state = Error + ret.failure(err) try: let res = waitFor tds[].ds.get(key) if res.isErr: @@ -130,7 +126,7 @@ proc putTask*( ) = without key =? kb.toKey(), err: - ret[].state = Error + ret.failure(err) let data = db.toSeq(byte) let res = (waitFor tds[].ds.put(key, data)).catch @@ -161,7 +157,7 @@ proc deleteTask*( ) = without key =? kb.toKey(), err: - ret[].state = Error + ret.failure(err) let res = (waitFor tds[].ds.delete(key)).catch # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index a67c9abc..dc44ee43 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -41,7 +41,7 @@ method has*( ret[].signal.close() # echo "\nSharedDataStore:has:value: ", ret[].repr - return success(ret[].value) + return ret.convert(bool) method delete*( self: ThreadProxyDatastore, @@ -77,7 +77,7 @@ method get*( key: Key ): Future[?!seq[byte]] {.async.} = - without ret =? newThreadResult(DataBuffer), err: + without ret =? newThreadResult(ValueBuffer), err: return failure(err) try: @@ -87,8 +87,8 @@ method get*( ret[].signal.close() # print "\nSharedDataStore:put:value: ", ret[] - let data = ret[].value.toSeq(byte) - return success(data) + # let data = ret[].value.toSeq(byte) + return ret.convert(seq[byte]) method put*( self: ThreadProxyDatastore, diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 8eec311a..01964e72 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -16,6 +16,7 @@ import ./querycommontests import pretty + suite "Test Basic ThreadProxyDatastore": var sds: ThreadProxyDatastore From 7a3c64d425da3adcc3b2b4e979c8b38a4fd0d365 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:43:05 -0700 Subject: [PATCH 082/445] switch to result type --- datastore/threadproxyds.nim | 12 ++++-------- tests/datastore/testthreadproxyds.nim | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index dc44ee43..a2395daf 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -37,7 +37,6 @@ method has*( has(ret, self.tds, key) await wait(ret[].signal) finally: - # echo "closing signal" ret[].signal.close() # echo "\nSharedDataStore:has:value: ", ret[].repr @@ -55,7 +54,6 @@ method delete*( delete(ret, self.tds, key) await wait(ret[].signal) finally: - # echo "closing signal" ret[].signal.close() # echo "\nSharedDataStore:put:value: ", ret[].repr @@ -105,7 +103,6 @@ method put*( finally: ret[].signal.close() - # echo "\nSharedDataStore:put:value: ", ret[].repr return success() method put*( @@ -133,17 +130,16 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() -proc newSharedDataStore*( +proc newThreadProxyDatastore*( ds: Datastore, ): ?!ThreadProxyDatastore = + ## create a new var self = ThreadProxyDatastore() - let value = newSharedPtr(ThreadDatastore) - echo "\nnewDataStore: threadId:", getThreadId() - # GC_ref(ds) - value[].ds = ds + # GC_ref(ds) ## TODO: is this needed? try: + value[].ds = ds value[].tp = Taskpool.new(num_threads = 2) except Exception as exc: return err((ref DatastoreError)(msg: exc.msg)) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 01964e72..2b982248 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -27,7 +27,7 @@ suite "Test Basic ThreadProxyDatastore": setupAll: mem = MemoryDatastore.new() - sds = newSharedDataStore(mem).expect("should work") + sds = newThreadProxyDatastore(mem).expect("should work") key1 = Key.init("/a").tryGet data = "value for 1".toBytes() @@ -62,7 +62,7 @@ suite "Test Basic ThreadProxyDatastore": setupAll: memStore = MemoryDatastore.new() - ds = newSharedDataStore(memStore).expect("should work") + ds = newThreadProxyDatastore(memStore).expect("should work") teardownAll: (await memStore.close()).get() From 9d6e6ae2201237d3aaa7997dd532ad71020189b6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:47:22 -0700 Subject: [PATCH 083/445] cleanup --- datastore/threadbackend.nim | 4 ++-- datastore/threadproxyds.nim | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 395d54b4..dd590f9a 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -12,8 +12,6 @@ import ./datastore import ./databuffer import threading/smartptrs -import pretty - export key, query, smartptrs, databuffer push: {.upraises: [].} @@ -58,6 +56,8 @@ proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableEr result.ok(ret[].results.get().toSeq(byte)) elif S is string: result.ok(ret[].results.get().toString()) + elif S is void: + result.ok() else: result.ok(ret[].results.get()) else: diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index a2395daf..158548b2 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -15,8 +15,6 @@ import ./datastore import ./threadbackend import ./fsds -import pretty - export key, query push: {.upraises: [].} @@ -39,7 +37,6 @@ method has*( finally: ret[].signal.close() - # echo "\nSharedDataStore:has:value: ", ret[].repr return ret.convert(bool) method delete*( @@ -56,8 +53,7 @@ method delete*( finally: ret[].signal.close() - # echo "\nSharedDataStore:put:value: ", ret[].repr - return success() + return ret.convert(void) method delete*( self: ThreadProxyDatastore, @@ -74,6 +70,11 @@ method get*( self: ThreadProxyDatastore, key: Key ): Future[?!seq[byte]] {.async.} = + ## implements batch get + ## + ## note: this implementation is rather naive and should + ## probably be switched to use a single ThreadSignal + ## for the entire batch without ret =? newThreadResult(ValueBuffer), err: return failure(err) @@ -84,8 +85,6 @@ method get*( finally: ret[].signal.close() - # print "\nSharedDataStore:put:value: ", ret[] - # let data = ret[].value.toSeq(byte) return ret.convert(seq[byte]) method put*( @@ -103,12 +102,17 @@ method put*( finally: ret[].signal.close() - return success() + return ret.convert(void) method put*( self: ThreadProxyDatastore, batch: seq[BatchEntry] ): Future[?!void] {.async.} = + ## implements batch put + ## + ## note: this implementation is rather naive and should + ## probably be switched to use a single ThreadSignal + ## for the entire batch for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: From e52f7949b95b8db55d64bc7ea0317ba3492cea4f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 14:58:33 -0700 Subject: [PATCH 084/445] add querybuffer type --- datastore/databuffer.nim | 12 ++++++++++++ datastore/query.nim | 13 +++++++++++++ datastore/threadbackend.nim | 29 +++++++++++++++++++++++++++++ datastore/threadproxyds.nim | 1 + 4 files changed, 55 insertions(+) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index fef4c784..69f45790 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -21,6 +21,18 @@ type CatchableErrorBuffer* = object msg: StringBuffer + QSortOrder* {.pure.} = enum + Ascending, + Descending + + QueryBuffer* = object + key*: KeyBuffer # Key to be queried + value*: bool # Flag to indicate if data should be returned + limit*: int # Max items to return - not available in all backends + offset*: int # Offset from which to start querying - not available in all backends + sort*: QSortOrder # Sort order - not available in all backends + + proc `=destroy`*(x: var DataBufferHolder) = ## copy pointer implementation if x.buf != nil: diff --git a/datastore/query.nim b/datastore/query.nim index 1c51a6e2..f3330fce 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -5,6 +5,7 @@ import pkg/questionable/results import ./key import ./types +import ./databuffer type SortOrder* {.pure.} = enum @@ -52,3 +53,15 @@ proc init*( sort: sort, offset: offset, limit: limit) + +proc toBuffer*(q: Query): QueryBuffer = + ## convert Query to thread-safe QueryBuffer + return QueryBuffer( + key: KeyBuffer.new(q.key), + value: q.value, + offset: q.offset, + sort: + case q.sort: + of SortOrder.Assending: QSortOrder.Ascending + of SortOrder.Descending: QSortOrder.Descending + ) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index dd590f9a..75b5000a 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -175,3 +175,32 @@ proc delete*( ) = let bkey = StringBuffer.new(key.id()) tds[].tp.spawn deleteTask(ret, tds, bkey) + +# proc keyIterator(self: ThreadProxyDatastore, +# queryKey: string +# ): iterator: KeyBuffer {.gcsafe.} = +# return iterator(): KeyBuffer {.closure.} = +# var keys = self.store.keys().toSeq() +# keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) +# for key in keys: +# if key.toString().startsWith(queryKey): +# yield key + +method queryTask*( + ret: TResult[void], + tds: ThreadDatastorePtr, + query: Query, +): Future[?!QueryIter] {.async.} = + + without key =? kb.toKey(), err: + ret.failure(err) + + let q = Query.init(key1) + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + + iter.next = next + return success iter diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 158548b2..c1c75842 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -134,6 +134,7 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() + proc newThreadProxyDatastore*( ds: Datastore, ): ?!ThreadProxyDatastore = From a2f77458aea793075bd0ad33898209cbaf877d66 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:15:00 -0700 Subject: [PATCH 085/445] add querybuffer type --- datastore/databuffer.nim | 11 -------- datastore/query.nim | 56 +++++++++++++++++++++++++++++++++++++ datastore/threadbackend.nim | 8 ++++++ 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 69f45790..e3fa207e 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -21,17 +21,6 @@ type CatchableErrorBuffer* = object msg: StringBuffer - QSortOrder* {.pure.} = enum - Ascending, - Descending - - QueryBuffer* = object - key*: KeyBuffer # Key to be queried - value*: bool # Flag to indicate if data should be returned - limit*: int # Max items to return - not available in all backends - offset*: int # Offset from which to start querying - not available in all backends - sort*: QSortOrder # Sort order - not available in all backends - proc `=destroy`*(x: var DataBufferHolder) = ## copy pointer implementation diff --git a/datastore/query.nim b/datastore/query.nim index f3330fce..c86bacb5 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -1,3 +1,4 @@ +import std/options import pkg/upraises import pkg/chronos import pkg/questionable @@ -6,6 +7,7 @@ import pkg/questionable/results import ./key import ./types import ./databuffer +export options type SortOrder* {.pure.} = enum @@ -54,6 +56,29 @@ proc init*( offset: offset, limit: limit) +type + QSortOrder* {.pure.} = enum + Ascending, + Descending + + QueryBuffer* = object + key*: KeyBuffer # Key to be queried + value*: bool # Flag to indicate if data should be returned + limit*: int # Max items to return - not available in all backends + offset*: int # Offset from which to start querying - not available in all backends + sort*: QSortOrder # Sort order - not available in all backends + + QueryResponseBuffer* = object + key*: KeyBuffer + data*: ValueBuffer + + # GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe, closure.} + # IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} + # QueryIter* = ref object + # finished*: bool + # next*: GetNext + # dispose*: IterDispose + proc toBuffer*(q: Query): QueryBuffer = ## convert Query to thread-safe QueryBuffer return QueryBuffer( @@ -65,3 +90,34 @@ proc toBuffer*(q: Query): QueryBuffer = of SortOrder.Assending: QSortOrder.Ascending of SortOrder.Descending: QSortOrder.Descending ) + +proc toBuffer*(q: QueryResponse): QueryResponseBuffer = + ## convert QueryReponses to thread safe type + var kb: KeyBuffer + if q.key.isSome(): + kb = KeyBuffer.new(q.key.get()) + var kv: KeyBuffer + if q.data.len() > 0: + kv = ValueBuffer.new(q.data) + + QueryResponseBuffer(key: kb, data: kv) + +proc toQueryResponse*(qb: QueryResponseBuffer): QueryResponse = + ## convert QueryReponseBuffer to regular QueryResponse + let key = + if qb.key.isNil: none(Key) + else: some qb.key.toKey().expect("key response should work") + let data = + if qb.data.isNil: EmptyBytes + else: qb.data.toSeq(byte) + + (key: key, data: data) + +# proc convert*(ret: TResult[QueryResponseBuffer], +# tp: typedesc[QueryResponse] +# ): Result[QueryResponse, ref CatchableError] = +# if ret[].results.isOk(): +# result.ok(ret[].results.get().toString()) +# else: +# let exc: ref CatchableError = ret[].results.error().toCatchable() +# result.err(exc) \ No newline at end of file diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 75b5000a..c067f1a2 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -204,3 +204,11 @@ method queryTask*( iter.next = next return success iter + +proc query*( + ret: TResult[void], + tds: ThreadDatastorePtr, + query: Query, +) = + let bkey = StringBuffer.new(key.id()) + tds[].tp.spawn deleteTask(ret, tds, bkey) From 3f8c471aedc464a6b31a73cab555396c398795fa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:20:57 -0700 Subject: [PATCH 086/445] add querybuffer type --- datastore/query.nim | 13 +++++++++++++ datastore/threadbackend.nim | 12 ++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index c86bacb5..741aedde 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -91,6 +91,19 @@ proc toBuffer*(q: Query): QueryBuffer = of SortOrder.Descending: QSortOrder.Descending ) +proc toQuery*(qb: QueryBuffer): Query = + ## convert QueryBuffer to regular Query + Query( + key: qb.key.toKey().expect("key expected"), + value: qb.value, + limit: qb.limit, + offset: qb.offset, + sort: + case qb.sort: + of QSortOrder.Ascending: SortOrder.Assending + of QSortOrder.Descending: SortOrder.Descending + ) + proc toBuffer*(q: QueryResponse): QueryResponseBuffer = ## convert QueryReponses to thread safe type var kb: KeyBuffer diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index c067f1a2..905be101 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -187,11 +187,12 @@ proc delete*( # yield key method queryTask*( - ret: TResult[void], + ret: TResult[QueryResponseBuffer], tds: ThreadDatastorePtr, - query: Query, -): Future[?!QueryIter] {.async.} = + qb: QueryBuffer, +) = + let query = qb.toQuery() without key =? kb.toKey(), err: ret.failure(err) @@ -201,7 +202,6 @@ method queryTask*( if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err - iter.next = next return success iter @@ -210,5 +210,5 @@ proc query*( tds: ThreadDatastorePtr, query: Query, ) = - let bkey = StringBuffer.new(key.id()) - tds[].tp.spawn deleteTask(ret, tds, bkey) + let bq = query.toBuffer() + tds[].tp.spawn queryTask(ret, tds, bq) From 120f770d1e0ecc2064b12d92343b6a5802b7a76f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:42:53 -0700 Subject: [PATCH 087/445] implementing query type --- datastore/threadbackend.nim | 46 ++++++++++++++++--------------------- datastore/threadproxyds.nim | 16 +++++++++++++ 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 905be101..f65e8cbe 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -47,7 +47,7 @@ proc success*[T](ret: TResult[T], value: T) = proc success*[T: void](ret: TResult[T]) = ret[].results.ok() -proc failure*[T](ret: TResult[T], exc: ref CatchableError) = +proc failure*[T](ret: TResult[T], exc: ref Exception) = ret[].results.err(exc.toBuffer()) proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableError] = @@ -176,39 +176,33 @@ proc delete*( let bkey = StringBuffer.new(key.id()) tds[].tp.spawn deleteTask(ret, tds, bkey) -# proc keyIterator(self: ThreadProxyDatastore, -# queryKey: string -# ): iterator: KeyBuffer {.gcsafe.} = -# return iterator(): KeyBuffer {.closure.} = -# var keys = self.store.keys().toSeq() -# keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) -# for key in keys: -# if key.toString().startsWith(queryKey): -# yield key - method queryTask*( ret: TResult[QueryResponseBuffer], tds: ThreadDatastorePtr, - qb: QueryBuffer, + qiter: QueryIter, ) = - let query = qb.toQuery() - without key =? kb.toKey(), err: - ret.failure(err) - - let q = Query.init(key1) + try: + without res =? waitFor(qiter.next()), err: + ret.failure(err) - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err + let qrb = res.toBuffer() + ret.success(qrb) - iter.next = next - return success iter + except Exception: + echo "failure" proc query*( - ret: TResult[void], + ret: TResult[QueryResponseBuffer], tds: ThreadDatastorePtr, - query: Query, + q: Query, ) = - let bq = query.toBuffer() - tds[].tp.spawn queryTask(ret, tds, bq) + + try: + without iter =? waitFor(tds[].ds.query(q)), err: + ret.failure(err) + + while not iter.finished: + tds[].tp.spawn queryTask(ret, tds, iter) + except Exception as exc: + ret.failure(exc) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index c1c75842..5896c3f7 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -120,6 +120,22 @@ method put*( return success() +method query*( + self: ThreadProxyDatastore, + query: Query +): Future[?!QueryIter] {.async.} = + + without ret =? newThreadResult(QueryResponseBuffer), err: + return failure(err) + + try: + delete(ret, self.tds, key) + await wait(ret[].signal) + finally: + ret[].signal.close() + + return ret.convert(void) + method close*( self: ThreadProxyDatastore ): Future[?!void] {.async.} = From 14c39478e9624151745526d2009ec4373275b04e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:55:42 -0700 Subject: [PATCH 088/445] implementing query type --- datastore/threadbackend.nim | 29 +++++++++-------------------- datastore/threadproxyds.nim | 19 ++++++++++++++++--- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f65e8cbe..8e10d080 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -17,8 +17,8 @@ export key, query, smartptrs, databuffer push: {.upraises: [].} type - - ThreadResult*[T: DataBuffer | void | bool | ThreadDatastorePtr] = object + ThreadSafeTypes* = DataBuffer | void | bool | ThreadDatastorePtr | QueryResponseBuffer + ThreadResult*[T: ThreadSafeTypes] = object signal*: ThreadSignalPtr results*: Result[T, CatchableErrorBuffer] @@ -30,6 +30,10 @@ type ThreadDatastorePtr* = SharedPtr[ThreadDatastore] + QueryIterStore* = object + it*: QueryIter + QueryIterPtr* = SharedPtr[QueryIterStore] + proc newThreadResult*[T]( tp: typedesc[T] ): Result[TResult[T], ref CatchableError] = @@ -176,14 +180,14 @@ proc delete*( let bkey = StringBuffer.new(key.id()) tds[].tp.spawn deleteTask(ret, tds, bkey) -method queryTask*( +proc queryTask*( ret: TResult[QueryResponseBuffer], tds: ThreadDatastorePtr, - qiter: QueryIter, + qiter: QueryIterPtr, ) = try: - without res =? waitFor(qiter.next()), err: + without res =? waitFor(qiter[].it.next()), err: ret.failure(err) let qrb = res.toBuffer() @@ -191,18 +195,3 @@ method queryTask*( except Exception: echo "failure" - -proc query*( - ret: TResult[QueryResponseBuffer], - tds: ThreadDatastorePtr, - q: Query, -) = - - try: - without iter =? waitFor(tds[].ds.query(q)), err: - ret.failure(err) - - while not iter.finished: - tds[].tp.spawn queryTask(ret, tds, iter) - except Exception as exc: - ret.failure(exc) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 5896c3f7..115bd5ef 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -129,12 +129,24 @@ method query*( return failure(err) try: - delete(ret, self.tds, key) - await wait(ret[].signal) + ## we need to setup the query iter on the main thread + ## to keep it's lifetime associated with this async Future + without it =? await self.tds[].ds.query(query), err: + ret.failure(err) + + var iter = newSharedPtr(QueryIterStore()) + ## note that bypasses SharedPtr isolation - may need `protect` here? + iter[].it = it + + while not iter[].it.finished: + self.tds[].tp.spawn queryTask(ret, self.tds, iter) + await wait(ret[].signal) + + iter[].it = nil # ensure our sharedptr doesn't try and dealloc finally: ret[].signal.close() - return ret.convert(void) + # return ret.convert(void) method close*( self: ThreadProxyDatastore @@ -150,6 +162,7 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() + self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc proc newThreadProxyDatastore*( ds: Datastore, From f1d2a0ed2656a6e6746ada2eccc33c1f006cb0b9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:57:15 -0700 Subject: [PATCH 089/445] implementing query type --- datastore/threadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index 115bd5ef..ad250eb1 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -142,7 +142,7 @@ method query*( self.tds[].tp.spawn queryTask(ret, self.tds, iter) await wait(ret[].signal) - iter[].it = nil # ensure our sharedptr doesn't try and dealloc + # iter[].it = nil # ensure our sharedptr doesn't try and dealloc finally: ret[].signal.close() @@ -162,7 +162,7 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() - self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc + # self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc proc newThreadProxyDatastore*( ds: Datastore, From 428b3e6a2e5d80edcfd4f4b023109629732139b8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 15:59:49 -0700 Subject: [PATCH 090/445] implementing query type --- datastore/threadbackend.nim | 13 +++++++++++-- datastore/threadproxyds.nim | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index 8e10d080..f4d06d83 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -193,5 +193,14 @@ proc queryTask*( let qrb = res.toBuffer() ret.success(qrb) - except Exception: - echo "failure" + except Exception as exc: + ret.failure(exc) + + discard ret[].signal.fireSync() + +proc query*( + ret: TResult[QueryResponseBuffer], + tds: ThreadDatastorePtr, + qiter: QueryIterPtr, +) = + tds[].tp.spawn queryTask(ret, tds, qiter) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index ad250eb1..f6550ee5 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -139,10 +139,10 @@ method query*( iter[].it = it while not iter[].it.finished: - self.tds[].tp.spawn queryTask(ret, self.tds, iter) + query(ret, self.tds, iter) await wait(ret[].signal) - # iter[].it = nil # ensure our sharedptr doesn't try and dealloc + iter[].it = nil # ensure our sharedptr doesn't try and dealloc finally: ret[].signal.close() @@ -162,7 +162,7 @@ method close*( ## this can block... how to handle? maybe just leak? self.tds[].tp.shutdown() - # self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc + self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc proc newThreadProxyDatastore*( ds: Datastore, From 2979ae08fdb4b6b4c09cd04de2cb70c1cb0241e5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 16:40:11 -0700 Subject: [PATCH 091/445] implementing query type --- datastore/query.nim | 25 ++++++--------------- tests/datastore/testthreadproxyds.nim | 32 ++++++++++----------------- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index 741aedde..dc0f2380 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -1,4 +1,5 @@ import std/options +import std/algorithm import pkg/upraises import pkg/chronos import pkg/questionable @@ -7,12 +8,9 @@ import pkg/questionable/results import ./key import ./types import ./databuffer -export options +export options, SortOrder type - SortOrder* {.pure.} = enum - Assending, - Descending Query* = object key*: Key # Key to be queried @@ -45,7 +43,7 @@ proc init*( T: type Query, key: Key, value = true, - sort = SortOrder.Assending, + sort = Ascending, offset = 0, limit = -1): T = @@ -57,16 +55,13 @@ proc init*( limit: limit) type - QSortOrder* {.pure.} = enum - Ascending, - Descending QueryBuffer* = object key*: KeyBuffer # Key to be queried value*: bool # Flag to indicate if data should be returned limit*: int # Max items to return - not available in all backends offset*: int # Offset from which to start querying - not available in all backends - sort*: QSortOrder # Sort order - not available in all backends + sort*: SortOrder # Sort order - not available in all backends QueryResponseBuffer* = object key*: KeyBuffer @@ -85,10 +80,7 @@ proc toBuffer*(q: Query): QueryBuffer = key: KeyBuffer.new(q.key), value: q.value, offset: q.offset, - sort: - case q.sort: - of SortOrder.Assending: QSortOrder.Ascending - of SortOrder.Descending: QSortOrder.Descending + sort: q.sort ) proc toQuery*(qb: QueryBuffer): Query = @@ -98,10 +90,7 @@ proc toQuery*(qb: QueryBuffer): Query = value: qb.value, limit: qb.limit, offset: qb.offset, - sort: - case qb.sort: - of QSortOrder.Ascending: SortOrder.Assending - of QSortOrder.Descending: SortOrder.Descending + sort: qb.sort ) proc toBuffer*(q: QueryResponse): QueryResponseBuffer = @@ -133,4 +122,4 @@ proc toQueryResponse*(qb: QueryResponseBuffer): QueryResponse = # result.ok(ret[].results.get().toString()) # else: # let exc: ref CatchableError = ret[].results.error().toCatchable() -# result.err(exc) \ No newline at end of file +# result.err(exc) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 2b982248..472f62e3 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -21,7 +21,6 @@ suite "Test Basic ThreadProxyDatastore": var sds: ThreadProxyDatastore mem: MemoryDatastore - res: ThreadProxyDatastore key1: Key data: seq[byte] @@ -69,25 +68,18 @@ suite "Test Basic ThreadProxyDatastore": basicStoreTests(ds, key, bytes, otherBytes) -# suite "Test Query": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath +suite "Test Query": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath -# var -# ds: FSDatastore - -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) - -# ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - -# teardown: + var + mem: MemoryDatastore + sds: ThreadProxyDatastore -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) + setup: + mem = MemoryDatastore.new() + sds = newThreadProxyDatastore(mem).expect("should work") -# queryTests(ds, false) + queryTests(sds, false) From 9ea1b585d494891c9d8fc65de68dd25b027003f5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 16:40:33 -0700 Subject: [PATCH 092/445] implementing query type --- datastore/threadproxyds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index f6550ee5..fc9a1622 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -134,7 +134,7 @@ method query*( without it =? await self.tds[].ds.query(query), err: ret.failure(err) - var iter = newSharedPtr(QueryIterStore()) + var iter = newSharedPtr(QueryIterStore) ## note that bypasses SharedPtr isolation - may need `protect` here? iter[].it = it From 76ec7e7f16e2bd84ef17552054a10854c05c5ca1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:00:11 -0700 Subject: [PATCH 093/445] implementing query type --- datastore/databuffer.nim | 1 + datastore/threadbackend.nim | 6 ++++++ datastore/threadproxyds.nim | 7 +++++++ tests/datastore/testthreadproxyds.nim | 6 +----- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index e3fa207e..b6ff1254 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -64,6 +64,7 @@ proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = proc toString*(data: DataBuffer): string = ## convert buffer to string type using copy + if data.isNil: return "" result = newString(data.len()) if data.len() > 0: copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index f4d06d83..d36acf6b 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -172,6 +172,8 @@ proc deleteTask*( discard ret[].signal.fireSync() +import pretty + proc delete*( ret: TResult[void], tds: ThreadDatastorePtr, @@ -191,7 +193,11 @@ proc queryTask*( ret.failure(err) let qrb = res.toBuffer() + print "queryTask: ", " res: ", res + ret.success(qrb) + print "queryTask: ", " qrb:key: ", ret[].results.get().key.toString() + print "queryTask: ", " qrb:data: ", ret[].results.get().data.toString() except Exception as exc: ret.failure(exc) diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index fc9a1622..da482754 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -120,6 +120,8 @@ method put*( return success() +import pretty + method query*( self: ThreadProxyDatastore, query: Query @@ -138,9 +140,14 @@ method query*( ## note that bypasses SharedPtr isolation - may need `protect` here? iter[].it = it + echo "\n\n=== Query Start === " while not iter[].it.finished: + echo "" query(ret, self.tds, iter) await wait(ret[].signal) + print "query:post: ", ret[].results + print "query:post: ", " qrb:key: ", ret[].results.get().key.toString() + print "query:post: ", " qrb:data: ", ret[].results.get().data.toString() iter[].it = nil # ensure our sharedptr doesn't try and dealloc finally: diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 472f62e3..30b91fcb 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -69,11 +69,6 @@ suite "Test Basic ThreadProxyDatastore": basicStoreTests(ds, key, bytes, otherBytes) suite "Test Query": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - var mem: MemoryDatastore sds: ThreadProxyDatastore @@ -83,3 +78,4 @@ suite "Test Query": sds = newThreadProxyDatastore(mem).expect("should work") queryTests(sds, false) + From 2c881d9eef1fc916add94f2627b7ae28ce419871 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:13:13 -0700 Subject: [PATCH 094/445] cleanup --- datastore/databuffer.nim | 1 - tests/datastore/testdatabuffer.nim | 17 ----------------- 2 files changed, 18 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index b6ff1254..431e5e87 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -1,4 +1,3 @@ -# import std/atomics import threading/smartptrs import std/hashes diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 1d48b109..92d8e1f3 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -10,23 +10,6 @@ import pkg/questionable/results include ../../datastore/databuffer -type - AtomicFreed* = ptr int - -proc newFreedValue*(val = 0): ptr int = - result = cast[ptr int](alloc0(sizeof(int))) - result[] = val - -proc getFreedValue*(x: ptr int): int = - atomicLoad(x, addr result, ATOMIC_ACQUIRE) - -proc incrFreedValue*(x: ptr int): int = - atomicAddFetch(x, 1, ATOMIC_ACQUIRE) - -proc decrFreedValue*(x: ptr int): int = - atomicSubFetch(x, 1, ATOMIC_ACQUIRE) - - var shareVal: DataBuffer lock: Lock From 3f3e4b8d783c6c2ffb8da65d714eeab77fbf1377 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:19:32 -0700 Subject: [PATCH 095/445] update nim --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a1b3c2fc..0e2e106d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: cache_nonce: [ 1 ] - nim_version: [ 1.6.6 ] + nim_version: [ 1.6.14 ] platform: - { icon: 🐧, From 64434b70b0cc0625e356b206bed592caaa499ab3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:19:58 -0700 Subject: [PATCH 096/445] stylechecks workaround --- datastore/databuffer.nim | 3 +++ datastore/threadbackend.nim | 3 +++ 2 files changed, 6 insertions(+) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 431e5e87..1d52f959 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -3,6 +3,9 @@ import std/hashes export hashes +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + type DataBufferHolder* = object buf: ptr UncheckedArray[byte] diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index d36acf6b..c9d3817b 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -14,6 +14,9 @@ import threading/smartptrs export key, query, smartptrs, databuffer +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + push: {.upraises: [].} type From da83b04efe13639d7cd3cd136ddc57bdf8c9cc1f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:21:51 -0700 Subject: [PATCH 097/445] remove stylechecks workaround --- datastore/databuffer.nim | 3 --- datastore/threadbackend.nim | 3 --- 2 files changed, 6 deletions(-) diff --git a/datastore/databuffer.nim b/datastore/databuffer.nim index 1d52f959..431e5e87 100644 --- a/datastore/databuffer.nim +++ b/datastore/databuffer.nim @@ -3,9 +3,6 @@ import std/hashes export hashes -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - type DataBufferHolder* = object buf: ptr UncheckedArray[byte] diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index c9d3817b..d36acf6b 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -14,9 +14,6 @@ import threading/smartptrs export key, query, smartptrs, databuffer -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - push: {.upraises: [].} type From a95cd9cd26f64e45698af5dcd649edc430065f1f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 17:22:13 -0700 Subject: [PATCH 098/445] bump required nim version --- datastore.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore.nimble b/datastore.nimble index ddc7bed9..07f1aa74 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -6,7 +6,7 @@ author = "Status Research & Development GmbH" description = "Simple, unified API for multiple data stores" license = "Apache License 2.0 or MIT" -requires "nim >= 1.2.0", +requires "nim >= 1.6.14", "asynctest >= 0.3.1 & < 0.4.0", "chronos", "questionable >= 0.10.3 & < 0.11.0", From b1a5b9cff8aff27dc8e34acb9c0416496bddaf24 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 19:43:28 -0700 Subject: [PATCH 099/445] implementing query type --- datastore/threadbackend.nim | 2 ++ datastore/threadproxyds.nim | 30 +++++++++++++++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index d36acf6b..ebcb25aa 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -62,6 +62,8 @@ proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableEr result.ok(ret[].results.get().toString()) elif S is void: result.ok() + elif S is QueryResponse: + result.ok(ret[].results.get().toQueryResponse()) else: result.ok(ret[].results.get()) else: diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index da482754..abac2e38 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -130,30 +130,38 @@ method query*( without ret =? newThreadResult(QueryResponseBuffer), err: return failure(err) - try: - ## we need to setup the query iter on the main thread - ## to keep it's lifetime associated with this async Future - without it =? await self.tds[].ds.query(query), err: - ret.failure(err) + ## we need to setup the query iter on the main thread + ## to keep it's lifetime associated with this async Future + without it =? await self.tds[].ds.query(query), err: + ret.failure(err) + + var iter = newSharedPtr(QueryIterStore) + ## note that bypasses SharedPtr isolation - may need `protect` here? + iter[].it = it - var iter = newSharedPtr(QueryIterStore) - ## note that bypasses SharedPtr isolation - may need `protect` here? - iter[].it = it + var iterWrapper = QueryIter.new() + proc next(): Future[?!QueryResponse] {.async.} = echo "\n\n=== Query Start === " - while not iter[].it.finished: + if not iter[].it.finished: echo "" query(ret, self.tds, iter) await wait(ret[].signal) print "query:post: ", ret[].results print "query:post: ", " qrb:key: ", ret[].results.get().key.toString() print "query:post: ", " qrb:data: ", ret[].results.get().data.toString() + return ret.convert(QueryResponse) + else: + iterWrapper.finished = true + proc dispose(): Future[?!void] {.async.} = iter[].it = nil # ensure our sharedptr doesn't try and dealloc - finally: ret[].signal.close() + return success() - # return ret.convert(void) + iterWrapper.next = next + iterWrapper.dispose = dispose + return success iterWrapper method close*( self: ThreadProxyDatastore From 52286b8b3946f61abb8ac92e2d361475d0eaf5d8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:15:54 -0700 Subject: [PATCH 100/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 6 ++++-- datastore/threadbackend.nim | 5 ++++- datastore/threadproxyds.nim | 12 ++++++++---- tests/datastore/querycommontests.nim | 12 +++++++++--- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index dc0f2380..bc6435a1 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,9 +29,11 @@ type next*: GetNext dispose*: IterDispose -iterator items*(q: QueryIter): Future[?!QueryResponse] = +proc collectAllQueries*(q: QueryIter) = + var qr: Future[?!QueryResponse] while not q.finished: - yield q.next() + qr = q.next() + yield qr proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim index ebcb25aa..087dc482 100644 --- a/datastore/threadbackend.nim +++ b/datastore/threadbackend.nim @@ -184,6 +184,8 @@ proc delete*( let bkey = StringBuffer.new(key.id()) tds[].tp.spawn deleteTask(ret, tds, bkey) +import os + proc queryTask*( ret: TResult[QueryResponseBuffer], tds: ThreadDatastorePtr, @@ -191,11 +193,12 @@ proc queryTask*( ) = try: + os.sleep(100) without res =? waitFor(qiter[].it.next()), err: ret.failure(err) let qrb = res.toBuffer() - print "queryTask: ", " res: ", res + # print "queryTask: ", " res: ", res ret.success(qrb) print "queryTask: ", " qrb:key: ", ret[].results.get().key.toString() diff --git a/datastore/threadproxyds.nim b/datastore/threadproxyds.nim index abac2e38..4410f103 100644 --- a/datastore/threadproxyds.nim +++ b/datastore/threadproxyds.nim @@ -130,6 +130,8 @@ method query*( without ret =? newThreadResult(QueryResponseBuffer), err: return failure(err) + echo "\n\n=== Query Start === " + ## we need to setup the query iter on the main thread ## to keep it's lifetime associated with this async Future without it =? await self.tds[].ds.query(query), err: @@ -142,17 +144,19 @@ method query*( var iterWrapper = QueryIter.new() proc next(): Future[?!QueryResponse] {.async.} = - echo "\n\n=== Query Start === " + print "query:next:start: " + iterWrapper.finished = iter[].it.finished if not iter[].it.finished: - echo "" query(ret, self.tds, iter) await wait(ret[].signal) + echo "" print "query:post: ", ret[].results + print "query:post:finished: ", iter[].it.finished print "query:post: ", " qrb:key: ", ret[].results.get().key.toString() print "query:post: ", " qrb:data: ", ret[].results.get().data.toString() - return ret.convert(QueryResponse) + result = ret.convert(QueryResponse) else: - iterWrapper.finished = true + result = success (Key.none, EmptyBytes) proc dispose(): Future[?!void] {.async.} = iter[].it = nil # ensure our sharedptr doesn't try and dealloc diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 4f0a15d8..86092fc9 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -9,6 +9,8 @@ import pkg/stew/byteutils import pkg/datastore +import pretty + template queryTests*(ds: Datastore, extended = true) {.dirty.} = var key1: Key @@ -36,9 +38,13 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + + var res: seq[QueryResponse] + while not iter.finished: + let val = await iter.next() + let qr = val.tryGet() + if qr.key.isSome: + res.add qr check: res.len == 3 From ed508b4be96cc595b2a088c81f516778d8dc8475 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:25:47 -0700 Subject: [PATCH 101/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 22 +++++++++++++++++----- tests/datastore/querycommontests.nim | 6 +----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index bc6435a1..1d98e3b4 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,11 +29,23 @@ type next*: GetNext dispose*: IterDispose -proc collectAllQueries*(q: QueryIter) = - var qr: Future[?!QueryResponse] - while not q.finished: - qr = q.next() - yield qr +proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = + ## for large blocks this would be *expensive* + var res: seq[QueryResponse] + without iter =? (await qi), err: + return failure err + + while not iter.finished: + let val = await iter.next() + if val.isOk(): + let qr = val.tryGet() + if qr.key.isSome: + res.add qr + else: + return failure val.error() + + await iter.dispose() + return success res proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 86092fc9..b842ab5c 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -68,10 +68,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 3 @@ -84,7 +81,6 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[2].key.get == key3 res[2].data.len == 0 - (await iter.dispose()).tryGet test "Key should not query parent": let From 9b004cde03e32d230925803092afb1e93892e694 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:32:28 -0700 Subject: [PATCH 102/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 10 +++++--- tests/datastore/querycommontests.nim | 36 +++++++--------------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index 1d98e3b4..e4055568 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,11 +29,9 @@ type next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] - without iter =? (await qi), err: - return failure err while not iter.finished: let val = await iter.next() @@ -47,6 +45,12 @@ proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryRespons await iter.dispose() return success res +proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = + without iter =? (await qi), err: + return failure err + await waitForAllQueryResults(iter) + + proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index b842ab5c..663b75a4 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -91,10 +91,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + iter = tryGet(await ds.query(q)) + res = tryGet(await iter.waitForAllQueryResults()) check: res.len == 2 @@ -119,9 +117,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = iter = (await ds.query(q)).tryGet var - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + res = tryGet(await ds.query(q).waitForAllQueryResults()) res.sort do (a, b: QueryResponse) -> int: cmp(a.key.get.id, b.key.get.id) @@ -153,16 +149,11 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 10 - (await iter.dispose()).tryGet - test "Should not apply offset": let key = Key.init("/a").tryGet @@ -176,16 +167,11 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 10 - (await iter.dispose()).tryGet - test "Should not apply offset and limit": let key = Key.init("/a").tryGet @@ -199,10 +185,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + iter = tryGet(await ds.query(q)) + res = tryGet(await iter.waitForAllQueryResults()) check: res.len == 5 @@ -238,10 +222,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = kvs = kvs.reversed let - iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + iter = tryGet(await ds.query(q)) + res = tryGet(await iter.waitForAllQueryResults()) check: res.len == 100 From 8fccc7738720f6642afc1236745230f949477b9a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:34:30 -0700 Subject: [PATCH 103/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 10 +++------- tests/datastore/querycommontests.nim | 16 +++------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index e4055568..1d98e3b4 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,9 +29,11 @@ type next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] + without iter =? (await qi), err: + return failure err while not iter.finished: let val = await iter.next() @@ -45,12 +47,6 @@ proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.as await iter.dispose() return success res -proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = - without iter =? (await qi), err: - return failure err - await waitForAllQueryResults(iter) - - proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 663b75a4..6153411c 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -9,7 +9,6 @@ import pkg/stew/byteutils import pkg/datastore -import pretty template queryTests*(ds: Datastore, extended = true) {.dirty.} = var @@ -91,8 +90,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - iter = tryGet(await ds.query(q)) - res = tryGet(await iter.waitForAllQueryResults()) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 2 @@ -102,8 +100,6 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[1].key.get == key3 res[1].data == val3 - (await iter.dispose()).tryGet - test "Key should all list all keys at the same level": let queryKey = Key.init("/a").tryGet @@ -185,8 +181,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - iter = tryGet(await ds.query(q)) - res = tryGet(await iter.waitForAllQueryResults()) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 5 @@ -200,8 +195,6 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[i].key.get == key res[i].data == val - (await iter.dispose()).tryGet - test "Should apply sort order - descending": let key = Key.init("/a").tryGet @@ -222,8 +215,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = kvs = kvs.reversed let - iter = tryGet(await ds.query(q)) - res = tryGet(await iter.waitForAllQueryResults()) + res = tryGet(await ds.query(q).waitForAllQueryResults()) check: res.len == 100 @@ -232,5 +224,3 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = check: res[i].key.get == kvs[i].key.get res[i].data == kvs[i].data - - (await iter.dispose()).tryGet From 3a9ee98f0247fe372905194e48375de67457da2e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:37:10 -0700 Subject: [PATCH 104/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index 1d98e3b4..aa26bdf2 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -32,8 +32,10 @@ type proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] - without iter =? (await qi), err: - return failure err + let iterRes = await qi + if iterRes.isErr(): + return failure iterRes.error() + let iter = iterRes.get() while not iter.finished: let val = await iter.next() From 221d193b80d0c4df85f28ad587c8db0aaa663270 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:38:29 -0700 Subject: [PATCH 105/445] query iterator using items is breaks when the DS isn't blocking --- datastore/query.nim | 11 +- tests/datastore/querycommontests.nim | 244 +++++++++++++-------------- 2 files changed, 128 insertions(+), 127 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index aa26bdf2..c286b654 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,13 +29,14 @@ type next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(qi: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(qi: QueryIter): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] - let iterRes = await qi - if iterRes.isErr(): - return failure iterRes.error() - let iter = iterRes.get() + # let iterRes = await qi + # if iterRes.isErr(): + # return failure iterRes.error() + # let iter = iterRes.get() + let iter = qi while not iter.finished: let val = await iter.next() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 6153411c..91277f3f 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -58,169 +58,169 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - test "Key should query all keys without values": - let - q = Query.init(key1, value = false) + # test "Key should query all keys without values": + # let + # q = Query.init(key1, value = false) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 3 - res[0].key.get == key1 - res[0].data.len == 0 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data.len == 0 - res[1].key.get == key2 - res[1].data.len == 0 + # res[1].key.get == key2 + # res[1].data.len == 0 - res[2].key.get == key3 - res[2].data.len == 0 + # res[2].key.get == key3 + # res[2].data.len == 0 - test "Key should not query parent": - let - q = Query.init(key2) + # test "Key should not query parent": + # let + # q = Query.init(key2) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 2 - res[0].key.get == key2 - res[0].data == val2 + # check: + # res.len == 2 + # res[0].key.get == key2 + # res[0].data == val2 - res[1].key.get == key3 - res[1].data == val3 + # res[1].key.get == key3 + # res[1].data == val3 - test "Key should all list all keys at the same level": - let - queryKey = Key.init("/a").tryGet - q = Query.init(queryKey) + # test "Key should all list all keys at the same level": + # let + # queryKey = Key.init("/a").tryGet + # q = Query.init(queryKey) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet + # let + # iter = (await ds.query(q)).tryGet - var - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # var + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - res.sort do (a, b: QueryResponse) -> int: - cmp(a.key.get.id, b.key.get.id) + # res.sort do (a, b: QueryResponse) -> int: + # cmp(a.key.get.id, b.key.get.id) - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data == val1 - res[1].key.get == key2 - res[1].data == val2 + # res[1].key.get == key2 + # res[1].data == val2 - res[2].key.get == key3 - res[2].data == val3 + # res[2].key.get == key3 + # res[2].data == val3 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - if extended: - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, limit = 10) + # if extended: + # test "Should apply limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, limit = 10) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 10 + # check: + # res.len == 10 - test "Should not apply offset": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 90) + # test "Should not apply offset": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 90) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 10 + # check: + # res.len == 10 - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 95, limit = 5) + # test "Should not apply offset and limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 95, limit = 5) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 5 + # check: + # res.len == 5 - for i in 0.. int: - cmp(a.key.get.id, b.key.get.id) + # # lexicographic sort, as it comes from the backend + # kvs.sort do (a, b: QueryResponse) -> int: + # cmp(a.key.get.id, b.key.get.id) - kvs = kvs.reversed - let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + # kvs = kvs.reversed + # let + # res = tryGet(await ds.query(q).waitForAllQueryResults()) - check: - res.len == 100 + # check: + # res.len == 100 - for i, r in res[1..^1]: - check: - res[i].key.get == kvs[i].key.get - res[i].data == kvs[i].data + # for i, r in res[1..^1]: + # check: + # res[i].key.get == kvs[i].key.get + # res[i].data == kvs[i].data From 2d2b663516e96867a65da0305b0bca508af3e383 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:42:53 -0700 Subject: [PATCH 106/445] compiler really doesn't like this --- datastore/query.nim | 10 ++++----- tests/datastore/querycommontests.nim | 33 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index c286b654..ff485101 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,14 +29,12 @@ type next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(qi: QueryIter): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] - # let iterRes = await qi - # if iterRes.isErr(): - # return failure iterRes.error() - # let iter = iterRes.get() - let iter = qi + # if qi.isErr(): + # return failure qi.error() + # let iter = qi.get() while not iter.finished: let val = await iter.next() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 91277f3f..0716e4bc 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -58,27 +58,28 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - # test "Key should query all keys without values": - # let - # q = Query.init(key1, value = false) + test "Key should query all keys without values": + let + q = Query.init(key1, value = false) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + let + all = waitForAllQueryResults(tryGet(await ds.query(q))) + res = tryGet(await all) - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data.len == 0 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 - # res[1].key.get == key2 - # res[1].data.len == 0 + res[1].key.get == key2 + res[1].data.len == 0 - # res[2].key.get == key3 - # res[2].data.len == 0 + res[2].key.get == key3 + res[2].data.len == 0 # test "Key should not query parent": From 1f0012530d800159464634a2f459dcf674441904 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 29 Aug 2023 20:49:43 -0700 Subject: [PATCH 107/445] fix compiler issue -- wasn't detecting discard on result correctly --- datastore/query.nim | 15 +- tests/datastore/querycommontests.nim | 214 +++++++++++++-------------- 2 files changed, 117 insertions(+), 112 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index ff485101..295ac836 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -29,12 +29,11 @@ type next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(qi: ?!QueryIter): Future[?!seq[QueryResponse]] {.async.} = ## for large blocks this would be *expensive* var res: seq[QueryResponse] - # if qi.isErr(): - # return failure qi.error() - # let iter = qi.get() + without iter =? qi, err: + return failure err while not iter.finished: let val = await iter.next() @@ -45,9 +44,15 @@ proc waitForAllQueryResults*(iter: QueryIter): Future[?!seq[QueryResponse]] {.as else: return failure val.error() - await iter.dispose() + let rd = await iter.dispose() + if rd.isErr(): + return failure(rd.error()) return success res +proc waitForAllQueryResults*(iter: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = + let res = await iter + await waitForAllQueryResults(res) + proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 0716e4bc..fe111428 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -67,7 +67,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - all = waitForAllQueryResults(tryGet(await ds.query(q))) + all = waitForAllQueryResults(await ds.query(q)) res = tryGet(await all) check: @@ -82,146 +82,146 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[2].data.len == 0 - # test "Key should not query parent": - # let - # q = Query.init(key2) + test "Key should not query parent": + let + q = Query.init(key2) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + let + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # check: - # res.len == 2 - # res[0].key.get == key2 - # res[0].data == val2 + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 - # res[1].key.get == key3 - # res[1].data == val3 + res[1].key.get == key3 + res[1].data == val3 - # test "Key should all list all keys at the same level": - # let - # queryKey = Key.init("/a").tryGet - # q = Query.init(queryKey) + test "Key should all list all keys at the same level": + let + queryKey = Key.init("/a").tryGet + q = Query.init(queryKey) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet + let + iter = (await ds.query(q)).tryGet - # var - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + var + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # res.sort do (a, b: QueryResponse) -> int: - # cmp(a.key.get.id, b.key.get.id) + res.sort do (a, b: QueryResponse) -> int: + cmp(a.key.get.id, b.key.get.id) - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data == val1 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 - # res[1].key.get == key2 - # res[1].data == val2 + res[1].key.get == key2 + res[1].data == val2 - # res[2].key.get == key3 - # res[2].data == val3 + res[2].key.get == key3 + res[2].data == val3 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # if extended: - # test "Should apply limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, limit = 10) + if extended: + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, limit = 10) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + let + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # check: - # res.len == 10 + check: + res.len == 10 - # test "Should not apply offset": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 90) + test "Should not apply offset": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 90) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + let + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # check: - # res.len == 10 + check: + res.len == 10 - # test "Should not apply offset and limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 95, limit = 5) + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 95, limit = 5) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + let + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # check: - # res.len == 5 + check: + res.len == 5 - # for i in 0.. int: - # cmp(a.key.get.id, b.key.get.id) + # lexicographic sort, as it comes from the backend + kvs.sort do (a, b: QueryResponse) -> int: + cmp(a.key.get.id, b.key.get.id) - # kvs = kvs.reversed - # let - # res = tryGet(await ds.query(q).waitForAllQueryResults()) + kvs = kvs.reversed + let + res = tryGet(await ds.query(q).waitForAllQueryResults()) - # check: - # res.len == 100 + check: + res.len == 100 - # for i, r in res[1..^1]: - # check: - # res[i].key.get == kvs[i].key.get - # res[i].data == kvs[i].data + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data From 0caa606f3ec28cca5e3c3c0d736953af98826d1e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:21:59 -0700 Subject: [PATCH 108/445] fix test --- tests/datastore/testdatastore.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/datastore/testdatastore.nim b/tests/datastore/testdatastore.nim index 4361eefa..cd529156 100644 --- a/tests/datastore/testdatastore.nim +++ b/tests/datastore/testdatastore.nim @@ -25,5 +25,6 @@ suite "Datastore (base)": test "query": expect Defect: - let iter = (await ds.query(Query.init(key))).tryGet - for n in iter: discard + let iter = tryGet(await waitForAllQueryResults ds.query(Query.init(key))) + for n in iter: + discard From 619d3e9d56e355bf1fb2af46520724f86fa32904 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:24:12 -0700 Subject: [PATCH 109/445] add tests for new ds'es --- tests/exampletaskpool.nim | 34 ---------------------------------- tests/testall.nim | 4 +++- 2 files changed, 3 insertions(+), 35 deletions(-) delete mode 100644 tests/exampletaskpool.nim diff --git a/tests/exampletaskpool.nim b/tests/exampletaskpool.nim deleted file mode 100644 index c4102351..00000000 --- a/tests/exampletaskpool.nim +++ /dev/null @@ -1,34 +0,0 @@ - - -import - std/[strutils, math, cpuinfo], - taskpools - -# From https://github.com/nim-lang/Nim/blob/v1.6.2/tests/parallel/tpi.nim -# Leibniz Formula https://en.wikipedia.org/wiki/Leibniz_formula_for_%CF%80 -proc term(k: int): float = - if k mod 2 == 1: - -4'f / float(2*k + 1) - else: - 4'f / float(2*k + 1) - -proc piApprox(tp: Taskpool, n: int): float = - var pendingFuts = newSeq[FlowVar[float]](n) - for k in 0 ..< pendingFuts.len: - pendingFuts[k] = tp.spawn term(k) # Schedule a task on the threadpool a return a handle to retrieve the result. - for k in 0 ..< pendingFuts.len: - result += sync pendingFuts[k] # Block until the result is available. - -proc main() = - var n = 1_000_000 - var nthreads = countProcessors() - - var tp = Taskpool.new(num_threads = nthreads) # Default to the number of hardware threads. - - echo formatFloat(tp.piApprox(n)) - - tp.syncAll() # Block until all pending tasks are processed (implied in tp.shutdown()) - tp.shutdown() - -# Compile with nim c -r -d:release --threads:on --outdir:build example.nim -main() \ No newline at end of file diff --git a/tests/testall.nim b/tests/testall.nim index a6aca018..b4911446 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -4,6 +4,8 @@ import ./datastore/testfsds, ./datastore/testsql, ./datastore/testtieredds, - ./datastore/testmountedds + ./datastore/testmountedds, + ./datastore/testmemoryds, + ./datastore/testthreadproxyds {.warning[UnusedImport]: off.} From 24ce85e49bac4540f5d2d9f5445a69d7cf32e136 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:46:30 -0700 Subject: [PATCH 110/445] add tests for new ds'es --- datastore/query.nim | 2 +- tests/datastore/testdatastore.nim | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index 295ac836..72e260b5 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -51,7 +51,7 @@ proc waitForAllQueryResults*(qi: ?!QueryIter): Future[?!seq[QueryResponse]] {.as proc waitForAllQueryResults*(iter: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = let res = await iter - await waitForAllQueryResults(res) + return await waitForAllQueryResults(res) proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() diff --git a/tests/datastore/testdatastore.nim b/tests/datastore/testdatastore.nim index cd529156..149badcf 100644 --- a/tests/datastore/testdatastore.nim +++ b/tests/datastore/testdatastore.nim @@ -25,6 +25,9 @@ suite "Datastore (base)": test "query": expect Defect: - let iter = tryGet(await waitForAllQueryResults ds.query(Query.init(key))) - for n in iter: + let + q = Query.init(key) + all = waitForAllQueryResults(await ds.query(q)) + res = tryGet(await all) + for n in res: discard From f2bfe7affcaeef88e4d90094555d439c252c7323 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:47:40 -0700 Subject: [PATCH 111/445] add tests for new ds'es --- datastore/query.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/query.nim b/datastore/query.nim index 72e260b5..92aafe9a 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -49,7 +49,8 @@ proc waitForAllQueryResults*(qi: ?!QueryIter): Future[?!seq[QueryResponse]] {.as return failure(rd.error()) return success res -proc waitForAllQueryResults*(iter: Future[?!QueryIter]): Future[?!seq[QueryResponse]] {.async.} = +proc waitForAllQueryResults*(iter: Future[?!QueryIter] + ): Future[?!seq[QueryResponse]] {.async.} = let res = await iter return await waitForAllQueryResults(res) From 2a04a36ed37a5d32f80ece17b64fc40b5d39e806 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:49:23 -0700 Subject: [PATCH 112/445] add tests for new ds'es --- datastore.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore.nimble b/datastore.nimble index 07f1aa74..6f3fe082 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -8,7 +8,7 @@ license = "Apache License 2.0 or MIT" requires "nim >= 1.6.14", "asynctest >= 0.3.1 & < 0.4.0", - "chronos", + "chronos >= 3.2.0", "questionable >= 0.10.3 & < 0.11.0", "sqlite3_abi", "stew", From 1760ae24d681186d7826cdafe22151efb92fc356 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 18:55:52 -0700 Subject: [PATCH 113/445] set manual version of chronos --- datastore.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore.nimble b/datastore.nimble index 6f3fe082..fade4122 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -8,7 +8,7 @@ license = "Apache License 2.0 or MIT" requires "nim >= 1.6.14", "asynctest >= 0.3.1 & < 0.4.0", - "chronos >= 3.2.0", + "chronos#0277b65be2c7a365ac13df002fba6e172be55537", "questionable >= 0.10.3 & < 0.11.0", "sqlite3_abi", "stew", From e84ccb21bac1cf4826b9bc7ed64cfb98a79caf5c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 30 Aug 2023 19:00:17 -0700 Subject: [PATCH 114/445] add taskpools --- datastore.nimble | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore.nimble b/datastore.nimble index fade4122..df032c4e 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -15,6 +15,7 @@ requires "nim >= 1.6.14", "unittest2", "pretty", "threading", + "taskpools", "upraises >= 0.1.0 & < 0.2.0" task coverage, "generates code coverage report": From 6c3b0d60fb46213ff5591bce2f6f7d440e21fd79 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 7 Sep 2023 17:33:33 -0600 Subject: [PATCH 115/445] fix unitest2 import --- tests/datastore/testmemoryds.nim | 2 +- tests/datastore/testthreadproxyds.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/testmemoryds.nim b/tests/datastore/testmemoryds.nim index fedb497a..2100f69f 100644 --- a/tests/datastore/testmemoryds.nim +++ b/tests/datastore/testmemoryds.nim @@ -3,7 +3,7 @@ import std/sequtils import std/os from std/algorithm import sort, reversed -import pkg/asynctest/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 30b91fcb..06415701 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -3,7 +3,7 @@ import std/sequtils import std/os import std/algorithm -import pkg/asynctest/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils From 146cbcb88a168f1338fc7f9ca6061a4f2df6d384 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 8 Sep 2023 15:09:15 -0600 Subject: [PATCH 116/445] wip --- datastore/datastore.nim | 8 +- datastore/memoryds.nim | 100 ++++---- datastore/query.nim | 104 ++++---- datastore/threadbackend.nim | 217 ---------------- datastore/{ => threads}/databuffer.nim | 19 +- datastore/threads/foreignbuffer.nim | 68 +++++ datastore/threads/threadproxyds.nim | 237 ++++++++++++++++++ .../threadproxyds_.nim} | 0 datastore/threads/threadresult.nim | 60 +++++ tests/datastore/testthreadproxyds.nim | 97 +++---- 10 files changed, 530 insertions(+), 380 deletions(-) delete mode 100644 datastore/threadbackend.nim rename datastore/{ => threads}/databuffer.nim (96%) create mode 100644 datastore/threads/foreignbuffer.nim create mode 100644 datastore/threads/threadproxyds.nim rename datastore/{threadproxyds.nim => threads/threadproxyds_.nim} (100%) create mode 100644 datastore/threads/threadresult.nim diff --git a/datastore/datastore.nim b/datastore/datastore.nim index d7fa45fd..3e908142 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -34,11 +34,11 @@ method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, loc method close*(self: Datastore): Future[?!void] {.base, async, locks: "unknown".} = raiseAssert("Not implemented!") -method query*( - self: Datastore, - query: Query): Future[?!QueryIter] {.base, gcsafe.} = +# method query*( +# self: Datastore, +# query: Query): Future[?!QueryIter] {.base, gcsafe.} = - raiseAssert("Not implemented!") +# raiseAssert("Not implemented!") proc contains*(self: Datastore, key: Key): Future[bool] {.async.} = return (await self.has(key)) |? false diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index dd3192fd..fd02fe96 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -11,33 +11,27 @@ import pkg/upraises import ./key import ./query import ./datastore -import ./databuffer export key, query push: {.upraises: [].} type - MemoryDatastore* = ref object of Datastore - store*: Table[KeyBuffer, ValueBuffer] + store*: Table[Key, seq[byte]] method has*( self: MemoryDatastore, key: Key ): Future[?!bool] {.async.} = - let dk = KeyBuffer.new(key) - return success self.store.hasKey(dk) + return success self.store.hasKey(key) method delete*( - self: MemoryDatastore, - key: Key -): Future[?!void] {.async.} = + self: MemoryDatastore, + key: Key): Future[?!void] {.async.} = - let dk = KeyBuffer.new(key) - var val: ValueBuffer - discard self.store.pop(dk, val) + self.store.del(key) return success() method delete*( @@ -51,26 +45,20 @@ method delete*( return success() method get*( - self: MemoryDatastore, - key: Key -): Future[?!seq[byte]] {.async.} = + self: MemoryDatastore, + key: Key): Future[?!seq[byte]] {.async.} = - let dk = KeyBuffer.new(key) - if self.store.hasKey(dk): - let res = self.store[dk].toSeq(byte) - return success res - else: + self.store.withValue(key, value): + return success value[] + do: return failure (ref DatastoreError)(msg: "no such key") method put*( - self: MemoryDatastore, - key: Key, - data: seq[byte] -): Future[?!void] {.async.} = + self: MemoryDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = - let dk = KeyBuffer.new(key) - let dv = ValueBuffer.new(data) - self.store[dk] = dv + self.store[key] = data return success() method put*( @@ -83,47 +71,47 @@ method put*( return success() -proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer {.gcsafe.} = - return iterator(): KeyBuffer {.closure.} = - var keys = self.store.keys().toSeq() - keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) - for key in keys: - if key.toString().startsWith(queryKey): - yield key +# proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer {.gcsafe.} = +# return iterator(): KeyBuffer {.closure.} = +# var keys = self.store.keys().toSeq() +# keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) +# for key in keys: +# if key.toString().startsWith(queryKey): +# yield key -method query*( - self: MemoryDatastore, - query: Query, -): Future[?!QueryIter] {.async.} = +# method query*( +# self: MemoryDatastore, +# query: Query, +# ): Future[?!QueryIter] {.async.} = - let - queryKey = query.key.id() - walker = keyIterator(self, queryKey) - var - iter = QueryIter.new() +# let +# queryKey = query.key.id() +# walker = keyIterator(self, queryKey) +# var +# iter = QueryIter.new() - proc next(): Future[?!QueryResponse] {.async.} = - let kb = walker() +# proc next(): Future[?!QueryResponse] {.async.} = +# let kb = walker() - if finished(walker): - iter.finished = true - return success (Key.none, EmptyBytes) +# if finished(walker): +# iter.finished = true +# return success (Key.none, EmptyBytes) - let key = kb.toKey().expect("should not fail") - var ds: ValueBuffer - if query.value: - ds = self.store[kb] - let data = if ds.isNil: EmptyBytes else: ds.toSeq(byte) +# let key = kb.toKey().expect("should not fail") +# var ds: ValueBuffer +# if query.value: +# ds = self.store[kb] +# let data = if ds.isNil: EmptyBytes else: ds.toSeq(byte) - return success (key.some, data) +# return success (key.some, data) - iter.next = next - return success iter +# iter.next = next +# return success iter method close*(self: MemoryDatastore): Future[?!void] {.async.} = self.store.clear() return success() -func new*(tp: typedesc[MemoryDatastore]): MemoryDatastore = +func new*(tp: type MemoryDatastore): MemoryDatastore = var self = tp() return self diff --git a/datastore/query.nim b/datastore/query.nim index 92aafe9a..5f9c9131 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -7,7 +7,7 @@ import pkg/questionable/results import ./key import ./types -import ./databuffer +import ./threads/databuffer export options, SortOrder type @@ -43,7 +43,7 @@ proc waitForAllQueryResults*(qi: ?!QueryIter): Future[?!seq[QueryResponse]] {.as res.add qr else: return failure val.error() - + let rd = await iter.dispose() if rd.isErr(): return failure(rd.error()) @@ -75,18 +75,18 @@ proc init*( offset: offset, limit: limit) -type +# type - QueryBuffer* = object - key*: KeyBuffer # Key to be queried - value*: bool # Flag to indicate if data should be returned - limit*: int # Max items to return - not available in all backends - offset*: int # Offset from which to start querying - not available in all backends - sort*: SortOrder # Sort order - not available in all backends +# QueryBuffer* = object +# key*: KeyBuffer # Key to be queried +# value*: bool # Flag to indicate if data should be returned +# limit*: int # Max items to return - not available in all backends +# offset*: int # Offset from which to start querying - not available in all backends +# sort*: SortOrder # Sort order - not available in all backends - QueryResponseBuffer* = object - key*: KeyBuffer - data*: ValueBuffer +# QueryResponseBuffer* = object +# key*: KeyBuffer +# data*: ValueBuffer # GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe, closure.} # IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} @@ -95,46 +95,46 @@ type # next*: GetNext # dispose*: IterDispose -proc toBuffer*(q: Query): QueryBuffer = - ## convert Query to thread-safe QueryBuffer - return QueryBuffer( - key: KeyBuffer.new(q.key), - value: q.value, - offset: q.offset, - sort: q.sort - ) - -proc toQuery*(qb: QueryBuffer): Query = - ## convert QueryBuffer to regular Query - Query( - key: qb.key.toKey().expect("key expected"), - value: qb.value, - limit: qb.limit, - offset: qb.offset, - sort: qb.sort - ) - -proc toBuffer*(q: QueryResponse): QueryResponseBuffer = - ## convert QueryReponses to thread safe type - var kb: KeyBuffer - if q.key.isSome(): - kb = KeyBuffer.new(q.key.get()) - var kv: KeyBuffer - if q.data.len() > 0: - kv = ValueBuffer.new(q.data) - - QueryResponseBuffer(key: kb, data: kv) - -proc toQueryResponse*(qb: QueryResponseBuffer): QueryResponse = - ## convert QueryReponseBuffer to regular QueryResponse - let key = - if qb.key.isNil: none(Key) - else: some qb.key.toKey().expect("key response should work") - let data = - if qb.data.isNil: EmptyBytes - else: qb.data.toSeq(byte) - - (key: key, data: data) +# proc toBuffer*(q: Query): QueryBuffer = +# ## convert Query to thread-safe QueryBuffer +# return QueryBuffer( +# key: KeyBuffer.new(q.key), +# value: q.value, +# offset: q.offset, +# sort: q.sort +# ) + +# proc toQuery*(qb: QueryBuffer): Query = +# ## convert QueryBuffer to regular Query +# Query( +# key: qb.key.toKey().expect("key expected"), +# value: qb.value, +# limit: qb.limit, +# offset: qb.offset, +# sort: qb.sort +# ) + +# proc toBuffer*(q: QueryResponse): QueryResponseBuffer = +# ## convert QueryReponses to thread safe type +# var kb: KeyBuffer +# if q.key.isSome(): +# kb = KeyBuffer.new(q.key.get()) +# var kv: KeyBuffer +# if q.data.len() > 0: +# kv = ValueBuffer.new(q.data) + +# QueryResponseBuffer(key: kb, data: kv) + +# proc toQueryResponse*(qb: QueryResponseBuffer): QueryResponse = +# ## convert QueryReponseBuffer to regular QueryResponse +# let key = +# if qb.key.isNil: none(Key) +# else: some qb.key.toKey().expect("key response should work") +# let data = +# if qb.data.isNil: EmptyBytes +# else: qb.data.toSeq(byte) + +# (key: key, data: data) # proc convert*(ret: TResult[QueryResponseBuffer], # tp: typedesc[QueryResponse] diff --git a/datastore/threadbackend.nim b/datastore/threadbackend.nim deleted file mode 100644 index 087dc482..00000000 --- a/datastore/threadbackend.nim +++ /dev/null @@ -1,217 +0,0 @@ -import pkg/chronos -import pkg/chronos/threadsync -import pkg/questionable -import pkg/questionable/results -import stew/results -import pkg/upraises -import pkg/taskpools - -import ./key -import ./query -import ./datastore -import ./databuffer -import threading/smartptrs - -export key, query, smartptrs, databuffer - -push: {.upraises: [].} - -type - ThreadSafeTypes* = DataBuffer | void | bool | ThreadDatastorePtr | QueryResponseBuffer - ThreadResult*[T: ThreadSafeTypes] = object - signal*: ThreadSignalPtr - results*: Result[T, CatchableErrorBuffer] - - TResult*[T] = SharedPtr[ThreadResult[T]] - - ThreadDatastore* = object - tp*: Taskpool - ds*: Datastore - - ThreadDatastorePtr* = SharedPtr[ThreadDatastore] - - QueryIterStore* = object - it*: QueryIter - QueryIterPtr* = SharedPtr[QueryIterStore] - -proc newThreadResult*[T]( - tp: typedesc[T] -): Result[TResult[T], ref CatchableError] = - let res = newSharedPtr(ThreadResult[T]) - let signal = ThreadSignalPtr.new() - if signal.isErr: - return err((ref CatchableError)(msg: signal.error())) - else: - res[].signal = signal.get() - ok res - -proc success*[T](ret: TResult[T], value: T) = - ret[].results.ok(value) - -proc success*[T: void](ret: TResult[T]) = - ret[].results.ok() - -proc failure*[T](ret: TResult[T], exc: ref Exception) = - ret[].results.err(exc.toBuffer()) - -proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableError] = - if ret[].results.isOk(): - when S is seq[byte]: - result.ok(ret[].results.get().toSeq(byte)) - elif S is string: - result.ok(ret[].results.get().toString()) - elif S is void: - result.ok() - elif S is QueryResponse: - result.ok(ret[].results.get().toQueryResponse()) - else: - result.ok(ret[].results.get()) - else: - let exc: ref CatchableError = ret[].results.error().toCatchable() - result.err(exc) - -proc hasTask*( - ret: TResult[bool], - tds: ThreadDatastorePtr, - kb: KeyBuffer, -) = - without key =? kb.toKey(), err: - ret.failure(err) - - try: - let res = waitFor tds[].ds.has(key) - if res.isErr: - ret.failure(res.error()) - else: - ret.success(res.get()) - discard ret[].signal.fireSync() - except CatchableError as err: - ret.failure(err) - -proc has*( - ret: TResult[bool], - tds: ThreadDatastorePtr, - key: Key, -) = - let bkey = StringBuffer.new(key.id()) - tds[].tp.spawn hasTask(ret, tds, bkey) - -proc getTask*( - ret: TResult[DataBuffer], - tds: ThreadDatastorePtr, - kb: KeyBuffer, -) = - without key =? kb.toKey(), err: - ret.failure(err) - try: - let res = waitFor tds[].ds.get(key) - if res.isErr: - ret.failure(res.error()) - else: - let db = DataBuffer.new res.get() - ret.success(db) - - discard ret[].signal.fireSync() - except CatchableError as err: - ret.failure(err) - -proc get*( - ret: TResult[DataBuffer], - tds: ThreadDatastorePtr, - key: Key, -) = - let bkey = StringBuffer.new(key.id()) - tds[].tp.spawn getTask(ret, tds, bkey) - - -proc putTask*( - ret: TResult[void], - tds: ThreadDatastorePtr, - kb: KeyBuffer, - db: DataBuffer, -) = - - without key =? kb.toKey(), err: - ret.failure(err) - - let data = db.toSeq(byte) - let res = (waitFor tds[].ds.put(key, data)).catch - # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() - if res.isErr: - ret.failure(res.error()) - else: - ret.success() - - discard ret[].signal.fireSync() - -proc put*( - ret: TResult[void], - tds: ThreadDatastorePtr, - key: Key, - data: seq[byte] -) = - let bkey = StringBuffer.new(key.id()) - let bval = DataBuffer.new(data) - - tds[].tp.spawn putTask(ret, tds, bkey, bval) - - -proc deleteTask*( - ret: TResult[void], - tds: ThreadDatastorePtr, - kb: KeyBuffer, -) = - - without key =? kb.toKey(), err: - ret.failure(err) - - let res = (waitFor tds[].ds.delete(key)).catch - # print "thrbackend: putTask: fire", ret[].signal.fireSync().get() - if res.isErr: - ret.failure(res.error()) - else: - ret.success() - - discard ret[].signal.fireSync() - -import pretty - -proc delete*( - ret: TResult[void], - tds: ThreadDatastorePtr, - key: Key, -) = - let bkey = StringBuffer.new(key.id()) - tds[].tp.spawn deleteTask(ret, tds, bkey) - -import os - -proc queryTask*( - ret: TResult[QueryResponseBuffer], - tds: ThreadDatastorePtr, - qiter: QueryIterPtr, -) = - - try: - os.sleep(100) - without res =? waitFor(qiter[].it.next()), err: - ret.failure(err) - - let qrb = res.toBuffer() - # print "queryTask: ", " res: ", res - - ret.success(qrb) - print "queryTask: ", " qrb:key: ", ret[].results.get().key.toString() - print "queryTask: ", " qrb:data: ", ret[].results.get().data.toString() - - except Exception as exc: - ret.failure(exc) - - discard ret[].signal.fireSync() - -proc query*( - ret: TResult[QueryResponseBuffer], - tds: ThreadDatastorePtr, - qiter: QueryIterPtr, -) = - tds[].tp.spawn queryTask(ret, tds, qiter) diff --git a/datastore/databuffer.nim b/datastore/threads/databuffer.nim similarity index 96% rename from datastore/databuffer.nim rename to datastore/threads/databuffer.nim index 431e5e87..67193e0d 100644 --- a/datastore/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -1,5 +1,12 @@ + import threading/smartptrs import std/hashes +import pkg/stew/results +import pkg/upraises + +push: {.upraises: [].} + +import ../key export hashes @@ -7,20 +14,20 @@ type DataBufferHolder* = object buf: ptr UncheckedArray[byte] size: int - + DataBuffer* = SharedPtr[DataBufferHolder] ##\ ## A fixed length data buffer using a SharedPtr. ## It is thread safe even with `refc` since ## it doesn't use string or seq types internally. - ## + ## KeyBuffer* = DataBuffer ValueBuffer* = DataBuffer StringBuffer* = DataBuffer + CatchableErrorBuffer* = object msg: StringBuffer - proc `=destroy`*(x: var DataBufferHolder) = ## copy pointer implementation if x.buf != nil: @@ -51,7 +58,7 @@ proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuffer = ## allocate new buffer and copies indata from openArray - ## + ## result = DataBuffer.new(data.len) if data.len() > 0: copyMem(result[].buf, unsafeAddr data[0], data.len) @@ -78,10 +85,8 @@ proc toBuffer*(err: ref Exception): CatchableErrorBuffer = msg: StringBuffer.new(err.msg) ) -import ./key -import stew/results - proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = KeyBuffer.new(key.id()) + proc toKey*(kb: KeyBuffer): Result[Key, ref CatchableError] = Key.init(kb.toString()) diff --git a/datastore/threads/foreignbuffer.nim b/datastore/threads/foreignbuffer.nim new file mode 100644 index 00000000..b2002ea8 --- /dev/null +++ b/datastore/threads/foreignbuffer.nim @@ -0,0 +1,68 @@ + +type + ## Copy foreign buffers between threads. + ## + ## This is meant to be used as a temporary holder a + ## pointer to a foreign buffer that is being passed + ## between threads. + ## + ## The receiving thread should copy the contained buffer + ## to it's local GC as soon as possible. Should only be + ## used with refgc. + ## + ForeignBuff*[T] = object + buf: ptr UncheckedArray[T] + len: int + cell: ForeignCell + +proc `=sink`[T](a: var ForeignBuff[T], b: ForeignBuff[T]) = + `=destroy`(a) + wasMoved(a) + a.len = b.len + a.buf = b.buf + a.cell = b.cell + +proc `=copy`[T](a: var ForeignBuff[T], b: ForeignBuff[T]) + {.error: "You can't copy the buffer, only it's contents!".} + +proc `=destroy`[T](self: var ForeignBuff[T]) = + if self.cell.data != nil: + echo "DESTROYING CELL" + dispose self.cell + +proc len*[T](self: ForeignBuff[T]): int = + return self.len + +template `[]`*[T](self: ForeignBuff[T], idx: int): T = + assert idx >= 0 and idx < self.len + return self.buf[idx] + +template `[]=`*[T](self: ForeignBuff[T], idx: int, val: T) = + assert idx >= 0 and idx < self.len + return self.buf[idx] + +proc get*[T](self: ForeignBuff[T]): ptr UncheckedArray[T] = + self.buf + +iterator items*[T](self: ForeignBuff[T]): T = + for i in 0 ..< self.len: + yield self.buf[i] + +iterator miterms*[T](self: ForeignBuff[T]): var T = + for i in 0 ..< self.len: + yield self.buf[i] + +proc attach*[T]( + self: var ForeignBuff[T], + buf: ptr UncheckedArray[T], + len: int, + cell: ForeignCell) = + ## Attach a foreign pointer to this buffer + ## + + self.buf = buf + self.len = len + self.cell = cell + +func init*[T](_: type ForeignBuff[T]): ForeignBuff[T] = + return ForeignBuff[T]() diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim new file mode 100644 index 00000000..71a1bbeb --- /dev/null +++ b/datastore/threads/threadproxyds.nim @@ -0,0 +1,237 @@ + +import pkg/upraises + +push: {.upraises: [].} + +import std/atomics + +import pkg/chronos +import pkg/chronos/threadsync +import pkg/questionable +import pkg/questionable/results +import pkg/stew/ptrops +import pkg/taskpools + +import ../key +import ../query +import ../datastore + +import ./foreignbuffer + +type + TaskRes = object + ok: Atomic[bool] + msg: ptr cstring + + TaskCtx = object + ds: ptr Datastore + res: TaskRes + signal: ThreadSignalPtr + + ThreadDatastore* = ref object of Datastore + tp*: Taskpool + ds*: Datastore + +proc hasTask( + ctx: ptr TaskCtx, + key: ptr Key, + doesHave: ptr bool) = + + let + res = (waitFor ctx[].ds[].has(key[])).catch + + if res.isErr: + var + err = cstring(res.error().msg) + ctx[].res.msg = addr err + else: + ctx[].res.msg = nil + doesHave[] = res.get().get() + + ctx[].res.ok.store(res.isOk) + discard ctx[].signal.fireSync() + +proc has*( + self: ThreadDatastore, + key: Key): Future[?!bool] {.async.} = + + var + signal = ThreadSignalPtr.new().valueOr: + return failure("Failed to create signal") + + key = key + ctx = TaskCtx( + ds: addr self.ds, + res: TaskRes(msg: nil), + signal: signal) + doesHave = false + + proc runTask() = + self.tp.spawn hasTask(addr ctx, addr key, addr doesHave) + + try: + runTask() + await wait(ctx.signal) + + var data: bool + if ctx.res.ok.load() == false: + return failure("error") + + return success(doesHave) + finally: + ctx.signal.close() + +proc delTask(ctx: ptr TaskCtx, key: ptr Key) = + + let + res = (waitFor ctx[].ds[].delete(key[])).catch + + if res.isErr: + var + err = cstring(res.error().msg) + ctx[].res.msg = addr err + else: + ctx[].res.msg = nil + + ctx[].res.ok.store(res.isOk) + discard ctx[].signal.fireSync() + +proc delete*( + self: ThreadDatastore, + key: Key): Future[?!void] {.async.} = + + var + signal = ThreadSignalPtr.new().valueOr: + return failure("Failed to create signal") + + key = key + ctx = TaskCtx( + ds: addr self.ds, + res: TaskRes(msg: nil), + signal: signal) + + proc runTask() = + self.tp.spawn delTask(addr ctx, addr key) + + try: + runTask() + await wait(ctx.signal) + + if ctx.res.ok.load() == false: + return failure("error") + + return success() + finally: + ctx.signal.close() + +proc putTask( + ctx: ptr TaskCtx, + key: ptr Key, + data: ptr UncheckedArray[byte], + len: int) = + ## run put in a thread task + ## + + let + res = (waitFor ctx[].ds[].put( + key[], + @(toOpenArray(data, 0, len - 1)))).catch + + if res.isErr: + var err = cstring(res.error().msg) + ctx[].res.msg = addr err + else: + ctx[].res.msg = nil + + ctx[].res.ok.store(res.isOk) + discard ctx[].signal.fireSync() + +proc put*( + self: ThreadDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + + var + signal = ThreadSignalPtr.new().valueOr: + return failure("Failed to create signal") + key = key + data = data + ctx = TaskCtx( + ds: addr self.ds, + res: TaskRes(msg: nil), + signal: signal) + + proc runTask() = + self.tp.spawn putTask( + addr ctx, + addr key, makeUncheckedArray(baseAddr data), + data.len) + + try: + runTask() + await wait(ctx.signal) + finally: + ctx.signal.close() + + if ctx.res.ok.load() == false: + return failure("error") + + return success() + +proc getTask( + ctx: ptr TaskCtx, + key: ptr Key, + buf: ptr ForeignBuff[byte]) = + ## Run get in a thread task + ## + + without res =? (waitFor ctx[].ds[].get(key[])).catch, error: + var err = cstring(error.msg) + ctx[].res.msg = addr err + return + + var + data = res.get() + cell = protect(addr data) + ctx[].res.msg = nil + buf[].attach( + makeUncheckedArray(baseAddr data), data.len, cell) + + ctx[].res.ok.store(res.isOk) + discard ctx[].signal.fireSync() + +proc get*( + self: ThreadDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + + var + signal = ThreadSignalPtr.new().valueOr: + return failure("Failed to create signal") + + key = key + buf = ForeignBuff[byte].init() + ctx = TaskCtx( + ds: addr self.ds, + res: TaskRes(msg: nil), + signal: signal) + + proc runTask() = + self.tp.spawn getTask(addr ctx, addr key, addr buf) + + try: + runTask() + await wait(ctx.signal) + + if ctx.res.ok.load() == false: + return failure("error") + + var data = @(toOpenArray(buf.get(), 0, buf.len - 1)) + return success(data) + finally: + ctx.signal.close() + +proc new*( + self: type ThreadDatastore, + ds: Datastore, + tp: Taskpool): ?!ThreadDatastore = + success ThreadDatastore(tp: tp, ds: ds) diff --git a/datastore/threadproxyds.nim b/datastore/threads/threadproxyds_.nim similarity index 100% rename from datastore/threadproxyds.nim rename to datastore/threads/threadproxyds_.nim diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim new file mode 100644 index 00000000..2692093b --- /dev/null +++ b/datastore/threads/threadresult.nim @@ -0,0 +1,60 @@ + +import threading/smartptrs + +import pkg/upraises +push: {.upraises: [].} + +import pkg/chronos/threadsync + +import ./foreignbuffer + +type + CatchableErrorBuffer = ForeignBuffer[ref CatchableError] + + ThreadResult*[T] = object + signal*: ThreadSignalPtr + results*: Result[T, CatchableErrorBuffer] + + TResult*[T] = SharedPtr[ThreadResult[T]] + +proc success*[T](ret: TResult[T], value: T) = + ret[].results.ok(value) + +proc success*[T: void](ret: TResult[T]) = + ret[].results.ok() + +proc failure*[T](ret: TResult[T], exc: ref Exception) = + ret[].results.err(exc.toBuffer()) + +proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableError] = + if ret[].results.isOk(): + when S is seq[byte]: + result.ok(ret[].results.get().toSeq(byte)) + elif S is string: + result.ok(ret[].results.get().toString()) + elif S is void: + result.ok() + # elif S is QueryResponse: + # result.ok(ret[].results.get().toQueryResponse()) + else: + result.ok(ret[].results.get()) + else: + let exc: ref CatchableError = ret[].results.error().toCatchable() + result.err(exc) + +proc new*[T]( + self: type ThreadResult, + tp: typedesc[T]): Result[TResult[T], ref CatchableError] = + ## Create a new ThreadResult for type T + ## + + let + res = newSharedPtr(ThreadResult[T]) + signal = ThreadSignalPtr.new() + + if signal.isErr: + return err((ref CatchableError)(msg: signal.error())) + else: + res[].signal = signal.get() + + ok res diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 06415701..0f795363 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -7,75 +7,84 @@ import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils +import pkg/taskpools import pkg/datastore/memoryds -import pkg/datastore/threadproxyds +import pkg/datastore/threads/threadproxyds import ./dscommontests -import ./querycommontests +# import ./querycommontests import pretty -suite "Test Basic ThreadProxyDatastore": - var - sds: ThreadProxyDatastore - mem: MemoryDatastore - key1: Key - data: seq[byte] - - setupAll: - mem = MemoryDatastore.new() - sds = newThreadProxyDatastore(mem).expect("should work") - key1 = Key.init("/a").tryGet - data = "value for 1".toBytes() - - test "check put": - echo "\n\n=== put ===" - let res1 = await sds.put(key1, data) - print "res1: ", res1 - - test "check get": - echo "\n\n=== get ===" - let res2 = await sds.get(key1) - check res2.get() == data - var val = "" - for c in res2.get(): - val &= char(c) - print "get res2: ", $val - - # echo "\n\n=== put cancel ===" - # # let res1 = await sds.put(key1, "value for 1".toBytes()) - # let res3 = sds.put(key1, "value for 1".toBytes()) - # res3.cancel() - # # print "res3: ", res3 +# suite "Test Basic ThreadProxyDatastore": +# var +# sds: ThreadDatastore +# mem: MemoryDatastore +# key1: Key +# data: seq[byte] +# taskPool: Taskpool + +# setupAll: +# mem = MemoryDatastore.new() +# taskPool = TaskPool.new(3) +# sds = ThreadDatastore.new(mem, taskPool).expect("should work") +# key1 = Key.init("/a").tryGet +# data = "value for 1".toBytes() + +# test "check put": +# echo "\n\n=== put ===" +# let res1 = await sds.put(key1, data) +# print "res1: ", res1 + +# test "check get": +# echo "\n\n=== get ===" +# let res2 = await sds.get(key1) +# check res2.get() == data +# var val = "" +# for c in res2.get(): +# val &= char(c) + +# print "get res2: ", $val + +# # echo "\n\n=== put cancel ===" +# # # let res1 = await sds.put(key1, "value for 1".toBytes()) +# # let res3 = sds.put(key1, "value for 1".toBytes()) +# # res3.cancel() +# # # print "res3: ", res3 suite "Test Basic ThreadProxyDatastore": var memStore: MemoryDatastore - ds: ThreadProxyDatastore + ds: ThreadDatastore key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes + taskPool: TaskPool setupAll: memStore = MemoryDatastore.new() - ds = newThreadProxyDatastore(memStore).expect("should work") + taskPool = TaskPool.new(3) + ds = ThreadDatastore.new(memStore, taskPool).expect("should work") teardownAll: (await memStore.close()).get() + # test "check put": + # (await ds.put(key, bytes)).tryGet() + basicStoreTests(ds, key, bytes, otherBytes) -suite "Test Query": - var - mem: MemoryDatastore - sds: ThreadProxyDatastore +# suite "Test Query": +# var +# mem: MemoryDatastore +# sds: ThreadProxyDatastore - setup: - mem = MemoryDatastore.new() - sds = newThreadProxyDatastore(mem).expect("should work") +# setup: +# mem = MemoryDatastore.new() +# sds = newThreadProxyDatastore(mem).expect("should work") - queryTests(sds, false) +# queryTests(sds, false) From b7454d6e3d2c89f4ec1dae72a33d27c5e7c7140f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 11 Sep 2023 14:48:53 -0600 Subject: [PATCH 117/445] foreign buffer --- datastore/memoryds.nim | 8 +- datastore/query.nim | 2 +- datastore/threads/databuffer.nim | 92 ------------ datastore/threads/foreignbuffer.nim | 82 ++++++----- datastore/threads/threadproxyds.nim | 130 +++++++++-------- datastore/threads/threadproxyds_.nim | 202 -------------------------- datastore/threads/threadresult.nim | 60 -------- tests/datastore/testthreadproxyds.nim | 9 +- 8 files changed, 120 insertions(+), 465 deletions(-) delete mode 100644 datastore/threads/databuffer.nim delete mode 100644 datastore/threads/threadproxyds_.nim delete mode 100644 datastore/threads/threadresult.nim diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index fd02fe96..5745bf3e 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -21,9 +21,8 @@ type store*: Table[Key, seq[byte]] method has*( - self: MemoryDatastore, - key: Key -): Future[?!bool] {.async.} = + self: MemoryDatastore, + key: Key): Future[?!bool] {.async.} = return success self.store.hasKey(key) @@ -113,5 +112,4 @@ method close*(self: MemoryDatastore): Future[?!void] {.async.} = return success() func new*(tp: type MemoryDatastore): MemoryDatastore = - var self = tp() - return self + MemoryDatastore() diff --git a/datastore/query.nim b/datastore/query.nim index 5f9c9131..e2ca1189 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -7,7 +7,7 @@ import pkg/questionable/results import ./key import ./types -import ./threads/databuffer +# import ./threads/databuffer export options, SortOrder type diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim deleted file mode 100644 index 67193e0d..00000000 --- a/datastore/threads/databuffer.nim +++ /dev/null @@ -1,92 +0,0 @@ - -import threading/smartptrs -import std/hashes -import pkg/stew/results -import pkg/upraises - -push: {.upraises: [].} - -import ../key - -export hashes - -type - DataBufferHolder* = object - buf: ptr UncheckedArray[byte] - size: int - - DataBuffer* = SharedPtr[DataBufferHolder] ##\ - ## A fixed length data buffer using a SharedPtr. - ## It is thread safe even with `refc` since - ## it doesn't use string or seq types internally. - ## - - KeyBuffer* = DataBuffer - ValueBuffer* = DataBuffer - StringBuffer* = DataBuffer - - CatchableErrorBuffer* = object - msg: StringBuffer - -proc `=destroy`*(x: var DataBufferHolder) = - ## copy pointer implementation - if x.buf != nil: - # when isMainModule or true: - # echo "buffer: FREE: ", repr x.buf.pointer - deallocShared(x.buf) - -proc len*(a: DataBuffer): int = a[].size - -proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) - -proc hash*(a: DataBuffer): Hash = - a[].buf.toOpenArray(0, a[].size-1).hash() - -proc `==`*(a, b: DataBuffer): bool = - if a.isNil and b.isNil: return true - elif a.isNil or b.isNil: return false - elif a[].size != b[].size: return false - elif a[].buf == b[].buf: return true - else: a.hash() == b.hash() - -proc new*(tp: typedesc[DataBuffer], size: int = 0): DataBuffer = - ## allocate new buffer with given size - newSharedPtr(DataBufferHolder( - buf: cast[typeof(result[].buf)](allocShared0(size)), - size: size, - )) - -proc new*[T: byte | char](tp: typedesc[DataBuffer], data: openArray[T]): DataBuffer = - ## allocate new buffer and copies indata from openArray - ## - result = DataBuffer.new(data.len) - if data.len() > 0: - copyMem(result[].buf, unsafeAddr data[0], data.len) - -proc toSeq*[T: byte | char](a: DataBuffer, tp: typedesc[T]): seq[T] = - ## convert buffer to a seq type using copy and either a byte or char - result = newSeq[T](a.len) - copyMem(addr result[0], unsafeAddr a[].buf[0], a.len) - -proc toString*(data: DataBuffer): string = - ## convert buffer to string type using copy - if data.isNil: return "" - result = newString(data.len()) - if data.len() > 0: - copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) - -proc toCatchable*(err: CatchableErrorBuffer): ref CatchableError = - ## convert back to a ref CatchableError - result = (ref CatchableError)(msg: err.msg.toString()) - -proc toBuffer*(err: ref Exception): CatchableErrorBuffer = - ## convert exception to an object with StringBuffer - return CatchableErrorBuffer( - msg: StringBuffer.new(err.msg) - ) - -proc new*(tp: typedesc[KeyBuffer], key: Key): KeyBuffer = - KeyBuffer.new(key.id()) - -proc toKey*(kb: KeyBuffer): Result[Key, ref CatchableError] = - Key.init(kb.toString()) diff --git a/datastore/threads/foreignbuffer.nim b/datastore/threads/foreignbuffer.nim index b2002ea8..54a8720f 100644 --- a/datastore/threads/foreignbuffer.nim +++ b/datastore/threads/foreignbuffer.nim @@ -1,8 +1,10 @@ +import std/locks + type - ## Copy foreign buffers between threads. + ## Pass foreign buffers between threads. ## - ## This is meant to be used as a temporary holder a + ## This is meant to be used as temporary holder ## pointer to a foreign buffer that is being passed ## between threads. ## @@ -11,58 +13,64 @@ type ## used with refgc. ## ForeignBuff*[T] = object + lock: Lock buf: ptr UncheckedArray[T] len: int cell: ForeignCell -proc `=sink`[T](a: var ForeignBuff[T], b: ForeignBuff[T]) = - `=destroy`(a) - wasMoved(a) - a.len = b.len - a.buf = b.buf - a.cell = b.cell +proc `=sink`[T](self: var ForeignBuff[T], b: ForeignBuff[T]) = + withLock(self.lock): + `=destroy`(self) + wasMoved(self) + self.len = b.len + self.buf = b.buf + self.cell = b.cell -proc `=copy`[T](a: var ForeignBuff[T], b: ForeignBuff[T]) - {.error: "You can't copy the buffer, only it's contents!".} +proc `=copy`[T](self: var ForeignBuff[T], b: ForeignBuff[T]) {.error.} proc `=destroy`[T](self: var ForeignBuff[T]) = - if self.cell.data != nil: - echo "DESTROYING CELL" - dispose self.cell + withLock(self.lock): + if self.cell.data != nil: + echo "DESTROYING CELL" + dispose self.cell proc len*[T](self: ForeignBuff[T]): int = return self.len -template `[]`*[T](self: ForeignBuff[T], idx: int): T = - assert idx >= 0 and idx < self.len - return self.buf[idx] - -template `[]=`*[T](self: ForeignBuff[T], idx: int, val: T) = - assert idx >= 0 and idx < self.len - return self.buf[idx] - proc get*[T](self: ForeignBuff[T]): ptr UncheckedArray[T] = self.buf -iterator items*[T](self: ForeignBuff[T]): T = - for i in 0 ..< self.len: - yield self.buf[i] - -iterator miterms*[T](self: ForeignBuff[T]): var T = - for i in 0 ..< self.len: - yield self.buf[i] - proc attach*[T]( self: var ForeignBuff[T], - buf: ptr UncheckedArray[T], - len: int, - cell: ForeignCell) = - ## Attach a foreign pointer to this buffer + buf: openArray[T]) = + ## Attach self foreign pointer to this buffer ## + withLock(self.lock): + self.buf = makeUncheckedArray[T](baseAddr buf) + self.len = buf.len + self.cell = protect(self.buf) - self.buf = buf - self.len = len - self.cell = cell +func attached*[T]() = + ## Check if self foreign pointer is attached to this buffer + ## + withLock(self.lock): + return self.but != nil and self.cell.data != nil + +## NOTE: Converters might return copies of the buffer, +## this should be overall safe since we want to copy +## the buffer local GC anyway. +converter toSeq*[T](self: ForeignBuff[T]): seq[T] | lent seq[T] = + @(self.buf.toOpenArray(0, self.len - 1)) + +converter toString*[T](self: ForeignBuff[T]): string | lent string = + $(self.buf.toOpenArray(0, self.len - 1)) + +converter getVal*[T](self: ForeignBuff[T]): ptr UncheckedArray[T] = + self.buf func init*[T](_: type ForeignBuff[T]): ForeignBuff[T] = - return ForeignBuff[T]() + var + lock = Lock() + + lock.initLock() + ForeignBuff[T](lock: lock) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 71a1bbeb..b0b44e96 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -1,4 +1,7 @@ +when not compileOption("threads"): + {.error: "This module requires --threads:on compilation flag".} + import pkg/upraises push: {.upraises: [].} @@ -19,50 +22,49 @@ import ../datastore import ./foreignbuffer type - TaskRes = object + ThreadResults = object ok: Atomic[bool] - msg: ptr cstring + msg: ForeignBuff[char] TaskCtx = object ds: ptr Datastore - res: TaskRes + res: ptr ThreadResults signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore tp*: Taskpool ds*: Datastore +proc success(self: var ThreadResults) {.inline.} = + self.ok.store(true) + +proc failure(self: var ThreadResults, msg: var string) {.inline.} = + self.ok.store(false) + self.msg.attach(msg.toOpenArray(0, msg.high)) + proc hasTask( ctx: ptr TaskCtx, key: ptr Key, doesHave: ptr bool) = - let - res = (waitFor ctx[].ds[].has(key[])).catch - - if res.isErr: - var - err = cstring(res.error().msg) - ctx[].res.msg = addr err - else: - ctx[].res.msg = nil - doesHave[] = res.get().get() + without res =? (waitFor ctx[].ds[].has(key[])).catch, error: + ctx[].res[].failure(error.msg) + return - ctx[].res.ok.store(res.isOk) + doesHave[] = res.get() + ctx[].res[].success() discard ctx[].signal.fireSync() -proc has*( - self: ThreadDatastore, - key: Key): Future[?!bool] {.async.} = - +method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var signal = ThreadSignalPtr.new().valueOr: return failure("Failed to create signal") key = key + res = ThreadResults() ctx = TaskCtx( ds: addr self.ds, - res: TaskRes(msg: nil), + res: addr res, signal: signal) doesHave = false @@ -73,30 +75,22 @@ proc has*( runTask() await wait(ctx.signal) - var data: bool if ctx.res.ok.load() == false: - return failure("error") + return failure($(ctx.res.msg)) return success(doesHave) finally: ctx.signal.close() proc delTask(ctx: ptr TaskCtx, key: ptr Key) = + without res =? (waitFor ctx[].ds[].delete(key[])).catch, error: + ctx[].res[].failure(error.msg) + return - let - res = (waitFor ctx[].ds[].delete(key[])).catch - - if res.isErr: - var - err = cstring(res.error().msg) - ctx[].res.msg = addr err - else: - ctx[].res.msg = nil - - ctx[].res.ok.store(res.isOk) + ctx[].res[].ok.store(true) discard ctx[].signal.fireSync() -proc delete*( +method delete*( self: ThreadDatastore, key: Key): Future[?!void] {.async.} = @@ -105,9 +99,10 @@ proc delete*( return failure("Failed to create signal") key = key + res = ThreadResults() ctx = TaskCtx( ds: addr self.ds, - res: TaskRes(msg: nil), + res: addr res, signal: signal) proc runTask() = @@ -124,6 +119,13 @@ proc delete*( finally: ctx.signal.close() +method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err + + return success() + proc putTask( ctx: ptr TaskCtx, key: ptr Key, @@ -132,21 +134,16 @@ proc putTask( ## run put in a thread task ## - let - res = (waitFor ctx[].ds[].put( + without res =? (waitFor ctx[].ds[].put( key[], - @(toOpenArray(data, 0, len - 1)))).catch - - if res.isErr: - var err = cstring(res.error().msg) - ctx[].res.msg = addr err - else: - ctx[].res.msg = nil + @(toOpenArray(data, 0, len - 1)))).catch, error: + ctx[].res[].failure(error.msg) + return - ctx[].res.ok.store(res.isOk) + ctx[].res[].ok.store(true) discard ctx[].signal.fireSync() -proc put*( +method put*( self: ThreadDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = @@ -156,15 +153,17 @@ proc put*( return failure("Failed to create signal") key = key data = data + res = ThreadResults() ctx = TaskCtx( ds: addr self.ds, - res: TaskRes(msg: nil), + res: addr res, signal: signal) proc runTask() = self.tp.spawn putTask( addr ctx, - addr key, makeUncheckedArray(baseAddr data), + addr key, + makeUncheckedArray(baseAddr data), data.len) try: @@ -173,8 +172,18 @@ proc put*( finally: ctx.signal.close() - if ctx.res.ok.load() == false: - return failure("error") + if ctx.res[].ok.load() == false: + return failure($(ctx.res[].msg)) + + return success() + +method put*( + self: ThreadDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err return success() @@ -186,21 +195,18 @@ proc getTask( ## without res =? (waitFor ctx[].ds[].get(key[])).catch, error: - var err = cstring(error.msg) - ctx[].res.msg = addr err + var err = error.msg + ctx[].res[].failure(error.msg) return var data = res.get() - cell = protect(addr data) - ctx[].res.msg = nil - buf[].attach( - makeUncheckedArray(baseAddr data), data.len, cell) - ctx[].res.ok.store(res.isOk) + buf[].attach(data) + ctx[].res[].ok.store(res.isOk) discard ctx[].signal.fireSync() -proc get*( +method get*( self: ThreadDatastore, key: Key): Future[?!seq[byte]] {.async.} = @@ -210,9 +216,10 @@ proc get*( key = key buf = ForeignBuff[byte].init() + res = ThreadResults() ctx = TaskCtx( ds: addr self.ds, - res: TaskRes(msg: nil), + res: addr res, signal: signal) proc runTask() = @@ -223,14 +230,13 @@ proc get*( await wait(ctx.signal) if ctx.res.ok.load() == false: - return failure("error") + return failure($(ctx.res[].msg)) - var data = @(toOpenArray(buf.get(), 0, buf.len - 1)) - return success(data) + return success(buf.toSeq()) finally: ctx.signal.close() -proc new*( +func new*( self: type ThreadDatastore, ds: Datastore, tp: Taskpool): ?!ThreadDatastore = diff --git a/datastore/threads/threadproxyds_.nim b/datastore/threads/threadproxyds_.nim deleted file mode 100644 index 4410f103..00000000 --- a/datastore/threads/threadproxyds_.nim +++ /dev/null @@ -1,202 +0,0 @@ -import std/tables - -import pkg/chronos -import pkg/chronos/threadsync -import pkg/questionable -import pkg/questionable/results -import pkg/upraises -import pkg/taskpools -import pkg/stew/results -import pkg/threading/smartptrs - -import ./key -import ./query -import ./datastore -import ./threadbackend -import ./fsds - -export key, query - -push: {.upraises: [].} - -type - ThreadProxyDatastore* = ref object of Datastore - tds: ThreadDatastorePtr - -method has*( - self: ThreadProxyDatastore, - key: Key -): Future[?!bool] {.async.} = - - without ret =? newThreadResult(bool), err: - return failure(err) - - try: - has(ret, self.tds, key) - await wait(ret[].signal) - finally: - ret[].signal.close() - - return ret.convert(bool) - -method delete*( - self: ThreadProxyDatastore, - key: Key -): Future[?!void] {.async.} = - - without ret =? newThreadResult(void), err: - return failure(err) - - try: - delete(ret, self.tds, key) - await wait(ret[].signal) - finally: - ret[].signal.close() - - return ret.convert(void) - -method delete*( - self: ThreadProxyDatastore, - keys: seq[Key] -): Future[?!void] {.async.} = - - for key in keys: - if err =? (await self.delete(key)).errorOption: - return failure err - - return success() - -method get*( - self: ThreadProxyDatastore, - key: Key -): Future[?!seq[byte]] {.async.} = - ## implements batch get - ## - ## note: this implementation is rather naive and should - ## probably be switched to use a single ThreadSignal - ## for the entire batch - - without ret =? newThreadResult(ValueBuffer), err: - return failure(err) - - try: - get(ret, self.tds, key) - await wait(ret[].signal) - finally: - ret[].signal.close() - - return ret.convert(seq[byte]) - -method put*( - self: ThreadProxyDatastore, - key: Key, - data: seq[byte] -): Future[?!void] {.async.} = - - without ret =? newThreadResult(void), err: - return failure(err) - - try: - put(ret, self.tds, key, data) - await wait(ret[].signal) - finally: - ret[].signal.close() - - return ret.convert(void) - -method put*( - self: ThreadProxyDatastore, - batch: seq[BatchEntry] -): Future[?!void] {.async.} = - ## implements batch put - ## - ## note: this implementation is rather naive and should - ## probably be switched to use a single ThreadSignal - ## for the entire batch - - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err - - return success() - -import pretty - -method query*( - self: ThreadProxyDatastore, - query: Query -): Future[?!QueryIter] {.async.} = - - without ret =? newThreadResult(QueryResponseBuffer), err: - return failure(err) - - echo "\n\n=== Query Start === " - - ## we need to setup the query iter on the main thread - ## to keep it's lifetime associated with this async Future - without it =? await self.tds[].ds.query(query), err: - ret.failure(err) - - var iter = newSharedPtr(QueryIterStore) - ## note that bypasses SharedPtr isolation - may need `protect` here? - iter[].it = it - - var iterWrapper = QueryIter.new() - - proc next(): Future[?!QueryResponse] {.async.} = - print "query:next:start: " - iterWrapper.finished = iter[].it.finished - if not iter[].it.finished: - query(ret, self.tds, iter) - await wait(ret[].signal) - echo "" - print "query:post: ", ret[].results - print "query:post:finished: ", iter[].it.finished - print "query:post: ", " qrb:key: ", ret[].results.get().key.toString() - print "query:post: ", " qrb:data: ", ret[].results.get().data.toString() - result = ret.convert(QueryResponse) - else: - result = success (Key.none, EmptyBytes) - - proc dispose(): Future[?!void] {.async.} = - iter[].it = nil # ensure our sharedptr doesn't try and dealloc - ret[].signal.close() - return success() - - iterWrapper.next = next - iterWrapper.dispose = dispose - return success iterWrapper - -method close*( - self: ThreadProxyDatastore -): Future[?!void] {.async.} = - # TODO: how to handle failed close? - result = success() - - without res =? self.tds[].ds.close(), err: - result = failure(err) - # GC_unref(self.tds[].ds) ## TODO: is this needed? - - if self.tds[].tp != nil: - ## this can block... how to handle? maybe just leak? - self.tds[].tp.shutdown() - - self[].tds[].ds = nil # ensure our sharedptr doesn't try and dealloc - -proc newThreadProxyDatastore*( - ds: Datastore, -): ?!ThreadProxyDatastore = - ## create a new - - var self = ThreadProxyDatastore() - let value = newSharedPtr(ThreadDatastore) - # GC_ref(ds) ## TODO: is this needed? - try: - value[].ds = ds - value[].tp = Taskpool.new(num_threads = 2) - except Exception as exc: - return err((ref DatastoreError)(msg: exc.msg)) - - self.tds = value - - success self diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim deleted file mode 100644 index 2692093b..00000000 --- a/datastore/threads/threadresult.nim +++ /dev/null @@ -1,60 +0,0 @@ - -import threading/smartptrs - -import pkg/upraises -push: {.upraises: [].} - -import pkg/chronos/threadsync - -import ./foreignbuffer - -type - CatchableErrorBuffer = ForeignBuffer[ref CatchableError] - - ThreadResult*[T] = object - signal*: ThreadSignalPtr - results*: Result[T, CatchableErrorBuffer] - - TResult*[T] = SharedPtr[ThreadResult[T]] - -proc success*[T](ret: TResult[T], value: T) = - ret[].results.ok(value) - -proc success*[T: void](ret: TResult[T]) = - ret[].results.ok() - -proc failure*[T](ret: TResult[T], exc: ref Exception) = - ret[].results.err(exc.toBuffer()) - -proc convert*[T, S](ret: TResult[T], tp: typedesc[S]): Result[S, ref CatchableError] = - if ret[].results.isOk(): - when S is seq[byte]: - result.ok(ret[].results.get().toSeq(byte)) - elif S is string: - result.ok(ret[].results.get().toString()) - elif S is void: - result.ok() - # elif S is QueryResponse: - # result.ok(ret[].results.get().toQueryResponse()) - else: - result.ok(ret[].results.get()) - else: - let exc: ref CatchableError = ret[].results.error().toCatchable() - result.err(exc) - -proc new*[T]( - self: type ThreadResult, - tp: typedesc[T]): Result[TResult[T], ref CatchableError] = - ## Create a new ThreadResult for type T - ## - - let - res = newSharedPtr(ThreadResult[T]) - signal = ThreadSignalPtr.new() - - if signal.isErr: - return err((ref CatchableError)(msg: signal.error())) - else: - res[].signal = signal.get() - - ok res diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 0f795363..8ca84818 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -62,19 +62,16 @@ suite "Test Basic ThreadProxyDatastore": key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - taskPool: TaskPool + taskPool: Taskpool setupAll: memStore = MemoryDatastore.new() - taskPool = TaskPool.new(3) - ds = ThreadDatastore.new(memStore, taskPool).expect("should work") + taskPool = Taskpool.new(2) + ds = ThreadDatastore.new(memStore, taskPool).tryGet() teardownAll: (await memStore.close()).get() - # test "check put": - # (await ds.put(key, bytes)).tryGet() - basicStoreTests(ds, key, bytes, otherBytes) # suite "Test Query": From 9bbf3ed59563c8f7086ebf19400b5d55460afadb Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:50:07 -0600 Subject: [PATCH 118/445] use shared table --- datastore/memoryds.nim | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 5745bf3e..90e7c6f5 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -1,4 +1,5 @@ import std/tables +import std/sharedtables import std/sequtils import std/strutils import std/algorithm @@ -18,13 +19,16 @@ push: {.upraises: [].} type MemoryDatastore* = ref object of Datastore - store*: Table[Key, seq[byte]] + store*: SharedTable[Key, seq[byte]] method has*( self: MemoryDatastore, key: Key): Future[?!bool] {.async.} = - return success self.store.hasKey(key) + self.store.withValue(key, value): + return success true + do: + return success(false) method delete*( self: MemoryDatastore, @@ -108,8 +112,12 @@ method put*( # return success iter method close*(self: MemoryDatastore): Future[?!void] {.async.} = - self.store.clear() + self.store.deinitSharedTable() return success() -func new*(tp: type MemoryDatastore): MemoryDatastore = - MemoryDatastore() +proc new*(tp: type MemoryDatastore): MemoryDatastore = + var + table: SharedTable[Key, seq[byte]] + + table.init() + MemoryDatastore(store: table) From 13e89bca5dbdb2c7ef350980803061b2ee9450eb Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:50:21 -0600 Subject: [PATCH 119/445] adding semaphore for async backpresure --- datastore/threads/asyncsemaphore.nim | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 datastore/threads/asyncsemaphore.nim diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim new file mode 100644 index 00000000..ee57ee00 --- /dev/null +++ b/datastore/threads/asyncsemaphore.nim @@ -0,0 +1,96 @@ +# Nim-datastore +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +{.push raises: [].} + +import sequtils +import chronos, chronicles + +# TODO: this should probably go in chronos + +logScope: + topics = "datastore semaphore" + +type + AsyncSemaphore* = ref object of RootObj + size*: int + count: int + queue: seq[Future[void]] + +proc new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = + AsyncSemaphore(size: size, count: size) + +proc `count`*(s: AsyncSemaphore): int = s.count + +proc tryAcquire*(s: AsyncSemaphore): bool = + ## Attempts to acquire a resource, if successful + ## returns true, otherwise false + ## + + if s.count > 0 and s.queue.len == 0: + s.count.dec + trace "Acquired slot", available = s.count, queue = s.queue.len + return true + +proc acquire*(s: AsyncSemaphore): Future[void] = + ## Acquire a resource and decrement the resource + ## counter. If no more resources are available, + ## the returned future will not complete until + ## the resource count goes above 0. + ## + + let fut = newFuture[void]("AsyncSemaphore.acquire") + if s.tryAcquire(): + fut.complete() + return fut + + proc cancellation(udata: pointer) {.gcsafe.} = + fut.cancelCallback = nil + if not fut.finished: + s.queue.keepItIf( it != fut ) + + fut.cancelCallback = cancellation + + s.queue.add(fut) + + trace "Queued slot", available = s.count, queue = s.queue.len + return fut + +proc forceAcquire*(s: AsyncSemaphore) = + ## ForceAcquire will always succeed, + ## creating a temporary slot if required. + ## This temporary slot will stay usable until + ## there is less `acquire`s than `release`s + s.count.dec + +proc release*(s: AsyncSemaphore) = + ## Release a resource from the semaphore, + ## by picking the first future from the queue + ## and completing it and incrementing the + ## internal resource count + ## + + doAssert(s.count <= s.size) + + if s.count < s.size: + trace "Releasing slot", available = s.count, + queue = s.queue.len + + s.count.inc + while s.queue.len > 0: + var fut = s.queue[0] + s.queue.delete(0) + if not fut.finished(): + s.count.dec + fut.complete() + break + + trace "Released slot", available = s.count, + queue = s.queue.len + return From 184420c4e842b1aa6c6e22d1de39d921f0d449b5 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:50:29 -0600 Subject: [PATCH 120/445] re-adding databuf --- datastore/threads/databuffer.nim | 87 ++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 datastore/threads/databuffer.nim diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim new file mode 100644 index 00000000..270aed8f --- /dev/null +++ b/datastore/threads/databuffer.nim @@ -0,0 +1,87 @@ +import threading/smartptrs +import std/hashes + +export hashes + +type + DataBufferHolder* = object + buf: ptr UncheckedArray[byte] + size: int + + DataBuffer* = SharedPtr[DataBufferHolder] ##\ + ## A fixed length data buffer using a SharedPtr. + ## It is thread safe even with `refc` since + ## it doesn't use string or seq types internally. + ## + +proc `=destroy`*(x: var DataBufferHolder) = + ## copy pointer implementation + ## + + if x.buf != nil: + # when isMainModule or true: + # echo "buffer: FREE: ", repr x.buf.pointer + deallocShared(x.buf) + +proc len*(a: DataBuffer): int = a[].size + +proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) + +proc hash*(a: DataBuffer): Hash = + a[].buf.toOpenArray(0, a[].size-1).hash() + +proc `==`*(a, b: DataBuffer): bool = + if a.isNil and b.isNil: return true + elif a.isNil or b.isNil: return false + elif a[].size != b[].size: return false + elif a[].buf == b[].buf: return true + else: a.hash() == b.hash() + +proc new*(tp: type DataBuffer, size: int = 0): DataBuffer = + ## allocate new buffer with given size + ## + + newSharedPtr(DataBufferHolder( + buf: cast[typeof(result[].buf)](allocShared0(size)), + size: size, + )) + +proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T]): DataBuffer = + ## allocate new buffer and copies indata from openArray + ## + result = DataBuffer.new(data.len) + if data.len() > 0: + copyMem(result[].buf, unsafeAddr data[0], data.len) + +proc toSequence(T: typedesc[byte | char], a: DataBuffer): seq[T] = + ## convert buffer to a seq type using copy and either a byte or char + result = newSeq[T](a.len) + copyMem(addr result[0], unsafeAddr a[].buf[0], a.len) + +converter toSeq*(a: DataBuffer): seq[byte] = + ## convert buffer to a seq type using copy and either a byte or char + ## + + byte.toSequence(a) + +proc `@`*(self: DataBuffer): seq[byte] = + ## Convert a buffer to a seq type using copy and + ## either a byte or char + ## + + self.toSeq() + +converter toString*(data: DataBuffer): string = + ## convert buffer to string type using copy + ## + + if data.isNil: return "" + result = newString(data.len()) + if data.len() > 0: + copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) + +converter toBuffer*(err: ref CatchableError): DataBuffer = + ## convert exception to an object with StringBuffer + ## + + return DataBuffer.new(err.msg) From 5adc7c96119143578ed634ee288fa9fdfe0733f6 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:50:42 -0600 Subject: [PATCH 121/445] this would never work --- datastore/threads/foreignbuffer.nim | 76 ----------------------------- 1 file changed, 76 deletions(-) delete mode 100644 datastore/threads/foreignbuffer.nim diff --git a/datastore/threads/foreignbuffer.nim b/datastore/threads/foreignbuffer.nim deleted file mode 100644 index 54a8720f..00000000 --- a/datastore/threads/foreignbuffer.nim +++ /dev/null @@ -1,76 +0,0 @@ - -import std/locks - -type - ## Pass foreign buffers between threads. - ## - ## This is meant to be used as temporary holder - ## pointer to a foreign buffer that is being passed - ## between threads. - ## - ## The receiving thread should copy the contained buffer - ## to it's local GC as soon as possible. Should only be - ## used with refgc. - ## - ForeignBuff*[T] = object - lock: Lock - buf: ptr UncheckedArray[T] - len: int - cell: ForeignCell - -proc `=sink`[T](self: var ForeignBuff[T], b: ForeignBuff[T]) = - withLock(self.lock): - `=destroy`(self) - wasMoved(self) - self.len = b.len - self.buf = b.buf - self.cell = b.cell - -proc `=copy`[T](self: var ForeignBuff[T], b: ForeignBuff[T]) {.error.} - -proc `=destroy`[T](self: var ForeignBuff[T]) = - withLock(self.lock): - if self.cell.data != nil: - echo "DESTROYING CELL" - dispose self.cell - -proc len*[T](self: ForeignBuff[T]): int = - return self.len - -proc get*[T](self: ForeignBuff[T]): ptr UncheckedArray[T] = - self.buf - -proc attach*[T]( - self: var ForeignBuff[T], - buf: openArray[T]) = - ## Attach self foreign pointer to this buffer - ## - withLock(self.lock): - self.buf = makeUncheckedArray[T](baseAddr buf) - self.len = buf.len - self.cell = protect(self.buf) - -func attached*[T]() = - ## Check if self foreign pointer is attached to this buffer - ## - withLock(self.lock): - return self.but != nil and self.cell.data != nil - -## NOTE: Converters might return copies of the buffer, -## this should be overall safe since we want to copy -## the buffer local GC anyway. -converter toSeq*[T](self: ForeignBuff[T]): seq[T] | lent seq[T] = - @(self.buf.toOpenArray(0, self.len - 1)) - -converter toString*[T](self: ForeignBuff[T]): string | lent string = - $(self.buf.toOpenArray(0, self.len - 1)) - -converter getVal*[T](self: ForeignBuff[T]): ptr UncheckedArray[T] = - self.buf - -func init*[T](_: type ForeignBuff[T]): ForeignBuff[T] = - var - lock = Lock() - - lock.initLock() - ForeignBuff[T](lock: lock) From 4c48383b88ac59791232b97d37a91fdaf02810d9 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:51:01 -0600 Subject: [PATCH 122/445] reworked with less copying --- datastore/threads/threadproxyds.nim | 181 ++++++++++++++-------------- 1 file changed, 88 insertions(+), 93 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index b0b44e96..312db405 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -7,6 +7,7 @@ import pkg/upraises push: {.upraises: [].} import std/atomics +import std/strutils import pkg/chronos import pkg/chronos/threadsync @@ -14,81 +15,91 @@ import pkg/questionable import pkg/questionable/results import pkg/stew/ptrops import pkg/taskpools +import pkg/threading/smartptrs +import pkg/stew/byteutils import ../key import ../query import ../datastore -import ./foreignbuffer +import ./asyncsemaphore +import ./databuffer type - ThreadResults = object - ok: Atomic[bool] - msg: ForeignBuff[char] + ThreadTypes = void | bool | SomeInteger | DataBuffer + ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] - TaskCtx = object + TaskCtx[T: ThreadTypes] = object ds: ptr Datastore - res: ptr ThreadResults + res: ptr ThreadResult[T] signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore - tp*: Taskpool - ds*: Datastore + tp: Taskpool + ds: Datastore + semaphore: AsyncSemaphore + tasks: seq[Future[void]] -proc success(self: var ThreadResults) {.inline.} = - self.ok.store(true) +template dispatchTask(self: ThreadDatastore, ctx: TaskCtx, runTask: proc): untyped = + let + fut = wait(ctx.signal) -proc failure(self: var ThreadResults, msg: var string) {.inline.} = - self.ok.store(false) - self.msg.attach(msg.toOpenArray(0, msg.high)) + try: + await self.semaphore.acquire() + runTask() + self.tasks.add(fut) + await fut + + if ctx.res[].isErr: + result = failure(ctx.res[].error()) + finally: + discard ctx.signal.close() + if ( + let idx = self.tasks.find(fut); + idx != -1): + self.tasks.del(idx) + + self.semaphore.release() proc hasTask( ctx: ptr TaskCtx, - key: ptr Key, - doesHave: ptr bool) = + key: ptr Key) = + + defer: + discard ctx[].signal.fireSync() without res =? (waitFor ctx[].ds[].has(key[])).catch, error: - ctx[].res[].failure(error.msg) + ctx[].res[].err(error) return - doesHave[] = res.get() - ctx[].res[].success() - discard ctx[].signal.fireSync() + ctx[].res[].ok(res.get()) method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var signal = ThreadSignalPtr.new().valueOr: return failure("Failed to create signal") - key = key - res = ThreadResults() - ctx = TaskCtx( + res = ThreadResult[bool]() + ctx = TaskCtx[bool]( ds: addr self.ds, res: addr res, signal: signal) - doesHave = false proc runTask() = - self.tp.spawn hasTask(addr ctx, addr key, addr doesHave) + self.tp.spawn hasTask(addr ctx, unsafeAddr key) - try: - runTask() - await wait(ctx.signal) - - if ctx.res.ok.load() == false: - return failure($(ctx.res.msg)) - - return success(doesHave) - finally: - ctx.signal.close() + self.dispatchTask(ctx, runTask) + return success(res.get()) proc delTask(ctx: ptr TaskCtx, key: ptr Key) = + defer: + discard ctx[].signal.fireSync() + without res =? (waitFor ctx[].ds[].delete(key[])).catch, error: - ctx[].res[].failure(error.msg) + ctx[].res[].err(error) return - ctx[].res[].ok.store(true) - discard ctx[].signal.fireSync() + ctx[].res[].ok() method delete*( self: ThreadDatastore, @@ -98,26 +109,17 @@ method delete*( signal = ThreadSignalPtr.new().valueOr: return failure("Failed to create signal") - key = key - res = ThreadResults() - ctx = TaskCtx( + res = ThreadResult[void]() + ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, signal: signal) proc runTask() = - self.tp.spawn delTask(addr ctx, addr key) - - try: - runTask() - await wait(ctx.signal) - - if ctx.res.ok.load() == false: - return failure("error") + self.tp.spawn delTask(addr ctx, unsafeAddr key) - return success() - finally: - ctx.signal.close() + self.dispatchTask(ctx, runTask) + return success() method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = for key in keys: @@ -129,19 +131,19 @@ method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} proc putTask( ctx: ptr TaskCtx, key: ptr Key, - data: ptr UncheckedArray[byte], + data: DataBuffer, len: int) = ## run put in a thread task ## - without res =? (waitFor ctx[].ds[].put( - key[], - @(toOpenArray(data, 0, len - 1)))).catch, error: - ctx[].res[].failure(error.msg) + defer: + discard ctx[].signal.fireSync() + + without res =? (waitFor ctx[].ds[].put(key[], @data)).catch, error: + ctx[].res[].err(error) return - ctx[].res[].ok.store(true) - discard ctx[].signal.fireSync() + ctx[].res[].ok() method put*( self: ThreadDatastore, @@ -151,10 +153,9 @@ method put*( var signal = ThreadSignalPtr.new().valueOr: return failure("Failed to create signal") - key = key - data = data - res = ThreadResults() - ctx = TaskCtx( + + res = ThreadResult[void]() + ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, signal: signal) @@ -162,19 +163,11 @@ method put*( proc runTask() = self.tp.spawn putTask( addr ctx, - addr key, - makeUncheckedArray(baseAddr data), + unsafeAddr key, + DataBuffer.new(data), data.len) - try: - runTask() - await wait(ctx.signal) - finally: - ctx.signal.close() - - if ctx.res[].ok.load() == false: - return failure($(ctx.res[].msg)) - + self.dispatchTask(ctx, runTask) return success() method put*( @@ -189,22 +182,22 @@ method put*( proc getTask( ctx: ptr TaskCtx, - key: ptr Key, - buf: ptr ForeignBuff[byte]) = + key: ptr Key) = ## Run get in a thread task ## + defer: + discard ctx[].signal.fireSync() + without res =? (waitFor ctx[].ds[].get(key[])).catch, error: var err = error.msg - ctx[].res[].failure(error.msg) + ctx[].res[].err(error) return var data = res.get() - buf[].attach(data) - ctx[].res[].ok.store(res.isOk) - discard ctx[].signal.fireSync() + ctx[].res[].ok(DataBuffer.new(data)) method get*( self: ThreadDatastore, @@ -214,30 +207,32 @@ method get*( signal = ThreadSignalPtr.new().valueOr: return failure("Failed to create signal") - key = key - buf = ForeignBuff[byte].init() - res = ThreadResults() - ctx = TaskCtx( + var + res = ThreadResult[DataBuffer]() + ctx = TaskCtx[DataBuffer]( ds: addr self.ds, res: addr res, signal: signal) proc runTask() = - self.tp.spawn getTask(addr ctx, addr key, addr buf) + self.tp.spawn getTask(addr ctx, unsafeAddr key) - try: - runTask() - await wait(ctx.signal) + self.dispatchTask(ctx, runTask) + return success(@(res.get())) - if ctx.res.ok.load() == false: - return failure($(ctx.res[].msg)) +method close*(self: ThreadDatastore): Future[?!void] {.async.} = + for task in self.tasks: + await task.cancelAndWait() - return success(buf.toSeq()) - finally: - ctx.signal.close() + await self.ds.close() func new*( self: type ThreadDatastore, ds: Datastore, tp: Taskpool): ?!ThreadDatastore = - success ThreadDatastore(tp: tp, ds: ds) + doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" + + success ThreadDatastore( + tp: tp, + ds: ds, + semaphore: AsyncSemaphore.new(tp.numThreads - 1)) # one thread is needed for the task dispatcher From 2829ac82a16551db304deba7f97b335dde8cd81c Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 12 Sep 2023 13:51:15 -0600 Subject: [PATCH 123/445] remove useless async annotation --- datastore/datastore.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 3e908142..169349af 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -31,7 +31,7 @@ method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -method close*(self: Datastore): Future[?!void] {.base, async, locks: "unknown".} = +method close*(self: Datastore): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") # method query*( From d6c4d97d91699a9cf2d2349598c878b16ffc554e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:41:01 -0600 Subject: [PATCH 124/445] add $ operator --- datastore/threads/databuffer.nim | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 270aed8f..818b3c7d 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -53,16 +53,13 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T]): DataBuffer = if data.len() > 0: copyMem(result[].buf, unsafeAddr data[0], data.len) -proc toSequence(T: typedesc[byte | char], a: DataBuffer): seq[T] = - ## convert buffer to a seq type using copy and either a byte or char - result = newSeq[T](a.len) - copyMem(addr result[0], unsafeAddr a[].buf[0], a.len) - -converter toSeq*(a: DataBuffer): seq[byte] = +converter toSeq*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char ## - byte.toSequence(a) + result = newSeq[byte](self.len) + if self.len() > 0: + copyMem(addr result[0], unsafeAddr self[].buf[0], self.len) proc `@`*(self: DataBuffer): seq[byte] = ## Convert a buffer to a seq type using copy and @@ -85,3 +82,9 @@ converter toBuffer*(err: ref CatchableError): DataBuffer = ## return DataBuffer.new(err.msg) + +proc `$`*(data: DataBuffer): string = + ## convert buffer to string type using copy + ## + + data.toString() From 19954c6f949dde1d85fc188e73bd4a9adfccd19b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:41:16 -0600 Subject: [PATCH 125/445] quick and dirty query --- datastore/threads/threadproxyds.nim | 87 ++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 312db405..d3ae21ae 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -15,7 +15,6 @@ import pkg/questionable import pkg/questionable/results import pkg/stew/ptrops import pkg/taskpools -import pkg/threading/smartptrs import pkg/stew/byteutils import ../key @@ -26,7 +25,7 @@ import ./asyncsemaphore import ./databuffer type - ThreadTypes = void | bool | SomeInteger | DataBuffer + ThreadTypes = void | bool | SomeInteger | DataBuffer | tuple ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] TaskCtx[T: ThreadTypes] = object @@ -68,11 +67,12 @@ proc hasTask( defer: discard ctx[].signal.fireSync() - without res =? (waitFor ctx[].ds[].has(key[])).catch, error: + without ret =? + (waitFor ctx[].ds[].has(key[])).catch and res =? ret, error: ctx[].res[].err(error) return - ctx[].res[].ok(res.get()) + ctx[].res[].ok(res) method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var @@ -189,14 +189,11 @@ proc getTask( defer: discard ctx[].signal.fireSync() - without res =? (waitFor ctx[].ds[].get(key[])).catch, error: - var err = error.msg + without res =? + (waitFor ctx[].ds[].get(key[])).catch and data =? res, error: ctx[].res[].err(error) return - var - data = res.get() - ctx[].res[].ok(DataBuffer.new(data)) method get*( @@ -226,6 +223,78 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.ds.close() +proc queryTask*( + ctx: ptr TaskCtx, + iter: ptr QueryIter) = + + defer: + discard ctx[].signal.fireSync() + + without ret =? (waitFor iter[].next()).catch and res =? ret, error: + ctx[].res[].err(error) + return + + if res.key.isNone: + ctx[].res[].ok((false, DataBuffer.new(), DataBuffer.new())) + return + + var + keyBuf = DataBuffer.new($(res.key.get())) + dataBuf = DataBuffer.new(res.data) + + ctx[].res[].ok((true, keyBuf, dataBuf)) + +method query*( + self: ThreadDatastore, + query: Query): Future[?!QueryIter] {.async.} = + + without var childIter =? await self.ds.query(query), error: + return failure error + + var + iter = QueryIter.init() + + let lock = newAsyncLock() + proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() + + if iter.finished == true: + return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + + await lock.acquire() + + if iter.finished == true: + return success (Key.none, EmptyBytes) + + var + signal = ThreadSignalPtr.new().valueOr: + return failure("Failed to create signal") + + res = ThreadResult[(bool, DataBuffer, DataBuffer)]() + ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( + ds: addr self.ds, + res: addr res, + signal: signal) + + proc runTask() = + self.tp.spawn queryTask(addr ctx, addr childIter) + + self.dispatchTask(ctx, runTask) + if err =? res.errorOption: + return failure err + + let (ok, key, data) = res.get() + if not ok: + iter.finished = true + return success (Key.none, EmptyBytes) + + return success (Key.init($key).expect("should not fail").some, @(data)) + + iter.next = next + return success iter + func new*( self: type ThreadDatastore, ds: Datastore, From 776c58f0915361f0da476f0d6eae2a03fc62e142 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:41:40 -0600 Subject: [PATCH 126/445] re-added query and added raises (do they work?) --- datastore/datastore.nim | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 169349af..98b95208 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -13,32 +13,32 @@ push: {.upraises: [].} type BatchEntry* = tuple[key: Key, data: seq[byte]] -method has*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown".} = +method has*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown".} = +method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method delete*(self: Datastore, keys: seq[Key]): Future[?!void] {.base, locks: "unknown".} = +method delete*(self: Datastore, keys: seq[Key]): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown".} = +method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown".} = +method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, locks: "unknown".} = +method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method close*(self: Datastore): Future[?!void] {.base, locks: "unknown".} = +method close*(self: Datastore): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -# method query*( -# self: Datastore, -# query: Query): Future[?!QueryIter] {.base, gcsafe.} = +method query*( + self: Datastore, + query: Query): Future[?!QueryIter] {.base, gcsafe, raises: [].} = -# raiseAssert("Not implemented!") + raiseAssert("Not implemented!") -proc contains*(self: Datastore, key: Key): Future[bool] {.async.} = +proc contains*(self: Datastore, key: Key): Future[bool] {.async, raises: [].} = return (await self.has(key)) |? false From 7a9bc11c33a3e6ff5547ca99e603e6d6bbc7c7d3 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:42:22 -0600 Subject: [PATCH 127/445] make concurrent (but don't need anymore) --- datastore/memoryds.nim | 108 +++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim index 90e7c6f5..fa7d10f4 100644 --- a/datastore/memoryds.nim +++ b/datastore/memoryds.nim @@ -1,8 +1,12 @@ import std/tables import std/sharedtables +import std/sharedlist import std/sequtils import std/strutils import std/algorithm +import std/locks + +import std/atomics import pkg/chronos import pkg/questionable @@ -20,21 +24,36 @@ push: {.upraises: [].} type MemoryDatastore* = ref object of Datastore store*: SharedTable[Key, seq[byte]] + keys*: SharedList[Key] + lock: Lock # yes, we need the lock since we need to update both the table and the list :facepalm: + +template withLock(lock: Lock, body: untyped) = + try: + lock.acquire() + body + finally: + lock.release() method has*( self: MemoryDatastore, key: Key): Future[?!bool] {.async.} = + let + keys = toSeq(self.keys) + + for k in keys: + if k == key: + return success true - self.store.withValue(key, value): - return success true - do: - return success(false) + return success false method delete*( self: MemoryDatastore, key: Key): Future[?!void] {.async.} = - self.store.del(key) + withLock(self.lock): + self.keys.iterAndMutate(proc(k: Key): bool = k == key) + self.store.del(key) + return success() method delete*( @@ -51,18 +70,25 @@ method get*( self: MemoryDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.store.withValue(key, value): - return success value[] - do: - return failure (ref DatastoreError)(msg: "no such key") + withLock(self.lock): + let + has = (await self.has(key)) + + if has.isOk and has.get: + return self.store.mget(key).catch() + + return failure (ref DatastoreError)(msg: "not found") method put*( self: MemoryDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = - self.store[key] = data - return success() + withLock(self.lock): + if not self.store.hasKeyOrPut(key, data): + self.keys.add(key) + else: + self.store[key] = data method put*( self: MemoryDatastore, @@ -74,50 +100,50 @@ method put*( return success() -# proc keyIterator(self: MemoryDatastore, queryKey: string): iterator: KeyBuffer {.gcsafe.} = -# return iterator(): KeyBuffer {.closure.} = -# var keys = self.store.keys().toSeq() -# keys.sort(proc (x, y: KeyBuffer): int = cmp(x.toString, y.toString)) -# for key in keys: -# if key.toString().startsWith(queryKey): -# yield key +method query*( + self: MemoryDatastore, + query: Query, +): Future[?!QueryIter] {.async.} = -# method query*( -# self: MemoryDatastore, -# query: Query, -# ): Future[?!QueryIter] {.async.} = + let + queryKey = query.key.id() + keys = toSeq(self.keys) -# let -# queryKey = query.key.id() -# walker = keyIterator(self, queryKey) -# var -# iter = QueryIter.new() + var + iter = QueryIter.new() + pos = 0 -# proc next(): Future[?!QueryResponse] {.async.} = -# let kb = walker() + proc next(): Future[?!QueryResponse] {.async.} = + defer: + pos.inc -# if finished(walker): -# iter.finished = true -# return success (Key.none, EmptyBytes) + if iter.finished: + return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") -# let key = kb.toKey().expect("should not fail") -# var ds: ValueBuffer -# if query.value: -# ds = self.store[kb] -# let data = if ds.isNil: EmptyBytes else: ds.toSeq(byte) + if pos > keys.len - 1: + iter.finished = true + return success (Key.none, EmptyBytes) -# return success (key.some, data) + return success ( + Key.init(keys[pos]).expect("Should not fail!").some, + if query.value: self.store.mget(keys[pos]) else: EmptyBytes) -# iter.next = next -# return success iter + iter.next = next + return success iter method close*(self: MemoryDatastore): Future[?!void] {.async.} = self.store.deinitSharedTable() + self.keys.deinitSharedList() + self.lock.deinitLock() return success() proc new*(tp: type MemoryDatastore): MemoryDatastore = var table: SharedTable[Key, seq[byte]] + keys: SharedList[Key] + lock: Lock table.init() - MemoryDatastore(store: table) + keys.init() + lock.initLock() + MemoryDatastore(store: table, keys: keys, lock: lock) From 6a3882ffa5283909c385b985127676bbe3ea5a5e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:42:37 -0600 Subject: [PATCH 128/445] reverted query back (it works as is) --- datastore/query.nim | 108 +++----------------------------------------- 1 file changed, 7 insertions(+), 101 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index e2ca1189..8b920429 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -1,17 +1,14 @@ -import std/options -import std/algorithm import pkg/upraises +import std/algorithm import pkg/chronos import pkg/questionable import pkg/questionable/results import ./key import ./types -# import ./threads/databuffer export options, SortOrder type - Query* = object key*: Key # Key to be queried value*: bool # Flag to indicate if data should be returned @@ -22,49 +19,28 @@ type QueryResponse* = tuple[key: ?Key, data: seq[byte]] QueryEndedError* = object of DatastoreError - GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe, closure.} + GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe.} IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} QueryIter* = ref object finished*: bool next*: GetNext dispose*: IterDispose -proc waitForAllQueryResults*(qi: ?!QueryIter): Future[?!seq[QueryResponse]] {.async.} = - ## for large blocks this would be *expensive* - var res: seq[QueryResponse] - without iter =? qi, err: - return failure err - - while not iter.finished: - let val = await iter.next() - if val.isOk(): - let qr = val.tryGet() - if qr.key.isSome: - res.add qr - else: - return failure val.error() - - let rd = await iter.dispose() - if rd.isErr(): - return failure(rd.error()) - return success res - -proc waitForAllQueryResults*(iter: Future[?!QueryIter] - ): Future[?!seq[QueryResponse]] {.async.} = - let res = await iter - return await waitForAllQueryResults(res) +iterator items*(q: QueryIter): Future[?!QueryResponse] = + while not q.finished: + yield q.next() proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() -proc new*(T: type QueryIter, dispose = defaultDispose): T = +proc init*(T: type QueryIter, dispose = defaultDispose): T = QueryIter(dispose: dispose) proc init*( T: type Query, key: Key, value = true, - sort = Ascending, + sort = SortOrder.Ascending, offset = 0, limit = -1): T = @@ -74,73 +50,3 @@ proc init*( sort: sort, offset: offset, limit: limit) - -# type - -# QueryBuffer* = object -# key*: KeyBuffer # Key to be queried -# value*: bool # Flag to indicate if data should be returned -# limit*: int # Max items to return - not available in all backends -# offset*: int # Offset from which to start querying - not available in all backends -# sort*: SortOrder # Sort order - not available in all backends - -# QueryResponseBuffer* = object -# key*: KeyBuffer -# data*: ValueBuffer - - # GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe, closure.} - # IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} - # QueryIter* = ref object - # finished*: bool - # next*: GetNext - # dispose*: IterDispose - -# proc toBuffer*(q: Query): QueryBuffer = -# ## convert Query to thread-safe QueryBuffer -# return QueryBuffer( -# key: KeyBuffer.new(q.key), -# value: q.value, -# offset: q.offset, -# sort: q.sort -# ) - -# proc toQuery*(qb: QueryBuffer): Query = -# ## convert QueryBuffer to regular Query -# Query( -# key: qb.key.toKey().expect("key expected"), -# value: qb.value, -# limit: qb.limit, -# offset: qb.offset, -# sort: qb.sort -# ) - -# proc toBuffer*(q: QueryResponse): QueryResponseBuffer = -# ## convert QueryReponses to thread safe type -# var kb: KeyBuffer -# if q.key.isSome(): -# kb = KeyBuffer.new(q.key.get()) -# var kv: KeyBuffer -# if q.data.len() > 0: -# kv = ValueBuffer.new(q.data) - -# QueryResponseBuffer(key: kb, data: kv) - -# proc toQueryResponse*(qb: QueryResponseBuffer): QueryResponse = -# ## convert QueryReponseBuffer to regular QueryResponse -# let key = -# if qb.key.isNil: none(Key) -# else: some qb.key.toKey().expect("key response should work") -# let data = -# if qb.data.isNil: EmptyBytes -# else: qb.data.toSeq(byte) - -# (key: key, data: data) - -# proc convert*(ret: TResult[QueryResponseBuffer], -# tp: typedesc[QueryResponse] -# ): Result[QueryResponse, ref CatchableError] = -# if ret[].results.isOk(): -# result.ok(ret[].results.get().toString()) -# else: -# let exc: ref CatchableError = ret[].results.error().toCatchable() -# result.err(exc) From 88eb96edb37024fe8fc50d5e3de1171b22755265 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:43:10 -0600 Subject: [PATCH 129/445] don't use toSeq --- tests/datastore/querycommontests.nim | 110 +++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 16 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index fe111428..b18e8224 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -9,7 +9,6 @@ import pkg/stew/byteutils import pkg/datastore - template queryTests*(ds: Datastore, extended = true) {.dirty.} = var key1: Key @@ -37,13 +36,20 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet - - var res: seq[QueryResponse] - while not iter.finished: - let val = await iter.next() - let qr = val.tryGet() - if qr.key.isSome: - res.add qr + res = block: + var + res: seq[QueryResponse] + cnt = 0 + + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + cnt.inc + + res check: res.len == 3 @@ -67,8 +73,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - all = waitForAllQueryResults(await ds.query(q)) - res = tryGet(await all) + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res check: res.len == 3 @@ -81,6 +96,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[2].key.get == key3 res[2].data.len == 0 + (await iter.dispose()).tryGet test "Key should not query parent": let @@ -91,7 +107,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key3, val3)).tryGet let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res check: res.len == 2 @@ -101,6 +127,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[1].key.get == key3 res[1].data == val3 + (await iter.dispose()).tryGet + test "Key should all list all keys at the same level": let queryKey = Key.init("/a").tryGet @@ -114,7 +142,16 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = iter = (await ds.query(q)).tryGet var - res = tryGet(await ds.query(q).waitForAllQueryResults()) + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res res.sort do (a, b: QueryResponse) -> int: cmp(a.key.get.id, b.key.get.id) @@ -146,11 +183,16 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + iter = (await ds.query(q)).tryGet + res = (await allFinished(toSeq(iter))) + .mapIt( it.read.tryGet ) + .filterIt( it.key.isSome ) check: res.len == 10 + (await iter.dispose()).tryGet + test "Should not apply offset": let key = Key.init("/a").tryGet @@ -164,11 +206,23 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res check: res.len == 10 + (await iter.dispose()).tryGet + test "Should not apply offset and limit": let key = Key.init("/a").tryGet @@ -182,7 +236,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await ds.put(key, val)).tryGet let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res check: res.len == 5 @@ -196,6 +260,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res[i].key.get == key res[i].data == val + (await iter.dispose()).tryGet + test "Should apply sort order - descending": let key = Key.init("/a").tryGet @@ -216,7 +282,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = kvs = kvs.reversed let - res = tryGet(await ds.query(q).waitForAllQueryResults()) + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break + + res.add((key, val)) + + res check: res.len == 100 @@ -225,3 +301,5 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = check: res[i].key.get == kvs[i].key.get res[i].data == kvs[i].data + + (await iter.dispose()).tryGet From 6c86d2b10e6726bb0fb814e6e073bda556343554 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 13 Sep 2023 14:43:25 -0600 Subject: [PATCH 130/445] make sure all tests pass --- tests/datastore/testthreadproxyds.nim | 76 ++++++++------------------- 1 file changed, 22 insertions(+), 54 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 8ca84818..4852afa5 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -1,6 +1,7 @@ import std/options import std/sequtils import std/os +import std/cpuinfo import std/algorithm import pkg/asynctest @@ -9,55 +10,16 @@ import pkg/stew/results import pkg/stew/byteutils import pkg/taskpools -import pkg/datastore/memoryds +import pkg/datastore/sql import pkg/datastore/threads/threadproxyds import ./dscommontests -# import ./querycommontests +import ./querycommontests -import pretty - - -# suite "Test Basic ThreadProxyDatastore": -# var -# sds: ThreadDatastore -# mem: MemoryDatastore -# key1: Key -# data: seq[byte] -# taskPool: Taskpool - -# setupAll: -# mem = MemoryDatastore.new() -# taskPool = TaskPool.new(3) -# sds = ThreadDatastore.new(mem, taskPool).expect("should work") -# key1 = Key.init("/a").tryGet -# data = "value for 1".toBytes() - -# test "check put": -# echo "\n\n=== put ===" -# let res1 = await sds.put(key1, data) -# print "res1: ", res1 - -# test "check get": -# echo "\n\n=== get ===" -# let res2 = await sds.get(key1) -# check res2.get() == data -# var val = "" -# for c in res2.get(): -# val &= char(c) - -# print "get res2: ", $val - -# # echo "\n\n=== put cancel ===" -# # # let res1 = await sds.put(key1, "value for 1".toBytes()) -# # let res3 = sds.put(key1, "value for 1".toBytes()) -# # res3.cancel() -# # # print "res3: ", res3 - -suite "Test Basic ThreadProxyDatastore": +suite "Test Basic ThreadDatastore": var - memStore: MemoryDatastore + memStore: Datastore ds: ThreadDatastore key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes @@ -65,23 +27,29 @@ suite "Test Basic ThreadProxyDatastore": taskPool: Taskpool setupAll: - memStore = MemoryDatastore.new() - taskPool = Taskpool.new(2) + memStore = SQLiteDatastore.new(Memory).tryGet() + taskPool = Taskpool.new(countProcessors() * 2) ds = ThreadDatastore.new(memStore, taskPool).tryGet() teardownAll: - (await memStore.close()).get() + (await ds.close()).tryGet() + taskPool.shutdown() basicStoreTests(ds, key, bytes, otherBytes) -# suite "Test Query": -# var -# mem: MemoryDatastore -# sds: ThreadProxyDatastore +suite "Test Query ThreadDatastore": + var + mem: Datastore + ds: ThreadDatastore + taskPool: Taskpool -# setup: -# mem = MemoryDatastore.new() -# sds = newThreadProxyDatastore(mem).expect("should work") + setup: + taskPool = Taskpool.new(countProcessors() * 2) + mem = SQLiteDatastore.new(Memory).tryGet() + ds = ThreadDatastore.new(mem, taskPool).tryGet() -# queryTests(sds, false) + teardown: + (await ds.close()).tryGet() + taskPool.shutdown() + queryTests(ds, false) From f0038127c83a7191970fe186d7d701dbec182f62 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 17:56:02 -0600 Subject: [PATCH 131/445] remove asyncsemaphore --- datastore/threads/asyncsemaphore.nim | 96 ---------------------------- datastore/threads/threadproxyds.nim | 16 ++--- 2 files changed, 7 insertions(+), 105 deletions(-) delete mode 100644 datastore/threads/asyncsemaphore.nim diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim deleted file mode 100644 index ee57ee00..00000000 --- a/datastore/threads/asyncsemaphore.nim +++ /dev/null @@ -1,96 +0,0 @@ -# Nim-datastore -# Copyright (c) 2023 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -# * MIT license ([LICENSE-MIT](LICENSE-MIT)) -# at your option. -# This file may not be copied, modified, or distributed except according to -# those terms. - -{.push raises: [].} - -import sequtils -import chronos, chronicles - -# TODO: this should probably go in chronos - -logScope: - topics = "datastore semaphore" - -type - AsyncSemaphore* = ref object of RootObj - size*: int - count: int - queue: seq[Future[void]] - -proc new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = - AsyncSemaphore(size: size, count: size) - -proc `count`*(s: AsyncSemaphore): int = s.count - -proc tryAcquire*(s: AsyncSemaphore): bool = - ## Attempts to acquire a resource, if successful - ## returns true, otherwise false - ## - - if s.count > 0 and s.queue.len == 0: - s.count.dec - trace "Acquired slot", available = s.count, queue = s.queue.len - return true - -proc acquire*(s: AsyncSemaphore): Future[void] = - ## Acquire a resource and decrement the resource - ## counter. If no more resources are available, - ## the returned future will not complete until - ## the resource count goes above 0. - ## - - let fut = newFuture[void]("AsyncSemaphore.acquire") - if s.tryAcquire(): - fut.complete() - return fut - - proc cancellation(udata: pointer) {.gcsafe.} = - fut.cancelCallback = nil - if not fut.finished: - s.queue.keepItIf( it != fut ) - - fut.cancelCallback = cancellation - - s.queue.add(fut) - - trace "Queued slot", available = s.count, queue = s.queue.len - return fut - -proc forceAcquire*(s: AsyncSemaphore) = - ## ForceAcquire will always succeed, - ## creating a temporary slot if required. - ## This temporary slot will stay usable until - ## there is less `acquire`s than `release`s - s.count.dec - -proc release*(s: AsyncSemaphore) = - ## Release a resource from the semaphore, - ## by picking the first future from the queue - ## and completing it and incrementing the - ## internal resource count - ## - - doAssert(s.count <= s.size) - - if s.count < s.size: - trace "Releasing slot", available = s.count, - queue = s.queue.len - - s.count.inc - while s.queue.len > 0: - var fut = s.queue[0] - s.queue.delete(0) - if not fut.finished(): - s.count.dec - fut.complete() - break - - trace "Released slot", available = s.count, - queue = s.queue.len - return diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d3ae21ae..09f448f3 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -21,7 +21,6 @@ import ../key import ../query import ../datastore -import ./asyncsemaphore import ./databuffer type @@ -36,15 +35,17 @@ type ThreadDatastore* = ref object of Datastore tp: Taskpool ds: Datastore - semaphore: AsyncSemaphore tasks: seq[Future[void]] -template dispatchTask(self: ThreadDatastore, ctx: TaskCtx, runTask: proc): untyped = +template dispatchTask( + self: ThreadDatastore, + ctx: TaskCtx, + runTask: proc): untyped = + let fut = wait(ctx.signal) try: - await self.semaphore.acquire() runTask() self.tasks.add(fut) await fut @@ -58,8 +59,6 @@ template dispatchTask(self: ThreadDatastore, ctx: TaskCtx, runTask: proc): untyp idx != -1): self.tasks.del(idx) - self.semaphore.release() - proc hasTask( ctx: ptr TaskCtx, key: ptr Key) = @@ -223,7 +222,7 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.ds.close() -proc queryTask*( +proc queryTask( ctx: ptr TaskCtx, iter: ptr QueryIter) = @@ -303,5 +302,4 @@ func new*( success ThreadDatastore( tp: tp, - ds: ds, - semaphore: AsyncSemaphore.new(tp.numThreads - 1)) # one thread is needed for the task dispatcher + ds: ds) From af310306adab731560283da7824faab2f7cb746e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 17:56:20 -0600 Subject: [PATCH 132/445] enable sqllite threading tests --- tests/datastore/querycommontests.nim | 81 +++++++++++++++++++-------- tests/datastore/testthreadproxyds.nim | 2 +- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index b18e8224..a0f93a7f 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -41,8 +41,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res: seq[QueryResponse] cnt = 0 - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break @@ -75,13 +75,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -109,13 +113,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -143,13 +151,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = var res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -184,9 +196,20 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet - res = (await allFinished(toSeq(iter))) - .mapIt( it.read.tryGet ) - .filterIt( it.key.isSome ) + res = block: + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break + + res.add((key, val)) + cnt.inc + + res check: res.len == 10 @@ -208,13 +231,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -238,13 +265,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -284,13 +315,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 4852afa5..8c387988 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -52,4 +52,4 @@ suite "Test Query ThreadDatastore": (await ds.close()).tryGet() taskPool.shutdown() - queryTests(ds, false) + queryTests(ds, true) From 35009136429b10e4e2739365fbe95a4af17ec268 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 17:47:37 -0600 Subject: [PATCH 133/445] adding semaphore --- datastore/threads/databuffer.nim | 15 +- datastore/threads/semaphore.nim | 57 ++++ datastore/threads/threadproxyds.nim | 61 +++- tests/datastore/dscommontests.nim | 7 + tests/datastore/querycommontests.nim | 393 ++++++++++++-------------- tests/datastore/testsemaphore.nim | 70 +++++ tests/datastore/testthreadproxyds.nim | 39 +-- 7 files changed, 395 insertions(+), 247 deletions(-) create mode 100644 datastore/threads/semaphore.nim create mode 100644 tests/datastore/testsemaphore.nim diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 818b3c7d..be1f0ca4 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -19,8 +19,7 @@ proc `=destroy`*(x: var DataBufferHolder) = ## if x.buf != nil: - # when isMainModule or true: - # echo "buffer: FREE: ", repr x.buf.pointer + # echo "buffer: FREE: ", repr x.buf.pointer deallocShared(x.buf) proc len*(a: DataBuffer): int = a[].size @@ -77,14 +76,14 @@ converter toString*(data: DataBuffer): string = if data.len() > 0: copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) -converter toBuffer*(err: ref CatchableError): DataBuffer = - ## convert exception to an object with StringBuffer - ## - - return DataBuffer.new(err.msg) - proc `$`*(data: DataBuffer): string = ## convert buffer to string type using copy ## data.toString() + +converter toBuffer*(err: ref CatchableError): DataBuffer = + ## convert exception to an object with StringBuffer + ## + + return DataBuffer.new(err.msg) diff --git a/datastore/threads/semaphore.nim b/datastore/threads/semaphore.nim new file mode 100644 index 00000000..5a2e4b9c --- /dev/null +++ b/datastore/threads/semaphore.nim @@ -0,0 +1,57 @@ +import std/atomics +import std/locks + +type + Semaphore* = object + count: int + size: int + lock {.align: 64.}: Lock + cond: Cond + +func `=`*(dst: var Semaphore, src: Semaphore) {.error: "A semaphore cannot be copied".} +func `=sink`*(dst: var Semaphore, src: Semaphore) {.error: "An semaphore cannot be moved".} + +proc init*(_: type Semaphore, count: uint): Semaphore = + var + lock: Lock + cond: Cond + + lock.initLock() + cond.initCond() + + Semaphore(count: count.int, size: count.int, lock: lock, cond: cond) + +proc `=destroy`*(self: var Semaphore) = + self.lock.deinitLock() + self.cond.deinitCond() + +proc count*(self: var Semaphore): int = + self.count + +proc size*(self: var Semaphore): int = + self.size + +proc acquire*(self: var Semaphore) {.inline.} = + self.lock.acquire() + while self.count <= 0: + self.cond.wait(self.lock) + + self.count -= 1 + self.lock.release() + +proc release*(self: var Semaphore) {.inline.} = + self.lock.acquire() + if self.count <= 0: + self.count += 1 + self.cond.signal() + self.lock.release() + + doAssert not (self.count > self.size), + "Semaphore count is greather than size: " & $self.size & " count is: " & $self.count + +template withSemaphore*(self: var Semaphore, body: untyped) = + self.acquire() + try: + body + finally: + self.release() diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 09f448f3..5f58e4d3 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -21,20 +21,25 @@ import ../key import ../query import ../datastore +import ./semaphore +import ./asyncsemaphore import ./databuffer type - ThreadTypes = void | bool | SomeInteger | DataBuffer | tuple + ThreadTypes = void | bool | SomeInteger | DataBuffer | tuple | Atomic ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] TaskCtx[T: ThreadTypes] = object ds: ptr Datastore res: ptr ThreadResult[T] + semaphore: ptr Semaphore signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore tp: Taskpool ds: Datastore + # semaphore: AsyncSemaphore + semaphore: Semaphore tasks: seq[Future[void]] template dispatchTask( @@ -46,12 +51,16 @@ template dispatchTask( fut = wait(ctx.signal) try: - runTask() + # await self.semaphore.acquire() self.tasks.add(fut) + runTask() await fut if ctx.res[].isErr: - result = failure(ctx.res[].error()) + result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed + except CancelledError as exc: + echo "Cancelling future!" + raise exc finally: discard ctx.signal.close() if ( @@ -59,13 +68,17 @@ template dispatchTask( idx != -1): self.tasks.del(idx) + # self.semaphore.release() + proc hasTask( ctx: ptr TaskCtx, key: ptr Key) = defer: discard ctx[].signal.fireSync() + ctx[].semaphore[].release() + ctx[].semaphore[].acquire() without ret =? (waitFor ctx[].ds[].has(key[])).catch and res =? ret, error: ctx[].res[].err(error) @@ -76,12 +89,13 @@ proc hasTask( method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var signal = ThreadSignalPtr.new().valueOr: - return failure("Failed to create signal") + return failure(error()) res = ThreadResult[bool]() ctx = TaskCtx[bool]( ds: addr self.ds, res: addr res, + semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -93,7 +107,9 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = proc delTask(ctx: ptr TaskCtx, key: ptr Key) = defer: discard ctx[].signal.fireSync() + ctx[].semaphore[].release() + ctx[].semaphore[].acquire() without res =? (waitFor ctx[].ds[].delete(key[])).catch, error: ctx[].res[].err(error) return @@ -106,12 +122,13 @@ method delete*( var signal = ThreadSignalPtr.new().valueOr: - return failure("Failed to create signal") + return failure(error()) res = ThreadResult[void]() ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, + semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -120,7 +137,10 @@ method delete*( self.dispatchTask(ctx, runTask) return success() -method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = +method delete*( + self: ThreadDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + for key in keys: if err =? (await self.delete(key)).errorOption: return failure err @@ -130,15 +150,18 @@ method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} proc putTask( ctx: ptr TaskCtx, key: ptr Key, - data: DataBuffer, + # data: DataBuffer, + data: ptr UncheckedArray[byte], len: int) = ## run put in a thread task ## defer: discard ctx[].signal.fireSync() + ctx[].semaphore[].release() - without res =? (waitFor ctx[].ds[].put(key[], @data)).catch, error: + ctx[].semaphore[].acquire() + without res =? (waitFor ctx[].ds[].put(key[], @(data.toOpenArray(0, len - 1)))).catch, error: ctx[].res[].err(error) return @@ -151,19 +174,20 @@ method put*( var signal = ThreadSignalPtr.new().valueOr: - return failure("Failed to create signal") + return failure(error()) res = ThreadResult[void]() ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, + semaphore: addr self.semaphore, signal: signal) proc runTask() = self.tp.spawn putTask( addr ctx, unsafeAddr key, - DataBuffer.new(data), + makeUncheckedArray(baseAddr data), data.len) self.dispatchTask(ctx, runTask) @@ -187,7 +211,9 @@ proc getTask( defer: discard ctx[].signal.fireSync() + ctx[].semaphore[].release() + ctx[].semaphore[].acquire() without res =? (waitFor ctx[].ds[].get(key[])).catch and data =? res, error: ctx[].res[].err(error) @@ -201,19 +227,23 @@ method get*( var signal = ThreadSignalPtr.new().valueOr: - return failure("Failed to create signal") + return failure(error()) var res = ThreadResult[DataBuffer]() ctx = TaskCtx[DataBuffer]( ds: addr self.ds, res: addr res, + semaphore: addr self.semaphore, signal: signal) proc runTask() = self.tp.spawn getTask(addr ctx, unsafeAddr key) self.dispatchTask(ctx, runTask) + if err =? res.errorOption: + return failure err + return success(@(res.get())) method close*(self: ThreadDatastore): Future[?!void] {.async.} = @@ -228,13 +258,15 @@ proc queryTask( defer: discard ctx[].signal.fireSync() + ctx[].semaphore[].release() + ctx[].semaphore[].acquire() without ret =? (waitFor iter[].next()).catch and res =? ret, error: ctx[].res[].err(error) return if res.key.isNone: - ctx[].res[].ok((false, DataBuffer.new(), DataBuffer.new())) + ctx[].res[].ok((false, default(DataBuffer), default(DataBuffer))) return var @@ -275,6 +307,7 @@ method query*( ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( ds: addr self.ds, res: addr res, + semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -302,4 +335,6 @@ func new*( success ThreadDatastore( tp: tp, - ds: ds) + ds: ds, + # semaphore: AsyncSemaphore.new(tp.numThreads - 1)) # one thread is needed for the task dispatcher + semaphore: Semaphore.init((tp.numThreads - 1).uint)) # one thread is needed for the task dispatcher diff --git a/tests/datastore/dscommontests.nim b/tests/datastore/dscommontests.nim index 1e0353cb..d1611323 100644 --- a/tests/datastore/dscommontests.nim +++ b/tests/datastore/dscommontests.nim @@ -3,6 +3,7 @@ import std/options import pkg/asynctest import pkg/chronos import pkg/stew/results +import pkg/questionable/results import pkg/datastore @@ -56,3 +57,9 @@ proc basicStoreTests*( for k in batch: check: not (await ds.has(k)).tryGet + + # test "handle missing key": + # let key = Key.init("/missing/key").tryGet() + + # # expect(ResultFailure): + # discard (await ds.get(key)).tryGet() # non existing key diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index a0f93a7f..5f63be02 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -26,118 +26,110 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = val2 = "value for 2".toBytes val3 = "value for 3".toBytes - test "Key should query all keys and all it's children": - let - q = Query.init(key1) + # test "Key should query all keys and all it's children": + # let + # q = Query.init(key1) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data == val1 - res[1].key.get == key2 - res[1].data == val2 + # res[1].key.get == key2 + # res[1].data == val2 - res[2].key.get == key3 - res[2].data == val3 + # res[2].key.get == key3 + # res[2].data == val3 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Key should query all keys without values": - let - q = Query.init(key1, value = false) + # test "Key should query all keys without values": + # let + # q = Query.init(key1, value = false) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # res.add((key, val)) - res.add((key, val)) - cnt.inc + # res - res + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data.len == 0 - check: - res.len == 3 - res[0].key.get == key1 - res[0].data.len == 0 + # res[1].key.get == key2 + # res[1].data.len == 0 - res[1].key.get == key2 - res[1].data.len == 0 + # res[2].key.get == key3 + # res[2].data.len == 0 - res[2].key.get == key3 - res[2].data.len == 0 + # (await iter.dispose()).tryGet - (await iter.dispose()).tryGet + # test "Key should not query parent": + # let + # q = Query.init(key2) - test "Key should not query parent": - let - q = Query.init(key2) + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 - - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break - - res.add((key, val)) - cnt.inc + # res.add((key, val)) - res + # res - check: - res.len == 2 - res[0].key.get == key2 - res[0].data == val2 + # check: + # res.len == 2 + # res[0].key.get == key2 + # res[0].data == val2 - res[1].key.get == key3 - res[1].data == val3 + # res[1].key.get == key3 + # res[1].data == val3 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Key should all list all keys at the same level": + test "Key should list all keys at the same level": let queryKey = Key.init("/a").tryGet q = Query.init(queryKey) @@ -181,160 +173,145 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - if extended: - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, limit = 10) - - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes - - (await ds.put(key, val)).tryGet - - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 - - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break - - res.add((key, val)) - cnt.inc + # if extended: + # test "Should apply limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, limit = 10) - res + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - check: - res.len == 10 + # echo "putting ", $key + # (await ds.put(key, val)).tryGet - (await iter.dispose()).tryGet + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - test "Should not apply offset": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 90) + # res.add((key, val)) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # res + + # check: + # res.len == 10 - (await ds.put(key, val)).tryGet + # (await iter.dispose()).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # test "Should not apply offset": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 90) - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - res.add((key, val)) - cnt.inc + # (await ds.put(key, val)).tryGet - res + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - check: - res.len == 10 + # res.add((key, val)) - (await iter.dispose()).tryGet + # res - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 95, limit = 5) + # check: + # res.len == 10 - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # (await iter.dispose()).tryGet - (await ds.put(key, val)).tryGet + # test "Should not apply offset and limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 95, limit = 5) - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # (await ds.put(key, val)).tryGet - res.add((key, val)) - cnt.inc + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - res + # res.add((key, val)) - check: - res.len == 5 + # res - for i in 0.. int: - cmp(a.key.get.id, b.key.get.id) + # kvs.add((k.some, val)) + # (await ds.put(k, val)).tryGet - kvs = kvs.reversed - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # # lexicographic sort, as it comes from the backend + # kvs.sort do (a, b: QueryResponse) -> int: + # cmp(a.key.get.id, b.key.get.id) - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # kvs = kvs.reversed + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var res: seq[QueryResponse] + # while not iter.finished: + # let (key, val) = (await iter.next()).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) - res + # res - check: - res.len == 100 + # check: + # res.len == 100 - for i, r in res[1..^1]: - check: - res[i].key.get == kvs[i].key.get - res[i].data == kvs[i].data + # for i, r in res[1..^1]: + # check: + # res[i].key.get == kvs[i].key.get + # res[i].data == kvs[i].data - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet diff --git a/tests/datastore/testsemaphore.nim b/tests/datastore/testsemaphore.nim new file mode 100644 index 00000000..80962394 --- /dev/null +++ b/tests/datastore/testsemaphore.nim @@ -0,0 +1,70 @@ +import std/os +import std/osproc + +import pkg/unittest2 +import pkg/taskpools + +import pkg/stew/ptrops + +import pkg/datastore/threads/semaphore + +suite "Test semaphore": + + test "Should work as a mutex/lock": + var + tp = TaskPool.new(countProcessors() * 2) + lock = Semaphore.init(1) # mutex/lock + resource = 1 + count = 0 + + const numTasks = 1000 + + proc task(lock: ptr Semaphore, resource: ptr int, count: ptr int) = + lock[].acquire() + resource[] -= 1 + doAssert resource[] == 0, "resource should be 0, but it's: " & $resource[] + + resource[] += 1 + doAssert resource[] == 1, "resource should be 1, but it's: " & $resource[] + + count[] += 1 + lock[].release() + + for i in 0.. Date: Thu, 14 Sep 2023 18:05:32 -0600 Subject: [PATCH 134/445] remove asyncsemaphore --- datastore/threads/threadproxyds.nim | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 5f58e4d3..edf027c0 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -22,7 +22,6 @@ import ../query import ../datastore import ./semaphore -import ./asyncsemaphore import ./databuffer type @@ -38,8 +37,7 @@ type ThreadDatastore* = ref object of Datastore tp: Taskpool ds: Datastore - # semaphore: AsyncSemaphore - semaphore: Semaphore + semaphore: Semaphore # semaphore is used for backpressure to avoid exhausting file descriptors tasks: seq[Future[void]] template dispatchTask( @@ -51,7 +49,6 @@ template dispatchTask( fut = wait(ctx.signal) try: - # await self.semaphore.acquire() self.tasks.add(fut) runTask() await fut @@ -68,8 +65,6 @@ template dispatchTask( idx != -1): self.tasks.del(idx) - # self.semaphore.release() - proc hasTask( ctx: ptr TaskCtx, key: ptr Key) = @@ -336,5 +331,4 @@ func new*( success ThreadDatastore( tp: tp, ds: ds, - # semaphore: AsyncSemaphore.new(tp.numThreads - 1)) # one thread is needed for the task dispatcher - semaphore: Semaphore.init((tp.numThreads - 1).uint)) # one thread is needed for the task dispatcher + semaphore: Semaphore.init((tp.numThreads - 1).uint)) From e39bd530d98f2ace35a26b5da6ec60e425669214 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:05:58 -0600 Subject: [PATCH 135/445] wip --- tests/datastore/querycommontests.nim | 368 +++++++++++++------------- tests/datastore/testthreadproxyds.nim | 2 +- 2 files changed, 185 insertions(+), 185 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 5f63be02..9f305b06 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -26,108 +26,108 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = val2 = "value for 2".toBytes val3 = "value for 3".toBytes - # test "Key should query all keys and all it's children": - # let - # q = Query.init(key1) + test "Key should query all keys and all it's children": + let + q = Query.init(key1) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data == val1 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 - # res[1].key.get == key2 - # res[1].data == val2 + res[1].key.get == key2 + res[1].data == val2 - # res[2].key.get == key3 - # res[2].data == val3 + res[2].key.get == key3 + res[2].data == val3 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Key should query all keys without values": - # let - # q = Query.init(key1, value = false) + test "Key should query all keys without values": + let + q = Query.init(key1, value = false) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res + res - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data.len == 0 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 - # res[1].key.get == key2 - # res[1].data.len == 0 + res[1].key.get == key2 + res[1].data.len == 0 - # res[2].key.get == key3 - # res[2].data.len == 0 + res[2].key.get == key3 + res[2].data.len == 0 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Key should not query parent": - # let - # q = Query.init(key2) + test "Key should not query parent": + let + q = Query.init(key2) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res + res - # check: - # res.len == 2 - # res[0].key.get == key2 - # res[0].data == val2 + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 - # res[1].key.get == key3 - # res[1].data == val3 + res[1].key.get == key3 + res[1].data == val3 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet test "Key should list all keys at the same level": let @@ -173,145 +173,145 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - # if extended: - # test "Should apply limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, limit = 10) + if extended: + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, limit = 10) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # echo "putting ", $key - # (await ds.put(key, val)).tryGet + echo "putting ", $key + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res - - # check: - # res.len == 10 + res + + check: + res.len == 10 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Should not apply offset": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 90) + test "Should not apply offset": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 90) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res + res - # check: - # res.len == 10 + check: + res.len == 10 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Should not apply offset and limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 95, limit = 5) + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 95, limit = 5) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res + res - # check: - # res.len == 5 + check: + res.len == 5 - # for i in 0.. int: - # cmp(a.key.get.id, b.key.get.id) + # lexicographic sort, as it comes from the backend + kvs.sort do (a, b: QueryResponse) -> int: + cmp(a.key.get.id, b.key.get.id) - # kvs = kvs.reversed - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var res: seq[QueryResponse] - # while not iter.finished: - # let (key, val) = (await iter.next()).tryGet - # if key.isNone: - # break + kvs = kvs.reversed + let + iter = (await ds.query(q)).tryGet + res = block: + var res: seq[QueryResponse] + while not iter.finished: + let (key, val) = (await iter.next()).tryGet + if key.isNone: + break - # res.add((key, val)) + res.add((key, val)) - # res + res - # check: - # res.len == 100 + check: + res.len == 100 - # for i, r in res[1..^1]: - # check: - # res[i].key.get == kvs[i].key.get - # res[i].data == kvs[i].data + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 45434e78..c415decf 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -39,6 +39,7 @@ suite "Test Basic ThreadDatastore with SQLite": taskPool.shutdown() basicStoreTests(ds, key, bytes, otherBytes) + queryTests(ds, true) # suite "Test Basic ThreadDatastore with fsds": @@ -55,4 +56,3 @@ suite "Test Basic ThreadDatastore with SQLite": # ds: ThreadDatastore # taskPool: Taskpool - queryTests(ds, true) From 812b4fd3b671ef138caad6995556dda7d2e92d97 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:08:04 -0600 Subject: [PATCH 136/445] use foreach iterator --- tests/datastore/querycommontests.nim | 67 +++++++++++++++++++--------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 9f305b06..a0f93a7f 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -41,8 +41,8 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = res: seq[QueryResponse] cnt = 0 - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break @@ -75,13 +75,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -109,13 +113,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -129,7 +137,7 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - test "Key should list all keys at the same level": + test "Key should all list all keys at the same level": let queryKey = Key.init("/a").tryGet q = Query.init(queryKey) @@ -184,19 +192,22 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = key = Key.init(key, Key.init("/" & $i).tryGet).tryGet val = ("val " & $i).toBytes - echo "putting ", $key (await ds.put(key, val)).tryGet let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -220,13 +231,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -250,13 +265,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res @@ -296,13 +315,17 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var res: seq[QueryResponse] - while not iter.finished: - let (key, val) = (await iter.next()).tryGet + var + res: seq[QueryResponse] + cnt = 0 + + for pair in iter: + let (key, val) = (await pair).tryGet if key.isNone: break res.add((key, val)) + cnt.inc res From 6d5c4716f2c90640beb0ff0dac7347e833558fb1 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:08:22 -0600 Subject: [PATCH 137/445] add sqlite query tests --- tests/datastore/testthreadproxyds.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index c415decf..4cdb4e64 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -39,6 +39,27 @@ suite "Test Basic ThreadDatastore with SQLite": taskPool.shutdown() basicStoreTests(ds, key, bytes, otherBytes) + + +suite "Test Query ThreadDatastore with SQLite": + + var + sqlStore: Datastore + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setup: + sqlStore = SQLiteDatastore.new(Memory).tryGet() + taskPool = Taskpool.new(countProcessors() * 2) + ds = ThreadDatastore.new(sqlStore, taskPool).tryGet() + + teardown: + (await ds.close()).tryGet() + taskPool.shutdown() + queryTests(ds, true) # suite "Test Basic ThreadDatastore with fsds": From a96de2ccd5ac5e3c133d1b8a1763ffc452937a2e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:12:56 -0600 Subject: [PATCH 138/445] fix --- tests/datastore/testdatastore.nim | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/datastore/testdatastore.nim b/tests/datastore/testdatastore.nim index 149badcf..e63e2483 100644 --- a/tests/datastore/testdatastore.nim +++ b/tests/datastore/testdatastore.nim @@ -25,9 +25,6 @@ suite "Datastore (base)": test "query": expect Defect: - let + let q = Query.init(key) - all = waitForAllQueryResults(await ds.query(q)) - res = tryGet(await all) - for n in res: - discard + discard (await ds.query(q)).tryGet From 59f1f0bc61c1a21a42a4c2744c1285f3d9e0c905 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:13:07 -0600 Subject: [PATCH 139/445] add semaphore tests --- tests/testall.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/testall.nim b/tests/testall.nim index b4911446..81c3a4e8 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -6,6 +6,7 @@ import ./datastore/testtieredds, ./datastore/testmountedds, ./datastore/testmemoryds, - ./datastore/testthreadproxyds + ./datastore/testthreadproxyds, + ./datastore/testsemaphore {.warning[UnusedImport]: off.} From 25678b2be2b4c5e7021cc5c3fc873397e3d8d48f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:19:14 -0600 Subject: [PATCH 140/445] change iter constructore back to new --- datastore/query.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/query.nim b/datastore/query.nim index 8b920429..097d33dc 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -33,7 +33,7 @@ iterator items*(q: QueryIter): Future[?!QueryResponse] = proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = return success() -proc init*(T: type QueryIter, dispose = defaultDispose): T = +proc new*(T: type QueryIter, dispose = defaultDispose): T = QueryIter(dispose: dispose) proc init*( From a17e9fe1022233a74c595bf8eea750b14c4af261 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:21:24 -0600 Subject: [PATCH 141/445] remove memdb (not needed) --- datastore/memoryds.nim | 149 ---------------------------- datastore/threads/threadproxyds.nim | 5 +- 2 files changed, 3 insertions(+), 151 deletions(-) delete mode 100644 datastore/memoryds.nim diff --git a/datastore/memoryds.nim b/datastore/memoryds.nim deleted file mode 100644 index fa7d10f4..00000000 --- a/datastore/memoryds.nim +++ /dev/null @@ -1,149 +0,0 @@ -import std/tables -import std/sharedtables -import std/sharedlist -import std/sequtils -import std/strutils -import std/algorithm -import std/locks - -import std/atomics - -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import pkg/upraises - -import ./key -import ./query -import ./datastore - -export key, query - -push: {.upraises: [].} - -type - MemoryDatastore* = ref object of Datastore - store*: SharedTable[Key, seq[byte]] - keys*: SharedList[Key] - lock: Lock # yes, we need the lock since we need to update both the table and the list :facepalm: - -template withLock(lock: Lock, body: untyped) = - try: - lock.acquire() - body - finally: - lock.release() - -method has*( - self: MemoryDatastore, - key: Key): Future[?!bool] {.async.} = - let - keys = toSeq(self.keys) - - for k in keys: - if k == key: - return success true - - return success false - -method delete*( - self: MemoryDatastore, - key: Key): Future[?!void] {.async.} = - - withLock(self.lock): - self.keys.iterAndMutate(proc(k: Key): bool = k == key) - self.store.del(key) - - return success() - -method delete*( - self: MemoryDatastore, - keys: seq[Key]): Future[?!void] {.async.} = - - for key in keys: - if err =? (await self.delete(key)).errorOption: - return failure err - - return success() - -method get*( - self: MemoryDatastore, - key: Key): Future[?!seq[byte]] {.async.} = - - withLock(self.lock): - let - has = (await self.has(key)) - - if has.isOk and has.get: - return self.store.mget(key).catch() - - return failure (ref DatastoreError)(msg: "not found") - -method put*( - self: MemoryDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async.} = - - withLock(self.lock): - if not self.store.hasKeyOrPut(key, data): - self.keys.add(key) - else: - self.store[key] = data - -method put*( - self: MemoryDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = - - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err - - return success() - -method query*( - self: MemoryDatastore, - query: Query, -): Future[?!QueryIter] {.async.} = - - let - queryKey = query.key.id() - keys = toSeq(self.keys) - - var - iter = QueryIter.new() - pos = 0 - - proc next(): Future[?!QueryResponse] {.async.} = - defer: - pos.inc - - if iter.finished: - return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - - if pos > keys.len - 1: - iter.finished = true - return success (Key.none, EmptyBytes) - - return success ( - Key.init(keys[pos]).expect("Should not fail!").some, - if query.value: self.store.mget(keys[pos]) else: EmptyBytes) - - iter.next = next - return success iter - -method close*(self: MemoryDatastore): Future[?!void] {.async.} = - self.store.deinitSharedTable() - self.keys.deinitSharedList() - self.lock.deinitLock() - return success() - -proc new*(tp: type MemoryDatastore): MemoryDatastore = - var - table: SharedTable[Key, seq[byte]] - keys: SharedList[Key] - lock: Lock - - table.init() - keys.init() - lock.initLock() - MemoryDatastore(store: table, keys: keys, lock: lock) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index edf027c0..f4975154 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -37,7 +37,8 @@ type ThreadDatastore* = ref object of Datastore tp: Taskpool ds: Datastore - semaphore: Semaphore # semaphore is used for backpressure to avoid exhausting file descriptors + semaphore: Semaphore # semaphore is used for backpressure \ + # to avoid exhausting file descriptors tasks: seq[Future[void]] template dispatchTask( @@ -278,7 +279,7 @@ method query*( return failure error var - iter = QueryIter.init() + iter = QueryIter.new() let lock = newAsyncLock() proc next(): Future[?!QueryResponse] {.async.} = From b2cb1fd3714e7d18d14b25cc507ada3272330d89 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:21:33 -0600 Subject: [PATCH 142/445] remove memdb tests --- tests/datastore/testmemoryds.nim | 68 -------------------------------- tests/testall.nim | 1 - 2 files changed, 69 deletions(-) delete mode 100644 tests/datastore/testmemoryds.nim diff --git a/tests/datastore/testmemoryds.nim b/tests/datastore/testmemoryds.nim deleted file mode 100644 index 2100f69f..00000000 --- a/tests/datastore/testmemoryds.nim +++ /dev/null @@ -1,68 +0,0 @@ -import std/options -import std/sequtils -import std/os -from std/algorithm import sort, reversed - -import pkg/asynctest -import pkg/chronos -import pkg/stew/results -import pkg/stew/byteutils - -import pkg/datastore/memoryds - -import ./dscommontests -import ./querycommontests - -suite "Test Basic MemoryDatastore": - let - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - var - memStore: MemoryDatastore - - setupAll: - memStore = MemoryDatastore.new() - - basicStoreTests(memStore, key, bytes, otherBytes) - -suite "Test Misc MemoryDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - bytes = "some bytes".toBytes - - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - teardown: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - -suite "Test Query": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - - var - ds: MemoryDatastore - - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - ds = MemoryDatastore.new() - - teardown: - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - queryTests(ds, false) diff --git a/tests/testall.nim b/tests/testall.nim index 81c3a4e8..6b57c297 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -5,7 +5,6 @@ import ./datastore/testsql, ./datastore/testtieredds, ./datastore/testmountedds, - ./datastore/testmemoryds, ./datastore/testthreadproxyds, ./datastore/testsemaphore From 0fde28a994973c48147abf7ac761cf45e9a2a4ba Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:23:37 -0600 Subject: [PATCH 143/445] move assert into lock --- datastore/threads/semaphore.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/threads/semaphore.nim b/datastore/threads/semaphore.nim index 5a2e4b9c..5087e3f9 100644 --- a/datastore/threads/semaphore.nim +++ b/datastore/threads/semaphore.nim @@ -44,11 +44,12 @@ proc release*(self: var Semaphore) {.inline.} = if self.count <= 0: self.count += 1 self.cond.signal() - self.lock.release() doAssert not (self.count > self.size), "Semaphore count is greather than size: " & $self.size & " count is: " & $self.count + self.lock.release() + template withSemaphore*(self: var Semaphore, body: untyped) = self.acquire() try: From 71c704f0f573c4a7b3f4ea84f1624f2dc6f0379f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:25:16 -0600 Subject: [PATCH 144/445] adding todo --- tests/datastore/testthreadproxyds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 4cdb4e64..a0f7ddfa 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -62,6 +62,7 @@ suite "Test Query ThreadDatastore with SQLite": queryTests(ds, true) +# TODO: needs a per key lock to work # suite "Test Basic ThreadDatastore with fsds": # let From 600dca61482e4b2f98e0ac4cb1bc373ebc7e1ac7 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:34:20 -0600 Subject: [PATCH 145/445] adding serialization to query iter --- datastore/fsds.nim | 13 +++++++++++++ datastore/sql/sqliteds.nim | 12 +++++++++++- datastore/threads/threadproxyds.nim | 3 +++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 163a52ab..5526d5cd 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -188,10 +188,23 @@ method query*( var iter = QueryIter.new() + let lock = newAsyncLock() proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() + + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") + let path = walker() + if iter.finished: + return failure "iterator is finished" + + await lock.acquire() + if finished(walker): iter.finished = true return success (Key.none, EmptyBytes) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index aa632747..e5fad22d 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -151,9 +151,19 @@ method query*( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) + let lock = newAsyncLock() proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() + + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") + if iter.finished: - return failure(newException(QueryEndedError, "Calling next on a finished query!")) + return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) + + await lock.acquire() let v = sqlite3_step(s) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index f4975154..673e674c 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -287,6 +287,9 @@ method query*( if lock.locked: lock.release() + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") + if iter.finished == true: return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") From 783ecc3ab3777c72f2a36b8144ddf144def8d74c Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:42:58 -0600 Subject: [PATCH 146/445] fix databuffer tests --- tests/datastore/testdatabuffer.nim | 37 ++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 92d8e1f3..7fc3ceba 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -7,8 +7,9 @@ import pkg/stew/byteutils import pkg/unittest2 import pkg/questionable import pkg/questionable/results +import pkg/datastore/key -include ../../datastore/databuffer +include ../../datastore/threads/databuffer var shareVal: DataBuffer @@ -44,7 +45,7 @@ proc thread2(val: int) {.thread.} = echo "thread2: receiving " let msg: DataBuffer = shareVal echo "thread2: received: ", msg - check msg.toSeq(char) == @"hello world" + check string.fromBytes(msg.toSeq()) == "hello world" # os.sleep(100) proc runBasicTest() = @@ -60,13 +61,19 @@ proc runBasicTest() = joinThreads(threads) suite "Share buffer test": + var + k1: Key + k2: Key + a: DataBuffer + b: DataBuffer + c: DataBuffer setup: - let k1 {.used.} = Key.init("/a/b").get() - let k2 {.used.} = Key.init("/a").get() - let a {.used.} = KeyBuffer.new(k1) - let b {.used.} = KeyBuffer.new(Key.init("/a/b").get) - let c {.used.} = KeyBuffer.new(k2) + k1 = Key.init("/a/b").tryGet() + k2 = Key.init("/a").tryGet() + a = DataBuffer.new($k1) + b = DataBuffer.new($Key.init("/a/b").tryGet()) + c = DataBuffer.new($k2) test "creation": let db = DataBuffer.new("abc") @@ -74,21 +81,27 @@ suite "Share buffer test": check db[].buf[0].char == 'a' check db[].buf[1].char == 'b' check db[].buf[2].char == 'c' + test "equality": check a == b + test "toString": - check a.toString() == "/a/b" + check $a == "/a/b" + test "hash": check a.hash() == b.hash() + test "hashes differ": check a.hash() != c.hash() + test "key conversion": - check a.toKey().get() == k1 + check Key.init(a).tryGet == k1 + test "seq conversion": - check a.toSeq(char) == @"/a/b" + check string.fromBytes(a.toSeq()) == "/a/b" + test "seq conversion": - check a.toSeq(byte) == "/a/b".toBytes + check a.toSeq() == "/a/b".toBytes test "basic thread test": runBasicTest() - From 7f6d95bc2903aa8598e45766cbc51a61b2805224 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:43:34 -0600 Subject: [PATCH 147/445] adding databuffer tests --- tests/testall.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testall.nim b/tests/testall.nim index 6b57c297..153261b4 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -5,6 +5,7 @@ import ./datastore/testsql, ./datastore/testtieredds, ./datastore/testmountedds, + ./datastore/testdatabuffer, ./datastore/testthreadproxyds, ./datastore/testsemaphore From 20d7234d509ebeda57b087d9fca3da24c7ae227d Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 14 Sep 2023 18:51:50 -0600 Subject: [PATCH 148/445] re-enable missing key check --- tests/datastore/dscommontests.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/datastore/dscommontests.nim b/tests/datastore/dscommontests.nim index d1611323..72a7394c 100644 --- a/tests/datastore/dscommontests.nim +++ b/tests/datastore/dscommontests.nim @@ -58,8 +58,8 @@ proc basicStoreTests*( for k in batch: check: not (await ds.has(k)).tryGet - # test "handle missing key": - # let key = Key.init("/missing/key").tryGet() + test "handle missing key": + let key = Key.init("/missing/key").tryGet() - # # expect(ResultFailure): - # discard (await ds.get(key)).tryGet() # non existing key + # TODO: map error correctly from threadproxy + check (await ds.get(key)).isErr() # non existing key From 1713c7674caf2ec3941b090ad5f7e93ec8e29476 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 11:53:20 -0600 Subject: [PATCH 149/445] adding back async semaphore --- datastore/threads/asyncsemaphore.nim | 96 ++++++++++++ tests/datastore/testasyncsemaphore.nim | 206 +++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 datastore/threads/asyncsemaphore.nim create mode 100644 tests/datastore/testasyncsemaphore.nim diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim new file mode 100644 index 00000000..78f768aa --- /dev/null +++ b/datastore/threads/asyncsemaphore.nim @@ -0,0 +1,96 @@ +# Nim-Libp2p +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +{.push raises: [].} + +import sequtils +import chronos, chronicles + +# TODO: this should probably go in chronos + +logScope: + topics = "libp2p semaphore" + +type + AsyncSemaphore* = ref object of RootObj + size*: int + count: int + queue: seq[Future[void]] + +func new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = + AsyncSemaphore(size: size, count: size) + +proc `count`*(s: AsyncSemaphore): int = s.count + +proc tryAcquire*(s: AsyncSemaphore): bool = + ## Attempts to acquire a resource, if successful + ## returns true, otherwise false + ## + + if s.count > 0 and s.queue.len == 0: + s.count.dec + trace "Acquired slot", available = s.count, queue = s.queue.len + return true + +proc acquire*(s: AsyncSemaphore): Future[void] = + ## Acquire a resource and decrement the resource + ## counter. If no more resources are available, + ## the returned future will not complete until + ## the resource count goes above 0. + ## + + let fut = newFuture[void]("AsyncSemaphore.acquire") + if s.tryAcquire(): + fut.complete() + return fut + + proc cancellation(udata: pointer) {.gcsafe.} = + fut.cancelCallback = nil + if not fut.finished: + s.queue.keepItIf( it != fut ) + + fut.cancelCallback = cancellation + + s.queue.add(fut) + + trace "Queued slot", available = s.count, queue = s.queue.len + return fut + +proc forceAcquire*(s: AsyncSemaphore) = + ## ForceAcquire will always succeed, + ## creating a temporary slot if required. + ## This temporary slot will stay usable until + ## there is less `acquire`s than `release`s + s.count.dec + +proc release*(s: AsyncSemaphore) = + ## Release a resource from the semaphore, + ## by picking the first future from the queue + ## and completing it and incrementing the + ## internal resource count + ## + + doAssert(s.count <= s.size) + + if s.count < s.size: + trace "Releasing slot", available = s.count, + queue = s.queue.len + + s.count.inc + while s.queue.len > 0: + var fut = s.queue[0] + s.queue.delete(0) + if not fut.finished(): + s.count.dec + fut.complete() + break + + trace "Released slot", available = s.count, + queue = s.queue.len + return diff --git a/tests/datastore/testasyncsemaphore.nim b/tests/datastore/testasyncsemaphore.nim new file mode 100644 index 00000000..b8688df6 --- /dev/null +++ b/tests/datastore/testasyncsemaphore.nim @@ -0,0 +1,206 @@ +{.used.} + +# Nim-Libp2p +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import random +import chronos + +import ../libp2p/utils/semaphore + +import ./helpers + +randomize() + +suite "AsyncSemaphore": + asyncTest "should acquire": + let sema = newAsyncSemaphore(3) + + await sema.acquire() + await sema.acquire() + await sema.acquire() + + check sema.count == 0 + + asyncTest "should release": + let sema = newAsyncSemaphore(3) + + await sema.acquire() + await sema.acquire() + await sema.acquire() + + check sema.count == 0 + sema.release() + sema.release() + sema.release() + check sema.count == 3 + + asyncTest "should queue acquire": + let sema = newAsyncSemaphore(1) + + await sema.acquire() + let fut = sema.acquire() + + check sema.count == 0 + sema.release() + sema.release() + check sema.count == 1 + + await sleepAsync(10.millis) + check fut.finished() + + asyncTest "should keep count == size": + let sema = newAsyncSemaphore(1) + sema.release() + sema.release() + sema.release() + check sema.count == 1 + + asyncTest "should tryAcquire": + let sema = newAsyncSemaphore(1) + await sema.acquire() + check sema.tryAcquire() == false + + asyncTest "should tryAcquire and acquire": + let sema = newAsyncSemaphore(4) + check sema.tryAcquire() == true + check sema.tryAcquire() == true + check sema.tryAcquire() == true + check sema.tryAcquire() == true + check sema.count == 0 + + let fut = sema.acquire() + check fut.finished == false + check sema.count == 0 + + sema.release() + sema.release() + sema.release() + sema.release() + sema.release() + + check fut.finished == true + check sema.count == 4 + + asyncTest "should restrict resource access": + let sema = newAsyncSemaphore(3) + var resource = 0 + + proc task() {.async.} = + try: + await sema.acquire() + resource.inc() + check resource > 0 and resource <= 3 + let sleep = rand(0..10).millis + # echo sleep + await sleepAsync(sleep) + finally: + resource.dec() + sema.release() + + var tasks: seq[Future[void]] + for i in 0..<10: + tasks.add(task()) + + await allFutures(tasks) + + asyncTest "should cancel sequential semaphore slot": + let sema = newAsyncSemaphore(1) + + await sema.acquire() + + let + tmp = sema.acquire() + tmp2 = sema.acquire() + check: + not tmp.finished() + not tmp2.finished() + + tmp.cancel() + sema.release() + + check tmp2.finished() + + sema.release() + + check await sema.acquire().withTimeout(10.millis) + + asyncTest "should handle out of order cancellations": + let sema = newAsyncSemaphore(1) + + await sema.acquire() # 1st acquire + let tmp1 = sema.acquire() # 2nd acquire + check not tmp1.finished() + + let tmp2 = sema.acquire() # 3rd acquire + check not tmp2.finished() + + let tmp3 = sema.acquire() # 4th acquire + check not tmp3.finished() + + # up to this point, we've called acquire 4 times + tmp1.cancel() # 1st release (implicit) + tmp2.cancel() # 2nd release (implicit) + + check not tmp3.finished() # check that we didn't release the wrong slot + + sema.release() # 3rd release (explicit) + check tmp3.finished() + + sema.release() # 4th release + check await sema.acquire().withTimeout(10.millis) + + asyncTest "should properly handle timeouts and cancellations": + let sema = newAsyncSemaphore(1) + + await sema.acquire() + check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel + sema.release() + + check await sema.acquire().withTimeout(10.millis) + + asyncTest "should handle forceAcquire properly": + let sema = newAsyncSemaphore(1) + + await sema.acquire() + check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel + + let + fut1 = sema.acquire() + fut2 = sema.acquire() + + sema.forceAcquire() + sema.release() + + await fut1 or fut2 or sleepAsync(1.millis) + check: + fut1.finished() + not fut2.finished() + + sema.release() + await fut1 or fut2 or sleepAsync(1.millis) + check: + fut1.finished() + fut2.finished() + + + sema.forceAcquire() + sema.forceAcquire() + + let + fut3 = sema.acquire() + fut4 = sema.acquire() + fut5 = sema.acquire() + sema.release() + sema.release() + await sleepAsync(1.millis) + check: + fut3.finished() + fut4.finished() + not fut5.finished() From d151c01cd810a4066127f216c3ab324b99e620ca Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 13:08:38 -0600 Subject: [PATCH 150/445] enable cancellations --- datastore/threads/threadproxyds.nim | 196 +++++++++++++++++++------- tests/datastore/testthreadproxyds.nim | 84 +++++++++++ 2 files changed, 226 insertions(+), 54 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 673e674c..56c818a2 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -8,6 +8,7 @@ push: {.upraises: [].} import std/atomics import std/strutils +import std/tables import pkg/chronos import pkg/chronos/threadsync @@ -16,30 +17,36 @@ import pkg/questionable/results import pkg/stew/ptrops import pkg/taskpools import pkg/stew/byteutils +import pkg/chronicles import ../key import ../query import ../datastore -import ./semaphore +import ./asyncsemaphore import ./databuffer type + ErrorEnum {.pure.} = enum + DatastoreErr, DatastoreKeyNotFoundErr, CatchableErr + ThreadTypes = void | bool | SomeInteger | DataBuffer | tuple | Atomic ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] TaskCtx[T: ThreadTypes] = object ds: ptr Datastore res: ptr ThreadResult[T] - semaphore: ptr Semaphore + cancelled: bool + semaphore: AsyncSemaphore signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore tp: Taskpool ds: Datastore - semaphore: Semaphore # semaphore is used for backpressure \ - # to avoid exhausting file descriptors + semaphore: AsyncSemaphore # semaphore is used for backpressure \ + # to avoid exhausting file descriptors tasks: seq[Future[void]] + locks: Table[Key, AsyncLock] template dispatchTask( self: ThreadDatastore, @@ -57,7 +64,9 @@ template dispatchTask( if ctx.res[].isErr: result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed except CancelledError as exc: - echo "Cancelling future!" + trace "Cancelling future!", exc = exc.msg + ctx.cancelled = true + await ctx.signal.fire() raise exc finally: discard ctx.signal.close() @@ -66,23 +75,54 @@ template dispatchTask( idx != -1): self.tasks.del(idx) -proc hasTask( - ctx: ptr TaskCtx, - key: ptr Key) = +proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = + ## Monitor the signal and cancel the future if + ## the cancellation flag is set + ## + + try: + await ctx[].signal.wait() + trace "Received signal" + + if ctx[].cancelled: # there could eventually be other flags + trace "Cancelling future" + if not fut.finished: + await fut.cancelAndWait() # cancel the `has` future + + discard ctx[].signal.fireSync() + except CatchableError as exc: + trace "Exception in thread signal monitor", exc = exc.msg + ctx[].res[].err(exc) + discard ctx[].signal.fireSync() +proc asyncHasTask( + ctx: ptr TaskCtx[bool], + key: ptr Key) {.async.} = defer: discard ctx[].signal.fireSync() - ctx[].semaphore[].release() - ctx[].semaphore[].acquire() - without ret =? - (waitFor ctx[].ds[].has(key[])).catch and res =? ret, error: + let + fut = ctx[].ds[].has(key[]) + + asyncSpawn signalMonitor(ctx, fut) + without ret =? (await fut).catch and res =? ret, error: ctx[].res[].err(error) return ctx[].res[].ok(res) +proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = + try: + waitFor asyncHasTask(ctx, key) + except CatchableError as exc: + raiseAssert exc.msg + method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = + defer: + self.semaphore.release() + + await self.semaphore.acquire() + var signal = ThreadSignalPtr.new().valueOr: return failure(error()) @@ -91,7 +131,6 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = ctx = TaskCtx[bool]( ds: addr self.ds, res: addr res, - semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -100,21 +139,34 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = self.dispatchTask(ctx, runTask) return success(res.get()) -proc delTask(ctx: ptr TaskCtx, key: ptr Key) = +proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = defer: discard ctx[].signal.fireSync() - ctx[].semaphore[].release() - ctx[].semaphore[].acquire() - without res =? (waitFor ctx[].ds[].delete(key[])).catch, error: + let + fut = ctx[].ds[].delete(key[]) + + asyncSpawn signalMonitor(ctx, fut) + without res =? (await fut).catch, error: ctx[].res[].err(error) return ctx[].res[].ok() + return + +proc delTask(ctx: ptr TaskCtx, key: ptr Key) = + try: + waitFor asyncDelTask(ctx, key) + except CatchableError as exc: + raiseAssert exc.msg method delete*( self: ThreadDatastore, key: Key): Future[?!void] {.async.} = + defer: + self.semaphore.release() + + await self.semaphore.acquire() var signal = ThreadSignalPtr.new().valueOr: @@ -124,7 +176,6 @@ method delete*( ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, - semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -143,30 +194,45 @@ method delete*( return success() -proc putTask( - ctx: ptr TaskCtx, +proc asyncPutTask( + ctx: ptr TaskCtx[void], key: ptr Key, - # data: DataBuffer, data: ptr UncheckedArray[byte], - len: int) = - ## run put in a thread task - ## - + len: int) {.async.} = defer: discard ctx[].signal.fireSync() - ctx[].semaphore[].release() - ctx[].semaphore[].acquire() - without res =? (waitFor ctx[].ds[].put(key[], @(data.toOpenArray(0, len - 1)))).catch, error: + let + fut = ctx[].ds[].put(key[], @(data.toOpenArray(0, len - 1))) + + asyncSpawn signalMonitor(ctx, fut) + without res =? (await fut).catch, error: ctx[].res[].err(error) return ctx[].res[].ok() +proc putTask( + ctx: ptr TaskCtx, + key: ptr Key, + data: ptr UncheckedArray[byte], + len: int) = + ## run put in a thread task + ## + + try: + waitFor asyncPutTask(ctx, key, data, len) + except CatchableError as exc: + raiseAssert exc.msg + method put*( self: ThreadDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = + defer: + self.semaphore.release() + + await self.semaphore.acquire() var signal = ThreadSignalPtr.new().valueOr: @@ -176,7 +242,6 @@ method put*( ctx = TaskCtx[void]( ds: addr self.ds, res: addr res, - semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -199,27 +264,41 @@ method put*( return success() -proc getTask( - ctx: ptr TaskCtx, - key: ptr Key) = - ## Run get in a thread task - ## - +proc asyncGetTask( + ctx: ptr TaskCtx[DataBuffer], + key: ptr Key) {.async.} = defer: discard ctx[].signal.fireSync() - ctx[].semaphore[].release() - ctx[].semaphore[].acquire() + let + fut = ctx[].ds[].get(key[]) + + asyncSpawn signalMonitor(ctx, fut) without res =? - (waitFor ctx[].ds[].get(key[])).catch and data =? res, error: + (waitFor fut).catch and data =? res, error: ctx[].res[].err(error) return ctx[].res[].ok(DataBuffer.new(data)) +proc getTask( + ctx: ptr TaskCtx, + key: ptr Key) = + ## Run get in a thread task + ## + + try: + waitFor asyncGetTask(ctx, key) + except CatchableError as exc: + raiseAssert exc.msg + method get*( self: ThreadDatastore, key: Key): Future[?!seq[byte]] {.async.} = + defer: + self.semaphore.release() + + await self.semaphore.acquire() var signal = ThreadSignalPtr.new().valueOr: @@ -230,7 +309,6 @@ method get*( ctx = TaskCtx[DataBuffer]( ds: addr self.ds, res: addr res, - semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -248,16 +326,17 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.ds.close() -proc queryTask( +proc asyncQueryTask( ctx: ptr TaskCtx, - iter: ptr QueryIter) = - + iter: ptr QueryIter) {.async.} = defer: discard ctx[].signal.fireSync() - ctx[].semaphore[].release() - ctx[].semaphore[].acquire() - without ret =? (waitFor iter[].next()).catch and res =? ret, error: + let + fut = iter[].next() + + asyncSpawn signalMonitor(ctx, fut) + without ret =? (waitFor fut).catch and res =? ret, error: ctx[].res[].err(error) return @@ -271,30 +350,40 @@ proc queryTask( ctx[].res[].ok((true, keyBuf, dataBuf)) +proc queryTask( + ctx: ptr TaskCtx, + iter: ptr QueryIter) = + + try: + waitFor asyncQueryTask(ctx, iter) + except CatchableError as exc: + raiseAssert exc.msg + method query*( self: ThreadDatastore, query: Query): Future[?!QueryIter] {.async.} = - without var childIter =? await self.ds.query(query), error: return failure error var iter = QueryIter.new() + locked = false - let lock = newAsyncLock() proc next(): Future[?!QueryResponse] {.async.} = defer: - if lock.locked: - lock.release() + locked = false + self.semaphore.release() + + await self.semaphore.acquire() - if lock.locked: + if locked: return failure (ref DatastoreError)(msg: "Should always await query features") + locked = true + if iter.finished == true: return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - await lock.acquire() - if iter.finished == true: return success (Key.none, EmptyBytes) @@ -306,7 +395,6 @@ method query*( ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( ds: addr self.ds, res: addr res, - semaphore: addr self.semaphore, signal: signal) proc runTask() = @@ -335,4 +423,4 @@ func new*( success ThreadDatastore( tp: tp, ds: ds, - semaphore: Semaphore.init((tp.numThreads - 1).uint)) + semaphore: AsyncSemaphore.new(tp.numThreads - 1)) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index a0f7ddfa..dac08fad 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -7,10 +7,12 @@ import std/importutils import pkg/asynctest import pkg/chronos +import pkg/chronos/threadsync import pkg/stew/results import pkg/stew/byteutils import pkg/taskpools import pkg/questionable/results +import pkg/chronicles import pkg/datastore/sql import pkg/datastore/fsds @@ -78,3 +80,85 @@ suite "Test Query ThreadDatastore with SQLite": # ds: ThreadDatastore # taskPool: Taskpool +suite "Test ThreadDatastore": + var + sqlStore: Datastore + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + privateAccess(ThreadDatastore) # expose private fields + privateAccess(TaskCtx) # expose private fields + + setupAll: + sqlStore = SQLiteDatastore.new(Memory).tryGet() + taskPool = Taskpool.new(countProcessors() * 2) + ds = ThreadDatastore.new(sqlStore, taskPool).tryGet() + + test "should monitor signal for cancellations and cancel": + var + signal = ThreadSignalPtr.new().tryGet() + res = ThreadResult[void]() + ctx = TaskCtx[void]( + ds: addr sqlStore, + res: addr res, + signal: signal) + fut = newFuture[void]("signalMonitor") + threadArgs = (addr ctx, addr fut) + + var + thread: Thread[type threadArgs] + + proc threadTask(args: type threadArgs) = + var (ctx, fut) = args + proc asyncTask() {.async.} = + let + monitor = signalMonitor(ctx, fut[]) + + await monitor + + waitFor asyncTask() + + createThread(thread, threadTask, threadArgs) + ctx.cancelled = true + check: ctx.signal.fireSync.tryGet + + joinThreads(thread) + + check: fut.cancelled + check: ctx.signal.close().isOk + + test "should monitor signal for cancellations and not cancel": + var + signal = ThreadSignalPtr.new().tryGet() + res = ThreadResult[void]() + ctx = TaskCtx[void]( + ds: addr sqlStore, + res: addr res, + signal: signal) + fut = newFuture[void]("signalMonitor") + threadArgs = (addr ctx, addr fut) + + var + thread: Thread[type threadArgs] + + proc threadTask(args: type threadArgs) = + var (ctx, fut) = args + proc asyncTask() {.async.} = + let + monitor = signalMonitor(ctx, fut[]) + + await monitor + + waitFor asyncTask() + + createThread(thread, threadTask, threadArgs) + ctx.cancelled = false + check: ctx.signal.fireSync.tryGet + + joinThreads(thread) + + check: not fut.cancelled + check: ctx.signal.close().isOk From 9e27eec291f5027b6d8c27db0eea21825dcbe7e2 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 13:26:10 -0600 Subject: [PATCH 151/445] adding chronicles and generating lock file --- datastore.nimble | 3 +- nimble.lock | 214 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 nimble.lock diff --git a/datastore.nimble b/datastore.nimble index df032c4e..3b5b28ef 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -16,7 +16,8 @@ requires "nim >= 1.6.14", "pretty", "threading", "taskpools", - "upraises >= 0.1.0 & < 0.2.0" + "upraises >= 0.1.0 & < 0.2.0", + "chronicles" task coverage, "generates code coverage report": var (output, exitCode) = gorgeEx("which lcov") diff --git a/nimble.lock b/nimble.lock new file mode 100644 index 00000000..3a208e32 --- /dev/null +++ b/nimble.lock @@ -0,0 +1,214 @@ +{ + "version": 2, + "packages": { + "upraises": { + "version": "0.1.0", + "vcsRevision": "d9f268db1021959fe0f2c7a5e49fba741f9932a0", + "url": "https://github.com/markspanbroek/upraises", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "176234f808b44a0be763df706ed634d6e8df17bb" + } + }, + "results": { + "version": "0.4.0", + "vcsRevision": "f3c666a272c69d70cb41e7245e7f6844797303ad", + "url": "https://github.com/arnetheduck/nim-results", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "51e08ca9524db98dc909fb39192272cc2b5451c7" + } + }, + "unittest2": { + "version": "0.1.0", + "vcsRevision": "2300fa9924a76e6c96bc4ea79d043e3a0f27120c", + "url": "https://github.com/status-im/nim-unittest2", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "914cf9a380c83b2ae40697981e5d94903505e87e" + } + }, + "stew": { + "version": "0.1.0", + "vcsRevision": "3159137d9a3110edb4024145ce0ba778975de40e", + "url": "https://github.com/status-im/nim-stew", + "downloadMethod": "git", + "dependencies": [ + "results", + "unittest2" + ], + "checksums": { + "sha1": "4ab494e272e997011853faddebe9e55183613776" + } + }, + "faststreams": { + "version": "0.3.0", + "vcsRevision": "720fc5e5c8e428d9d0af618e1e27c44b42350309", + "url": "https://github.com/status-im/nim-faststreams", + "downloadMethod": "git", + "dependencies": [ + "stew", + "unittest2" + ], + "checksums": { + "sha1": "ab178ba25970b95d953434b5d86b4d60396ccb64" + } + }, + "serialization": { + "version": "0.2.0", + "vcsRevision": "4bdbc29e54fe54049950e352bb969aab97173b35", + "url": "https://github.com/status-im/nim-serialization", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "unittest2", + "stew" + ], + "checksums": { + "sha1": "c8c99a387aae488e7008aded909ebfe662e74450" + } + }, + "sqlite3_abi": { + "version": "3.40.1.1", + "vcsRevision": "362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "8e91db8156a82383d9c48f53b33e48f4e93077b1" + } + }, + "testutils": { + "version": "0.5.0", + "vcsRevision": "dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34", + "url": "https://github.com/status-im/nim-testutils", + "downloadMethod": "git", + "dependencies": [ + "unittest2" + ], + "checksums": { + "sha1": "756d0757c4dd06a068f9d38c7f238576ba5ee897" + } + }, + "json_serialization": { + "version": "0.1.5", + "vcsRevision": "85b7ea093cb85ee4f433a617b97571bd709d30df", + "url": "https://github.com/status-im/nim-json-serialization", + "downloadMethod": "git", + "dependencies": [ + "serialization", + "stew" + ], + "checksums": { + "sha1": "c6b30565292acf199b8be1c62114726e354af59e" + } + }, + "chronicles": { + "version": "0.10.3", + "vcsRevision": "32ac8679680ea699f7dbc046e8e0131cac97d41a", + "url": "https://github.com/status-im/nim-chronicles", + "downloadMethod": "git", + "dependencies": [ + "testutils", + "json_serialization" + ], + "checksums": { + "sha1": "79f09526d4d9b9196dd2f6a75310d71a890c4f88" + } + }, + "asynctest": { + "version": "0.3.2", + "vcsRevision": "a236a5f0f3031573ac2cb082b63dbf6e170e06e7", + "url": "https://github.com/markspanbroek/asynctest", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "0ef50d086659835b0a23a4beb77cb11747695448" + } + }, + "httputils": { + "version": "0.3.0", + "vcsRevision": "87b7cbf032c90b9e6b446081f4a647e950362cec", + "url": "https://github.com/status-im/nim-http-utils", + "downloadMethod": "git", + "dependencies": [ + "stew", + "unittest2" + ], + "checksums": { + "sha1": "72a138157a9951f0986a9c4afc8c9a83ce3979a8" + } + }, + "taskpools": { + "version": "0.0.4", + "vcsRevision": "b3673c7a7a959ccacb393bd9b47e997bbd177f5a", + "url": "https://github.com/status-im/nim-taskpools", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "e43c5170d4e9ef1b27dd0956ffa0db46f992f9a6" + } + }, + "pretty": { + "version": "0.1.0", + "vcsRevision": "60850f8c595d4f0b9e55f3a99c7c471244a3182a", + "url": "https://github.com/treeform/pretty", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "b0a1cf8607d169050acd2bdca8d76fe850c23648" + } + }, + "bearssl": { + "version": "0.2.1", + "vcsRevision": "e4157639db180e52727712a47deaefcbbac6ec86", + "url": "https://github.com/status-im/nim-bearssl", + "downloadMethod": "git", + "dependencies": [ + "unittest2" + ], + "checksums": { + "sha1": "a5086fd5c0af2b852f34c0cc6e4cff93a98f97ec" + } + }, + "chronos": { + "version": "3.2.0", + "vcsRevision": "0277b65be2c7a365ac13df002fba6e172be55537", + "url": "https://github.com/status-im/nim-chronos", + "downloadMethod": "git", + "dependencies": [ + "stew", + "bearssl", + "httputils", + "unittest2" + ], + "checksums": { + "sha1": "78a41db7fb05b937196d4fa2f1e3fb4353b36a07" + } + }, + "questionable": { + "version": "0.10.10", + "vcsRevision": "b3cf35ac450fd42c9ea83dc084f5cba2efc55da3", + "url": "https://github.com/markspanbroek/questionable", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "8bb23a05d7f21619010471aa009e27d3fa73d93a" + } + }, + "threading": { + "version": "0.2.0", + "vcsRevision": "bcc284991ba928d1ed299a81a93b7113cc7de04f", + "url": "https://github.com/nim-lang/threading", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "08dfc46cedc3fc202e5de6ef80a655598331eb89" + } + } + }, + "tasks": {} +} From f6acaa6f322e5a023a81951a518f8c31ddb4eddc Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 13:29:57 -0600 Subject: [PATCH 152/445] add async semaphore tests --- tests/datastore/testasyncsemaphore.nim | 50 +++++++++++++------------- tests/testall.nim | 1 + 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/datastore/testasyncsemaphore.nim b/tests/datastore/testasyncsemaphore.nim index b8688df6..3e5cb318 100644 --- a/tests/datastore/testasyncsemaphore.nim +++ b/tests/datastore/testasyncsemaphore.nim @@ -10,17 +10,17 @@ # those terms. import random -import chronos -import ../libp2p/utils/semaphore +import pkg/chronos +import pkg/asynctest -import ./helpers +import pkg/datastore/threads/asyncsemaphore randomize() suite "AsyncSemaphore": - asyncTest "should acquire": - let sema = newAsyncSemaphore(3) + test "should acquire": + let sema = AsyncSemaphore.new(3) await sema.acquire() await sema.acquire() @@ -28,8 +28,8 @@ suite "AsyncSemaphore": check sema.count == 0 - asyncTest "should release": - let sema = newAsyncSemaphore(3) + test "should release": + let sema = AsyncSemaphore.new(3) await sema.acquire() await sema.acquire() @@ -41,8 +41,8 @@ suite "AsyncSemaphore": sema.release() check sema.count == 3 - asyncTest "should queue acquire": - let sema = newAsyncSemaphore(1) + test "should queue acquire": + let sema = AsyncSemaphore.new(1) await sema.acquire() let fut = sema.acquire() @@ -55,20 +55,20 @@ suite "AsyncSemaphore": await sleepAsync(10.millis) check fut.finished() - asyncTest "should keep count == size": - let sema = newAsyncSemaphore(1) + test "should keep count == size": + let sema = AsyncSemaphore.new(1) sema.release() sema.release() sema.release() check sema.count == 1 - asyncTest "should tryAcquire": - let sema = newAsyncSemaphore(1) + test "should tryAcquire": + let sema = AsyncSemaphore.new(1) await sema.acquire() check sema.tryAcquire() == false - asyncTest "should tryAcquire and acquire": - let sema = newAsyncSemaphore(4) + test "should tryAcquire and acquire": + let sema = AsyncSemaphore.new(4) check sema.tryAcquire() == true check sema.tryAcquire() == true check sema.tryAcquire() == true @@ -88,8 +88,8 @@ suite "AsyncSemaphore": check fut.finished == true check sema.count == 4 - asyncTest "should restrict resource access": - let sema = newAsyncSemaphore(3) + test "should restrict resource access": + let sema = AsyncSemaphore.new(3) var resource = 0 proc task() {.async.} = @@ -110,8 +110,8 @@ suite "AsyncSemaphore": await allFutures(tasks) - asyncTest "should cancel sequential semaphore slot": - let sema = newAsyncSemaphore(1) + test "should cancel sequential semaphore slot": + let sema = AsyncSemaphore.new(1) await sema.acquire() @@ -131,8 +131,8 @@ suite "AsyncSemaphore": check await sema.acquire().withTimeout(10.millis) - asyncTest "should handle out of order cancellations": - let sema = newAsyncSemaphore(1) + test "should handle out of order cancellations": + let sema = AsyncSemaphore.new(1) await sema.acquire() # 1st acquire let tmp1 = sema.acquire() # 2nd acquire @@ -156,8 +156,8 @@ suite "AsyncSemaphore": sema.release() # 4th release check await sema.acquire().withTimeout(10.millis) - asyncTest "should properly handle timeouts and cancellations": - let sema = newAsyncSemaphore(1) + test "should properly handle timeouts and cancellations": + let sema = AsyncSemaphore.new(1) await sema.acquire() check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel @@ -165,8 +165,8 @@ suite "AsyncSemaphore": check await sema.acquire().withTimeout(10.millis) - asyncTest "should handle forceAcquire properly": - let sema = newAsyncSemaphore(1) + test "should handle forceAcquire properly": + let sema = AsyncSemaphore.new(1) await sema.acquire() check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel diff --git a/tests/testall.nim b/tests/testall.nim index 153261b4..83bdf381 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -7,6 +7,7 @@ import ./datastore/testmountedds, ./datastore/testdatabuffer, ./datastore/testthreadproxyds, + ./datastore/testasyncsemaphore, ./datastore/testsemaphore {.warning[UnusedImport]: off.} From bee79ffe72eb92a6331af6060b543d1ca89abc03 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 16:40:46 -0600 Subject: [PATCH 153/445] added (ugly!) locking capabilities --- datastore/fsds.nim | 2 +- datastore/threads/threadproxyds.nim | 140 ++++++++++++++++++++-------- 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 5526d5cd..6c695bb9 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -188,7 +188,7 @@ method query*( var iter = QueryIter.new() - let lock = newAsyncLock() + var lock = newAsyncLock() # serialize querying under threads proc next(): Future[?!QueryResponse] {.async.} = defer: if lock.locked: diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 56c818a2..1cf4d46d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -9,6 +9,7 @@ push: {.upraises: [].} import std/atomics import std/strutils import std/tables +import std/sequtils import pkg/chronos import pkg/chronos/threadsync @@ -45,35 +46,66 @@ type ds: Datastore semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors - tasks: seq[Future[void]] - locks: Table[Key, AsyncLock] + case withLocks: bool + of true: + tasks: Table[Key, Future[void]] + queryLock: AsyncLock # global query lock, this is only really \ + # needed for the fsds, but it is expensive! + else: + futs: seq[Future[void]] # keep a list of the futures to the signals around + +template withLocks( + self: ThreadDatastore, + ctx: TaskCtx, + key: ?Key = Key.none, + fut: Future[void], + body: untyped) = + try: + case self.withLocks: + of true: + if key.isSome and + key.get in self.tasks: + await self.tasks[key.get] + await self.queryLock.acquire() # lock query or wait to finish + + self.tasks[key.get] = fut + else: + self.futs.add(fut) + + body + finally: + case self.withLocks: + of true: + if key.isSome: + self.tasks.del(key.get) + if self.queryLock.locked: + self.queryLock.release() + else: + self.futs.keepItIf(it != fut) template dispatchTask( self: ThreadDatastore, ctx: TaskCtx, + key: ?Key = Key.none, runTask: proc): untyped = let fut = wait(ctx.signal) - try: - self.tasks.add(fut) - runTask() - await fut - - if ctx.res[].isErr: - result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed - except CancelledError as exc: - trace "Cancelling future!", exc = exc.msg - ctx.cancelled = true - await ctx.signal.fire() - raise exc - finally: - discard ctx.signal.close() - if ( - let idx = self.tasks.find(fut); - idx != -1): - self.tasks.del(idx) + withLocks(self, ctx, key, fut): + try: + runTask() + await fut + + if ctx.res[].isErr: + result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed + except CancelledError as exc: + trace "Cancelling thread future!", exc = exc.msg + ctx.cancelled = true + await ctx.signal.fire() + raise exc + finally: + discard ctx.signal.close() proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = ## Monitor the signal and cancel the future if @@ -115,6 +147,7 @@ proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = try: waitFor asyncHasTask(ctx, key) except CatchableError as exc: + trace "Unexpected exception thrown in asyncHasTask", error = error.msg raiseAssert exc.msg method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = @@ -136,7 +169,7 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = proc runTask() = self.tp.spawn hasTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, runTask) + self.dispatchTask(ctx, key.some, runTask) return success(res.get()) proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = @@ -148,6 +181,7 @@ proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: + trace "Error in asyncDelTask", error = error.msg ctx[].res[].err(error) return @@ -158,6 +192,7 @@ proc delTask(ctx: ptr TaskCtx, key: ptr Key) = try: waitFor asyncDelTask(ctx, key) except CatchableError as exc: + trace "Unexpected exception thrown in asyncDelTask", error = error.msg raiseAssert exc.msg method delete*( @@ -181,7 +216,7 @@ method delete*( proc runTask() = self.tp.spawn delTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, runTask) + self.dispatchTask(ctx, key.some, runTask) return success() method delete*( @@ -207,6 +242,7 @@ proc asyncPutTask( asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: + trace "Error in asyncPutTask", error = error.msg ctx[].res[].err(error) return @@ -221,8 +257,9 @@ proc putTask( ## try: - waitFor asyncPutTask(ctx, key, data, len) + waitFor asyncPutTask(ctx, key, data, len) except CatchableError as exc: + trace "Unexpected exception thrown in asyncPutTask", error = error.msg raiseAssert exc.msg method put*( @@ -251,7 +288,7 @@ method put*( makeUncheckedArray(baseAddr data), data.len) - self.dispatchTask(ctx, runTask) + self.dispatchTask(ctx, key.some, runTask) return success() method put*( @@ -274,8 +311,8 @@ proc asyncGetTask( fut = ctx[].ds[].get(key[]) asyncSpawn signalMonitor(ctx, fut) - without res =? - (waitFor fut).catch and data =? res, error: + without res =? (await fut).catch and data =? res, error: + trace "Error in asyncGetTask", error = error.msg ctx[].res[].err(error) return @@ -290,6 +327,7 @@ proc getTask( try: waitFor asyncGetTask(ctx, key) except CatchableError as exc: + trace "Unexpected exception thrown in asyncGetTask", error = error.msg raiseAssert exc.msg method get*( @@ -314,15 +352,20 @@ method get*( proc runTask() = self.tp.spawn getTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, runTask) + self.dispatchTask(ctx, key.some, runTask) if err =? res.errorOption: return failure err return success(@(res.get())) method close*(self: ThreadDatastore): Future[?!void] {.async.} = - for task in self.tasks: - await task.cancelAndWait() + var futs = if self.withLocks: + self.tasks.values.toSeq # toSeq(...) doesn't work here??? + else: + self.futs + + for fut in futs: + await fut.cancelAndWait() await self.ds.close() @@ -336,7 +379,8 @@ proc asyncQueryTask( fut = iter[].next() asyncSpawn signalMonitor(ctx, fut) - without ret =? (waitFor fut).catch and res =? ret, error: + without ret =? (await fut).catch and res =? ret, error: + trace "Error in asyncQueryTask", error = error.msg ctx[].res[].err(error) return @@ -357,6 +401,7 @@ proc queryTask( try: waitFor asyncQueryTask(ctx, iter) except CatchableError as exc: + trace "Unexpected exception thrown in asyncQueryTask", error = error.msg raiseAssert exc.msg method query*( @@ -373,8 +418,17 @@ method query*( defer: locked = false self.semaphore.release() - + case self.withLocks: + of true: + if self.queryLock.locked: + self.queryLock.release() + else: + discard + + trace "About to query" await self.semaphore.acquire() + if self.withLocks: + await self.queryLock.acquire() if locked: return failure (ref DatastoreError)(msg: "Should always await query features") @@ -400,8 +454,9 @@ method query*( proc runTask() = self.tp.spawn queryTask(addr ctx, addr childIter) - self.dispatchTask(ctx, runTask) + self.dispatchTask(ctx, Key.none, runTask) if err =? res.errorOption: + trace "Query failed", err = err return failure err let (ok, key, data) = res.get() @@ -414,13 +469,24 @@ method query*( iter.next = next return success iter -func new*( +proc new*( self: type ThreadDatastore, ds: Datastore, + withLocks = static false, tp: Taskpool): ?!ThreadDatastore = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" - success ThreadDatastore( - tp: tp, - ds: ds, - semaphore: AsyncSemaphore.new(tp.numThreads - 1)) + case withLocks: + of true: + success ThreadDatastore( + tp: tp, + ds: ds, + withLocks: true, + queryLock: newAsyncLock(), + semaphore: AsyncSemaphore.new(tp.numThreads - 1)) + else: + success ThreadDatastore( + tp: tp, + ds: ds, + withLocks: false, + semaphore: AsyncSemaphore.new(tp.numThreads - 1)) From 84681cd8efb16d5bb9d1b0e61704b583219794da Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 16:41:03 -0600 Subject: [PATCH 154/445] make all tests pass --- tests/datastore/testasyncsemaphore.nim | 26 ++++---- tests/datastore/testthreadproxyds.nim | 84 ++++++++++++++++++++------ 2 files changed, 77 insertions(+), 33 deletions(-) diff --git a/tests/datastore/testasyncsemaphore.nim b/tests/datastore/testasyncsemaphore.nim index 3e5cb318..3c8bb84b 100644 --- a/tests/datastore/testasyncsemaphore.nim +++ b/tests/datastore/testasyncsemaphore.nim @@ -19,7 +19,7 @@ import pkg/datastore/threads/asyncsemaphore randomize() suite "AsyncSemaphore": - test "should acquire": + test "Should acquire": let sema = AsyncSemaphore.new(3) await sema.acquire() @@ -28,7 +28,7 @@ suite "AsyncSemaphore": check sema.count == 0 - test "should release": + test "Should release": let sema = AsyncSemaphore.new(3) await sema.acquire() @@ -41,7 +41,7 @@ suite "AsyncSemaphore": sema.release() check sema.count == 3 - test "should queue acquire": + test "Should queue acquire": let sema = AsyncSemaphore.new(1) await sema.acquire() @@ -55,19 +55,19 @@ suite "AsyncSemaphore": await sleepAsync(10.millis) check fut.finished() - test "should keep count == size": + test "Should keep count == size": let sema = AsyncSemaphore.new(1) sema.release() sema.release() sema.release() check sema.count == 1 - test "should tryAcquire": + test "Should tryAcquire": let sema = AsyncSemaphore.new(1) await sema.acquire() check sema.tryAcquire() == false - test "should tryAcquire and acquire": + test "Should tryAcquire and acquire": let sema = AsyncSemaphore.new(4) check sema.tryAcquire() == true check sema.tryAcquire() == true @@ -88,7 +88,7 @@ suite "AsyncSemaphore": check fut.finished == true check sema.count == 4 - test "should restrict resource access": + test "Should restrict resource access": let sema = AsyncSemaphore.new(3) var resource = 0 @@ -110,7 +110,7 @@ suite "AsyncSemaphore": await allFutures(tasks) - test "should cancel sequential semaphore slot": + test "Should cancel sequential semaphore slot": let sema = AsyncSemaphore.new(1) await sema.acquire() @@ -131,7 +131,7 @@ suite "AsyncSemaphore": check await sema.acquire().withTimeout(10.millis) - test "should handle out of order cancellations": + test "Should handle out of order cancellations": let sema = AsyncSemaphore.new(1) await sema.acquire() # 1st acquire @@ -156,20 +156,20 @@ suite "AsyncSemaphore": sema.release() # 4th release check await sema.acquire().withTimeout(10.millis) - test "should properly handle timeouts and cancellations": + test "Should properly handle timeouts and cancellations": let sema = AsyncSemaphore.new(1) await sema.acquire() - check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel + check not(await sema.acquire().withTimeout(1.millis)) # Should not acquire but cancel sema.release() check await sema.acquire().withTimeout(10.millis) - test "should handle forceAcquire properly": + test "Should handle forceAcquire properly": let sema = AsyncSemaphore.new(1) await sema.acquire() - check not(await sema.acquire().withTimeout(1.millis)) # should not acquire but cancel + check not(await sema.acquire().withTimeout(1.millis)) # Should not acquire but cancel let fut1 = sema.acquire() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index dac08fad..92b832cf 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -34,7 +34,7 @@ suite "Test Basic ThreadDatastore with SQLite": setupAll: sqlStore = SQLiteDatastore.new(Memory).tryGet() taskPool = Taskpool.new(countProcessors() * 2) - ds = ThreadDatastore.new(sqlStore, taskPool).tryGet() + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardownAll: (await ds.close()).tryGet() @@ -42,7 +42,6 @@ suite "Test Basic ThreadDatastore with SQLite": basicStoreTests(ds, key, bytes, otherBytes) - suite "Test Query ThreadDatastore with SQLite": var @@ -56,7 +55,7 @@ suite "Test Query ThreadDatastore with SQLite": setup: sqlStore = SQLiteDatastore.new(Memory).tryGet() taskPool = Taskpool.new(countProcessors() * 2) - ds = ThreadDatastore.new(sqlStore, taskPool).tryGet() + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: (await ds.close()).tryGet() @@ -64,23 +63,68 @@ suite "Test Query ThreadDatastore with SQLite": queryTests(ds, true) -# TODO: needs a per key lock to work -# suite "Test Basic ThreadDatastore with fsds": +suite "Test Basic ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + var + fsStore: FSDatastore + ds: ThreadDatastore + taskPool: Taskpool + + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + taskPool = Taskpool.new(countProcessors() * 2) + ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() + + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + basicStoreTests(fsStore, key, bytes, otherBytes) + +suite "Test Query ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + + var + fsStore: FSDatastore + ds: ThreadDatastore + taskPool: Taskpool + + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + fsStore = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() + taskPool = Taskpool.new(countProcessors() * 2) + ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() + + teardown: + (await ds.close()).tryGet() + taskPool.shutdown() -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) -# var -# fsStore: FSDatastore -# ds: ThreadDatastore -# taskPool: Taskpool + queryTests(ds, false) -suite "Test ThreadDatastore": +suite "Test ThreadDatastore cancelations": var sqlStore: Datastore ds: ThreadDatastore @@ -95,9 +139,9 @@ suite "Test ThreadDatastore": setupAll: sqlStore = SQLiteDatastore.new(Memory).tryGet() taskPool = Taskpool.new(countProcessors() * 2) - ds = ThreadDatastore.new(sqlStore, taskPool).tryGet() + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - test "should monitor signal for cancellations and cancel": + test "Should monitor signal and cancel": var signal = ThreadSignalPtr.new().tryGet() res = ThreadResult[void]() @@ -130,7 +174,7 @@ suite "Test ThreadDatastore": check: fut.cancelled check: ctx.signal.close().isOk - test "should monitor signal for cancellations and not cancel": + test "Should monitor and not cancel": var signal = ThreadSignalPtr.new().tryGet() res = ThreadResult[void]() From f849c1cf296563500100f31d27390a057dc7f412 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 16:49:35 -0600 Subject: [PATCH 155/445] re-export threaded datastore --- datastore.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore.nim b/datastore.nim index 6d43a209..9e42c472 100644 --- a/datastore.nim +++ b/datastore.nim @@ -3,5 +3,6 @@ import ./datastore/fsds import ./datastore/sql import ./datastore/mountedds import ./datastore/tieredds +import ./datastore/threads/threadproxyds -export datastore, fsds, mountedds, tieredds, sql +export datastore, fsds, mountedds, tieredds, sql, threadproxyds From 52d6a857ac8bee8cec0e39913ab9a0483b4c98ab Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 15 Sep 2023 16:54:50 -0600 Subject: [PATCH 156/445] fixing dumb error in trace --- datastore/threads/threadproxyds.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 1cf4d46d..55591269 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -147,7 +147,7 @@ proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = try: waitFor asyncHasTask(ctx, key) except CatchableError as exc: - trace "Unexpected exception thrown in asyncHasTask", error = error.msg + trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = @@ -192,7 +192,7 @@ proc delTask(ctx: ptr TaskCtx, key: ptr Key) = try: waitFor asyncDelTask(ctx, key) except CatchableError as exc: - trace "Unexpected exception thrown in asyncDelTask", error = error.msg + trace "Unexpected exception thrown in asyncDelTask", exc = exc.msg raiseAssert exc.msg method delete*( @@ -259,7 +259,7 @@ proc putTask( try: waitFor asyncPutTask(ctx, key, data, len) except CatchableError as exc: - trace "Unexpected exception thrown in asyncPutTask", error = error.msg + trace "Unexpected exception thrown in asyncPutTask", exc = exc.msg raiseAssert exc.msg method put*( @@ -327,7 +327,7 @@ proc getTask( try: waitFor asyncGetTask(ctx, key) except CatchableError as exc: - trace "Unexpected exception thrown in asyncGetTask", error = error.msg + trace "Unexpected exception thrown in asyncGetTask", exc = exc.msg raiseAssert exc.msg method get*( @@ -401,7 +401,7 @@ proc queryTask( try: waitFor asyncQueryTask(ctx, iter) except CatchableError as exc: - trace "Unexpected exception thrown in asyncQueryTask", error = error.msg + trace "Unexpected exception thrown in asyncQueryTask", exc = exc.msg raiseAssert exc.msg method query*( From 7306a0bfd43ab3a45bee78a79206f92375dd96f4 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 18 Sep 2023 13:18:51 -0600 Subject: [PATCH 157/445] simplify locking --- .gitignore | 2 + config.nims | 4 ++ datastore/threads/threadproxyds.nim | 53 ++++++-------------- tests/datastore/testthreadproxyds.nim | 72 ++++++++++++++------------- 4 files changed, 58 insertions(+), 73 deletions(-) diff --git a/.gitignore b/.gitignore index 38068420..de000360 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ datastore.nims nimcache TODO nim.cfg +nimble.develop +nimble.paths diff --git a/config.nims b/config.nims index 052a576c..ed3a5315 100644 --- a/config.nims +++ b/config.nims @@ -10,3 +10,7 @@ when (NimMajor, NimMinor) == (1, 2): when (NimMajor, NimMinor) > (1, 2): switch("hint", "XCannotRaiseY:off") +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 55591269..c9b0408e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -46,13 +46,10 @@ type ds: Datastore semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors - case withLocks: bool - of true: - tasks: Table[Key, Future[void]] - queryLock: AsyncLock # global query lock, this is only really \ - # needed for the fsds, but it is expensive! - else: - futs: seq[Future[void]] # keep a list of the futures to the signals around + withLocks: bool + tasks: Table[Key, Future[void]] + queryLock: AsyncLock # global query lock, this is only really \ + # needed for the fsds, but it is expensive! template withLocks( self: ThreadDatastore, @@ -61,27 +58,21 @@ template withLocks( fut: Future[void], body: untyped) = try: - case self.withLocks: - of true: - if key.isSome and - key.get in self.tasks: + if key.isSome and key.get in self.tasks: + if self.withLocks: await self.tasks[key.get] - await self.queryLock.acquire() # lock query or wait to finish + self.tasks[key.get] = fut # we alway want to store the future, but only await if we're using locks - self.tasks[key.get] = fut - else: - self.futs.add(fut) + if self.withLocks: + await self.queryLock.acquire() # only lock if it's required (fsds) body finally: - case self.withLocks: - of true: - if key.isSome: + if self.withLocks: + if key.isSome and key.get in self.tasks: self.tasks.del(key.get) - if self.queryLock.locked: - self.queryLock.release() - else: - self.futs.keepItIf(it != fut) + if self.queryLock.locked: + self.queryLock.release() template dispatchTask( self: ThreadDatastore, @@ -359,13 +350,8 @@ method get*( return success(@(res.get())) method close*(self: ThreadDatastore): Future[?!void] {.async.} = - var futs = if self.withLocks: - self.tasks.values.toSeq # toSeq(...) doesn't work here??? - else: - self.futs - - for fut in futs: - await fut.cancelAndWait() + for fut in self.tasks.values.toSeq: + await fut.cancelAndWait() # probably want to store the signal, instead of the future (or both?) await self.ds.close() @@ -418,18 +404,9 @@ method query*( defer: locked = false self.semaphore.release() - case self.withLocks: - of true: - if self.queryLock.locked: - self.queryLock.release() - else: - discard trace "About to query" await self.semaphore.acquire() - if self.withLocks: - await self.queryLock.acquire() - if locked: return failure (ref DatastoreError)(msg: "Should always await query features") diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 92b832cf..7019e528 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -21,47 +21,49 @@ import pkg/datastore/threads/threadproxyds {.all.} import ./dscommontests import ./querycommontests -suite "Test Basic ThreadDatastore with SQLite": +const NumThreads = 200 # IO threads aren't attached to CPU count - var - sqlStore: Datastore - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes +# suite "Test Basic ThreadDatastore with SQLite": - setupAll: - sqlStore = SQLiteDatastore.new(Memory).tryGet() - taskPool = Taskpool.new(countProcessors() * 2) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() +# var +# sqlStore: Datastore +# ds: ThreadDatastore +# taskPool: Taskpool +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() +# setupAll: +# sqlStore = SQLiteDatastore.new(Memory).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - basicStoreTests(ds, key, bytes, otherBytes) +# teardownAll: +# (await ds.close()).tryGet() +# taskPool.shutdown() -suite "Test Query ThreadDatastore with SQLite": +# basicStoreTests(ds, key, bytes, otherBytes) - var - sqlStore: Datastore - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes +# suite "Test Query ThreadDatastore with SQLite": - setup: - sqlStore = SQLiteDatastore.new(Memory).tryGet() - taskPool = Taskpool.new(countProcessors() * 2) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() +# var +# sqlStore: Datastore +# ds: ThreadDatastore +# taskPool: Taskpool +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes - teardown: - (await ds.close()).tryGet() - taskPool.shutdown() +# setup: +# sqlStore = SQLiteDatastore.new(Memory).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + +# teardown: +# (await ds.close()).tryGet() +# taskPool.shutdown() - queryTests(ds, true) +# queryTests(ds, true) suite "Test Basic ThreadDatastore with fsds": let @@ -83,7 +85,7 @@ suite "Test Basic ThreadDatastore with fsds": createDir(basePathAbs) fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - taskPool = Taskpool.new(countProcessors() * 2) + taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() teardownAll: @@ -112,7 +114,7 @@ suite "Test Query ThreadDatastore with fsds": createDir(basePathAbs) fsStore = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - taskPool = Taskpool.new(countProcessors() * 2) + taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() teardown: @@ -138,7 +140,7 @@ suite "Test ThreadDatastore cancelations": setupAll: sqlStore = SQLiteDatastore.new(Memory).tryGet() - taskPool = Taskpool.new(countProcessors() * 2) + taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() test "Should monitor signal and cancel": From ed09b9c936c97ee62ccf918c10f3c976ef75107b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 18 Sep 2023 13:19:33 -0600 Subject: [PATCH 158/445] re-enable tests --- tests/datastore/testthreadproxyds.nim | 82 +++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 7019e528..e85132ea 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -23,47 +23,47 @@ import ./querycommontests const NumThreads = 200 # IO threads aren't attached to CPU count -# suite "Test Basic ThreadDatastore with SQLite": - -# var -# sqlStore: Datastore -# ds: ThreadDatastore -# taskPool: Taskpool -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes - -# setupAll: -# sqlStore = SQLiteDatastore.new(Memory).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - -# teardownAll: -# (await ds.close()).tryGet() -# taskPool.shutdown() - -# basicStoreTests(ds, key, bytes, otherBytes) - -# suite "Test Query ThreadDatastore with SQLite": - -# var -# sqlStore: Datastore -# ds: ThreadDatastore -# taskPool: Taskpool -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes - -# setup: -# sqlStore = SQLiteDatastore.new(Memory).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - -# teardown: -# (await ds.close()).tryGet() -# taskPool.shutdown() - -# queryTests(ds, true) +suite "Test Basic ThreadDatastore with SQLite": + + var + sqlStore: Datastore + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setupAll: + sqlStore = SQLiteDatastore.new(Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() + + basicStoreTests(ds, key, bytes, otherBytes) + +suite "Test Query ThreadDatastore with SQLite": + + var + sqlStore: Datastore + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setup: + sqlStore = SQLiteDatastore.new(Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + + teardown: + (await ds.close()).tryGet() + taskPool.shutdown() + + queryTests(ds, true) suite "Test Basic ThreadDatastore with fsds": let From 75fa37f5672e9c4eb6895fbd878c58db0ff594bc Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 18 Sep 2023 13:40:44 -0600 Subject: [PATCH 159/445] avoid duplicating code --- datastore/threads/threadproxyds.nim | 86 ++++++++--------------------- 1 file changed, 24 insertions(+), 62 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c9b0408e..42ed8d0f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -65,7 +65,6 @@ template withLocks( if self.withLocks: await self.queryLock.acquire() # only lock if it's required (fsds) - body finally: if self.withLocks: @@ -74,29 +73,35 @@ template withLocks( if self.queryLock.locked: self.queryLock.release() +# TODO: needs rework, we can't use `result` with async template dispatchTask( self: ThreadDatastore, ctx: TaskCtx, key: ?Key = Key.none, runTask: proc): untyped = + try: + await self.semaphore.acquire() + ctx.signal = ThreadSignalPtr.new().valueOr: + result = failure(error()) + return - let - fut = wait(ctx.signal) + let + fut = wait(ctx.signal) - withLocks(self, ctx, key, fut): - try: + withLocks(self, ctx, key, fut): runTask() await fut if ctx.res[].isErr: - result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed - except CancelledError as exc: - trace "Cancelling thread future!", exc = exc.msg - ctx.cancelled = true - await ctx.signal.fire() - raise exc - finally: - discard ctx.signal.close() + result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed from a thread + except CancelledError as exc: + trace "Cancelling thread future!", exc = exc.msg + ctx.cancelled = true + await ctx.signal.fire() + raise exc + finally: + discard ctx.signal.close() + self.semaphore.release() proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = ## Monitor the signal and cancel the future if @@ -142,20 +147,11 @@ proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = raiseAssert exc.msg method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = - defer: - self.semaphore.release() - - await self.semaphore.acquire() - var - signal = ThreadSignalPtr.new().valueOr: - return failure(error()) - res = ThreadResult[bool]() ctx = TaskCtx[bool]( ds: addr self.ds, - res: addr res, - signal: signal) + res: addr res) proc runTask() = self.tp.spawn hasTask(addr ctx, unsafeAddr key) @@ -189,20 +185,11 @@ proc delTask(ctx: ptr TaskCtx, key: ptr Key) = method delete*( self: ThreadDatastore, key: Key): Future[?!void] {.async.} = - defer: - self.semaphore.release() - - await self.semaphore.acquire() - var - signal = ThreadSignalPtr.new().valueOr: - return failure(error()) - res = ThreadResult[void]() ctx = TaskCtx[void]( ds: addr self.ds, - res: addr res, - signal: signal) + res: addr res) proc runTask() = self.tp.spawn delTask(addr ctx, unsafeAddr key) @@ -257,20 +244,11 @@ method put*( self: ThreadDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = - defer: - self.semaphore.release() - - await self.semaphore.acquire() - var - signal = ThreadSignalPtr.new().valueOr: - return failure(error()) - res = ThreadResult[void]() ctx = TaskCtx[void]( ds: addr self.ds, - res: addr res, - signal: signal) + res: addr res) proc runTask() = self.tp.spawn putTask( @@ -324,21 +302,11 @@ proc getTask( method get*( self: ThreadDatastore, key: Key): Future[?!seq[byte]] {.async.} = - defer: - self.semaphore.release() - - await self.semaphore.acquire() - - var - signal = ThreadSignalPtr.new().valueOr: - return failure(error()) - var res = ThreadResult[DataBuffer]() ctx = TaskCtx[DataBuffer]( ds: addr self.ds, - res: addr res, - signal: signal) + res: addr res) proc runTask() = self.tp.spawn getTask(addr ctx, unsafeAddr key) @@ -403,10 +371,8 @@ method query*( proc next(): Future[?!QueryResponse] {.async.} = defer: locked = false - self.semaphore.release() trace "About to query" - await self.semaphore.acquire() if locked: return failure (ref DatastoreError)(msg: "Should always await query features") @@ -419,21 +385,17 @@ method query*( return success (Key.none, EmptyBytes) var - signal = ThreadSignalPtr.new().valueOr: - return failure("Failed to create signal") - res = ThreadResult[(bool, DataBuffer, DataBuffer)]() ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( ds: addr self.ds, - res: addr res, - signal: signal) + res: addr res) proc runTask() = self.tp.spawn queryTask(addr ctx, addr childIter) self.dispatchTask(ctx, Key.none, runTask) if err =? res.errorOption: - trace "Query failed", err = err + trace "Query failed", err = $err return failure err let (ok, key, data) = res.get() From 2c5186eab62b2b4a32ab7b3471952d756944323b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 18 Sep 2023 13:46:58 -0600 Subject: [PATCH 160/445] use baseAddr --- datastore/threads/databuffer.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index be1f0ca4..66bb7e66 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -1,5 +1,6 @@ import threading/smartptrs import std/hashes +import pkg/stew/ptrops export hashes @@ -50,7 +51,9 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T]): DataBuffer = ## result = DataBuffer.new(data.len) if data.len() > 0: - copyMem(result[].buf, unsafeAddr data[0], data.len) + # TODO: we might want to copy data, otherwise the GC might + # release it on stack-unwind + copyMem(result[].buf, baseAddr data, data.len) converter toSeq*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char @@ -58,7 +61,7 @@ converter toSeq*(self: DataBuffer): seq[byte] = result = newSeq[byte](self.len) if self.len() > 0: - copyMem(addr result[0], unsafeAddr self[].buf[0], self.len) + copyMem(addr result[0], addr self[].buf[0], self.len) proc `@`*(self: DataBuffer): seq[byte] = ## Convert a buffer to a seq type using copy and @@ -74,7 +77,7 @@ converter toString*(data: DataBuffer): string = if data.isNil: return "" result = newString(data.len()) if data.len() > 0: - copyMem(addr result[0], unsafeAddr data[].buf[0], data.len) + copyMem(addr result[0], addr data[].buf[0], data.len) proc `$`*(data: DataBuffer): string = ## convert buffer to string type using copy From d5a1b344b38be591ed613cee46a8ac73f49fd56b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 09:16:08 -0600 Subject: [PATCH 161/445] remove ptr to Datastore in TaskCtx, it's a ref --- datastore/threads/threadproxyds.nim | 20 ++++++++++---------- tests/datastore/testthreadproxyds.nim | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 42ed8d0f..f0638e92 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -35,7 +35,7 @@ type ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] TaskCtx[T: ThreadTypes] = object - ds: ptr Datastore + ds: Datastore res: ptr ThreadResult[T] cancelled: bool semaphore: AsyncSemaphore @@ -130,7 +130,7 @@ proc asyncHasTask( discard ctx[].signal.fireSync() let - fut = ctx[].ds[].has(key[]) + fut = ctx[].ds.has(key[]) asyncSpawn signalMonitor(ctx, fut) without ret =? (await fut).catch and res =? ret, error: @@ -150,7 +150,7 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var res = ThreadResult[bool]() ctx = TaskCtx[bool]( - ds: addr self.ds, + ds: self.ds, res: addr res) proc runTask() = @@ -164,7 +164,7 @@ proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = discard ctx[].signal.fireSync() let - fut = ctx[].ds[].delete(key[]) + fut = ctx[].ds.delete(key[]) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: @@ -188,7 +188,7 @@ method delete*( var res = ThreadResult[void]() ctx = TaskCtx[void]( - ds: addr self.ds, + ds: self.ds, res: addr res) proc runTask() = @@ -216,7 +216,7 @@ proc asyncPutTask( discard ctx[].signal.fireSync() let - fut = ctx[].ds[].put(key[], @(data.toOpenArray(0, len - 1))) + fut = ctx[].ds.put(key[], @(data.toOpenArray(0, len - 1))) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: @@ -247,7 +247,7 @@ method put*( var res = ThreadResult[void]() ctx = TaskCtx[void]( - ds: addr self.ds, + ds: self.ds, res: addr res) proc runTask() = @@ -277,7 +277,7 @@ proc asyncGetTask( discard ctx[].signal.fireSync() let - fut = ctx[].ds[].get(key[]) + fut = ctx[].ds.get(key[]) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch and data =? res, error: @@ -305,7 +305,7 @@ method get*( var res = ThreadResult[DataBuffer]() ctx = TaskCtx[DataBuffer]( - ds: addr self.ds, + ds: self.ds, res: addr res) proc runTask() = @@ -387,7 +387,7 @@ method query*( var res = ThreadResult[(bool, DataBuffer, DataBuffer)]() ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( - ds: addr self.ds, + ds: self.ds, res: addr res) proc runTask() = diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index e85132ea..10197dfa 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -148,7 +148,7 @@ suite "Test ThreadDatastore cancelations": signal = ThreadSignalPtr.new().tryGet() res = ThreadResult[void]() ctx = TaskCtx[void]( - ds: addr sqlStore, + ds: sqlStore, res: addr res, signal: signal) fut = newFuture[void]("signalMonitor") @@ -181,7 +181,7 @@ suite "Test ThreadDatastore cancelations": signal = ThreadSignalPtr.new().tryGet() res = ThreadResult[void]() ctx = TaskCtx[void]( - ds: addr sqlStore, + ds: sqlStore, res: addr res, signal: signal) fut = newFuture[void]("signalMonitor") From 3d33820729eb369eb33869a71b882c3ce73b0f65 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 18:42:24 -0600 Subject: [PATCH 162/445] change logscope --- datastore/threads/asyncsemaphore.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim index 78f768aa..c9a76276 100644 --- a/datastore/threads/asyncsemaphore.nim +++ b/datastore/threads/asyncsemaphore.nim @@ -15,7 +15,7 @@ import chronos, chronicles # TODO: this should probably go in chronos logScope: - topics = "libp2p semaphore" + topics = "datastore semaphore" type AsyncSemaphore* = ref object of RootObj From 0beb3d3c905a22ec459b32b7d7014cbf753bea7a Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 18:43:00 -0600 Subject: [PATCH 163/445] handle error passing and conversion better --- datastore/query.nim | 3 +- datastore/threads/threadproxyds.nim | 110 ++++++++++++---------------- datastore/threads/threadresult.nim | 43 +++++++++++ datastore/types.nim | 1 + 4 files changed, 93 insertions(+), 64 deletions(-) create mode 100644 datastore/threads/threadresult.nim diff --git a/datastore/query.nim b/datastore/query.nim index 097d33dc..7cae8974 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -6,6 +6,8 @@ import pkg/questionable/results import ./key import ./types + +export types export options, SortOrder type @@ -17,7 +19,6 @@ type sort*: SortOrder # Sort order - not available in all backends QueryResponse* = tuple[key: ?Key, data: seq[byte]] - QueryEndedError* = object of DatastoreError GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe.} IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index f0638e92..fc69a690 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -26,14 +26,14 @@ import ../datastore import ./asyncsemaphore import ./databuffer +import ./threadresult -type - ErrorEnum {.pure.} = enum - DatastoreErr, DatastoreKeyNotFoundErr, CatchableErr +export threadresult - ThreadTypes = void | bool | SomeInteger | DataBuffer | tuple | Atomic - ThreadResult[T: ThreadTypes] = Result[T, DataBuffer] +logScope: + topics = "datastore threadproxyds" +type TaskCtx[T: ThreadTypes] = object ds: Datastore res: ptr ThreadResult[T] @@ -56,7 +56,7 @@ template withLocks( ctx: TaskCtx, key: ?Key = Key.none, fut: Future[void], - body: untyped) = + body: untyped): untyped = try: if key.isSome and key.get in self.tasks: if self.withLocks: @@ -65,7 +65,9 @@ template withLocks( if self.withLocks: await self.queryLock.acquire() # only lock if it's required (fsds) - body + + block: + body finally: if self.withLocks: if key.isSome and key.get in self.tasks: @@ -74,26 +76,31 @@ template withLocks( self.queryLock.release() # TODO: needs rework, we can't use `result` with async -template dispatchTask( +template dispatchTask[T]( self: ThreadDatastore, - ctx: TaskCtx, + ctx: TaskCtx[T], key: ?Key = Key.none, - runTask: proc): untyped = + runTask: proc): auto = try: await self.semaphore.acquire() - ctx.signal = ThreadSignalPtr.new().valueOr: - result = failure(error()) - return - - let - fut = wait(ctx.signal) - - withLocks(self, ctx, key, fut): - runTask() - await fut - - if ctx.res[].isErr: - result = failure(ctx.res[].error()) # TODO: fix this, result shouldn't be accessed from a thread + let signal = ThreadSignalPtr.new() + if signal.isErr: + failure(signal.error) + else: + ctx.signal = signal.get() + let + fut = wait(ctx.signal) + + withLocks(self, ctx, key, fut): + runTask() + await fut + if ctx.res[].isErr: + failure ctx.res[].error + else: + when result.T isnot void: + success result.T(ctx.res[].get) + else: + success() except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.cancelled = true @@ -156,8 +163,7 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = proc runTask() = self.tp.spawn hasTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, key.some, runTask) - return success(res.get()) + return self.dispatchTask(ctx, key.some, runTask) proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = defer: @@ -194,8 +200,7 @@ method delete*( proc runTask() = self.tp.spawn delTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, key.some, runTask) - return success() + return self.dispatchTask(ctx, key.some, runTask) method delete*( self: ThreadDatastore, @@ -257,8 +262,7 @@ method put*( makeUncheckedArray(baseAddr data), data.len) - self.dispatchTask(ctx, key.some, runTask) - return success() + return self.dispatchTask(ctx, key.some, runTask) method put*( self: ThreadDatastore, @@ -285,6 +289,7 @@ proc asyncGetTask( ctx[].res[].err(error) return + trace "Got data in get", data ctx[].res[].ok(DataBuffer.new(data)) proc getTask( @@ -311,11 +316,7 @@ method get*( proc runTask() = self.tp.spawn getTask(addr ctx, unsafeAddr key) - self.dispatchTask(ctx, key.some, runTask) - if err =? res.errorOption: - return failure err - - return success(@(res.get())) + return self.dispatchTask(ctx, key.some, runTask) method close*(self: ThreadDatastore): Future[?!void] {.async.} = for fut in self.tasks.values.toSeq: @@ -339,14 +340,15 @@ proc asyncQueryTask( return if res.key.isNone: - ctx[].res[].ok((false, default(DataBuffer), default(DataBuffer))) + ctx[].res[].ok((default(DataBuffer), default(DataBuffer))) return var keyBuf = DataBuffer.new($(res.key.get())) dataBuf = DataBuffer.new(res.data) - ctx[].res[].ok((true, keyBuf, dataBuf)) + trace "Got query result", key = $res.key.get(), data = res.data + ctx[].res[].ok((keyBuf, dataBuf)) proc queryTask( ctx: ptr TaskCtx, @@ -385,25 +387,15 @@ method query*( return success (Key.none, EmptyBytes) var - res = ThreadResult[(bool, DataBuffer, DataBuffer)]() - ctx = TaskCtx[(bool, DataBuffer, DataBuffer)]( + res = ThreadResult[ThreadQueryRes]() + ctx = TaskCtx[ThreadQueryRes]( ds: self.ds, res: addr res) proc runTask() = self.tp.spawn queryTask(addr ctx, addr childIter) - self.dispatchTask(ctx, Key.none, runTask) - if err =? res.errorOption: - trace "Query failed", err = $err - return failure err - - let (ok, key, data) = res.get() - if not ok: - iter.finished = true - return success (Key.none, EmptyBytes) - - return success (Key.init($key).expect("should not fail").some, @(data)) + return self.dispatchTask(ctx, Key.none, runTask) iter.next = next return success iter @@ -415,17 +407,9 @@ proc new*( tp: Taskpool): ?!ThreadDatastore = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" - case withLocks: - of true: - success ThreadDatastore( - tp: tp, - ds: ds, - withLocks: true, - queryLock: newAsyncLock(), - semaphore: AsyncSemaphore.new(tp.numThreads - 1)) - else: - success ThreadDatastore( - tp: tp, - ds: ds, - withLocks: false, - semaphore: AsyncSemaphore.new(tp.numThreads - 1)) + success ThreadDatastore( + tp: tp, + ds: ds, + withLocks: withLocks, + queryLock: newAsyncLock(), + semaphore: AsyncSemaphore.new(tp.numThreads - 1)) diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim new file mode 100644 index 00000000..fcb7ffdd --- /dev/null +++ b/datastore/threads/threadresult.nim @@ -0,0 +1,43 @@ +import std/atomics +import std/options + +import pkg/questionable/results +import pkg/results + +import ../types +import ../query +import ../key + +import ./databuffer + +type + ErrorEnum* {.pure.} = enum + DatastoreErr, + DatastoreKeyNotFoundErr, + QueryEndedErr, + CatchableErr + + ThreadTypes* = void | bool | SomeInteger | DataBuffer | tuple | Atomic + ThreadResErr* = (ErrorEnum, DataBuffer) + ThreadQueryRes* = (DataBuffer, DataBuffer) + ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] + +converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = + if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) + elif e of QueryEndedError: (ErrorEnum.QueryEndedErr, DataBuffer.new(e.msg)) + elif e of DatastoreError: (DatastoreErr, DataBuffer.new(e.msg)) + elif e of CatchableError: (CatchableErr, DataBuffer.new(e.msg)) + else: raise (ref Defect)(msg: e.msg) + +converter toExc*(e: ThreadResErr): ref CatchableError = + case e[0]: + of ErrorEnum.DatastoreKeyNotFoundErr: (ref DatastoreKeyNotFound)(msg: $e[1]) + of ErrorEnum.QueryEndedErr: (ref QueryEndedError)(msg: $e[1]) + of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) + of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) + +converter toQueryResponse*(r: ThreadQueryRes): QueryResponse = + if not r[0].isNil and r[0].len > 0 and key =? Key.init($r[0]): + (key.some, @(r[1])) + else: + (Key.none, EmptyBytes) diff --git a/datastore/types.nim b/datastore/types.nim index b019cdb0..473db527 100644 --- a/datastore/types.nim +++ b/datastore/types.nim @@ -6,5 +6,6 @@ const type DatastoreError* = object of CatchableError DatastoreKeyNotFound* = object of DatastoreError + QueryEndedError* = object of DatastoreError Datastore* = ref object of RootObj From bb304a2f9d452102cb354224336e8895054174be Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 18:43:34 -0600 Subject: [PATCH 164/445] fix broken key not found test --- tests/datastore/dscommontests.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/dscommontests.nim b/tests/datastore/dscommontests.nim index 72a7394c..24dd7d39 100644 --- a/tests/datastore/dscommontests.nim +++ b/tests/datastore/dscommontests.nim @@ -61,5 +61,5 @@ proc basicStoreTests*( test "handle missing key": let key = Key.init("/missing/key").tryGet() - # TODO: map error correctly from threadproxy - check (await ds.get(key)).isErr() # non existing key + expect(DatastoreKeyNotFound): + discard (await ds.get(key)).tryGet() # non existing key From 181168d0738bb3ac66ee630099873fef39e5a29a Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 19:03:40 -0600 Subject: [PATCH 165/445] fix tired ds --- datastore/tieredds.nim | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/datastore/tieredds.nim b/datastore/tieredds.nim index 4eca23be..8448f9bf 100644 --- a/datastore/tieredds.nim +++ b/datastore/tieredds.nim @@ -74,13 +74,16 @@ method get*( bytes: seq[byte] for store in self.stores: - without bytes =? (await store.get(key)): - continue + without bytes =? (await store.get(key)), err: + if err of DatastoreKeyNotFound: + continue + else: + return failure(err) if bytes.len <= 0: continue - # put found data into stores logically in front of the current store + # put found data into stores in front of the current store for s in self.stores: if s == store: break if( @@ -88,7 +91,10 @@ method get*( res.isErr): return failure res.error - return success bytes + if bytes.len > 0: + return success bytes + + return failure (ref DatastoreKeyNotFound)(msg: "Key not found") method put*( self: TieredDatastore, From 14f8c3a71cbc7682dc4c61bf386f18825e395003 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 21:10:38 -0600 Subject: [PATCH 166/445] check for nil ctx and set iter.finished correctly --- datastore/threads/threadproxyds.nim | 52 ++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index fc69a690..cf331601 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -115,6 +115,10 @@ proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = ## the cancellation flag is set ## + if ctx.isNil: + trace "ctx is nil" + return + try: await ctx[].signal.wait() trace "Received signal" @@ -134,7 +138,12 @@ proc asyncHasTask( ctx: ptr TaskCtx[bool], key: ptr Key) {.async.} = defer: - discard ctx[].signal.fireSync() + if not ctx.isNil: + discard ctx[].signal.fireSync() + + if ctx.isNil: + trace "ctx is nil" + return let fut = ctx[].ds.has(key[]) @@ -167,7 +176,12 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = defer: - discard ctx[].signal.fireSync() + if not ctx.isNil: + discard ctx[].signal.fireSync() + + if ctx.isNil: + trace "ctx is nil" + return let fut = ctx[].ds.delete(key[]) @@ -218,7 +232,12 @@ proc asyncPutTask( data: ptr UncheckedArray[byte], len: int) {.async.} = defer: - discard ctx[].signal.fireSync() + if not ctx.isNil: + discard ctx[].signal.fireSync() + + if ctx.isNil: + trace "ctx is nil" + return let fut = ctx[].ds.put(key[], @(data.toOpenArray(0, len - 1))) @@ -278,7 +297,12 @@ proc asyncGetTask( ctx: ptr TaskCtx[DataBuffer], key: ptr Key) {.async.} = defer: - discard ctx[].signal.fireSync() + if not ctx.isNil: + discard ctx[].signal.fireSync() + + if ctx.isNil: + trace "ctx is nil" + return let fut = ctx[].ds.get(key[]) @@ -328,7 +352,12 @@ proc asyncQueryTask( ctx: ptr TaskCtx, iter: ptr QueryIter) {.async.} = defer: - discard ctx[].signal.fireSync() + if not ctx.isNil: + discard ctx[].signal.fireSync() + + if ctx.isNil or iter.isNil: + trace "ctx is nil" + return let fut = iter[].next() @@ -368,24 +397,23 @@ method query*( var iter = QueryIter.new() - locked = false + lock = newAsyncLock() # serialize querying under threads proc next(): Future[?!QueryResponse] {.async.} = defer: - locked = false + if lock.locked: + lock.release() trace "About to query" - if locked: + if lock.locked: return failure (ref DatastoreError)(msg: "Should always await query features") - locked = true + await lock.acquire() if iter.finished == true: return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - if iter.finished == true: - return success (Key.none, EmptyBytes) - + iter.finished = childIter.finished var res = ThreadResult[ThreadQueryRes]() ctx = TaskCtx[ThreadQueryRes]( From 7ceccf9a72e42541a14c03431c2f71deb5a751fd Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 19 Sep 2023 21:54:02 -0600 Subject: [PATCH 167/445] get rid of unsafeAddr everywhere --- datastore/sql/sqliteutils.nim | 3 +- datastore/threads/threadproxyds.nim | 53 ++++++++++++++++------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index bffb6e8c..4d2254ab 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -50,7 +50,8 @@ proc bindParam( # to the return from sqlite3_bind_*(). The object and pointer to it # must remain valid until then. SQLite will then manage the lifetime of # its private copy." - sqlite3_bind_blob(s, n.cint, unsafeAddr val[0], val.len.cint, + var val = val + sqlite3_bind_blob(s, n.cint, addr val[0], val.len.cint, SQLITE_TRANSIENT) else: sqlite3_bind_null(s, n.cint) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index cf331601..3fd08a5c 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -137,10 +137,6 @@ proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = proc asyncHasTask( ctx: ptr TaskCtx[bool], key: ptr Key) {.async.} = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - if ctx.isNil: trace "ctx is nil" return @@ -156,6 +152,10 @@ proc asyncHasTask( ctx[].res[].ok(res) proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = + defer: + if not ctx.isNil: + discard ctx[].signal.fireSync() + try: waitFor asyncHasTask(ctx, key) except CatchableError as exc: @@ -164,21 +164,18 @@ proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var + key = key res = ThreadResult[bool]() ctx = TaskCtx[bool]( ds: self.ds, res: addr res) proc runTask() = - self.tp.spawn hasTask(addr ctx, unsafeAddr key) + self.tp.spawn hasTask(addr ctx, addr key) return self.dispatchTask(ctx, key.some, runTask) proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - if ctx.isNil: trace "ctx is nil" return @@ -196,6 +193,10 @@ proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = return proc delTask(ctx: ptr TaskCtx, key: ptr Key) = + defer: + if not ctx.isNil: + discard ctx[].signal.fireSync() + try: waitFor asyncDelTask(ctx, key) except CatchableError as exc: @@ -206,13 +207,14 @@ method delete*( self: ThreadDatastore, key: Key): Future[?!void] {.async.} = var + key = key res = ThreadResult[void]() ctx = TaskCtx[void]( ds: self.ds, res: addr res) proc runTask() = - self.tp.spawn delTask(addr ctx, unsafeAddr key) + self.tp.spawn delTask(addr ctx, addr key) return self.dispatchTask(ctx, key.some, runTask) @@ -231,9 +233,6 @@ proc asyncPutTask( key: ptr Key, data: ptr UncheckedArray[byte], len: int) {.async.} = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() if ctx.isNil: trace "ctx is nil" @@ -258,6 +257,10 @@ proc putTask( ## run put in a thread task ## + defer: + if not ctx.isNil: + discard ctx[].signal.fireSync() + try: waitFor asyncPutTask(ctx, key, data, len) except CatchableError as exc: @@ -269,6 +272,8 @@ method put*( key: Key, data: seq[byte]): Future[?!void] {.async.} = var + key = key + data = data res = ThreadResult[void]() ctx = TaskCtx[void]( ds: self.ds, @@ -277,8 +282,8 @@ method put*( proc runTask() = self.tp.spawn putTask( addr ctx, - unsafeAddr key, - makeUncheckedArray(baseAddr data), + addr key, + makeUncheckedArray(addr data[0]), data.len) return self.dispatchTask(ctx, key.some, runTask) @@ -296,10 +301,6 @@ method put*( proc asyncGetTask( ctx: ptr TaskCtx[DataBuffer], key: ptr Key) {.async.} = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - if ctx.isNil: trace "ctx is nil" return @@ -322,6 +323,10 @@ proc getTask( ## Run get in a thread task ## + defer: + if not ctx.isNil: + discard ctx[].signal.fireSync() + try: waitFor asyncGetTask(ctx, key) except CatchableError as exc: @@ -332,13 +337,14 @@ method get*( self: ThreadDatastore, key: Key): Future[?!seq[byte]] {.async.} = var + key = key res = ThreadResult[DataBuffer]() ctx = TaskCtx[DataBuffer]( ds: self.ds, res: addr res) proc runTask() = - self.tp.spawn getTask(addr ctx, unsafeAddr key) + self.tp.spawn getTask(addr ctx, addr key) return self.dispatchTask(ctx, key.some, runTask) @@ -351,9 +357,6 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = proc asyncQueryTask( ctx: ptr TaskCtx, iter: ptr QueryIter) {.async.} = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() if ctx.isNil or iter.isNil: trace "ctx is nil" @@ -383,6 +386,10 @@ proc queryTask( ctx: ptr TaskCtx, iter: ptr QueryIter) = + defer: + if not ctx.isNil: + discard ctx[].signal.fireSync() + try: waitFor asyncQueryTask(ctx, iter) except CatchableError as exc: From 2eb120bb668f5df2d36d1088503d492f8c8ab375 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 20 Sep 2023 09:42:37 -0600 Subject: [PATCH 168/445] make path threadvar --- datastore/fsds.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 6c695bb9..f0c1c6ae 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -155,6 +155,9 @@ method put*( return success() proc dirWalker(path: string): iterator: string {.gcsafe.} = + var localPath {.threadvar.} + + localPath = path return iterator(): string = try: for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): From 81372c96d7f640058b10a724494d19e15c283899 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 20 Sep 2023 09:43:27 -0600 Subject: [PATCH 169/445] duh --- datastore/fsds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index f0c1c6ae..65f5cdd4 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -155,7 +155,7 @@ method put*( return success() proc dirWalker(path: string): iterator: string {.gcsafe.} = - var localPath {.threadvar.} + var localPath {.threadvar.}: string localPath = path return iterator(): string = From 3d84781d3e443612d338bc36dfe86b2d346a7a62 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 20 Sep 2023 14:00:29 -0600 Subject: [PATCH 170/445] copy data and keys to thread local gc --- datastore/threads/threadproxyds.nim | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 3fd08a5c..31a65baf 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -142,7 +142,8 @@ proc asyncHasTask( return let - fut = ctx[].ds.has(key[]) + key = key[] + fut = ctx[].ds.has(key) asyncSpawn signalMonitor(ctx, fut) without ret =? (await fut).catch and res =? ret, error: @@ -181,7 +182,8 @@ proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = return let - fut = ctx[].ds.delete(key[]) + key = key[] + fut = ctx[].ds.delete(key) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: @@ -239,7 +241,9 @@ proc asyncPutTask( return let - fut = ctx[].ds.put(key[], @(data.toOpenArray(0, len - 1))) + key = key[] + data = @(data.toOpenArray(0, len - 1)) + fut = ctx[].ds.put(key, data) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch, error: @@ -306,7 +310,8 @@ proc asyncGetTask( return let - fut = ctx[].ds.get(key[]) + key = key[] + fut = ctx[].ds.get(key) asyncSpawn signalMonitor(ctx, fut) without res =? (await fut).catch and data =? res, error: @@ -314,7 +319,7 @@ proc asyncGetTask( ctx[].res[].err(error) return - trace "Got data in get", data + trace "Got data in get" ctx[].res[].ok(DataBuffer.new(data)) proc getTask( From 9013a0c61900e54bf8be8d7c4ed5fe581bf1a58c Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 20 Sep 2023 14:01:08 -0600 Subject: [PATCH 171/445] GC_fullcollect() in tests to check mem consistency --- tests/datastore/testthreadproxyds.nim | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 10197dfa..528a3e7c 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -38,6 +38,9 @@ suite "Test Basic ThreadDatastore with SQLite": taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + teardown: + GC_fullCollect() + teardownAll: (await ds.close()).tryGet() taskPool.shutdown() @@ -60,6 +63,8 @@ suite "Test Query ThreadDatastore with SQLite": ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: + GC_fullCollect() + (await ds.close()).tryGet() taskPool.shutdown() @@ -88,6 +93,9 @@ suite "Test Basic ThreadDatastore with fsds": taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() + teardown: + GC_fullCollect() + teardownAll: (await ds.close()).tryGet() taskPool.shutdown() @@ -118,6 +126,7 @@ suite "Test Query ThreadDatastore with fsds": ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() teardown: + GC_fullCollect() (await ds.close()).tryGet() taskPool.shutdown() @@ -143,6 +152,13 @@ suite "Test ThreadDatastore cancelations": taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + teardown: + GC_fullCollect() + + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() + test "Should monitor signal and cancel": var signal = ThreadSignalPtr.new().tryGet() From 215d3344006b9168a8501d991ff259f8e4e71fb5 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 20 Sep 2023 17:29:25 -0600 Subject: [PATCH 172/445] avoid segfault on fullcollect --- tests/datastore/testthreadproxyds.nim | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 528a3e7c..e81a2356 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -140,9 +140,6 @@ suite "Test ThreadDatastore cancelations": sqlStore: Datastore ds: ThreadDatastore taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes privateAccess(ThreadDatastore) # expose private fields privateAccess(TaskCtx) # expose private fields @@ -153,7 +150,7 @@ suite "Test ThreadDatastore cancelations": ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: - GC_fullCollect() + GC_fullCollect() # run full collect after each test teardownAll: (await ds.close()).tryGet() @@ -169,8 +166,6 @@ suite "Test ThreadDatastore cancelations": signal: signal) fut = newFuture[void]("signalMonitor") threadArgs = (addr ctx, addr fut) - - var thread: Thread[type threadArgs] proc threadTask(args: type threadArgs) = @@ -191,6 +186,7 @@ suite "Test ThreadDatastore cancelations": check: fut.cancelled check: ctx.signal.close().isOk + fut = nil test "Should monitor and not cancel": var @@ -202,8 +198,6 @@ suite "Test ThreadDatastore cancelations": signal: signal) fut = newFuture[void]("signalMonitor") threadArgs = (addr ctx, addr fut) - - var thread: Thread[type threadArgs] proc threadTask(args: type threadArgs) = @@ -224,3 +218,4 @@ suite "Test ThreadDatastore cancelations": check: not fut.cancelled check: ctx.signal.close().isOk + fut = nil From 84bbfa687e0fd68ce66f31cd90adb834a36baee1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 17:12:19 -0700 Subject: [PATCH 173/445] refactor sqliteds --- datastore/sql.nim | 248 ++++++++++++++++++++++++++++++++++++- datastore/sql/sqliteds.nim | 54 ++++---- 2 files changed, 269 insertions(+), 33 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 9c30c9e5..e5fad22d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -1,3 +1,247 @@ -import ./sql/sqliteds +import std/times +import std/options -export sqliteds +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/sqlite3_abi +from pkg/stew/results as stewResults import isErr +import pkg/upraises + +import ../datastore +import ./sqlitedsdb + +export datastore, sqlitedsdb + +push: {.upraises: [].} + +type + SQLiteDatastore* = ref object of Datastore + readOnly: bool + db: SQLiteDsDb + +proc path*(self: SQLiteDatastore): string = + self.db.dbPath + +proc `readOnly=`*(self: SQLiteDatastore): bool + {.error: "readOnly should not be assigned".} + +proc timestamp*(t = epochTime()): int64 = + (t * 1_000_000).int64 + +method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = + var + exists = false + + proc onData(s: RawStmtPtr) = + exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool + + if err =? self.db.containsStmt.query((key.id), onData).errorOption: + return failure err + + return success exists + +method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = + return self.db.deleteStmt.exec((key.id)) + +method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = + if err =? self.db.beginStmt.exec().errorOption: + return failure(err) + + for key in keys: + if err =? self.db.deleteStmt.exec((key.id)).errorOption: + if err =? self.db.rollbackStmt.exec().errorOption: + return failure err.msg + + return failure err.msg + + if err =? self.db.endStmt.exec().errorOption: + return failure err.msg + + return success() + +method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = + # see comment in ./filesystem_datastore re: finer control of memory + # allocation in `method get`, could apply here as well if bytes were read + # incrementally with `sqlite3_blob_read` + + var + bytes: seq[byte] + + proc onData(s: RawStmtPtr) = + bytes = self.db.getDataCol() + + if err =? self.db.getStmt.query((key.id), onData).errorOption: + return failure(err) + + if bytes.len <= 0: + return failure( + newException(DatastoreKeyNotFound, "Key doesn't exist")) + + return success bytes + +method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = + return self.db.putStmt.exec((key.id, data, timestamp())) + +method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = + if err =? self.db.beginStmt.exec().errorOption: + return failure err + + for entry in batch: + if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: + if err =? self.db.rollbackStmt.exec().errorOption: + return failure err + + return failure err + + if err =? self.db.endStmt.exec().errorOption: + return failure err + + return success() + +method close*(self: SQLiteDatastore): Future[?!void] {.async.} = + self.db.close() + + return success() + +method query*( + self: SQLiteDatastore, + query: Query): Future[?!QueryIter] {.async.} = + + var + iter = QueryIter() + queryStr = if query.value: + QueryStmtDataIdStr + else: + QueryStmtIdStr + + if query.sort == SortOrder.Descending: + queryStr &= QueryStmtOrderDescending + else: + queryStr &= QueryStmtOrderAscending + + if query.limit != 0: + queryStr &= QueryStmtLimit + + if query.offset != 0: + queryStr &= QueryStmtOffset + + let + queryStmt = QueryStmt.prepare( + self.db.env, queryStr).expect("should not fail") + + s = RawStmtPtr(queryStmt) + + var + v = sqlite3_bind_text( + s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + if query.limit != 0: + v = sqlite3_bind_int(s, 2.cint, query.limit.cint) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + if query.offset != 0: + v = sqlite3_bind_int(s, 3.cint, query.offset.cint) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + let lock = newAsyncLock() + proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() + + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") + + if iter.finished: + return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) + + await lock.acquire() + + let + v = sqlite3_step(s) + + case v + of SQLITE_ROW: + let + key = Key.init( + $sqlite3_column_text_not_null(s, QueryStmtIdCol)) + .expect("should not fail") + + blob: ?pointer = + if query.value: + sqlite3_column_blob(s, QueryStmtDataCol).some + else: + pointer.none + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let + v = sqlite3_errcode(sqlite3_db_handle(s)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + iter.finished = true + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + data = if blob.isSome: + @( + toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), + 0, + dataLen - 1)) + else: + @[] + + return success (key.some, data) + of SQLITE_DONE: + iter.finished = true + return success (Key.none, EmptyBytes) + else: + iter.finished = true + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + iter.dispose = proc(): Future[?!void] {.async.} = + discard sqlite3_reset(s) + discard sqlite3_clear_bindings(s) + s.dispose + return success() + + iter.next = next + return success iter + +proc new*( + T: type SQLiteDatastore, + path: string, + readOnly = false): ?!T = + + let + flags = + if readOnly: SQLITE_OPEN_READONLY + else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE + + success T( + db: ? SQLiteDsDb.open(path, flags), + readOnly: readOnly) + +proc new*( + T: type SQLiteDatastore, + db: SQLiteDsDb): ?!T = + + success T( + db: db, + readOnly: db.readOnly) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index e5fad22d..8cd7f858 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -1,7 +1,6 @@ import std/times import std/options -import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/sqlite3_abi @@ -16,20 +15,18 @@ export datastore, sqlitedsdb push: {.upraises: [].} type - SQLiteDatastore* = ref object of Datastore - readOnly: bool + SQLiteDatastore* = object db: SQLiteDsDb proc path*(self: SQLiteDatastore): string = self.db.dbPath -proc `readOnly=`*(self: SQLiteDatastore): bool - {.error: "readOnly should not be assigned".} +proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = +proc has*(self: SQLiteDatastore, key: Key): ?!bool = var exists = false @@ -41,10 +38,10 @@ method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = return success exists -method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = +proc delete*(self: SQLiteDatastore, key: Key): ?!void = return self.db.deleteStmt.exec((key.id)) -method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = +proc delete*(self: SQLiteDatastore, keys: seq[Key]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) @@ -60,9 +57,9 @@ method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} return success() -method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = +proc get*(self: SQLiteDatastore, key: Key): ?!seq[byte] = # see comment in ./filesystem_datastore re: finer control of memory - # allocation in `method get`, could apply here as well if bytes were read + # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` var @@ -80,10 +77,10 @@ method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = return success bytes -method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = +proc put*(self: SQLiteDatastore, key: Key, data: seq[byte]): ?!void = return self.db.putStmt.exec((key.id, data, timestamp())) -method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = +proc put*(self: SQLiteDatastore, batch: seq[BatchEntry]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err @@ -99,14 +96,14 @@ method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.asy return success() -method close*(self: SQLiteDatastore): Future[?!void] {.async.} = +proc close*(self: SQLiteDatastore): ?!void = self.db.close() return success() -method query*( - self: SQLiteDatastore, - query: Query): Future[?!QueryIter] {.async.} = +proc query*(self: SQLiteDatastore, + query: Query + ): ?!QueryIter {.async.} = var iter = QueryIter() @@ -152,7 +149,7 @@ method query*( return failure newException(DatastoreError, $sqlite3_errstr(v)) let lock = newAsyncLock() - proc next(): Future[?!QueryResponse] {.async.} = + proc next(): ?!QueryResponse = defer: if lock.locked: lock.release() @@ -215,7 +212,7 @@ method query*( iter.finished = true return failure newException(DatastoreError, $sqlite3_errstr(v)) - iter.dispose = proc(): Future[?!void] {.async.} = + iter.dispose = proc(): ?!void = discard sqlite3_reset(s) discard sqlite3_clear_bindings(s) s.dispose @@ -224,24 +221,19 @@ method query*( iter.next = next return success iter -proc new*( - T: type SQLiteDatastore, - path: string, - readOnly = false): ?!T = +proc new*(T: type SQLiteDatastore, + path: string, + readOnly = false): ?!T = let flags = if readOnly: SQLITE_OPEN_READONLY else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - success T( - db: ? SQLiteDsDb.open(path, flags), - readOnly: readOnly) + success SQLiteDatastore(db: ? SQLiteDsDb.open(path, flags)) + -proc new*( - T: type SQLiteDatastore, - db: SQLiteDsDb): ?!T = +proc new*(T: type SQLiteDatastore, + db: SQLiteDsDb): ?!T = - success T( - db: db, - readOnly: db.readOnly) + success SQLiteDatastore(db: db) From ea3546f5b33b8929cae1906f083c0d87fcf36f95 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 17:19:03 -0700 Subject: [PATCH 174/445] refactor sqliteds --- datastore/sql/sqliteds.nim | 5 +++-- datastore/threads/threadresult.nim | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 8cd7f858..42412924 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -101,9 +101,10 @@ proc close*(self: SQLiteDatastore): ?!void = return success() -proc query*(self: SQLiteDatastore, + +iterator query*(self: SQLiteDatastore, query: Query - ): ?!QueryIter {.async.} = + ): ?!ThreadQueryRes = var iter = QueryIter() diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index fcb7ffdd..07c5108e 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -22,6 +22,9 @@ type ThreadQueryRes* = (DataBuffer, DataBuffer) ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] + DbKey* = tuple[data: DataBuffer] + DbValue* = tuple[data: DataBuffer] + converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) elif e of QueryEndedError: (ErrorEnum.QueryEndedErr, DataBuffer.new(e.msg)) From 1833155e53d0f34bc5b1beeefcc184abda7182bc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 17:27:14 -0700 Subject: [PATCH 175/445] add key/value --- datastore/threads/threadresult.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 07c5108e..746e45ed 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -25,6 +25,18 @@ type DbKey* = tuple[data: DataBuffer] DbValue* = tuple[data: DataBuffer] +proc toDb*(key: Key): DbKey {.inline, raises: [].} = + (data: DataBuffer.new(key.id())) + +proc toKey*(key: DbKey): Key {.inline, raises: [].} = + Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") + +proc toDb*(value: sink seq[byte]): DbValue {.inline, raises: [].} = + (data: DataBuffer.new(value)) + +proc toValue*(value: DbValue): seq[byte] {.inline, raises: [].} = + value.data.toSeq() + converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) elif e of QueryEndedError: (ErrorEnum.QueryEndedErr, DataBuffer.new(e.msg)) From f7a933a60ab17a622c2fbb3730ec2c647e5eaac8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 17:49:57 -0700 Subject: [PATCH 176/445] openArray --- datastore/backend.nim | 10 ++++++++++ datastore/sql/sqliteds.nim | 22 +++++++++++----------- datastore/sql/sqliteutils.nim | 2 +- datastore/threads/databuffer.nim | 7 +++++++ datastore/threads/threadresult.nim | 3 +++ tests/datastore/testdatabuffer.nim | 10 ++++++++++ 6 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 datastore/backend.nim diff --git a/datastore/backend.nim b/datastore/backend.nim new file mode 100644 index 00000000..9559f50b --- /dev/null +++ b/datastore/backend.nim @@ -0,0 +1,10 @@ +import pkg/questionable/results +import pkg/upraises + +import ./threads/databuffer +import ./threads/threadresult +import ./threads/semaphore +import ./types + +export databuffer, threadresult, semaphore, types +export upraises, results diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 42412924..136ac719 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -7,10 +7,10 @@ import pkg/sqlite3_abi from pkg/stew/results as stewResults import isErr import pkg/upraises -import ../datastore +import ../backend import ./sqlitedsdb -export datastore, sqlitedsdb +export backend, sqlitedsdb push: {.upraises: [].} @@ -26,7 +26,7 @@ proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*(self: SQLiteDatastore, key: Key): ?!bool = +proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = var exists = false @@ -38,10 +38,10 @@ proc has*(self: SQLiteDatastore, key: Key): ?!bool = return success exists -proc delete*(self: SQLiteDatastore, key: Key): ?!void = - return self.db.deleteStmt.exec((key.id)) +proc delete*(self: SQLiteDatastore, key: DbKey): ?!void = + return self.db.deleteStmt.exec((key.data)) -proc delete*(self: SQLiteDatastore, keys: seq[Key]): ?!void = +proc delete*(self: SQLiteDatastore, keys: seq[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) @@ -57,7 +57,7 @@ proc delete*(self: SQLiteDatastore, keys: seq[Key]): ?!void = return success() -proc get*(self: SQLiteDatastore, key: Key): ?!seq[byte] = +proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = # see comment in ./filesystem_datastore re: finer control of memory # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` @@ -73,11 +73,11 @@ proc get*(self: SQLiteDatastore, key: Key): ?!seq[byte] = if bytes.len <= 0: return failure( - newException(DatastoreKeyNotFound, "Key doesn't exist")) + newException(DatastoreKeyNotFound, "DbKey doesn't exist")) return success bytes -proc put*(self: SQLiteDatastore, key: Key, data: seq[byte]): ?!void = +proc put*(self: SQLiteDatastore, key: DbKey, data: seq[byte]): ?!void = return self.db.putStmt.exec((key.id, data, timestamp())) proc put*(self: SQLiteDatastore, batch: seq[BatchEntry]): ?!void = @@ -169,7 +169,7 @@ iterator query*(self: SQLiteDatastore, case v of SQLITE_ROW: let - key = Key.init( + key = DbKey.init( $sqlite3_column_text_not_null(s, QueryStmtIdCol)) .expect("should not fail") @@ -208,7 +208,7 @@ iterator query*(self: SQLiteDatastore, return success (key.some, data) of SQLITE_DONE: iter.finished = true - return success (Key.none, EmptyBytes) + return success (DbKey.none, EmptyBytes) else: iter.finished = true return failure newException(DatastoreError, $sqlite3_errstr(v)) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 4d2254ab..54b20907 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -61,7 +61,7 @@ proc bindParam( sqlite3_bind_int64(s, n.cint, val.int64) elif val is float32 | float64: sqlite3_bind_double(s, n.cint, val.float64) - elif val is string: + elif val is string|openArray[char]: # `-1` implies string length is num bytes up to first null-terminator; # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it must diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 66bb7e66..130490e1 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -90,3 +90,10 @@ converter toBuffer*(err: ref CatchableError): DataBuffer = ## return DataBuffer.new(err.msg) + +template toOpenArray*[T: byte | char](data: DataBuffer, t: typedesc[T]): openArray[T] = + ## get openArray from DataBuffer as char + ## + ## this is explicit since sqlite treats string differently from openArray[byte] + let bf = cast[ptr UncheckedArray[T]](data[].buf) + bf.toOpenArray(0, data[].size-1) diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 746e45ed..15a442a4 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -37,6 +37,9 @@ proc toDb*(value: sink seq[byte]): DbValue {.inline, raises: [].} = proc toValue*(value: DbValue): seq[byte] {.inline, raises: [].} = value.data.toSeq() +template toOpenArray*(x: DbKey): openArray[char] = + x.data[].buf.toOpenArray(0, x[].size-1) + converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) elif e of QueryEndedError: (ErrorEnum.QueryEndedErr, DataBuffer.new(e.msg)) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 7fc3ceba..4969f9c4 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -103,5 +103,15 @@ suite "Share buffer test": test "seq conversion": check a.toSeq() == "/a/b".toBytes + test "basic openArray test": + proc letters(val: openArray[char]): int = + val.len() + proc bytes(val: openArray[byte]): int = + val.len() + + check a.toOpenArray(char).letters() == a.len() + check a.toOpenArray(byte).bytes() == a.len() + # check a.toOpenArray(char).bytes() == a.len() + test "basic thread test": runBasicTest() From 77efc50469bf6c5a840373fb8b4d5dfa765d27ed Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 18:08:27 -0700 Subject: [PATCH 177/445] openArray --- datastore/sql/sqliteutils.nim | 11 ++++++++++- datastore/threads/databuffer.nim | 9 +++++++-- datastore/threads/threadresult.nim | 21 ++++++++++++++------- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 54b20907..d070ce24 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -3,6 +3,8 @@ import pkg/questionable/results import pkg/sqlite3_abi import pkg/upraises +import ../backend + export sqlite3_abi # Adapted from: @@ -61,7 +63,14 @@ proc bindParam( sqlite3_bind_int64(s, n.cint, val.int64) elif val is float32 | float64: sqlite3_bind_double(s, n.cint, val.float64) - elif val is string|openArray[char]: + elif val is string: + # `-1` implies string length is num bytes up to first null-terminator; + # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior + # to the return from sqlite3_bind_*(). The object and pointer to it must + # remain valid until then. SQLite will then manage the lifetime of its + # private copy." + sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) + elif val is DbKey: # `-1` implies string length is num bytes up to first null-terminator; # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it must diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 130490e1..4de7b053 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -51,10 +51,15 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T]): DataBuffer = ## result = DataBuffer.new(data.len) if data.len() > 0: - # TODO: we might want to copy data, otherwise the GC might - # release it on stack-unwind copyMem(result[].buf, baseAddr data, data.len) +proc setData*[T: byte | char](db: DataBuffer, data: openArray[T]) = + ## allocate new buffer and copies indata from openArray + ## + if data.len() > db.len(): + raise newException(IndexDefect, "data too large for buffer") + copyMem(db[].buf, baseAddr data, data.len) + converter toSeq*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char ## diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 15a442a4..f430d8fa 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -22,23 +22,30 @@ type ThreadQueryRes* = (DataBuffer, DataBuffer) ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] - DbKey* = tuple[data: DataBuffer] - DbValue* = tuple[data: DataBuffer] + DbKey* = object + data: DataBuffer + DbVal* = object + data: DataBuffer proc toDb*(key: Key): DbKey {.inline, raises: [].} = - (data: DataBuffer.new(key.id())) + let id: string = key.id() + let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat + db.setData(id) + DbKey(data: db) proc toKey*(key: DbKey): Key {.inline, raises: [].} = Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") -proc toDb*(value: sink seq[byte]): DbValue {.inline, raises: [].} = - (data: DataBuffer.new(value)) +proc toDb*(value: sink seq[byte]): DbVal {.inline, raises: [].} = + DbVal(data: DataBuffer.new(value)) -proc toValue*(value: DbValue): seq[byte] {.inline, raises: [].} = +proc toValue*(value: DbVal): seq[byte] {.inline, raises: [].} = value.data.toSeq() template toOpenArray*(x: DbKey): openArray[char] = - x.data[].buf.toOpenArray(0, x[].size-1) + x.data.toOpenArray(char) +template toOpenArray*(x: DbVal): openArray[byte] = + x.data.toOpenArray(byte) converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) From 1627e4d286341340c6f2099cb4da58092b328596 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 19:30:23 -0700 Subject: [PATCH 178/445] add null tests --- datastore/threads/databuffer.nim | 31 ++++++++++++++++++++++-------- tests/datastore/testdatabuffer.nim | 6 ++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 4de7b053..a7b0f3fe 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -5,9 +5,13 @@ import pkg/stew/ptrops export hashes type + DataBufferOpt* = enum + dbNullTerminate + DataBufferHolder* = object buf: ptr UncheckedArray[byte] size: int + cap: int DataBuffer* = SharedPtr[DataBufferHolder] ##\ ## A fixed length data buffer using a SharedPtr. @@ -24,6 +28,7 @@ proc `=destroy`*(x: var DataBufferHolder) = deallocShared(x.buf) proc len*(a: DataBuffer): int = a[].size +proc capacity*(a: DataBuffer): int = a[].cap proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) @@ -37,28 +42,38 @@ proc `==`*(a, b: DataBuffer): bool = elif a[].buf == b[].buf: return true else: a.hash() == b.hash() -proc new*(tp: type DataBuffer, size: int = 0): DataBuffer = - ## allocate new buffer with given size +proc new*(tp: type DataBuffer, capacity: int = 0): DataBuffer = + ## allocate new buffer with given capacity ## newSharedPtr(DataBufferHolder( - buf: cast[typeof(result[].buf)](allocShared0(size)), - size: size, + buf: cast[typeof(result[].buf)](allocShared0(capacity)), + size: 0, + cap: capacity, )) -proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T]): DataBuffer = +proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[DataBufferOpt] = {}): DataBuffer = ## allocate new buffer and copies indata from openArray ## - result = DataBuffer.new(data.len) + let dataCap = + if dbNullTerminate in opts: data.len() + 1 + else: data.len() + result = DataBuffer.new(dataCap) if data.len() > 0: - copyMem(result[].buf, baseAddr data, data.len) + copyMem(result[].buf, baseAddr data, data.len()) + result[].size = data.len() + +proc clear*(db: DataBuffer) = + zeroMem(db[].buf, db[].cap) proc setData*[T: byte | char](db: DataBuffer, data: openArray[T]) = ## allocate new buffer and copies indata from openArray ## if data.len() > db.len(): raise newException(IndexDefect, "data too large for buffer") - copyMem(db[].buf, baseAddr data, data.len) + db.clear() # this is expensive, but we can optimize later + copyMem(db[].buf, baseAddr data, data.len()) + db[].size = data.len() converter toSeq*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 4969f9c4..58d1e827 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -103,6 +103,12 @@ suite "Share buffer test": test "seq conversion": check a.toSeq() == "/a/b".toBytes + test "basic null terminate test": + let cstr = DataBuffer.new("test", {dbNullTerminate}) + check cstr.len() == 4 + check cstr.capacity() == 5 + check "test" == cstr.toString() + test "basic openArray test": proc letters(val: openArray[char]): int = val.len() From b388d242bba2df2f9078f199bd952c14bc6fca86 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 19:48:31 -0700 Subject: [PATCH 179/445] openArray compare --- datastore/threads/databuffer.nim | 8 +++++++- tests/datastore/testdatabuffer.nim | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index a7b0f3fe..3fcad0e2 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -42,6 +42,11 @@ proc `==`*(a, b: DataBuffer): bool = elif a[].buf == b[].buf: return true else: a.hash() == b.hash() +template `==`*[T: char | byte](a: DataBuffer, b: openArray[T]): bool = + if a.isNil: false + elif a[].size != b.len: false + else: a.hash() == b.hash() + proc new*(tp: type DataBuffer, capacity: int = 0): DataBuffer = ## allocate new buffer with given capacity ## @@ -65,11 +70,12 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[Dat proc clear*(db: DataBuffer) = zeroMem(db[].buf, db[].cap) + db[].size = 0 proc setData*[T: byte | char](db: DataBuffer, data: openArray[T]) = ## allocate new buffer and copies indata from openArray ## - if data.len() > db.len(): + if data.len() > db[].cap: raise newException(IndexDefect, "data too large for buffer") db.clear() # this is expensive, but we can optimize later copyMem(db[].buf, baseAddr data, data.len()) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 58d1e827..0c6fa469 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -109,6 +109,16 @@ suite "Share buffer test": check cstr.capacity() == 5 check "test" == cstr.toString() + test "basic clear test": + let test = DataBuffer.new("test", {dbNullTerminate}) + test.clear() + check "" == test.toString() + test.setData("hi") + check "hi" == test.toString() + + test "check openArray compare": + assert a == toOpenArray(@"/a/b", 0, 3) + test "basic openArray test": proc letters(val: openArray[char]): int = val.len() From 29ff227d3763b8d1f18541b7dbb5360f456b4d56 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:04:44 -0700 Subject: [PATCH 180/445] databuffer type --- datastore/sql/sqliteds.nim | 3 ++- datastore/sql/sqlitedsdb.nim | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 136ac719..9b5604fc 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -29,11 +29,12 @@ proc timestamp*(t = epochTime()): int64 = proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = var exists = false + key = $key proc onData(s: RawStmtPtr) = exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool - if err =? self.db.containsStmt.query((key.id), onData).errorOption: + if err =? self.db.containsStmt.query((key), onData).errorOption: return failure err return success exists diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 503dea45..7dc1c928 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -4,13 +4,14 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises +import ../backend import ./sqliteutils export sqliteutils type BoundIdCol* = proc (): string {.closure, gcsafe, upraises: [].} - BoundDataCol* = proc (): seq[byte] {.closure, gcsafe, upraises: [].} + BoundDataCol* = proc (): DataBuffer {.closure, gcsafe, upraises: [].} BoundTimestampCol* = proc (): int64 {.closure, gcsafe, upraises: [].} # feels odd to use `void` for prepared statements corresponding to SELECT @@ -19,7 +20,7 @@ type ContainsStmt* = SQLiteStmt[(string), void] DeleteStmt* = SQLiteStmt[(string), void] GetStmt* = SQLiteStmt[(string), void] - PutStmt* = SQLiteStmt[(string, seq[byte], int64), void] + PutStmt* = SQLiteStmt[(string, DataBuffer, int64), void] QueryStmt* = SQLiteStmt[(string), void] BeginStmt* = NoParamsStmt EndStmt* = NoParamsStmt @@ -162,7 +163,7 @@ proc dataCol*( checkColMetadata(s, index, DataColName) - return proc (): seq[byte] = + return proc (): DataBuffer = let i = index.cint blob = sqlite3_column_blob(s, i) @@ -186,7 +187,8 @@ proc dataCol*( dataLen = sqlite3_column_bytes(s, i) dataBytes = cast[ptr UncheckedArray[byte]](blob) - @(toOpenArray(dataBytes, 0, dataLen - 1)) + # copy data out, since sqlite will free it + DataBuffer.new(toOpenArray(dataBytes, 0, dataLen - 1)) proc timestampCol*( s: RawStmtPtr, From ab5f8da7366ee4ec939b3368f90a1ad5abda9678 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:14:08 -0700 Subject: [PATCH 181/445] databuffer type --- datastore/backend.nim | 23 +++++++++++++++++++++++ datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 7 ++++++- datastore/threads/threadresult.nim | 24 ------------------------ 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 9559f50b..8b6daa29 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -4,7 +4,30 @@ import pkg/upraises import ./threads/databuffer import ./threads/threadresult import ./threads/semaphore +import ./key import ./types export databuffer, threadresult, semaphore, types export upraises, results + +type + KeyId* = object + ## serialized Key ID, equivalent to `key.id()` + data: DataBuffer + + DbKey* = string | KeyId + DbVal* = seq[byte] | DataBuffer + +proc `$`*(id: KeyId): string = $(id.data) + +proc toDb*(key: Key): DbKey {.inline, raises: [].} = + let id: string = key.id() + let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat + db.setData(id) + DbKey(data: db) + +proc toKey*(key: DbKey): Key {.inline, raises: [].} = + Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") + +template toOpenArray*(x: DbKey): openArray[char] = + x.data.toOpenArray(char) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 9b5604fc..2401095e 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -26,7 +26,7 @@ proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = +proc has*(self: SQLiteDatastore, key: DbKey|string): ?!bool = var exists = false key = $key diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 7dc1c928..c4572c6d 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -20,7 +20,8 @@ type ContainsStmt* = SQLiteStmt[(string), void] DeleteStmt* = SQLiteStmt[(string), void] GetStmt* = SQLiteStmt[(string), void] - PutStmt* = SQLiteStmt[(string, DataBuffer, int64), void] + PutStmt* = SQLiteStmt[(string, seq[byte], int64), void] + PutBufferStmt* = SQLiteStmt[(string, DataBuffer, int64), void] QueryStmt* = SQLiteStmt[(string), void] BeginStmt* = NoParamsStmt EndStmt* = NoParamsStmt @@ -268,6 +269,7 @@ proc open*( deleteStmt: DeleteStmt getStmt: GetStmt putStmt: PutStmt + putBufferStmt: PutStmt beginStmt: BeginStmt endStmt: EndStmt rollbackStmt: RollbackStmt @@ -281,6 +283,9 @@ proc open*( putStmt = ? PutStmt.prepare( env.val, PutStmtStr, SQLITE_PREPARE_PERSISTENT) + putBufferStmt = ? PutBufferStmt.prepare( + env.val, PutStmtStr, SQLITE_PREPARE_PERSISTENT) + beginStmt = ? BeginStmt.prepare( env.val, BeginTransactionStr, SQLITE_PREPARE_PERSISTENT) diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index f430d8fa..b2c2d916 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -22,30 +22,6 @@ type ThreadQueryRes* = (DataBuffer, DataBuffer) ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] - DbKey* = object - data: DataBuffer - DbVal* = object - data: DataBuffer - -proc toDb*(key: Key): DbKey {.inline, raises: [].} = - let id: string = key.id() - let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat - db.setData(id) - DbKey(data: db) - -proc toKey*(key: DbKey): Key {.inline, raises: [].} = - Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") - -proc toDb*(value: sink seq[byte]): DbVal {.inline, raises: [].} = - DbVal(data: DataBuffer.new(value)) - -proc toValue*(value: DbVal): seq[byte] {.inline, raises: [].} = - value.data.toSeq() - -template toOpenArray*(x: DbKey): openArray[char] = - x.data.toOpenArray(char) -template toOpenArray*(x: DbVal): openArray[byte] = - x.data.toOpenArray(byte) converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) From 8b4f388bb37389710951c4d0a6a99fd067c687b6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:26:34 -0700 Subject: [PATCH 182/445] databuffer type --- datastore/backend.nim | 15 ++++++++++++++- datastore/sql/sqliteds.nim | 4 ++-- datastore/threads/threadresult.nim | 9 +++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 8b6daa29..59a9bf3f 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -1,6 +1,8 @@ import pkg/questionable/results import pkg/upraises +import std/algorithm +import std/options import ./threads/databuffer import ./threads/threadresult import ./threads/semaphore @@ -13,11 +15,22 @@ export upraises, results type KeyId* = object ## serialized Key ID, equivalent to `key.id()` - data: DataBuffer + data*: DataBuffer DbKey* = string | KeyId DbVal* = seq[byte] | DataBuffer + DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] + + DbQuery* = object + key*: KeyId # Key to be queried + value*: bool # Flag to indicate if data should be returned + limit*: int # Max items to return - not available in all backends + offset*: int # Offset from which to start querying - not available in all backends + sort*: SortOrder # Sort order - not available in all backends + + DbQueryResponse* = tuple[key: Option[string], data: seq[byte]] | tuple[key: Option[KeyId], data: DataBuffer] + proc `$`*(id: KeyId): string = $(id.data) proc toDb*(key: Key): DbKey {.inline, raises: [].} = diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 2401095e..13dd5854 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -26,7 +26,7 @@ proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*(self: SQLiteDatastore, key: DbKey|string): ?!bool = +proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = var exists = false key = $key @@ -81,7 +81,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = proc put*(self: SQLiteDatastore, key: DbKey, data: seq[byte]): ?!void = return self.db.putStmt.exec((key.id, data, timestamp())) -proc put*(self: SQLiteDatastore, batch: seq[BatchEntry]): ?!void = +proc put*(self: SQLiteDatastore, batch: iterator (): DbBatchEntry): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index b2c2d916..47bdcf36 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -8,6 +8,7 @@ import ../types import ../query import ../key +import ../backend import ./databuffer type @@ -19,7 +20,7 @@ type ThreadTypes* = void | bool | SomeInteger | DataBuffer | tuple | Atomic ThreadResErr* = (ErrorEnum, DataBuffer) - ThreadQueryRes* = (DataBuffer, DataBuffer) + # ThreadQueryRes* = tuple[key: KeyId, val: DataBuffer] ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] @@ -37,8 +38,8 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) -converter toQueryResponse*(r: ThreadQueryRes): QueryResponse = - if not r[0].isNil and r[0].len > 0 and key =? Key.init($r[0]): - (key.some, @(r[1])) +converter toQueryResponse*(r: DbQueryResponse): QueryResponse = + if not r.key.data.isNil and r.key.data.len > 0 and key =? Key.init($r.key.data): + (key.some, @(r.val)) else: (Key.none, EmptyBytes) From 8800a2ccc02b90483602ec2f934cd11b4d0af0f4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:31:42 -0700 Subject: [PATCH 183/445] databuffer type --- datastore/backend.nim | 4 ++-- datastore/sql/sqliteds.nim | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 59a9bf3f..9a9dc8b5 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -10,7 +10,7 @@ import ./key import ./types export databuffer, threadresult, semaphore, types -export upraises, results +export upraises, results, SortOrder type KeyId* = object @@ -29,7 +29,7 @@ type offset*: int # Offset from which to start querying - not available in all backends sort*: SortOrder # Sort order - not available in all backends - DbQueryResponse* = tuple[key: Option[string], data: seq[byte]] | tuple[key: Option[KeyId], data: DataBuffer] + DbQueryResponse* = tuple[key: Option[KeyId], val: DataBuffer] proc `$`*(id: KeyId): string = $(id.data) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 13dd5854..a1a52798 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -104,11 +104,10 @@ proc close*(self: SQLiteDatastore): ?!void = iterator query*(self: SQLiteDatastore, - query: Query - ): ?!ThreadQueryRes = + query: DbQuery + ): ?!DbQueryResponse {.closure.} = var - iter = QueryIter() queryStr = if query.value: QueryStmtDataIdStr else: From 10d4031c5cea800195f26448ef7ed0826ab74ef3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:57:47 -0700 Subject: [PATCH 184/445] refactoring to non-async --- datastore/backend.nim | 3 + datastore/sql/sqliteds.nim | 109 +++++++++++------------------ datastore/threads/databuffer.nim | 3 + datastore/threads/threadresult.nim | 10 +-- 4 files changed, 53 insertions(+), 72 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 9a9dc8b5..89877207 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -33,6 +33,9 @@ type proc `$`*(id: KeyId): string = $(id.data) +proc new*(tp: typedesc[KeyId], id: cstring): KeyId = + KeyId(data: DataBuffer.new(id.pointer, 0, id.len()-1)) + proc toDb*(key: Key): DbKey {.inline, raises: [].} = let id: string = key.id() let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index a1a52798..3e9809b3 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -103,9 +103,9 @@ proc close*(self: SQLiteDatastore): ?!void = return success() -iterator query*(self: SQLiteDatastore, - query: DbQuery - ): ?!DbQueryResponse {.closure.} = +proc query*(self: SQLiteDatastore, + query: DbQuery + ): Result[iterator(): ?!DbQueryResponse {.closure.}, ref CatchableError] = var queryStr = if query.value: @@ -132,7 +132,7 @@ iterator query*(self: SQLiteDatastore, var v = sqlite3_bind_text( - s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) + s, 1.cint, ($query.key & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) @@ -149,78 +149,53 @@ iterator query*(self: SQLiteDatastore, if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - let lock = newAsyncLock() - proc next(): ?!QueryResponse = - defer: - if lock.locked: - lock.release() + let iter = iterator(): ?!DbQueryResponse {.closure.} = - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") + try: + let + v = sqlite3_step(s) - if iter.finished: - return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) + case v + of SQLITE_ROW: + let + key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) - await lock.acquire() + blob: ?pointer = + if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some + else: pointer.none - let - v = sqlite3_step(s) + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html - case v - of SQLITE_ROW: - let - key = DbKey.init( - $sqlite3_column_text_not_null(s, QueryStmtIdCol)) - .expect("should not fail") - - blob: ?pointer = - if query.value: - sqlite3_column_blob(s, QueryStmtDataCol).some - else: - pointer.none - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html - - # the "data" column can be NULL so in order to detect an out-of-memory - # error it is necessary to check that the result is a null pointer and - # that the result code is an error code - if blob.isSome and blob.get().isNil: - let - v = sqlite3_errcode(sqlite3_db_handle(s)) + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let v = sqlite3_errcode(sqlite3_db_handle(s)) - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - iter.finished = true - return failure newException(DatastoreError, $sqlite3_errstr(v)) + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + return failure newException(DatastoreError, $sqlite3_errstr(v)) - let - dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) - data = if blob.isSome: - @( - toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), - 0, - dataLen - 1)) - else: - @[] - - return success (key.some, data) - of SQLITE_DONE: - iter.finished = true - return success (DbKey.none, EmptyBytes) - else: - iter.finished = true - return failure newException(DatastoreError, $sqlite3_errstr(v)) + let + dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + data = + if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) + else: DataBuffer.new(0) + + return success (key.some, data) + of SQLITE_DONE: + return success (KeyId.none, DataBuffer.new(0)) + else: + return failure newException(DatastoreError, $sqlite3_errstr(v)) - iter.dispose = proc(): ?!void = - discard sqlite3_reset(s) - discard sqlite3_clear_bindings(s) - s.dispose - return success() + finally: + discard sqlite3_reset(s) + discard sqlite3_clear_bindings(s) + s.dispose() + return - iter.next = next - return success iter proc new*(T: type SQLiteDatastore, path: string, diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 3fcad0e2..561d41ae 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -68,6 +68,9 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[Dat copyMem(result[].buf, baseAddr data, data.len()) result[].size = data.len() +proc new*(tp: type DataBuffer, data: pointer, first, last: int): DataBuffer = + DataBuffer.new(toOpenArray(cast[ptr UncheckedArray[byte]](data), first, last)) + proc clear*(db: DataBuffer) = zeroMem(db[].buf, db[].cap) db[].size = 0 diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 47bdcf36..5d17677d 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -38,8 +38,8 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) -converter toQueryResponse*(r: DbQueryResponse): QueryResponse = - if not r.key.data.isNil and r.key.data.len > 0 and key =? Key.init($r.key.data): - (key.some, @(r.val)) - else: - (Key.none, EmptyBytes) +# converter toQueryResponse*(r: DbQueryResponse): QueryResponse = +# if not r.key.data.isNil and r.key.data.len > 0 and key =? Key.init($r.key.data): +# (key.some, @(r.val)) +# else: +# (Key.none, EmptyBytes) From 5752eb01e12fd528a23c36fe5019ff8591a160e4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 20:59:06 -0700 Subject: [PATCH 185/445] refactoring to non-async --- datastore/sql/sqliteds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 3e9809b3..b8d7a88e 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -149,7 +149,7 @@ proc query*(self: SQLiteDatastore, if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - let iter = iterator(): ?!DbQueryResponse {.closure.} = + success iterator(): ?!DbQueryResponse {.closure.} = try: let @@ -195,6 +195,7 @@ proc query*(self: SQLiteDatastore, discard sqlite3_clear_bindings(s) s.dispose() return + proc new*(T: type SQLiteDatastore, From 357ab44b90880a3b82d25404cde4c439f78f04ce Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 21:21:15 -0700 Subject: [PATCH 186/445] refactoring to non-async --- datastore/sql/sqliteds.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index b8d7a88e..b2f6fc39 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -42,7 +42,7 @@ proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = proc delete*(self: SQLiteDatastore, key: DbKey): ?!void = return self.db.deleteStmt.exec((key.data)) -proc delete*(self: SQLiteDatastore, keys: seq[DbKey]): ?!void = +proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) @@ -78,10 +78,15 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = return success bytes -proc put*(self: SQLiteDatastore, key: DbKey, data: seq[byte]): ?!void = - return self.db.putStmt.exec((key.id, data, timestamp())) +proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = + when DbVal is seq[byte]: + return self.db.putStmt.exec((key.id, data, timestamp())) + elif DbVal is DataBuffer: + return self.db.putBufferStmt.exec((key.id, data, timestamp())) + else: + {.error: "unknown type".} -proc put*(self: SQLiteDatastore, batch: iterator (): DbBatchEntry): ?!void = +proc put*(self: SQLiteDatastore, batch: openArray[DbBatchEntry]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err @@ -102,7 +107,6 @@ proc close*(self: SQLiteDatastore): ?!void = return success() - proc query*(self: SQLiteDatastore, query: DbQuery ): Result[iterator(): ?!DbQueryResponse {.closure.}, ref CatchableError] = @@ -186,7 +190,7 @@ proc query*(self: SQLiteDatastore, return success (key.some, data) of SQLITE_DONE: - return success (KeyId.none, DataBuffer.new(0)) + return else: return failure newException(DatastoreError, $sqlite3_errstr(v)) From 61bdc6b88ac284b28e690c2b64eb171110c40246 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 21:57:31 -0700 Subject: [PATCH 187/445] fix test --- datastore/sql/sqlitedsdb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index c4572c6d..454c5f02 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -269,7 +269,7 @@ proc open*( deleteStmt: DeleteStmt getStmt: GetStmt putStmt: PutStmt - putBufferStmt: PutStmt + putBufferStmt: PutBufferStmt beginStmt: BeginStmt endStmt: EndStmt rollbackStmt: RollbackStmt From 3b66afba6ebfc6888fe8ce3beeed8c00e09d0ac2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:12:53 -0700 Subject: [PATCH 188/445] fix test --- datastore.nim | 3 +- datastore/sql.nim | 372 +++++++++++++-------------- datastore/sql/sqliteds.nim | 9 +- datastore/threads/threadproxyds.nim | 5 +- tests/datastore/sql/testsqliteds.nim | 94 +++---- 5 files changed, 237 insertions(+), 246 deletions(-) diff --git a/datastore.nim b/datastore.nim index 9e42c472..6d43a209 100644 --- a/datastore.nim +++ b/datastore.nim @@ -3,6 +3,5 @@ import ./datastore/fsds import ./datastore/sql import ./datastore/mountedds import ./datastore/tieredds -import ./datastore/threads/threadproxyds -export datastore, fsds, mountedds, tieredds, sql, threadproxyds +export datastore, fsds, mountedds, tieredds, sql diff --git a/datastore/sql.nim b/datastore/sql.nim index e5fad22d..55a23e03 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -9,239 +9,239 @@ from pkg/stew/results as stewResults import isErr import pkg/upraises import ../datastore -import ./sqlitedsdb +# import ./sqlitedsdb -export datastore, sqlitedsdb +export datastore push: {.upraises: [].} -type - SQLiteDatastore* = ref object of Datastore - readOnly: bool - db: SQLiteDsDb +# type +# SQLiteDatastore* = ref object of Datastore +# readOnly: bool +# db: SQLiteDsDb -proc path*(self: SQLiteDatastore): string = - self.db.dbPath +# proc path*(self: SQLiteDatastore): string = +# self.db.dbPath -proc `readOnly=`*(self: SQLiteDatastore): bool - {.error: "readOnly should not be assigned".} +# proc `readOnly=`*(self: SQLiteDatastore): bool +# {.error: "readOnly should not be assigned".} -proc timestamp*(t = epochTime()): int64 = - (t * 1_000_000).int64 +# proc timestamp*(t = epochTime()): int64 = +# (t * 1_000_000).int64 -method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = - var - exists = false +# method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = +# var +# exists = false - proc onData(s: RawStmtPtr) = - exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool +# proc onData(s: RawStmtPtr) = +# exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool - if err =? self.db.containsStmt.query((key.id), onData).errorOption: - return failure err +# if err =? self.db.containsStmt.query((key.id), onData).errorOption: +# return failure err - return success exists +# return success exists -method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = - return self.db.deleteStmt.exec((key.id)) +# method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = +# return self.db.deleteStmt.exec((key.id)) -method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = - if err =? self.db.beginStmt.exec().errorOption: - return failure(err) +# method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = +# if err =? self.db.beginStmt.exec().errorOption: +# return failure(err) - for key in keys: - if err =? self.db.deleteStmt.exec((key.id)).errorOption: - if err =? self.db.rollbackStmt.exec().errorOption: - return failure err.msg +# for key in keys: +# if err =? self.db.deleteStmt.exec((key.id)).errorOption: +# if err =? self.db.rollbackStmt.exec().errorOption: +# return failure err.msg - return failure err.msg +# return failure err.msg - if err =? self.db.endStmt.exec().errorOption: - return failure err.msg +# if err =? self.db.endStmt.exec().errorOption: +# return failure err.msg - return success() +# return success() -method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - # see comment in ./filesystem_datastore re: finer control of memory - # allocation in `method get`, could apply here as well if bytes were read - # incrementally with `sqlite3_blob_read` +# method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = +# # see comment in ./filesystem_datastore re: finer control of memory +# # allocation in `method get`, could apply here as well if bytes were read +# # incrementally with `sqlite3_blob_read` - var - bytes: seq[byte] +# var +# bytes: seq[byte] - proc onData(s: RawStmtPtr) = - bytes = self.db.getDataCol() +# proc onData(s: RawStmtPtr) = +# bytes = self.db.getDataCol() - if err =? self.db.getStmt.query((key.id), onData).errorOption: - return failure(err) +# if err =? self.db.getStmt.query((key.id), onData).errorOption: +# return failure(err) - if bytes.len <= 0: - return failure( - newException(DatastoreKeyNotFound, "Key doesn't exist")) +# if bytes.len <= 0: +# return failure( +# newException(DatastoreKeyNotFound, "Key doesn't exist")) - return success bytes +# return success bytes -method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = - return self.db.putStmt.exec((key.id, data, timestamp())) +# method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = +# return self.db.putStmt.exec((key.id, data, timestamp())) -method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = - if err =? self.db.beginStmt.exec().errorOption: - return failure err +# method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = +# if err =? self.db.beginStmt.exec().errorOption: +# return failure err - for entry in batch: - if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: - if err =? self.db.rollbackStmt.exec().errorOption: - return failure err +# for entry in batch: +# if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: +# if err =? self.db.rollbackStmt.exec().errorOption: +# return failure err - return failure err +# return failure err - if err =? self.db.endStmt.exec().errorOption: - return failure err +# if err =? self.db.endStmt.exec().errorOption: +# return failure err - return success() +# return success() -method close*(self: SQLiteDatastore): Future[?!void] {.async.} = - self.db.close() +# method close*(self: SQLiteDatastore): Future[?!void] {.async.} = +# self.db.close() - return success() +# return success() -method query*( - self: SQLiteDatastore, - query: Query): Future[?!QueryIter] {.async.} = +# method query*( +# self: SQLiteDatastore, +# query: Query): Future[?!QueryIter] {.async.} = - var - iter = QueryIter() - queryStr = if query.value: - QueryStmtDataIdStr - else: - QueryStmtIdStr +# var +# iter = QueryIter() +# queryStr = if query.value: +# QueryStmtDataIdStr +# else: +# QueryStmtIdStr - if query.sort == SortOrder.Descending: - queryStr &= QueryStmtOrderDescending - else: - queryStr &= QueryStmtOrderAscending +# if query.sort == SortOrder.Descending: +# queryStr &= QueryStmtOrderDescending +# else: +# queryStr &= QueryStmtOrderAscending - if query.limit != 0: - queryStr &= QueryStmtLimit +# if query.limit != 0: +# queryStr &= QueryStmtLimit - if query.offset != 0: - queryStr &= QueryStmtOffset +# if query.offset != 0: +# queryStr &= QueryStmtOffset - let - queryStmt = QueryStmt.prepare( - self.db.env, queryStr).expect("should not fail") +# let +# queryStmt = QueryStmt.prepare( +# self.db.env, queryStr).expect("should not fail") - s = RawStmtPtr(queryStmt) +# s = RawStmtPtr(queryStmt) - var - v = sqlite3_bind_text( - s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) +# var +# v = sqlite3_bind_text( +# s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) - if not (v == SQLITE_OK): - return failure newException(DatastoreError, $sqlite3_errstr(v)) +# if not (v == SQLITE_OK): +# return failure newException(DatastoreError, $sqlite3_errstr(v)) - if query.limit != 0: - v = sqlite3_bind_int(s, 2.cint, query.limit.cint) +# if query.limit != 0: +# v = sqlite3_bind_int(s, 2.cint, query.limit.cint) - if not (v == SQLITE_OK): - return failure newException(DatastoreError, $sqlite3_errstr(v)) +# if not (v == SQLITE_OK): +# return failure newException(DatastoreError, $sqlite3_errstr(v)) - if query.offset != 0: - v = sqlite3_bind_int(s, 3.cint, query.offset.cint) +# if query.offset != 0: +# v = sqlite3_bind_int(s, 3.cint, query.offset.cint) - if not (v == SQLITE_OK): - return failure newException(DatastoreError, $sqlite3_errstr(v)) +# if not (v == SQLITE_OK): +# return failure newException(DatastoreError, $sqlite3_errstr(v)) - let lock = newAsyncLock() - proc next(): Future[?!QueryResponse] {.async.} = - defer: - if lock.locked: - lock.release() +# let lock = newAsyncLock() +# proc next(): Future[?!QueryResponse] {.async.} = +# defer: +# if lock.locked: +# lock.release() - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") +# if lock.locked: +# return failure (ref DatastoreError)(msg: "Should always await query features") - if iter.finished: - return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) +# if iter.finished: +# return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) - await lock.acquire() +# await lock.acquire() - let - v = sqlite3_step(s) +# let +# v = sqlite3_step(s) - case v - of SQLITE_ROW: - let - key = Key.init( - $sqlite3_column_text_not_null(s, QueryStmtIdCol)) - .expect("should not fail") - - blob: ?pointer = - if query.value: - sqlite3_column_blob(s, QueryStmtDataCol).some - else: - pointer.none - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html - - # the "data" column can be NULL so in order to detect an out-of-memory - # error it is necessary to check that the result is a null pointer and - # that the result code is an error code - if blob.isSome and blob.get().isNil: - let - v = sqlite3_errcode(sqlite3_db_handle(s)) - - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - iter.finished = true - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - let - dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) - data = if blob.isSome: - @( - toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), - 0, - dataLen - 1)) - else: - @[] - - return success (key.some, data) - of SQLITE_DONE: - iter.finished = true - return success (Key.none, EmptyBytes) - else: - iter.finished = true - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - iter.dispose = proc(): Future[?!void] {.async.} = - discard sqlite3_reset(s) - discard sqlite3_clear_bindings(s) - s.dispose - return success() - - iter.next = next - return success iter - -proc new*( - T: type SQLiteDatastore, - path: string, - readOnly = false): ?!T = - - let - flags = - if readOnly: SQLITE_OPEN_READONLY - else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - - success T( - db: ? SQLiteDsDb.open(path, flags), - readOnly: readOnly) - -proc new*( - T: type SQLiteDatastore, - db: SQLiteDsDb): ?!T = - - success T( - db: db, - readOnly: db.readOnly) +# case v +# of SQLITE_ROW: +# let +# key = Key.init( +# $sqlite3_column_text_not_null(s, QueryStmtIdCol)) +# .expect("should not fail") + +# blob: ?pointer = +# if query.value: +# sqlite3_column_blob(s, QueryStmtDataCol).some +# else: +# pointer.none + +# # detect out-of-memory error +# # see the conversion table and final paragraph of: +# # https://www.sqlite.org/c3ref/column_blob.html +# # see also https://www.sqlite.org/rescode.html + +# # the "data" column can be NULL so in order to detect an out-of-memory +# # error it is necessary to check that the result is a null pointer and +# # that the result code is an error code +# if blob.isSome and blob.get().isNil: +# let +# v = sqlite3_errcode(sqlite3_db_handle(s)) + +# if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): +# iter.finished = true +# return failure newException(DatastoreError, $sqlite3_errstr(v)) + +# let +# dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) +# data = if blob.isSome: +# @( +# toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), +# 0, +# dataLen - 1)) +# else: +# @[] + +# return success (key.some, data) +# of SQLITE_DONE: +# iter.finished = true +# return success (Key.none, EmptyBytes) +# else: +# iter.finished = true +# return failure newException(DatastoreError, $sqlite3_errstr(v)) + +# iter.dispose = proc(): Future[?!void] {.async.} = +# discard sqlite3_reset(s) +# discard sqlite3_clear_bindings(s) +# s.dispose +# return success() + +# iter.next = next +# return success iter + +# proc new*( +# T: type SQLiteDatastore, +# path: string, +# readOnly = false): ?!T = + +# let +# flags = +# if readOnly: SQLITE_OPEN_READONLY +# else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE + +# success T( +# db: ? SQLiteDsDb.open(path, flags), +# readOnly: readOnly) + +# proc new*( +# T: type SQLiteDatastore, +# db: SQLiteDsDb): ?!T = + +# success T( +# db: db, +# readOnly: db.readOnly) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index b2f6fc39..a823485d 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -69,7 +69,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = proc onData(s: RawStmtPtr) = bytes = self.db.getDataCol() - if err =? self.db.getStmt.query((key.id), onData).errorOption: + if err =? self.db.getStmt.query((key), onData).errorOption: return failure(err) if bytes.len <= 0: @@ -80,7 +80,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = when DbVal is seq[byte]: - return self.db.putStmt.exec((key.id, data, timestamp())) + return self.db.putStmt.exec((key, data, timestamp())) elif DbVal is DataBuffer: return self.db.putBufferStmt.exec((key.id, data, timestamp())) else: @@ -199,7 +199,10 @@ proc query*(self: SQLiteDatastore, discard sqlite3_clear_bindings(s) s.dispose() return - + + +proc contains*(self: SQLiteDatastore, key: DbKey): bool = + return self.has(key) proc new*(T: type SQLiteDatastore, diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 31a65baf..2a920562 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -23,6 +23,7 @@ import pkg/chronicles import ../key import ../query import ../datastore +import ../backend import ./asyncsemaphore import ./databuffer @@ -427,8 +428,8 @@ method query*( iter.finished = childIter.finished var - res = ThreadResult[ThreadQueryRes]() - ctx = TaskCtx[ThreadQueryRes]( + res = ThreadResult[DbQueryResponse]() + ctx = TaskCtx[DbQueryResponse]( ds: self.ds, res: addr res) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index c629eb0c..6e4b5850 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -3,87 +3,75 @@ import std/os import std/sequtils from std/algorithm import sort, reversed -import pkg/asynctest +import pkg/unittest2 import pkg/chronos import pkg/stew/results import pkg/stew/byteutils import pkg/datastore/sql/sqliteds +import pkg/datastore/key import ../dscommontests import ../querycommontests suite "Test Basic SQLiteDatastore": + let ds = SQLiteDatastore.new(Memory).tryGet() - key = Key.init("a:b/c/d:e").tryGet() + key = Key.init("a:b/c/d:e").tryGet().id() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - teardownAll: - (await ds.close()).tryGet() - - basicStoreTests(ds, key, bytes, otherBytes) - -suite "Test Read Only SQLiteDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - filename = "test_store" & DbExt - dbPathAbs = basePathAbs / filename - key = Key.init("a:b/c/d:e").tryGet() - bytes = "some bytes".toBytes + teardown: + ds.close().tryGet() - var - dsDb: SQLiteDatastore - readOnlyDb: SQLiteDatastore + test "put": + ds.put(key, bytes).tryGet() - setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) + test "get": + check: + ds.get(key).tryGet() == bytes - dsDb = SQLiteDatastore.new(path = dbPathAbs).tryGet() - readOnlyDb = SQLiteDatastore.new(path = dbPathAbs, readOnly = true).tryGet() + test "put update": + ds.put(key, otherBytes).tryGet() - teardownAll: - (await dsDb.close()).tryGet() - (await readOnlyDb.close()).tryGet() + test "get updated": + check: + ds.get(key).tryGet() == otherBytes - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + test "delete": + ds.delete(key).tryGet() - test "put": + test "contains": check: - (await readOnlyDb.put(key, bytes)).isErr + not await (key in ds) - (await dsDb.put(key, bytes)).tryGet() + test "put batch": + var + batch: seq[BatchEntry] - test "get": - check: - (await readOnlyDb.get(key)).tryGet() == bytes - (await dsDb.get(key)).tryGet() == bytes + for k in 0..<100: + batch.add((Key.init(key.id, $k).tryGet, @[k.byte])) - test "delete": - check: - (await readOnlyDb.delete(key)).isErr + ds.put(batch).tryGet - (await dsDb.delete(key)).tryGet() + for k in batch: + check: ds.has(k.key).tryGet - test "contains": - check: - not (await readOnlyDb.has(key)).tryGet() - not (await dsDb.has(key)).tryGet() + test "delete batch": + var + batch: seq[Key] -suite "Test Query": - var - ds: SQLiteDatastore + for k in 0..<100: + batch.add(Key.init(key.id, $k).tryGet) - setup: - ds = SQLiteDatastore.new(Memory).tryGet() + ds.delete(batch).tryGet - teardown: - (await ds.close()).tryGet + for k in batch: + check: not ds.has(k).tryGet + + test "handle missing key": + let key = Key.init("/missing/key").tryGet() - queryTests(ds) + expect(DatastoreKeyNotFound): + discard ds.get(key).tryGet() # non existing key From d35bbea741127e1a5784ef2f792f4e8f317cfc45 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:13:01 -0700 Subject: [PATCH 189/445] fix test --- tests/datastore/sql/testsqlite.nim | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/datastore/sql/testsqlite.nim diff --git a/tests/datastore/sql/testsqlite.nim b/tests/datastore/sql/testsqlite.nim new file mode 100644 index 00000000..c629eb0c --- /dev/null +++ b/tests/datastore/sql/testsqlite.nim @@ -0,0 +1,89 @@ +import std/options +import std/os +import std/sequtils +from std/algorithm import sort, reversed + +import pkg/asynctest +import pkg/chronos +import pkg/stew/results +import pkg/stew/byteutils + +import pkg/datastore/sql/sqliteds + +import ../dscommontests +import ../querycommontests + +suite "Test Basic SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + key = Key.init("a:b/c/d:e").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + teardownAll: + (await ds.close()).tryGet() + + basicStoreTests(ds, key, bytes, otherBytes) + +suite "Test Read Only SQLiteDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + filename = "test_store" & DbExt + dbPathAbs = basePathAbs / filename + key = Key.init("a:b/c/d:e").tryGet() + bytes = "some bytes".toBytes + + var + dsDb: SQLiteDatastore + readOnlyDb: SQLiteDatastore + + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + dsDb = SQLiteDatastore.new(path = dbPathAbs).tryGet() + readOnlyDb = SQLiteDatastore.new(path = dbPathAbs, readOnly = true).tryGet() + + teardownAll: + (await dsDb.close()).tryGet() + (await readOnlyDb.close()).tryGet() + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + test "put": + check: + (await readOnlyDb.put(key, bytes)).isErr + + (await dsDb.put(key, bytes)).tryGet() + + test "get": + check: + (await readOnlyDb.get(key)).tryGet() == bytes + (await dsDb.get(key)).tryGet() == bytes + + test "delete": + check: + (await readOnlyDb.delete(key)).isErr + + (await dsDb.delete(key)).tryGet() + + test "contains": + check: + not (await readOnlyDb.has(key)).tryGet() + not (await dsDb.has(key)).tryGet() + +suite "Test Query": + var + ds: SQLiteDatastore + + setup: + ds = SQLiteDatastore.new(Memory).tryGet() + + teardown: + (await ds.close()).tryGet + + queryTests(ds) From 9362fcbb07dbaf20a141b981af3282122fb44ae4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:23:18 -0700 Subject: [PATCH 190/445] fix test --- datastore/backend.nim | 2 +- datastore/sql/sqliteds.nim | 6 +++--- tests/datastore/sql/testsqliteds.nim | 13 +++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 89877207..b5135f77 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -20,7 +20,7 @@ type DbKey* = string | KeyId DbVal* = seq[byte] | DataBuffer - DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] + DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] DbQuery* = object key*: KeyId # Key to be queried diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index a823485d..f9d486c3 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -40,7 +40,7 @@ proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = return success exists proc delete*(self: SQLiteDatastore, key: DbKey): ?!void = - return self.db.deleteStmt.exec((key.data)) + return self.db.deleteStmt.exec((key)) proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: @@ -91,7 +91,7 @@ proc put*(self: SQLiteDatastore, batch: openArray[DbBatchEntry]): ?!void = return failure err for entry in batch: - if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: + if err =? self.db.putStmt.exec((entry.key, entry.data, timestamp())).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err @@ -202,7 +202,7 @@ proc query*(self: SQLiteDatastore, proc contains*(self: SQLiteDatastore, key: DbKey): bool = - return self.has(key) + return self.has(key).get() proc new*(T: type SQLiteDatastore, diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 6e4b5850..fff73540 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -18,7 +18,8 @@ suite "Test Basic SQLiteDatastore": let ds = SQLiteDatastore.new(Memory).tryGet() - key = Key.init("a:b/c/d:e").tryGet().id() + keyFull = Key.init("a:b/c/d:e").tryGet() + key = keyFull.id() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes @@ -43,15 +44,15 @@ suite "Test Basic SQLiteDatastore": ds.delete(key).tryGet() test "contains": - check: - not await (key in ds) + check key notin ds test "put batch": var - batch: seq[BatchEntry] + batch: seq[tuple[key: string, data: seq[byte]]] for k in 0..<100: - batch.add((Key.init(key.id, $k).tryGet, @[k.byte])) + let kk = Key.init(key, $k).tryGet().id() + batch.add((kk, @[k.byte])) ds.put(batch).tryGet @@ -63,7 +64,7 @@ suite "Test Basic SQLiteDatastore": batch: seq[Key] for k in 0..<100: - batch.add(Key.init(key.id, $k).tryGet) + batch.add(Key.init(key, $k).tryGet) ds.delete(batch).tryGet From 8b494907c6027a7f522dd1f17ccec30cc6b3f574 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:43:56 -0700 Subject: [PATCH 191/445] fix test --- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 56 +++++++++++++------------- tests/datastore/sql/testsqliteds.nim | 6 +-- tests/datastore/sql/testsqlitedsdb.nim | 8 +--- 4 files changed, 34 insertions(+), 38 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index f9d486c3..99a6aa4b 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -47,7 +47,7 @@ proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = return failure(err) for key in keys: - if err =? self.db.deleteStmt.exec((key.id)).errorOption: + if err =? self.db.deleteStmt.exec((key)).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err.msg diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 454c5f02..572bc443 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -33,7 +33,7 @@ type containsStmt*: ContainsStmt deleteStmt*: DeleteStmt env*: SQLite - getDataCol*: BoundDataCol + getDataCol*: (RawStmtPtr, int) getStmt*: GetStmt putStmt*: PutStmt beginStmt*: BeginStmt @@ -158,38 +158,38 @@ proc idCol*( return proc (): string = $sqlite3_column_text_not_null(s, index.cint) -proc dataCol*( - s: RawStmtPtr, - index: int): BoundDataCol = +proc dataCol*(data: (RawStmtPtr, int)): DataBuffer = + + let s = data[0] + let index = data[1] checkColMetadata(s, index, DataColName) - return proc (): DataBuffer = + let + i = index.cint + blob = sqlite3_column_blob(s, i) + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory error + # it is necessary to check that the result is a null pointer and that the + # result code is an error code + if blob.isNil: let - i = index.cint - blob = sqlite3_column_blob(s, i) - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html + v = sqlite3_errcode(sqlite3_db_handle(s)) - # the "data" column can be NULL so in order to detect an out-of-memory error - # it is necessary to check that the result is a null pointer and that the - # result code is an error code - if blob.isNil: - let - v = sqlite3_errcode(sqlite3_db_handle(s)) + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + raise (ref Defect)(msg: $sqlite3_errstr(v)) - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - raise (ref Defect)(msg: $sqlite3_errstr(v)) - - let - dataLen = sqlite3_column_bytes(s, i) - dataBytes = cast[ptr UncheckedArray[byte]](blob) + let + dataLen = sqlite3_column_bytes(s, i) + dataBytes = cast[ptr UncheckedArray[byte]](blob) - # copy data out, since sqlite will free it - DataBuffer.new(toOpenArray(dataBytes, 0, dataLen - 1)) + # copy data out, since sqlite will free it + DataBuffer.new(toOpenArray(dataBytes, 0, dataLen - 1)) proc timestampCol*( s: RawStmtPtr, @@ -306,9 +306,9 @@ proc open*( # "SQL logic error" let - getDataCol = dataCol(RawStmtPtr(getStmt), GetStmtDataCol) + getDataCol = (RawStmtPtr(getStmt), GetStmtDataCol) - success T( + success SQLiteDsDb( readOnly: readOnly, dbPath: path, containsStmt: containsStmt, diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index fff73540..32836b38 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -61,10 +61,10 @@ suite "Test Basic SQLiteDatastore": test "delete batch": var - batch: seq[Key] + batch: seq[string] for k in 0..<100: - batch.add(Key.init(key, $k).tryGet) + batch.add(Key.init(key, $k).tryGet().id()) ds.delete(batch).tryGet @@ -72,7 +72,7 @@ suite "Test Basic SQLiteDatastore": check: not ds.has(k).tryGet test "handle missing key": - let key = Key.init("/missing/key").tryGet() + let key = Key.init("/missing/key").tryGet().id() expect(DatastoreKeyNotFound): discard ds.get(key).tryGet() # non existing key diff --git a/tests/datastore/sql/testsqlitedsdb.nim b/tests/datastore/sql/testsqlitedsdb.nim index d1049331..e5bd1062 100644 --- a/tests/datastore/sql/testsqlitedsdb.nim +++ b/tests/datastore/sql/testsqlitedsdb.nim @@ -111,12 +111,10 @@ suite "Test SQLite Datastore DB operations": dsDb.putStmt.exec((key.id, data, timestamp())).tryGet() test "Should select key": - let - dataCol = dsDb.getDataCol var bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = dataCol() + bytes = dataCol(dsDb.getDataCol) check: dsDb.getStmt.query((key.id), onData).tryGet() @@ -129,12 +127,10 @@ suite "Test SQLite Datastore DB operations": dsDb.putStmt.exec((key.id, otherData, timestamp())).tryGet() test "Should select updated key": - let - dataCol = dsDb.getDataCol var bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = dataCol() + bytes = dataCol(dsDb.getDataCol) check: dsDb.getStmt.query((key.id), onData).tryGet() From 76b952c9287fdf89c67fb0e2fab0b5b5f062b760 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:44:37 -0700 Subject: [PATCH 192/445] fix test --- datastore/sql/sqliteds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 99a6aa4b..2126fbe4 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -67,7 +67,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = self.db.getDataCol() + bytes = dataCol(self.db.getDataCol) if err =? self.db.getStmt.query((key), onData).errorOption: return failure(err) From cf2cbd3e1a5b8b34f131b73b0539f30e0be183ec Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:45:47 -0700 Subject: [PATCH 193/445] fix test --- tests/datastore/sql/testsqliteds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 32836b38..73e00f94 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -23,7 +23,7 @@ suite "Test Basic SQLiteDatastore": bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - teardown: + suiteTeardown: ds.close().tryGet() test "put": From 937e1b79e89e3922784f12967c7e09872de61164 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 22:49:11 -0700 Subject: [PATCH 194/445] test generic --- tests/datastore/sql/testsqliteds.nim | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 73e00f94..38b83fc9 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -14,17 +14,12 @@ import pkg/datastore/key import ../dscommontests import ../querycommontests -suite "Test Basic SQLiteDatastore": - - let - ds = SQLiteDatastore.new(Memory).tryGet() - keyFull = Key.init("a:b/c/d:e").tryGet() - key = keyFull.id() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - suiteTeardown: - ds.close().tryGet() +proc testBasic( + ds: SQLiteDatastore, + key: string, + bytes: seq[byte], + otherBytes: seq[byte], +) = test "put": ds.put(key, bytes).tryGet() @@ -76,3 +71,16 @@ suite "Test Basic SQLiteDatastore": expect(DatastoreKeyNotFound): discard ds.get(key).tryGet() # non existing key + +suite "Test Basic SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + keyFull = Key.init("a:b/c/d:e").tryGet() + key = keyFull.id() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + suiteTeardown: + ds.close().tryGet() + + testBasic(ds, key, bytes, otherBytes) From 8c7165559355b31872ace8755f502ac4e28ac8fe Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:07:52 -0700 Subject: [PATCH 195/445] add index --- datastore/backend.nim | 13 +++++++++++-- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 4 +++- datastore/sql/sqliteutils.nim | 6 +++--- datastore/threads/databuffer.nim | 8 ++++++++ tests/datastore/sql/testsqliteds.nim | 20 ++++++++++++++++---- tests/datastore/testdatabuffer.nim | 6 ++++++ 7 files changed, 48 insertions(+), 11 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index b5135f77..3d757bcb 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -34,7 +34,16 @@ type proc `$`*(id: KeyId): string = $(id.data) proc new*(tp: typedesc[KeyId], id: cstring): KeyId = - KeyId(data: DataBuffer.new(id.pointer, 0, id.len()-1)) + ## copy cstring including null terminator + KeyId(data: DataBuffer.new(id.pointer, 0, id.len())) + +proc new*(tp: typedesc[KeyId], id: string): KeyId = + ## copy cstring including null terminator + KeyId(data: DataBuffer.new(id, {dbNullTerminate})) + +proc toCString*(key: KeyId): cstring = + ## copy cstring including null terminator + cast[cstring](baseAddr key.data) proc toDb*(key: Key): DbKey {.inline, raises: [].} = let id: string = key.id() @@ -42,7 +51,7 @@ proc toDb*(key: Key): DbKey {.inline, raises: [].} = db.setData(id) DbKey(data: db) -proc toKey*(key: DbKey): Key {.inline, raises: [].} = +proc toKey*(key: KeyId): Key {.inline, raises: [].} = Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") template toOpenArray*(x: DbKey): openArray[char] = diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 2126fbe4..66beed1c 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -82,7 +82,7 @@ proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = when DbVal is seq[byte]: return self.db.putStmt.exec((key, data, timestamp())) elif DbVal is DataBuffer: - return self.db.putBufferStmt.exec((key.id, data, timestamp())) + return self.db.putBufferStmt.exec((key, data, timestamp())) else: {.error: "unknown type".} diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 572bc443..ea06e500 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -21,7 +21,7 @@ type DeleteStmt* = SQLiteStmt[(string), void] GetStmt* = SQLiteStmt[(string), void] PutStmt* = SQLiteStmt[(string, seq[byte], int64), void] - PutBufferStmt* = SQLiteStmt[(string, DataBuffer, int64), void] + PutBufferStmt* = SQLiteStmt[(KeyId, DataBuffer, int64), void] QueryStmt* = SQLiteStmt[(string), void] BeginStmt* = NoParamsStmt EndStmt* = NoParamsStmt @@ -36,6 +36,7 @@ type getDataCol*: (RawStmtPtr, int) getStmt*: GetStmt putStmt*: PutStmt + putBufferStmt*: PutBufferStmt beginStmt*: BeginStmt endStmt*: EndStmt rollbackStmt*: RollbackStmt @@ -317,6 +318,7 @@ proc open*( getStmt: getStmt, getDataCol: getDataCol, putStmt: putStmt, + putBufferStmt: putBufferStmt, beginStmt: beginStmt, endStmt: endStmt, rollbackStmt: rollbackStmt) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index d070ce24..213438f7 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -46,7 +46,7 @@ proc bindParam( n: int, val: auto): cint = - when val is openArray[byte]|seq[byte]: + when val is openArray[byte]|seq[byte]|DataBuffer: if val.len > 0: # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it @@ -70,13 +70,13 @@ proc bindParam( # remain valid until then. SQLite will then manage the lifetime of its # private copy." sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) - elif val is DbKey: + elif val is KeyId: # `-1` implies string length is num bytes up to first null-terminator; # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it must # remain valid until then. SQLite will then manage the lifetime of its # private copy." - sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) + sqlite3_bind_text(s, n.cint, val.toCString(), -1.cint, SQLITE_TRANSIENT) else: {.fatal: "Please add support for the '" & $typeof(val) & "' type".} diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 561d41ae..cae4b65e 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -35,6 +35,11 @@ proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) proc hash*(a: DataBuffer): Hash = a[].buf.toOpenArray(0, a[].size-1).hash() +proc `[]`*(db: DataBuffer, idx: int): byte = + if idx >= db.len(): + raise newException(IndexDefect, "index out of bounds") + db[].buf[idx] + proc `==`*(a, b: DataBuffer): bool = if a.isNil and b.isNil: return true elif a.isNil or b.isNil: return false @@ -71,6 +76,9 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[Dat proc new*(tp: type DataBuffer, data: pointer, first, last: int): DataBuffer = DataBuffer.new(toOpenArray(cast[ptr UncheckedArray[byte]](data), first, last)) +proc baseAddr*(db: DataBuffer): pointer = + db[].buf + proc clear*(db: DataBuffer) = zeroMem(db[].buf, db[].cap) db[].size = 0 diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 38b83fc9..240892e8 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -14,11 +14,11 @@ import pkg/datastore/key import ../dscommontests import ../querycommontests -proc testBasic( +proc testBasic[K, V]( ds: SQLiteDatastore, - key: string, - bytes: seq[byte], - otherBytes: seq[byte], + key: K, + bytes: V, + otherBytes: V, ) = test "put": @@ -84,3 +84,15 @@ suite "Test Basic SQLiteDatastore": ds.close().tryGet() testBasic(ds, key, bytes, otherBytes) + +suite "Test DataBuffer SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + key = KeyId.new Key.init("a:b/c/d:e").tryGet().id() + bytes = DataBuffer.new "some bytes" + otherBytes = DataBuffer.new "some other bytes" + + suiteTeardown: + ds.close().tryGet() + + testBasic(ds, key, bytes, otherBytes) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 0c6fa469..5ec8127b 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -88,6 +88,12 @@ suite "Share buffer test": test "toString": check $a == "/a/b" + test "index": + check c[0] == '/'.byte + check c[1] == 'a'.byte + expect IndexDefect: + check c[2] == 'c'.byte + test "hash": check a.hash() == b.hash() From 4d26f707e50c2d62fb43f5cb38f45d0e8b6e3b40 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:16:24 -0700 Subject: [PATCH 196/445] key batch --- datastore/sql/sqliteds.nim | 4 ++-- datastore/threads/databuffer.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 30 +++++++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 66beed1c..754d5f28 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -40,7 +40,7 @@ proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = return success exists proc delete*(self: SQLiteDatastore, key: DbKey): ?!void = - return self.db.deleteStmt.exec((key)) + return self.db.deleteStmt.exec(($key)) proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: @@ -69,7 +69,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = proc onData(s: RawStmtPtr) = bytes = dataCol(self.db.getDataCol) - if err =? self.db.getStmt.query((key), onData).errorOption: + if err =? self.db.getStmt.query(($key), onData).errorOption: return failure(err) if bytes.len <= 0: diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index cae4b65e..7be334fd 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -35,7 +35,7 @@ proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) proc hash*(a: DataBuffer): Hash = a[].buf.toOpenArray(0, a[].size-1).hash() -proc `[]`*(db: DataBuffer, idx: int): byte = +proc `[]`*(db: DataBuffer, idx: int): var byte = if idx >= db.len(): raise newException(IndexDefect, "index out of bounds") db[].buf[idx] diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 240892e8..38935b14 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -14,11 +14,12 @@ import pkg/datastore/key import ../dscommontests import ../querycommontests -proc testBasic[K, V]( +proc testBasic[K, V, B]( ds: SQLiteDatastore, key: K, bytes: V, otherBytes: V, + batches: B, ) = test "put": @@ -80,19 +81,24 @@ suite "Test Basic SQLiteDatastore": bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes + var batch: seq[tuple[key: string, data: seq[byte]]] + for k in 0..<100: + let kk = Key.init(key, $k).tryGet().id() + batch.add( (kk, @[k.byte]) ) + suiteTeardown: ds.close().tryGet() - - testBasic(ds, key, bytes, otherBytes) -suite "Test DataBuffer SQLiteDatastore": - let - ds = SQLiteDatastore.new(Memory).tryGet() - key = KeyId.new Key.init("a:b/c/d:e").tryGet().id() - bytes = DataBuffer.new "some bytes" - otherBytes = DataBuffer.new "some other bytes" + testBasic(ds, key, bytes, otherBytes, batch) - suiteTeardown: - ds.close().tryGet() +# suite "Test DataBuffer SQLiteDatastore": +# let +# ds = SQLiteDatastore.new(Memory).tryGet() +# key = KeyId.new Key.init("a:b/c/d:e").tryGet().id() +# bytes = DataBuffer.new "some bytes" +# otherBytes = DataBuffer.new "some other bytes" + +# suiteTeardown: +# ds.close().tryGet() - testBasic(ds, key, bytes, otherBytes) +# testBasic(ds, key, bytes, otherBytes) From 398342be76f46811dc94ec8c1935e6cff8acd166 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:28:50 -0700 Subject: [PATCH 197/445] key batch --- datastore/sql/sqliteds.nim | 11 +++++-- tests/datastore/sql/testsqliteds.nim | 48 +++++++++++++--------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 754d5f28..423c4a16 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -47,7 +47,7 @@ proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = return failure(err) for key in keys: - if err =? self.db.deleteStmt.exec((key)).errorOption: + if err =? self.db.deleteStmt.exec(($key)).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err.msg @@ -91,7 +91,14 @@ proc put*(self: SQLiteDatastore, batch: openArray[DbBatchEntry]): ?!void = return failure err for entry in batch: - if err =? self.db.putStmt.exec((entry.key, entry.data, timestamp())).errorOption: + # DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] + when entry.key is string: + let putStmt = self.db.putStmt + elif entry.key is KeyId: + let putStmt = self.db.putBufferStmt + else: + {.error: "unhandled type".} + if err =? putStmt.exec((entry.key, entry.data, timestamp())).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 38935b14..c5dfb461 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -19,7 +19,7 @@ proc testBasic[K, V, B]( key: K, bytes: V, otherBytes: V, - batches: B, + batch: B, ) = test "put": @@ -43,28 +43,20 @@ proc testBasic[K, V, B]( check key notin ds test "put batch": - var - batch: seq[tuple[key: string, data: seq[byte]]] - - for k in 0..<100: - let kk = Key.init(key, $k).tryGet().id() - batch.add((kk, @[k.byte])) ds.put(batch).tryGet - for k in batch: - check: ds.has(k.key).tryGet + for (k, v) in batch: + check: ds.has(k).tryGet test "delete batch": - var - batch: seq[string] - - for k in 0..<100: - batch.add(Key.init(key, $k).tryGet().id()) + var keys: seq[K] + for (k, v) in batch: + keys.add(k) - ds.delete(batch).tryGet + ds.delete(keys).tryGet - for k in batch: + for (k, v) in batch: check: not ds.has(k).tryGet test "handle missing key": @@ -91,14 +83,20 @@ suite "Test Basic SQLiteDatastore": testBasic(ds, key, bytes, otherBytes, batch) -# suite "Test DataBuffer SQLiteDatastore": -# let -# ds = SQLiteDatastore.new(Memory).tryGet() -# key = KeyId.new Key.init("a:b/c/d:e").tryGet().id() -# bytes = DataBuffer.new "some bytes" -# otherBytes = DataBuffer.new "some other bytes" +suite "Test DataBuffer SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + keyFull = Key.init("a:b/c/d:e").tryGet() + key = KeyId.new keyFull.id() + bytes = DataBuffer.new "some bytes" + otherBytes = DataBuffer.new "some other bytes" + + var batch: seq[tuple[key: KeyId, data: DataBuffer]] + for k in 0..<100: + let kk = Key.init(keyFull.id(), $k).tryGet().id() + batch.add( (KeyId.new kk, DataBuffer.new @[k.byte]) ) -# suiteTeardown: -# ds.close().tryGet() + suiteTeardown: + ds.close().tryGet() -# testBasic(ds, key, bytes, otherBytes) + testBasic(ds, key, bytes, otherBytes, batch) From 9243b6b9769f0bcb37abca95f4baf474384b5829 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:35:02 -0700 Subject: [PATCH 198/445] key batch --- datastore/sql/sqliteutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 213438f7..dab5b816 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -76,7 +76,7 @@ proc bindParam( # to the return from sqlite3_bind_*(). The object and pointer to it must # remain valid until then. SQLite will then manage the lifetime of its # private copy." - sqlite3_bind_text(s, n.cint, val.toCString(), -1.cint, SQLITE_TRANSIENT) + sqlite3_bind_text(s, n.cint, val.toCString(), val.data.len().cint, SQLITE_TRANSIENT) else: {.fatal: "Please add support for the '" & $typeof(val) & "' type".} From 8fe5e3df057bb96be23396cd2f5c270c7695f893 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:36:20 -0700 Subject: [PATCH 199/445] key batch --- datastore/sql/sqliteutils.nim | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index dab5b816..a235eb6a 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -69,13 +69,9 @@ proc bindParam( # to the return from sqlite3_bind_*(). The object and pointer to it must # remain valid until then. SQLite will then manage the lifetime of its # private copy." - sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) + sqlite3_bind_text(s, n.cint, val.cstring, val.len().cint, SQLITE_TRANSIENT) elif val is KeyId: - # `-1` implies string length is num bytes up to first null-terminator; - # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior - # to the return from sqlite3_bind_*(). The object and pointer to it must - # remain valid until then. SQLite will then manage the lifetime of its - # private copy." + # same as previous sqlite3_bind_text(s, n.cint, val.toCString(), val.data.len().cint, SQLITE_TRANSIENT) else: {.fatal: "Please add support for the '" & $typeof(val) & "' type".} From d165ff2482a7215d6f1113a9a10b00d782cf4039 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:44:09 -0700 Subject: [PATCH 200/445] tests --- tests/datastore/sql/testsqliteds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index c5dfb461..72c0e435 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -98,5 +98,5 @@ suite "Test DataBuffer SQLiteDatastore": suiteTeardown: ds.close().tryGet() - + testBasic(ds, key, bytes, otherBytes, batch) From bab5cb989c59148276afc72eab7e422d783ce80b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:48:29 -0700 Subject: [PATCH 201/445] tests --- datastore/sql/sqliteutils.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index a235eb6a..f994f6c1 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -52,9 +52,7 @@ proc bindParam( # to the return from sqlite3_bind_*(). The object and pointer to it # must remain valid until then. SQLite will then manage the lifetime of # its private copy." - var val = val - sqlite3_bind_blob(s, n.cint, addr val[0], val.len.cint, - SQLITE_TRANSIENT) + sqlite3_bind_blob(s, n.cint, unsafeAddr val[0], val.len.cint, SQLITE_TRANSIENT) else: sqlite3_bind_null(s, n.cint) elif val is int32: From 6745c29efd1a52e4ff09f2c295490903e7511455 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:50:57 -0700 Subject: [PATCH 202/445] tests --- datastore/sql/sqliteutils.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index f994f6c1..66e0f02a 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -52,7 +52,8 @@ proc bindParam( # to the return from sqlite3_bind_*(). The object and pointer to it # must remain valid until then. SQLite will then manage the lifetime of # its private copy." - sqlite3_bind_blob(s, n.cint, unsafeAddr val[0], val.len.cint, SQLITE_TRANSIENT) + var val = val + sqlite3_bind_blob(s, n.cint, addr val[0], val.len.cint, SQLITE_TRANSIENT) else: sqlite3_bind_null(s, n.cint) elif val is int32: From 25bb865e208ed456823a8d816e538ed15bca8bd0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 00:00:46 -0700 Subject: [PATCH 203/445] tests --- datastore/sql/sqliteutils.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 66e0f02a..a235eb6a 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -53,7 +53,8 @@ proc bindParam( # must remain valid until then. SQLite will then manage the lifetime of # its private copy." var val = val - sqlite3_bind_blob(s, n.cint, addr val[0], val.len.cint, SQLITE_TRANSIENT) + sqlite3_bind_blob(s, n.cint, addr val[0], val.len.cint, + SQLITE_TRANSIENT) else: sqlite3_bind_null(s, n.cint) elif val is int32: From b224c1ce3e245d62216050dad1edb2b9854c4b9b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 00:04:05 -0700 Subject: [PATCH 204/445] cleanup --- datastore/threads/threadresult.nim | 7 ------- 1 file changed, 7 deletions(-) diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 5d17677d..c0fb7ed5 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -20,7 +20,6 @@ type ThreadTypes* = void | bool | SomeInteger | DataBuffer | tuple | Atomic ThreadResErr* = (ErrorEnum, DataBuffer) - # ThreadQueryRes* = tuple[key: KeyId, val: DataBuffer] ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] @@ -37,9 +36,3 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.QueryEndedErr: (ref QueryEndedError)(msg: $e[1]) of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) - -# converter toQueryResponse*(r: DbQueryResponse): QueryResponse = -# if not r.key.data.isNil and r.key.data.len > 0 and key =? Key.init($r.key.data): -# (key.some, @(r.val)) -# else: -# (Key.none, EmptyBytes) From 113f59cecc551eb21ecdc96e6caa2bcb206a5f66 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 17:21:20 -0700 Subject: [PATCH 205/445] don't store string --- datastore.nim | 2 +- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 4 ++-- datastore/sql/sqliteutils.nim | 2 +- tests/datastore/dscommontests.nim | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datastore.nim b/datastore.nim index 6d43a209..8242f6df 100644 --- a/datastore.nim +++ b/datastore.nim @@ -4,4 +4,4 @@ import ./datastore/sql import ./datastore/mountedds import ./datastore/tieredds -export datastore, fsds, mountedds, tieredds, sql +export datastore, fsds, mountedds, tieredds diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 423c4a16..8ccc97ad 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -214,7 +214,7 @@ proc contains*(self: SQLiteDatastore, key: DbKey): bool = proc new*(T: type SQLiteDatastore, path: string, - readOnly = false): ?!T = + readOnly = false): ?!SQLiteDatastore = let flags = diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index ea06e500..902be644 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -29,7 +29,7 @@ type SQLiteDsDb* = object readOnly*: bool - dbPath*: string + dbPath*: DataBuffer containsStmt*: ContainsStmt deleteStmt*: DeleteStmt env*: SQLite @@ -311,7 +311,7 @@ proc open*( success SQLiteDsDb( readOnly: readOnly, - dbPath: path, + dbPath: DataBuffer.new path, containsStmt: containsStmt, deleteStmt: deleteStmt, env: env.release, diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index a235eb6a..98840835 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -22,7 +22,7 @@ type AutoDisposed*[T: ptr|ref] = object val*: T - DataProc* = proc(s: RawStmtPtr) {.closure, gcsafe.} + DataProc* = proc(s: RawStmtPtr) {.gcsafe.} NoParams* = tuple # empty tuple diff --git a/tests/datastore/dscommontests.nim b/tests/datastore/dscommontests.nim index 24dd7d39..beda975e 100644 --- a/tests/datastore/dscommontests.nim +++ b/tests/datastore/dscommontests.nim @@ -5,7 +5,7 @@ import pkg/chronos import pkg/stew/results import pkg/questionable/results -import pkg/datastore +import pkg/datastore/datastore proc basicStoreTests*( ds: Datastore, From 06cb6fc8b0762c18867b1f4b52f480977cb84aaa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 17:31:47 -0700 Subject: [PATCH 206/445] rename to SQLiteBackend --- datastore/sql/sqliteds.nim | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 8ccc97ad..1faf7991 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -15,18 +15,18 @@ export backend, sqlitedsdb push: {.upraises: [].} type - SQLiteDatastore* = object + SQLiteBackend* = object db: SQLiteDsDb -proc path*(self: SQLiteDatastore): string = +proc path*(self: SQLiteBackend): string = self.db.dbPath -proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly +proc readOnly*(self: SQLiteBackend): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = +proc has*(self: SQLiteBackend, key: DbKey): ?!bool = var exists = false key = $key @@ -39,10 +39,10 @@ proc has*(self: SQLiteDatastore, key: DbKey): ?!bool = return success exists -proc delete*(self: SQLiteDatastore, key: DbKey): ?!void = +proc delete*(self: SQLiteBackend, key: DbKey): ?!void = return self.db.deleteStmt.exec(($key)) -proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = +proc delete*(self: SQLiteBackend, keys: openArray[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) @@ -58,7 +58,7 @@ proc delete*(self: SQLiteDatastore, keys: openArray[DbKey]): ?!void = return success() -proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = +proc get*(self: SQLiteBackend, key: DbKey): ?!seq[byte] = # see comment in ./filesystem_datastore re: finer control of memory # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` @@ -78,7 +78,7 @@ proc get*(self: SQLiteDatastore, key: DbKey): ?!seq[byte] = return success bytes -proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = +proc put*(self: SQLiteBackend, key: DbKey, data: DbVal): ?!void = when DbVal is seq[byte]: return self.db.putStmt.exec((key, data, timestamp())) elif DbVal is DataBuffer: @@ -86,7 +86,7 @@ proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = else: {.error: "unknown type".} -proc put*(self: SQLiteDatastore, batch: openArray[DbBatchEntry]): ?!void = +proc put*(self: SQLiteBackend, batch: openArray[DbBatchEntry]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err @@ -109,12 +109,12 @@ proc put*(self: SQLiteDatastore, batch: openArray[DbBatchEntry]): ?!void = return success() -proc close*(self: SQLiteDatastore): ?!void = +proc close*(self: SQLiteBackend): ?!void = self.db.close() return success() -proc query*(self: SQLiteDatastore, +proc query*(self: SQLiteBackend, query: DbQuery ): Result[iterator(): ?!DbQueryResponse {.closure.}, ref CatchableError] = @@ -208,23 +208,23 @@ proc query*(self: SQLiteDatastore, return -proc contains*(self: SQLiteDatastore, key: DbKey): bool = +proc contains*(self: SQLiteBackend, key: DbKey): bool = return self.has(key).get() -proc new*(T: type SQLiteDatastore, +proc new*(T: type SQLiteBackend, path: string, - readOnly = false): ?!SQLiteDatastore = + readOnly = false): ?!SQLiteBackend = let flags = if readOnly: SQLITE_OPEN_READONLY else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - success SQLiteDatastore(db: ? SQLiteDsDb.open(path, flags)) + success SQLiteBackend(db: ? SQLiteDsDb.open(path, flags)) -proc new*(T: type SQLiteDatastore, +proc new*(T: type SQLiteBackend, db: SQLiteDsDb): ?!T = - success SQLiteDatastore(db: db) + success SQLiteBackend(db: db) From 8356f6cd972d65d2765e48a2b052f3882723cc54 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 17:52:06 -0700 Subject: [PATCH 207/445] re-add async frontend --- datastore/sql.nim | 285 ++++++++++--------------------------- datastore/sql/sqliteds.nim | 5 +- 2 files changed, 81 insertions(+), 209 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 55a23e03..7e13da7d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -8,240 +8,111 @@ import pkg/sqlite3_abi from pkg/stew/results as stewResults import isErr import pkg/upraises +import std/sequtils import ../datastore -# import ./sqlitedsdb +import ./backend +import ./sql/sqliteds export datastore push: {.upraises: [].} -# type -# SQLiteDatastore* = ref object of Datastore -# readOnly: bool -# db: SQLiteDsDb +type + SQLiteDatastore* = ref object of Datastore + db: SQLiteBackend -# proc path*(self: SQLiteDatastore): string = -# self.db.dbPath +proc path*(self: SQLiteDatastore): string = + self.db.path() -# proc `readOnly=`*(self: SQLiteDatastore): bool -# {.error: "readOnly should not be assigned".} +proc readOnly*(self: SQLiteDatastore): bool = + self.db.readOnly() -# proc timestamp*(t = epochTime()): int64 = -# (t * 1_000_000).int64 +method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = + return self.db.has(KeyId.new key.id()) -# method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = -# var -# exists = false +method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = + return self.db.delete(KeyId.new key.id()) -# proc onData(s: RawStmtPtr) = -# exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool +method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = + let dkeys = keys.mapIt(KeyId.new it.id()) + return self.db.delete(dkeys) -# if err =? self.db.containsStmt.query((key.id), onData).errorOption: -# return failure err +method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = + self.db.get(KeyId.new key.id()) -# return success exists +method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = + self.db.put(KeyId.new key.id(), DataBuffer.new data) -# method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = -# return self.db.deleteStmt.exec((key.id)) +method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = + var dbatch: seq[tuple[key: string, data: seq[byte]]] + for entry in batch: + dbatch.add((entry.key.id(), entry.data)) + self.db.put(dbatch) -# method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = -# if err =? self.db.beginStmt.exec().errorOption: -# return failure(err) +method close*(self: SQLiteDatastore): Future[?!void] {.async.} = + self.db.close() -# for key in keys: -# if err =? self.db.deleteStmt.exec((key.id)).errorOption: -# if err =? self.db.rollbackStmt.exec().errorOption: -# return failure err.msg +method query*( + self: SQLiteDatastore, + query: Query +): Future[?!QueryIter] {.async.} = -# return failure err.msg + var iter = QueryIter() + let dbquery = DbQuery( + key: KeyId.new query.key.id(), + value: query.value, + limit: query.limit, + offset: query.offset, + sort: query.sort, + ) + var queries = ? self.db.query(dbquery) -# if err =? self.db.endStmt.exec().errorOption: -# return failure err.msg + let lock = newAsyncLock() + proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() -# return success() + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") -# method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = -# # see comment in ./filesystem_datastore re: finer control of memory -# # allocation in `method get`, could apply here as well if bytes were read -# # incrementally with `sqlite3_blob_read` + if iter.finished: + return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) -# var -# bytes: seq[byte] + await lock.acquire() -# proc onData(s: RawStmtPtr) = -# bytes = self.db.getDataCol() + let res = queries() + iter.result = res -# if err =? self.db.getStmt.query((key.id), onData).errorOption: -# return failure(err) + iter.finished = true -# if bytes.len <= 0: -# return failure( -# newException(DatastoreKeyNotFound, "Key doesn't exist")) + iter.dispose = proc(): Future[?!void] {.async.} = + discard sqlite3_reset(s) + discard sqlite3_clear_bindings(s) + s.dispose + return success() -# return success bytes + iter.next = next + return success iter -# method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = -# return self.db.putStmt.exec((key.id, data, timestamp())) +proc new*( + T: type SQLiteDatastore, + path: string, + readOnly = false): ?!T = -# method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = -# if err =? self.db.beginStmt.exec().errorOption: -# return failure err + let + flags = + if readOnly: SQLITE_OPEN_READONLY + else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE -# for entry in batch: -# if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: -# if err =? self.db.rollbackStmt.exec().errorOption: -# return failure err + success T( + db: ? SQLiteDsDb.open(path, flags), + readOnly: readOnly) -# return failure err +proc new*( + T: type SQLiteDatastore, + db: SQLiteDsDb): ?!T = -# if err =? self.db.endStmt.exec().errorOption: -# return failure err - -# return success() - -# method close*(self: SQLiteDatastore): Future[?!void] {.async.} = -# self.db.close() - -# return success() - -# method query*( -# self: SQLiteDatastore, -# query: Query): Future[?!QueryIter] {.async.} = - -# var -# iter = QueryIter() -# queryStr = if query.value: -# QueryStmtDataIdStr -# else: -# QueryStmtIdStr - -# if query.sort == SortOrder.Descending: -# queryStr &= QueryStmtOrderDescending -# else: -# queryStr &= QueryStmtOrderAscending - -# if query.limit != 0: -# queryStr &= QueryStmtLimit - -# if query.offset != 0: -# queryStr &= QueryStmtOffset - -# let -# queryStmt = QueryStmt.prepare( -# self.db.env, queryStr).expect("should not fail") - -# s = RawStmtPtr(queryStmt) - -# var -# v = sqlite3_bind_text( -# s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) - -# if not (v == SQLITE_OK): -# return failure newException(DatastoreError, $sqlite3_errstr(v)) - -# if query.limit != 0: -# v = sqlite3_bind_int(s, 2.cint, query.limit.cint) - -# if not (v == SQLITE_OK): -# return failure newException(DatastoreError, $sqlite3_errstr(v)) - -# if query.offset != 0: -# v = sqlite3_bind_int(s, 3.cint, query.offset.cint) - -# if not (v == SQLITE_OK): -# return failure newException(DatastoreError, $sqlite3_errstr(v)) - -# let lock = newAsyncLock() -# proc next(): Future[?!QueryResponse] {.async.} = -# defer: -# if lock.locked: -# lock.release() - -# if lock.locked: -# return failure (ref DatastoreError)(msg: "Should always await query features") - -# if iter.finished: -# return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) - -# await lock.acquire() - -# let -# v = sqlite3_step(s) - -# case v -# of SQLITE_ROW: -# let -# key = Key.init( -# $sqlite3_column_text_not_null(s, QueryStmtIdCol)) -# .expect("should not fail") - -# blob: ?pointer = -# if query.value: -# sqlite3_column_blob(s, QueryStmtDataCol).some -# else: -# pointer.none - -# # detect out-of-memory error -# # see the conversion table and final paragraph of: -# # https://www.sqlite.org/c3ref/column_blob.html -# # see also https://www.sqlite.org/rescode.html - -# # the "data" column can be NULL so in order to detect an out-of-memory -# # error it is necessary to check that the result is a null pointer and -# # that the result code is an error code -# if blob.isSome and blob.get().isNil: -# let -# v = sqlite3_errcode(sqlite3_db_handle(s)) - -# if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): -# iter.finished = true -# return failure newException(DatastoreError, $sqlite3_errstr(v)) - -# let -# dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) -# data = if blob.isSome: -# @( -# toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), -# 0, -# dataLen - 1)) -# else: -# @[] - -# return success (key.some, data) -# of SQLITE_DONE: -# iter.finished = true -# return success (Key.none, EmptyBytes) -# else: -# iter.finished = true -# return failure newException(DatastoreError, $sqlite3_errstr(v)) - -# iter.dispose = proc(): Future[?!void] {.async.} = -# discard sqlite3_reset(s) -# discard sqlite3_clear_bindings(s) -# s.dispose -# return success() - -# iter.next = next -# return success iter - -# proc new*( -# T: type SQLiteDatastore, -# path: string, -# readOnly = false): ?!T = - -# let -# flags = -# if readOnly: SQLITE_OPEN_READONLY -# else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - -# success T( -# db: ? SQLiteDsDb.open(path, flags), -# readOnly: readOnly) - -# proc new*( -# T: type SQLiteDatastore, -# db: SQLiteDsDb): ?!T = - -# success T( -# db: db, -# readOnly: db.readOnly) + success T( + db: db, + readOnly: db.readOnly) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 1faf7991..21f067f4 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -19,7 +19,7 @@ type db: SQLiteDsDb proc path*(self: SQLiteBackend): string = - self.db.dbPath + $self.db.dbPath proc readOnly*(self: SQLiteBackend): bool = self.db.readOnly @@ -197,11 +197,12 @@ proc query*(self: SQLiteBackend, return success (key.some, data) of SQLITE_DONE: - return + return success (KeyId.none, DataBuffer.new()) else: return failure newException(DatastoreError, $sqlite3_errstr(v)) finally: + echo "sqlite backend: query: finally close" discard sqlite3_reset(s) discard sqlite3_clear_bindings(s) s.dispose() From 4750ac69df61dbc87b1c2301f239dde70784272e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 18:08:00 -0700 Subject: [PATCH 208/445] implementing query --- datastore.nim | 2 +- datastore/sql.nim | 6 +++--- datastore/sql/sqliteds.nim | 3 ++- tests/datastore/sql/testsqliteds.nim | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/datastore.nim b/datastore.nim index 8242f6df..5a971a5c 100644 --- a/datastore.nim +++ b/datastore.nim @@ -1,6 +1,6 @@ import ./datastore/datastore import ./datastore/fsds -import ./datastore/sql +# import ./datastore/sql import ./datastore/mountedds import ./datastore/tieredds diff --git a/datastore/sql.nim b/datastore/sql.nim index 7e13da7d..3c323687 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -81,10 +81,10 @@ method query*( await lock.acquire() - let res = queries() - iter.result = res + without res =? queries(), err: + iter.finished = true + return failure err - iter.finished = true iter.dispose = proc(): Future[?!void] {.async.} = discard sqlite3_reset(s) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 21f067f4..c4e62308 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -140,10 +140,11 @@ proc query*(self: SQLiteBackend, self.db.env, queryStr).expect("should not fail") s = RawStmtPtr(queryStmt) + queryKey = $query.key & "*" var v = sqlite3_bind_text( - s, 1.cint, ($query.key & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) + s, 1.cint, queryKey.cstring, queryKey.len().cint, SQLITE_TRANSIENT_GCSAFE) if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 72c0e435..0187e84d 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -15,7 +15,7 @@ import ../dscommontests import ../querycommontests proc testBasic[K, V, B]( - ds: SQLiteDatastore, + ds: SQLiteBackend, key: K, bytes: V, otherBytes: V, @@ -67,7 +67,7 @@ proc testBasic[K, V, B]( suite "Test Basic SQLiteDatastore": let - ds = SQLiteDatastore.new(Memory).tryGet() + ds = SQLiteBackend.new(Memory).tryGet() keyFull = Key.init("a:b/c/d:e").tryGet() key = keyFull.id() bytes = "some bytes".toBytes @@ -85,7 +85,7 @@ suite "Test Basic SQLiteDatastore": suite "Test DataBuffer SQLiteDatastore": let - ds = SQLiteDatastore.new(Memory).tryGet() + ds = SQLiteBackend.new(Memory).tryGet() keyFull = Key.init("a:b/c/d:e").tryGet() key = KeyId.new keyFull.id() bytes = DataBuffer.new "some bytes" From cbb38a51fb42539d7b9e1565b26d46a73ee66f37 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 18:29:07 -0700 Subject: [PATCH 209/445] trying to fixup query --- datastore/backend.nim | 3 +++ datastore/sql/sqliteds.nim | 23 +++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 3d757bcb..3ad4920a 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -29,6 +29,9 @@ type offset*: int # Offset from which to start querying - not available in all backends sort*: SortOrder # Sort order - not available in all backends + DbQueryHandle* = ref object + cancel*: bool + DbQueryResponse* = tuple[key: Option[KeyId], val: DataBuffer] proc `$`*(id: KeyId): string = $(id.data) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index c4e62308..09ce3488 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -114,9 +114,11 @@ proc close*(self: SQLiteBackend): ?!void = return success() -proc query*(self: SQLiteBackend, - query: DbQuery - ): Result[iterator(): ?!DbQueryResponse {.closure.}, ref CatchableError] = +proc query*( + self: SQLiteBackend, + query: DbQuery +): Result[(DbQueryHandle, iterator(): ?!DbQueryResponse {.closure.}), + ref CatchableError] = var queryStr = if query.value: @@ -161,11 +163,14 @@ proc query*(self: SQLiteBackend, if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - success iterator(): ?!DbQueryResponse {.closure.} = + let handle = DbQueryHandle() + let iter = iterator(): ?!DbQueryResponse {.closure.} = try: - let - v = sqlite3_step(s) + if handle.cancel: + return + + let v = sqlite3_step(s) case v of SQLITE_ROW: @@ -196,9 +201,9 @@ proc query*(self: SQLiteBackend, if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) else: DataBuffer.new(0) - return success (key.some, data) + yield success (key.some, data) of SQLITE_DONE: - return success (KeyId.none, DataBuffer.new()) + return else: return failure newException(DatastoreError, $sqlite3_errstr(v)) @@ -208,6 +213,8 @@ proc query*(self: SQLiteBackend, discard sqlite3_clear_bindings(s) s.dispose() return + + success (handle, iter) proc contains*(self: SQLiteBackend, key: DbKey): bool = From 3ae1c60fa1b8e569a12fc85e90feb48ebe49d6c4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 18:49:08 -0700 Subject: [PATCH 210/445] porting query tests --- datastore/backend.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 215 +++++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 1 deletion(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 3ad4920a..0b609c72 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -32,7 +32,7 @@ type DbQueryHandle* = ref object cancel*: bool - DbQueryResponse* = tuple[key: Option[KeyId], val: DataBuffer] + DbQueryResponse* = tuple[key: Option[KeyId], data: DataBuffer] proc `$`*(id: KeyId): string = $(id.data) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 0187e84d..ac548c84 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -100,3 +100,218 @@ suite "Test DataBuffer SQLiteDatastore": ds.close().tryGet() testBasic(ds, key, bytes, otherBytes, batch) + +suite "queryTests": + let + ds = SQLiteBackend.new(Memory).tryGet() + + var + key1: KeyId + key2: KeyId + key3: KeyId + val1: DataBuffer + val2: DataBuffer + val3: DataBuffer + + setup: + key1 = KeyId.new "/a" + key2 = KeyId.new "/a/b" + key3 = KeyId.new "/a/b/c" + val1 = DataBuffer.new "value for 1" + val2 = DataBuffer.new "value for 2" + val3 = DataBuffer.new "value for 3" + + test "Key should query all keys and all it's children": + let + q = DbQuery(key: key1) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 + + res[1].key.get == key2 + res[1].data == val2 + + res[2].key.get == key3 + res[2].data == val3 + + test "Key should query all keys without values": + let + q = DbQuery(key: key1, value: false) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 + + res[1].key.get == key2 + res[1].data.len == 0 + + res[2].key.get == key3 + res[2].data.len == 0 + + + test "Key should not query parent": + let + q = DbQuery(key: key1) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 + + res[1].key.get == key3 + res[1].data == val3 + + test "Key should all list all keys at the same level": + let + queryKey = Key.init("/a").tryGet + q = DbQuery(key: key1) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + res.sort do (a, b: DbQueryResponse) -> int: + cmp($a.key.get, $b.key.get) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 + + res[1].key.get == key2 + res[1].data == val2 + + res[2].key.get == key3 + res[2].data == val3 + + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = DbQuery(key: key1, value: false) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes + + ds.put(key, val).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 10 + + test "Should not apply offset": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = DbQuery(key: KeyId.new $key, offset: 90) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) + + ds.put(keyId, val).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 10 + + + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = DbQuery(key: keyId, offset: 95, limit: 5) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) + + ds.put(key, val).tryGet + + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 5 + + for i in 0.. int: + cmp($a.key.get, $b.key.get) + + kvs = kvs.reversed + let + (handle, iter) = ds.query(q).tryGet + res = iter.mapIt(it.tryGet()) + + check: + res.len == 100 + + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data From a339bb3c95c2720562e9bf99588aa17a212902dd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 18:50:48 -0700 Subject: [PATCH 211/445] porting query tests --- tests/datastore/sql/testsqliteds.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index ac548c84..9e4ce018 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -223,7 +223,7 @@ suite "queryTests": for i in 0..<100: let key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + val = DataBuffer.new("val " & $i) ds.put(key, val).tryGet @@ -277,8 +277,8 @@ suite "queryTests": for i in 0.. Date: Thu, 21 Sep 2023 18:54:42 -0700 Subject: [PATCH 212/445] porting query tests --- datastore/sql/sqliteds.nim | 98 ++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 09ce3488..2565a43c 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -163,58 +163,62 @@ proc query*( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - let handle = DbQueryHandle() - let iter = iterator(): ?!DbQueryResponse {.closure.} = - - try: - if handle.cancel: - return - - let v = sqlite3_step(s) - - case v - of SQLITE_ROW: - let - key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) - - blob: ?pointer = - if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some - else: pointer.none - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html - - # the "data" column can be NULL so in order to detect an out-of-memory - # error it is necessary to check that the result is a null pointer and - # that the result code is an error code - if blob.isSome and blob.get().isNil: - let v = sqlite3_errcode(sqlite3_db_handle(s)) - - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - let - dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) - data = - if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) - else: DataBuffer.new(0) - - yield success (key.some, data) - of SQLITE_DONE: - return - else: - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - finally: + proc doClose() = echo "sqlite backend: query: finally close" discard sqlite3_reset(s) discard sqlite3_clear_bindings(s) s.dispose() return - - success (handle, iter) + + let handle = DbQueryHandle() + let iter = iterator(): ?!DbQueryResponse {.closure.} = + + if handle.cancel: + doClose() + return + + let v = sqlite3_step(s) + + case v + of SQLITE_ROW: + echo "SQLITE ROW" + let + key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) + + blob: ?pointer = + if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some + else: pointer.none + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let v = sqlite3_errcode(sqlite3_db_handle(s)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + data = + if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) + else: DataBuffer.new(0) + + echo "SQLITE ROW: yield" + yield success (key.some, data) + of SQLITE_DONE: + echo "SQLITE DONE: return" + doClose() + return + else: + echo "SQLITE ERROR: return" + doClose() + return failure newException(DatastoreError, $sqlite3_errstr(v)) proc contains*(self: SQLiteBackend, key: DbKey): bool = From d6d5978d5c3047a226e34485f3eb84e65a1d0a69 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 19:08:19 -0700 Subject: [PATCH 213/445] porting query tests --- datastore/sql/sqliteds.nim | 96 +++++----- tests/datastore/sql/testsqliteds.nim | 258 +++++++++++++-------------- 2 files changed, 179 insertions(+), 175 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 2565a43c..c2b9ef13 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -173,52 +173,56 @@ proc query*( let handle = DbQueryHandle() let iter = iterator(): ?!DbQueryResponse {.closure.} = - if handle.cancel: - doClose() - return - - let v = sqlite3_step(s) - - case v - of SQLITE_ROW: - echo "SQLITE ROW" - let - key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) - - blob: ?pointer = - if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some - else: pointer.none - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html - - # the "data" column can be NULL so in order to detect an out-of-memory - # error it is necessary to check that the result is a null pointer and - # that the result code is an error code - if blob.isSome and blob.get().isNil: - let v = sqlite3_errcode(sqlite3_db_handle(s)) - - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - let - dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) - data = - if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) - else: DataBuffer.new(0) - - echo "SQLITE ROW: yield" - yield success (key.some, data) - of SQLITE_DONE: - echo "SQLITE DONE: return" - doClose() - return - else: - echo "SQLITE ERROR: return" - doClose() - return failure newException(DatastoreError, $sqlite3_errstr(v)) + while true: + + if handle.cancel: + doClose() + return + + let v = sqlite3_step(s) + + case v + of SQLITE_ROW: + echo "SQLITE ROW" + let + key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) + + blob: ?pointer = + if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some + else: pointer.none + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let v = sqlite3_errcode(sqlite3_db_handle(s)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + data = + if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) + else: DataBuffer.new(0) + + echo "SQLITE ROW: yield" + yield success (key.some, data) + of SQLITE_DONE: + echo "SQLITE DONE: return" + doClose() + return + else: + echo "SQLITE ERROR: return" + doClose() + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + success (handle, iter) proc contains*(self: SQLiteBackend, key: DbKey): bool = diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 9e4ce018..4f5df8b6 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -131,7 +131,7 @@ suite "queryTests": let (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + res = iter.mapIt(it.tryGet()).reversed() check: res.len == 3 @@ -144,175 +144,175 @@ suite "queryTests": res[2].key.get == key3 res[2].data == val3 - test "Key should query all keys without values": - let - q = DbQuery(key: key1, value: false) + # test "Key should query all keys without values": + # let + # q = DbQuery(key: key1, value: false) - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet + # ds.put(key1, val1).tryGet + # ds.put(key2, val2).tryGet + # ds.put(key3, val3).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 3 - res[0].key.get == key1 - res[0].data.len == 0 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data.len == 0 - res[1].key.get == key2 - res[1].data.len == 0 + # res[1].key.get == key2 + # res[1].data.len == 0 - res[2].key.get == key3 - res[2].data.len == 0 + # res[2].key.get == key3 + # res[2].data.len == 0 - test "Key should not query parent": - let - q = DbQuery(key: key1) + # test "Key should not query parent": + # let + # q = DbQuery(key: key1) - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet + # ds.put(key1, val1).tryGet + # ds.put(key2, val2).tryGet + # ds.put(key3, val3).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 2 - res[0].key.get == key2 - res[0].data == val2 + # check: + # res.len == 2 + # res[0].key.get == key2 + # res[0].data == val2 - res[1].key.get == key3 - res[1].data == val3 + # res[1].key.get == key3 + # res[1].data == val3 - test "Key should all list all keys at the same level": - let - queryKey = Key.init("/a").tryGet - q = DbQuery(key: key1) + # test "Key should all list all keys at the same level": + # let + # queryKey = Key.init("/a").tryGet + # q = DbQuery(key: key1) - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet + # ds.put(key1, val1).tryGet + # ds.put(key2, val2).tryGet + # ds.put(key3, val3).tryGet - var - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # var + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - res.sort do (a, b: DbQueryResponse) -> int: - cmp($a.key.get, $b.key.get) + # res.sort do (a, b: DbQueryResponse) -> int: + # cmp($a.key.get, $b.key.get) - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data == val1 - res[1].key.get == key2 - res[1].data == val2 + # res[1].key.get == key2 + # res[1].data == val2 - res[2].key.get == key3 - res[2].data == val3 + # res[2].key.get == key3 + # res[2].data == val3 - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = DbQuery(key: key1, value: false) + # test "Should apply limit": + # let + # key = Key.init("/a").tryGet + # q = DbQuery(key: key1, value: false) - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) + # for i in 0..<100: + # let + # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = DataBuffer.new("val " & $i) - ds.put(key, val).tryGet + # ds.put(key, val).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 10 + # check: + # res.len == 10 - test "Should not apply offset": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = DbQuery(key: KeyId.new $key, offset: 90) + # test "Should not apply offset": + # let + # key = Key.init("/a").tryGet + # keyId = KeyId.new $key + # q = DbQuery(key: KeyId.new $key, offset: 90) - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) + # for i in 0..<100: + # let + # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = DataBuffer.new("val " & $i) - ds.put(keyId, val).tryGet + # ds.put(keyId, val).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 10 + # check: + # res.len == 10 - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = DbQuery(key: keyId, offset: 95, limit: 5) + # test "Should not apply offset and limit": + # let + # key = Key.init("/a").tryGet + # keyId = KeyId.new $key + # q = DbQuery(key: keyId, offset: 95, limit: 5) - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) + # for i in 0..<100: + # let + # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = DataBuffer.new("val " & $i) - ds.put(key, val).tryGet + # ds.put(key, val).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 5 + # check: + # res.len == 5 - for i in 0.. int: - cmp($a.key.get, $b.key.get) + # # lexicographic sort, as it comes from the backend + # kvs.sort do (a, b: DbQueryResponse) -> int: + # cmp($a.key.get, $b.key.get) - kvs = kvs.reversed - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()) + # kvs = kvs.reversed + # let + # (handle, iter) = ds.query(q).tryGet + # res = iter.mapIt(it.tryGet()) - check: - res.len == 100 + # check: + # res.len == 100 - for i, r in res[1..^1]: - check: - res[i].key.get == kvs[i].key.get - res[i].data == kvs[i].data + # for i, r in res[1..^1]: + # check: + # res[i].key.get == kvs[i].key.get + # res[i].data == kvs[i].data From 0971986863331b0d0ce3b7d2b700863f4a6c45d5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 19:19:29 -0700 Subject: [PATCH 214/445] porting query tests --- datastore/backend.nim | 2 +- datastore/sql/sqliteds.nim | 4 +++- datastore/threads/databuffer.nim | 24 +++++++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 0b609c72..70414b12 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -38,7 +38,7 @@ proc `$`*(id: KeyId): string = $(id.data) proc new*(tp: typedesc[KeyId], id: cstring): KeyId = ## copy cstring including null terminator - KeyId(data: DataBuffer.new(id.pointer, 0, id.len())) + KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1), {dbNullTerminate})) proc new*(tp: typedesc[KeyId], id: string): KeyId = ## copy cstring including null terminator diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index c2b9ef13..5405b881 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -208,7 +208,9 @@ proc query*( let dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) data = - if blob.isSome: DataBuffer.new(blob.get(), 0, dataLen - 1) + if blob.isSome: + let arr = cast[ptr UncheckedArray[byte]](blob) + DataBuffer.new(arr.toOpenArray(0, dataLen-1)) else: DataBuffer.new(0) echo "SQLITE ROW: yield" diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 7be334fd..dcc8a728 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -40,13 +40,6 @@ proc `[]`*(db: DataBuffer, idx: int): var byte = raise newException(IndexDefect, "index out of bounds") db[].buf[idx] -proc `==`*(a, b: DataBuffer): bool = - if a.isNil and b.isNil: return true - elif a.isNil or b.isNil: return false - elif a[].size != b[].size: return false - elif a[].buf == b[].buf: return true - else: a.hash() == b.hash() - template `==`*[T: char | byte](a: DataBuffer, b: openArray[T]): bool = if a.isNil: false elif a[].size != b.len: false @@ -73,8 +66,8 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[Dat copyMem(result[].buf, baseAddr data, data.len()) result[].size = data.len() -proc new*(tp: type DataBuffer, data: pointer, first, last: int): DataBuffer = - DataBuffer.new(toOpenArray(cast[ptr UncheckedArray[byte]](data), first, last)) +# proc new*(tp: type DataBuffer, data: pointer, first, last: int): DataBuffer = +# DataBuffer.new(toOpenArray(cast[ptr UncheckedArray[byte]](data), first, last)) proc baseAddr*(db: DataBuffer): pointer = db[].buf @@ -122,6 +115,19 @@ proc `$`*(data: DataBuffer): string = data.toString() +proc `==`*(a, b: DataBuffer): bool = + echo "DB == ", a.toString, " ", b.toString + echo "DB == len: ", a.len, " ", b.len + echo "DB == size: ", a[].size, " ", b[].size + echo "DB == cap: ", a[].cap, " ", b[].cap + echo "DB == ", a[].buf.pointer.repr, " ", b[].buf.pointer.repr + echo "DB == ", a[].hash, " ", b[].hash + if a.isNil and b.isNil: return true + elif a.isNil or b.isNil: return false + elif a[].size != b[].size: return false + elif a[].buf == b[].buf: return true + else: a.hash() == b.hash() + converter toBuffer*(err: ref CatchableError): DataBuffer = ## convert exception to an object with StringBuffer ## From 185f162050c18cb418dbf936507d3d0767919888 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 21 Sep 2023 19:23:23 -0700 Subject: [PATCH 215/445] porting query tests --- datastore/sql/sqliteds.nim | 2 +- datastore/threads/databuffer.nim | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 5405b881..3fa7010c 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -211,7 +211,7 @@ proc query*( if blob.isSome: let arr = cast[ptr UncheckedArray[byte]](blob) DataBuffer.new(arr.toOpenArray(0, dataLen-1)) - else: DataBuffer.new(0) + else: DataBuffer.new("") echo "SQLITE ROW: yield" yield success (key.some, data) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index dcc8a728..928ffd6d 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -45,7 +45,7 @@ template `==`*[T: char | byte](a: DataBuffer, b: openArray[T]): bool = elif a[].size != b.len: false else: a.hash() == b.hash() -proc new*(tp: type DataBuffer, capacity: int = 0): DataBuffer = +proc new(tp: type DataBuffer, capacity: int = 0): DataBuffer = ## allocate new buffer with given capacity ## @@ -122,11 +122,13 @@ proc `==`*(a, b: DataBuffer): bool = echo "DB == cap: ", a[].cap, " ", b[].cap echo "DB == ", a[].buf.pointer.repr, " ", b[].buf.pointer.repr echo "DB == ", a[].hash, " ", b[].hash - if a.isNil and b.isNil: return true - elif a.isNil or b.isNil: return false - elif a[].size != b[].size: return false - elif a[].buf == b[].buf: return true - else: a.hash() == b.hash() + if a.isNil and b.isNil: result = true + elif a.isNil or b.isNil: result = false + elif a[].size != b[].size: result = false + elif a[].buf == b[].buf: result = true + else: result = a.hash() == b.hash() + echo "DB == ", result + echo "" converter toBuffer*(err: ref CatchableError): DataBuffer = ## convert exception to an object with StringBuffer From 10ab970cdfaa41caff2358f9d5ec28c1cb3053ec Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 15:31:22 -0700 Subject: [PATCH 216/445] changes --- datastore/backend.nim | 4 ++-- datastore/threads/databuffer.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 70414b12..1f01d595 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -38,11 +38,11 @@ proc `$`*(id: KeyId): string = $(id.data) proc new*(tp: typedesc[KeyId], id: cstring): KeyId = ## copy cstring including null terminator - KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1), {dbNullTerminate})) + KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1))) proc new*(tp: typedesc[KeyId], id: string): KeyId = ## copy cstring including null terminator - KeyId(data: DataBuffer.new(id, {dbNullTerminate})) + KeyId(data: DataBuffer.new(id)) proc toCString*(key: KeyId): cstring = ## copy cstring including null terminator diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 928ffd6d..622d137a 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -121,7 +121,7 @@ proc `==`*(a, b: DataBuffer): bool = echo "DB == size: ", a[].size, " ", b[].size echo "DB == cap: ", a[].cap, " ", b[].cap echo "DB == ", a[].buf.pointer.repr, " ", b[].buf.pointer.repr - echo "DB == ", a[].hash, " ", b[].hash + echo "DB == ", a.hash, " ", b.hash if a.isNil and b.isNil: result = true elif a.isNil or b.isNil: result = false elif a[].size != b[].size: result = false diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 4f5df8b6..1982bafd 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -136,13 +136,13 @@ suite "queryTests": check: res.len == 3 res[0].key.get == key1 - res[0].data == val1 + # res[0].data == val1 - res[1].key.get == key2 - res[1].data == val2 + # res[1].key.get == key2 + # res[1].data == val2 - res[2].key.get == key3 - res[2].data == val3 + # res[2].key.get == key3 + # res[2].data == val3 # test "Key should query all keys without values": # let From 0a86a3466ae945550271342d554a153ac1482382 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 15:43:27 -0700 Subject: [PATCH 217/445] change to non-closure iterator --- datastore/backend.nim | 3 ++- datastore/sql/sqliteds.nim | 34 +++++++++++++--------------- tests/datastore/sql/testsqliteds.nim | 2 +- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 1f01d595..1c5e739d 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -29,8 +29,9 @@ type offset*: int # Offset from which to start querying - not available in all backends sort*: SortOrder # Sort order - not available in all backends - DbQueryHandle* = ref object + DbQueryHandle*[T] = object cancel*: bool + env*: T DbQueryResponse* = tuple[key: Option[KeyId], data: DataBuffer] diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 3fa7010c..cb3457e3 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -91,7 +91,6 @@ proc put*(self: SQLiteBackend, batch: openArray[DbBatchEntry]): ?!void = return failure err for entry in batch: - # DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] when entry.key is string: let putStmt = self.db.putStmt elif entry.key is KeyId: @@ -117,8 +116,7 @@ proc close*(self: SQLiteBackend): ?!void = proc query*( self: SQLiteBackend, query: DbQuery -): Result[(DbQueryHandle, iterator(): ?!DbQueryResponse {.closure.}), - ref CatchableError] = +): Result[DbQueryHandle, ref CatchableError] = var queryStr = if query.value: @@ -163,32 +161,32 @@ proc query*( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - proc doClose() = - echo "sqlite backend: query: finally close" - discard sqlite3_reset(s) - discard sqlite3_clear_bindings(s) - s.dispose() - return +proc close*(handle: DbQueryHandle[RawStmtPtr]) = + echo "sqlite backend: query: finally close" + discard sqlite3_reset(handle.env) + discard sqlite3_clear_bindings(handle.env) + handle.env.dispose() + return - let handle = DbQueryHandle() +iterator iter*(handle: var DbQueryHandle): ?!DbQueryResponse = let iter = iterator(): ?!DbQueryResponse {.closure.} = while true: if handle.cancel: - doClose() + handle.close() return - let v = sqlite3_step(s) + let v = sqlite3_step(handle.env) case v of SQLITE_ROW: echo "SQLITE ROW" let - key = KeyId.new(sqlite3_column_text_not_null(s, QueryStmtIdCol)) + key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) blob: ?pointer = - if query.value: sqlite3_column_blob(s, QueryStmtDataCol).some + if query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some else: pointer.none # detect out-of-memory error @@ -200,13 +198,13 @@ proc query*( # error it is necessary to check that the result is a null pointer and # that the result code is an error code if blob.isSome and blob.get().isNil: - let v = sqlite3_errcode(sqlite3_db_handle(s)) + let v = sqlite3_errcode(sqlite3_db_handle(handle.env)) if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): return failure newException(DatastoreError, $sqlite3_errstr(v)) let - dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + dataLen = sqlite3_column_bytes(handle.env, QueryStmtDataCol) data = if blob.isSome: let arr = cast[ptr UncheckedArray[byte]](blob) @@ -217,11 +215,11 @@ proc query*( yield success (key.some, data) of SQLITE_DONE: echo "SQLITE DONE: return" - doClose() + handle.close() return else: echo "SQLITE ERROR: return" - doClose() + handle.close() return failure newException(DatastoreError, $sqlite3_errstr(v)) success (handle, iter) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 1982bafd..419402b6 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -136,7 +136,7 @@ suite "queryTests": check: res.len == 3 res[0].key.get == key1 - # res[0].data == val1 + res[0].data == val1 # res[1].key.get == key2 # res[1].data == val2 From f8ba1e768433d1ba7f0d1fc52b748811d4d8e200 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 15:46:03 -0700 Subject: [PATCH 218/445] change to non-closure iterator --- datastore/backend.nim | 1 + datastore/sql/sqliteds.nim | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 1c5e739d..8f1fb7b6 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -30,6 +30,7 @@ type sort*: SortOrder # Sort order - not available in all backends DbQueryHandle*[T] = object + query*: DbQuery cancel*: bool env*: T diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index cb3457e3..75f9bab5 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -116,7 +116,7 @@ proc close*(self: SQLiteBackend): ?!void = proc query*( self: SQLiteBackend, query: DbQuery -): Result[DbQueryHandle, ref CatchableError] = +): Result[DbQueryHandle[RawStmtPtr], ref CatchableError] = var queryStr = if query.value: @@ -161,6 +161,8 @@ proc query*( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) + success DbQueryHandle[RawStmtPtr](query: query, env: s) + proc close*(handle: DbQueryHandle[RawStmtPtr]) = echo "sqlite backend: query: finally close" discard sqlite3_reset(handle.env) @@ -168,7 +170,7 @@ proc close*(handle: DbQueryHandle[RawStmtPtr]) = handle.env.dispose() return -iterator iter*(handle: var DbQueryHandle): ?!DbQueryResponse = +iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = let iter = iterator(): ?!DbQueryResponse {.closure.} = while true: @@ -186,7 +188,7 @@ iterator iter*(handle: var DbQueryHandle): ?!DbQueryResponse = key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) blob: ?pointer = - if query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some + if handle.query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some else: pointer.none # detect out-of-memory error @@ -222,7 +224,6 @@ iterator iter*(handle: var DbQueryHandle): ?!DbQueryResponse = handle.close() return failure newException(DatastoreError, $sqlite3_errstr(v)) - success (handle, iter) proc contains*(self: SQLiteBackend, key: DbKey): bool = From 598f63c65113f0446c86badd58a42006f45bb270 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 15:54:07 -0700 Subject: [PATCH 219/445] change to non-closure iterator --- datastore/sql/sqliteds.nim | 102 +++++++++++++-------------- tests/datastore/sql/testsqliteds.nim | 10 ++- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 75f9bab5..d14080b9 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -171,59 +171,55 @@ proc close*(handle: DbQueryHandle[RawStmtPtr]) = return iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = - let iter = iterator(): ?!DbQueryResponse {.closure.} = - - while true: - - if handle.cancel: - handle.close() - return - - let v = sqlite3_step(handle.env) - - case v - of SQLITE_ROW: - echo "SQLITE ROW" - let - key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) - - blob: ?pointer = - if handle.query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some - else: pointer.none - - # detect out-of-memory error - # see the conversion table and final paragraph of: - # https://www.sqlite.org/c3ref/column_blob.html - # see also https://www.sqlite.org/rescode.html - - # the "data" column can be NULL so in order to detect an out-of-memory - # error it is necessary to check that the result is a null pointer and - # that the result code is an error code - if blob.isSome and blob.get().isNil: - let v = sqlite3_errcode(sqlite3_db_handle(handle.env)) - - if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - return failure newException(DatastoreError, $sqlite3_errstr(v)) - - let - dataLen = sqlite3_column_bytes(handle.env, QueryStmtDataCol) - data = - if blob.isSome: - let arr = cast[ptr UncheckedArray[byte]](blob) - DataBuffer.new(arr.toOpenArray(0, dataLen-1)) - else: DataBuffer.new("") - - echo "SQLITE ROW: yield" - yield success (key.some, data) - of SQLITE_DONE: - echo "SQLITE DONE: return" - handle.close() - return - else: - echo "SQLITE ERROR: return" - handle.close() - return failure newException(DatastoreError, $sqlite3_errstr(v)) - + while not handle.cancel: + + let v = sqlite3_step(handle.env) + + case v + of SQLITE_ROW: + echo "SQLITE ROW" + let + key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) + + blob: ?pointer = + if handle.query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some + else: pointer.none + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let v = sqlite3_errcode(sqlite3_db_handle(handle.env)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + handle.cancel = true + yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(handle.env, QueryStmtDataCol) + data = + if blob.isSome: + let arr = cast[ptr UncheckedArray[byte]](blob) + DataBuffer.new(arr.toOpenArray(0, dataLen-1)) + else: DataBuffer.new("") + + echo "SQLITE ROW: yield" + yield success (key.some, data) + of SQLITE_DONE: + echo "SQLITE DONE: return" + break + else: + echo "SQLITE ERROR: return" + handle.cancel = true + yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) + + handle.close() + proc contains*(self: SQLiteBackend, key: DbKey): bool = diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 419402b6..25521c51 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -129,9 +129,13 @@ suite "queryTests": ds.put(key2, val2).tryGet ds.put(key3, val3).tryGet - let - (handle, iter) = ds.query(q).tryGet - res = iter.mapIt(it.tryGet()).reversed() + var + handle = ds.query(q).tryGet + # res = handle.iter().mapIt(it.tryGet()).reversed() + + var res: seq[DbQueryResponse] + for item in handle.iter(): + res.insert(item.tryGet(), 0) check: res.len == 3 From 7bb11c0e0b80d26379c30c762c2238fd4c7758fe Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 16:06:09 -0700 Subject: [PATCH 220/445] change to non-closure iterator --- datastore/backend.nim | 1 + datastore/sql/sqliteds.nim | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 8f1fb7b6..c2134567 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -32,6 +32,7 @@ type DbQueryHandle*[T] = object query*: DbQuery cancel*: bool + closed*: bool env*: T DbQueryResponse* = tuple[key: Option[KeyId], data: DataBuffer] diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index d14080b9..8185e57a 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -163,12 +163,13 @@ proc query*( success DbQueryHandle[RawStmtPtr](query: query, env: s) -proc close*(handle: DbQueryHandle[RawStmtPtr]) = +proc close*(handle: var DbQueryHandle[RawStmtPtr]) = + if not handle.closed: + handle.closed = true echo "sqlite backend: query: finally close" discard sqlite3_reset(handle.env) discard sqlite3_clear_bindings(handle.env) handle.env.dispose() - return iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = while not handle.cancel: @@ -211,15 +212,16 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = echo "SQLITE ROW: yield" yield success (key.some, data) of SQLITE_DONE: - echo "SQLITE DONE: return" + echo "SQLITE DONE: yield" + handle.close() break else: - echo "SQLITE ERROR: return" + echo "SQLITE ERROR: yield" handle.cancel = true yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) + break handle.close() - proc contains*(self: SQLiteBackend, key: DbKey): bool = From f0fc8ce224ea60cbc1e45c59a1ecb870b3db44b4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 16:08:59 -0700 Subject: [PATCH 221/445] change to non-closure iterator --- datastore/sql/sqliteds.nim | 1 + datastore/threads/databuffer.nim | 8 -------- tests/datastore/sql/testsqliteds.nim | 10 +++++----- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 8185e57a..0673acc5 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -195,6 +195,7 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = # error it is necessary to check that the result is a null pointer and # that the result code is an error code if blob.isSome and blob.get().isNil: + echo "BLOB: isSome" let v = sqlite3_errcode(sqlite3_db_handle(handle.env)) if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 622d137a..be08c02d 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -116,19 +116,11 @@ proc `$`*(data: DataBuffer): string = data.toString() proc `==`*(a, b: DataBuffer): bool = - echo "DB == ", a.toString, " ", b.toString - echo "DB == len: ", a.len, " ", b.len - echo "DB == size: ", a[].size, " ", b[].size - echo "DB == cap: ", a[].cap, " ", b[].cap - echo "DB == ", a[].buf.pointer.repr, " ", b[].buf.pointer.repr - echo "DB == ", a.hash, " ", b.hash if a.isNil and b.isNil: result = true elif a.isNil or b.isNil: result = false elif a[].size != b[].size: result = false elif a[].buf == b[].buf: result = true else: result = a.hash() == b.hash() - echo "DB == ", result - echo "" converter toBuffer*(err: ref CatchableError): DataBuffer = ## convert exception to an object with StringBuffer diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 25521c51..9dae616b 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -123,7 +123,7 @@ suite "queryTests": test "Key should query all keys and all it's children": let - q = DbQuery(key: key1) + q = DbQuery(key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -142,11 +142,11 @@ suite "queryTests": res[0].key.get == key1 res[0].data == val1 - # res[1].key.get == key2 - # res[1].data == val2 + res[1].key.get == key2 + res[1].data == val2 - # res[2].key.get == key3 - # res[2].data == val3 + res[2].key.get == key3 + res[2].data == val3 # test "Key should query all keys without values": # let From 0efc7f6c4bf7e25a45e091ba0704859a51254030 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 16:15:35 -0700 Subject: [PATCH 222/445] change to non-closure iterator --- datastore/sql/sqliteds.nim | 5 ---- tests/datastore/sql/testsqliteds.nim | 36 ++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 0673acc5..8e906bac 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -178,7 +178,6 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = case v of SQLITE_ROW: - echo "SQLITE ROW" let key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) @@ -195,7 +194,6 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = # error it is necessary to check that the result is a null pointer and # that the result code is an error code if blob.isSome and blob.get().isNil: - echo "BLOB: isSome" let v = sqlite3_errcode(sqlite3_db_handle(handle.env)) if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): @@ -210,14 +208,11 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = DataBuffer.new(arr.toOpenArray(0, dataLen-1)) else: DataBuffer.new("") - echo "SQLITE ROW: yield" yield success (key.some, data) of SQLITE_DONE: - echo "SQLITE DONE: yield" handle.close() break else: - echo "SQLITE ERROR: yield" handle.cancel = true yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) break diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 9dae616b..48bc734a 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -131,11 +131,7 @@ suite "queryTests": var handle = ds.query(q).tryGet - # res = handle.iter().mapIt(it.tryGet()).reversed() - - var res: seq[DbQueryResponse] - for item in handle.iter(): - res.insert(item.tryGet(), 0) + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() check: res.len == 3 @@ -148,6 +144,36 @@ suite "queryTests": res[2].key.get == key3 res[2].data == val3 + test "query should cancel": + let + q = DbQuery(key: key1, value: true) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + + var res: seq[DbQueryResponse] + var cnt = 0 + for item in handle.iter(): + cnt.inc + res.insert(item.tryGet(), 0) + if cnt > 1: + handle.cancel = true + + check: + handle.cancel == true + handle.closed == true + res.len == 2 + + res[0].key.get == key2 + res[0].data == val2 + + res[1].key.get == key3 + res[1].data == val3 + # test "Key should query all keys without values": # let # q = DbQuery(key: key1, value: false) From 85352e8bc547a286330c4dd821ab4ff9648aa573 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 16:16:00 -0700 Subject: [PATCH 223/445] change to non-closure iterator --- datastore/sql/sqliteds.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 8e906bac..ac163578 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -166,7 +166,6 @@ proc query*( proc close*(handle: var DbQueryHandle[RawStmtPtr]) = if not handle.closed: handle.closed = true - echo "sqlite backend: query: finally close" discard sqlite3_reset(handle.env) discard sqlite3_clear_bindings(handle.env) handle.env.dispose() From 0336b9336048769415f82254dff9d45235302621 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 17:35:37 -0700 Subject: [PATCH 224/445] change to non-closure iterator --- datastore/sql/sqliteds.nim | 62 ++++++++++++---------------- datastore/sql/sqlitedsdb.nim | 49 ++++++++++------------ tests/datastore/sql/testsqliteds.nim | 10 ++--- 3 files changed, 52 insertions(+), 69 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index ac163578..0494097f 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -15,21 +15,21 @@ export backend, sqlitedsdb push: {.upraises: [].} type - SQLiteBackend* = object - db: SQLiteDsDb + SQLiteBackend*[K, V] = object + db: SQLiteDsDb[K, V] -proc path*(self: SQLiteBackend): string = +proc path*[K,V](self: SQLiteBackend[K,V]): string = $self.db.dbPath -proc readOnly*(self: SQLiteBackend): bool = self.db.readOnly +proc readOnly*[K,V](self: SQLiteBackend[K,V]): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*(self: SQLiteBackend, key: DbKey): ?!bool = +proc has*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!bool = var exists = false - key = $key + key = key proc onData(s: RawStmtPtr) = exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool @@ -39,15 +39,15 @@ proc has*(self: SQLiteBackend, key: DbKey): ?!bool = return success exists -proc delete*(self: SQLiteBackend, key: DbKey): ?!void = - return self.db.deleteStmt.exec(($key)) +proc delete*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!void = + return self.db.deleteStmt.exec((key)) -proc delete*(self: SQLiteBackend, keys: openArray[DbKey]): ?!void = +proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[DbKey]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) for key in keys: - if err =? self.db.deleteStmt.exec(($key)).errorOption: + if err =? self.db.deleteStmt.exec((key)).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err.msg @@ -58,7 +58,7 @@ proc delete*(self: SQLiteBackend, keys: openArray[DbKey]): ?!void = return success() -proc get*(self: SQLiteBackend, key: DbKey): ?!seq[byte] = +proc get*[K,V](self: SQLiteBackend[K,V], key: KeyId): ?!seq[byte] = # see comment in ./filesystem_datastore re: finer control of memory # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` @@ -69,7 +69,7 @@ proc get*(self: SQLiteBackend, key: DbKey): ?!seq[byte] = proc onData(s: RawStmtPtr) = bytes = dataCol(self.db.getDataCol) - if err =? self.db.getStmt.query(($key), onData).errorOption: + if err =? self.db.getStmt.query((key), onData).errorOption: return failure(err) if bytes.len <= 0: @@ -78,25 +78,15 @@ proc get*(self: SQLiteBackend, key: DbKey): ?!seq[byte] = return success bytes -proc put*(self: SQLiteBackend, key: DbKey, data: DbVal): ?!void = - when DbVal is seq[byte]: - return self.db.putStmt.exec((key, data, timestamp())) - elif DbVal is DataBuffer: - return self.db.putBufferStmt.exec((key, data, timestamp())) - else: - {.error: "unknown type".} +proc put*[K,V](self: SQLiteBackend[K,V], key: KeyId, data: DataBuffer): ?!void = + return self.db.putStmt.exec((key, data, timestamp())) -proc put*(self: SQLiteBackend, batch: openArray[DbBatchEntry]): ?!void = +proc put*[K,V](self: SQLiteBackend[K,V], batch: openArray[DbBatchEntry]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err for entry in batch: - when entry.key is string: - let putStmt = self.db.putStmt - elif entry.key is KeyId: - let putStmt = self.db.putBufferStmt - else: - {.error: "unhandled type".} + let putStmt = self.db.putStmt if err =? putStmt.exec((entry.key, entry.data, timestamp())).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err @@ -108,13 +98,13 @@ proc put*(self: SQLiteBackend, batch: openArray[DbBatchEntry]): ?!void = return success() -proc close*(self: SQLiteBackend): ?!void = +proc close*[K,V](self: SQLiteBackend[K,V]): ?!void = self.db.close() return success() -proc query*( - self: SQLiteBackend, +proc query*[K,V]( + self: SQLiteBackend[K,V], query: DbQuery ): Result[DbQueryHandle[RawStmtPtr], ref CatchableError] = @@ -219,23 +209,23 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = handle.close() -proc contains*(self: SQLiteBackend, key: DbKey): bool = +proc contains*[K,V](self: SQLiteBackend[K,V], key: DbKey): bool = return self.has(key).get() -proc new*(T: type SQLiteBackend, +proc newSQLiteBackend*[K,V]( path: string, - readOnly = false): ?!SQLiteBackend = + readOnly = false): ?!SQLiteBackend[K,V] = let flags = if readOnly: SQLITE_OPEN_READONLY else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - success SQLiteBackend(db: ? SQLiteDsDb.open(path, flags)) + success SQLiteBackend[K,V](db: ? SQLiteDsDb[KeyId,DataBuffer].open(path, flags)) -proc new*(T: type SQLiteBackend, - db: SQLiteDsDb): ?!T = +proc newSQLiteBackend*[K,V]( + db: SQLiteDsDb[K,V]): ?!SQLiteBackend[K,V] = - success SQLiteBackend(db: db) + success SQLiteBackend[K,V](db: db) diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 902be644..95a1550c 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -17,26 +17,24 @@ type # feels odd to use `void` for prepared statements corresponding to SELECT # queries but it fits with the rest of the SQLite wrapper adapted from # status-im/nwaku, at least in its current form in ./sqlite - ContainsStmt* = SQLiteStmt[(string), void] - DeleteStmt* = SQLiteStmt[(string), void] - GetStmt* = SQLiteStmt[(string), void] - PutStmt* = SQLiteStmt[(string, seq[byte], int64), void] - PutBufferStmt* = SQLiteStmt[(KeyId, DataBuffer, int64), void] + ContainsStmt*[K] = SQLiteStmt[(K), void] + DeleteStmt*[K] = SQLiteStmt[(K), void] + GetStmt*[K] = SQLiteStmt[(K), void] + PutStmt*[K, V] = SQLiteStmt[(K, V, int64), void] QueryStmt* = SQLiteStmt[(string), void] BeginStmt* = NoParamsStmt EndStmt* = NoParamsStmt RollbackStmt* = NoParamsStmt - SQLiteDsDb* = object + SQLiteDsDb*[K,V] = object readOnly*: bool dbPath*: DataBuffer - containsStmt*: ContainsStmt - deleteStmt*: DeleteStmt + containsStmt*: ContainsStmt[K] + deleteStmt*: DeleteStmt[K] env*: SQLite getDataCol*: (RawStmtPtr, int) - getStmt*: GetStmt - putStmt*: PutStmt - putBufferStmt*: PutBufferStmt + getStmt*: GetStmt[K] + putStmt*: PutStmt[K,V] beginStmt*: BeginStmt endStmt*: EndStmt rollbackStmt*: RollbackStmt @@ -230,10 +228,10 @@ proc close*(self: SQLiteDsDb) = self.env.dispose -proc open*( - T: type SQLiteDsDb, +proc open*[K,V]( + T: type SQLiteDsDb[K,V], path = Memory, - flags = SQLITE_OPEN_READONLY): ?!SQLiteDsDb = + flags = SQLITE_OPEN_READONLY): ?!SQLiteDsDb[K, V] = # make it optional to enable WAL with it enabled being the default? @@ -266,11 +264,10 @@ proc open*( checkExec(pragmaStmt) var - containsStmt: ContainsStmt - deleteStmt: DeleteStmt - getStmt: GetStmt - putStmt: PutStmt - putBufferStmt: PutBufferStmt + containsStmt: ContainsStmt[K] + deleteStmt: DeleteStmt[K] + getStmt: GetStmt[K] + putStmt: PutStmt[K,V] beginStmt: BeginStmt endStmt: EndStmt rollbackStmt: RollbackStmt @@ -278,13 +275,10 @@ proc open*( if not readOnly: checkExec(env.val, CreateStmtStr) - deleteStmt = ? DeleteStmt.prepare( + deleteStmt = ? DeleteStmt[K].prepare( env.val, DeleteStmtStr, SQLITE_PREPARE_PERSISTENT) - putStmt = ? PutStmt.prepare( - env.val, PutStmtStr, SQLITE_PREPARE_PERSISTENT) - - putBufferStmt = ? PutBufferStmt.prepare( + putStmt = ? PutStmt[K,V].prepare( env.val, PutStmtStr, SQLITE_PREPARE_PERSISTENT) beginStmt = ? BeginStmt.prepare( @@ -296,10 +290,10 @@ proc open*( rollbackStmt = ? RollbackStmt.prepare( env.val, RollbackTransactionStr, SQLITE_PREPARE_PERSISTENT) - containsStmt = ? ContainsStmt.prepare( + containsStmt = ? ContainsStmt[K].prepare( env.val, ContainsStmtStr, SQLITE_PREPARE_PERSISTENT) - getStmt = ? GetStmt.prepare( + getStmt = ? GetStmt[K].prepare( env.val, GetStmtStr, SQLITE_PREPARE_PERSISTENT) # if a readOnly/existing database does not satisfy the expected schema @@ -309,7 +303,7 @@ proc open*( let getDataCol = (RawStmtPtr(getStmt), GetStmtDataCol) - success SQLiteDsDb( + success SQLiteDsDb[K,V]( readOnly: readOnly, dbPath: DataBuffer.new path, containsStmt: containsStmt, @@ -318,7 +312,6 @@ proc open*( getStmt: getStmt, getDataCol: getDataCol, putStmt: putStmt, - putBufferStmt: putBufferStmt, beginStmt: beginStmt, endStmt: endStmt, rollbackStmt: rollbackStmt) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 48bc734a..dd931fd2 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -60,14 +60,14 @@ proc testBasic[K, V, B]( check: not ds.has(k).tryGet test "handle missing key": - let key = Key.init("/missing/key").tryGet().id() + let key = KeyId.new Key.init("/missing/key").tryGet().id() expect(DatastoreKeyNotFound): discard ds.get(key).tryGet() # non existing key suite "Test Basic SQLiteDatastore": let - ds = SQLiteBackend.new(Memory).tryGet() + ds = newSQLiteBackend[KeyId, DataBuffer](path=Memory).tryGet() keyFull = Key.init("a:b/c/d:e").tryGet() key = keyFull.id() bytes = "some bytes".toBytes @@ -81,11 +81,11 @@ suite "Test Basic SQLiteDatastore": suiteTeardown: ds.close().tryGet() - testBasic(ds, key, bytes, otherBytes, batch) + # testBasic(ds, key, bytes, otherBytes, batch) suite "Test DataBuffer SQLiteDatastore": let - ds = SQLiteBackend.new(Memory).tryGet() + ds = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() keyFull = Key.init("a:b/c/d:e").tryGet() key = KeyId.new keyFull.id() bytes = DataBuffer.new "some bytes" @@ -103,7 +103,7 @@ suite "Test DataBuffer SQLiteDatastore": suite "queryTests": let - ds = SQLiteBackend.new(Memory).tryGet() + ds = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() var key1: KeyId From 92dcbf70ae017170e82e1d1d36cb4a83be10f309 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 17:49:17 -0700 Subject: [PATCH 225/445] change to non-closure iterator --- datastore/backend.nim | 13 ++++++++----- datastore/sql/sqliteds.nim | 18 +++++++++--------- tests/datastore/sql/testsqliteds.nim | 19 +++++++++++-------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index c2134567..80204cfb 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -20,17 +20,17 @@ type DbKey* = string | KeyId DbVal* = seq[byte] | DataBuffer - DbBatchEntry* = tuple[key: string, data: seq[byte]] | tuple[key: KeyId, data: DataBuffer] + DbBatchEntry*[K, V] = tuple[key: K, data: V] - DbQuery* = object - key*: KeyId # Key to be queried + DbQuery*[K] = object + key*: K # Key to be queried value*: bool # Flag to indicate if data should be returned limit*: int # Max items to return - not available in all backends offset*: int # Offset from which to start querying - not available in all backends sort*: SortOrder # Sort order - not available in all backends - DbQueryHandle*[T] = object - query*: DbQuery + DbQueryHandle*[K, T] = object + query*: DbQuery[K] cancel*: bool closed*: bool env*: T @@ -39,6 +39,9 @@ type proc `$`*(id: KeyId): string = $(id.data) +proc toKey*(tp: typedesc[KeyId], id: cstring): KeyId = KeyId.new(id) +proc toKey*(tp: typedesc[string], id: cstring): string = $(id) + proc new*(tp: typedesc[KeyId], id: cstring): KeyId = ## copy cstring including null terminator KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1))) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 0494097f..0f763d11 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -39,7 +39,7 @@ proc has*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!bool = return success exists -proc delete*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!void = +proc delete*[K,V](self: SQLiteBackend[K,V], key: K): ?!void = return self.db.deleteStmt.exec((key)) proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[DbKey]): ?!void = @@ -58,7 +58,7 @@ proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[DbKey]): ?!void = return success() -proc get*[K,V](self: SQLiteBackend[K,V], key: KeyId): ?!seq[byte] = +proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!seq[byte] = # see comment in ./filesystem_datastore re: finer control of memory # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` @@ -78,7 +78,7 @@ proc get*[K,V](self: SQLiteBackend[K,V], key: KeyId): ?!seq[byte] = return success bytes -proc put*[K,V](self: SQLiteBackend[K,V], key: KeyId, data: DataBuffer): ?!void = +proc put*[K,V](self: SQLiteBackend[K,V], key: K, data: V): ?!void = return self.db.putStmt.exec((key, data, timestamp())) proc put*[K,V](self: SQLiteBackend[K,V], batch: openArray[DbBatchEntry]): ?!void = @@ -106,7 +106,7 @@ proc close*[K,V](self: SQLiteBackend[K,V]): ?!void = proc query*[K,V]( self: SQLiteBackend[K,V], query: DbQuery -): Result[DbQueryHandle[RawStmtPtr], ref CatchableError] = +): Result[DbQueryHandle[K, RawStmtPtr], ref CatchableError] = var queryStr = if query.value: @@ -151,16 +151,16 @@ proc query*[K,V]( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - success DbQueryHandle[RawStmtPtr](query: query, env: s) + success DbQueryHandle[K, RawStmtPtr](query: query, env: s) -proc close*(handle: var DbQueryHandle[RawStmtPtr]) = +proc close*[K](handle: var DbQueryHandle[K, RawStmtPtr]) = if not handle.closed: handle.closed = true discard sqlite3_reset(handle.env) discard sqlite3_clear_bindings(handle.env) handle.env.dispose() -iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = +iterator iter*[K](handle: var DbQueryHandle[K, RawStmtPtr]): ?!DbQueryResponse = while not handle.cancel: let v = sqlite3_step(handle.env) @@ -168,7 +168,7 @@ iterator iter*(handle: var DbQueryHandle[RawStmtPtr]): ?!DbQueryResponse = case v of SQLITE_ROW: let - key = KeyId.new(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) + key = K.toKey(sqlite3_column_text_not_null(handle.env, QueryStmtIdCol)) blob: ?pointer = if handle.query.value: sqlite3_column_blob(handle.env, QueryStmtDataCol).some @@ -222,7 +222,7 @@ proc newSQLiteBackend*[K,V]( if readOnly: SQLITE_OPEN_READONLY else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - success SQLiteBackend[K,V](db: ? SQLiteDsDb[KeyId,DataBuffer].open(path, flags)) + success SQLiteBackend[K,V](db: ? SQLiteDsDb[K,V].open(path, flags)) proc newSQLiteBackend*[K,V]( diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index dd931fd2..c8767d51 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -14,12 +14,12 @@ import pkg/datastore/key import ../dscommontests import ../querycommontests -proc testBasic[K, V, B]( - ds: SQLiteBackend, +proc testBasic[K, V]( + ds: SQLiteBackend[K,V], key: K, bytes: V, otherBytes: V, - batch: B, + batch: seq[DbBatchEntry[K, V]], ) = test "put": @@ -60,14 +60,17 @@ proc testBasic[K, V, B]( check: not ds.has(k).tryGet test "handle missing key": - let key = KeyId.new Key.init("/missing/key").tryGet().id() + when K is KeyId: + let key = KeyId.new Key.init("/missing/key").tryGet().id() + elif K is string: + let key = $KeyId.new Key.init("/missing/key").tryGet().id() expect(DatastoreKeyNotFound): discard ds.get(key).tryGet() # non existing key suite "Test Basic SQLiteDatastore": let - ds = newSQLiteBackend[KeyId, DataBuffer](path=Memory).tryGet() + ds = newSQLiteBackend[string, seq[byte]](path=Memory).tryGet() keyFull = Key.init("a:b/c/d:e").tryGet() key = keyFull.id() bytes = "some bytes".toBytes @@ -81,7 +84,7 @@ suite "Test Basic SQLiteDatastore": suiteTeardown: ds.close().tryGet() - # testBasic(ds, key, bytes, otherBytes, batch) + testBasic(ds, key, bytes, otherBytes, batch) suite "Test DataBuffer SQLiteDatastore": let @@ -123,7 +126,7 @@ suite "queryTests": test "Key should query all keys and all it's children": let - q = DbQuery(key: key1, value: true) + q = DbQuery[KeyId](key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -146,7 +149,7 @@ suite "queryTests": test "query should cancel": let - q = DbQuery(key: key1, value: true) + q = DbQuery[KeyId](key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet From 269aec68f39e4d3da4bb226cc563f53e7eeb579d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 17:51:54 -0700 Subject: [PATCH 226/445] change to generics --- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 0f763d11..6d916d5f 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -15,7 +15,7 @@ export backend, sqlitedsdb push: {.upraises: [].} type - SQLiteBackend*[K, V] = object + SQLiteBackend*[K: DbKey, V: DbVal] = object db: SQLiteDsDb[K, V] proc path*[K,V](self: SQLiteBackend[K,V]): string = diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 95a1550c..a8d4f415 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -26,7 +26,7 @@ type EndStmt* = NoParamsStmt RollbackStmt* = NoParamsStmt - SQLiteDsDb*[K,V] = object + SQLiteDsDb*[K: DbKey, V: DbVal] = object readOnly*: bool dbPath*: DataBuffer containsStmt*: ContainsStmt[K] From 24ca5f96c9702af64e000539c6bdee4909ed5326 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 17:59:18 -0700 Subject: [PATCH 227/445] change to generics --- datastore/backend.nim | 19 +++++++++++-------- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 4 ++-- datastore/sql/sqliteutils.nim | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 80204cfb..13828333 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -42,6 +42,9 @@ proc `$`*(id: KeyId): string = $(id.data) proc toKey*(tp: typedesc[KeyId], id: cstring): KeyId = KeyId.new(id) proc toKey*(tp: typedesc[string], id: cstring): string = $(id) +template toVal*(tp: typedesc[DataBuffer], id: openArray[byte]): DataBuffer = DataBuffer.new(id) +template toVal*(tp: typedesc[seq[byte]], id: openArray[byte]): seq[byte] = @(id) + proc new*(tp: typedesc[KeyId], id: cstring): KeyId = ## copy cstring including null terminator KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1))) @@ -50,15 +53,15 @@ proc new*(tp: typedesc[KeyId], id: string): KeyId = ## copy cstring including null terminator KeyId(data: DataBuffer.new(id)) -proc toCString*(key: KeyId): cstring = - ## copy cstring including null terminator - cast[cstring](baseAddr key.data) +# proc toCString*(key: KeyId): cstring = +# ## copy cstring including null terminator +# cast[cstring](baseAddr key.data) -proc toDb*(key: Key): DbKey {.inline, raises: [].} = - let id: string = key.id() - let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat - db.setData(id) - DbKey(data: db) +# proc toDb*(key: Key): DbKey {.inline, raises: [].} = +# let id: string = key.id() +# let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat +# db.setData(id) +# DbKey(data: db) proc toKey*(key: KeyId): Key {.inline, raises: [].} = Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 6d916d5f..f02e4414 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -67,7 +67,7 @@ proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!seq[byte] = bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = dataCol(self.db.getDataCol) + bytes = dataCol[V](self.db.getDataCol) if err =? self.db.getStmt.query((key), onData).errorOption: return failure(err) diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index a8d4f415..295e43c6 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -157,7 +157,7 @@ proc idCol*( return proc (): string = $sqlite3_column_text_not_null(s, index.cint) -proc dataCol*(data: (RawStmtPtr, int)): DataBuffer = +proc dataCol*[V: DbVal](data: (RawStmtPtr, int)): V = let s = data[0] let index = data[1] @@ -188,7 +188,7 @@ proc dataCol*(data: (RawStmtPtr, int)): DataBuffer = dataBytes = cast[ptr UncheckedArray[byte]](blob) # copy data out, since sqlite will free it - DataBuffer.new(toOpenArray(dataBytes, 0, dataLen - 1)) + V.toVal(toOpenArray(dataBytes, 0, dataLen - 1)) proc timestampCol*( s: RawStmtPtr, diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 98840835..884527b7 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -72,7 +72,7 @@ proc bindParam( sqlite3_bind_text(s, n.cint, val.cstring, val.len().cint, SQLITE_TRANSIENT) elif val is KeyId: # same as previous - sqlite3_bind_text(s, n.cint, val.toCString(), val.data.len().cint, SQLITE_TRANSIENT) + sqlite3_bind_text(s, n.cint, cast[cstring](baseAddr val.data), val.data.len().cint, SQLITE_TRANSIENT) else: {.fatal: "Please add support for the '" & $typeof(val) & "' type".} From 9fe9951b9130a46d0d7998f56f07b11a54732de4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:02:34 -0700 Subject: [PATCH 228/445] change to generics --- datastore/sql/sqlitedsdb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 295e43c6..ae381e1e 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -213,7 +213,7 @@ proc getDBFilePath*(path: string): ?!string = except CatchableError as exc: return failure(exc.msg) -proc close*(self: SQLiteDsDb) = +proc close*[K, V](self: SQLiteDsDb[K, V]) = self.containsStmt.dispose self.getStmt.dispose self.beginStmt.dispose From dbfb54f026e3d28a6c99a8ab42cc9cb416cca8c1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:05:35 -0700 Subject: [PATCH 229/445] change to generics --- tests/datastore/sql/testsqliteds.nim | 37 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index c8767d51..8debb415 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -177,28 +177,29 @@ suite "queryTests": res[1].key.get == key3 res[1].data == val3 - # test "Key should query all keys without values": - # let - # q = DbQuery(key: key1, value: false) + test "Key should query all keys without values": + let + q = DbQuery[KeyId](key: key1, value: false) - # ds.put(key1, val1).tryGet - # ds.put(key2, val2).tryGet - # ds.put(key3, val3).tryGet + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) - - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data.len == 0 + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 - # res[1].key.get == key2 - # res[1].data.len == 0 + res[1].key.get == key2 + res[1].data.len == 0 - # res[2].key.get == key3 - # res[2].data.len == 0 + res[2].key.get == key3 + res[2].data.len == 0 # test "Key should not query parent": From 04a30da700da1d5caad8811873aa4154b4827001 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:12:48 -0700 Subject: [PATCH 230/445] change to generics --- datastore/backend.nim | 4 ++-- datastore/sql/sqliteds.nim | 14 +++++------ tests/datastore/sql/testsqliteds.nim | 35 +++++++++++++++------------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 13828333..54c7c626 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -29,13 +29,13 @@ type offset*: int # Offset from which to start querying - not available in all backends sort*: SortOrder # Sort order - not available in all backends - DbQueryHandle*[K, T] = object + DbQueryHandle*[K, V, T] = object query*: DbQuery[K] cancel*: bool closed*: bool env*: T - DbQueryResponse* = tuple[key: Option[KeyId], data: DataBuffer] + DbQueryResponse*[K, V] = tuple[key: Option[K], data: V] proc `$`*(id: KeyId): string = $(id.data) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index f02e4414..b3b7c4e5 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -106,7 +106,7 @@ proc close*[K,V](self: SQLiteBackend[K,V]): ?!void = proc query*[K,V]( self: SQLiteBackend[K,V], query: DbQuery -): Result[DbQueryHandle[K, RawStmtPtr], ref CatchableError] = +): Result[DbQueryHandle[K,V,RawStmtPtr], ref CatchableError] = var queryStr = if query.value: @@ -151,16 +151,16 @@ proc query*[K,V]( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - success DbQueryHandle[K, RawStmtPtr](query: query, env: s) + success DbQueryHandle[K,V,RawStmtPtr](query: query, env: s) -proc close*[K](handle: var DbQueryHandle[K, RawStmtPtr]) = +proc close*[K,V](handle: var DbQueryHandle[K,V,RawStmtPtr]) = if not handle.closed: handle.closed = true discard sqlite3_reset(handle.env) discard sqlite3_clear_bindings(handle.env) handle.env.dispose() -iterator iter*[K](handle: var DbQueryHandle[K, RawStmtPtr]): ?!DbQueryResponse = +iterator iter*[K, V](handle: var DbQueryHandle[K, V, RawStmtPtr]): ?!DbQueryResponse[K, V] = while not handle.cancel: let v = sqlite3_step(handle.env) @@ -187,14 +187,14 @@ iterator iter*[K](handle: var DbQueryHandle[K, RawStmtPtr]): ?!DbQueryResponse = if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): handle.cancel = true - yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) + yield DbQueryResponse[K,V].failure newException(DatastoreError, $sqlite3_errstr(v)) let dataLen = sqlite3_column_bytes(handle.env, QueryStmtDataCol) data = if blob.isSome: let arr = cast[ptr UncheckedArray[byte]](blob) - DataBuffer.new(arr.toOpenArray(0, dataLen-1)) + V.toVal(arr.toOpenArray(0, dataLen-1)) else: DataBuffer.new("") yield success (key.some, data) @@ -203,7 +203,7 @@ iterator iter*[K](handle: var DbQueryHandle[K, RawStmtPtr]): ?!DbQueryResponse = break else: handle.cancel = true - yield DbQueryResponse.failure newException(DatastoreError, $sqlite3_errstr(v)) + yield DbQueryResponse[K,V].failure newException(DatastoreError, $sqlite3_errstr(v)) break handle.close() diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 8debb415..7cfd12e2 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -158,7 +158,7 @@ suite "queryTests": var handle = ds.query(q).tryGet - var res: seq[DbQueryResponse] + var res: seq[DbQueryResponse[KeyId, DataBuffer]] var cnt = 0 for item in handle.iter(): cnt.inc @@ -202,25 +202,28 @@ suite "queryTests": res[2].data.len == 0 - # test "Key should not query parent": - # let - # q = DbQuery(key: key1) + test "Key should not query parent": + let + q = DbQuery[KeyId](key: key1) - # ds.put(key1, val1).tryGet - # ds.put(key2, val2).tryGet - # ds.put(key3, val3).tryGet + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) - # check: - # res.len == 2 - # res[0].key.get == key2 - # res[0].data == val2 + echo "res: ", res.mapIt($it.key) - # res[1].key.get == key3 - # res[1].data == val3 + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 + + res[1].key.get == key3 + res[1].data == val3 # test "Key should all list all keys at the same level": # let From f9e59925550d5acc1819b1e3479d63f089905258 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:17:43 -0700 Subject: [PATCH 231/445] query fixes --- datastore/sql/sqliteds.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index b3b7c4e5..2cc0f537 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -195,7 +195,9 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, RawStmtPtr]): ?!DbQueryResp if blob.isSome: let arr = cast[ptr UncheckedArray[byte]](blob) V.toVal(arr.toOpenArray(0, dataLen-1)) - else: DataBuffer.new("") + else: + var empty: array[0, byte] + V.toVal(empty.toOpenArray(0,-1)) yield success (key.some, data) of SQLITE_DONE: From 3b30cc8b8a6c7cd379523b5eb7badc88be94f294 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:18:29 -0700 Subject: [PATCH 232/445] query fixes --- datastore/backend.nim | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 54c7c626..15246004 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -46,23 +46,11 @@ template toVal*(tp: typedesc[DataBuffer], id: openArray[byte]): DataBuffer = Dat template toVal*(tp: typedesc[seq[byte]], id: openArray[byte]): seq[byte] = @(id) proc new*(tp: typedesc[KeyId], id: cstring): KeyId = - ## copy cstring including null terminator KeyId(data: DataBuffer.new(id.toOpenArray(0, id.len()-1))) proc new*(tp: typedesc[KeyId], id: string): KeyId = - ## copy cstring including null terminator KeyId(data: DataBuffer.new(id)) -# proc toCString*(key: KeyId): cstring = -# ## copy cstring including null terminator -# cast[cstring](baseAddr key.data) - -# proc toDb*(key: Key): DbKey {.inline, raises: [].} = -# let id: string = key.id() -# let db = DataBuffer.new(id.len()+1) # include room for null for cstring compat -# db.setData(id) -# DbKey(data: db) - proc toKey*(key: KeyId): Key {.inline, raises: [].} = Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") From ca108f53bdef1c22e63020fc437673e39b76fe54 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:57:02 -0700 Subject: [PATCH 233/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 7cfd12e2..4175fe12 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -204,7 +204,7 @@ suite "queryTests": test "Key should not query parent": let - q = DbQuery[KeyId](key: key1) + q = DbQuery[KeyId](key: key2, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -213,9 +213,7 @@ suite "queryTests": var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - echo "res: ", res.mapIt($it.key) + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() check: res.len == 2 From ccd5d642d9ce9d8285ada607edbdef6ba09a5dd1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 18:59:03 -0700 Subject: [PATCH 234/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 4175fe12..66c9209e 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -223,32 +223,32 @@ suite "queryTests": res[1].key.get == key3 res[1].data == val3 - # test "Key should all list all keys at the same level": - # let - # queryKey = Key.init("/a").tryGet - # q = DbQuery(key: key1) + test "Key should all list all keys at the same level": + let + queryKey = Key.init("/a").tryGet + q = DbQuery[KeyId](key: key1, value: true) - # ds.put(key1, val1).tryGet - # ds.put(key2, val2).tryGet - # ds.put(key3, val3).tryGet + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet - # var - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() - # res.sort do (a, b: DbQueryResponse) -> int: - # cmp($a.key.get, $b.key.get) + res.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: + cmp($a.key.get, $b.key.get) - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data == val1 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 - # res[1].key.get == key2 - # res[1].data == val2 + res[1].key.get == key2 + res[1].data == val2 - # res[2].key.get == key3 - # res[2].data == val3 + res[2].key.get == key3 + res[2].data == val3 # test "Should apply limit": # let From 1f23fe57d3a561f24089c33521c8cd92f5c57a5f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 19:03:59 -0700 Subject: [PATCH 235/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 66c9209e..0ea23648 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -20,6 +20,7 @@ proc testBasic[K, V]( bytes: V, otherBytes: V, batch: seq[DbBatchEntry[K, V]], + extended = true ) = test "put": @@ -250,24 +251,26 @@ suite "queryTests": res[2].key.get == key3 res[2].data == val3 - # test "Should apply limit": - # let - # key = Key.init("/a").tryGet - # q = DbQuery(key: key1, value: false) + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = DbQuery[KeyId](key: key1, limit: 10, value: false) - # for i in 0..<100: - # let - # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = DataBuffer.new("val " & $i) + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) - # ds.put(key, val).tryGet + ds.put(key, val).tryGet - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() - # check: - # res.len == 10 + echo "RES: ", res.mapIt(it.key) + check: + res.len == 10 # test "Should not apply offset": # let From 62c9e7c583476c9a79a1a6fce2cf6789939202db Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 19:14:10 -0700 Subject: [PATCH 236/445] query fixes --- datastore/backend.nim | 7 ++++++- datastore/sql/sqliteds.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 14 +++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 15246004..350f5906 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -13,6 +13,11 @@ export databuffer, threadresult, semaphore, types export upraises, results, SortOrder type + + DbSortOrder* {.pure.} = enum + Ascending, + Descending + KeyId* = object ## serialized Key ID, equivalent to `key.id()` data*: DataBuffer @@ -27,7 +32,7 @@ type value*: bool # Flag to indicate if data should be returned limit*: int # Max items to return - not available in all backends offset*: int # Offset from which to start querying - not available in all backends - sort*: SortOrder # Sort order - not available in all backends + sort*: DbSortOrder # Sort order - not available in all backends DbQueryHandle*[K, V, T] = object query*: DbQuery[K] diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 2cc0f537..3f96dd8d 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -114,7 +114,7 @@ proc query*[K,V]( else: QueryStmtIdStr - if query.sort == SortOrder.Descending: + if query.sort == DbSortOrder.Descending: queryStr &= QueryStmtOrderDescending else: queryStr &= QueryStmtOrderAscending diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 0ea23648..1d644a2f 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -135,7 +135,7 @@ suite "queryTests": var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) check: res.len == 3 @@ -175,8 +175,8 @@ suite "queryTests": res[0].key.get == key2 res[0].data == val2 - res[1].key.get == key3 - res[1].data == val3 + res[1].key.get == key1 + res[1].data == val1 test "Key should query all keys without values": let @@ -189,7 +189,7 @@ suite "queryTests": var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) check: res.len == 3 @@ -214,7 +214,7 @@ suite "queryTests": var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) check: res.len == 2 @@ -235,7 +235,7 @@ suite "queryTests": var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) res.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: cmp($a.key.get, $b.key.get) @@ -266,7 +266,7 @@ suite "queryTests": var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) echo "RES: ", res.mapIt(it.key) check: From ebed992f5a09649af4ca09835bc0535be4937242 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 19:54:22 -0700 Subject: [PATCH 237/445] query fixes --- datastore/backend.nim | 51 +++++++++++++++------------- datastore/query.nim | 36 ++++++++------------ datastore/sql/sqliteds.nim | 11 +++--- tests/datastore/sql/testsqliteds.nim | 49 +++++++++++++------------- 4 files changed, 75 insertions(+), 72 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 350f5906..de32a431 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -1,22 +1,23 @@ -import pkg/questionable/results -import pkg/upraises - import std/algorithm import std/options + +import pkg/questionable/results + import ./threads/databuffer -import ./threads/threadresult -import ./threads/semaphore -import ./key -import ./types -export databuffer, threadresult, semaphore, types -export upraises, results, SortOrder +export databuffer +export SortOrder type - DbSortOrder* {.pure.} = enum - Ascending, - Descending + DbQueryResponse*[K, V] = tuple[key: Option[K], data: V] + + DbQuery*[K] = object + key*: K # Key to be queried + value*: bool # Flag to indicate if data should be returned + limit*: int # Max items to return - not available in all backends + offset*: int # Offset from which to start querying - not available in all backends + sort*: SortOrder # Sort order - not available in all backends KeyId* = object ## serialized Key ID, equivalent to `key.id()` @@ -27,20 +28,27 @@ type DbBatchEntry*[K, V] = tuple[key: K, data: V] - DbQuery*[K] = object - key*: K # Key to be queried - value*: bool # Flag to indicate if data should be returned - limit*: int # Max items to return - not available in all backends - offset*: int # Offset from which to start querying - not available in all backends - sort*: DbSortOrder # Sort order - not available in all backends - DbQueryHandle*[K, V, T] = object query*: DbQuery[K] cancel*: bool closed*: bool env*: T - DbQueryResponse*[K, V] = tuple[key: Option[K], data: V] +proc dbQuery*[K]( + key: K, + value = true, + sort = SortOrder.Ascending, + offset = 0, + limit = -1 +): DbQuery[K] = + + DbQuery[K]( + key: key, + value: value, + sort: sort, + offset: offset, + limit: limit) + proc `$`*(id: KeyId): string = $(id.data) @@ -56,8 +64,5 @@ proc new*(tp: typedesc[KeyId], id: cstring): KeyId = proc new*(tp: typedesc[KeyId], id: string): KeyId = KeyId(data: DataBuffer.new(id)) -proc toKey*(key: KeyId): Key {.inline, raises: [].} = - Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") - template toOpenArray*(x: DbKey): openArray[char] = x.data.toOpenArray(char) diff --git a/datastore/query.nim b/datastore/query.nim index 7cae8974..4f2ce602 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -6,19 +6,17 @@ import pkg/questionable/results import ./key import ./types +import ./backend export types export options, SortOrder type - Query* = object - key*: Key # Key to be queried - value*: bool # Flag to indicate if data should be returned - limit*: int # Max items to return - not available in all backends - offset*: int # Offset from which to start querying - not available in all backends - sort*: SortOrder # Sort order - not available in all backends - QueryResponse* = tuple[key: ?Key, data: seq[byte]] + ## Front end types + Query* = DbQuery[Key] + + QueryResponse* = DbQueryResponse[Key, seq[byte]] GetNext* = proc(): Future[?!QueryResponse] {.upraises: [], gcsafe.} IterDispose* = proc(): Future[?!void] {.upraises: [], gcsafe.} @@ -37,17 +35,13 @@ proc defaultDispose(): Future[?!void] {.upraises: [], gcsafe, async.} = proc new*(T: type QueryIter, dispose = defaultDispose): T = QueryIter(dispose: dispose) -proc init*( - T: type Query, - key: Key, - value = true, - sort = SortOrder.Ascending, - offset = 0, - limit = -1): T = - - T( - key: key, - value: value, - sort: sort, - offset: offset, - limit: limit) +proc init*(T: type Query, + key: Key, + value = true, + sort = SortOrder.Ascending, + offset = 0, + limit = -1): Query = + dbQuery[Key](key, value, sort, offset, limit) + +proc toKey*(key: KeyId): Key {.inline, raises: [].} = + Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 3f96dd8d..6a45502c 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -105,7 +105,7 @@ proc close*[K,V](self: SQLiteBackend[K,V]): ?!void = proc query*[K,V]( self: SQLiteBackend[K,V], - query: DbQuery + query: DbQuery[K] ): Result[DbQueryHandle[K,V,RawStmtPtr], ref CatchableError] = var @@ -114,9 +114,10 @@ proc query*[K,V]( else: QueryStmtIdStr - if query.sort == DbSortOrder.Descending: + case query.sort: + of Descending: queryStr &= QueryStmtOrderDescending - else: + of Ascending: queryStr &= QueryStmtOrderAscending if query.limit != 0: @@ -125,9 +126,9 @@ proc query*[K,V]( if query.offset != 0: queryStr &= QueryStmtOffset + echo "QUERY_STR: ", queryStr let - queryStmt = QueryStmt.prepare( - self.db.env, queryStr).expect("should not fail") + queryStmt = ? QueryStmt.prepare(self.db.env, queryStr) s = RawStmtPtr(queryStmt) queryKey = $query.key & "*" diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 1d644a2f..86007b9c 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -12,7 +12,6 @@ import pkg/datastore/sql/sqliteds import pkg/datastore/key import ../dscommontests -import ../querycommontests proc testBasic[K, V]( ds: SQLiteBackend[K,V], @@ -127,7 +126,7 @@ suite "queryTests": test "Key should query all keys and all it's children": let - q = DbQuery[KeyId](key: key1, value: true) + q = dbQuery(key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -150,7 +149,7 @@ suite "queryTests": test "query should cancel": let - q = DbQuery[KeyId](key: key1, value: true) + q = dbQuery(key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -180,7 +179,7 @@ suite "queryTests": test "Key should query all keys without values": let - q = DbQuery[KeyId](key: key1, value: false) + q = dbQuery(key: key1, value: false) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -205,7 +204,7 @@ suite "queryTests": test "Key should not query parent": let - q = DbQuery[KeyId](key: key2, value: true) + q = dbQuery(key: key2, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -227,7 +226,7 @@ suite "queryTests": test "Key should all list all keys at the same level": let queryKey = Key.init("/a").tryGet - q = DbQuery[KeyId](key: key1, value: true) + q = dbQuery(key: key1, value: true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -254,7 +253,7 @@ suite "queryTests": test "Should apply limit": let key = Key.init("/a").tryGet - q = DbQuery[KeyId](key: key1, limit: 10, value: false) + q = dbQuery(key: key1, limit: 10, value: false) for i in 0..<100: let @@ -268,29 +267,33 @@ suite "queryTests": let res = handle.iter().toSeq().mapIt(it.tryGet()) - echo "RES: ", res.mapIt(it.key) check: res.len == 10 - # test "Should not apply offset": - # let - # key = Key.init("/a").tryGet - # keyId = KeyId.new $key - # q = DbQuery(key: KeyId.new $key, offset: 90) + test "Should not apply offset": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key: keyId, offset: 90) - # for i in 0..<100: - # let - # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = DataBuffer.new("val " & $i) + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) - # ds.put(keyId, val).tryGet + ds.put(key, val).tryGet - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) + var + qr = ds.query(q) + echo "RES: ", qr.repr - # check: - # res.len == 10 + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 10 # test "Should not apply offset and limit": From 77173a5fac249f0bef930defd72fe6ec5858fd16 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 19:55:12 -0700 Subject: [PATCH 238/445] query fixes --- datastore/backend.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index de32a431..1fa955a9 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -4,9 +4,9 @@ import std/options import pkg/questionable/results import ./threads/databuffer +import ./types -export databuffer -export SortOrder +export databuffer, types, SortOrder type From a16cdaab29474bb0ba12e18e43d651f06896141f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 19:58:03 -0700 Subject: [PATCH 239/445] query fixes --- datastore/backend.nim | 2 +- datastore/query.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 1fa955a9..fe2969ac 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -36,7 +36,7 @@ type proc dbQuery*[K]( key: K, - value = true, + value = false, sort = SortOrder.Ascending, offset = 0, limit = -1 diff --git a/datastore/query.nim b/datastore/query.nim index 4f2ce602..1295ff99 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -37,7 +37,7 @@ proc new*(T: type QueryIter, dispose = defaultDispose): T = proc init*(T: type Query, key: Key, - value = true, + value = false, sort = SortOrder.Ascending, offset = 0, limit = -1): Query = diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 86007b9c..3d62da3f 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -126,7 +126,7 @@ suite "queryTests": test "Key should query all keys and all it's children": let - q = dbQuery(key: key1, value: true) + q = dbQuery(key=key1, value=true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -149,7 +149,7 @@ suite "queryTests": test "query should cancel": let - q = dbQuery(key: key1, value: true) + q = dbQuery(key= key1, value= true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -179,7 +179,7 @@ suite "queryTests": test "Key should query all keys without values": let - q = dbQuery(key: key1, value: false) + q = dbQuery(key= key1, value= false) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -204,7 +204,7 @@ suite "queryTests": test "Key should not query parent": let - q = dbQuery(key: key2, value: true) + q = dbQuery(key= key2, value= true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -226,7 +226,7 @@ suite "queryTests": test "Key should all list all keys at the same level": let queryKey = Key.init("/a").tryGet - q = dbQuery(key: key1, value: true) + q = dbQuery(key= key1, value= true) ds.put(key1, val1).tryGet ds.put(key2, val2).tryGet @@ -253,7 +253,7 @@ suite "queryTests": test "Should apply limit": let key = Key.init("/a").tryGet - q = dbQuery(key: key1, limit: 10, value: false) + q = dbQuery(key= key1, limit= 10, value= false) for i in 0..<100: let @@ -274,7 +274,7 @@ suite "queryTests": let key = Key.init("/a").tryGet keyId = KeyId.new $key - q = dbQuery(key: keyId, offset: 90) + q = dbQuery(key= keyId, offset= 90) for i in 0..<100: let From 2e2da52b300b809176deb238eae4a3d5d612b0da Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:02:01 -0700 Subject: [PATCH 240/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 3d62da3f..cdd6cd83 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -105,24 +105,16 @@ suite "Test DataBuffer SQLiteDatastore": testBasic(ds, key, bytes, otherBytes, batch) suite "queryTests": - let - ds = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - - var - key1: KeyId - key2: KeyId - key3: KeyId - val1: DataBuffer - val2: DataBuffer - val3: DataBuffer setup: - key1 = KeyId.new "/a" - key2 = KeyId.new "/a/b" - key3 = KeyId.new "/a/b/c" - val1 = DataBuffer.new "value for 1" - val2 = DataBuffer.new "value for 2" - val3 = DataBuffer.new "value for 3" + let + ds = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + key1 = KeyId.new "/a" + key2 = KeyId.new "/a/b" + key3 = KeyId.new "/a/b/c" + val1 = DataBuffer.new "value for 1" + val2 = DataBuffer.new "value for 2" + val3 = DataBuffer.new "value for 3" test "Key should query all keys and all it's children": let @@ -292,10 +284,10 @@ suite "queryTests": let res = handle.iter().toSeq().mapIt(it.tryGet()) + echo "RES: ", res.mapIt(it.key) check: res.len == 10 - # test "Should not apply offset and limit": # let # key = Key.init("/a").tryGet From 206751a09069ada6e8f27cf28fbf3740ba6ed9e9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:03:10 -0700 Subject: [PATCH 241/445] query fixes --- datastore/backend.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index fe2969ac..1566afa9 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -49,7 +49,6 @@ proc dbQuery*[K]( offset: offset, limit: limit) - proc `$`*(id: KeyId): string = $(id.data) proc toKey*(tp: typedesc[KeyId], id: cstring): KeyId = KeyId.new(id) From 2de2650fb599141bf7a656c3c82e31897be8052f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:19:33 -0700 Subject: [PATCH 242/445] query fixes --- datastore/sql/sqliteds.nim | 1 - tests/datastore/sql/testsqliteds.nim | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 6a45502c..17600878 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -126,7 +126,6 @@ proc query*[K,V]( if query.offset != 0: queryStr &= QueryStmtOffset - echo "QUERY_STR: ", queryStr let queryStmt = ? QueryStmt.prepare(self.db.env, queryStr) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index cdd6cd83..2448d7dd 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -277,14 +277,14 @@ suite "queryTests": var qr = ds.query(q) - echo "RES: ", qr.repr + # echo "RES: ", qr.repr var handle = ds.query(q).tryGet let res = handle.iter().toSeq().mapIt(it.tryGet()) - echo "RES: ", res.mapIt(it.key) + # echo "RES: ", res.mapIt(it.key) check: res.len == 10 From d3eb55fc7528b8d38a20605ae163fcfcab64f5a9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:20:37 -0700 Subject: [PATCH 243/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 44 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 2448d7dd..c48a6953 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -288,34 +288,34 @@ suite "queryTests": check: res.len == 10 - # test "Should not apply offset and limit": - # let - # key = Key.init("/a").tryGet - # keyId = KeyId.new $key - # q = DbQuery(key: keyId, offset: 95, limit: 5) + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key= keyId, offset= 95, limit= 5) - # for i in 0..<100: - # let - # key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = DataBuffer.new("val " & $i) + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) - # ds.put(key, val).tryGet + ds.put(key, val).tryGet - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() - # check: - # res.len == 5 + check: + res.len == 5 - # for i in 0.. Date: Mon, 25 Sep 2023 20:24:25 -0700 Subject: [PATCH 244/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index c48a6953..29aec12a 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -303,7 +303,7 @@ suite "queryTests": var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()).reversed() + res = handle.iter().toSeq().mapIt(it.tryGet()) check: res.len == 5 @@ -315,7 +315,7 @@ suite "queryTests": check: res[i].key.get == key - res[i].data == val + # res[i].data == val # test "Should apply sort order - descending": From a637e6693527426ae742e3b283c2f73b51a1ad88 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:26:26 -0700 Subject: [PATCH 245/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 61 ++++++++++++++-------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 29aec12a..0737dd94 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -317,35 +317,34 @@ suite "queryTests": res[i].key.get == key # res[i].data == val + test "Should apply sort order - descending": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key= keyId, sort= SortOrder.Descending) + + var kvs: seq[DbQueryResponse[KeyId, DataBuffer]] + for i in 0..<100: + let + k = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new ("val " & $i) + + kvs.add((k.some, val)) + ds.put(k, val).tryGet + + # lexicographic sort, as it comes from the backend + kvs.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: + cmp($a.key.get, $b.key.get) + + kvs = kvs.reversed + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 100 - # test "Should apply sort order - descending": - # let - # key = Key.init("/a").tryGet - # keyId = KeyId.new $key - # q = DbQuery(key: keyId, sort: SortOrder.Descending) - - # var kvs: seq[DbQueryResponse] - # for i in 0..<100: - # let - # k = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = DataBuffer.new ("val " & $i) - - # kvs.add((k.some, val)) - # ds.put(k, val).tryGet - - # # lexicographic sort, as it comes from the backend - # kvs.sort do (a, b: DbQueryResponse) -> int: - # cmp($a.key.get, $b.key.get) - - # kvs = kvs.reversed - # let - # (handle, iter) = ds.query(q).tryGet - # res = iter.mapIt(it.tryGet()) - - # check: - # res.len == 100 - - # for i, r in res[1..^1]: - # check: - # res[i].key.get == kvs[i].key.get - # res[i].data == kvs[i].data + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data From 84cdc1d8caef27bb1e77004ec157743399a34d63 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:27:44 -0700 Subject: [PATCH 246/445] query fixes --- tests/datastore/sql/testsqliteds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 0737dd94..1309dcd3 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -321,7 +321,7 @@ suite "queryTests": let key = Key.init("/a").tryGet keyId = KeyId.new $key - q = dbQuery(key= keyId, sort= SortOrder.Descending) + q = dbQuery(key= keyId, value=true, sort= SortOrder.Descending) var kvs: seq[DbQueryResponse[KeyId, DataBuffer]] for i in 0..<100: From 0c72ad8478836703f8500c465579d824bc5d5963 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:37:34 -0700 Subject: [PATCH 247/445] query fixes --- datastore/sql.nim | 21 ++++++++++++++------- datastore/sql/sqliteds.nim | 5 +++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 3c323687..c75ad5c3 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -19,7 +19,7 @@ push: {.upraises: [].} type SQLiteDatastore* = ref object of Datastore - db: SQLiteBackend + db: SQLiteBackend[KeyId, DataBuffer] proc path*(self: SQLiteDatastore): string = self.db.path() @@ -27,23 +27,30 @@ proc path*(self: SQLiteDatastore): string = proc readOnly*(self: SQLiteDatastore): bool = self.db.readOnly() -method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = +method has*(self: SQLiteDatastore, + key: Key): Future[?!bool] {.async.} = return self.db.has(KeyId.new key.id()) -method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = +method delete*(self: SQLiteDatastore, + key: Key): Future[?!void] {.async.} = return self.db.delete(KeyId.new key.id()) -method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = +method delete*(self: SQLiteDatastore, + keys: seq[Key]): Future[?!void] {.async.} = let dkeys = keys.mapIt(KeyId.new it.id()) return self.db.delete(dkeys) -method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = +method get*(self: SQLiteDatastore, + key: Key): Future[?!seq[byte]] {.async.} = self.db.get(KeyId.new key.id()) -method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = +method put*(self: SQLiteDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = self.db.put(KeyId.new key.id(), DataBuffer.new data) -method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = +method put*(self: SQLiteDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = var dbatch: seq[tuple[key: string, data: seq[byte]]] for entry in batch: dbatch.add((entry.key.id(), entry.data)) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 17600878..1ce8d559 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -81,13 +81,14 @@ proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!seq[byte] = proc put*[K,V](self: SQLiteBackend[K,V], key: K, data: V): ?!void = return self.db.putStmt.exec((key, data, timestamp())) -proc put*[K,V](self: SQLiteBackend[K,V], batch: openArray[DbBatchEntry]): ?!void = +proc put*[K,V](self: SQLiteBackend[K,V], batch: openArray[DbBatchEntry[K,V]]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure err for entry in batch: let putStmt = self.db.putStmt - if err =? putStmt.exec((entry.key, entry.data, timestamp())).errorOption: + let item = (entry.key, entry.data, timestamp()) + if err =? putStmt.exec(item).errorOption: if err =? self.db.rollbackStmt.exec().errorOption: return failure err From 73ec70fa9548725cad52158c974f5116ec1d0083 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 20:58:26 -0700 Subject: [PATCH 248/445] query fixes --- datastore/sql.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index c75ad5c3..de1f08ef 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -51,9 +51,9 @@ method put*(self: SQLiteDatastore, method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = - var dbatch: seq[tuple[key: string, data: seq[byte]]] + var dbatch: seq[tuple[key: KeyId, data: DataBuffer]] for entry in batch: - dbatch.add((entry.key.id(), entry.data)) + dbatch.add((KeyId.new entry.key.id(), DataBuffer.new entry.data)) self.db.put(dbatch) method close*(self: SQLiteDatastore): Future[?!void] {.async.} = @@ -65,12 +65,12 @@ method query*( ): Future[?!QueryIter] {.async.} = var iter = QueryIter() - let dbquery = DbQuery( - key: KeyId.new query.key.id(), - value: query.value, - limit: query.limit, - offset: query.offset, - sort: query.sort, + let dbquery = dbQuery( + key= KeyId.new query.key.id(), + value= query.value, + limit= query.limit, + offset= query.offset, + sort= query.sort, ) var queries = ? self.db.query(dbquery) From 2861eba890af83e418b43ce45af6d54ddc5ebd2c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:05:07 -0700 Subject: [PATCH 249/445] simple iterator --- datastore/sql.nim | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index de1f08ef..d9619993 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -62,9 +62,8 @@ method close*(self: SQLiteDatastore): Future[?!void] {.async.} = method query*( self: SQLiteDatastore, query: Query -): Future[?!QueryIter] {.async.} = +): ?!(iterator(): ?!QueryResponse) = - var iter = QueryIter() let dbquery = dbQuery( key= KeyId.new query.key.id(), value= query.value, @@ -72,35 +71,14 @@ method query*( offset= query.offset, sort= query.sort, ) - var queries = ? self.db.query(dbquery) - - let lock = newAsyncLock() - proc next(): Future[?!QueryResponse] {.async.} = - defer: - if lock.locked: - lock.release() - - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") - - if iter.finished: - return failure((ref QueryEndedError)(msg: "Calling next on a finished query!")) - - await lock.acquire() - - without res =? queries(), err: - iter.finished = true - return failure err - - - iter.dispose = proc(): Future[?!void] {.async.} = - discard sqlite3_reset(s) - discard sqlite3_clear_bindings(s) - s.dispose - return success() - - iter.next = next - return success iter + var qhandle = ? self.db.query(dbquery) + + let iter = iterator(): ?!QueryResponse = + for resp in qhandle.iter(): + without qres =? resp, err: + yield QueryResponse.failure err + + success iter proc new*( T: type SQLiteDatastore, From 7710dd7c262fc7230464cdbf914972285ceb039b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:07:42 -0700 Subject: [PATCH 250/445] updates --- datastore/datastore.nim | 11 ++++++++--- datastore/sql.nim | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 98b95208..c0f1f0d1 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -34,11 +34,16 @@ method put*(self: Datastore, batch: seq[BatchEntry]): Future[?!void] {.base, loc method close*(self: Datastore): Future[?!void] {.base, locks: "unknown", raises: [].} = raiseAssert("Not implemented!") -method query*( - self: Datastore, - query: Query): Future[?!QueryIter] {.base, gcsafe, raises: [].} = +method query*(self: Datastore, + query: Query + ): Future[?!QueryIter] {.base, gcsafe, raises: [].} = raiseAssert("Not implemented!") +method queryIter*(self: Datastore, + query: Query + ): ?!(iterator(): ?!QueryResponse) {.base, gcsafe, raises: [].} = + raiseAssert("Not implemented!") + proc contains*(self: Datastore, key: Key): Future[bool] {.async, raises: [].} = return (await self.has(key)) |? false diff --git a/datastore/sql.nim b/datastore/sql.nim index d9619993..fd22ed6d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -59,7 +59,7 @@ method put*(self: SQLiteDatastore, method close*(self: SQLiteDatastore): Future[?!void] {.async.} = self.db.close() -method query*( +method queryIter*( self: SQLiteDatastore, query: Query ): ?!(iterator(): ?!QueryResponse) = From be9426694bfcade23346eff5ade1787195d93578 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:12:06 -0700 Subject: [PATCH 251/445] updates --- datastore/sql.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datastore/sql.nim b/datastore/sql.nim index fd22ed6d..4550858b 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -77,6 +77,9 @@ method queryIter*( for resp in qhandle.iter(): without qres =? resp, err: yield QueryResponse.failure err + let k = qres.key.map(proc(k: KeyId): Key = Key.init($k).expect("valid key")) + let v: seq[byte] = qres.data + yield success (k, v) success iter From c25eb3a207d5cd4b3597111cb451bf8b033e7650 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:13:23 -0700 Subject: [PATCH 252/445] updates --- datastore/sql.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 4550858b..5b68b53d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -77,7 +77,8 @@ method queryIter*( for resp in qhandle.iter(): without qres =? resp, err: yield QueryResponse.failure err - let k = qres.key.map(proc(k: KeyId): Key = Key.init($k).expect("valid key")) + let k = qres.key.map() do(k: KeyId) -> Key: + Key.init($k).expect("valid key") let v: seq[byte] = qres.data yield success (k, v) From 46a29128de0b2cfd18ad4b29686149d88a82287e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:16:43 -0700 Subject: [PATCH 253/445] updates --- datastore/threads/threadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 2a920562..da586091 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -428,8 +428,8 @@ method query*( iter.finished = childIter.finished var - res = ThreadResult[DbQueryResponse]() - ctx = TaskCtx[DbQueryResponse]( + res = ThreadResult[QueryResponse]() + ctx = TaskCtx[QueryResponse]( ds: self.ds, res: addr res) From 1a6065b89d7789c39548b6128e8608d96f1e7047 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:27:35 -0700 Subject: [PATCH 254/445] updates --- datastore/backend.nim | 9 +------ datastore/threads/threadproxyds.nim | 39 ++++++++++++----------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 1566afa9..f6af9838 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -9,7 +9,6 @@ import ./types export databuffer, types, SortOrder type - DbQueryResponse*[K, V] = tuple[key: Option[K], data: V] DbQuery*[K] = object @@ -41,13 +40,7 @@ proc dbQuery*[K]( offset = 0, limit = -1 ): DbQuery[K] = - - DbQuery[K]( - key: key, - value: value, - sort: sort, - offset: offset, - limit: limit) + DbQuery[K](key: key, value: value, sort: sort, offset: offset, limit: limit) proc `$`*(id: KeyId): string = $(id.data) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index da586091..44f1e2d3 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -24,6 +24,7 @@ import ../key import ../query import ../datastore import ../backend +import ../sql/sqliteds import ./asyncsemaphore import ./databuffer @@ -35,8 +36,18 @@ logScope: topics = "datastore threadproxyds" type - TaskCtx[T: ThreadTypes] = object - ds: Datastore + ThreadBackendKinds* = enum + Sqlite + # Filesystem + + ThreadBackend* = object + case kind*: ThreadBackendKinds + of Sqlite: + sql*: SQLiteBackend[KeyId,DataBuffer] + + + TaskCtx[D; T: ThreadTypes] = object + ds: D res: ptr ThreadResult[T] cancelled: bool semaphore: AsyncSemaphore @@ -77,9 +88,9 @@ template withLocks( self.queryLock.release() # TODO: needs rework, we can't use `result` with async -template dispatchTask[T]( +template dispatchTask[D, T]( self: ThreadDatastore, - ctx: TaskCtx[T], + ctx: TaskCtx[D, T], key: ?Key = Key.none, runTask: proc): auto = try: @@ -135,31 +146,13 @@ proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = ctx[].res[].err(exc) discard ctx[].signal.fireSync() -proc asyncHasTask( - ctx: ptr TaskCtx[bool], - key: ptr Key) {.async.} = - if ctx.isNil: - trace "ctx is nil" - return - - let - key = key[] - fut = ctx[].ds.has(key) - - asyncSpawn signalMonitor(ctx, fut) - without ret =? (await fut).catch and res =? ret, error: - ctx[].res[].err(error) - return - - ctx[].res[].ok(res) - proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = defer: if not ctx.isNil: discard ctx[].signal.fireSync() try: - waitFor asyncHasTask(ctx, key) + has(ctx.ds, key) except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg From 37dbd1c2342ff2c711c415d7d6f3cc3b7d7f07ee Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 21:44:26 -0700 Subject: [PATCH 255/445] simplifying --- datastore/threads/threadproxyds.nim | 566 ++++++++++++++-------------- 1 file changed, 283 insertions(+), 283 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 44f1e2d3..5e11472e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -48,32 +48,30 @@ type TaskCtx[D; T: ThreadTypes] = object ds: D - res: ptr ThreadResult[T] + res: ThreadResult[T] cancelled: bool semaphore: AsyncSemaphore signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore tp: Taskpool - ds: Datastore + backend: ThreadBackend semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors withLocks: bool - tasks: Table[Key, Future[void]] + tasks: Table[KeyId, Future[void]] queryLock: AsyncLock # global query lock, this is only really \ # needed for the fsds, but it is expensive! template withLocks( self: ThreadDatastore, ctx: TaskCtx, - key: ?Key = Key.none, - fut: Future[void], + key: ?KeyId = KeyId.none, body: untyped): untyped = try: if key.isSome and key.get in self.tasks: if self.withLocks: await self.tasks[key.get] - self.tasks[key.get] = fut # we alway want to store the future, but only await if we're using locks if self.withLocks: await self.queryLock.acquire() # only lock if it's required (fsds) @@ -91,7 +89,7 @@ template withLocks( template dispatchTask[D, T]( self: ThreadDatastore, ctx: TaskCtx[D, T], - key: ?Key = Key.none, + key: ?KeyId = KeyId.none, runTask: proc): auto = try: await self.semaphore.acquire() @@ -103,14 +101,14 @@ template dispatchTask[D, T]( let fut = wait(ctx.signal) - withLocks(self, ctx, key, fut): + withLocks(self, ctx, key): runTask() await fut - if ctx.res[].isErr: - failure ctx.res[].error + if ctx.res.isErr: + failure ctx.res.error else: when result.T isnot void: - success result.T(ctx.res[].get) + success result.T(ctx.res.get) else: success() except CancelledError as exc: @@ -146,293 +144,295 @@ proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = ctx[].res[].err(exc) discard ctx[].signal.fireSync() -proc hasTask(ctx: ptr TaskCtx, key: ptr Key) = +proc hasTask(ctx: ptr TaskCtx, key: KeyId) = defer: if not ctx.isNil: discard ctx[].signal.fireSync() try: - has(ctx.ds, key) + let res = has(ctx.ds, key) except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var - key = key - res = ThreadResult[bool]() - ctx = TaskCtx[bool]( - ds: self.ds, - res: addr res) - - proc runTask() = - self.tp.spawn hasTask(addr ctx, addr key) - - return self.dispatchTask(ctx, key.some, runTask) - -proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = - if ctx.isNil: - trace "ctx is nil" - return - - let - key = key[] - fut = ctx[].ds.delete(key) - - asyncSpawn signalMonitor(ctx, fut) - without res =? (await fut).catch, error: - trace "Error in asyncDelTask", error = error.msg - ctx[].res[].err(error) - return - - ctx[].res[].ok() - return - -proc delTask(ctx: ptr TaskCtx, key: ptr Key) = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - - try: - waitFor asyncDelTask(ctx, key) - except CatchableError as exc: - trace "Unexpected exception thrown in asyncDelTask", exc = exc.msg - raiseAssert exc.msg - -method delete*( - self: ThreadDatastore, - key: Key): Future[?!void] {.async.} = - var - key = key - res = ThreadResult[void]() - ctx = TaskCtx[void]( - ds: self.ds, - res: addr res) - - proc runTask() = - self.tp.spawn delTask(addr ctx, addr key) - - return self.dispatchTask(ctx, key.some, runTask) - -method delete*( - self: ThreadDatastore, - keys: seq[Key]): Future[?!void] {.async.} = - - for key in keys: - if err =? (await self.delete(key)).errorOption: - return failure err - - return success() - -proc asyncPutTask( - ctx: ptr TaskCtx[void], - key: ptr Key, - data: ptr UncheckedArray[byte], - len: int) {.async.} = - - if ctx.isNil: - trace "ctx is nil" - return - - let - key = key[] - data = @(data.toOpenArray(0, len - 1)) - fut = ctx[].ds.put(key, data) - - asyncSpawn signalMonitor(ctx, fut) - without res =? (await fut).catch, error: - trace "Error in asyncPutTask", error = error.msg - ctx[].res[].err(error) - return - - ctx[].res[].ok() - -proc putTask( - ctx: ptr TaskCtx, - key: ptr Key, - data: ptr UncheckedArray[byte], - len: int) = - ## run put in a thread task - ## - - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - - try: - waitFor asyncPutTask(ctx, key, data, len) - except CatchableError as exc: - trace "Unexpected exception thrown in asyncPutTask", exc = exc.msg - raiseAssert exc.msg - -method put*( - self: ThreadDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async.} = - var - key = key - data = data - res = ThreadResult[void]() - ctx = TaskCtx[void]( - ds: self.ds, - res: addr res) - - proc runTask() = - self.tp.spawn putTask( - addr ctx, - addr key, - makeUncheckedArray(addr data[0]), - data.len) - - return self.dispatchTask(ctx, key.some, runTask) - -method put*( - self: ThreadDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = - - for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: - return failure err - - return success() - -proc asyncGetTask( - ctx: ptr TaskCtx[DataBuffer], - key: ptr Key) {.async.} = - if ctx.isNil: - trace "ctx is nil" - return - - let - key = key[] - fut = ctx[].ds.get(key) - - asyncSpawn signalMonitor(ctx, fut) - without res =? (await fut).catch and data =? res, error: - trace "Error in asyncGetTask", error = error.msg - ctx[].res[].err(error) - return - - trace "Got data in get" - ctx[].res[].ok(DataBuffer.new(data)) - -proc getTask( - ctx: ptr TaskCtx, - key: ptr Key) = - ## Run get in a thread task - ## - - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - - try: - waitFor asyncGetTask(ctx, key) - except CatchableError as exc: - trace "Unexpected exception thrown in asyncGetTask", exc = exc.msg - raiseAssert exc.msg - -method get*( - self: ThreadDatastore, - key: Key): Future[?!seq[byte]] {.async.} = - var - key = key - res = ThreadResult[DataBuffer]() - ctx = TaskCtx[DataBuffer]( - ds: self.ds, - res: addr res) - - proc runTask() = - self.tp.spawn getTask(addr ctx, addr key) - - return self.dispatchTask(ctx, key.some, runTask) - -method close*(self: ThreadDatastore): Future[?!void] {.async.} = - for fut in self.tasks.values.toSeq: - await fut.cancelAndWait() # probably want to store the signal, instead of the future (or both?) - - await self.ds.close() - -proc asyncQueryTask( - ctx: ptr TaskCtx, - iter: ptr QueryIter) {.async.} = - - if ctx.isNil or iter.isNil: - trace "ctx is nil" - return - - let - fut = iter[].next() - - asyncSpawn signalMonitor(ctx, fut) - without ret =? (await fut).catch and res =? ret, error: - trace "Error in asyncQueryTask", error = error.msg - ctx[].res[].err(error) - return - - if res.key.isNone: - ctx[].res[].ok((default(DataBuffer), default(DataBuffer))) - return - - var - keyBuf = DataBuffer.new($(res.key.get())) - dataBuf = DataBuffer.new(res.data) - - trace "Got query result", key = $res.key.get(), data = res.data - ctx[].res[].ok((keyBuf, dataBuf)) - -proc queryTask( - ctx: ptr TaskCtx, - iter: ptr QueryIter) = - - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - - try: - waitFor asyncQueryTask(ctx, iter) - except CatchableError as exc: - trace "Unexpected exception thrown in asyncQueryTask", exc = exc.msg - raiseAssert exc.msg - -method query*( - self: ThreadDatastore, - query: Query): Future[?!QueryIter] {.async.} = - without var childIter =? await self.ds.query(query), error: - return failure error - - var - iter = QueryIter.new() - lock = newAsyncLock() # serialize querying under threads - - proc next(): Future[?!QueryResponse] {.async.} = - defer: - if lock.locked: - lock.release() - - trace "About to query" - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") - - await lock.acquire() - - if iter.finished == true: - return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - - iter.finished = childIter.finished + key = KeyId.new key.id() + + case self.backend.kind: + of Sqlite: var - res = ThreadResult[QueryResponse]() - ctx = TaskCtx[QueryResponse]( - ds: self.ds, - res: addr res) + ds = self.backend.sql + ctx = TaskCtx[typeof(ds), bool](ds: ds) proc runTask() = - self.tp.spawn queryTask(addr ctx, addr childIter) - - return self.dispatchTask(ctx, Key.none, runTask) + self.tp.spawn hasTask(addr ctx, key) + + return self.dispatchTask(ctx, key.some, runTask) + +# proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = +# if ctx.isNil: +# trace "ctx is nil" +# return + +# let +# key = key[] +# fut = ctx[].ds.delete(key) + +# asyncSpawn signalMonitor(ctx, fut) +# without res =? (await fut).catch, error: +# trace "Error in asyncDelTask", error = error.msg +# ctx[].res[].err(error) +# return + +# ctx[].res[].ok() +# return + +# proc delTask(ctx: ptr TaskCtx, key: ptr Key) = +# defer: +# if not ctx.isNil: +# discard ctx[].signal.fireSync() + +# try: +# waitFor asyncDelTask(ctx, key) +# except CatchableError as exc: +# trace "Unexpected exception thrown in asyncDelTask", exc = exc.msg +# raiseAssert exc.msg + +# method delete*( +# self: ThreadDatastore, +# key: Key): Future[?!void] {.async.} = +# var +# key = key +# res = ThreadResult[void]() +# ctx = TaskCtx[void]( +# ds: self.ds, +# res: addr res) + +# proc runTask() = +# self.tp.spawn delTask(addr ctx, addr key) + +# return self.dispatchTask(ctx, key.some, runTask) + +# method delete*( +# self: ThreadDatastore, +# keys: seq[Key]): Future[?!void] {.async.} = + +# for key in keys: +# if err =? (await self.delete(key)).errorOption: +# return failure err + +# return success() + +# proc asyncPutTask( +# ctx: ptr TaskCtx[void], +# key: ptr Key, +# data: ptr UncheckedArray[byte], +# len: int) {.async.} = + +# if ctx.isNil: +# trace "ctx is nil" +# return + +# let +# key = key[] +# data = @(data.toOpenArray(0, len - 1)) +# fut = ctx[].ds.put(key, data) + +# asyncSpawn signalMonitor(ctx, fut) +# without res =? (await fut).catch, error: +# trace "Error in asyncPutTask", error = error.msg +# ctx[].res[].err(error) +# return + +# ctx[].res[].ok() + +# proc putTask( +# ctx: ptr TaskCtx, +# key: ptr Key, +# data: ptr UncheckedArray[byte], +# len: int) = +# ## run put in a thread task +# ## + +# defer: +# if not ctx.isNil: +# discard ctx[].signal.fireSync() + +# try: +# waitFor asyncPutTask(ctx, key, data, len) +# except CatchableError as exc: +# trace "Unexpected exception thrown in asyncPutTask", exc = exc.msg +# raiseAssert exc.msg + +# method put*( +# self: ThreadDatastore, +# key: Key, +# data: seq[byte]): Future[?!void] {.async.} = +# var +# key = key +# data = data +# res = ThreadResult[void]() +# ctx = TaskCtx[void]( +# ds: self.ds, +# res: addr res) + +# proc runTask() = +# self.tp.spawn putTask( +# addr ctx, +# addr key, +# makeUncheckedArray(addr data[0]), +# data.len) + +# return self.dispatchTask(ctx, key.some, runTask) + +# method put*( +# self: ThreadDatastore, +# batch: seq[BatchEntry]): Future[?!void] {.async.} = + +# for entry in batch: +# if err =? (await self.put(entry.key, entry.data)).errorOption: +# return failure err + +# return success() + +# proc asyncGetTask( +# ctx: ptr TaskCtx[DataBuffer], +# key: ptr Key) {.async.} = +# if ctx.isNil: +# trace "ctx is nil" +# return + +# let +# key = key[] +# fut = ctx[].ds.get(key) + +# asyncSpawn signalMonitor(ctx, fut) +# without res =? (await fut).catch and data =? res, error: +# trace "Error in asyncGetTask", error = error.msg +# ctx[].res[].err(error) +# return + +# trace "Got data in get" +# ctx[].res[].ok(DataBuffer.new(data)) + +# proc getTask( +# ctx: ptr TaskCtx, +# key: ptr Key) = +# ## Run get in a thread task +# ## + +# defer: +# if not ctx.isNil: +# discard ctx[].signal.fireSync() + +# try: +# waitFor asyncGetTask(ctx, key) +# except CatchableError as exc: +# trace "Unexpected exception thrown in asyncGetTask", exc = exc.msg +# raiseAssert exc.msg + +# method get*( +# self: ThreadDatastore, +# key: Key): Future[?!seq[byte]] {.async.} = +# var +# key = key +# res = ThreadResult[DataBuffer]() +# ctx = TaskCtx[DataBuffer]( +# ds: self.ds, +# res: addr res) + +# proc runTask() = +# self.tp.spawn getTask(addr ctx, addr key) + +# return self.dispatchTask(ctx, key.some, runTask) + +# method close*(self: ThreadDatastore): Future[?!void] {.async.} = +# for fut in self.tasks.values.toSeq: +# await fut.cancelAndWait() # probably want to store the signal, instead of the future (or both?) + +# await self.ds.close() + +# proc asyncQueryTask( +# ctx: ptr TaskCtx, +# iter: ptr QueryIter) {.async.} = + +# if ctx.isNil or iter.isNil: +# trace "ctx is nil" +# return + +# let +# fut = iter[].next() + +# asyncSpawn signalMonitor(ctx, fut) +# without ret =? (await fut).catch and res =? ret, error: +# trace "Error in asyncQueryTask", error = error.msg +# ctx[].res[].err(error) +# return + +# if res.key.isNone: +# ctx[].res[].ok((default(DataBuffer), default(DataBuffer))) +# return + +# var +# keyBuf = DataBuffer.new($(res.key.get())) +# dataBuf = DataBuffer.new(res.data) + +# trace "Got query result", key = $res.key.get(), data = res.data +# ctx[].res[].ok((keyBuf, dataBuf)) + +# proc queryTask( +# ctx: ptr TaskCtx, +# iter: ptr QueryIter) = + +# defer: +# if not ctx.isNil: +# discard ctx[].signal.fireSync() + +# try: +# waitFor asyncQueryTask(ctx, iter) +# except CatchableError as exc: +# trace "Unexpected exception thrown in asyncQueryTask", exc = exc.msg +# raiseAssert exc.msg + +# method query*( +# self: ThreadDatastore, +# query: Query): Future[?!QueryIter] {.async.} = +# without var childIter =? await self.ds.query(query), error: +# return failure error + +# var +# iter = QueryIter.new() +# lock = newAsyncLock() # serialize querying under threads + +# proc next(): Future[?!QueryResponse] {.async.} = +# defer: +# if lock.locked: +# lock.release() + +# trace "About to query" +# if lock.locked: +# return failure (ref DatastoreError)(msg: "Should always await query features") + +# await lock.acquire() + +# if iter.finished == true: +# return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + +# iter.finished = childIter.finished +# var +# res = ThreadResult[QueryResponse]() +# ctx = TaskCtx[QueryResponse]( +# ds: self.ds, +# res: addr res) + +# proc runTask() = +# self.tp.spawn queryTask(addr ctx, addr childIter) + +# return self.dispatchTask(ctx, Key.none, runTask) - iter.next = next - return success iter +# iter.next = next +# return success iter proc new*( self: type ThreadDatastore, From 2aa8cfa3aa84d0ea481dc5eb68d5a19d7b43d356 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:09:26 -0700 Subject: [PATCH 256/445] simplifying --- datastore/threads/threadproxyds.nim | 94 ++++++----------------------- 1 file changed, 19 insertions(+), 75 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 5e11472e..e5d2265e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -50,7 +50,6 @@ type ds: D res: ThreadResult[T] cancelled: bool - semaphore: AsyncSemaphore signal: ThreadSignalPtr ThreadDatastore* = ref object of Datastore @@ -58,59 +57,18 @@ type backend: ThreadBackend semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors - withLocks: bool - tasks: Table[KeyId, Future[void]] - queryLock: AsyncLock # global query lock, this is only really \ - # needed for the fsds, but it is expensive! - -template withLocks( - self: ThreadDatastore, - ctx: TaskCtx, - key: ?KeyId = KeyId.none, - body: untyped): untyped = - try: - if key.isSome and key.get in self.tasks: - if self.withLocks: - await self.tasks[key.get] - if self.withLocks: - await self.queryLock.acquire() # only lock if it's required (fsds) - block: - body - finally: - if self.withLocks: - if key.isSome and key.get in self.tasks: - self.tasks.del(key.get) - if self.queryLock.locked: - self.queryLock.release() - -# TODO: needs rework, we can't use `result` with async -template dispatchTask[D, T]( - self: ThreadDatastore, - ctx: TaskCtx[D, T], - key: ?KeyId = KeyId.none, - runTask: proc): auto = +proc dispatchTask[D, T]( + self: ThreadDatastore, + ctx: var TaskCtx[D, T], + key: ?KeyId = KeyId.none, + runTask: proc +): Future[?!T] {.async.} = try: - await self.semaphore.acquire() - let signal = ThreadSignalPtr.new() - if signal.isErr: - failure(signal.error) - else: - ctx.signal = signal.get() - let - fut = wait(ctx.signal) - - withLocks(self, ctx, key): - runTask() - await fut - if ctx.res.isErr: - failure ctx.res.error - else: - when result.T isnot void: - success result.T(ctx.res.get) - else: - success() + runTask() + let fut = wait(ctx.signal) + except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.cancelled = true @@ -120,29 +78,12 @@ template dispatchTask[D, T]( discard ctx.signal.close() self.semaphore.release() -proc signalMonitor[T](ctx: ptr TaskCtx, fut: Future[T]) {.async.} = - ## Monitor the signal and cancel the future if - ## the cancellation flag is set - ## - - if ctx.isNil: - trace "ctx is nil" - return - - try: - await ctx[].signal.wait() - trace "Received signal" - - if ctx[].cancelled: # there could eventually be other flags - trace "Cancelling future" - if not fut.finished: - await fut.cancelAndWait() # cancel the `has` future - - discard ctx[].signal.fireSync() - except CatchableError as exc: - trace "Exception in thread signal monitor", exc = exc.msg - ctx[].res[].err(exc) - discard ctx[].signal.fireSync() +proc acquireSignal(): ?!ThreadSignalPtr = + let signal = ThreadSignalPtr.new() + if signal.isErr(): + failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) + else: + success signal.get() proc hasTask(ctx: ptr TaskCtx, key: KeyId) = defer: @@ -159,11 +100,14 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var key = KeyId.new key.id() + await self.semaphore.acquire() + let signal = ? acquireSignal() + case self.backend.kind: of Sqlite: var ds = self.backend.sql - ctx = TaskCtx[typeof(ds), bool](ds: ds) + ctx = TaskCtx[typeof(ds), bool](ds: ds, signal: signal) proc runTask() = self.tp.spawn hasTask(addr ctx, key) From 575d97311837464efa2c1792ec4892287b35e328 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:35:34 -0700 Subject: [PATCH 257/445] simplifying --- datastore/threads/threadproxyds.nim | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e5d2265e..715a0512 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -10,6 +10,7 @@ import std/atomics import std/strutils import std/tables import std/sequtils +import std/locks import pkg/chronos import pkg/chronos/threadsync @@ -36,6 +37,8 @@ logScope: topics = "datastore threadproxyds" type + SqliteDB = SQLiteBackend[KeyId,DataBuffer] + ThreadBackendKinds* = enum Sqlite # Filesystem @@ -49,8 +52,9 @@ type TaskCtx[D; T: ThreadTypes] = object ds: D res: ThreadResult[T] - cancelled: bool signal: ThreadSignalPtr + running: bool + cancelled: bool ThreadDatastore* = ref object of Datastore tp: Taskpool @@ -58,21 +62,30 @@ type semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors +var finishLock: Lock -proc dispatchTask[D, T]( - self: ThreadDatastore, - ctx: var TaskCtx[D, T], - key: ?KeyId = KeyId.none, - runTask: proc -): Future[?!T] {.async.} = +finishLock.initLock() + +proc setCancelled(ctx: var TaskCtx): bool = + withLock(finishLock): + if ctx.running: + return false + else: + ctx.cancelled = true + return true + +proc dispatchTask[D, T](self: ThreadDatastore, + ctx: var TaskCtx[D, T], + key: ?KeyId = KeyId.none, + runTask: proc(): void + ): Future[?!T] {.async.} = try: runTask() - let fut = wait(ctx.signal) - + await wait(ctx.signal) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg - ctx.cancelled = true - await ctx.signal.fire() + while not ctx.setCancelled(): + await sleepAsync(10.milliseconds) raise exc finally: discard ctx.signal.close() @@ -92,27 +105,29 @@ proc hasTask(ctx: ptr TaskCtx, key: KeyId) = try: let res = has(ctx.ds, key) + ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: + e.toThreadErr() except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = - var - key = KeyId.new key.id() + var key = KeyId.new key.id() await self.semaphore.acquire() - let signal = ? acquireSignal() + without signal =? acquireSignal(), err: + return bool.failure err case self.backend.kind: of Sqlite: var ds = self.backend.sql - ctx = TaskCtx[typeof(ds), bool](ds: ds, signal: signal) + ctx = TaskCtx[SqliteDB, bool](ds: self.backend.sql, signal: signal) proc runTask() = self.tp.spawn hasTask(addr ctx, key) - return self.dispatchTask(ctx, key.some, runTask) + return await self.dispatchTask(ctx, key.some, runTask) # proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = # if ctx.isNil: From 77e53d2bf64ad72d8e7b5e90d762c707bd5821d1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:50:11 -0700 Subject: [PATCH 258/445] simplifying --- datastore/sql/sqliteds.nim | 8 +++---- datastore/threads/threadproxyds.nim | 37 +++++++++++++---------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 1ce8d559..8b07c392 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -26,7 +26,7 @@ proc readOnly*[K,V](self: SQLiteBackend[K,V]): bool = self.db.readOnly proc timestamp*(t = epochTime()): int64 = (t * 1_000_000).int64 -proc has*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!bool = +proc has*[K,V](self: SQLiteBackend[K,V], key: K): ?!bool = var exists = false key = key @@ -42,7 +42,7 @@ proc has*[K,V](self: SQLiteBackend[K,V], key: DbKey): ?!bool = proc delete*[K,V](self: SQLiteBackend[K,V], key: K): ?!void = return self.db.deleteStmt.exec((key)) -proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[DbKey]): ?!void = +proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[K]): ?!void = if err =? self.db.beginStmt.exec().errorOption: return failure(err) @@ -74,7 +74,7 @@ proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!seq[byte] = if bytes.len <= 0: return failure( - newException(DatastoreKeyNotFound, "DbKey doesn't exist")) + newException(DatastoreKeyNotFound, "key doesn't exist")) return success bytes @@ -212,7 +212,7 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, RawStmtPtr]): ?!DbQueryResp handle.close() -proc contains*[K,V](self: SQLiteBackend[K,V], key: DbKey): bool = +proc contains*[K,V](self: SQLiteBackend[K,V], key: K): bool = return self.has(key).get() diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 715a0512..eb635de7 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -50,7 +50,7 @@ type TaskCtx[D; T: ThreadTypes] = object - ds: D + key: KeyId res: ThreadResult[T] signal: ThreadSignalPtr running: bool @@ -74,14 +74,20 @@ proc setCancelled(ctx: var TaskCtx): bool = ctx.cancelled = true return true -proc dispatchTask[D, T](self: ThreadDatastore, - ctx: var TaskCtx[D, T], - key: ?KeyId = KeyId.none, - runTask: proc(): void - ): Future[?!T] {.async.} = +template dispatchTask(self: ThreadDatastore, + signal: ThreadSignalPtr, + fn: untyped): auto = + var + ctx = TaskCtx[SqliteDB, bool](signal: signal) try: - runTask() - await wait(ctx.signal) + case self.backend.kind: + of Sqlite: + var ds = self.backend.sql + proc runTask() = + `fn`(addr ctx, ds) + runTask() + + await wait(ctx.signal) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg while not ctx.setCancelled(): @@ -98,13 +104,13 @@ proc acquireSignal(): ?!ThreadSignalPtr = else: success signal.get() -proc hasTask(ctx: ptr TaskCtx, key: KeyId) = +proc hasTask[D](ctx: ptr TaskCtx, ds: D) = defer: if not ctx.isNil: discard ctx[].signal.fireSync() try: - let res = has(ctx.ds, key) + let res = has(ds, key) ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: e.toThreadErr() except CatchableError as exc: @@ -118,16 +124,7 @@ method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = without signal =? acquireSignal(), err: return bool.failure err - case self.backend.kind: - of Sqlite: - var - ds = self.backend.sql - ctx = TaskCtx[SqliteDB, bool](ds: self.backend.sql, signal: signal) - - proc runTask() = - self.tp.spawn hasTask(addr ctx, key) - - return await self.dispatchTask(ctx, key.some, runTask) + dispatchTask(self, signal, hasTask) # proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = # if ctx.isNil: From 53eedd946c16f20034cc1962f2f407eb73eb3e72 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:51:34 -0700 Subject: [PATCH 259/445] simplifying --- datastore/threads/threadproxyds.nim | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index eb635de7..c5899984 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -63,7 +63,6 @@ type # to avoid exhausting file descriptors var finishLock: Lock - finishLock.initLock() proc setCancelled(ctx: var TaskCtx): bool = @@ -74,6 +73,13 @@ proc setCancelled(ctx: var TaskCtx): bool = ctx.cancelled = true return true +proc acquireSignal(): ?!ThreadSignalPtr = + let signal = ThreadSignalPtr.new() + if signal.isErr(): + failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) + else: + success signal.get() + template dispatchTask(self: ThreadDatastore, signal: ThreadSignalPtr, fn: untyped): auto = @@ -97,20 +103,13 @@ template dispatchTask(self: ThreadDatastore, discard ctx.signal.close() self.semaphore.release() -proc acquireSignal(): ?!ThreadSignalPtr = - let signal = ThreadSignalPtr.new() - if signal.isErr(): - failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) - else: - success signal.get() - proc hasTask[D](ctx: ptr TaskCtx, ds: D) = defer: if not ctx.isNil: discard ctx[].signal.fireSync() try: - let res = has(ds, key) + let res = has(ds, ctx.key) ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: e.toThreadErr() except CatchableError as exc: From 8e573bdfc7e88be8c53731f6791a174dfc5215b3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:53:28 -0700 Subject: [PATCH 260/445] simplifying --- datastore/threads/threadproxyds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c5899984..8425caa5 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -90,7 +90,7 @@ template dispatchTask(self: ThreadDatastore, of Sqlite: var ds = self.backend.sql proc runTask() = - `fn`(addr ctx, ds) + self.tp.spawn `fn`(addr ctx, ds) runTask() await wait(ctx.signal) From cbad3c1396925c0bf51d5f04d70e56e251f2542f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:54:58 -0700 Subject: [PATCH 261/445] simplifying --- datastore/threads/threadproxyds.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8425caa5..18590e6f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -48,7 +48,6 @@ type of Sqlite: sql*: SQLiteBackend[KeyId,DataBuffer] - TaskCtx[D; T: ThreadTypes] = object key: KeyId res: ThreadResult[T] @@ -62,11 +61,11 @@ type semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors -var finishLock: Lock -finishLock.initLock() +var ctxLock: Lock +ctxLock.initLock() proc setCancelled(ctx: var TaskCtx): bool = - withLock(finishLock): + withLock(ctxLock): if ctx.running: return false else: @@ -110,8 +109,9 @@ proc hasTask[D](ctx: ptr TaskCtx, ds: D) = try: let res = has(ds, ctx.key) - ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: - e.toThreadErr() + withLock(ctxLock): + ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: + e.toThreadErr() except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg From cdb55c0f089e8e5ba0d94ad697bff2f7980616ea Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 22:55:16 -0700 Subject: [PATCH 262/445] simplifying --- datastore/threads/threadproxyds.nim | 5 ----- 1 file changed, 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 18590e6f..0060d784 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -6,19 +6,14 @@ import pkg/upraises push: {.upraises: [].} -import std/atomics -import std/strutils import std/tables -import std/sequtils import std/locks import pkg/chronos import pkg/chronos/threadsync import pkg/questionable import pkg/questionable/results -import pkg/stew/ptrops import pkg/taskpools -import pkg/stew/byteutils import pkg/chronicles import ../key From d8d6ad771a7d0b3446dc392c618f010fa9e86c60 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:05:26 -0700 Subject: [PATCH 263/445] integrate setups --- datastore/threads/threadproxyds.nim | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 0060d784..1da13d32 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -91,25 +91,38 @@ template dispatchTask(self: ThreadDatastore, except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg while not ctx.setCancelled(): + warn "waiting to cancel thread future!", fn = astToStr(fn), key = $ctx.key await sleepAsync(10.milliseconds) raise exc finally: discard ctx.signal.close() self.semaphore.release() -proc hasTask[D](ctx: ptr TaskCtx, ds: D) = - defer: - if not ctx.isNil: - discard ctx[].signal.fireSync() - +proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsafe.}) {.gcsafe.} = try: - let res = has(ds, ctx.key) withLock(ctxLock): + if ctx.cancelled: + return + ctx.running = true + + ## run backend command + let res = cb(ctx) + # let res = has(ds, ctx.key) + + withLock(ctxLock): + ctx.running = false ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: e.toThreadErr() except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg + finally: + discard ctx[].signal.fireSync() + +proc hasTask[D](ctx: ptr TaskCtx, ds: D) {.gcsafe.} = + ## run backend command + runTask(ctx, ds) do(ctx: ptr TaskCtx) -> ?!bool: + has(ds, ctx.key) method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = var key = KeyId.new key.id() From 0926eaf2457268c78bca5d237a2c140642c887ad Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:19:23 -0700 Subject: [PATCH 264/445] integrate setups --- datastore/threads/threadproxyds.nim | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 1da13d32..130d642d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -44,7 +44,6 @@ type sql*: SQLiteBackend[KeyId,DataBuffer] TaskCtx[D; T: ThreadTypes] = object - key: KeyId res: ThreadResult[T] signal: ThreadSignalPtr running: bool @@ -76,22 +75,24 @@ proc acquireSignal(): ?!ThreadSignalPtr = template dispatchTask(self: ThreadDatastore, signal: ThreadSignalPtr, - fn: untyped): auto = + blk: untyped + ): auto = var - ctx = TaskCtx[SqliteDB, bool](signal: signal) + ctx {.inject.} = TaskCtx[SqliteDB, bool](signal: signal) try: case self.backend.kind: of Sqlite: - var ds = self.backend.sql + var ds {.inject.} = self.backend.sql proc runTask() = - self.tp.spawn `fn`(addr ctx, ds) + # self.tp.spawn `fn`(addr ctx, ds, args) + `blk` runTask() await wait(ctx.signal) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg while not ctx.setCancelled(): - warn "waiting to cancel thread future!", fn = astToStr(fn), key = $ctx.key + warn "waiting to cancel thread future!", fn = astToStr(fn) await sleepAsync(10.milliseconds) raise exc finally: @@ -107,7 +108,6 @@ proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsa ## run backend command let res = cb(ctx) - # let res = has(ds, ctx.key) withLock(ctxLock): ctx.running = false @@ -119,19 +119,19 @@ proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsa finally: discard ctx[].signal.fireSync() -proc hasTask[D](ctx: ptr TaskCtx, ds: D) {.gcsafe.} = +proc hasTask[DB](ctx: ptr TaskCtx, ds: DB, key: KeyId) {.gcsafe.} = ## run backend command runTask(ctx, ds) do(ctx: ptr TaskCtx) -> ?!bool: - has(ds, ctx.key) + has(ds, key) method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = - var key = KeyId.new key.id() - await self.semaphore.acquire() without signal =? acquireSignal(), err: return bool.failure err - dispatchTask(self, signal, hasTask) + let key = KeyId.new key.id() + dispatchTask(self, signal): + self.tp.spawn hasTask(addr ctx, ds, key) # proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = # if ctx.isNil: From 73df67899b422602c9f26e7e2edc6652425921ea Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:21:08 -0700 Subject: [PATCH 265/445] integrate setups --- datastore/threads/threadproxyds.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 130d642d..07f8f878 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -99,7 +99,7 @@ template dispatchTask(self: ThreadDatastore, discard ctx.signal.close() self.semaphore.release() -proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsafe.}) {.gcsafe.} = +template runTask(ctx: ptr TaskCtx, blk: untyped) = try: withLock(ctxLock): if ctx.cancelled: @@ -107,7 +107,7 @@ proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsa ctx.running = true ## run backend command - let res = cb(ctx) + let res = `blk` withLock(ctxLock): ctx.running = false @@ -121,7 +121,7 @@ proc runTask[D, T](ctx: ptr TaskCtx, ds: D, cb: proc(ctx: ptr TaskCtx): T {.gcsa proc hasTask[DB](ctx: ptr TaskCtx, ds: DB, key: KeyId) {.gcsafe.} = ## run backend command - runTask(ctx, ds) do(ctx: ptr TaskCtx) -> ?!bool: + runTask(ctx): has(ds, key) method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = From f159effadf08feb7125df079057a55b5fafa388c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:22:06 -0700 Subject: [PATCH 266/445] integrate setups --- datastore/threads/threadproxyds.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 07f8f878..c2d02862 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -84,7 +84,6 @@ template dispatchTask(self: ThreadDatastore, of Sqlite: var ds {.inject.} = self.backend.sql proc runTask() = - # self.tp.spawn `fn`(addr ctx, ds, args) `blk` runTask() From 8eef3e644204a1a79b50b88dba278424538acd26 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:26:59 -0700 Subject: [PATCH 267/445] integrate setups --- datastore/threads/threadproxyds.nim | 76 +++++++++-------------------- 1 file changed, 24 insertions(+), 52 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c2d02862..56f84906 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -98,7 +98,7 @@ template dispatchTask(self: ThreadDatastore, discard ctx.signal.close() self.semaphore.release() -template runTask(ctx: ptr TaskCtx, blk: untyped) = +template executeTask(ctx: ptr TaskCtx, blk: untyped) = try: withLock(ctxLock): if ctx.cancelled: @@ -120,71 +120,43 @@ template runTask(ctx: ptr TaskCtx, blk: untyped) = proc hasTask[DB](ctx: ptr TaskCtx, ds: DB, key: KeyId) {.gcsafe.} = ## run backend command - runTask(ctx): + executeTask(ctx): has(ds, key) -method has*(self: ThreadDatastore, key: Key): Future[?!bool] {.async.} = +method has*(self: ThreadDatastore, + key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() without signal =? acquireSignal(), err: - return bool.failure err + return failure err let key = KeyId.new key.id() dispatchTask(self, signal): self.tp.spawn hasTask(addr ctx, ds, key) -# proc asyncDelTask(ctx: ptr TaskCtx[void], key: ptr Key) {.async.} = -# if ctx.isNil: -# trace "ctx is nil" -# return - -# let -# key = key[] -# fut = ctx[].ds.delete(key) - -# asyncSpawn signalMonitor(ctx, fut) -# without res =? (await fut).catch, error: -# trace "Error in asyncDelTask", error = error.msg -# ctx[].res[].err(error) -# return - -# ctx[].res[].ok() -# return - -# proc delTask(ctx: ptr TaskCtx, key: ptr Key) = -# defer: -# if not ctx.isNil: -# discard ctx[].signal.fireSync() - -# try: -# waitFor asyncDelTask(ctx, key) -# except CatchableError as exc: -# trace "Unexpected exception thrown in asyncDelTask", exc = exc.msg -# raiseAssert exc.msg - -# method delete*( -# self: ThreadDatastore, -# key: Key): Future[?!void] {.async.} = -# var -# key = key -# res = ThreadResult[void]() -# ctx = TaskCtx[void]( -# ds: self.ds, -# res: addr res) +proc deleteTask[DB](ctx: ptr TaskCtx, ds: DB; + key: KeyId) {.gcsafe.} = + ## run backend command + executeTask(ctx): + delete(ds, key) -# proc runTask() = -# self.tp.spawn delTask(addr ctx, addr key) +method delete*(self: ThreadDatastore, + key: Key): Future[?!void] {.async.} = + await self.semaphore.acquire() + without signal =? acquireSignal(), err: + return failure err -# return self.dispatchTask(ctx, key.some, runTask) + let key = KeyId.new key.id() + dispatchTask(self, signal): + self.tp.spawn deleteTask(addr ctx, ds, key) -# method delete*( -# self: ThreadDatastore, -# keys: seq[Key]): Future[?!void] {.async.} = +method delete*(self: ThreadDatastore, + keys: seq[Key]): Future[?!void] {.async.} = -# for key in keys: -# if err =? (await self.delete(key)).errorOption: -# return failure err + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err -# return success() + return success() # proc asyncPutTask( # ctx: ptr TaskCtx[void], From 9c7c47e66a3dbf54b607671ac6e3c11cbb7d8d30 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:29:22 -0700 Subject: [PATCH 268/445] integrate setups --- datastore/threads/threadproxyds.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 56f84906..9e5ce859 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -73,12 +73,12 @@ proc acquireSignal(): ?!ThreadSignalPtr = else: success signal.get() -template dispatchTask(self: ThreadDatastore, - signal: ThreadSignalPtr, - blk: untyped - ): auto = +template dispatchTask[T](self: ThreadDatastore, + signal: ThreadSignalPtr, + blk: untyped + ): auto = var - ctx {.inject.} = TaskCtx[SqliteDB, bool](signal: signal) + ctx {.inject.} = TaskCtx[SqliteDB, T](signal: signal) try: case self.backend.kind: of Sqlite: @@ -130,7 +130,7 @@ method has*(self: ThreadDatastore, return failure err let key = KeyId.new key.id() - dispatchTask(self, signal): + dispatchTask[bool](self, signal): self.tp.spawn hasTask(addr ctx, ds, key) proc deleteTask[DB](ctx: ptr TaskCtx, ds: DB; @@ -146,7 +146,7 @@ method delete*(self: ThreadDatastore, return failure err let key = KeyId.new key.id() - dispatchTask(self, signal): + dispatchTask[void](self, signal): self.tp.spawn deleteTask(addr ctx, ds, key) method delete*(self: ThreadDatastore, From 0bd6bd58018ecb03dd5397a36dcdaa832e839422 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 25 Sep 2023 23:48:17 -0700 Subject: [PATCH 269/445] integrate setups --- datastore/threads/threadproxyds.nim | 176 ++++++++-------------------- 1 file changed, 52 insertions(+), 124 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 9e5ce859..4d089e4a 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -43,7 +43,7 @@ type of Sqlite: sql*: SQLiteBackend[KeyId,DataBuffer] - TaskCtx[D; T: ThreadTypes] = object + TaskCtx[T: ThreadTypes] = object res: ThreadResult[T] signal: ThreadSignalPtr running: bool @@ -78,7 +78,7 @@ template dispatchTask[T](self: ThreadDatastore, blk: untyped ): auto = var - ctx {.inject.} = TaskCtx[SqliteDB, T](signal: signal) + ctx {.inject.} = TaskCtx[T](signal: signal) try: case self.backend.kind: of Sqlite: @@ -98,7 +98,7 @@ template dispatchTask[T](self: ThreadDatastore, discard ctx.signal.close() self.semaphore.release() -template executeTask(ctx: ptr TaskCtx, blk: untyped) = +template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = try: withLock(ctxLock): if ctx.cancelled: @@ -107,18 +107,23 @@ template executeTask(ctx: ptr TaskCtx, blk: untyped) = ## run backend command let res = `blk` + if res.isOk(): + when T is void: + ctx.res.ok() + else: + ctx.res.ok(res.get()) + else: + ctx.res.err res.error().toThreadErr() withLock(ctxLock): ctx.running = false - ctx.res = res.mapErr() do(e: ref CatchableError) -> ThreadResErr: - e.toThreadErr() except CatchableError as exc: trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg raiseAssert exc.msg finally: discard ctx[].signal.fireSync() -proc hasTask[DB](ctx: ptr TaskCtx, ds: DB, key: KeyId) {.gcsafe.} = +proc hasTask[T, DB](ctx: ptr TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): has(ds, key) @@ -133,8 +138,8 @@ method has*(self: ThreadDatastore, dispatchTask[bool](self, signal): self.tp.spawn hasTask(addr ctx, ds, key) -proc deleteTask[DB](ctx: ptr TaskCtx, ds: DB; - key: KeyId) {.gcsafe.} = +proc deleteTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; + key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): delete(ds, key) @@ -158,128 +163,51 @@ method delete*(self: ThreadDatastore, return success() -# proc asyncPutTask( -# ctx: ptr TaskCtx[void], -# key: ptr Key, -# data: ptr UncheckedArray[byte], -# len: int) {.async.} = - -# if ctx.isNil: -# trace "ctx is nil" -# return - -# let -# key = key[] -# data = @(data.toOpenArray(0, len - 1)) -# fut = ctx[].ds.put(key, data) - -# asyncSpawn signalMonitor(ctx, fut) -# without res =? (await fut).catch, error: -# trace "Error in asyncPutTask", error = error.msg -# ctx[].res[].err(error) -# return - -# ctx[].res[].ok() - -# proc putTask( -# ctx: ptr TaskCtx, -# key: ptr Key, -# data: ptr UncheckedArray[byte], -# len: int) = -# ## run put in a thread task -# ## - -# defer: -# if not ctx.isNil: -# discard ctx[].signal.fireSync() - -# try: -# waitFor asyncPutTask(ctx, key, data, len) -# except CatchableError as exc: -# trace "Unexpected exception thrown in asyncPutTask", exc = exc.msg -# raiseAssert exc.msg - -# method put*( -# self: ThreadDatastore, -# key: Key, -# data: seq[byte]): Future[?!void] {.async.} = -# var -# key = key -# data = data -# res = ThreadResult[void]() -# ctx = TaskCtx[void]( -# ds: self.ds, -# res: addr res) - -# proc runTask() = -# self.tp.spawn putTask( -# addr ctx, -# addr key, -# makeUncheckedArray(addr data[0]), -# data.len) - -# return self.dispatchTask(ctx, key.some, runTask) - -# method put*( -# self: ThreadDatastore, -# batch: seq[BatchEntry]): Future[?!void] {.async.} = - -# for entry in batch: -# if err =? (await self.put(entry.key, entry.data)).errorOption: -# return failure err - -# return success() - -# proc asyncGetTask( -# ctx: ptr TaskCtx[DataBuffer], -# key: ptr Key) {.async.} = -# if ctx.isNil: -# trace "ctx is nil" -# return - -# let -# key = key[] -# fut = ctx[].ds.get(key) - -# asyncSpawn signalMonitor(ctx, fut) -# without res =? (await fut).catch and data =? res, error: -# trace "Error in asyncGetTask", error = error.msg -# ctx[].res[].err(error) -# return - -# trace "Got data in get" -# ctx[].res[].ok(DataBuffer.new(data)) +proc putTask[DB](ctx: ptr TaskCtx, ds: DB; + key: KeyId, + data: DataBuffer) {.gcsafe, nimcall.} = + ## run backend command + executeTask(ctx): + put(ds, key, data) -# proc getTask( -# ctx: ptr TaskCtx, -# key: ptr Key) = -# ## Run get in a thread task -# ## +method put*(self: ThreadDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + await self.semaphore.acquire() + without signal =? acquireSignal(), err: + return failure err -# defer: -# if not ctx.isNil: -# discard ctx[].signal.fireSync() + let key = KeyId.new key.id() + let data = DataBuffer.new data + dispatchTask[void](self, signal): + self.tp.spawn putTask(addr ctx, ds, key, data) + +method put*( + self: ThreadDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err -# try: -# waitFor asyncGetTask(ctx, key) -# except CatchableError as exc: -# trace "Unexpected exception thrown in asyncGetTask", exc = exc.msg -# raiseAssert exc.msg + return success() -# method get*( -# self: ThreadDatastore, -# key: Key): Future[?!seq[byte]] {.async.} = -# var -# key = key -# res = ThreadResult[DataBuffer]() -# ctx = TaskCtx[DataBuffer]( -# ds: self.ds, -# res: addr res) +proc getTask[DB](ctx: ptr TaskCtx, ds: DB; + key: KeyId) {.gcsafe, nimcall.} = + ## run backend command + executeTask(ctx): + get(ds, key) -# proc runTask() = -# self.tp.spawn getTask(addr ctx, addr key) +method get*(self: ThreadDatastore, + key: Key, + ): Future[?!seq[byte]] {.async.} = + await self.semaphore.acquire() + without signal =? acquireSignal(), err: + return failure err -# return self.dispatchTask(ctx, key.some, runTask) + let key = KeyId.new key.id() + dispatchTask[void](self, signal): + self.tp.spawn getTask(addr ctx, ds, key) # method close*(self: ThreadDatastore): Future[?!void] {.async.} = # for fut in self.tasks.values.toSeq: From fddbb99a9875df0ee656358336bca24f37e6c452 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 00:00:25 -0700 Subject: [PATCH 270/445] await all --- datastore/threads/asyncsemaphore.nim | 7 +- datastore/threads/threadproxyds.nim | 106 ++++++++------------------- 2 files changed, 37 insertions(+), 76 deletions(-) diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim index c9a76276..17354603 100644 --- a/datastore/threads/asyncsemaphore.nim +++ b/datastore/threads/asyncsemaphore.nim @@ -21,6 +21,7 @@ type AsyncSemaphore* = ref object of RootObj size*: int count: int + exit: bool queue: seq[Future[void]] func new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = @@ -28,12 +29,16 @@ func new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = proc `count`*(s: AsyncSemaphore): int = s.count +proc waitAll*(s: AsyncSemaphore) {.async.} = + s.exit = true + await allFutures(s.queue) + proc tryAcquire*(s: AsyncSemaphore): bool = ## Attempts to acquire a resource, if successful ## returns true, otherwise false ## - if s.count > 0 and s.queue.len == 0: + if s.count > 0 and s.queue.len == 0 and not s.exit: s.count.dec trace "Acquired slot", available = s.count, queue = s.queue.len return true diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4d089e4a..c0fc8a0f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -32,7 +32,6 @@ logScope: topics = "datastore threadproxyds" type - SqliteDB = SQLiteBackend[KeyId,DataBuffer] ThreadBackendKinds* = enum Sqlite @@ -209,92 +208,49 @@ method get*(self: ThreadDatastore, dispatchTask[void](self, signal): self.tp.spawn getTask(addr ctx, ds, key) -# method close*(self: ThreadDatastore): Future[?!void] {.async.} = -# for fut in self.tasks.values.toSeq: -# await fut.cancelAndWait() # probably want to store the signal, instead of the future (or both?) +method close*(self: ThreadDatastore): Future[?!void] {.async.} = -# await self.ds.close() + await self.ds.close() -# proc asyncQueryTask( -# ctx: ptr TaskCtx, -# iter: ptr QueryIter) {.async.} = -# if ctx.isNil or iter.isNil: -# trace "ctx is nil" -# return - -# let -# fut = iter[].next() - -# asyncSpawn signalMonitor(ctx, fut) -# without ret =? (await fut).catch and res =? ret, error: -# trace "Error in asyncQueryTask", error = error.msg -# ctx[].res[].err(error) -# return - -# if res.key.isNone: -# ctx[].res[].ok((default(DataBuffer), default(DataBuffer))) -# return - -# var -# keyBuf = DataBuffer.new($(res.key.get())) -# dataBuf = DataBuffer.new(res.data) - -# trace "Got query result", key = $res.key.get(), data = res.data -# ctx[].res[].ok((keyBuf, dataBuf)) - -# proc queryTask( -# ctx: ptr TaskCtx, -# iter: ptr QueryIter) = - -# defer: -# if not ctx.isNil: -# discard ctx[].signal.fireSync() - -# try: -# waitFor asyncQueryTask(ctx, iter) -# except CatchableError as exc: -# trace "Unexpected exception thrown in asyncQueryTask", exc = exc.msg -# raiseAssert exc.msg - -# method query*( -# self: ThreadDatastore, -# query: Query): Future[?!QueryIter] {.async.} = -# without var childIter =? await self.ds.query(query), error: -# return failure error +method query*( + self: ThreadDatastore, + query: Query): Future[?!QueryIter] {.async.} = + without var childIter =? await self.ds.query(query), error: + return failure error -# var -# iter = QueryIter.new() -# lock = newAsyncLock() # serialize querying under threads + var + iter = QueryIter.new() + lock = newAsyncLock() # serialize querying under threads -# proc next(): Future[?!QueryResponse] {.async.} = -# defer: -# if lock.locked: -# lock.release() + proc next(): Future[?!QueryResponse] {.async.} = + defer: + if lock.locked: + lock.release() -# trace "About to query" -# if lock.locked: -# return failure (ref DatastoreError)(msg: "Should always await query features") + trace "About to query" + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") -# await lock.acquire() + await lock.acquire() -# if iter.finished == true: -# return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + if iter.finished == true: + return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") -# iter.finished = childIter.finished -# var -# res = ThreadResult[QueryResponse]() -# ctx = TaskCtx[QueryResponse]( -# ds: self.ds, -# res: addr res) + iter.finished = childIter.finished + var + res = ThreadResult[QueryResponse]() + ctx = TaskCtx[QueryResponse]( + ds: self.ds, + res: addr res) -# proc runTask() = -# self.tp.spawn queryTask(addr ctx, addr childIter) + proc runTask() = + self.tp.spawn queryTask(addr ctx, addr childIter) -# return self.dispatchTask(ctx, Key.none, runTask) + return self.dispatchTask(ctx, Key.none, runTask) -# iter.next = next -# return success iter + iter.next = next + return success iter proc new*( self: type ThreadDatastore, From ca9edb27980cec4e9010563b294ee373b5c1db45 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 00:02:44 -0700 Subject: [PATCH 271/445] await all --- datastore/threads/asyncsemaphore.nim | 2 +- datastore/threads/threadproxyds.nim | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/datastore/threads/asyncsemaphore.nim b/datastore/threads/asyncsemaphore.nim index 17354603..b561882c 100644 --- a/datastore/threads/asyncsemaphore.nim +++ b/datastore/threads/asyncsemaphore.nim @@ -29,7 +29,7 @@ func new*(_: type AsyncSemaphore, size: int): AsyncSemaphore = proc `count`*(s: AsyncSemaphore): int = s.count -proc waitAll*(s: AsyncSemaphore) {.async.} = +proc closeAll*(s: AsyncSemaphore) {.async.} = s.exit = true await allFutures(s.queue) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c0fc8a0f..35237bcf 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -209,8 +209,10 @@ method get*(self: ThreadDatastore, self.tp.spawn getTask(addr ctx, ds, key) method close*(self: ThreadDatastore): Future[?!void] {.async.} = - - await self.ds.close() + await self.semaphore.closeAll() + case self.backend.kind: + of Sqlite: + self.backend.sql.close() method query*( From d34cbd7df5ca1a834c7f9a8d85e9a6aff4c6e164 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 12:51:28 -0700 Subject: [PATCH 272/445] integrate setups --- datastore/threads/threadproxyds.nim | 62 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 35237bcf..33f0d81a 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -72,31 +72,6 @@ proc acquireSignal(): ?!ThreadSignalPtr = else: success signal.get() -template dispatchTask[T](self: ThreadDatastore, - signal: ThreadSignalPtr, - blk: untyped - ): auto = - var - ctx {.inject.} = TaskCtx[T](signal: signal) - try: - case self.backend.kind: - of Sqlite: - var ds {.inject.} = self.backend.sql - proc runTask() = - `blk` - runTask() - - await wait(ctx.signal) - except CancelledError as exc: - trace "Cancelling thread future!", exc = exc.msg - while not ctx.setCancelled(): - warn "waiting to cancel thread future!", fn = astToStr(fn) - await sleepAsync(10.milliseconds) - raise exc - finally: - discard ctx.signal.close() - self.semaphore.release() - template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = try: withLock(ctxLock): @@ -122,6 +97,31 @@ template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = finally: discard ctx[].signal.fireSync() +template dispatchTask[T](self: ThreadDatastore, + signal: ThreadSignalPtr, + blk: untyped + ): auto = + var + ctx {.inject.} = TaskCtx[T](signal: signal) + try: + case self.backend.kind: + of Sqlite: + var ds {.inject.} = self.backend.sql + proc runTask() = + `blk` + runTask() + + await wait(ctx.signal) + except CancelledError as exc: + trace "Cancelling thread future!", exc = exc.msg + while not ctx.setCancelled(): + warn "waiting to cancel thread future!", fn = astToStr(fn) + await sleepAsync(10.milliseconds) + raise exc + finally: + discard ctx.signal.close() + self.semaphore.release() + proc hasTask[T, DB](ctx: ptr TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -214,15 +214,18 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = of Sqlite: self.backend.sql.close() +proc queryTask[DB](ctx: ptr TaskCtx, ds: DB; + key: DbQuery[KeyId]) {.gcsafe, nimcall.} = + ## run backend command + executeTask(ctx): + let handle = ds.query(q) + query(ds, key) method query*( self: ThreadDatastore, query: Query): Future[?!QueryIter] {.async.} = - without var childIter =? await self.ds.query(query), error: - return failure error var - iter = QueryIter.new() lock = newAsyncLock() # serialize querying under threads proc next(): Future[?!QueryResponse] {.async.} = @@ -242,9 +245,6 @@ method query*( iter.finished = childIter.finished var res = ThreadResult[QueryResponse]() - ctx = TaskCtx[QueryResponse]( - ds: self.ds, - res: addr res) proc runTask() = self.tp.spawn queryTask(addr ctx, addr childIter) From b2de461c41f55ba91233c38bf6ac3ec586090be9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:01:53 -0700 Subject: [PATCH 273/445] setup query --- datastore/threads/threadproxyds.nim | 32 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 33f0d81a..a752d63e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -162,7 +162,7 @@ method delete*(self: ThreadDatastore, return success() -proc putTask[DB](ctx: ptr TaskCtx, ds: DB; +proc putTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; key: KeyId, data: DataBuffer) {.gcsafe, nimcall.} = ## run backend command @@ -191,7 +191,7 @@ method put*( return success() -proc getTask[DB](ctx: ptr TaskCtx, ds: DB; +proc getTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; key: KeyId) {.gcsafe, nimcall.} = ## run backend command executeTask(ctx): @@ -214,17 +214,35 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = of Sqlite: self.backend.sql.close() -proc queryTask[DB](ctx: ptr TaskCtx, ds: DB; - key: DbQuery[KeyId]) {.gcsafe, nimcall.} = +proc queryTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; + query: DbQuery[KeyId]) {.gcsafe, nimcall.} = ## run backend command + var handle = ds.query(query) + executeTask(ctx): + handle + executeTask(ctx): - let handle = ds.query(q) query(ds, key) method query*( self: ThreadDatastore, query: Query): Future[?!QueryIter] {.async.} = + await self.semaphore.acquire() + without signal =? acquireSignal(), err: + return failure err + + let dq = dbQuery( + key=query.key, + value=query.value, + limit=query.limit, + offset=query.offset, + sort=query.sort, + ) + + dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): + self.tp.spawn deleteTask(addr ctx, ds, dq) + var lock = newAsyncLock() # serialize querying under threads @@ -246,10 +264,6 @@ method query*( var res = ThreadResult[QueryResponse]() - proc runTask() = - self.tp.spawn queryTask(addr ctx, addr childIter) - - return self.dispatchTask(ctx, Key.none, runTask) iter.next = next return success iter From 85958782440ef7e2972bd9282f09dcb1b33e830e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:02:34 -0700 Subject: [PATCH 274/445] setup query --- datastore/threads/threadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index a752d63e..c1a72b13 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -233,7 +233,7 @@ method query*( return failure err let dq = dbQuery( - key=query.key, + key= KeyId.new query.key.id(), value=query.value, limit=query.limit, offset=query.offset, @@ -241,7 +241,7 @@ method query*( ) dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn deleteTask(addr ctx, ds, dq) + self.tp.spawn queryTask(addr ctx, ds, dq) var lock = newAsyncLock() # serialize querying under threads From ca143227c63267418b3f41b7b6b427a85d08a0f2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:25:15 -0700 Subject: [PATCH 275/445] setup query --- datastore/threads/threadproxyds.nim | 51 ++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c1a72b13..4d068b75 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -65,6 +65,15 @@ proc setCancelled(ctx: var TaskCtx): bool = ctx.cancelled = true return true +proc setRunning[T](ctx: ptr TaskCtx[T]): bool = + withLock(ctxLock): + if ctx.cancelled: + return + ctx.running = true +proc setDone[T](ctx: ptr TaskCtx[T]) = + withLock(ctxLock): + ctx.running = false + proc acquireSignal(): ?!ThreadSignalPtr = let signal = ThreadSignalPtr.new() if signal.isErr(): @@ -74,10 +83,8 @@ proc acquireSignal(): ?!ThreadSignalPtr = template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = try: - withLock(ctxLock): - if ctx.cancelled: - return - ctx.running = true + if not ctx.setRunning(): + return ## run backend command let res = `blk` @@ -88,13 +95,11 @@ template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = ctx.res.ok(res.get()) else: ctx.res.err res.error().toThreadErr() - - withLock(ctxLock): - ctx.running = false except CatchableError as exc: - trace "Unexpected exception thrown in asyncHasTask", exc = exc.msg - raiseAssert exc.msg + trace "Unexpected exception thrown in async task", exc = exc.msg + ctx[].res.err exc.toThreadErr() finally: + ctx.setDone() discard ctx[].signal.fireSync() template dispatchTask[T](self: ThreadDatastore, @@ -214,13 +219,27 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = of Sqlite: self.backend.sql.close() -proc queryTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; - query: DbQuery[KeyId]) {.gcsafe, nimcall.} = - ## run backend command - var handle = ds.query(query) - executeTask(ctx): - handle - +type + QResult = DbQueryResponse[KeyId, DataBuffer] + +proc queryTask[DB]( + ctx: ptr TaskCtx[QResult], + ds: DB, + dq: DbQuery[KeyId] +) {.gcsafe, nimcall.} = + ## run query command + if not ctx.setRunning(): + return + + var qh = ds.query(dq) + if qh .isOk(): + (?!QResult).ok(default(QResult)) + else: + (?!QResult).err(qh.error()) + + var handle = qh.get() + + for item in executeTask(ctx): query(ds, key) From 7a0c660788a908d0c04ef5de519e1813a3a8d04d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:31:24 -0700 Subject: [PATCH 276/445] setup query --- datastore/threads/threadproxyds.nim | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4d068b75..ed2b2de5 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -228,20 +228,19 @@ proc queryTask[DB]( dq: DbQuery[KeyId] ) {.gcsafe, nimcall.} = ## run query command - if not ctx.setRunning(): - return - - var qh = ds.query(dq) - if qh .isOk(): - (?!QResult).ok(default(QResult)) - else: - (?!QResult).err(qh.error()) + var qh: typeof(ds.query(dq)) + executeTask(ctx): + qh = ds.query(dq) + if qh.isOk(): + (?!QResult).ok(default(QResult)) + else: + (?!QResult).err(qh.error()) var handle = qh.get() - for item in - executeTask(ctx): - query(ds, key) + for item in handle.iter(): + executeTask(ctx): + item method query*( self: ThreadDatastore, From 7a4b9acedc2919c870bb1f071bea1059ef376dc2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:34:20 -0700 Subject: [PATCH 277/445] setup query --- datastore/threads/threadproxyds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ed2b2de5..db2c6f1d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -263,6 +263,7 @@ method query*( var lock = newAsyncLock() # serialize querying under threads + iter = QueryIter.new() proc next(): Future[?!QueryResponse] {.async.} = defer: From 86ee5d912c1ff26b3e198dbf1ebcb715533a57d5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:35:58 -0700 Subject: [PATCH 278/445] setup query --- datastore/threads/threadproxyds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index db2c6f1d..cf1960ea 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -235,9 +235,10 @@ proc queryTask[DB]( (?!QResult).ok(default(QResult)) else: (?!QResult).err(qh.error()) + if qh.isErr(): + return var handle = qh.get() - for item in handle.iter(): executeTask(ctx): item From 57e2a702767b4f3171018ec8c3f9cbda74343261 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:40:57 -0700 Subject: [PATCH 279/445] setup query --- datastore/threads/threadproxyds.nim | 3 +++ datastore/threads/threadresult.nim | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index cf1960ea..8ad8ba29 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -98,6 +98,9 @@ template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = except CatchableError as exc: trace "Unexpected exception thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() + except Exception as exc: + trace "Unexpected defect thrown in async task", exc = exc.msg + ctx[].res.err exc.toThreadErr() finally: ctx.setDone() discard ctx[].signal.fireSync() diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index c0fb7ed5..b1694f42 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -16,18 +16,20 @@ type DatastoreErr, DatastoreKeyNotFoundErr, QueryEndedErr, - CatchableErr + CatchableErr, + DefectErr ThreadTypes* = void | bool | SomeInteger | DataBuffer | tuple | Atomic ThreadResErr* = (ErrorEnum, DataBuffer) ThreadResult*[T: ThreadTypes] = Result[T, ThreadResErr] -converter toThreadErr*(e: ref CatchableError): ThreadResErr {.inline, raises: [].} = +converter toThreadErr*(e: ref Exception): ThreadResErr {.inline, raises: [].} = if e of DatastoreKeyNotFound: (ErrorEnum.DatastoreKeyNotFoundErr, DataBuffer.new(e.msg)) elif e of QueryEndedError: (ErrorEnum.QueryEndedErr, DataBuffer.new(e.msg)) elif e of DatastoreError: (DatastoreErr, DataBuffer.new(e.msg)) elif e of CatchableError: (CatchableErr, DataBuffer.new(e.msg)) + elif e of Defect: (DefectErr, DataBuffer.new(e.msg)) else: raise (ref Defect)(msg: e.msg) converter toExc*(e: ThreadResErr): ref CatchableError = @@ -36,3 +38,4 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.QueryEndedErr: (ref QueryEndedError)(msg: $e[1]) of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) + of ErrorEnum.DefectErr: (ref CatchableError)(msg: "defect: " & $e[1]) From 8699cfe6719c1ca305856f4d5d6ae8ffdeb2ab9b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:44:00 -0700 Subject: [PATCH 280/445] setup query --- datastore/threads/threadproxyds.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8ad8ba29..67228c8e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -245,6 +245,9 @@ proc queryTask[DB]( for item in handle.iter(): executeTask(ctx): item + + executeTask(ctx): + (ref QueryEndedError)(msg: "done") method query*( self: ThreadDatastore, From f8f0a727d7b4f4749245a1365a61b75115e5c723 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 13:51:36 -0700 Subject: [PATCH 281/445] setup query --- datastore/threads/threadproxyds.nim | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 67228c8e..67ad9e60 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -280,15 +280,24 @@ method query*( trace "About to query" if lock.locked: return failure (ref DatastoreError)(msg: "Should always await query features") + if iter.finished == true: + return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") await lock.acquire() - if iter.finished == true: - return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + dispatchTask[void](self, signal): + discard ctx.signal.fireSync() - iter.finished = childIter.finished - var - res = ThreadResult[QueryResponse]() + let res = ctx.res + + if res.isErr() and res.error()[0] == ErrorEnum.QueryEndedErr: + iter.finished = true + else: + if res.isErr(): + return err(res.error()) + else: + let qres = res.get() + return ok(res.get()) iter.next = next From 2600d244f19a5eab7365207a8c9fb389206f9ad3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 14:02:31 -0700 Subject: [PATCH 282/445] setup query end --- datastore/threads/threadproxyds.nim | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 67ad9e60..304c735d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -245,9 +245,9 @@ proc queryTask[DB]( for item in handle.iter(): executeTask(ctx): item - + executeTask(ctx): - (ref QueryEndedError)(msg: "done") + (?!QResult).err((ref QueryEndedError)(msg: "done").toThreadErr()) method query*( self: ThreadDatastore, @@ -288,17 +288,14 @@ method query*( dispatchTask[void](self, signal): discard ctx.signal.fireSync() - let res = ctx.res - - if res.isErr() and res.error()[0] == ErrorEnum.QueryEndedErr: - iter.finished = true - else: - if res.isErr(): - return err(res.error()) - else: - let qres = res.get() - return ok(res.get()) - + # if ctx.res.isErr() and ctx.res.error()[0] == ErrorEnum.QueryEndedErr: + # iter.finished = true + # return + # elif ctx.res.isErr(): + # return err(ctx.res.error()) + # else: + # let qres = ctx.res.get() + # return ok(default) iter.next = next return success iter From 33ca4346e4265211924725bc0939a016cca49a3c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 14:09:38 -0700 Subject: [PATCH 283/445] setup query end --- datastore/threads/threadproxyds.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 304c735d..8fc0d1e4 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -288,14 +288,14 @@ method query*( dispatchTask[void](self, signal): discard ctx.signal.fireSync() - # if ctx.res.isErr() and ctx.res.error()[0] == ErrorEnum.QueryEndedErr: - # iter.finished = true - # return - # elif ctx.res.isErr(): - # return err(ctx.res.error()) - # else: - # let qres = ctx.res.get() - # return ok(default) + if ctx.res.isErr() and ctx.res.error()[0] == ErrorEnum.QueryEndedErr: + iter.finished = true + return + elif ctx.res.isErr(): + return err(ctx.res.error()) + else: + let qres = ctx.res.get() + return ok(default) iter.next = next return success iter From 173d42631a082fb84fce91dadc21cbe37f68c176 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 14:53:12 -0700 Subject: [PATCH 284/445] setup query end --- datastore/threads/threadproxyds.nim | 67 +++++++++++++++-------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8fc0d1e4..4117d00d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -15,6 +15,7 @@ import pkg/questionable import pkg/questionable/results import pkg/taskpools import pkg/chronicles +import pkg/threading/smartptrs import ../key import ../query @@ -42,12 +43,14 @@ type of Sqlite: sql*: SQLiteBackend[KeyId,DataBuffer] - TaskCtx[T: ThreadTypes] = object + TaskCtxObj[T: ThreadTypes] = object res: ThreadResult[T] signal: ThreadSignalPtr running: bool cancelled: bool + TaskCtx[T] = SharedPtr[TaskCtxObj[T]] + ThreadDatastore* = ref object of Datastore tp: Taskpool backend: ThreadBackend @@ -57,22 +60,22 @@ type var ctxLock: Lock ctxLock.initLock() -proc setCancelled(ctx: var TaskCtx): bool = +proc setCancelled[T](ctx: TaskCtx[T]): bool = withLock(ctxLock): - if ctx.running: + if ctx[].running: return false else: - ctx.cancelled = true + ctx[].cancelled = true return true -proc setRunning[T](ctx: ptr TaskCtx[T]): bool = +proc setRunning[T](ctx: TaskCtx[T]): bool = withLock(ctxLock): - if ctx.cancelled: + if ctx[].cancelled: return - ctx.running = true -proc setDone[T](ctx: ptr TaskCtx[T]) = + ctx[].running = true +proc setDone[T](ctx: TaskCtx[T]) = withLock(ctxLock): - ctx.running = false + ctx[].running = false proc acquireSignal(): ?!ThreadSignalPtr = let signal = ThreadSignalPtr.new() @@ -81,7 +84,7 @@ proc acquireSignal(): ?!ThreadSignalPtr = else: success signal.get() -template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = +template executeTask[T](ctx: TaskCtx[T], blk: untyped) = try: if not ctx.setRunning(): return @@ -90,11 +93,11 @@ template executeTask[T](ctx: ptr TaskCtx[T], blk: untyped) = let res = `blk` if res.isOk(): when T is void: - ctx.res.ok() + ctx[].res.ok() else: - ctx.res.ok(res.get()) + ctx[].res.ok(res.get()) else: - ctx.res.err res.error().toThreadErr() + ctx[].res.err res.error().toThreadErr() except CatchableError as exc: trace "Unexpected exception thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() @@ -110,7 +113,7 @@ template dispatchTask[T](self: ThreadDatastore, blk: untyped ): auto = var - ctx {.inject.} = TaskCtx[T](signal: signal) + ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) try: case self.backend.kind: of Sqlite: @@ -119,7 +122,7 @@ template dispatchTask[T](self: ThreadDatastore, `blk` runTask() - await wait(ctx.signal) + await wait(ctx[].signal) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg while not ctx.setCancelled(): @@ -127,10 +130,10 @@ template dispatchTask[T](self: ThreadDatastore, await sleepAsync(10.milliseconds) raise exc finally: - discard ctx.signal.close() + discard ctx[].signal.close() self.semaphore.release() -proc hasTask[T, DB](ctx: ptr TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = +proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): has(ds, key) @@ -143,9 +146,9 @@ method has*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[bool](self, signal): - self.tp.spawn hasTask(addr ctx, ds, key) + self.tp.spawn hasTask(ctx, ds, key) -proc deleteTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; +proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -159,7 +162,7 @@ method delete*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[void](self, signal): - self.tp.spawn deleteTask(addr ctx, ds, key) + self.tp.spawn deleteTask(ctx, ds, key) method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = @@ -170,7 +173,7 @@ method delete*(self: ThreadDatastore, return success() -proc putTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; +proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId, data: DataBuffer) {.gcsafe, nimcall.} = ## run backend command @@ -187,7 +190,7 @@ method put*(self: ThreadDatastore, let key = KeyId.new key.id() let data = DataBuffer.new data dispatchTask[void](self, signal): - self.tp.spawn putTask(addr ctx, ds, key, data) + self.tp.spawn putTask(ctx, ds, key, data) method put*( self: ThreadDatastore, @@ -199,7 +202,7 @@ method put*( return success() -proc getTask[T, DB](ctx: ptr TaskCtx[T], ds: DB; +proc getTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe, nimcall.} = ## run backend command executeTask(ctx): @@ -214,7 +217,7 @@ method get*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[void](self, signal): - self.tp.spawn getTask(addr ctx, ds, key) + self.tp.spawn getTask(ctx, ds, key) method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.semaphore.closeAll() @@ -226,7 +229,7 @@ type QResult = DbQueryResponse[KeyId, DataBuffer] proc queryTask[DB]( - ctx: ptr TaskCtx[QResult], + ctx: TaskCtx[QResult], ds: DB, dq: DbQuery[KeyId] ) {.gcsafe, nimcall.} = @@ -266,7 +269,7 @@ method query*( ) dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn queryTask(addr ctx, ds, dq) + self.tp.spawn queryTask(ctx, ds, dq) var lock = newAsyncLock() # serialize querying under threads @@ -286,16 +289,16 @@ method query*( await lock.acquire() dispatchTask[void](self, signal): - discard ctx.signal.fireSync() + discard ctx[].signal.fireSync() - if ctx.res.isErr() and ctx.res.error()[0] == ErrorEnum.QueryEndedErr: + if ctx[].res.isErr() and ctx[].res.error()[0] == ErrorEnum.QueryEndedErr: iter.finished = true return - elif ctx.res.isErr(): - return err(ctx.res.error()) + elif ctx[].res.isErr(): + return err(ctx[].res.error()) else: - let qres = ctx.res.get() - return ok(default) + # let qres = ctx[].res.get() + return (?!QueryResponse).ok(default(QueryResponse)) iter.next = next return success iter From 4be88df7154a96097d5d511bc313056e944573b2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:04:18 -0700 Subject: [PATCH 285/445] setup query end --- datastore/threads/threadproxyds.nim | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4117d00d..055e5d0d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -237,10 +237,8 @@ proc queryTask[DB]( var qh: typeof(ds.query(dq)) executeTask(ctx): qh = ds.query(dq) - if qh.isOk(): - (?!QResult).ok(default(QResult)) - else: - (?!QResult).err(qh.error()) + if qh.isOk(): (?!QResult).ok(default(QResult)) + else: (?!QResult).err(qh.error()) if qh.isErr(): return @@ -254,19 +252,15 @@ proc queryTask[DB]( method query*( self: ThreadDatastore, - query: Query): Future[?!QueryIter] {.async.} = + q: Query): Future[?!QueryIter] {.async.} = await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err let dq = dbQuery( - key= KeyId.new query.key.id(), - value=query.value, - limit=query.limit, - offset=query.offset, - sort=query.sort, - ) + key= KeyId.new q.key.id(), + value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): self.tp.spawn queryTask(ctx, ds, dq) @@ -288,17 +282,19 @@ method query*( await lock.acquire() - dispatchTask[void](self, signal): - discard ctx[].signal.fireSync() + discard ctx[].signal.fireSync() + await ctx[].signal.wait() if ctx[].res.isErr() and ctx[].res.error()[0] == ErrorEnum.QueryEndedErr: iter.finished = true - return + return success (key: Key.none, data: EmptyBytes) elif ctx[].res.isErr(): return err(ctx[].res.error()) else: - # let qres = ctx[].res.get() - return (?!QueryResponse).ok(default(QueryResponse)) + let qres = ctx[].res.get() + let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) + let data = qres.data.toSeq() + return (?!QueryResponse).ok((key: key, data: data)) iter.next = next return success iter From 77a147cad27fc4a1dec73b6a6b60170eed9e9168 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:09:09 -0700 Subject: [PATCH 286/445] setup query end --- datastore/threads/threadproxyds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 055e5d0d..a95b9cd0 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -245,6 +245,7 @@ proc queryTask[DB]( var handle = qh.get() for item in handle.iter(): executeTask(ctx): + discard ctx[].signal.waitSync().get() item executeTask(ctx): From b6baefc19c85214cdbd48cf44272f909a464c56b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:29:35 -0700 Subject: [PATCH 287/445] setup query end --- datastore/threads/threadproxyds.nim | 45 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index a95b9cd0..f4b6b595 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -60,13 +60,9 @@ type var ctxLock: Lock ctxLock.initLock() -proc setCancelled[T](ctx: TaskCtx[T]): bool = +proc setCancelled[T](ctx: TaskCtx[T]) = withLock(ctxLock): - if ctx[].running: - return false - else: - ctx[].cancelled = true - return true + ctx[].cancelled = true proc setRunning[T](ctx: TaskCtx[T]): bool = withLock(ctxLock): @@ -125,9 +121,7 @@ template dispatchTask[T](self: ThreadDatastore, await wait(ctx[].signal) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg - while not ctx.setCancelled(): - warn "waiting to cancel thread future!", fn = astToStr(fn) - await sleepAsync(10.milliseconds) + ctx.setCancelled() raise exc finally: discard ctx[].signal.close() @@ -234,21 +228,28 @@ proc queryTask[DB]( dq: DbQuery[KeyId] ) {.gcsafe, nimcall.} = ## run query command - var qh: typeof(ds.query(dq)) executeTask(ctx): - qh = ds.query(dq) - if qh.isOk(): (?!QResult).ok(default(QResult)) - else: (?!QResult).err(qh.error()) - if qh.isErr(): - return - - var handle = qh.get() - for item in handle.iter(): - executeTask(ctx): - discard ctx[].signal.waitSync().get() - item + let qh = ds.query(dq) + if qh.isOk(): + ctx[].res.ok default(QResult) + else: + ctx[].res.err qh.error().toThreadErr() + return + + var handle = qh.get() + + for item in handle.iter(): + if ctx[].cancelled: + # cancel iter, then run next cycle so it'll finish and close + handle.cancel = true + continue + else: + # wait for next request from async thread + discard ctx[].signal.waitSync().get() + ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: + exc + discard ctx[].signal.fireSync() - executeTask(ctx): (?!QResult).err((ref QueryEndedError)(msg: "done").toThreadErr()) method query*( From eec66c9238e151a6e6044ec3db463adc9373d073 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:37:19 -0700 Subject: [PATCH 288/445] setup query end --- datastore/threads/threadproxyds.nim | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index f4b6b595..ce808bac 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -104,12 +104,10 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = ctx.setDone() discard ctx[].signal.fireSync() -template dispatchTask[T](self: ThreadDatastore, +template dispatchTaskWrap[T](self: ThreadDatastore, signal: ThreadSignalPtr, blk: untyped ): auto = - var - ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) try: case self.backend.kind: of Sqlite: @@ -127,6 +125,13 @@ template dispatchTask[T](self: ThreadDatastore, discard ctx[].signal.close() self.semaphore.release() +template dispatchTask[T](self: ThreadDatastore, + signal: ThreadSignalPtr, + blk: untyped + ): auto = + let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) + dispatchTaskWrap[T](self, signal, blk) + proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -284,6 +289,8 @@ method query*( await lock.acquire() + dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): + self.tp.spawn queryTask(ctx, ds, dq) discard ctx[].signal.fireSync() await ctx[].signal.wait() From 911de835375e5f79174487351961b9e1e9cf80b6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:40:28 -0700 Subject: [PATCH 289/445] setup query end --- datastore/threads/threadproxyds.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ce808bac..1e73d34c 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -277,6 +277,7 @@ method query*( iter = QueryIter.new() proc next(): Future[?!QueryResponse] {.async.} = + let ctx = ctx defer: if lock.locked: lock.release() @@ -289,10 +290,9 @@ method query*( await lock.acquire() - dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn queryTask(ctx, ds, dq) - discard ctx[].signal.fireSync() - await ctx[].signal.wait() + dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): + # trigger query task to iterate then wait for new result! + discard ctx[].signal.fireSync() if ctx[].res.isErr() and ctx[].res.error()[0] == ErrorEnum.QueryEndedErr: iter.finished = true From 9c6f2fc74a477638993beb0d90b7252f5710eff0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:50:22 -0700 Subject: [PATCH 290/445] setup query --- datastore/threads/threadproxyds.nim | 34 +++++++++++++++-------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 1e73d34c..bcefd78d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -235,27 +235,29 @@ proc queryTask[DB]( ## run query command executeTask(ctx): let qh = ds.query(dq) - if qh.isOk(): - ctx[].res.ok default(QResult) + if qh.isErr(): + # set error and exit, which will fire final signal + (?!QResult).err(qh.error()) else: - ctx[].res.err qh.error().toThreadErr() - return - - var handle = qh.get() + # otherwise set empty ok result + ctx[].res.ok (KeyId.none, DataBuffer.new()) + discard ctx[].signal.fireSync() - for item in handle.iter(): - if ctx[].cancelled: - # cancel iter, then run next cycle so it'll finish and close - handle.cancel = true - continue - else: + var handle = qh.get() + for item in handle.iter(): # wait for next request from async thread discard ctx[].signal.waitSync().get() - ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: - exc - discard ctx[].signal.fireSync() - (?!QResult).err((ref QueryEndedError)(msg: "done").toThreadErr()) + if ctx[].cancelled: + # cancel iter, then run next cycle so it'll finish and close + handle.cancel = true + continue + else: + ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: + exc + discard ctx[].signal.fireSync() + + (?!QResult).err((ref QueryEndedError)(msg: "done").toThreadErr()) method query*( self: ThreadDatastore, From 5796d4dfa1c530f637253933e4953b2a3e8ef2ba Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:54:43 -0700 Subject: [PATCH 291/445] setup query --- datastore/threads/threadproxyds.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index bcefd78d..93782c01 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -234,12 +234,14 @@ proc queryTask[DB]( ) {.gcsafe, nimcall.} = ## run query command executeTask(ctx): + # we execute this all inside `executeTask` + # so we need to return a final result let qh = ds.query(dq) if qh.isErr(): - # set error and exit, which will fire final signal + # set error and exit executeTask, which will fire final signal (?!QResult).err(qh.error()) else: - # otherwise set empty ok result + # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer.new()) discard ctx[].signal.fireSync() @@ -257,7 +259,8 @@ proc queryTask[DB]( exc discard ctx[].signal.fireSync() - (?!QResult).err((ref QueryEndedError)(msg: "done").toThreadErr()) + # set final result + (?!QResult).ok((KeyId.none, DataBuffer.new())) method query*( self: ThreadDatastore, From abfd12f7a60ea9dc1e9328373db1d77e72bff720 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 15:56:09 -0700 Subject: [PATCH 292/445] setup query --- datastore/threads/threadproxyds.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 93782c01..77c4f1f6 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -299,10 +299,10 @@ method query*( # trigger query task to iterate then wait for new result! discard ctx[].signal.fireSync() - if ctx[].res.isErr() and ctx[].res.error()[0] == ErrorEnum.QueryEndedErr: + if not ctx[].running: iter.finished = true - return success (key: Key.none, data: EmptyBytes) - elif ctx[].res.isErr(): + + if ctx[].res.isErr(): return err(ctx[].res.error()) else: let qres = ctx[].res.get() From 892ec385eeda84e125ddde8949adb43f073088e4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:02:42 -0700 Subject: [PATCH 293/445] fixup databuffer --- datastore/threads/databuffer.nim | 6 ++++-- datastore/threads/threadproxyds.nim | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index be08c02d..9e192e70 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -27,8 +27,10 @@ proc `=destroy`*(x: var DataBufferHolder) = # echo "buffer: FREE: ", repr x.buf.pointer deallocShared(x.buf) -proc len*(a: DataBuffer): int = a[].size -proc capacity*(a: DataBuffer): int = a[].cap +proc len*(a: DataBuffer): int = + if a.isNil: 0 else: a[].size +proc capacity*(a: DataBuffer): int = + if a.isNil: 0 else: a[].cap proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 77c4f1f6..ad77bf0e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -242,7 +242,7 @@ proc queryTask[DB]( (?!QResult).err(qh.error()) else: # otherwise manually an set empty ok result - ctx[].res.ok (KeyId.none, DataBuffer.new()) + ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() var handle = qh.get() @@ -260,7 +260,7 @@ proc queryTask[DB]( discard ctx[].signal.fireSync() # set final result - (?!QResult).ok((KeyId.none, DataBuffer.new())) + (?!QResult).ok((KeyId.none, DataBuffer())) method query*( self: ThreadDatastore, @@ -313,16 +313,17 @@ method query*( iter.next = next return success iter -proc new*( - self: type ThreadDatastore, - ds: Datastore, - withLocks = static false, - tp: Taskpool): ?!ThreadDatastore = +proc new*(self: type ThreadDatastore, + backend: ThreadBackendKinds, + withLocks = static false, + tp: Taskpool + ): ?!ThreadDatastore = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" success ThreadDatastore( tp: tp, - ds: ds, + ds: ThreadBackend(), withLocks: withLocks, queryLock: newAsyncLock(), - semaphore: AsyncSemaphore.new(tp.numThreads - 1)) + semaphore: AsyncSemaphore.new(tp.numThreads - 1) + ) From 555f1fe77a4ee260546b864c0d8947a2ab4b8cef Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:12:55 -0700 Subject: [PATCH 294/445] fixup databuffer --- datastore/threads/threadproxyds.nim | 19 +- tests/datastore/testthreadproxyds.nim | 238 +++++++++++++------------- 2 files changed, 130 insertions(+), 127 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ad77bf0e..e4b98ecc 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -43,13 +43,13 @@ type of Sqlite: sql*: SQLiteBackend[KeyId,DataBuffer] - TaskCtxObj[T: ThreadTypes] = object + TaskCtxObj*[T: ThreadTypes] = object res: ThreadResult[T] signal: ThreadSignalPtr running: bool cancelled: bool - TaskCtx[T] = SharedPtr[TaskCtxObj[T]] + TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] ThreadDatastore* = ref object of Datastore tp: Taskpool @@ -313,17 +313,22 @@ method query*( iter.next = next return success iter -proc new*(self: type ThreadDatastore, - backend: ThreadBackendKinds, +proc new*[DB](self: type ThreadDatastore, + db: DB, withLocks = static false, tp: Taskpool ): ?!ThreadDatastore = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" + when DB is SQLiteBackend[KeyId,DataBuffer]: + let backend = ThreadBackend(kind: Sqlite, sql: db) + else: + {.error: "unsupported backend: " & $typeof(db).} + success ThreadDatastore( tp: tp, - ds: ThreadBackend(), - withLocks: withLocks, - queryLock: newAsyncLock(), + backend: backend, + # withLocks: withLocks, + # queryLock: newAsyncLock(), semaphore: AsyncSemaphore.new(tp.numThreads - 1) ) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index e81a2356..3c76ce03 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -13,8 +13,9 @@ import pkg/stew/byteutils import pkg/taskpools import pkg/questionable/results import pkg/chronicles +import pkg/threading/smartptrs -import pkg/datastore/sql +import pkg/datastore/sql/sqliteds import pkg/datastore/fsds import pkg/datastore/threads/threadproxyds {.all.} @@ -26,7 +27,7 @@ const NumThreads = 200 # IO threads aren't attached to CPU count suite "Test Basic ThreadDatastore with SQLite": var - sqlStore: Datastore + sqlStore: SQLiteBackend[KeyId,DataBuffer] ds: ThreadDatastore taskPool: Taskpool key = Key.init("/a/b").tryGet() @@ -34,7 +35,7 @@ suite "Test Basic ThreadDatastore with SQLite": otherBytes = "some other bytes".toBytes setupAll: - sqlStore = SQLiteDatastore.new(Memory).tryGet() + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() @@ -50,7 +51,7 @@ suite "Test Basic ThreadDatastore with SQLite": suite "Test Query ThreadDatastore with SQLite": var - sqlStore: Datastore + sqlStore: SQLiteBackend[KeyId,DataBuffer] ds: ThreadDatastore taskPool: Taskpool key = Key.init("/a/b").tryGet() @@ -58,7 +59,7 @@ suite "Test Query ThreadDatastore with SQLite": otherBytes = "some other bytes".toBytes setup: - sqlStore = SQLiteDatastore.new(Memory).tryGet() + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() @@ -70,74 +71,74 @@ suite "Test Query ThreadDatastore with SQLite": queryTests(ds, true) -suite "Test Basic ThreadDatastore with fsds": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes +# suite "Test Basic ThreadDatastore with fsds": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes - var - fsStore: FSDatastore - ds: ThreadDatastore - taskPool: Taskpool +# var +# fsStore: FSDatastore +# ds: ThreadDatastore +# taskPool: Taskpool - setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) +# setupAll: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) - fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() +# fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() - teardown: - GC_fullCollect() +# teardown: +# GC_fullCollect() - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() +# teardownAll: +# (await ds.close()).tryGet() +# taskPool.shutdown() - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) - basicStoreTests(fsStore, key, bytes, otherBytes) +# basicStoreTests(fsStore, key, bytes, otherBytes) -suite "Test Query ThreadDatastore with fsds": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath +# suite "Test Query ThreadDatastore with fsds": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath - var - fsStore: FSDatastore - ds: ThreadDatastore - taskPool: Taskpool +# var +# fsStore: FSDatastore +# ds: ThreadDatastore +# taskPool: Taskpool - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) +# setup: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) - fsStore = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() +# fsStore = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() - teardown: - GC_fullCollect() - (await ds.close()).tryGet() - taskPool.shutdown() +# teardown: +# GC_fullCollect() +# (await ds.close()).tryGet() +# taskPool.shutdown() - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) - queryTests(ds, false) +# queryTests(ds, false) suite "Test ThreadDatastore cancelations": var - sqlStore: Datastore + sqlStore: SQLiteBackend[KeyId,DataBuffer] ds: ThreadDatastore taskPool: Taskpool @@ -145,7 +146,7 @@ suite "Test ThreadDatastore cancelations": privateAccess(TaskCtx) # expose private fields setupAll: - sqlStore = SQLiteDatastore.new(Memory).tryGet() + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() @@ -156,66 +157,63 @@ suite "Test ThreadDatastore cancelations": (await ds.close()).tryGet() taskPool.shutdown() - test "Should monitor signal and cancel": - var - signal = ThreadSignalPtr.new().tryGet() - res = ThreadResult[void]() - ctx = TaskCtx[void]( - ds: sqlStore, - res: addr res, - signal: signal) - fut = newFuture[void]("signalMonitor") - threadArgs = (addr ctx, addr fut) - thread: Thread[type threadArgs] - - proc threadTask(args: type threadArgs) = - var (ctx, fut) = args - proc asyncTask() {.async.} = - let - monitor = signalMonitor(ctx, fut[]) - - await monitor - - waitFor asyncTask() - - createThread(thread, threadTask, threadArgs) - ctx.cancelled = true - check: ctx.signal.fireSync.tryGet - - joinThreads(thread) - - check: fut.cancelled - check: ctx.signal.close().isOk - fut = nil - - test "Should monitor and not cancel": - var - signal = ThreadSignalPtr.new().tryGet() - res = ThreadResult[void]() - ctx = TaskCtx[void]( - ds: sqlStore, - res: addr res, - signal: signal) - fut = newFuture[void]("signalMonitor") - threadArgs = (addr ctx, addr fut) - thread: Thread[type threadArgs] - - proc threadTask(args: type threadArgs) = - var (ctx, fut) = args - proc asyncTask() {.async.} = - let - monitor = signalMonitor(ctx, fut[]) - - await monitor - - waitFor asyncTask() - - createThread(thread, threadTask, threadArgs) - ctx.cancelled = false - check: ctx.signal.fireSync.tryGet - - joinThreads(thread) - - check: not fut.cancelled - check: ctx.signal.close().isOk - fut = nil + # test "Should monitor signal and cancel": + # var + # signal = ThreadSignalPtr.new().tryGet() + # res = ThreadResult[void]() + # ctx = newSharedPtr(TaskCtxObj[void](signal: signal)) + # fut = newFuture[void]("signalMonitor") + # threadArgs = (addr ctx, addr fut) + # thread: Thread[type threadArgs] + + # proc threadTask(args: type threadArgs) = + # var (ctx, fut) = args + # proc asyncTask() {.async.} = + # let + # monitor = signalMonitor(ctx, fut[]) + + # await monitor + + # waitFor asyncTask() + + # createThread(thread, threadTask, threadArgs) + # ctx.cancelled = true + # check: ctx.signal.fireSync.tryGet + + # joinThreads(thread) + + # check: fut.cancelled + # check: ctx.signal.close().isOk + # fut = nil + + # test "Should monitor and not cancel": + # var + # signal = ThreadSignalPtr.new().tryGet() + # res = ThreadResult[void]() + # ctx = TaskCtx[void]( + # ds: sqlStore, + # res: addr res, + # signal: signal) + # fut = newFuture[void]("signalMonitor") + # threadArgs = (addr ctx, addr fut) + # thread: Thread[type threadArgs] + + # proc threadTask(args: type threadArgs) = + # var (ctx, fut) = args + # proc asyncTask() {.async.} = + # let + # monitor = signalMonitor(ctx, fut[]) + + # await monitor + + # waitFor asyncTask() + + # createThread(thread, threadTask, threadArgs) + # ctx.cancelled = false + # check: ctx.signal.fireSync.tryGet + + # joinThreads(thread) + + # check: not fut.cancelled + # check: ctx.signal.close().isOk + # fut = nil From 61e8fd49041456629bcd77771424d3463e3fb885 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:13:11 -0700 Subject: [PATCH 295/445] fixup databuffer --- tests/datastore/testthreadproxyds.nim | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 3c76ce03..a6286b8d 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -48,28 +48,28 @@ suite "Test Basic ThreadDatastore with SQLite": basicStoreTests(ds, key, bytes, otherBytes) -suite "Test Query ThreadDatastore with SQLite": +# suite "Test Query ThreadDatastore with SQLite": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes +# var +# sqlStore: SQLiteBackend[KeyId,DataBuffer] +# ds: ThreadDatastore +# taskPool: Taskpool +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes - setup: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() +# setup: +# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - teardown: - GC_fullCollect() +# teardown: +# GC_fullCollect() - (await ds.close()).tryGet() - taskPool.shutdown() +# (await ds.close()).tryGet() +# taskPool.shutdown() - queryTests(ds, true) +# queryTests(ds, true) # suite "Test Basic ThreadDatastore with fsds": # let From 6e9325ed26faa38236184a64d40351f4e4e6e78e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:22:41 -0700 Subject: [PATCH 296/445] testing --- datastore/threads/threadproxyds.nim | 9 +++++ tests/datastore/testthreadproxyds.nim | 56 +++++++++++++++++++++------ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e4b98ecc..fd0d3842 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -177,6 +177,10 @@ proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; data: DataBuffer) {.gcsafe, nimcall.} = ## run backend command executeTask(ctx): + echo "putTask:key: ", key + echo "putTask:data: ", data + echo "putTask:ctx: ", ctx.repr() + echo "" put(ds, key, data) method put*(self: ThreadDatastore, @@ -188,7 +192,12 @@ method put*(self: ThreadDatastore, let key = KeyId.new key.id() let data = DataBuffer.new data + dispatchTask[void](self, signal): + echo "put:key: ", key + echo "put:data: ", data + echo "put:ctx: ", ctx.repr() + echo "" self.tp.spawn putTask(ctx, ds, key, data) method put*( diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index a6286b8d..d2b2ec04 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,29 +24,63 @@ import ./querycommontests const NumThreads = 200 # IO threads aren't attached to CPU count -suite "Test Basic ThreadDatastore with SQLite": - +suite "Test Basic ThreadProxyDatastore": var sqlStore: SQLiteBackend[KeyId,DataBuffer] ds: ThreadDatastore taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes + key = Key.init("/a").tryGet() + data = "some bytes".toBytes setupAll: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - teardown: - GC_fullCollect() - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() + echo "teardown done" + + test "check put": + echo "\n\n=== put ===" + let res1 = await ds.put(key, data) + echo "res1: ", res1.repr + check res1.isOk + + test "check get": + echo "\n\n=== get ===" + echo "get send key: ", key.repr + let res2 = await ds.get(key) + echo "get key post: ", key.repr + echo "get res2: ", res2.repr + echo res2.get() == data + var val = "" + for c in res2.get(): + val &= char(c) + echo "get res2: ", $val + +# suite "Test Basic ThreadDatastore with SQLite": + +# var +# sqlStore: SQLiteBackend[KeyId,DataBuffer] +# ds: ThreadDatastore +# taskPool: Taskpool +# key = Key.init("/a/b").tryGet() +# bytes = "some bytes".toBytes +# otherBytes = "some other bytes".toBytes + +# setupAll: +# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + +# teardown: +# GC_fullCollect() + +# teardownAll: +# (await ds.close()).tryGet() +# taskPool.shutdown() - basicStoreTests(ds, key, bytes, otherBytes) +# basicStoreTests(ds, key, bytes, otherBytes) # suite "Test Query ThreadDatastore with SQLite": From 54a04127a8f124086c45943854227f3df9f5de66 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:23:33 -0700 Subject: [PATCH 297/445] testing --- tests/datastore/testthreadproxyds.nim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index d2b2ec04..046773d4 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -46,17 +46,17 @@ suite "Test Basic ThreadProxyDatastore": echo "res1: ", res1.repr check res1.isOk - test "check get": - echo "\n\n=== get ===" - echo "get send key: ", key.repr - let res2 = await ds.get(key) - echo "get key post: ", key.repr - echo "get res2: ", res2.repr - echo res2.get() == data - var val = "" - for c in res2.get(): - val &= char(c) - echo "get res2: ", $val + # test "check get": + # echo "\n\n=== get ===" + # echo "get send key: ", key.repr + # let res2 = await ds.get(key) + # echo "get key post: ", key.repr + # echo "get res2: ", res2.repr + # echo res2.get() == data + # var val = "" + # for c in res2.get(): + # val &= char(c) + # echo "get res2: ", $val # suite "Test Basic ThreadDatastore with SQLite": From c31f1890b2ec7d9fbd038aaf30601e82f35dc6bd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 16:38:46 -0700 Subject: [PATCH 298/445] testing --- datastore/threads/threadproxyds.nim | 30 ++++++++++++++++++++----- datastore/threads/threadresult.nim | 4 ++++ tests/datastore/testthreadproxyds.nim | 32 +++++++++++++-------------- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index fd0d3842..03877815 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -67,8 +67,9 @@ proc setCancelled[T](ctx: TaskCtx[T]) = proc setRunning[T](ctx: TaskCtx[T]): bool = withLock(ctxLock): if ctx[].cancelled: - return + return false ctx[].running = true + return true proc setDone[T](ctx: TaskCtx[T]) = withLock(ctxLock): ctx[].running = false @@ -82,18 +83,26 @@ proc acquireSignal(): ?!ThreadSignalPtr = template executeTask[T](ctx: TaskCtx[T], blk: untyped) = try: + echo "executeTask:start:" if not ctx.setRunning(): + echo "executeTask:notRunning!" return ## run backend command + echo "executeTask:run:" let res = `blk` if res.isOk(): + echo "executeTask:run:ok" when T is void: ctx[].res.ok() else: ctx[].res.ok(res.get()) else: + echo "executeTask:run:err" ctx[].res.err res.error().toThreadErr() + echo "executeTask:run:done: ", ctx[].res.repr + echo "" + except CatchableError as exc: trace "Unexpected exception thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() @@ -101,8 +110,10 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = trace "Unexpected defect thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() finally: + echo "executeTask:finally:setDone" ctx.setDone() discard ctx[].signal.fireSync() + echo "executeTask:finally:done\n" template dispatchTaskWrap[T](self: ThreadDatastore, signal: ThreadSignalPtr, @@ -111,12 +122,15 @@ template dispatchTaskWrap[T](self: ThreadDatastore, try: case self.backend.kind: of Sqlite: - var ds {.inject.} = self.backend.sql + echo "dispatchTask:sql:" + var ds {.used, inject.} = self.backend.sql proc runTask() = `blk` runTask() - + echo "dispatchTask:wait:start" await wait(ctx[].signal) + echo "dispatchTask:wait:done" + except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() @@ -172,16 +186,19 @@ method delete*(self: ThreadDatastore, return success() + proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; - key: KeyId, - data: DataBuffer) {.gcsafe, nimcall.} = + key: KeyId, + data: DataBuffer) {.gcsafe, nimcall.} = ## run backend command + echo "\n\nputTask:start " executeTask(ctx): echo "putTask:key: ", key echo "putTask:data: ", data echo "putTask:ctx: ", ctx.repr() echo "" put(ds, key, data) + echo "putTask:done" method put*(self: ThreadDatastore, key: Key, @@ -200,6 +217,8 @@ method put*(self: ThreadDatastore, echo "" self.tp.spawn putTask(ctx, ds, key, data) + return ctx[].res + method put*( self: ThreadDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = @@ -208,7 +227,6 @@ method put*( if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err - return success() proc getTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe, nimcall.} = diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index b1694f42..f0a226bb 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -39,3 +39,7 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.DatastoreErr: (ref DatastoreError)(msg: $e[1]) of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) of ErrorEnum.DefectErr: (ref CatchableError)(msg: "defect: " & $e[1]) + +converter toExcRes*[T](res: ThreadResult[T]): ?!T = + res.mapErr() do(exc: ThreadResErr) -> ref CatchableError: + exc.toExc() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 046773d4..93eb84bf 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -170,26 +170,26 @@ suite "Test Basic ThreadProxyDatastore": # queryTests(ds, false) -suite "Test ThreadDatastore cancelations": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool +# suite "Test ThreadDatastore cancelations": +# var +# sqlStore: SQLiteBackend[KeyId,DataBuffer] +# ds: ThreadDatastore +# taskPool: Taskpool - privateAccess(ThreadDatastore) # expose private fields - privateAccess(TaskCtx) # expose private fields +# privateAccess(ThreadDatastore) # expose private fields +# privateAccess(TaskCtx) # expose private fields - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() +# setupAll: +# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() +# taskPool = Taskpool.new(NumThreads) +# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - teardown: - GC_fullCollect() # run full collect after each test +# teardown: +# GC_fullCollect() # run full collect after each test - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() +# teardownAll: +# (await ds.close()).tryGet() +# taskPool.shutdown() # test "Should monitor signal and cancel": # var From 36e6679a799dc10c5407fb828f5ce1f84f2fedc0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:01:21 -0700 Subject: [PATCH 299/445] fixup backend types --- datastore/query.nim | 2 +- datastore/sql/sqliteds.nim | 5 ++--- datastore/threads/databuffer.nim | 4 ++-- datastore/threads/threadproxyds.nim | 26 +++++++------------------- tests/datastore/testthreadproxyds.nim | 22 +++++++++++----------- 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/datastore/query.nim b/datastore/query.nim index 1295ff99..477d04d1 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -44,4 +44,4 @@ proc init*(T: type Query, dbQuery[Key](key, value, sort, offset, limit) proc toKey*(key: KeyId): Key {.inline, raises: [].} = - Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") + Key.init($key.data).expect("expected valid key here for but got `" & $key.data & "`") diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 8b07c392..feb0785b 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -58,13 +58,12 @@ proc delete*[K,V](self: SQLiteBackend[K,V], keys: openArray[K]): ?!void = return success() -proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!seq[byte] = +proc get*[K,V](self: SQLiteBackend[K,V], key: K): ?!V = # see comment in ./filesystem_datastore re: finer control of memory # allocation in `proc get`, could apply here as well if bytes were read # incrementally with `sqlite3_blob_read` - var - bytes: seq[byte] + var bytes: V proc onData(s: RawStmtPtr) = bytes = dataCol[V](self.db.getDataCol) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 9e192e70..5ea7b4be 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -87,7 +87,7 @@ proc setData*[T: byte | char](db: DataBuffer, data: openArray[T]) = copyMem(db[].buf, baseAddr data, data.len()) db[].size = data.len() -converter toSeq*(self: DataBuffer): seq[byte] = +proc toSeq*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char ## @@ -102,7 +102,7 @@ proc `@`*(self: DataBuffer): seq[byte] = self.toSeq() -converter toString*(data: DataBuffer): string = +proc toString*(data: DataBuffer): string = ## convert buffer to string type using copy ## diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 03877815..6582c0e6 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -122,14 +122,11 @@ template dispatchTaskWrap[T](self: ThreadDatastore, try: case self.backend.kind: of Sqlite: - echo "dispatchTask:sql:" var ds {.used, inject.} = self.backend.sql proc runTask() = `blk` runTask() - echo "dispatchTask:wait:start" await wait(ctx[].signal) - echo "dispatchTask:wait:done" except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg @@ -191,14 +188,8 @@ proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId, data: DataBuffer) {.gcsafe, nimcall.} = ## run backend command - echo "\n\nputTask:start " executeTask(ctx): - echo "putTask:key: ", key - echo "putTask:data: ", data - echo "putTask:ctx: ", ctx.repr() - echo "" put(ds, key, data) - echo "putTask:done" method put*(self: ThreadDatastore, key: Key, @@ -209,14 +200,8 @@ method put*(self: ThreadDatastore, let key = KeyId.new key.id() let data = DataBuffer.new data - dispatchTask[void](self, signal): - echo "put:key: ", key - echo "put:data: ", data - echo "put:ctx: ", ctx.repr() - echo "" self.tp.spawn putTask(ctx, ds, key, data) - return ctx[].res method put*( @@ -227,12 +212,14 @@ method put*( if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err - -proc getTask[T, DB](ctx: TaskCtx[T], ds: DB; +proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; key: KeyId) {.gcsafe, nimcall.} = ## run backend command executeTask(ctx): - get(ds, key) + let res = get(ds, key) + static: + echo "getTask:type: ", res.typeof + res method get*(self: ThreadDatastore, key: Key, @@ -242,8 +229,9 @@ method get*(self: ThreadDatastore, return failure err let key = KeyId.new key.id() - dispatchTask[void](self, signal): + dispatchTask[DataBuffer](self, signal): self.tp.spawn getTask(ctx, ds, key) + # return ctx[].res method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.semaphore.closeAll() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 93eb84bf..a9ae8438 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -46,17 +46,17 @@ suite "Test Basic ThreadProxyDatastore": echo "res1: ", res1.repr check res1.isOk - # test "check get": - # echo "\n\n=== get ===" - # echo "get send key: ", key.repr - # let res2 = await ds.get(key) - # echo "get key post: ", key.repr - # echo "get res2: ", res2.repr - # echo res2.get() == data - # var val = "" - # for c in res2.get(): - # val &= char(c) - # echo "get res2: ", $val + test "check get": + echo "\n\n=== get ===" + echo "get send key: ", key.repr + let res2 = await ds.get(key) + echo "get key post: ", key.repr + echo "get res2: ", res2.repr + echo res2.get() == data + var val = "" + for c in res2.get(): + val &= char(c) + echo "get res2: ", $val # suite "Test Basic ThreadDatastore with SQLite": From cfc743ce94c296d225d229ceeccfe0a2804aa0ab Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:16:13 -0700 Subject: [PATCH 300/445] result types --- datastore/threads/threadproxyds.nim | 5 +++-- datastore/threads/threadresult.nim | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 6582c0e6..d77ec494 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -202,7 +202,7 @@ method put*(self: ThreadDatastore, let data = DataBuffer.new data dispatchTask[void](self, signal): self.tp.spawn putTask(ctx, ds, key, data) - return ctx[].res + return ctx[].res.toRes() method put*( self: ThreadDatastore, @@ -231,7 +231,8 @@ method get*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[DataBuffer](self, signal): self.tp.spawn getTask(ctx, ds, key) - # return ctx[].res + return ctx[].res.toRes() do(v: DataBuffer) -> seq[byte]: + v.toSeq() method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.semaphore.closeAll() diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index f0a226bb..7754f121 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -40,6 +40,12 @@ converter toExc*(e: ThreadResErr): ref CatchableError = of ErrorEnum.CatchableErr: (ref CatchableError)(msg: $e[1]) of ErrorEnum.DefectErr: (ref CatchableError)(msg: "defect: " & $e[1]) -converter toExcRes*[T](res: ThreadResult[T]): ?!T = - res.mapErr() do(exc: ThreadResErr) -> ref CatchableError: - exc.toExc() +proc toRes*(res: ThreadResult[void]): ?!void = + res.mapErr() do(e: ThreadResErr) -> ref CatchableError: + e.toExc() + +proc toRes*[T,S](res: ThreadResult[T], m: proc(v: T): S = proc(v: T): T = v): ?!S = + if res.isErr(): + result.err res.error().toExc() + else: + result.ok m(res.get()) From e75081f832c470c340c9b766a0eee32dfbf997d5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:23:42 -0700 Subject: [PATCH 301/445] result types --- datastore/threads/threadproxyds.nim | 5 +++-- datastore/threads/threadresult.nim | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d77ec494..7d6541d2 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -8,6 +8,8 @@ push: {.upraises: [].} import std/tables import std/locks +import std/sugar + import pkg/chronos import pkg/chronos/threadsync @@ -231,8 +233,7 @@ method get*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[DataBuffer](self, signal): self.tp.spawn getTask(ctx, ds, key) - return ctx[].res.toRes() do(v: DataBuffer) -> seq[byte]: - v.toSeq() + return ctx[].res.toRes(v => v.toSeq()) method close*(self: ThreadDatastore): Future[?!void] {.async.} = await self.semaphore.closeAll() diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 7754f121..da0df6dc 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -44,7 +44,9 @@ proc toRes*(res: ThreadResult[void]): ?!void = res.mapErr() do(e: ThreadResErr) -> ref CatchableError: e.toExc() -proc toRes*[T,S](res: ThreadResult[T], m: proc(v: T): S = proc(v: T): T = v): ?!S = +proc toRes*[T,S](res: ThreadResult[T], + m: proc(v: T): S = proc(v: T): T = v): ?!S = + # todo: cleaner way to do this? if res.isErr(): result.err res.error().toExc() else: From c1f5d437750456c3a91a202ec8cc3239eb70b29e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:25:43 -0700 Subject: [PATCH 302/445] result types --- datastore/threads/threadproxyds.nim | 2 ++ tests/datastore/testthreadproxyds.nim | 36 +++++++++++++-------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 7d6541d2..d5a53dc6 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -159,6 +159,7 @@ method has*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[bool](self, signal): self.tp.spawn hasTask(ctx, ds, key) + return ctx[].res.toRes(v => v) proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = @@ -175,6 +176,7 @@ method delete*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[void](self, signal): self.tp.spawn deleteTask(ctx, ds, key) + return ctx[].res.toRes() method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index a9ae8438..1a2c6633 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -58,29 +58,29 @@ suite "Test Basic ThreadProxyDatastore": val &= char(c) echo "get res2: ", $val -# suite "Test Basic ThreadDatastore with SQLite": +suite "Test Basic ThreadDatastore with SQLite": -# var -# sqlStore: SQLiteBackend[KeyId,DataBuffer] -# ds: ThreadDatastore -# taskPool: Taskpool -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes -# setupAll: -# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + setupAll: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() -# teardown: -# GC_fullCollect() + teardown: + GC_fullCollect() -# teardownAll: -# (await ds.close()).tryGet() -# taskPool.shutdown() + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() -# basicStoreTests(ds, key, bytes, otherBytes) + basicStoreTests(ds, key, bytes, otherBytes) # suite "Test Query ThreadDatastore with SQLite": From da740f1bc837c8bf99eda7d786877a3e9e03a6dc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:26:41 -0700 Subject: [PATCH 303/445] result types --- datastore/threads/threadproxyds.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d5a53dc6..deb01b3e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -202,10 +202,11 @@ method put*(self: ThreadDatastore, without signal =? acquireSignal(), err: return failure err - let key = KeyId.new key.id() - let data = DataBuffer.new data dispatchTask[void](self, signal): + let key = KeyId.new key.id() + let data = DataBuffer.new data self.tp.spawn putTask(ctx, ds, key, data) + return ctx[].res.toRes() method put*( @@ -221,8 +222,6 @@ proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; ## run backend command executeTask(ctx): let res = get(ds, key) - static: - echo "getTask:type: ", res.typeof res method get*(self: ThreadDatastore, @@ -235,6 +234,7 @@ method get*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[DataBuffer](self, signal): self.tp.spawn getTask(ctx, ds, key) + return ctx[].res.toRes(v => v.toSeq()) method close*(self: ThreadDatastore): Future[?!void] {.async.} = From b8dad8bb60f7620bcf2b181c6f02296920b0032d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:27:50 -0700 Subject: [PATCH 304/445] result types --- datastore/threads/threadproxyds.nim | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index deb01b3e..fb41fbe2 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -85,25 +85,18 @@ proc acquireSignal(): ?!ThreadSignalPtr = template executeTask[T](ctx: TaskCtx[T], blk: untyped) = try: - echo "executeTask:start:" if not ctx.setRunning(): - echo "executeTask:notRunning!" return ## run backend command - echo "executeTask:run:" let res = `blk` if res.isOk(): - echo "executeTask:run:ok" when T is void: ctx[].res.ok() else: ctx[].res.ok(res.get()) else: - echo "executeTask:run:err" ctx[].res.err res.error().toThreadErr() - echo "executeTask:run:done: ", ctx[].res.repr - echo "" except CatchableError as exc: trace "Unexpected exception thrown in async task", exc = exc.msg @@ -112,10 +105,8 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = trace "Unexpected defect thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() finally: - echo "executeTask:finally:setDone" ctx.setDone() discard ctx[].signal.fireSync() - echo "executeTask:finally:done\n" template dispatchTaskWrap[T](self: ThreadDatastore, signal: ThreadSignalPtr, @@ -176,6 +167,7 @@ method delete*(self: ThreadDatastore, let key = KeyId.new key.id() dispatchTask[void](self, signal): self.tp.spawn deleteTask(ctx, ds, key) + return ctx[].res.toRes() method delete*(self: ThreadDatastore, From 1aec1a460d58702030804fd33ede47f4debba424 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:30:20 -0700 Subject: [PATCH 305/445] result types --- datastore/threads/threadproxyds.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index fb41fbe2..695af788 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -136,6 +136,7 @@ template dispatchTask[T](self: ThreadDatastore, let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) dispatchTaskWrap[T](self, signal, blk) + proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -152,6 +153,7 @@ method has*(self: ThreadDatastore, self.tp.spawn hasTask(ctx, ds, key) return ctx[].res.toRes(v => v) + proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command @@ -208,6 +210,9 @@ method put*( for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err + + return success() + proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; key: KeyId) {.gcsafe, nimcall.} = From d0ee284d9092c04cca4caa26d3d385ee9ffbc649 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:32:10 -0700 Subject: [PATCH 306/445] result types --- datastore/threads/threadproxyds.nim | 7 +++--- tests/datastore/testthreadproxyds.nim | 34 +++++++++++++-------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 695af788..bcf06803 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -162,6 +162,7 @@ proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; method delete*(self: ThreadDatastore, key: Key): Future[?!void] {.async.} = + ## delete key await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err @@ -174,7 +175,7 @@ method delete*(self: ThreadDatastore, method delete*(self: ThreadDatastore, keys: seq[Key]): Future[?!void] {.async.} = - + ## delete batch for key in keys: if err =? (await self.delete(key)).errorOption: return failure err @@ -185,13 +186,13 @@ method delete*(self: ThreadDatastore, proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId, data: DataBuffer) {.gcsafe, nimcall.} = - ## run backend command executeTask(ctx): put(ds, key, data) method put*(self: ThreadDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = + ## put key with data await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err @@ -206,7 +207,7 @@ method put*(self: ThreadDatastore, method put*( self: ThreadDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = - + ## put batch data for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 1a2c6633..cfef8e6c 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -82,28 +82,28 @@ suite "Test Basic ThreadDatastore with SQLite": basicStoreTests(ds, key, bytes, otherBytes) -# suite "Test Query ThreadDatastore with SQLite": +suite "Test Query ThreadDatastore with SQLite": -# var -# sqlStore: SQLiteBackend[KeyId,DataBuffer] -# ds: ThreadDatastore -# taskPool: Taskpool -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes -# setup: -# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + setup: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() -# teardown: -# GC_fullCollect() + teardown: + GC_fullCollect() -# (await ds.close()).tryGet() -# taskPool.shutdown() + (await ds.close()).tryGet() + taskPool.shutdown() -# queryTests(ds, true) + queryTests(ds, true) # suite "Test Basic ThreadDatastore with fsds": # let From f59cf06419326350f2af41f81f54caf1d9df2ca2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:38:13 -0700 Subject: [PATCH 307/445] test query --- datastore/threads/threadproxyds.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index bcf06803..03ac3216 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -250,21 +250,28 @@ proc queryTask[DB]( dq: DbQuery[KeyId] ) {.gcsafe, nimcall.} = ## run query command + echo "\n\tqueryTask:init" executeTask(ctx): + echo "\tqueryTask:exec:" # we execute this all inside `executeTask` # so we need to return a final result let qh = ds.query(dq) + echo "\tqueryTask:query: ", qh if qh.isErr(): # set error and exit executeTask, which will fire final signal + echo "\tqueryTask:query:err " (?!QResult).err(qh.error()) else: + echo "\tqueryTask:query:ok " # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() + echo "\tqueryTask:query:fireSync " var handle = qh.get() for item in handle.iter(): # wait for next request from async thread + echo "\tqueryTask:query:iter:wait " discard ctx[].signal.waitSync().get() if ctx[].cancelled: @@ -274,6 +281,7 @@ proc queryTask[DB]( else: ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc + echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() # set final result @@ -283,22 +291,31 @@ method query*( self: ThreadDatastore, q: Query): Future[?!QueryIter] {.async.} = + echo "\nquery:" await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err + echo "query:dbQuery:" let dq = dbQuery( key= KeyId.new q.key.id(), value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) + echo "query:init:dispatch:" dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): + echo "query:init:dispatch:queryTask" self.tp.spawn queryTask(ctx, ds, dq) + echo "query:init:dispatch:queryTask:done" var lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() + echo "query:asyncLock:done" + echo "query:next:ready: " + proc next(): Future[?!QueryResponse] {.async.} = + echo "query:next:exec: " let ctx = ctx defer: if lock.locked: From b4ce2868ed9a7d3f9cead6a4e1c8334dc35abccc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:43:41 -0700 Subject: [PATCH 308/445] test query --- datastore/threads/threadproxyds.nim | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 03ac3216..fe28cb7f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -306,7 +306,9 @@ method query*( echo "query:init:dispatch:queryTask" self.tp.spawn queryTask(ctx, ds, dq) - echo "query:init:dispatch:queryTask:done" + echo "query:init:dispatch:started" + echo "query:init:dispatch:res: ", ctx[].res + var lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() @@ -323,22 +325,31 @@ method query*( trace "About to query" if lock.locked: + echo "query:next:lock:fail:alreadyLock " return failure (ref DatastoreError)(msg: "Should always await query features") if iter.finished == true: + echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + echo "query:next:acquire:lock" await lock.acquire() + echo "query:next:iter:dispatch" dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): # trigger query task to iterate then wait for new result! discard ctx[].signal.fireSync() + echo "query:next:iter:dispatch:fire" + echo "query:next:iter:res: " if not ctx[].running: + echo "query:next:iter:finished " iter.finished = true if ctx[].res.isErr(): + echo "query:next:iter:res:err " return err(ctx[].res.error()) else: + echo "query:next:iter:res:ok " let qres = ctx[].res.get() let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) let data = qres.data.toSeq() From 690b3d4453e63850d38208dcf6e64a2cb5bcd8cf Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:52:35 -0700 Subject: [PATCH 309/445] test query --- datastore/threads/threadproxyds.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index fe28cb7f..e06b1033 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -106,6 +106,7 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = ctx[].res.err exc.toThreadErr() finally: ctx.setDone() + # echo "\t\texecuteTask:fireSync!" discard ctx[].signal.fireSync() template dispatchTaskWrap[T](self: ThreadDatastore, @@ -119,6 +120,7 @@ template dispatchTaskWrap[T](self: ThreadDatastore, proc runTask() = `blk` runTask() + # echo "\t\tdispatchTask:wait!" await wait(ctx[].signal) except CancelledError as exc: @@ -271,14 +273,17 @@ proc queryTask[DB]( var handle = qh.get() for item in handle.iter(): # wait for next request from async thread - echo "\tqueryTask:query:iter:wait " + echo "\tqueryTask:query:iter:wait! " discard ctx[].signal.waitSync().get() + echo "\tqueryTask:query:iter:wait:done " if ctx[].cancelled: + echo "\tqueryTask:query:iter:cancelled" # cancel iter, then run next cycle so it'll finish and close handle.cancel = true continue else: + echo "\tqueryTask:query:iter:done" ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc echo "\tqueryTask:query:iter:fireSync " @@ -335,10 +340,11 @@ method query*( await lock.acquire() echo "query:next:iter:dispatch" + await ctx[].signal.fire() dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): # trigger query task to iterate then wait for new result! - discard ctx[].signal.fireSync() - echo "query:next:iter:dispatch:fire" + echo "query:next:iter:dispatch:fireSync" + echo "query:next:iter:dispatch:wait" echo "query:next:iter:res: " if not ctx[].running: From 8aa3618805672eafbb90188c13ef29608cb06157 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:56:07 -0700 Subject: [PATCH 310/445] test query --- datastore/threads/threadproxyds.nim | 6 +- tests/datastore/querycommontests.nim | 408 +++++++++++++-------------- 2 files changed, 207 insertions(+), 207 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e06b1033..d21b1526 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -101,9 +101,9 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = except CatchableError as exc: trace "Unexpected exception thrown in async task", exc = exc.msg ctx[].res.err exc.toThreadErr() - except Exception as exc: - trace "Unexpected defect thrown in async task", exc = exc.msg - ctx[].res.err exc.toThreadErr() + # except Exception as exc: + # trace "Unexpected defect thrown in async task", exc = exc.msg + # ctx[].res.err exc.toThreadErr() finally: ctx.setDone() # echo "\t\texecuteTask:fireSync!" diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index a0f93a7f..63d18c19 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -64,277 +64,277 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - test "Key should query all keys without values": - let - q = Query.init(key1, value = false) + # test "Key should query all keys without values": + # let + # q = Query.init(key1, value = false) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 3 - res[0].key.get == key1 - res[0].data.len == 0 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data.len == 0 - res[1].key.get == key2 - res[1].data.len == 0 + # res[1].key.get == key2 + # res[1].data.len == 0 - res[2].key.get == key3 - res[2].data.len == 0 + # res[2].key.get == key3 + # res[2].data.len == 0 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Key should not query parent": - let - q = Query.init(key2) + # test "Key should not query parent": + # let + # q = Query.init(key2) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 2 - res[0].key.get == key2 - res[0].data == val2 + # check: + # res.len == 2 + # res[0].key.get == key2 + # res[0].data == val2 - res[1].key.get == key3 - res[1].data == val3 + # res[1].key.get == key3 + # res[1].data == val3 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Key should all list all keys at the same level": - let - queryKey = Key.init("/a").tryGet - q = Query.init(queryKey) + # test "Key should all list all keys at the same level": + # let + # queryKey = Key.init("/a").tryGet + # q = Query.init(queryKey) - (await ds.put(key1, val1)).tryGet - (await ds.put(key2, val2)).tryGet - (await ds.put(key3, val3)).tryGet + # (await ds.put(key1, val1)).tryGet + # (await ds.put(key2, val2)).tryGet + # (await ds.put(key3, val3)).tryGet - let - iter = (await ds.query(q)).tryGet + # let + # iter = (await ds.query(q)).tryGet - var - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # var + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - res.sort do (a, b: QueryResponse) -> int: - cmp(a.key.get.id, b.key.get.id) + # res.sort do (a, b: QueryResponse) -> int: + # cmp(a.key.get.id, b.key.get.id) - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 + # check: + # res.len == 3 + # res[0].key.get == key1 + # res[0].data == val1 - res[1].key.get == key2 - res[1].data == val2 + # res[1].key.get == key2 + # res[1].data == val2 - res[2].key.get == key3 - res[2].data == val3 + # res[2].key.get == key3 + # res[2].data == val3 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - if extended: - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, limit = 10) + # if extended: + # test "Should apply limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, limit = 10) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 10 + # check: + # res.len == 10 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Should not apply offset": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 90) + # test "Should not apply offset": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 90) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 10 + # check: + # res.len == 10 - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - q = Query.init(key, offset = 95, limit = 5) + # test "Should not apply offset and limit": + # let + # key = Key.init("/a").tryGet + # q = Query.init(key, offset = 95, limit = 5) - for i in 0..<100: - let - key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = ("val " & $i).toBytes + # for i in 0..<100: + # let + # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + # val = ("val " & $i).toBytes - (await ds.put(key, val)).tryGet + # (await ds.put(key, val)).tryGet - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 5 + # check: + # res.len == 5 - for i in 0.. int: - cmp(a.key.get.id, b.key.get.id) + # # lexicographic sort, as it comes from the backend + # kvs.sort do (a, b: QueryResponse) -> int: + # cmp(a.key.get.id, b.key.get.id) - kvs = kvs.reversed - let - iter = (await ds.query(q)).tryGet - res = block: - var - res: seq[QueryResponse] - cnt = 0 + # kvs = kvs.reversed + # let + # iter = (await ds.query(q)).tryGet + # res = block: + # var + # res: seq[QueryResponse] + # cnt = 0 - for pair in iter: - let (key, val) = (await pair).tryGet - if key.isNone: - break + # for pair in iter: + # let (key, val) = (await pair).tryGet + # if key.isNone: + # break - res.add((key, val)) - cnt.inc + # res.add((key, val)) + # cnt.inc - res + # res - check: - res.len == 100 + # check: + # res.len == 100 - for i, r in res[1..^1]: - check: - res[i].key.get == kvs[i].key.get - res[i].data == kvs[i].data + # for i, r in res[1..^1]: + # check: + # res[i].key.get == kvs[i].key.get + # res[i].data == kvs[i].data - (await iter.dispose()).tryGet + # (await iter.dispose()).tryGet From 4b4403bf4d3b3141adff8271230970b0d25c8a3f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 17:59:05 -0700 Subject: [PATCH 311/445] test query --- datastore/threads/threadproxyds.nim | 36 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d21b1526..7e4e6bfc 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -113,16 +113,22 @@ template dispatchTaskWrap[T](self: ThreadDatastore, signal: ThreadSignalPtr, blk: untyped ): auto = - try: - case self.backend.kind: - of Sqlite: - var ds {.used, inject.} = self.backend.sql - proc runTask() = - `blk` - runTask() - # echo "\t\tdispatchTask:wait!" - await wait(ctx[].signal) + case self.backend.kind: + of Sqlite: + var ds {.used, inject.} = self.backend.sql + proc runTask() = + `blk` + runTask() + # echo "\t\tdispatchTask:wait!" + await wait(ctx[].signal) +template dispatchTask[T](self: ThreadDatastore, + signal: ThreadSignalPtr, + blk: untyped + ): auto = + let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) + try: + dispatchTaskWrap[T](self, signal, blk) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() @@ -131,13 +137,6 @@ template dispatchTaskWrap[T](self: ThreadDatastore, discard ctx[].signal.close() self.semaphore.release() -template dispatchTask[T](self: ThreadDatastore, - signal: ThreadSignalPtr, - blk: untyped - ): auto = - let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) - dispatchTaskWrap[T](self, signal, blk) - proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command @@ -307,18 +306,17 @@ method query*( value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) echo "query:init:dispatch:" - dispatchTask[DbQueryResponse[KeyId, DataBuffer]](self, signal): + let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) + dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): echo "query:init:dispatch:queryTask" self.tp.spawn queryTask(ctx, ds, dq) - echo "query:init:dispatch:started" echo "query:init:dispatch:res: ", ctx[].res var lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() - echo "query:asyncLock:done" echo "query:next:ready: " proc next(): Future[?!QueryResponse] {.async.} = From a4c3574ff2b5397713073e5d7f80213fdfa4688c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:10:20 -0700 Subject: [PATCH 312/445] test query --- datastore/threads/threadproxyds.nim | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 7e4e6bfc..c79a1810 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -248,7 +248,8 @@ type proc queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, - dq: DbQuery[KeyId] + query: DbQuery[KeyId], + nextSignal: ThreadSignalPtr ) {.gcsafe, nimcall.} = ## run query command echo "\n\tqueryTask:init" @@ -256,12 +257,12 @@ proc queryTask[DB]( echo "\tqueryTask:exec:" # we execute this all inside `executeTask` # so we need to return a final result - let qh = ds.query(dq) - echo "\tqueryTask:query: ", qh - if qh.isErr(): + let handleRes = ds.query(query) + echo "\tqueryTask:query: ", handleRes + if handleRes.isErr(): # set error and exit executeTask, which will fire final signal echo "\tqueryTask:query:err " - (?!QResult).err(qh.error()) + (?!QResult).err(handleRes.error()) else: echo "\tqueryTask:query:ok " # otherwise manually an set empty ok result @@ -269,7 +270,7 @@ proc queryTask[DB]( discard ctx[].signal.fireSync() echo "\tqueryTask:query:fireSync " - var handle = qh.get() + var handle = handleRes.get() for item in handle.iter(): # wait for next request from async thread echo "\tqueryTask:query:iter:wait! " @@ -282,13 +283,14 @@ proc queryTask[DB]( handle.cancel = true continue else: - echo "\tqueryTask:query:iter:done" + echo "\tqueryTask:query:iter:result:" ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() # set final result + echo "\tqueryTask:query:iter:done " (?!QResult).ok((KeyId.none, DataBuffer())) method query*( @@ -299,9 +301,11 @@ method query*( await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err + without nextSignal =? acquireSignal(), err: + return failure err echo "query:dbQuery:" - let dq = dbQuery( + let query = dbQuery( key= KeyId.new q.key.id(), value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) @@ -309,7 +313,7 @@ method query*( let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): echo "query:init:dispatch:queryTask" - self.tp.spawn queryTask(ctx, ds, dq) + self.tp.spawn queryTask(ctx, ds, query, nextSignal) echo "query:init:dispatch:res: ", ctx[].res @@ -339,12 +343,11 @@ method query*( echo "query:next:iter:dispatch" await ctx[].signal.fire() - dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): - # trigger query task to iterate then wait for new result! - echo "query:next:iter:dispatch:fireSync" - echo "query:next:iter:dispatch:wait" + echo "query:next:iter:dispatch:wait" + await wait(ctx[].signal) + + echo "query:next:iter:res: ", ctx[].res - echo "query:next:iter:res: " if not ctx[].running: echo "query:next:iter:finished " iter.finished = true From 6c95c29267045b48a9aef286dc0ede8fe7da8420 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:11:06 -0700 Subject: [PATCH 313/445] test query --- datastore/threads/threadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c79a1810..c9861081 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -274,7 +274,7 @@ proc queryTask[DB]( for item in handle.iter(): # wait for next request from async thread echo "\tqueryTask:query:iter:wait! " - discard ctx[].signal.waitSync().get() + discard nextSignal.waitSync().get() echo "\tqueryTask:query:iter:wait:done " if ctx[].cancelled: @@ -344,7 +344,7 @@ method query*( echo "query:next:iter:dispatch" await ctx[].signal.fire() echo "query:next:iter:dispatch:wait" - await wait(ctx[].signal) + await wait(nextSignal) echo "query:next:iter:res: ", ctx[].res From 4e63ca48f09192e30164f8faf70d210926753b32 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:11:47 -0700 Subject: [PATCH 314/445] test query --- datastore/threads/threadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c9861081..afe69a35 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -342,9 +342,9 @@ method query*( await lock.acquire() echo "query:next:iter:dispatch" - await ctx[].signal.fire() + await nextSignal.fire() echo "query:next:iter:dispatch:wait" - await wait(nextSignal) + await wait(ctx[].signal) echo "query:next:iter:res: ", ctx[].res From d411b561755003721809918d4d7e4111816e3b6b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:14:30 -0700 Subject: [PATCH 315/445] test query --- datastore/threads/threadproxyds.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index afe69a35..369481cd 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -276,16 +276,15 @@ proc queryTask[DB]( echo "\tqueryTask:query:iter:wait! " discard nextSignal.waitSync().get() - echo "\tqueryTask:query:iter:wait:done " if ctx[].cancelled: echo "\tqueryTask:query:iter:cancelled" # cancel iter, then run next cycle so it'll finish and close handle.cancel = true continue else: - echo "\tqueryTask:query:iter:result:" ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc + echo "\tqueryTask:query:iter:result: ", ctx[].res echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() From 24748cd769c5ae20c3f9fd19962c6f0e546afb8b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:18:05 -0700 Subject: [PATCH 316/445] test query --- datastore/threads/threadproxyds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 369481cd..d2af41e1 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -274,7 +274,6 @@ proc queryTask[DB]( for item in handle.iter(): # wait for next request from async thread echo "\tqueryTask:query:iter:wait! " - discard nextSignal.waitSync().get() if ctx[].cancelled: echo "\tqueryTask:query:iter:cancelled" @@ -287,6 +286,7 @@ proc queryTask[DB]( echo "\tqueryTask:query:iter:result: ", ctx[].res echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() + discard nextSignal.waitSync().get() # set final result echo "\tqueryTask:query:iter:done " From be70ddb25099d085430661fb098c9101269b96db Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:20:05 -0700 Subject: [PATCH 317/445] test query --- datastore/threads/threadproxyds.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d2af41e1..fd6f09f5 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -345,17 +345,15 @@ method query*( echo "query:next:iter:dispatch:wait" await wait(ctx[].signal) - echo "query:next:iter:res: ", ctx[].res + echo "query:next:iter:res: ", ctx[].res, "\n" if not ctx[].running: echo "query:next:iter:finished " iter.finished = true if ctx[].res.isErr(): - echo "query:next:iter:res:err " return err(ctx[].res.error()) else: - echo "query:next:iter:res:ok " let qres = ctx[].res.get() let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) let data = qres.data.toSeq() From b746a266e754f8e7f36f2c44325eadcb4c13e656 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:24:48 -0700 Subject: [PATCH 318/445] test query --- datastore/threads/threadproxyds.nim | 9 +++++---- tests/datastore/querycommontests.nim | 9 ++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index fd6f09f5..d0655d72 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -337,6 +337,11 @@ method query*( echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + if not ctx[].running: + echo "query:next:iter:finished " + iter.finished = true + return + echo "query:next:acquire:lock" await lock.acquire() @@ -347,10 +352,6 @@ method query*( echo "query:next:iter:res: ", ctx[].res, "\n" - if not ctx[].running: - echo "query:next:iter:finished " - iter.finished = true - if ctx[].res.isErr(): return err(ctx[].res.error()) else: diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 63d18c19..8d4324a2 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -37,18 +37,13 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = let iter = (await ds.query(q)).tryGet res = block: - var - res: seq[QueryResponse] - cnt = 0 - + var res: seq[QueryResponse] for pair in iter: let (key, val) = (await pair).tryGet + echo "test:query:result: ", $key if key.isNone: break - res.add((key, val)) - cnt.inc - res check: From 70b5956773a125fdfd8c2c2c6704d68912d00992 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:27:18 -0700 Subject: [PATCH 319/445] test query --- datastore/threads/threadproxyds.nim | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d0655d72..c0154912 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -320,6 +320,9 @@ method query*( lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() + echo "query:first:iter:dispatch" + await nextSignal.fire() + echo "query:next:ready: " proc next(): Future[?!QueryResponse] {.async.} = @@ -337,19 +340,16 @@ method query*( echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - if not ctx[].running: - echo "query:next:iter:finished " - iter.finished = true - return - echo "query:next:acquire:lock" await lock.acquire() - echo "query:next:iter:dispatch" - await nextSignal.fire() - echo "query:next:iter:dispatch:wait" await wait(ctx[].signal) + if not ctx[].running: + echo "query:next:iter:finished " + iter.finished = true + # return + echo "query:next:iter:res: ", ctx[].res, "\n" if ctx[].res.isErr(): @@ -360,6 +360,9 @@ method query*( let data = qres.data.toSeq() return (?!QueryResponse).ok((key: key, data: data)) + echo "query:next:iter:dispatch" + await nextSignal.fire() + iter.next = next return success iter From 02de983dbd71f0bd7de828483b3da80378cd874f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:33:36 -0700 Subject: [PATCH 320/445] test query --- datastore/threads/threadproxyds.nim | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c0154912..ffba89fb 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -252,28 +252,23 @@ proc queryTask[DB]( nextSignal: ThreadSignalPtr ) {.gcsafe, nimcall.} = ## run query command - echo "\n\tqueryTask:init" executeTask(ctx): - echo "\tqueryTask:exec:" # we execute this all inside `executeTask` # so we need to return a final result let handleRes = ds.query(query) - echo "\tqueryTask:query: ", handleRes if handleRes.isErr(): # set error and exit executeTask, which will fire final signal echo "\tqueryTask:query:err " (?!QResult).err(handleRes.error()) else: - echo "\tqueryTask:query:ok " # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) - discard ctx[].signal.fireSync() echo "\tqueryTask:query:fireSync " + discard ctx[].signal.fireSync() var handle = handleRes.get() for item in handle.iter(): # wait for next request from async thread - echo "\tqueryTask:query:iter:wait! " if ctx[].cancelled: echo "\tqueryTask:query:iter:cancelled" @@ -286,6 +281,8 @@ proc queryTask[DB]( echo "\tqueryTask:query:iter:result: ", ctx[].res echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() + + echo "\tqueryTask:query:iter:nextSignal:wait! " discard nextSignal.waitSync().get() # set final result @@ -320,13 +317,13 @@ method query*( lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() - echo "query:first:iter:dispatch" - await nextSignal.fire() + # echo "query:first:iter:dispatch" + # await nextSignal.fire() - echo "query:next:ready: " + echo "query:first:ready!\n" proc next(): Future[?!QueryResponse] {.async.} = - echo "query:next:exec: " + echo "\n\nquery:next:exec: " let ctx = ctx defer: if lock.locked: @@ -351,6 +348,9 @@ method query*( # return echo "query:next:iter:res: ", ctx[].res, "\n" + defer: + echo "query:iter:nextSignal:fire!" + await nextSignal.fire() if ctx[].res.isErr(): return err(ctx[].res.error()) @@ -360,8 +360,6 @@ method query*( let data = qres.data.toSeq() return (?!QueryResponse).ok((key: key, data: data)) - echo "query:next:iter:dispatch" - await nextSignal.fire() iter.next = next return success iter From 0fedfcfc002bcb16bdd9c0189a258d2300b97ba2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:40:04 -0700 Subject: [PATCH 321/445] test query - it runs! --- datastore/query.nim | 2 +- datastore/threads/threadproxyds.nim | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/datastore/query.nim b/datastore/query.nim index 477d04d1..256ad97e 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -37,7 +37,7 @@ proc new*(T: type QueryIter, dispose = defaultDispose): T = proc init*(T: type Query, key: Key, - value = false, + value = true, sort = SortOrder.Ascending, offset = 0, limit = -1): Query = diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ffba89fb..a704d401 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -276,9 +276,12 @@ proc queryTask[DB]( handle.cancel = true continue else: + if item.isOk: + echo "\tqueryTask:query:iter:result:data: ", $item.get().data ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc echo "\tqueryTask:query:iter:result: ", ctx[].res + echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() From 6b90743de1509aad3b169e1e7ae2108e5a2df814 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:43:35 -0700 Subject: [PATCH 322/445] test query - it runs! --- tests/datastore/querycommontests.nim | 56 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 8d4324a2..a575e056 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -59,43 +59,43 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - # test "Key should query all keys without values": - # let - # q = Query.init(key1, value = false) + test "Key should query all keys without values": + let + q = Query.init(key1, value = false) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data.len == 0 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 - # res[1].key.get == key2 - # res[1].data.len == 0 + res[1].key.get == key2 + res[1].data.len == 0 - # res[2].key.get == key3 - # res[2].data.len == 0 + res[2].key.get == key3 + res[2].data.len == 0 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet # test "Key should not query parent": # let From f4d070553bc42442e272340e85d27310cc150c55 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:44:13 -0700 Subject: [PATCH 323/445] test query - it runs! --- tests/datastore/querycommontests.nim | 52 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index a575e056..d94246b2 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -97,40 +97,40 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - # test "Key should not query parent": - # let - # q = Query.init(key2) + test "Key should not query parent": + let + q = Query.init(key2) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 2 - # res[0].key.get == key2 - # res[0].data == val2 + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 - # res[1].key.get == key3 - # res[1].data == val3 + res[1].key.get == key3 + res[1].data == val3 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet # test "Key should all list all keys at the same level": # let From 9cd0ae7ea53515b47273374b8d52820301894b2d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:44:38 -0700 Subject: [PATCH 324/445] test query - it runs! --- tests/datastore/querycommontests.nim | 240 +++++++++++++-------------- 1 file changed, 120 insertions(+), 120 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index d94246b2..46dcee57 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -176,160 +176,160 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = # (await iter.dispose()).tryGet - # if extended: - # test "Should apply limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, limit = 10) + if extended: + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, limit = 10) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 10 + check: + res.len == 10 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Should not apply offset": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 90) - - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes - - # (await ds.put(key, val)).tryGet + test "Should not apply offset": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 90) + + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes + + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 10 + check: + res.len == 10 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet - # test "Should not apply offset and limit": - # let - # key = Key.init("/a").tryGet - # q = Query.init(key, offset = 95, limit = 5) + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + q = Query.init(key, offset = 95, limit = 5) - # for i in 0..<100: - # let - # key = Key.init(key, Key.init("/" & $i).tryGet).tryGet - # val = ("val " & $i).toBytes + for i in 0..<100: + let + key = Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = ("val " & $i).toBytes - # (await ds.put(key, val)).tryGet + (await ds.put(key, val)).tryGet - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 5 + check: + res.len == 5 - # for i in 0.. int: - # cmp(a.key.get.id, b.key.get.id) + # lexicographic sort, as it comes from the backend + kvs.sort do (a, b: QueryResponse) -> int: + cmp(a.key.get.id, b.key.get.id) - # kvs = kvs.reversed - # let - # iter = (await ds.query(q)).tryGet - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + kvs = kvs.reversed + let + iter = (await ds.query(q)).tryGet + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # check: - # res.len == 100 + check: + res.len == 100 - # for i, r in res[1..^1]: - # check: - # res[i].key.get == kvs[i].key.get - # res[i].data == kvs[i].data + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet From f8446de63857c6af11987b55f25cf0ab8e1b520b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:45:52 -0700 Subject: [PATCH 325/445] test query - it runs! --- tests/datastore/querycommontests.nim | 64 ++++++++++++++-------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 46dcee57..2ee4f5d3 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -132,49 +132,49 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = (await iter.dispose()).tryGet - # test "Key should all list all keys at the same level": - # let - # queryKey = Key.init("/a").tryGet - # q = Query.init(queryKey) + test "Key should all list all keys at the same level": + let + queryKey = Key.init("/a").tryGet + q = Query.init(queryKey) - # (await ds.put(key1, val1)).tryGet - # (await ds.put(key2, val2)).tryGet - # (await ds.put(key3, val3)).tryGet + (await ds.put(key1, val1)).tryGet + (await ds.put(key2, val2)).tryGet + (await ds.put(key3, val3)).tryGet - # let - # iter = (await ds.query(q)).tryGet + let + iter = (await ds.query(q)).tryGet - # var - # res = block: - # var - # res: seq[QueryResponse] - # cnt = 0 + var + res = block: + var + res: seq[QueryResponse] + cnt = 0 - # for pair in iter: - # let (key, val) = (await pair).tryGet - # if key.isNone: - # break + for pair in iter: + let (key, val) = (await pair).tryGet + if key.isNone: + break - # res.add((key, val)) - # cnt.inc + res.add((key, val)) + cnt.inc - # res + res - # res.sort do (a, b: QueryResponse) -> int: - # cmp(a.key.get.id, b.key.get.id) + res.sort do (a, b: QueryResponse) -> int: + cmp(a.key.get.id, b.key.get.id) - # check: - # res.len == 3 - # res[0].key.get == key1 - # res[0].data == val1 + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 - # res[1].key.get == key2 - # res[1].data == val2 + res[1].key.get == key2 + res[1].data == val2 - # res[2].key.get == key3 - # res[2].data == val3 + res[2].key.get == key3 + res[2].data == val3 - # (await iter.dispose()).tryGet + (await iter.dispose()).tryGet if extended: test "Should apply limit": From 317d42c511e7256f6c2521526534b3724e456d96 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 18:47:27 -0700 Subject: [PATCH 326/445] test query - it runs! --- datastore/threads/threadproxyds.nim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index a704d401..ed11f189 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -328,9 +328,9 @@ method query*( proc next(): Future[?!QueryResponse] {.async.} = echo "\n\nquery:next:exec: " let ctx = ctx - defer: - if lock.locked: - lock.release() + # defer: + # if lock.locked: + # lock.release() trace "About to query" if lock.locked: @@ -340,9 +340,10 @@ method query*( echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - echo "query:next:acquire:lock" - await lock.acquire() + # echo "query:next:acquire:lock" + # await lock.acquire() + echo "query:next:wait:signal" await wait(ctx[].signal) if not ctx[].running: From dacc28ab02e891cb53bbf42d21dbf5a83a7971d3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:21:09 -0700 Subject: [PATCH 327/445] fix occasional deadlock --- datastore/threads/threadproxyds.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ed11f189..d7bcd180 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -245,6 +245,8 @@ method close*(self: ThreadDatastore): Future[?!void] {.async.} = type QResult = DbQueryResponse[KeyId, DataBuffer] +import os + proc queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, @@ -265,6 +267,8 @@ proc queryTask[DB]( ctx[].res.ok (KeyId.none, DataBuffer(), ) echo "\tqueryTask:query:fireSync " discard ctx[].signal.fireSync() + echo "\tqueryTask:query:nextSignal:wait " + discard nextSignal.waitSync().get() var handle = handleRes.get() for item in handle.iter(): @@ -313,6 +317,10 @@ method query*( dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): echo "query:init:dispatch:queryTask" self.tp.spawn queryTask(ctx, ds, query, nextSignal) + + # await wait(ctx[].signal) + await nextSignal.fire() + # await wait(ctx[].signal) echo "query:init:dispatch:res: ", ctx[].res @@ -328,9 +336,6 @@ method query*( proc next(): Future[?!QueryResponse] {.async.} = echo "\n\nquery:next:exec: " let ctx = ctx - # defer: - # if lock.locked: - # lock.release() trace "About to query" if lock.locked: @@ -340,9 +345,6 @@ method query*( echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - # echo "query:next:acquire:lock" - # await lock.acquire() - echo "query:next:wait:signal" await wait(ctx[].signal) From 5c2cc57ccc720d916f2d94a2569d0b94b31a019c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:23:13 -0700 Subject: [PATCH 328/445] fix occasional deadlock --- datastore/threads/threadproxyds.nim | 33 +---------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index d7bcd180..4b46795f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -260,14 +260,11 @@ proc queryTask[DB]( let handleRes = ds.query(query) if handleRes.isErr(): # set error and exit executeTask, which will fire final signal - echo "\tqueryTask:query:err " (?!QResult).err(handleRes.error()) else: # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) - echo "\tqueryTask:query:fireSync " discard ctx[].signal.fireSync() - echo "\tqueryTask:query:nextSignal:wait " discard nextSignal.waitSync().get() var handle = handleRes.get() @@ -275,87 +272,59 @@ proc queryTask[DB]( # wait for next request from async thread if ctx[].cancelled: - echo "\tqueryTask:query:iter:cancelled" # cancel iter, then run next cycle so it'll finish and close handle.cancel = true continue else: - if item.isOk: - echo "\tqueryTask:query:iter:result:data: ", $item.get().data ctx[].res = item.mapErr() do(exc: ref CatchableError) -> ThreadResErr: exc - echo "\tqueryTask:query:iter:result: ", ctx[].res - echo "\tqueryTask:query:iter:fireSync " discard ctx[].signal.fireSync() - echo "\tqueryTask:query:iter:nextSignal:wait! " discard nextSignal.waitSync().get() # set final result - echo "\tqueryTask:query:iter:done " (?!QResult).ok((KeyId.none, DataBuffer())) method query*( self: ThreadDatastore, q: Query): Future[?!QueryIter] {.async.} = - echo "\nquery:" await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err without nextSignal =? acquireSignal(), err: return failure err - echo "query:dbQuery:" let query = dbQuery( key= KeyId.new q.key.id(), value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) - echo "query:init:dispatch:" + # setup initial queryTask let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): - echo "query:init:dispatch:queryTask" self.tp.spawn queryTask(ctx, ds, query, nextSignal) - - # await wait(ctx[].signal) await nextSignal.fire() - # await wait(ctx[].signal) - - echo "query:init:dispatch:res: ", ctx[].res var lock = newAsyncLock() # serialize querying under threads iter = QueryIter.new() - # echo "query:first:iter:dispatch" - # await nextSignal.fire() - - echo "query:first:ready!\n" - proc next(): Future[?!QueryResponse] {.async.} = - echo "\n\nquery:next:exec: " let ctx = ctx trace "About to query" if lock.locked: - echo "query:next:lock:fail:alreadyLock " return failure (ref DatastoreError)(msg: "Should always await query features") if iter.finished == true: - echo "query:next:iter:finished" return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - echo "query:next:wait:signal" await wait(ctx[].signal) if not ctx[].running: - echo "query:next:iter:finished " iter.finished = true - # return - echo "query:next:iter:res: ", ctx[].res, "\n" defer: - echo "query:iter:nextSignal:fire!" await nextSignal.fire() if ctx[].res.isErr(): From 4ff162e0c52ce6f8d602e51c3c65434aedbda349 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:25:05 -0700 Subject: [PATCH 329/445] fix occasional deadlock --- datastore/threads/threadproxyds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4b46795f..dcff5e07 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -265,7 +265,8 @@ proc queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - discard nextSignal.waitSync().get() + if not nextSignal.waitSync(1.seconds).get(): + raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") var handle = handleRes.get() for item in handle.iter(): From 7169a7d20fd2ecab64f594ca30c058217bfd1dda Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:31:07 -0700 Subject: [PATCH 330/445] fix occasional deadlock --- datastore/threads/threadproxyds.nim | 109 ++++++++++++++++------------ 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index dcff5e07..4e2a4298 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -265,7 +265,7 @@ proc queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - if not nextSignal.waitSync(1.seconds).get(): + if not nextSignal.waitSync(10.seconds).get(): raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") var handle = handleRes.get() @@ -287,58 +287,73 @@ proc queryTask[DB]( # set final result (?!QResult).ok((KeyId.none, DataBuffer())) -method query*( - self: ThreadDatastore, - q: Query): Future[?!QueryIter] {.async.} = - +method query*(self: ThreadDatastore, + q: Query + ): Future[?!QueryIter] {.async.} = + ## performs async query + ## keeps one thread running queryTask until finished + ## await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err without nextSignal =? acquireSignal(), err: return failure err - let query = dbQuery( - key= KeyId.new q.key.id(), - value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) - - # setup initial queryTask - let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) - dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn queryTask(ctx, ds, query, nextSignal) - await nextSignal.fire() - - var - lock = newAsyncLock() # serialize querying under threads - iter = QueryIter.new() - - proc next(): Future[?!QueryResponse] {.async.} = - let ctx = ctx - - trace "About to query" - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") - if iter.finished == true: - return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") - - await wait(ctx[].signal) - - if not ctx[].running: - iter.finished = true - - defer: - await nextSignal.fire() - - if ctx[].res.isErr(): - return err(ctx[].res.error()) - else: - let qres = ctx[].res.get() - let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) - let data = qres.data.toSeq() - return (?!QueryResponse).ok((key: key, data: data)) - - - iter.next = next - return success iter + try: + let query = dbQuery( + key= KeyId.new q.key.id(), + value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) + + # setup initial queryTask + let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) + dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): + self.tp.spawn queryTask(ctx, ds, query, nextSignal) + await nextSignal.fire() + + var + lock = newAsyncLock() # serialize querying under threads + iter = QueryIter.new() + + proc next(): Future[?!QueryResponse] {.async.} = + let ctx = ctx + try: + trace "About to query" + if lock.locked: + return failure (ref DatastoreError)(msg: "Should always await query features") + if iter.finished == true: + return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + + await wait(ctx[].signal) + + if not ctx[].running: + iter.finished = true + + defer: + await nextSignal.fire() + + if ctx[].res.isErr(): + return err(ctx[].res.error()) + else: + let qres = ctx[].res.get() + let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) + let data = qres.data.toSeq() + return (?!QueryResponse).ok((key: key, data: data)) + except CancelledError as exc: + trace "Cancelling thread future!", exc = exc.msg + ctx.setCancelled() + discard ctx[].signal.close() + discard nextSignal.close() + self.semaphore.release() + raise exc + + iter.next = next + return success iter + except CancelledError as exc: + trace "Cancelling thread future!", exc = exc.msg + discard signal.close() + discard nextSignal.close() + self.semaphore.release() + raise exc proc new*[DB](self: type ThreadDatastore, db: DB, From 2d843857cdfecffa575b940fa813b40afa940b88 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:35:25 -0700 Subject: [PATCH 331/445] cleanup --- datastore/threads/threadproxyds.nim | 6 +++--- tests/datastore/querycommontests.nim | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4e2a4298..ae147824 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -63,17 +63,17 @@ var ctxLock: Lock ctxLock.initLock() proc setCancelled[T](ctx: TaskCtx[T]) = - withLock(ctxLock): + # withLock(ctxLock): ctx[].cancelled = true proc setRunning[T](ctx: TaskCtx[T]): bool = - withLock(ctxLock): + # withLock(ctxLock): if ctx[].cancelled: return false ctx[].running = true return true proc setDone[T](ctx: TaskCtx[T]) = - withLock(ctxLock): + # withLock(ctxLock): ctx[].running = false proc acquireSignal(): ?!ThreadSignalPtr = diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 2ee4f5d3..460804b7 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -40,7 +40,6 @@ template queryTests*(ds: Datastore, extended = true) {.dirty.} = var res: seq[QueryResponse] for pair in iter: let (key, val) = (await pair).tryGet - echo "test:query:result: ", $key if key.isNone: break res.add((key, val)) From 55b118c055833c28d6fca4eaae7aeba10a74c4b8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 19:56:17 -0700 Subject: [PATCH 332/445] cleanup --- datastore/sql.nim | 21 +++++++--------- datastore/threads/threadproxyds.nim | 33 ++++++++++++++++---------- tests/datastore/sql/testsqlite.nim | 2 +- tests/datastore/testasyncsemaphore.nim | 1 - 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 5b68b53d..28b1f3d0 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -13,7 +13,7 @@ import ../datastore import ./backend import ./sql/sqliteds -export datastore +export datastore, sqliteds push: {.upraises: [].} @@ -42,7 +42,8 @@ method delete*(self: SQLiteDatastore, method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.db.get(KeyId.new key.id()) + self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: + d.toSeq() method put*(self: SQLiteDatastore, key: Key, @@ -79,7 +80,7 @@ method queryIter*( yield QueryResponse.failure err let k = qres.key.map() do(k: KeyId) -> Key: Key.init($k).expect("valid key") - let v: seq[byte] = qres.data + let v: seq[byte] = qres.data.toSeq() yield success (k, v) success iter @@ -87,20 +88,14 @@ method queryIter*( proc new*( T: type SQLiteDatastore, path: string, - readOnly = false): ?!T = + readOnly = false): ?!SQLiteDatastore = - let - flags = - if readOnly: SQLITE_OPEN_READONLY - else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - - success T( - db: ? SQLiteDsDb.open(path, flags), - readOnly: readOnly) + success SQLiteDatastore( + db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) proc new*( T: type SQLiteDatastore, - db: SQLiteDsDb): ?!T = + db: SQLiteBackend[KeyId, DataBuffer]): ?!T = success T( db: db, diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ae147824..795274ea 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -50,6 +50,7 @@ type signal: ThreadSignalPtr running: bool cancelled: bool + nextSignal: (Lock, Cond) TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] @@ -251,7 +252,6 @@ proc queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, query: DbQuery[KeyId], - nextSignal: ThreadSignalPtr ) {.gcsafe, nimcall.} = ## run query command executeTask(ctx): @@ -265,8 +265,11 @@ proc queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - if not nextSignal.waitSync(10.seconds).get(): - raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") + + withLock(ctx[].nextSignal[0]): + wait(ctx[].nextSignal[1], ctx[].nextSignal[0]) + # if not nextSignal.waitSync(10.seconds).get(): + # raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") var handle = handleRes.get() for item in handle.iter(): @@ -282,7 +285,9 @@ proc queryTask[DB]( discard ctx[].signal.fireSync() - discard nextSignal.waitSync().get() + # discard nextSignal.waitSync().get() + withLock(ctx[].nextSignal[0]): + wait(ctx[].nextSignal[1], ctx[].nextSignal[0]) # set final result (?!QResult).ok((KeyId.none, DataBuffer())) @@ -296,8 +301,11 @@ method query*(self: ThreadDatastore, await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - without nextSignal =? acquireSignal(), err: - return failure err + # without nextSignal =? acquireSignal(), err: + # return failure err + let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) + ctx[].nextSignal[0].initLock() + ctx[].nextSignal[1].initCond() try: let query = dbQuery( @@ -305,10 +313,10 @@ method query*(self: ThreadDatastore, value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) # setup initial queryTask - let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn queryTask(ctx, ds, query, nextSignal) - await nextSignal.fire() + self.tp.spawn queryTask(ctx, ds, query) + withLock(ctx[].nextSignal[0]): + signal(ctx[].nextSignal[1]) var lock = newAsyncLock() # serialize querying under threads @@ -329,7 +337,8 @@ method query*(self: ThreadDatastore, iter.finished = true defer: - await nextSignal.fire() + withLock(ctx[].nextSignal[0]): + signal(ctx[].nextSignal[1]) if ctx[].res.isErr(): return err(ctx[].res.error()) @@ -342,7 +351,7 @@ method query*(self: ThreadDatastore, trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() discard ctx[].signal.close() - discard nextSignal.close() + # discard nextSignal.close() self.semaphore.release() raise exc @@ -351,7 +360,7 @@ method query*(self: ThreadDatastore, except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg discard signal.close() - discard nextSignal.close() + # discard nextSignal.close() self.semaphore.release() raise exc diff --git a/tests/datastore/sql/testsqlite.nim b/tests/datastore/sql/testsqlite.nim index c629eb0c..3e15c436 100644 --- a/tests/datastore/sql/testsqlite.nim +++ b/tests/datastore/sql/testsqlite.nim @@ -8,7 +8,7 @@ import pkg/chronos import pkg/stew/results import pkg/stew/byteutils -import pkg/datastore/sql/sqliteds +import pkg/datastore/sql import ../dscommontests import ../querycommontests diff --git a/tests/datastore/testasyncsemaphore.nim b/tests/datastore/testasyncsemaphore.nim index 3c8bb84b..aefff19c 100644 --- a/tests/datastore/testasyncsemaphore.nim +++ b/tests/datastore/testasyncsemaphore.nim @@ -98,7 +98,6 @@ suite "AsyncSemaphore": resource.inc() check resource > 0 and resource <= 3 let sleep = rand(0..10).millis - # echo sleep await sleepAsync(sleep) finally: resource.dec() From 2255ea1c1cfbf2823baabee36ec811f2e226e002 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:03:22 -0700 Subject: [PATCH 333/445] Revert "cleanup" This reverts commit 55b118c055833c28d6fca4eaae7aeba10a74c4b8. --- datastore/sql.nim | 21 +++++++++------- datastore/threads/threadproxyds.nim | 33 ++++++++++---------------- tests/datastore/sql/testsqlite.nim | 2 +- tests/datastore/testasyncsemaphore.nim | 1 + 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 28b1f3d0..5b68b53d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -13,7 +13,7 @@ import ../datastore import ./backend import ./sql/sqliteds -export datastore, sqliteds +export datastore push: {.upraises: [].} @@ -42,8 +42,7 @@ method delete*(self: SQLiteDatastore, method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: - d.toSeq() + self.db.get(KeyId.new key.id()) method put*(self: SQLiteDatastore, key: Key, @@ -80,7 +79,7 @@ method queryIter*( yield QueryResponse.failure err let k = qres.key.map() do(k: KeyId) -> Key: Key.init($k).expect("valid key") - let v: seq[byte] = qres.data.toSeq() + let v: seq[byte] = qres.data yield success (k, v) success iter @@ -88,14 +87,20 @@ method queryIter*( proc new*( T: type SQLiteDatastore, path: string, - readOnly = false): ?!SQLiteDatastore = + readOnly = false): ?!T = - success SQLiteDatastore( - db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) + let + flags = + if readOnly: SQLITE_OPEN_READONLY + else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE + + success T( + db: ? SQLiteDsDb.open(path, flags), + readOnly: readOnly) proc new*( T: type SQLiteDatastore, - db: SQLiteBackend[KeyId, DataBuffer]): ?!T = + db: SQLiteDsDb): ?!T = success T( db: db, diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 795274ea..ae147824 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -50,7 +50,6 @@ type signal: ThreadSignalPtr running: bool cancelled: bool - nextSignal: (Lock, Cond) TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] @@ -252,6 +251,7 @@ proc queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, query: DbQuery[KeyId], + nextSignal: ThreadSignalPtr ) {.gcsafe, nimcall.} = ## run query command executeTask(ctx): @@ -265,11 +265,8 @@ proc queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - - withLock(ctx[].nextSignal[0]): - wait(ctx[].nextSignal[1], ctx[].nextSignal[0]) - # if not nextSignal.waitSync(10.seconds).get(): - # raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") + if not nextSignal.waitSync(10.seconds).get(): + raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") var handle = handleRes.get() for item in handle.iter(): @@ -285,9 +282,7 @@ proc queryTask[DB]( discard ctx[].signal.fireSync() - # discard nextSignal.waitSync().get() - withLock(ctx[].nextSignal[0]): - wait(ctx[].nextSignal[1], ctx[].nextSignal[0]) + discard nextSignal.waitSync().get() # set final result (?!QResult).ok((KeyId.none, DataBuffer())) @@ -301,11 +296,8 @@ method query*(self: ThreadDatastore, await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - # without nextSignal =? acquireSignal(), err: - # return failure err - let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) - ctx[].nextSignal[0].initLock() - ctx[].nextSignal[1].initCond() + without nextSignal =? acquireSignal(), err: + return failure err try: let query = dbQuery( @@ -313,10 +305,10 @@ method query*(self: ThreadDatastore, value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) # setup initial queryTask + let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): - self.tp.spawn queryTask(ctx, ds, query) - withLock(ctx[].nextSignal[0]): - signal(ctx[].nextSignal[1]) + self.tp.spawn queryTask(ctx, ds, query, nextSignal) + await nextSignal.fire() var lock = newAsyncLock() # serialize querying under threads @@ -337,8 +329,7 @@ method query*(self: ThreadDatastore, iter.finished = true defer: - withLock(ctx[].nextSignal[0]): - signal(ctx[].nextSignal[1]) + await nextSignal.fire() if ctx[].res.isErr(): return err(ctx[].res.error()) @@ -351,7 +342,7 @@ method query*(self: ThreadDatastore, trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() discard ctx[].signal.close() - # discard nextSignal.close() + discard nextSignal.close() self.semaphore.release() raise exc @@ -360,7 +351,7 @@ method query*(self: ThreadDatastore, except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg discard signal.close() - # discard nextSignal.close() + discard nextSignal.close() self.semaphore.release() raise exc diff --git a/tests/datastore/sql/testsqlite.nim b/tests/datastore/sql/testsqlite.nim index 3e15c436..c629eb0c 100644 --- a/tests/datastore/sql/testsqlite.nim +++ b/tests/datastore/sql/testsqlite.nim @@ -8,7 +8,7 @@ import pkg/chronos import pkg/stew/results import pkg/stew/byteutils -import pkg/datastore/sql +import pkg/datastore/sql/sqliteds import ../dscommontests import ../querycommontests diff --git a/tests/datastore/testasyncsemaphore.nim b/tests/datastore/testasyncsemaphore.nim index aefff19c..3c8bb84b 100644 --- a/tests/datastore/testasyncsemaphore.nim +++ b/tests/datastore/testasyncsemaphore.nim @@ -98,6 +98,7 @@ suite "AsyncSemaphore": resource.inc() check resource > 0 and resource <= 3 let sleep = rand(0..10).millis + # echo sleep await sleepAsync(sleep) finally: resource.dec() From c67b61aa5060276d422eaf259f5d20ff2180efe5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:07:02 -0700 Subject: [PATCH 334/445] cleanup --- datastore/sql.nim | 21 ++++++++------------- tests/datastore/sql/testsqlite.nim | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 5b68b53d..28b1f3d0 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -13,7 +13,7 @@ import ../datastore import ./backend import ./sql/sqliteds -export datastore +export datastore, sqliteds push: {.upraises: [].} @@ -42,7 +42,8 @@ method delete*(self: SQLiteDatastore, method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.db.get(KeyId.new key.id()) + self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: + d.toSeq() method put*(self: SQLiteDatastore, key: Key, @@ -79,7 +80,7 @@ method queryIter*( yield QueryResponse.failure err let k = qres.key.map() do(k: KeyId) -> Key: Key.init($k).expect("valid key") - let v: seq[byte] = qres.data + let v: seq[byte] = qres.data.toSeq() yield success (k, v) success iter @@ -87,20 +88,14 @@ method queryIter*( proc new*( T: type SQLiteDatastore, path: string, - readOnly = false): ?!T = + readOnly = false): ?!SQLiteDatastore = - let - flags = - if readOnly: SQLITE_OPEN_READONLY - else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - - success T( - db: ? SQLiteDsDb.open(path, flags), - readOnly: readOnly) + success SQLiteDatastore( + db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) proc new*( T: type SQLiteDatastore, - db: SQLiteDsDb): ?!T = + db: SQLiteBackend[KeyId, DataBuffer]): ?!T = success T( db: db, diff --git a/tests/datastore/sql/testsqlite.nim b/tests/datastore/sql/testsqlite.nim index c629eb0c..b63bac7c 100644 --- a/tests/datastore/sql/testsqlite.nim +++ b/tests/datastore/sql/testsqlite.nim @@ -8,7 +8,7 @@ import pkg/chronos import pkg/stew/results import pkg/stew/byteutils -import pkg/datastore/sql/sqliteds +import pkg/datastore/sql import ../dscommontests import ../querycommontests @@ -25,6 +25,7 @@ suite "Test Basic SQLiteDatastore": basicStoreTests(ds, key, bytes, otherBytes) + suite "Test Read Only SQLiteDatastore": let path = currentSourcePath() # get this file's name @@ -76,14 +77,14 @@ suite "Test Read Only SQLiteDatastore": not (await readOnlyDb.has(key)).tryGet() not (await dsDb.has(key)).tryGet() -suite "Test Query": - var - ds: SQLiteDatastore +# suite "Test Query": +# var +# ds: SQLiteDatastore - setup: - ds = SQLiteDatastore.new(Memory).tryGet() +# setup: +# ds = SQLiteDatastore.new(Memory).tryGet() - teardown: - (await ds.close()).tryGet +# teardown: +# (await ds.close()).tryGet - queryTests(ds) +# queryTests(ds) From e2b6005918240791e4c37d4953f4a6199a5af195 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:10:46 -0700 Subject: [PATCH 335/445] cleanup --- tests/datastore/sql/testsqlitedsdb.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/datastore/sql/testsqlitedsdb.nim b/tests/datastore/sql/testsqlitedsdb.nim index e5bd1062..03fd98fe 100644 --- a/tests/datastore/sql/testsqlitedsdb.nim +++ b/tests/datastore/sql/testsqlitedsdb.nim @@ -28,7 +28,7 @@ suite "Test Open SQLite Datastore DB": test "Should create and open datastore DB": let - dsDb = SQLiteDsDb.open( + dsDb = SQLiteDsDb[string, seq[byte]].open( path = dbPathAbs, flags = SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE).tryGet() @@ -40,7 +40,7 @@ suite "Test Open SQLite Datastore DB": test "Should open existing DB": let - dsDb = SQLiteDsDb.open( + dsDb = SQLiteDsDb[string, seq[byte]].open( path = dbPathAbs, flags = SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE).tryGet() @@ -55,7 +55,7 @@ suite "Test Open SQLite Datastore DB": fileExists(dbPathAbs) let - dsDb = SQLiteDsDb.open( + dsDb = SQLiteDsDb[string, seq[byte]].open( path = dbPathAbs, flags = SQLITE_OPEN_READONLY).tryGet() @@ -66,7 +66,7 @@ suite "Test Open SQLite Datastore DB": removeDir(basePathAbs) check: not fileExists(dbPathAbs) - SQLiteDsDb.open(path = dbPathAbs).isErr + SQLiteDsDb[string, seq[byte]].open(path = dbPathAbs).isErr suite "Test SQLite Datastore DB operations": let @@ -81,19 +81,19 @@ suite "Test SQLite Datastore DB operations": otherData = "some other data".toBytes var - dsDb: SQLiteDsDb - readOnlyDb: SQLiteDsDb + dsDb: SQLiteDsDb[string, seq[byte]] + readOnlyDb: SQLiteDsDb[string, seq[byte]] setupAll: removeDir(basePathAbs) require(not dirExists(basePathAbs)) createDir(basePathAbs) - dsDb = SQLiteDsDb.open( + dsDb = SQLiteDsDb[string, seq[byte]].open( path = dbPathAbs, flags = SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE).tryGet() - readOnlyDb = SQLiteDsDb.open( + readOnlyDb = SQLiteDsDb[string, seq[byte]].open( path = dbPathAbs, flags = SQLITE_OPEN_READONLY).tryGet() @@ -114,7 +114,7 @@ suite "Test SQLite Datastore DB operations": var bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = dataCol(dsDb.getDataCol) + bytes = dataCol[seq[byte]](dsDb.getDataCol) check: dsDb.getStmt.query((key.id), onData).tryGet() @@ -130,7 +130,7 @@ suite "Test SQLite Datastore DB operations": var bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = dataCol(dsDb.getDataCol) + bytes = dataCol[seq[byte]](dsDb.getDataCol) check: dsDb.getStmt.query((key.id), onData).tryGet() From 76590d420771121cbb48fbe4ee1f10c75e201633 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:11:09 -0700 Subject: [PATCH 336/445] cleanup --- tests/datastore/testdatabuffer.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 5ec8127b..eb979317 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -101,7 +101,7 @@ suite "Share buffer test": check a.hash() != c.hash() test "key conversion": - check Key.init(a).tryGet == k1 + check Key.init($a).tryGet == k1 test "seq conversion": check string.fromBytes(a.toSeq()) == "/a/b" From c7284b3b6fc9a950b04598a6a68eb2d015a4a690 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:34:47 -0700 Subject: [PATCH 337/445] cleanup --- datastore/backend.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datastore/backend.nim b/datastore/backend.nim index f6af9838..01bae107 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -8,6 +8,12 @@ import ./types export databuffer, types, SortOrder +## Backend type for Datastores. +## +## These should be syncrhonous and work with both GC types +## and DataBuffer's. This makes it easier to make them threadsafe. +## + type DbQueryResponse*[K, V] = tuple[key: Option[K], data: V] @@ -22,6 +28,7 @@ type ## serialized Key ID, equivalent to `key.id()` data*: DataBuffer + ## Accepted backend key and value types DbKey* = string | KeyId DbVal* = seq[byte] | DataBuffer From 686ea8e4b3641d31fe6504a1b395d2f2c685866a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:37:40 -0700 Subject: [PATCH 338/445] cleanup --- datastore/backend.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 01bae107..5790b121 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -8,7 +8,7 @@ import ./types export databuffer, types, SortOrder -## Backend type for Datastores. +## Types for datastore backends. ## ## These should be syncrhonous and work with both GC types ## and DataBuffer's. This makes it easier to make them threadsafe. From 25e16f94759d3bab316a4dfbbb686e20544c8e2c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:41:57 -0700 Subject: [PATCH 339/445] docs --- datastore/threads/threadproxyds.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ae147824..2583d2cf 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -48,10 +48,12 @@ type TaskCtxObj*[T: ThreadTypes] = object res: ThreadResult[T] signal: ThreadSignalPtr - running: bool - cancelled: bool + running: bool ## used to mark when a task worker is running + cancelled: bool ## used to cancel a task before it's started TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] + ## Task context object. + ## This is a SharedPtr to make the query iter simpler ThreadDatastore* = ref object of Datastore tp: Taskpool From c168c8ea7f64587f1709b28717c06f5588dce59b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:46:36 -0700 Subject: [PATCH 340/445] docs --- datastore/threads/threadproxyds.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 2583d2cf..ef640158 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -41,6 +41,7 @@ type # Filesystem ThreadBackend* = object + ## backend case type to avoid needing to make ThreadDatastore generic case kind*: ThreadBackendKinds of Sqlite: sql*: SQLiteBackend[KeyId,DataBuffer] @@ -86,6 +87,8 @@ proc acquireSignal(): ?!ThreadSignalPtr = success signal.get() template executeTask[T](ctx: TaskCtx[T], blk: untyped) = + ## executes a task on a thread work and handles cleanup after cancels/errors + ## try: if not ctx.setRunning(): return @@ -108,7 +111,6 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = # ctx[].res.err exc.toThreadErr() finally: ctx.setDone() - # echo "\t\texecuteTask:fireSync!" discard ctx[].signal.fireSync() template dispatchTaskWrap[T](self: ThreadDatastore, @@ -121,13 +123,15 @@ template dispatchTaskWrap[T](self: ThreadDatastore, proc runTask() = `blk` runTask() - # echo "\t\tdispatchTask:wait!" await wait(ctx[].signal) template dispatchTask[T](self: ThreadDatastore, signal: ThreadSignalPtr, blk: untyped ): auto = + ## handles dispatching a task from an async context + ## `blk` is the actions, it has `ctx` and `ds` variables in scope. + ## note that `ds` is a generic let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) try: dispatchTaskWrap[T](self, signal, blk) From 10bee6885d45534003bf1b3da5645e1681707acb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:47:43 -0700 Subject: [PATCH 341/445] docs --- datastore/threads/threadproxyds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ef640158..772dc721 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -376,6 +376,7 @@ proc new*[DB](self: type ThreadDatastore, success ThreadDatastore( tp: tp, backend: backend, + # TODO: are these needed anymore?? # withLocks: withLocks, # queryLock: newAsyncLock(), semaphore: AsyncSemaphore.new(tp.numThreads - 1) From d1f503fa964b271398fa710582f6936d70fdbee7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 20:53:28 -0700 Subject: [PATCH 342/445] reduce thread count --- tests/datastore/testthreadproxyds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index cfef8e6c..14a41b69 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -22,7 +22,7 @@ import pkg/datastore/threads/threadproxyds {.all.} import ./dscommontests import ./querycommontests -const NumThreads = 200 # IO threads aren't attached to CPU count +const NumThreads = 20 # IO threads aren't attached to CPU count suite "Test Basic ThreadProxyDatastore": var From 51d2b47205d8414e58be3f6a9c2f880bb4f7b78f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:03:07 -0700 Subject: [PATCH 343/445] initial take at making fsds synchronous --- datastore/fsds.nim | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 65f5cdd4..ffb3575d 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -8,6 +8,7 @@ import pkg/questionable/results from pkg/stew/results as stewResults import get, isErr import pkg/upraises +import ./backend import ./datastore export datastore @@ -16,15 +17,15 @@ push: {.upraises: [].} type FSDatastore* = ref object of Datastore - root*: string + root*: DataBuffer ignoreProtected: bool depth: int proc validDepth*(self: FSDatastore, key: Key): bool = key.len <= self.depth -proc isRootSubdir*(self: FSDatastore, path: string): bool = - path.startsWith(self.root) +proc isRootSubdir*(root, path: string): bool = + path.startsWith(root) proc path*(self: FSDatastore, key: Key): ?!string = ## Return filename corresponding to the key @@ -53,21 +54,22 @@ proc path*(self: FSDatastore, key: Key): ?!string = segments.add(ns.field / ns.value) let - fullname = (self.root / segments.joinPath()) + root = $self.root + fullname = (root / segments.joinPath()) .absolutePath() .catch() .get() .addFileExt(FileExt) - if not self.isRootSubdir(fullname): + if not root.isRootSubdir(fullname): return failure "Path is outside of `root` directory!" return success fullname -method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = +proc has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = return self.path(key).?fileExists() -method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = +proc delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = without path =? self.path(key), error: return failure error @@ -81,7 +83,7 @@ method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = return success() -method delete*(self: FSDatastore, keys: seq[Key]): Future[?!void] {.async.} = +proc delete*(self: FSDatastore, keys: seq[Key]): Future[?!void] {.async.} = for key in keys: if err =? (await self.delete(key)).errorOption: return failure err @@ -118,7 +120,7 @@ proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = except CatchableError as e: return failure e -method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = +proc get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = without path =? self.path(key), error: return failure error @@ -128,7 +130,7 @@ method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = return self.readFile(path) -method put*( +proc put*( self: FSDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = @@ -144,7 +146,7 @@ method put*( return success() -method put*( +proc put*( self: FSDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = @@ -165,10 +167,10 @@ proc dirWalker(path: string): iterator: string {.gcsafe.} = except CatchableError as exc: raise newException(Defect, exc.msg) -method close*(self: FSDatastore): Future[?!void] {.async.} = +proc close*(self: FSDatastore): Future[?!void] {.async.} = return success() -method query*( +proc query*( self: FSDatastore, query: Query): Future[?!QueryIter] {.async.} = @@ -201,6 +203,7 @@ method query*( return failure (ref DatastoreError)(msg: "Should always await query features") let + root = $self.root path = walker() if iter.finished: @@ -215,7 +218,7 @@ method query*( var keyPath = basePath - keyPath.removePrefix(self.root) + keyPath.removePrefix(root) keyPath = keyPath / path.changeFileExt("") keyPath = keyPath.replace("\\", "/") From 24ac1a870807afc090d2efd64e8b8262ccba328b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:15:41 -0700 Subject: [PATCH 344/445] initial take at making fsds synchronous --- datastore/fsds.nim | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index ffb3575d..d16fc704 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -2,7 +2,6 @@ import std/os import std/options import std/strutils -import pkg/chronos import pkg/questionable import pkg/questionable/results from pkg/stew/results as stewResults import get, isErr @@ -16,7 +15,7 @@ export datastore push: {.upraises: [].} type - FSDatastore* = ref object of Datastore + FSDatastore* = object root*: DataBuffer ignoreProtected: bool depth: int @@ -66,10 +65,13 @@ proc path*(self: FSDatastore, key: Key): ?!string = return success fullname -proc has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = +proc has*(self: FSDatastore, key: KeyId): ?!bool = + let key = key.toKey() return self.path(key).?fileExists() -proc delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = +proc delete*(self: FSDatastore, key: KeyId): ?!void = + let key = key.toKey() + without path =? self.path(key), error: return failure error @@ -83,9 +85,9 @@ proc delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = return success() -proc delete*(self: FSDatastore, keys: seq[Key]): Future[?!void] {.async.} = +proc delete*(self: FSDatastore, keys: openArray[KeyId]): ?!void = for key in keys: - if err =? (await self.delete(key)).errorOption: + if err =? self.delete(key).errorOption: return failure err return success() @@ -120,7 +122,7 @@ proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = except CatchableError as e: return failure e -proc get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = +proc get*(self: FSDatastore, key: Key): ?!seq[byte] = without path =? self.path(key), error: return failure error @@ -133,7 +135,7 @@ proc get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = proc put*( self: FSDatastore, key: Key, - data: seq[byte]): Future[?!void] {.async.} = + data: seq[byte]): ?!void = without path =? self.path(key), error: return failure error @@ -148,10 +150,10 @@ proc put*( proc put*( self: FSDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = + batch: seq[BatchEntry]): ?!void = for entry in batch: - if err =? (await self.put(entry.key, entry.data)).errorOption: + if err =? self.put(entry.key, entry.data).errorOption: return failure err return success() @@ -167,12 +169,12 @@ proc dirWalker(path: string): iterator: string {.gcsafe.} = except CatchableError as exc: raise newException(Defect, exc.msg) -proc close*(self: FSDatastore): Future[?!void] {.async.} = +proc close*(self: FSDatastore): ?!void = return success() proc query*( self: FSDatastore, - query: Query): Future[?!QueryIter] {.async.} = + query: Query): ?!QueryIter = without path =? self.path(query.key), error: return failure error @@ -193,14 +195,14 @@ proc query*( var iter = QueryIter.new() - var lock = newAsyncLock() # serialize querying under threads - proc next(): Future[?!QueryResponse] {.async.} = - defer: - if lock.locked: - lock.release() + # var lock = newAsyncLock() # serialize querying under threads + proc next(): ?!QueryResponse = + # defer: + # if lock.locked: + # lock.release() - if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") + # if lock.locked: + # return failure (ref DatastoreError)(msg: "Should always await query features") let root = $self.root @@ -209,7 +211,7 @@ proc query*( if iter.finished: return failure "iterator is finished" - await lock.acquire() + # await lock.acquire() if finished(walker): iter.finished = true From 653ec99b5341e3ea90ed2bdddf9719c5025c3edd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:27:46 -0700 Subject: [PATCH 345/445] initial take at making fsds synchronous --- datastore/fsds.nim | 26 ++++++++++++++++---------- datastore/threads/databuffer.nim | 9 ++++++--- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index d16fc704..2e9ad406 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -92,7 +92,7 @@ proc delete*(self: FSDatastore, keys: openArray[KeyId]): ?!void = return success() -proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = +proc readFile*[V](self: FSDatastore, path: string): ?!V = var file: File @@ -104,14 +104,19 @@ proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = try: let - size = file.getFileSize + size = file.getFileSize().int + when V is seq[byte]: + var bytes = newSeq[byte](size) + elif V is DataBuffer: + var bytes = DataBuffer.new(capacity=size) + else: + {.error: "unhandled result type".} var - bytes = newSeq[byte](size) read = 0 while read < size: - read += file.readBytes(bytes, read, size) + read += file.readBytes(bytes.toOpenArray(), read, size) if read < size: return failure $read & " bytes were read from " & path & @@ -122,7 +127,8 @@ proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = except CatchableError as e: return failure e -proc get*(self: FSDatastore, key: Key): ?!seq[byte] = +proc get*(self: FSDatastore, key: KeyId): ?!DataBuffer = + let key = key.toKey() without path =? self.path(key), error: return failure error @@ -130,12 +136,12 @@ proc get*(self: FSDatastore, key: Key): ?!seq[byte] = return failure( newException(DatastoreKeyNotFound, "Key doesn't exist")) - return self.readFile(path) + return readFile[DataBuffer](self, path) proc put*( self: FSDatastore, - key: Key, - data: seq[byte]): ?!void = + key: KeyId, + data: DataBuffer): ?!void = without path =? self.path(key), error: return failure error @@ -174,7 +180,7 @@ proc close*(self: FSDatastore): ?!void = proc query*( self: FSDatastore, - query: Query): ?!QueryIter = + query: DbQuery[KeyId, DataBuffer]): ?!QueryIter = without path =? self.path(query.key), error: return failure error @@ -228,7 +234,7 @@ proc query*( key = Key.init(keyPath).expect("should not fail") data = if query.value: - self.readFile((basePath / path).absolutePath) + self.readFile[DataBuffer]((basePath / path).absolutePath) .expect("Should read file") else: @[] diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 5ea7b4be..1b5e6d25 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -47,7 +47,7 @@ template `==`*[T: char | byte](a: DataBuffer, b: openArray[T]): bool = elif a[].size != b.len: false else: a.hash() == b.hash() -proc new(tp: type DataBuffer, capacity: int = 0): DataBuffer = +proc new*(tp: type DataBuffer, capacity: int = 0): DataBuffer = ## allocate new buffer with given capacity ## @@ -130,9 +130,12 @@ converter toBuffer*(err: ref CatchableError): DataBuffer = return DataBuffer.new(err.msg) -template toOpenArray*[T: byte | char](data: DataBuffer, t: typedesc[T]): openArray[T] = +template toOpenArray*[T: byte | char](data: var DataBuffer, t: typedesc[T]): var openArray[T] = ## get openArray from DataBuffer as char ## ## this is explicit since sqlite treats string differently from openArray[byte] - let bf = cast[ptr UncheckedArray[T]](data[].buf) + var bf = cast[ptr UncheckedArray[T]](data[].buf) bf.toOpenArray(0, data[].size-1) + +template toOpenArray*(data: var DataBuffer): var openArray[byte] = + toOpenArray(data, byte) From 5242b85b3693131764d7bdd0552177c3561ad00d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:28:48 -0700 Subject: [PATCH 346/445] initial take at making fsds synchronous --- datastore/fsds.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 2e9ad406..24b3f06d 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -142,13 +142,15 @@ proc put*( self: FSDatastore, key: KeyId, data: DataBuffer): ?!void = + let key = key.toKey() without path =? self.path(key), error: return failure error try: + var data = data createDir(parentDir(path)) - writeFile(path, data) + writeFile(path, data.toOpenArray()) except CatchableError as e: return failure e From bb9e343e9ffe59511f5fd8afe188a90692b793ec Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:29:12 -0700 Subject: [PATCH 347/445] initial take at making fsds synchronous --- datastore/fsds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 24b3f06d..35585bd3 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -158,7 +158,7 @@ proc put*( proc put*( self: FSDatastore, - batch: seq[BatchEntry]): ?!void = + batch: seq[DbBatchEntry[KeyId, DataBuffer]]): ?!void = for entry in batch: if err =? self.put(entry.key, entry.data).errorOption: @@ -182,7 +182,7 @@ proc close*(self: FSDatastore): ?!void = proc query*( self: FSDatastore, - query: DbQuery[KeyId, DataBuffer]): ?!QueryIter = + query: DbQuery[KeyId]): ?!QueryIter = without path =? self.path(query.key), error: return failure error From 235232a84f5ca5c2f8198edafa34ce4e35df0851 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 26 Sep 2023 21:30:59 -0700 Subject: [PATCH 348/445] initial take at making fsds synchronous --- datastore/fsds.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 35585bd3..b3162d82 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -184,7 +184,8 @@ proc query*( self: FSDatastore, query: DbQuery[KeyId]): ?!QueryIter = - without path =? self.path(query.key), error: + let key = query.key.toKey() + without path =? self.path(key), error: return failure error let basePath = @@ -236,7 +237,7 @@ proc query*( key = Key.init(keyPath).expect("should not fail") data = if query.value: - self.readFile[DataBuffer]((basePath / path).absolutePath) + readFile[DataBuffer](self, (basePath / path).absolutePath) .expect("Should read file") else: @[] From 5e424262c3cb39863d259fc8bcf7c683144d2158 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 11:51:55 -0700 Subject: [PATCH 349/445] rework tuple types --- datastore/threads/threadproxyds.nim | 96 ++++++++++++----------------- 1 file changed, 41 insertions(+), 55 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 772dc721..f9d11894 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -36,16 +36,6 @@ logScope: type - ThreadBackendKinds* = enum - Sqlite - # Filesystem - - ThreadBackend* = object - ## backend case type to avoid needing to make ThreadDatastore generic - case kind*: ThreadBackendKinds - of Sqlite: - sql*: SQLiteBackend[KeyId,DataBuffer] - TaskCtxObj*[T: ThreadTypes] = object res: ThreadResult[T] signal: ThreadSignalPtr @@ -56,15 +46,18 @@ type ## Task context object. ## This is a SharedPtr to make the query iter simpler - ThreadDatastore* = ref object of Datastore + ThreadDatastore*[BT] = ref object of Datastore tp: Taskpool - backend: ThreadBackend + backend: BT semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors var ctxLock: Lock ctxLock.initLock() +proc newTaskCtx*[T](signal: ThreadSignalPtr): TaskCtx[T] = + newSharedPtr(TaskCtxObj[T](signal: signal)) + proc setCancelled[T](ctx: TaskCtx[T]) = # withLock(ctxLock): ctx[].cancelled = true @@ -113,28 +106,26 @@ template executeTask[T](ctx: TaskCtx[T], blk: untyped) = ctx.setDone() discard ctx[].signal.fireSync() -template dispatchTaskWrap[T](self: ThreadDatastore, +template dispatchTaskWrap[BT](self: ThreadDatastore[BT], + signal: ThreadSignalPtr, + blk: untyped + ): auto = + var ds {.used, inject.} = self.backend + proc runTask() = + `blk` + runTask() + await wait(ctx[].signal) + +template dispatchTask[BT](self: ThreadDatastore[BT], signal: ThreadSignalPtr, blk: untyped - ): auto = - case self.backend.kind: - of Sqlite: - var ds {.used, inject.} = self.backend.sql - proc runTask() = - `blk` - runTask() - await wait(ctx[].signal) - -template dispatchTask[T](self: ThreadDatastore, - signal: ThreadSignalPtr, - blk: untyped - ): auto = + ): auto = ## handles dispatching a task from an async context ## `blk` is the actions, it has `ctx` and `ds` variables in scope. ## note that `ds` is a generic - let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) + # let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) try: - dispatchTaskWrap[T](self, signal, blk) + dispatchTaskWrap[BT](self, signal, blk) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() @@ -149,38 +140,39 @@ proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = executeTask(ctx): has(ds, key) -method has*(self: ThreadDatastore, +method has*[BT](self: ThreadDatastore[BT], key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - let key = KeyId.new key.id() - dispatchTask[bool](self, signal): + let ctx = newTaskCtx(bool, signal: signal) + dispatchTask(self, signal): + let key = KeyId.new key.id() self.tp.spawn hasTask(ctx, ds, key) return ctx[].res.toRes(v => v) - proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): delete(ds, key) -method delete*(self: ThreadDatastore, +method delete*[BT](self: ThreadDatastore[BT], key: Key): Future[?!void] {.async.} = ## delete key await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - let key = KeyId.new key.id() - dispatchTask[void](self, signal): + let ctx = newTaskCtx[void](signal: signal) + dispatchTask(self, signal): + let key = KeyId.new key.id() self.tp.spawn deleteTask(ctx, ds, key) return ctx[].res.toRes() -method delete*(self: ThreadDatastore, +method delete*[BT](self: ThreadDatastore[BT], keys: seq[Key]): Future[?!void] {.async.} = ## delete batch for key in keys: @@ -196,7 +188,7 @@ proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; executeTask(ctx): put(ds, key, data) -method put*(self: ThreadDatastore, +method put*[BT](self: ThreadDatastore[BT], key: Key, data: seq[byte]): Future[?!void] {.async.} = ## put key with data @@ -204,7 +196,8 @@ method put*(self: ThreadDatastore, without signal =? acquireSignal(), err: return failure err - dispatchTask[void](self, signal): + let ctx = newTaskCtx[void](signal: signal) + dispatchTask(self, signal): let key = KeyId.new key.id() let data = DataBuffer.new data self.tp.spawn putTask(ctx, ds, key, data) @@ -229,24 +222,22 @@ proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; let res = get(ds, key) res -method get*(self: ThreadDatastore, - key: Key, - ): Future[?!seq[byte]] {.async.} = +method get*[BT](self: ThreadDatastore[BT], + key: Key, + ): Future[?!seq[byte]] {.async.} = await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - let key = KeyId.new key.id() + let ctx = newTaskCtx[void](signal: signal) dispatchTask[DataBuffer](self, signal): self.tp.spawn getTask(ctx, ds, key) return ctx[].res.toRes(v => v.toSeq()) -method close*(self: ThreadDatastore): Future[?!void] {.async.} = +method close*[BT](self: ThreadDatastore[BT]): Future[?!void] {.async.} = await self.semaphore.closeAll() - case self.backend.kind: - of Sqlite: - self.backend.sql.close() + self.backend.close() type QResult = DbQueryResponse[KeyId, DataBuffer] @@ -293,7 +284,7 @@ proc queryTask[DB]( # set final result (?!QResult).ok((KeyId.none, DataBuffer())) -method query*(self: ThreadDatastore, +method query*[BT](self: ThreadDatastore[BT], q: Query ): Future[?!QueryIter] {.async.} = ## performs async query @@ -311,8 +302,8 @@ method query*(self: ThreadDatastore, value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) # setup initial queryTask - let ctx {.inject.} = newSharedPtr(TaskCtxObj[QResult](signal: signal)) - dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer]](self, signal): + let ctx = newTaskCtx[QResult](signal: signal) + dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer], BT](self, signal): self.tp.spawn queryTask(ctx, ds, query, nextSignal) await nextSignal.fire() @@ -368,14 +359,9 @@ proc new*[DB](self: type ThreadDatastore, ): ?!ThreadDatastore = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" - when DB is SQLiteBackend[KeyId,DataBuffer]: - let backend = ThreadBackend(kind: Sqlite, sql: db) - else: - {.error: "unsupported backend: " & $typeof(db).} - success ThreadDatastore( tp: tp, - backend: backend, + backend: db, # TODO: are these needed anymore?? # withLocks: withLocks, # queryLock: newAsyncLock(), From 49d846f468cf846b81da3d264c7e0a66fe0fb04f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:02:04 -0700 Subject: [PATCH 350/445] make Threadproxyds generic --- datastore/threads/threadproxyds.nim | 24 +++++++++++------------ tests/datastore/testthreadproxyds.nim | 28 +++++++-------------------- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index f9d11894..c8e98ad5 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -55,7 +55,7 @@ type var ctxLock: Lock ctxLock.initLock() -proc newTaskCtx*[T](signal: ThreadSignalPtr): TaskCtx[T] = +proc newTaskCtx*[T](tp: typedesc[T], signal: ThreadSignalPtr): TaskCtx[T] = newSharedPtr(TaskCtxObj[T](signal: signal)) proc setCancelled[T](ctx: TaskCtx[T]) = @@ -123,7 +123,6 @@ template dispatchTask[BT](self: ThreadDatastore[BT], ## handles dispatching a task from an async context ## `blk` is the actions, it has `ctx` and `ds` variables in scope. ## note that `ds` is a generic - # let ctx {.inject.} = newSharedPtr(TaskCtxObj[T](signal: signal)) try: dispatchTaskWrap[BT](self, signal, blk) except CancelledError as exc: @@ -146,13 +145,13 @@ method has*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err - let ctx = newTaskCtx(bool, signal: signal) + let ctx = newTaskCtx(bool, signal=signal) dispatchTask(self, signal): let key = KeyId.new key.id() self.tp.spawn hasTask(ctx, ds, key) return ctx[].res.toRes(v => v) -proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; +method deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -165,7 +164,7 @@ method delete*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err - let ctx = newTaskCtx[void](signal: signal) + let ctx = newTaskCtx(void, signal=signal) dispatchTask(self, signal): let key = KeyId.new key.id() self.tp.spawn deleteTask(ctx, ds, key) @@ -196,7 +195,7 @@ method put*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err - let ctx = newTaskCtx[void](signal: signal) + let ctx = newTaskCtx(void, signal=signal) dispatchTask(self, signal): let key = KeyId.new key.id() let data = DataBuffer.new data @@ -229,8 +228,9 @@ method get*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err - let ctx = newTaskCtx[void](signal: signal) - dispatchTask[DataBuffer](self, signal): + let ctx = newTaskCtx(DataBuffer, signal=signal) + dispatchTask(self, signal): + let key = KeyId.new key.id() self.tp.spawn getTask(ctx, ds, key) return ctx[].res.toRes(v => v.toSeq()) @@ -302,8 +302,8 @@ method query*[BT](self: ThreadDatastore[BT], value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) # setup initial queryTask - let ctx = newTaskCtx[QResult](signal: signal) - dispatchTaskWrap[DbQueryResponse[KeyId, DataBuffer], BT](self, signal): + let ctx = newTaskCtx(QResult, signal=signal) + dispatchTaskWrap(self, signal): self.tp.spawn queryTask(ctx, ds, query, nextSignal) await nextSignal.fire() @@ -356,10 +356,10 @@ proc new*[DB](self: type ThreadDatastore, db: DB, withLocks = static false, tp: Taskpool - ): ?!ThreadDatastore = + ): ?!ThreadDatastore[DB] = doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" - success ThreadDatastore( + success ThreadDatastore[DB]( tp: tp, backend: db, # TODO: are these needed anymore?? diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 14a41b69..72758b04 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -25,14 +25,10 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count suite "Test Basic ThreadProxyDatastore": + var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool key = Key.init("/a").tryGet() data = "some bytes".toBytes - - setupAll: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() @@ -61,17 +57,12 @@ suite "Test Basic ThreadProxyDatastore": suite "Test Basic ThreadDatastore with SQLite": var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setupAll: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes teardown: GC_fullCollect() @@ -85,17 +76,12 @@ suite "Test Basic ThreadDatastore with SQLite": suite "Test Query ThreadDatastore with SQLite": var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setup: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes teardown: GC_fullCollect() From 6a4e460e58002448286830c1b544b1e3270fcd3d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:41:27 -0700 Subject: [PATCH 351/445] queryLocks can be done at FsDs backend level --- datastore/threads/threadproxyds.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 772dc721..dcc16800 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -376,8 +376,5 @@ proc new*[DB](self: type ThreadDatastore, success ThreadDatastore( tp: tp, backend: backend, - # TODO: are these needed anymore?? - # withLocks: withLocks, - # queryLock: newAsyncLock(), semaphore: AsyncSemaphore.new(tp.numThreads - 1) ) From b4b534bf67d8ac1928492a70fece524759148ef2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:54:22 -0700 Subject: [PATCH 352/445] loop tests --- datastore/threads/threadproxyds.nim | 2 + tests/datastore/testthreadproxyds.nim | 136 ++++++++++---------------- 2 files changed, 56 insertions(+), 82 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index dcc16800..432eb6f8 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -348,6 +348,7 @@ method query*(self: ThreadDatastore, trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() discard ctx[].signal.close() + echo "nextSignal:CLOSE!" discard nextSignal.close() self.semaphore.release() raise exc @@ -357,6 +358,7 @@ method query*(self: ThreadDatastore, except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg discard signal.close() + echo "nextSignal:CLOSE!" discard nextSignal.close() self.semaphore.release() raise exc diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 14a41b69..27c8e829 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -22,88 +22,60 @@ import pkg/datastore/threads/threadproxyds {.all.} import ./dscommontests import ./querycommontests -const NumThreads = 20 # IO threads aren't attached to CPU count - -suite "Test Basic ThreadProxyDatastore": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a").tryGet() - data = "some bytes".toBytes - - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardownAll: - echo "teardown done" - - test "check put": - echo "\n\n=== put ===" - let res1 = await ds.put(key, data) - echo "res1: ", res1.repr - check res1.isOk - - test "check get": - echo "\n\n=== get ===" - echo "get send key: ", key.repr - let res2 = await ds.get(key) - echo "get key post: ", key.repr - echo "get res2: ", res2.repr - echo res2.get() == data - var val = "" - for c in res2.get(): - val &= char(c) - echo "get res2: ", $val - -suite "Test Basic ThreadDatastore with SQLite": - - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() - - basicStoreTests(ds, key, bytes, otherBytes) - -suite "Test Query ThreadDatastore with SQLite": - - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setup: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - (await ds.close()).tryGet() - taskPool.shutdown() - - queryTests(ds, true) +const + NumThreads = 20 # IO threads aren't attached to CPU count + N = 100 + +for i in 1..N: + suite "Test Basic ThreadDatastore with SQLite": + + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setupAll: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + + teardown: + GC_fullCollect() + + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() + + basicStoreTests(ds, key, bytes, otherBytes) + GC_fullCollect() + +for i in 1..N: + suite "Test Query ThreadDatastore with SQLite": + + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setup: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + + teardown: + GC_fullCollect() + + (await ds.close()).tryGet() + taskPool.shutdown() + + queryTests(ds, true) + GC_fullCollect() # suite "Test Basic ThreadDatastore with fsds": # let From 030084186ef68472f4e0c7f60d962f95aee9ad93 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:55:48 -0700 Subject: [PATCH 353/445] loop tests --- tests/datastore/testthreadproxyds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 27c8e829..b42cbb0a 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,7 +24,8 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count - N = 100 + ThreadTestLoops {.intdefine.} = 10 + N = ThreadTestLoops for i in 1..N: suite "Test Basic ThreadDatastore with SQLite": From 1fd80c6f2be318b9da26a5ffea34326cc3f236fb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:12:32 -0700 Subject: [PATCH 354/445] fix merge --- datastore/threads/threadproxyds.nim | 2 +- tests/datastore/testthreadproxyds.nim | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index b573bf1e..422c548e 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -204,7 +204,7 @@ method put*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes() method put*[DB]( - self: ThreadDatastore, + self: ThreadDatastore[DB], batch: seq[BatchEntry]): Future[?!void] {.async.} = ## put batch data for entry in batch: diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index f415ba61..000c52d7 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -63,7 +63,6 @@ for i in 1..N: key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - setup: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() From c51d354f72a40ea7c0b9b09c126ab80e2294b6a9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:16:30 -0700 Subject: [PATCH 355/445] add nextSignal using mutex --- datastore/threads/threadproxyds.nim | 8 ++++++++ tests/datastore/testthreadproxyds.nim | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 422c548e..e273e062 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -41,6 +41,7 @@ type signal: ThreadSignalPtr running: bool ## used to mark when a task worker is running cancelled: bool ## used to cancel a task before it's started + nextSignal: (Lock, Cond) TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] ## Task context object. @@ -72,6 +73,13 @@ proc setDone[T](ctx: TaskCtx[T]) = # withLock(ctxLock): ctx[].running = false +proc waitSync*(sig: var (Lock, Cond)) = + withLock(sig[0]): + wait(sig[1], sig[0]) +proc fireSync*(sig: var (Lock, Cond)) = + withLock(sig[0]): + signal(sig[1]) + proc acquireSignal(): ?!ThreadSignalPtr = let signal = ThreadSignalPtr.new() if signal.isErr(): diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 000c52d7..57fea886 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,7 +24,7 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count - ThreadTestLoops {.intdefine.} = 10 + ThreadTestLoops {.intdefine.} = 100 N = ThreadTestLoops for i in 1..N: From 30728c1d0f0ae85068bd774f8e0ca420eaee293c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:24:32 -0700 Subject: [PATCH 356/445] add nextSignal using mutex --- datastore/threads/threadproxyds.nim | 17 ++++------------- datastore/threads/threadresult.nim | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e273e062..1c5528ba 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -41,7 +41,7 @@ type signal: ThreadSignalPtr running: bool ## used to mark when a task worker is running cancelled: bool ## used to cancel a task before it's started - nextSignal: (Lock, Cond) + nextSignal: MutexSignal TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] ## Task context object. @@ -73,13 +73,6 @@ proc setDone[T](ctx: TaskCtx[T]) = # withLock(ctxLock): ctx[].running = false -proc waitSync*(sig: var (Lock, Cond)) = - withLock(sig[0]): - wait(sig[1], sig[0]) -proc fireSync*(sig: var (Lock, Cond)) = - withLock(sig[0]): - signal(sig[1]) - proc acquireSignal(): ?!ThreadSignalPtr = let signal = ThreadSignalPtr.new() if signal.isErr(): @@ -254,7 +247,6 @@ method queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, query: DbQuery[KeyId], - nextSignal: ThreadSignalPtr ) {.gcsafe, nimcall.} = ## run query command executeTask(ctx): @@ -299,8 +291,8 @@ method query*[BT](self: ThreadDatastore[BT], await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - without nextSignal =? acquireSignal(), err: - return failure err + let ctx = newTaskCtx(QResult, signal=signal) + ctx[].nextSignal.init() try: let query = dbQuery( @@ -308,9 +300,8 @@ method query*[BT](self: ThreadDatastore[BT], value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) # setup initial queryTask - let ctx = newTaskCtx(QResult, signal=signal) dispatchTaskWrap(self, signal): - self.tp.spawn queryTask(ctx, ds, query, nextSignal) + self.tp.spawn queryTask(ctx, ds, query) await nextSignal.fire() var diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index da0df6dc..9caa70c9 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -1,5 +1,6 @@ import std/atomics import std/options +import std/locks import pkg/questionable/results import pkg/results @@ -51,3 +52,24 @@ proc toRes*[T,S](res: ThreadResult[T], result.err res.error().toExc() else: result.ok m(res.get()) + +type + MutexSignal* = tuple[lock: Lock, cond: Cond, open: bool] + +proc open*(sig: var MutexSignal) = + sig.lock.initLock() + sig.cond.initCond() + sig.open = true + +proc waitSync*(sig: var MutexSignal) = + withLock(sig.lock): + wait(sig.cond, sig.lock) + +proc fireSync*(sig: var MutexSignal) = + withLock(sig.lock): + signal(sig.cond) + +proc close*(sig: var MutexSignal) = + if sig.open: + sig.lock.deinitLock() + sig.cond.deinitCond() From e571d2116a986fb7f2233e084170a182afee9b05 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:28:58 -0700 Subject: [PATCH 357/445] add nextSignal using mutex --- datastore/threads/threadproxyds.nim | 14 ++++++-------- datastore/threads/threadresult.nim | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 1c5528ba..8a543b81 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -260,8 +260,7 @@ method queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - if not nextSignal.waitSync(10.seconds).get(): - raise newException(DeadThreadDefect, "query task timeout; possible deadlock!") + ctx[].nextSignal.wait() var handle = handleRes.get() for item in handle.iter(): @@ -276,8 +275,7 @@ method queryTask[DB]( exc discard ctx[].signal.fireSync() - - discard nextSignal.waitSync().get() + ctx[].nextSignal.wait() # set final result (?!QResult).ok((KeyId.none, DataBuffer())) @@ -302,7 +300,7 @@ method query*[BT](self: ThreadDatastore[BT], # setup initial queryTask dispatchTaskWrap(self, signal): self.tp.spawn queryTask(ctx, ds, query) - await nextSignal.fire() + ctx[].nextSignal.fire() var lock = newAsyncLock() # serialize querying under threads @@ -323,7 +321,7 @@ method query*[BT](self: ThreadDatastore[BT], iter.finished = true defer: - await nextSignal.fire() + ctx[].nextSignal.fire() if ctx[].res.isErr(): return err(ctx[].res.error()) @@ -337,7 +335,7 @@ method query*[BT](self: ThreadDatastore[BT], ctx.setCancelled() discard ctx[].signal.close() echo "nextSignal:CLOSE!" - discard nextSignal.close() + ctx[].nextSignal.close() self.semaphore.release() raise exc @@ -347,7 +345,7 @@ method query*[BT](self: ThreadDatastore[BT], trace "Cancelling thread future!", exc = exc.msg discard signal.close() echo "nextSignal:CLOSE!" - discard nextSignal.close() + ctx[].nextSignal.close() self.semaphore.release() raise exc diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 9caa70c9..3508e9ee 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -56,16 +56,16 @@ proc toRes*[T,S](res: ThreadResult[T], type MutexSignal* = tuple[lock: Lock, cond: Cond, open: bool] -proc open*(sig: var MutexSignal) = +proc init*(sig: var MutexSignal) = sig.lock.initLock() sig.cond.initCond() sig.open = true -proc waitSync*(sig: var MutexSignal) = +proc wait*(sig: var MutexSignal) = withLock(sig.lock): wait(sig.cond, sig.lock) -proc fireSync*(sig: var MutexSignal) = +proc fire*(sig: var MutexSignal) = withLock(sig.lock): signal(sig.cond) From 3cc21b3a59279cc01b971b12e6258c5e0dcaa56c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:30:14 -0700 Subject: [PATCH 358/445] add nextSignal using mutex --- tests/datastore/testthreadproxyds.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 57fea886..c5b6ab0f 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,7 +24,7 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count - ThreadTestLoops {.intdefine.} = 100 + ThreadTestLoops {.intdefine.} = 1000 N = ThreadTestLoops for i in 1..N: @@ -60,9 +60,6 @@ for i in 1..N: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes setup: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() From 1da59ba730c03e1dae30bd2607b2c29ab0991782 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:35:14 -0700 Subject: [PATCH 359/445] add nextSignal using mutex --- datastore/threads/threadproxyds.nim | 3 +++ tests/datastore/testthreadproxyds.nim | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8a543b81..ae9f64c7 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -131,6 +131,7 @@ template dispatchTask[BT](self: ThreadDatastore[BT], ctx.setCancelled() raise exc finally: + echo "signal:CLOSE!" discard ctx[].signal.close() self.semaphore.release() @@ -333,6 +334,7 @@ method query*[BT](self: ThreadDatastore[BT], except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() + echo "signal:CLOSE!" discard ctx[].signal.close() echo "nextSignal:CLOSE!" ctx[].nextSignal.close() @@ -343,6 +345,7 @@ method query*[BT](self: ThreadDatastore[BT], return success iter except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg + echo "signal:CLOSE!" discard signal.close() echo "nextSignal:CLOSE!" ctx[].nextSignal.close() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index c5b6ab0f..e0cb8229 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,8 +24,10 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count - ThreadTestLoops {.intdefine.} = 1000 + ThreadTestLoops {.intdefine.} = 1 N = ThreadTestLoops + ThreadTestInnerLoops {.intdefine.} = 1 + M = ThreadTestInnerLoops for i in 1..N: suite "Test Basic ThreadDatastore with SQLite": @@ -50,7 +52,8 @@ for i in 1..N: (await ds.close()).tryGet() taskPool.shutdown() - basicStoreTests(ds, key, bytes, otherBytes) + for i in 1..M: + basicStoreTests(ds, key, bytes, otherBytes) GC_fullCollect() for i in 1..N: @@ -72,7 +75,8 @@ for i in 1..N: (await ds.close()).tryGet() taskPool.shutdown() - queryTests(ds, true) + for i in 1..M: + queryTests(ds, true) GC_fullCollect() # suite "Test Basic ThreadDatastore with fsds": From ac77917146c3bdeed72dc2bc9e4d7e75f61f6da6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:36:16 -0700 Subject: [PATCH 360/445] remove ctx locks --- datastore/threads/threadproxyds.nim | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index ae9f64c7..8c84fc72 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -53,27 +53,22 @@ type semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors -var ctxLock: Lock -ctxLock.initLock() - proc newTaskCtx*[T](tp: typedesc[T], signal: ThreadSignalPtr): TaskCtx[T] = newSharedPtr(TaskCtxObj[T](signal: signal)) proc setCancelled[T](ctx: TaskCtx[T]) = - # withLock(ctxLock): ctx[].cancelled = true proc setRunning[T](ctx: TaskCtx[T]): bool = - # withLock(ctxLock): if ctx[].cancelled: return false ctx[].running = true return true proc setDone[T](ctx: TaskCtx[T]) = - # withLock(ctxLock): ctx[].running = false proc acquireSignal(): ?!ThreadSignalPtr = + echo "signal:OPEN!" let signal = ThreadSignalPtr.new() if signal.isErr(): failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) From ca5695f2c8fc87dd2d5e8803ddc35184e1d34111 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:37:12 -0700 Subject: [PATCH 361/445] remove ctx locks --- datastore/threads/threadproxyds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8c84fc72..dbe25541 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -286,6 +286,7 @@ method query*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err let ctx = newTaskCtx(QResult, signal=signal) + echo "nextSignal:OPEN!" ctx[].nextSignal.init() try: From fbc00613c477b9c603316f43c2116094843f5d11 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 13:50:21 -0700 Subject: [PATCH 362/445] remove ctx locks --- datastore/threads/threadproxyds.nim | 31 +++++++++++++-------------- tests/datastore/testthreadproxyds.nim | 4 ++-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index dbe25541..4a6e5146 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -289,6 +289,13 @@ method query*[BT](self: ThreadDatastore[BT], echo "nextSignal:OPEN!" ctx[].nextSignal.init() + proc iterDispose() = + echo "signal:CLOSE!" + discard signal.close() + echo "nextSignal:CLOSE!" + ctx[].nextSignal.close() + self.semaphore.release() + try: let query = dbQuery( key= KeyId.new q.key.id(), @@ -299,11 +306,13 @@ method query*[BT](self: ThreadDatastore[BT], self.tp.spawn queryTask(ctx, ds, query) ctx[].nextSignal.fire() - var - lock = newAsyncLock() # serialize querying under threads - iter = QueryIter.new() + var lock = newAsyncLock() # serialize querying under threads + var iter = QueryIter.new() + iter.dispose = proc (): Future[?!void] {.async.} = + iterDispose() + success() - proc next(): Future[?!QueryResponse] {.async.} = + iter.next = proc(): Future[?!QueryResponse] {.async.} = let ctx = ctx try: trace "About to query" @@ -313,7 +322,6 @@ method query*[BT](self: ThreadDatastore[BT], return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") await wait(ctx[].signal) - if not ctx[].running: iter.finished = true @@ -330,22 +338,13 @@ method query*[BT](self: ThreadDatastore[BT], except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() - echo "signal:CLOSE!" - discard ctx[].signal.close() - echo "nextSignal:CLOSE!" - ctx[].nextSignal.close() - self.semaphore.release() + iterDispose() raise exc - iter.next = next return success iter except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg - echo "signal:CLOSE!" - discard signal.close() - echo "nextSignal:CLOSE!" - ctx[].nextSignal.close() - self.semaphore.release() + iterDispose() raise exc proc new*[DB](self: type ThreadDatastore, diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index e0cb8229..0926f07e 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -52,8 +52,8 @@ for i in 1..N: (await ds.close()).tryGet() taskPool.shutdown() - for i in 1..M: - basicStoreTests(ds, key, bytes, otherBytes) + # for i in 1..M: + # basicStoreTests(ds, key, bytes, otherBytes) GC_fullCollect() for i in 1..N: From efd2e1d19d9f792fe7b5e4642e737634c2d34191 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 14:01:21 -0700 Subject: [PATCH 363/445] running 1000+ outer loops --- datastore/threads/threadproxyds.nim | 10 +++++----- tests/datastore/testthreadproxyds.nim | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 4a6e5146..3085d6c6 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -68,7 +68,7 @@ proc setDone[T](ctx: TaskCtx[T]) = ctx[].running = false proc acquireSignal(): ?!ThreadSignalPtr = - echo "signal:OPEN!" + # echo "signal:OPEN!" let signal = ThreadSignalPtr.new() if signal.isErr(): failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) @@ -126,7 +126,7 @@ template dispatchTask[BT](self: ThreadDatastore[BT], ctx.setCancelled() raise exc finally: - echo "signal:CLOSE!" + # echo "signal:CLOSE!" discard ctx[].signal.close() self.semaphore.release() @@ -286,13 +286,13 @@ method query*[BT](self: ThreadDatastore[BT], without signal =? acquireSignal(), err: return failure err let ctx = newTaskCtx(QResult, signal=signal) - echo "nextSignal:OPEN!" + # echo "nextSignal:OPEN!" ctx[].nextSignal.init() proc iterDispose() = - echo "signal:CLOSE!" + # echo "signal:CLOSE!" discard signal.close() - echo "nextSignal:CLOSE!" + # echo "nextSignal:CLOSE!" ctx[].nextSignal.close() self.semaphore.release() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 0926f07e..8ab53399 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -33,9 +33,9 @@ for i in 1..N: suite "Test Basic ThreadDatastore with SQLite": var - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + sqlStore: SQLiteBackend[KeyId, DataBuffer] + taskPool: Taskpool + ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes @@ -52,17 +52,17 @@ for i in 1..N: (await ds.close()).tryGet() taskPool.shutdown() - # for i in 1..M: - # basicStoreTests(ds, key, bytes, otherBytes) + for i in 1..M: + basicStoreTests(ds, key, bytes, otherBytes) GC_fullCollect() for i in 1..N: suite "Test Query ThreadDatastore with SQLite": var - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + sqlStore: SQLiteBackend[KeyId, DataBuffer] + taskPool: Taskpool + ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] setup: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() From a4748ef4c6585164734e56643fe284522c832f21 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 15:30:21 -0700 Subject: [PATCH 364/445] running 1000+ outer loops --- tests/datastore/testthreadproxyds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 8ab53399..841dfcdf 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -30,7 +30,7 @@ const M = ThreadTestInnerLoops for i in 1..N: - suite "Test Basic ThreadDatastore with SQLite": + suite "Test Basic ThreadDatastore with SQLite " & $i: var sqlStore: SQLiteBackend[KeyId, DataBuffer] @@ -57,7 +57,7 @@ for i in 1..N: GC_fullCollect() for i in 1..N: - suite "Test Query ThreadDatastore with SQLite": + suite "Test Query ThreadDatastore with SQLite " & $i: var sqlStore: SQLiteBackend[KeyId, DataBuffer] From 78ea3b117e9916904e513425b1fb77c42480649f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 16:01:40 -0700 Subject: [PATCH 365/445] global taskpool --- datastore/threads/threadproxyds.nim | 2 ++ tests/datastore/testthreadproxyds.nim | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 3085d6c6..c192b063 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -291,6 +291,8 @@ method query*[BT](self: ThreadDatastore[BT], proc iterDispose() = # echo "signal:CLOSE!" + ctx.setCancelled() + ctx[].nextSignal.fire() discard signal.close() # echo "nextSignal:CLOSE!" ctx[].nextSignal.close() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 841dfcdf..9bb4849b 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -29,12 +29,14 @@ const ThreadTestInnerLoops {.intdefine.} = 1 M = ThreadTestInnerLoops -for i in 1..N: +var + taskPool: Taskpool = Taskpool.new(NumThreads) + +for i in 1..1: suite "Test Basic ThreadDatastore with SQLite " & $i: var sqlStore: SQLiteBackend[KeyId, DataBuffer] - taskPool: Taskpool ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes @@ -42,7 +44,7 @@ for i in 1..N: setupAll: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) + # taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: @@ -50,30 +52,31 @@ for i in 1..N: teardownAll: (await ds.close()).tryGet() - taskPool.shutdown() + # taskPool.shutdown() for i in 1..M: basicStoreTests(ds, key, bytes, otherBytes) GC_fullCollect() + for i in 1..N: suite "Test Query ThreadDatastore with SQLite " & $i: var sqlStore: SQLiteBackend[KeyId, DataBuffer] - taskPool: Taskpool + # taskPool: Taskpool ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] setup: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) + # taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: GC_fullCollect() (await ds.close()).tryGet() - taskPool.shutdown() + # taskPool.shutdown() for i in 1..M: queryTests(ds, true) From 5afec2b3d8cfefb74216ee8ac3c9cad21ed3b011 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 16:16:18 -0700 Subject: [PATCH 366/445] change nextSignal back to ThreadSignalPtr for timeouts --- datastore/threads/threadproxyds.nim | 38 ++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index c192b063..8be8a8a5 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -41,7 +41,7 @@ type signal: ThreadSignalPtr running: bool ## used to mark when a task worker is running cancelled: bool ## used to cancel a task before it's started - nextSignal: MutexSignal + nextSignal: ThreadSignalPtr TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] ## Task context object. @@ -53,8 +53,10 @@ type semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors -proc newTaskCtx*[T](tp: typedesc[T], signal: ThreadSignalPtr): TaskCtx[T] = - newSharedPtr(TaskCtxObj[T](signal: signal)) +proc newTaskCtx*[T](tp: typedesc[T], + signal: ThreadSignalPtr, + nextSignal: ThreadSignalPtr = nil): TaskCtx[T] = + newSharedPtr(TaskCtxObj[T](signal: signal, nextSignal: nextSignal)) proc setCancelled[T](ctx: TaskCtx[T]) = ctx[].cancelled = true @@ -137,7 +139,7 @@ proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = has(ds, key) method has*[BT](self: ThreadDatastore[BT], - key: Key): Future[?!bool] {.async.} = + key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err @@ -256,7 +258,8 @@ method queryTask[DB]( # otherwise manually an set empty ok result ctx[].res.ok (KeyId.none, DataBuffer(), ) discard ctx[].signal.fireSync() - ctx[].nextSignal.wait() + if not ctx[].nextSignal.waitSync(10.seconds).get(): + raise newException(DeadThreadDefect, "queryTask timed out") var handle = handleRes.get() for item in handle.iter(): @@ -271,7 +274,8 @@ method queryTask[DB]( exc discard ctx[].signal.fireSync() - ctx[].nextSignal.wait() + if not ctx[].nextSignal.waitSync(10.seconds).get(): + raise newException(DeadThreadDefect, "queryTask timed out") # set final result (?!QResult).ok((KeyId.none, DataBuffer())) @@ -285,17 +289,17 @@ method query*[BT](self: ThreadDatastore[BT], await self.semaphore.acquire() without signal =? acquireSignal(), err: return failure err - let ctx = newTaskCtx(QResult, signal=signal) - # echo "nextSignal:OPEN!" - ctx[].nextSignal.init() + without nextSignal =? acquireSignal(), err: + return failure err + let ctx = newTaskCtx(QResult, signal=signal, nextSignal=nextSignal) - proc iterDispose() = + proc iterDispose() {.async.} = # echo "signal:CLOSE!" ctx.setCancelled() - ctx[].nextSignal.fire() - discard signal.close() + await ctx[].nextSignal.fire() + discard ctx[].signal.close() # echo "nextSignal:CLOSE!" - ctx[].nextSignal.close() + discard ctx[].nextSignal.close() self.semaphore.release() try: @@ -306,7 +310,7 @@ method query*[BT](self: ThreadDatastore[BT], # setup initial queryTask dispatchTaskWrap(self, signal): self.tp.spawn queryTask(ctx, ds, query) - ctx[].nextSignal.fire() + await ctx[].nextSignal.fire() var lock = newAsyncLock() # serialize querying under threads var iter = QueryIter.new() @@ -328,7 +332,7 @@ method query*[BT](self: ThreadDatastore[BT], iter.finished = true defer: - ctx[].nextSignal.fire() + await ctx[].nextSignal.fire() if ctx[].res.isErr(): return err(ctx[].res.error()) @@ -340,13 +344,13 @@ method query*[BT](self: ThreadDatastore[BT], except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() - iterDispose() + await iterDispose() # todo: is this valid? raise exc return success iter except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg - iterDispose() + await iterDispose() raise exc proc new*[DB](self: type ThreadDatastore, From 69529876268f305651f0e65eca300395e7bfa66b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 16:17:02 -0700 Subject: [PATCH 367/445] running 1000+ outer loops --- tests/datastore/testthreadproxyds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 9bb4849b..d59992cc 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -32,7 +32,7 @@ const var taskPool: Taskpool = Taskpool.new(NumThreads) -for i in 1..1: +for i in 1..N: suite "Test Basic ThreadDatastore with SQLite " & $i: var From 74953e175b58dc5a7e8d9d09661c2e1a10ece2c4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 16:30:47 -0700 Subject: [PATCH 368/445] try newer questionable? --- datastore.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore.nimble b/datastore.nimble index 3b5b28ef..bcd31398 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -9,7 +9,7 @@ license = "Apache License 2.0 or MIT" requires "nim >= 1.6.14", "asynctest >= 0.3.1 & < 0.4.0", "chronos#0277b65be2c7a365ac13df002fba6e172be55537", - "questionable >= 0.10.3 & < 0.11.0", + "questionable", "sqlite3_abi", "stew", "unittest2", From 9fdba84aac85090c00bb27a92c9da3690f12e270 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 17:07:50 -0700 Subject: [PATCH 369/445] try newer questionable? --- datastore/fsds.nim | 54 +++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index b3162d82..d4c9ceff 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -20,18 +20,18 @@ type ignoreProtected: bool depth: int -proc validDepth*(self: FSDatastore, key: Key): bool = - key.len <= self.depth - proc isRootSubdir*(root, path: string): bool = path.startsWith(root) -proc path*(self: FSDatastore, key: Key): ?!string = +proc path*(root: string, depth: int, key: Key): ?!string = ## Return filename corresponding to the key ## or failure if the key doesn't correspond to a valid filename ## - if not self.validDepth(key): + proc validDepth(depth: int, key: Key): bool = + key.len <= depth + + if not validDepth(depth, key): return failure "Path has invalid depth!" var @@ -53,7 +53,7 @@ proc path*(self: FSDatastore, key: Key): ?!string = segments.add(ns.field / ns.value) let - root = $self.root + root = root fullname = (root / segments.joinPath()) .absolutePath() .catch() @@ -65,6 +65,9 @@ proc path*(self: FSDatastore, key: Key): ?!string = return success fullname +proc path*(self: FSDatastore, key: Key): ?!string = + path($self.root, self.depth, key) + proc has*(self: FSDatastore, key: KeyId): ?!bool = let key = key.toKey() return self.path(key).?fileExists() @@ -166,23 +169,23 @@ proc put*( return success() -proc dirWalker(path: string): iterator: string {.gcsafe.} = - var localPath {.threadvar.}: string - - localPath = path - return iterator(): string = - try: - for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): - yield p - except CatchableError as exc: - raise newException(Defect, exc.msg) +iterator dirIter(path: string): string {.gcsafe.} = + try: + for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): + yield p + except CatchableError as exc: + raise newException(Defect, exc.msg) proc close*(self: FSDatastore): ?!void = return success() +type + FsQueryEnv* = tuple[path: DataBuffer, root: DataBuffer] + proc query*( self: FSDatastore, - query: DbQuery[KeyId]): ?!QueryIter = + query: DbQuery[KeyId], +): Result[DbQueryHandle[KeyId, DataBuffer, KeyId], ref CatchableError] = let key = query.key.toKey() without path =? self.path(key), error: @@ -198,20 +201,13 @@ proc query*( else: path.changeFileExt("") - let - walker = dirWalker(basePath) - var - iter = QueryIter.new() - # var lock = newAsyncLock() # serialize querying under threads - proc next(): ?!QueryResponse = - # defer: - # if lock.locked: - # lock.release() +iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResponse[K, V] = - # if lock.locked: - # return failure (ref DatastoreError)(msg: "Should always await query features") + let root = $(handle.env) + + for path in root.dirIter(): let root = $self.root @@ -240,7 +236,7 @@ proc query*( readFile[DataBuffer](self, (basePath / path).absolutePath) .expect("Should read file") else: - @[] + DataBuffer.new(0) return success (key.some, data) From 10bad7e70d3a05857ac8e30982174a894fabaf48 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 17:09:38 -0700 Subject: [PATCH 370/445] try newer questionable? --- datastore/fsds.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index d4c9ceff..5b221b0e 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -23,7 +23,7 @@ type proc isRootSubdir*(root, path: string): bool = path.startsWith(root) -proc path*(root: string, depth: int, key: Key): ?!string = +proc findPath*(root: string, depth: int, key: Key): ?!string = ## Return filename corresponding to the key ## or failure if the key doesn't correspond to a valid filename ## @@ -65,17 +65,17 @@ proc path*(root: string, depth: int, key: Key): ?!string = return success fullname -proc path*(self: FSDatastore, key: Key): ?!string = - path($self.root, self.depth, key) +proc findPath*(self: FSDatastore, key: Key): ?!string = + findPath($self.root, self.depth, key) proc has*(self: FSDatastore, key: KeyId): ?!bool = let key = key.toKey() - return self.path(key).?fileExists() + return self.findPath(key).?fileExists() proc delete*(self: FSDatastore, key: KeyId): ?!void = let key = key.toKey() - without path =? self.path(key), error: + without path =? self.findPath(key), error: return failure error if not path.fileExists(): @@ -132,7 +132,7 @@ proc readFile*[V](self: FSDatastore, path: string): ?!V = proc get*(self: FSDatastore, key: KeyId): ?!DataBuffer = let key = key.toKey() - without path =? self.path(key), error: + without path =? self.findPath(key), error: return failure error if not path.fileExists(): @@ -147,7 +147,7 @@ proc put*( data: DataBuffer): ?!void = let key = key.toKey() - without path =? self.path(key), error: + without path =? self.findPath(key), error: return failure error try: @@ -188,7 +188,7 @@ proc query*( ): Result[DbQueryHandle[KeyId, DataBuffer, KeyId], ref CatchableError] = let key = query.key.toKey() - without path =? self.path(key), error: + without path =? self.findPath(key), error: return failure error let basePath = From 2eb0ee6e4ec8044404b31958f99bf0b915e02fe3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 17:18:29 -0700 Subject: [PATCH 371/445] update fsds --- datastore/fsds.nim | 50 +++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 5b221b0e..3e4891fd 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -23,15 +23,15 @@ type proc isRootSubdir*(root, path: string): bool = path.startsWith(root) -proc findPath*(root: string, depth: int, key: Key): ?!string = +proc validDepth(self: FSDatastore, key: Key): bool = + key.len <= self.depth + +proc findPath*(self: FSDatastore, key: Key): ?!string = ## Return filename corresponding to the key ## or failure if the key doesn't correspond to a valid filename ## - - proc validDepth(depth: int, key: Key): bool = - key.len <= depth - - if not validDepth(depth, key): + let root = $self.root + if not self.validDepth(key): return failure "Path has invalid depth!" var @@ -53,7 +53,6 @@ proc findPath*(root: string, depth: int, key: Key): ?!string = segments.add(ns.field / ns.value) let - root = root fullname = (root / segments.joinPath()) .absolutePath() .catch() @@ -65,9 +64,6 @@ proc findPath*(root: string, depth: int, key: Key): ?!string = return success fullname -proc findPath*(self: FSDatastore, key: Key): ?!string = - findPath($self.root, self.depth, key) - proc has*(self: FSDatastore, key: KeyId): ?!bool = let key = key.toKey() return self.findPath(key).?fileExists() @@ -95,7 +91,7 @@ proc delete*(self: FSDatastore, keys: openArray[KeyId]): ?!void = return success() -proc readFile*[V](self: FSDatastore, path: string): ?!V = +proc readFile[V](self: FSDatastore, path: string): ?!V = var file: File @@ -180,7 +176,7 @@ proc close*(self: FSDatastore): ?!void = return success() type - FsQueryEnv* = tuple[path: DataBuffer, root: DataBuffer] + FsQueryEnv* = tuple[basePath: DataBuffer, self: FSDatastore] proc query*( self: FSDatastore, @@ -201,29 +197,14 @@ proc query*( else: path.changeFileExt("") - - iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResponse[K, V] = - let root = $(handle.env) - - for path in root.dirIter(): - - let - root = $self.root - path = walker() - - if iter.finished: - return failure "iterator is finished" - # await lock.acquire() - - if finished(walker): - iter.finished = true - return success (Key.none, EmptyBytes) + for path in root.dirIter(): + if handle.cancel: + return - var - keyPath = basePath + var keyPath = handle.basePath keyPath.removePrefix(root) keyPath = keyPath / path.changeFileExt("") @@ -233,16 +214,13 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResp key = Key.init(keyPath).expect("should not fail") data = if query.value: - readFile[DataBuffer](self, (basePath / path).absolutePath) - .expect("Should read file") + let fl = (handle.env.basePath / path).absolutePath() + readFile[DataBuffer](handle.env.self, fl).expect("Should read file") else: DataBuffer.new(0) return success (key.some, data) - iter.next = next - return success iter - proc new*( T: type FSDatastore, root: string, From 131712071fddc98ead38d833cc71fb05c3e087ad Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 17:26:23 -0700 Subject: [PATCH 372/445] update fsds --- datastore/fsds.nim | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 3e4891fd..8f308c9e 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -211,15 +211,18 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResp keyPath = keyPath.replace("\\", "/") let + fl = (handle.env.basePath / path).absolutePath() key = Key.init(keyPath).expect("should not fail") data = if query.value: - let fl = (handle.env.basePath / path).absolutePath() - readFile[DataBuffer](handle.env.self, fl).expect("Should read file") + let res = readFile[DataBuffer](handle.env.self, fl) + if res.isErr(): + yield failure res.error() + res.get() else: - DataBuffer.new(0) + DataBuffer.new() - return success (key.some, data) + yield success (key.some, data) proc new*( T: type FSDatastore, From bda61df77e10dc40aa8350dd3fb6b0df65098d02 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 17:31:22 -0700 Subject: [PATCH 373/445] update fsds --- datastore/fsds.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 8f308c9e..13181581 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -176,12 +176,12 @@ proc close*(self: FSDatastore): ?!void = return success() type - FsQueryEnv* = tuple[basePath: DataBuffer, self: FSDatastore] + FsQueryEnv* = tuple[self: FSDatastore, basePath: DataBuffer] proc query*( self: FSDatastore, query: DbQuery[KeyId], -): Result[DbQueryHandle[KeyId, DataBuffer, KeyId], ref CatchableError] = +): Result[DbQueryHandle[KeyId, DataBuffer, FsQueryEnv], ref CatchableError] = let key = query.key.toKey() without path =? self.findPath(key), error: @@ -196,6 +196,9 @@ proc query*( path.parentDir else: path.changeFileExt("") + + let env: FsQueryEnv = (self: self, basePath: DataBuffer.new(basePath)) + success DbQueryHandle[KeyId, DataBuffer, FsQueryEnv](env: env) iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResponse[K, V] = let root = $(handle.env) From 36ec858dcca9225a27c200f516d19f65c33eb96b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:06:23 -0700 Subject: [PATCH 374/445] refactor tests --- tests/datastore/backendCommonTests.nim | 301 +++++++++++++++++++++++++ tests/datastore/querycommontests.nim | 9 +- tests/datastore/sql/testsqliteds.nim | 60 +---- 3 files changed, 305 insertions(+), 65 deletions(-) create mode 100644 tests/datastore/backendCommonTests.nim diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim new file mode 100644 index 00000000..d3273e03 --- /dev/null +++ b/tests/datastore/backendCommonTests.nim @@ -0,0 +1,301 @@ + +template testBasicBackend*[K, V, DB]( + ds: DB, + key: K, + bytes: V, + otherBytes: V, + batch: untyped, + extended = true +): untyped = + + test "put": + ds.put(key, bytes).tryGet() + + test "get": + check: + ds.get(key).tryGet() == bytes + + test "put update": + ds.put(key, otherBytes).tryGet() + + test "get updated": + check: + ds.get(key).tryGet() == otherBytes + + test "delete": + ds.delete(key).tryGet() + + test "contains": + check key notin ds + + test "put batch": + + ds.put(batch).tryGet + + for (k, v) in batch: + check: ds.has(k).tryGet + + test "delete batch": + var keys: seq[K] + for (k, v) in batch: + keys.add(k) + + ds.delete(keys).tryGet + + for (k, v) in batch: + check: not ds.has(k).tryGet + + test "handle missing key": + when K is KeyId: + let key = KeyId.new Key.init("/missing/key").tryGet().id() + elif K is string: + let key = $KeyId.new Key.init("/missing/key").tryGet().id() + + expect(DatastoreKeyNotFound): + discard ds.get(key).tryGet() # non existing key + +template queryTests*[K, V, DB]( + ds: DB, +) = + + setup: + let + key1 = KeyId.new "/a" + key2 = KeyId.new "/a/b" + key3 = KeyId.new "/a/b/c" + val1 = DataBuffer.new "value for 1" + val2 = DataBuffer.new "value for 2" + val3 = DataBuffer.new "value for 3" + + test "Key should query all keys and all it's children": + let + q = dbQuery(key=key1, value=true) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 + + res[1].key.get == key2 + res[1].data == val2 + + res[2].key.get == key3 + res[2].data == val3 + + test "query should cancel": + let + q = dbQuery(key= key1, value= true) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + + var res: seq[DbQueryResponse[KeyId, DataBuffer]] + var cnt = 0 + for item in handle.iter(): + cnt.inc + res.insert(item.tryGet(), 0) + if cnt > 1: + handle.cancel = true + + check: + handle.cancel == true + handle.closed == true + res.len == 2 + + res[0].key.get == key2 + res[0].data == val2 + + res[1].key.get == key1 + res[1].data == val1 + + test "Key should query all keys without values": + let + q = dbQuery(key= key1, value= false) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data.len == 0 + + res[1].key.get == key2 + res[1].data.len == 0 + + res[2].key.get == key3 + res[2].data.len == 0 + + + test "Key should not query parent": + let + q = dbQuery(key= key2, value= true) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 2 + res[0].key.get == key2 + res[0].data == val2 + + res[1].key.get == key3 + res[1].data == val3 + + test "Key should all list all keys at the same level": + let + queryKey = Key.init("/a").tryGet + q = dbQuery(key= key1, value= true) + + ds.put(key1, val1).tryGet + ds.put(key2, val2).tryGet + ds.put(key3, val3).tryGet + + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + res.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: + cmp($a.key.get, $b.key.get) + + check: + res.len == 3 + res[0].key.get == key1 + res[0].data == val1 + + res[1].key.get == key2 + res[1].data == val2 + + res[2].key.get == key3 + res[2].data == val3 + + test "Should apply limit": + let + key = Key.init("/a").tryGet + q = dbQuery(key= key1, limit= 10, value= false) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) + + ds.put(key, val).tryGet + + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 10 + + test "Should not apply offset": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key= keyId, offset= 90) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) + + ds.put(key, val).tryGet + + var + qr = ds.query(q) + # echo "RES: ", qr.repr + + var + handle = ds.query(q).tryGet + let + res = handle.iter().toSeq().mapIt(it.tryGet()) + + # echo "RES: ", res.mapIt(it.key) + check: + res.len == 10 + + test "Should not apply offset and limit": + let + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key= keyId, offset= 95, limit= 5) + + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) + + ds.put(key, val).tryGet + + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 5 + + for i in 0.. int: + cmp($a.key.get, $b.key.get) + + kvs = kvs.reversed + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 100 + + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data \ No newline at end of file diff --git a/tests/datastore/querycommontests.nim b/tests/datastore/querycommontests.nim index 460804b7..0bba6e45 100644 --- a/tests/datastore/querycommontests.nim +++ b/tests/datastore/querycommontests.nim @@ -1,11 +1,4 @@ -import std/options -import std/sequtils -from std/algorithm import sort, reversed - -import pkg/asynctest -import pkg/chronos -import pkg/stew/results -import pkg/stew/byteutils + import pkg/datastore diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 1309dcd3..ef57cbf6 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -11,62 +11,8 @@ import pkg/stew/byteutils import pkg/datastore/sql/sqliteds import pkg/datastore/key -import ../dscommontests +import ../backendCommonTests -proc testBasic[K, V]( - ds: SQLiteBackend[K,V], - key: K, - bytes: V, - otherBytes: V, - batch: seq[DbBatchEntry[K, V]], - extended = true -) = - - test "put": - ds.put(key, bytes).tryGet() - - test "get": - check: - ds.get(key).tryGet() == bytes - - test "put update": - ds.put(key, otherBytes).tryGet() - - test "get updated": - check: - ds.get(key).tryGet() == otherBytes - - test "delete": - ds.delete(key).tryGet() - - test "contains": - check key notin ds - - test "put batch": - - ds.put(batch).tryGet - - for (k, v) in batch: - check: ds.has(k).tryGet - - test "delete batch": - var keys: seq[K] - for (k, v) in batch: - keys.add(k) - - ds.delete(keys).tryGet - - for (k, v) in batch: - check: not ds.has(k).tryGet - - test "handle missing key": - when K is KeyId: - let key = KeyId.new Key.init("/missing/key").tryGet().id() - elif K is string: - let key = $KeyId.new Key.init("/missing/key").tryGet().id() - - expect(DatastoreKeyNotFound): - discard ds.get(key).tryGet() # non existing key suite "Test Basic SQLiteDatastore": let @@ -84,7 +30,7 @@ suite "Test Basic SQLiteDatastore": suiteTeardown: ds.close().tryGet() - testBasic(ds, key, bytes, otherBytes, batch) + testBasicBackend(ds, key, bytes, otherBytes, batch) suite "Test DataBuffer SQLiteDatastore": let @@ -102,7 +48,7 @@ suite "Test DataBuffer SQLiteDatastore": suiteTeardown: ds.close().tryGet() - testBasic(ds, key, bytes, otherBytes, batch) + testBasicBackend(ds, key, bytes, otherBytes, batch) suite "queryTests": From 439fd92d50d120f66d6c1859b4321a7ebf8cb894 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:11:28 -0700 Subject: [PATCH 375/445] refactor tests --- tests/datastore/backendCommonTests.nim | 20 +- tests/datastore/sql/testsqliteds.nim | 253 ++----------------------- 2 files changed, 23 insertions(+), 250 deletions(-) diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index d3273e03..e09e6121 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -54,18 +54,22 @@ template testBasicBackend*[K, V, DB]( expect(DatastoreKeyNotFound): discard ds.get(key).tryGet() # non existing key -template queryTests*[K, V, DB]( - ds: DB, +template queryTests*( + ds: untyped, + key1, key2, key3: untyped, + val1, val2, val3: untyped, + extended = true ) = setup: let - key1 = KeyId.new "/a" - key2 = KeyId.new "/a/b" - key3 = KeyId.new "/a/b/c" - val1 = DataBuffer.new "value for 1" - val2 = DataBuffer.new "value for 2" - val3 = DataBuffer.new "value for 3" + ds = dsNew() + key1 = key1 + key2 = key2 + key3 = key3 + val1 = val1 + val2 = val2 + val3 = val3 test "Key should query all keys and all it's children": let diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index ef57cbf6..18454ba6 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -52,245 +52,14 @@ suite "Test DataBuffer SQLiteDatastore": suite "queryTests": - setup: - let - ds = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - key1 = KeyId.new "/a" - key2 = KeyId.new "/a/b" - key3 = KeyId.new "/a/b/c" - val1 = DataBuffer.new "value for 1" - val2 = DataBuffer.new "value for 2" - val3 = DataBuffer.new "value for 3" - - test "Key should query all keys and all it's children": - let - q = dbQuery(key=key1, value=true) - - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet - - var - handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 - - res[1].key.get == key2 - res[1].data == val2 - - res[2].key.get == key3 - res[2].data == val3 - - test "query should cancel": - let - q = dbQuery(key= key1, value= true) - - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet - - var - handle = ds.query(q).tryGet - - var res: seq[DbQueryResponse[KeyId, DataBuffer]] - var cnt = 0 - for item in handle.iter(): - cnt.inc - res.insert(item.tryGet(), 0) - if cnt > 1: - handle.cancel = true - - check: - handle.cancel == true - handle.closed == true - res.len == 2 - - res[0].key.get == key2 - res[0].data == val2 - - res[1].key.get == key1 - res[1].data == val1 - - test "Key should query all keys without values": - let - q = dbQuery(key= key1, value= false) - - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 3 - res[0].key.get == key1 - res[0].data.len == 0 - - res[1].key.get == key2 - res[1].data.len == 0 - - res[2].key.get == key3 - res[2].data.len == 0 - - - test "Key should not query parent": - let - q = dbQuery(key= key2, value= true) - - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 2 - res[0].key.get == key2 - res[0].data == val2 - - res[1].key.get == key3 - res[1].data == val3 - - test "Key should all list all keys at the same level": - let - queryKey = Key.init("/a").tryGet - q = dbQuery(key= key1, value= true) - - ds.put(key1, val1).tryGet - ds.put(key2, val2).tryGet - ds.put(key3, val3).tryGet - - var - handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) - - res.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: - cmp($a.key.get, $b.key.get) - - check: - res.len == 3 - res[0].key.get == key1 - res[0].data == val1 - - res[1].key.get == key2 - res[1].data == val2 - - res[2].key.get == key3 - res[2].data == val3 - - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = dbQuery(key= key1, limit= 10, value= false) - - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) - - ds.put(key, val).tryGet - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 10 - - test "Should not apply offset": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = dbQuery(key= keyId, offset= 90) - - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) - - ds.put(key, val).tryGet - - var - qr = ds.query(q) - # echo "RES: ", qr.repr - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - # echo "RES: ", res.mapIt(it.key) - check: - res.len == 10 - - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = dbQuery(key= keyId, offset= 95, limit= 5) - - for i in 0..<100: - let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) - - ds.put(key, val).tryGet - - var - handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 5 - - for i in 0.. int: - cmp($a.key.get, $b.key.get) - - kvs = kvs.reversed - var - handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) - - check: - res.len == 100 - - for i, r in res[1..^1]: - check: - res[i].key.get == kvs[i].key.get - res[i].data == kvs[i].data + let + dsNew = proc(): SQLiteBackend[KeyId, DataBuffer] = + newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + key1 = KeyId.new "/a" + key2 = KeyId.new "/a/b" + key3 = KeyId.new "/a/b/c" + val1 = DataBuffer.new "value for 1" + val2 = DataBuffer.new "value for 2" + val3 = DataBuffer.new "value for 3" + + queryTests(ds, key1, key2, key3, val1, val2, val3, extended=true) From c215f9cb1a8105d41d54a9dde07739326954bc3c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:20:46 -0700 Subject: [PATCH 376/445] refactor tests --- datastore/fsds.nim | 5 +- tests/datastore/testfsds.nim | 236 ++++++++++++++++++----------------- 2 files changed, 124 insertions(+), 117 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 13181581..459b8605 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -68,6 +68,9 @@ proc has*(self: FSDatastore, key: KeyId): ?!bool = let key = key.toKey() return self.findPath(key).?fileExists() +proc contains*[K](self: FSDatastore, key: K): bool = + return self.has(key).get() + proc delete*(self: FSDatastore, key: KeyId): ?!void = let key = key.toKey() @@ -243,6 +246,6 @@ proc new*( return failure "directory does not exist: " & root success T( - root: root, + root: DataBuffer.new root, ignoreProtected: ignoreProtected, depth: depth) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 26336772..5400fc4e 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -3,136 +3,140 @@ import std/sequtils import std/os from std/algorithm import sort, reversed -import pkg/asynctest +import pkg/unittest2 import pkg/chronos import pkg/stew/results import pkg/stew/byteutils import pkg/datastore/fsds +import pkg/datastore/key +import pkg/datastore/backend -import ./dscommontests -import ./querycommontests +import ./backendCommonTests suite "Test Basic FSDatastore": let path = currentSourcePath() # get this file's name basePath = "tests_data" basePathAbs = path.parentDir / basePath - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes + keyFull = Key.init("/a/b").tryGet() + key = KeyId.new keyFull.id() + bytes = DataBuffer.new "some bytes" + otherBytes = DataBuffer.new "some other bytes".toBytes - var - fsStore: FSDatastore - - setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - - teardownAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + var batch: seq[tuple[key: KeyId, data: DataBuffer]] + for k in 0..<100: + let kk = Key.init($keyFull, $k).tryGet().id() + batch.add( (KeyId.new kk, DataBuffer.new @[k.byte]) ) - basicStoreTests(fsStore, key, bytes, otherBytes) - -suite "Test Misc FSDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - bytes = "some bytes".toBytes - - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - teardown: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - test "Test validDepth()": - let - fs = FSDatastore.new(root = "/", depth = 3).tryGet() - invalid = Key.init("/a/b/c/d").tryGet() - valid = Key.init("/a/b/c").tryGet() - - check: - not fs.validDepth(invalid) - fs.validDepth(valid) - - test "Test invalid key (path) depth": - let - fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/b/c/d").tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr - - test "Test valid key (path) depth": - let - fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/b/c").tryGet() - - check: - (await fs.put(key, bytes)).isOk - (await fs.get(key)).isOk - (await fs.delete(key)).isOk - (await fs.has(key)).isOk - - test "Test key cannot write outside of root": - let - fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - key = Key.init("/a/../../c").tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr - - test "Test key cannot convert to invalid path": - let - fs = FSDatastore.new(root = basePathAbs).tryGet() - - for c in invalidFilenameChars: - if c == ':': continue - if c == '/': continue - - let - key = Key.init("/" & c).tryGet() - - check: - (await fs.put(key, bytes)).isErr - (await fs.get(key)).isErr - (await fs.delete(key)).isErr - (await fs.has(key)).isErr - -suite "Test Query": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) var - ds: FSDatastore - - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - - teardown: - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - queryTests(ds, false) + testBasicBackend(fsStore, key, bytes, otherBytes, batch) + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + +# suite "Test Misc FSDatastore": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath +# bytes = "some bytes".toBytes + +# setup: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) + +# teardown: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) + +# test "Test validDepth()": +# let +# fs = FSDatastore.new(root = "/", depth = 3).tryGet() +# invalid = Key.init("/a/b/c/d").tryGet() +# valid = Key.init("/a/b/c").tryGet() + +# check: +# not fs.validDepth(invalid) +# fs.validDepth(valid) + +# test "Test invalid key (path) depth": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/b/c/d").tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + +# test "Test valid key (path) depth": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/b/c").tryGet() + +# check: +# (await fs.put(key, bytes)).isOk +# (await fs.get(key)).isOk +# (await fs.delete(key)).isOk +# (await fs.has(key)).isOk + +# test "Test key cannot write outside of root": +# let +# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() +# key = Key.init("/a/../../c").tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + +# test "Test key cannot convert to invalid path": +# let +# fs = FSDatastore.new(root = basePathAbs).tryGet() + +# for c in invalidFilenameChars: +# if c == ':': continue +# if c == '/': continue + +# let +# key = Key.init("/" & c).tryGet() + +# check: +# (await fs.put(key, bytes)).isErr +# (await fs.get(key)).isErr +# (await fs.delete(key)).isErr +# (await fs.has(key)).isErr + + +# suite "Test Query": +# let +# path = currentSourcePath() # get this file's name +# basePath = "tests_data" +# basePathAbs = path.parentDir / basePath + +# var +# ds: FSDatastore + +# setup: +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) +# createDir(basePathAbs) + +# ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() + +# teardown: + +# removeDir(basePathAbs) +# require(not dirExists(basePathAbs)) + +# queryTests(ds, false) From ef5a30f7d311f82075164d18e6c60fac6873804d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:27:29 -0700 Subject: [PATCH 377/445] refactor tests --- datastore/fsds.nim | 3 ++- datastore/threads/databuffer.nim | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 459b8605..a49f800d 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -111,12 +111,13 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = when V is seq[byte]: var bytes = newSeq[byte](size) elif V is DataBuffer: - var bytes = DataBuffer.new(capacity=size) + var bytes = DataBuffer.new(size=size) else: {.error: "unhandled result type".} var read = 0 + echo "BYTES: ", bytes.repr while read < size: read += file.readBytes(bytes.toOpenArray(), read, size) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 1b5e6d25..3cb414a7 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -47,14 +47,14 @@ template `==`*[T: char | byte](a: DataBuffer, b: openArray[T]): bool = elif a[].size != b.len: false else: a.hash() == b.hash() -proc new*(tp: type DataBuffer, capacity: int = 0): DataBuffer = +proc new*(tp: type DataBuffer, size: int = 0): DataBuffer = ## allocate new buffer with given capacity ## newSharedPtr(DataBufferHolder( - buf: cast[typeof(result[].buf)](allocShared0(capacity)), - size: 0, - cap: capacity, + buf: cast[typeof(result[].buf)](allocShared0(size)), + size: size, + cap: size, )) proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[DataBufferOpt] = {}): DataBuffer = From ed34fe2d3cfaef63773202e5d7550fb8de35920b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:29:04 -0700 Subject: [PATCH 378/445] refactor tests --- tests/datastore/testfsds.nim | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 5400fc4e..bd67777a 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -41,6 +41,32 @@ suite "Test Basic FSDatastore": removeDir(basePathAbs) require(not dirExists(basePathAbs)) +suite "Test Basic FSDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + var batch: seq[tuple[key: string, data: seq[byte]]] + for k in 0..<100: + let kk = Key.init($key, $k).tryGet().id() + batch.add( (kk, @[k.byte]) ) + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + var + fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + + testBasicBackend(fsStore, key, bytes, otherBytes, batch) + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + # suite "Test Misc FSDatastore": # let # path = currentSourcePath() # get this file's name From 2a96b1bcef29ffd666ab91aa1cba0b05b841f346 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:36:32 -0700 Subject: [PATCH 379/445] refactor tests --- datastore/fsds.nim | 35 +++++++++++++++++------------------ datastore/sql/sqliteds.nim | 4 ++-- tests/datastore/testfsds.nim | 4 ++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index a49f800d..9c725468 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -15,7 +15,7 @@ export datastore push: {.upraises: [].} type - FSDatastore* = object + FSDatastore*[K, V] = object root*: DataBuffer ignoreProtected: bool depth: int @@ -26,11 +26,12 @@ proc isRootSubdir*(root, path: string): bool = proc validDepth(self: FSDatastore, key: Key): bool = key.len <= self.depth -proc findPath*(self: FSDatastore, key: Key): ?!string = +proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = ## Return filename corresponding to the key ## or failure if the key doesn't correspond to a valid filename ## let root = $self.root + let key = Key.init($key).get() if not self.validDepth(key): return failure "Path has invalid depth!" @@ -64,14 +65,14 @@ proc findPath*(self: FSDatastore, key: Key): ?!string = return success fullname -proc has*(self: FSDatastore, key: KeyId): ?!bool = +proc has*[K,V](self: FSDatastore[K,V], key: KeyId): ?!bool = let key = key.toKey() return self.findPath(key).?fileExists() proc contains*[K](self: FSDatastore, key: K): bool = return self.has(key).get() -proc delete*(self: FSDatastore, key: KeyId): ?!void = +proc delete*[K,V](self: FSDatastore[K,V], key: KeyId): ?!void = let key = key.toKey() without path =? self.findPath(key), error: @@ -87,7 +88,7 @@ proc delete*(self: FSDatastore, key: KeyId): ?!void = return success() -proc delete*(self: FSDatastore, keys: openArray[KeyId]): ?!void = +proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[KeyId]): ?!void = for key in keys: if err =? self.delete(key).errorOption: return failure err @@ -130,7 +131,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = except CatchableError as e: return failure e -proc get*(self: FSDatastore, key: KeyId): ?!DataBuffer = +proc get*[K,V](self: FSDatastore[K,V], key: KeyId): ?!DataBuffer = let key = key.toKey() without path =? self.findPath(key), error: return failure error @@ -141,11 +142,10 @@ proc get*(self: FSDatastore, key: KeyId): ?!DataBuffer = return readFile[DataBuffer](self, path) -proc put*( - self: FSDatastore, +proc put*[K,V]( + self: FSDatastore[K,V], key: KeyId, data: DataBuffer): ?!void = - let key = key.toKey() without path =? self.findPath(key), error: return failure error @@ -176,11 +176,11 @@ iterator dirIter(path: string): string {.gcsafe.} = except CatchableError as exc: raise newException(Defect, exc.msg) -proc close*(self: FSDatastore): ?!void = +proc close*[K,V](self: FSDatastore[K,V]): ?!void = return success() type - FsQueryEnv* = tuple[self: FSDatastore, basePath: DataBuffer] + FsQueryEnv*[K,V] = tuple[self: FSDatastore[K,V], basePath: DataBuffer] proc query*( self: FSDatastore, @@ -231,12 +231,11 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResp yield success (key.some, data) -proc new*( - T: type FSDatastore, - root: string, - depth = 2, - caseSensitive = true, - ignoreProtected = false): ?!T = +proc newFSDatastore*[K,V](root: string, + depth = 2, + caseSensitive = true, + ignoreProtected = false + ): ?!FSDatastore[K,V] = let root = ? ( block: @@ -246,7 +245,7 @@ proc new*( if not dirExists(root): return failure "directory does not exist: " & root - success T( + success FSDatastore[K,V]( root: DataBuffer.new root, ignoreProtected: ignoreProtected, depth: depth) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index feb0785b..e86fb905 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -227,7 +227,7 @@ proc newSQLiteBackend*[K,V]( success SQLiteBackend[K,V](db: ? SQLiteDsDb[K,V].open(path, flags)) -proc newSQLiteBackend*[K,V]( - db: SQLiteDsDb[K,V]): ?!SQLiteBackend[K,V] = +proc newSQLiteBackend*[K,V](db: SQLiteDsDb[K,V] + ): ?!SQLiteBackend[K,V] = success SQLiteBackend[K,V](db: db) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index bd67777a..6a829b8b 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -34,7 +34,7 @@ suite "Test Basic FSDatastore": createDir(basePathAbs) var - fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() testBasicBackend(fsStore, key, bytes, otherBytes, batch) @@ -60,7 +60,7 @@ suite "Test Basic FSDatastore": createDir(basePathAbs) var - fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() testBasicBackend(fsStore, key, bytes, otherBytes, batch) From 4fa532e72a19a0cdc38f59269b899358fc66775e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:39:20 -0700 Subject: [PATCH 380/445] refactor tests --- datastore/fsds.nim | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 9c725468..2b649759 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -65,16 +65,15 @@ proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = return success fullname -proc has*[K,V](self: FSDatastore[K,V], key: KeyId): ?!bool = - let key = key.toKey() - return self.findPath(key).?fileExists() +proc has*[K,V](self: FSDatastore[K,V], key: K): ?!bool = + without path =? self.findPath(key), error: + return failure error + success path.fileExists() proc contains*[K](self: FSDatastore, key: K): bool = return self.has(key).get() -proc delete*[K,V](self: FSDatastore[K,V], key: KeyId): ?!void = - let key = key.toKey() - +proc delete*[K,V](self: FSDatastore[K,V], key: K): ?!void = without path =? self.findPath(key), error: return failure error @@ -88,7 +87,7 @@ proc delete*[K,V](self: FSDatastore[K,V], key: KeyId): ?!void = return success() -proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[KeyId]): ?!void = +proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[K]): ?!void = for key in keys: if err =? self.delete(key).errorOption: return failure err @@ -131,8 +130,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = except CatchableError as e: return failure e -proc get*[K,V](self: FSDatastore[K,V], key: KeyId): ?!DataBuffer = - let key = key.toKey() +proc get*[K,V](self: FSDatastore[K,V], key: K): ?!DataBuffer = without path =? self.findPath(key), error: return failure error @@ -159,9 +157,9 @@ proc put*[K,V]( return success() -proc put*( +proc put*[K,V]( self: FSDatastore, - batch: seq[DbBatchEntry[KeyId, DataBuffer]]): ?!void = + batch: seq[DbBatchEntry[K, V]]): ?!void = for entry in batch: if err =? self.put(entry.key, entry.data).errorOption: From 1b7866aad2e95acb83472b3e36f06d2f16ed10a8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:48:48 -0700 Subject: [PATCH 381/445] refactor --- datastore/backend.nim | 3 --- datastore/fsds.nim | 40 ++++++++++++++++---------------- datastore/threads/databuffer.nim | 4 ++-- tests/datastore/testfsds.nim | 2 +- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index 5790b121..bb722b9b 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -62,6 +62,3 @@ proc new*(tp: typedesc[KeyId], id: cstring): KeyId = proc new*(tp: typedesc[KeyId], id: string): KeyId = KeyId(data: DataBuffer.new(id)) - -template toOpenArray*(x: DbKey): openArray[char] = - x.data.toOpenArray(char) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 2b649759..8f2c7705 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -110,8 +110,8 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = when V is seq[byte]: var bytes = newSeq[byte](size) - elif V is DataBuffer: - var bytes = DataBuffer.new(size=size) + elif V is V: + var bytes = V.new(size=size) else: {.error: "unhandled result type".} var @@ -119,7 +119,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = echo "BYTES: ", bytes.repr while read < size: - read += file.readBytes(bytes.toOpenArray(), read, size) + read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) if read < size: return failure $read & " bytes were read from " & path & @@ -130,7 +130,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = except CatchableError as e: return failure e -proc get*[K,V](self: FSDatastore[K,V], key: K): ?!DataBuffer = +proc get*[K,V](self: FSDatastore[K,V], key: K): ?!V = without path =? self.findPath(key), error: return failure error @@ -138,12 +138,12 @@ proc get*[K,V](self: FSDatastore[K,V], key: K): ?!DataBuffer = return failure( newException(DatastoreKeyNotFound, "Key doesn't exist")) - return readFile[DataBuffer](self, path) + return readFile[V](self, path) -proc put*[K,V]( - self: FSDatastore[K,V], - key: KeyId, - data: DataBuffer): ?!void = +proc put*[K,V](self: FSDatastore[K,V], + key: K, + data: V + ): ?!void = without path =? self.findPath(key), error: return failure error @@ -151,7 +151,7 @@ proc put*[K,V]( try: var data = data createDir(parentDir(path)) - writeFile(path, data.toOpenArray()) + writeFile(path, data.toOpenArray(0, data.len()-1)) except CatchableError as e: return failure e @@ -178,12 +178,12 @@ proc close*[K,V](self: FSDatastore[K,V]): ?!void = return success() type - FsQueryEnv*[K,V] = tuple[self: FSDatastore[K,V], basePath: DataBuffer] + FsQueryEnv*[K,V] = tuple[self: FSDatastore[K,V], basePath: V] -proc query*( - self: FSDatastore, - query: DbQuery[KeyId], -): Result[DbQueryHandle[KeyId, DataBuffer, FsQueryEnv], ref CatchableError] = +proc query*[K,V]( + self: FSDatastore[K,V], + query: DbQuery[K], +): Result[DbQueryHandle[KeyId, V, FsQueryEnv], ref CatchableError] = let key = query.key.toKey() without path =? self.findPath(key), error: @@ -199,10 +199,10 @@ proc query*( else: path.changeFileExt("") - let env: FsQueryEnv = (self: self, basePath: DataBuffer.new(basePath)) - success DbQueryHandle[KeyId, DataBuffer, FsQueryEnv](env: env) + let env: FsQueryEnv = (self: self, basePath: V.new(basePath)) + success DbQueryHandle[KeyId, V, FsQueryEnv](env: env) -iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResponse[K, V] = +iterator iter*[K, V](handle: var DbQueryHandle[K, V, V]): ?!DbQueryResponse[K, V] = let root = $(handle.env) for path in root.dirIter(): @@ -220,12 +220,12 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, DataBuffer]): ?!DbQueryResp key = Key.init(keyPath).expect("should not fail") data = if query.value: - let res = readFile[DataBuffer](handle.env.self, fl) + let res = readFile[V](handle.env.self, fl) if res.isErr(): yield failure res.error() res.get() else: - DataBuffer.new() + V.new() yield success (key.some, data) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 3cb414a7..5f80cbfe 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -137,5 +137,5 @@ template toOpenArray*[T: byte | char](data: var DataBuffer, t: typedesc[T]): var var bf = cast[ptr UncheckedArray[T]](data[].buf) bf.toOpenArray(0, data[].size-1) -template toOpenArray*(data: var DataBuffer): var openArray[byte] = - toOpenArray(data, byte) +template toOpenArray*(data: var DataBuffer, first, last: int): var openArray[byte] = + toOpenArray(data, byte).toOpenArray(first, last) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 6a829b8b..d1ad8052 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -60,7 +60,7 @@ suite "Test Basic FSDatastore": createDir(basePathAbs) var - fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() + fsStore = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() testBasicBackend(fsStore, key, bytes, otherBytes, batch) From f587677be850666aae943e498a34052ad0dc8a18 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:50:42 -0700 Subject: [PATCH 382/445] refactor - tests --- tests/datastore/testfsds.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index d1ad8052..d2053422 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -50,9 +50,9 @@ suite "Test Basic FSDatastore": bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - var batch: seq[tuple[key: string, data: seq[byte]]] + var batch: seq[tuple[key: Key, data: seq[byte]]] for k in 0..<100: - let kk = Key.init($key, $k).tryGet().id() + let kk = Key.init($key, $k).tryGet() batch.add( (kk, @[k.byte]) ) removeDir(basePathAbs) From 9e946e683e175ac097262eb23dae10ec6ec25a05 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:53:46 -0700 Subject: [PATCH 383/445] refactor - tests --- datastore/fsds.nim | 2 +- tests/datastore/testfsds.nim | 148 +++++++++++++++++------------------ 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 8f2c7705..fb027c9c 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -23,7 +23,7 @@ type proc isRootSubdir*(root, path: string): bool = path.startsWith(root) -proc validDepth(self: FSDatastore, key: Key): bool = +proc validDepth*(self: FSDatastore, key: Key): bool = key.len <= self.depth proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index d2053422..33c1a051 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -67,81 +67,81 @@ suite "Test Basic FSDatastore": removeDir(basePathAbs) require(not dirExists(basePathAbs)) -# suite "Test Misc FSDatastore": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath -# bytes = "some bytes".toBytes - -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) - -# teardown: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) +suite "Test Misc FSDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + bytes = "some bytes".toBytes -# test "Test validDepth()": -# let -# fs = FSDatastore.new(root = "/", depth = 3).tryGet() -# invalid = Key.init("/a/b/c/d").tryGet() -# valid = Key.init("/a/b/c").tryGet() - -# check: -# not fs.validDepth(invalid) -# fs.validDepth(valid) - -# test "Test invalid key (path) depth": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/b/c/d").tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr - -# test "Test valid key (path) depth": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/b/c").tryGet() - -# check: -# (await fs.put(key, bytes)).isOk -# (await fs.get(key)).isOk -# (await fs.delete(key)).isOk -# (await fs.has(key)).isOk - -# test "Test key cannot write outside of root": -# let -# fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# key = Key.init("/a/../../c").tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr - -# test "Test key cannot convert to invalid path": -# let -# fs = FSDatastore.new(root = basePathAbs).tryGet() - -# for c in invalidFilenameChars: -# if c == ':': continue -# if c == '/': continue - -# let -# key = Key.init("/" & c).tryGet() - -# check: -# (await fs.put(key, bytes)).isErr -# (await fs.get(key)).isErr -# (await fs.delete(key)).isErr -# (await fs.has(key)).isErr + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + teardown: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + test "Test validDepth()": + let + fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + invalid = Key.init("/a/b/c/d").tryGet() + valid = Key.init("/a/b/c").tryGet() + + check: + not fs.validDepth(invalid) + fs.validDepth(valid) + + test "Test invalid key (path) depth": + let + fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/b/c/d").tryGet() + + check: + (fs.put(key, bytes)).isErr + (fs.get(key)).isErr + (fs.delete(key)).isErr + (fs.has(key)).isErr + + test "Test valid key (path) depth": + let + fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/b/c").tryGet() + + check: + (fs.put(key, bytes)).isOk + (fs.get(key)).isOk + (fs.delete(key)).isOk + (fs.has(key)).isOk + + test "Test key cannot write outside of root": + let + fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + key = Key.init("/a/../../c").tryGet() + + check: + (fs.put(key, bytes)).isErr + (fs.get(key)).isErr + (fs.delete(key)).isErr + (fs.has(key)).isErr + + test "Test key cannot convert to invalid path": + let + fs = newFSDatastore[Key, seq[byte]](root = basePathAbs).tryGet() + + for c in invalidFilenameChars: + if c == ':': continue + if c == '/': continue + + let + key = Key.init("/" & c).tryGet() + + check: + (fs.put(key, bytes)).isErr + (fs.get(key)).isErr + (fs.delete(key)).isErr + (fs.has(key)).isErr # suite "Test Query": From 1c2c5f10200d9d93d92f6f690ed28e03a8830b3f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 18:57:17 -0700 Subject: [PATCH 384/445] refactor - tests --- tests/datastore/backendCommonTests.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 2 +- tests/datastore/testfsds.nim | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index e09e6121..b3fecbe6 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -55,7 +55,7 @@ template testBasicBackend*[K, V, DB]( discard ds.get(key).tryGet() # non existing key template queryTests*( - ds: untyped, + dsNew: untyped, key1, key2, key3: untyped, val1, val2, val3: untyped, extended = true diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 18454ba6..e52d90b1 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -62,4 +62,4 @@ suite "queryTests": val2 = DataBuffer.new "value for 2" val3 = DataBuffer.new "value for 3" - queryTests(ds, key1, key2, key3, val1, val2, val3, extended=true) + queryTests(dsNew, key1, key2, key3, val1, val2, val3, extended=true) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 33c1a051..4eb4db30 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -166,3 +166,29 @@ suite "Test Misc FSDatastore": # require(not dirExists(basePathAbs)) # queryTests(ds, false) + +suite "queryTests": + + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + let + fsNew = proc(): FSDatastore[KeyId, DataBuffer] = + newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() + key1 = KeyId.new "/a" + key2 = KeyId.new "/a/b" + key3 = KeyId.new "/a/b/c" + val1 = DataBuffer.new "value for 1" + val2 = DataBuffer.new "value for 2" + val3 = DataBuffer.new "value for 3" + + queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=true) + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) From 534555015b823a4ae002c593b5f11e11c68961d8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:00:54 -0700 Subject: [PATCH 385/445] refactor - tests --- datastore/fsds.nim | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index fb027c9c..5fe34326 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -178,14 +178,16 @@ proc close*[K,V](self: FSDatastore[K,V]): ?!void = return success() type - FsQueryEnv*[K,V] = tuple[self: FSDatastore[K,V], basePath: V] + FsQueryEnv*[K,V] = object + self: FSDatastore[K,V] + basePath: DataBuffer proc query*[K,V]( self: FSDatastore[K,V], query: DbQuery[K], -): Result[DbQueryHandle[KeyId, V, FsQueryEnv], ref CatchableError] = +): Result[DbQueryHandle[K, V, FsQueryEnv[K,V]], ref CatchableError] = - let key = query.key.toKey() + let key = query.key without path =? self.findPath(key), error: return failure error @@ -199,8 +201,8 @@ proc query*[K,V]( else: path.changeFileExt("") - let env: FsQueryEnv = (self: self, basePath: V.new(basePath)) - success DbQueryHandle[KeyId, V, FsQueryEnv](env: env) + let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) + success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](env: env) iterator iter*[K, V](handle: var DbQueryHandle[K, V, V]): ?!DbQueryResponse[K, V] = let root = $(handle.env) From e778fdbb8e19621ccf7d5c8686a1b587e3aff1b4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:08:58 -0700 Subject: [PATCH 386/445] refactor - tests --- datastore/fsds.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 5fe34326..4ffd523a 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -202,29 +202,35 @@ proc query*[K,V]( path.changeFileExt("") let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) - success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](env: env) + success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](query: query, env: env) -iterator iter*[K, V](handle: var DbQueryHandle[K, V, V]): ?!DbQueryResponse[K, V] = +iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQueryResponse[K, V] = let root = $(handle.env) for path in root.dirIter(): if handle.cancel: - return + break - var keyPath = handle.basePath + var + basePath = $handle.env.basePath + keyPath = basePath keyPath.removePrefix(root) keyPath = keyPath / path.changeFileExt("") keyPath = keyPath.replace("\\", "/") let - fl = (handle.env.basePath / path).absolutePath() - key = Key.init(keyPath).expect("should not fail") + flres = (basePath / path).absolutePath().catch + if flres.isErr(): + yield DbQueryResponse[K,V].failure flres.error() + + let + key = K.toKey(keyPath) data = - if query.value: - let res = readFile[V](handle.env.self, fl) + if handle.query.value: + let res = readFile[V](handle.env.self, flres.get) if res.isErr(): - yield failure res.error() + yield DbQueryResponse[K,V].failure res.error() res.get() else: V.new() From d5850ebe9002995b16c2d393637d56071ee0958b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:12:42 -0700 Subject: [PATCH 387/445] refactor - tests --- datastore/fsds.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 4ffd523a..4cbaac40 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -117,7 +117,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = var read = 0 - echo "BYTES: ", bytes.repr + # echo "BYTES: ", bytes.repr while read < size: read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) @@ -208,7 +208,9 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQuer let root = $(handle.env) for path in root.dirIter(): + echo "FS:path: ", path if handle.cancel: + echo "FS:CANCELLED!" break var @@ -222,6 +224,7 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQuer let flres = (basePath / path).absolutePath().catch if flres.isErr(): + echo "FS:ERROR: ", flres.error() yield DbQueryResponse[K,V].failure flres.error() let @@ -230,11 +233,13 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQuer if handle.query.value: let res = readFile[V](handle.env.self, flres.get) if res.isErr(): + echo "FS:ERROR: ", res.error() yield DbQueryResponse[K,V].failure res.error() res.get() else: V.new() + echo "FS:SUCCESS: ", key yield success (key.some, data) proc newFSDatastore*[K,V](root: string, From ade0898fe78fdef1ea6f7c3cc117b4b2c1775458 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:13:41 -0700 Subject: [PATCH 388/445] refactor - tests --- datastore/fsds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 4cbaac40..14eeafe8 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -206,6 +206,7 @@ proc query*[K,V]( iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQueryResponse[K, V] = let root = $(handle.env) + echo "FS:root: ", root for path in root.dirIter(): echo "FS:path: ", path From cac5d52b350351cd1ca5a0ed1744eba355f41886 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:18:37 -0700 Subject: [PATCH 389/445] refactor - tests --- datastore/backend.nim | 5 +++-- datastore/fsds.nim | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index bb722b9b..410f5b11 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -51,8 +51,9 @@ proc dbQuery*[K]( proc `$`*(id: KeyId): string = $(id.data) -proc toKey*(tp: typedesc[KeyId], id: cstring): KeyId = KeyId.new(id) -proc toKey*(tp: typedesc[string], id: cstring): string = $(id) +proc toKey*(tp: typedesc[KeyId], id: string|cstring): KeyId = KeyId.new($id) +proc toKey*(tp: typedesc[string], id: string|cstring): string = $(id) +# proc toKey*(tp: typedesc[Key], id: string|cstring): KeyId = Key.init($id).expect("valid key") template toVal*(tp: typedesc[DataBuffer], id: openArray[byte]): DataBuffer = DataBuffer.new(id) template toVal*(tp: typedesc[seq[byte]], id: openArray[byte]): seq[byte] = @(id) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 14eeafe8..e9c9946f 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -204,8 +204,9 @@ proc query*[K,V]( let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](query: query, env: env) -iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]]): ?!DbQueryResponse[K, V] = - let root = $(handle.env) +iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] + ): ?!DbQueryResponse[K, V] = + let root = $(handle.env.self.root) echo "FS:root: ", root for path in root.dirIter(): From d4a7549e12daf648eaecd96a1c8186000a47c87d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:19:20 -0700 Subject: [PATCH 390/445] refactor - tests --- datastore/fsds.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index e9c9946f..81f2bec9 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -230,7 +230,7 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] yield DbQueryResponse[K,V].failure flres.error() let - key = K.toKey(keyPath) + key = K.toKey($Key.init(keyPath).expect("valid key")) data = if handle.query.value: let res = readFile[V](handle.env.self, flres.get) From 17337ea7cd8cfd5fc8b88ac17446809e4a1d9b98 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:21:39 -0700 Subject: [PATCH 391/445] refactor - tests --- datastore/fsds.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 81f2bec9..5da60d88 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -204,6 +204,10 @@ proc query*[K,V]( let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](query: query, env: env) +proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv]) = + if not handle.closed: + handle.closed = true + iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] ): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) From f0591a0f9f5393651938a9321ad1b1836ed5de08 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:22:55 -0700 Subject: [PATCH 392/445] refactor - tests --- datastore/fsds.nim | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 5da60d88..f81ec26e 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -204,19 +204,19 @@ proc query*[K,V]( let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](query: query, env: env) -proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv]) = +proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv[K,V]]) = if not handle.closed: handle.closed = true iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] ): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) - echo "FS:root: ", root + # echo "FS:root: ", root for path in root.dirIter(): - echo "FS:path: ", path + # echo "FS:path: ", path if handle.cancel: - echo "FS:CANCELLED!" + # echo "FS:CANCELLED!" break var @@ -230,7 +230,7 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] let flres = (basePath / path).absolutePath().catch if flres.isErr(): - echo "FS:ERROR: ", flres.error() + # echo "FS:ERROR: ", flres.error() yield DbQueryResponse[K,V].failure flres.error() let @@ -239,14 +239,15 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] if handle.query.value: let res = readFile[V](handle.env.self, flres.get) if res.isErr(): - echo "FS:ERROR: ", res.error() + # echo "FS:ERROR: ", res.error() yield DbQueryResponse[K,V].failure res.error() res.get() else: V.new() - echo "FS:SUCCESS: ", key + # echo "FS:SUCCESS: ", key yield success (key.some, data) + handle.close() proc newFSDatastore*[K,V](root: string, depth = 2, From 16bd98d32630de313c086c128516e4d6b1a16378 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:23:47 -0700 Subject: [PATCH 393/445] refactor - tests --- datastore/fsds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index f81ec26e..cad4a8e2 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -241,7 +241,8 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] if res.isErr(): # echo "FS:ERROR: ", res.error() yield DbQueryResponse[K,V].failure res.error() - res.get() + else: + res.get() else: V.new() From ae311856a86003f5c5c69ae9eea82f7d23dc5bfd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:27:43 -0700 Subject: [PATCH 394/445] refactor - tests --- datastore/fsds.nim | 5 +- tests/datastore/backendCommonTests.nim | 151 +++++++++++++------------ tests/datastore/testfsds.nim | 2 +- 3 files changed, 80 insertions(+), 78 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index cad4a8e2..3e09210e 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -232,6 +232,7 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] if flres.isErr(): # echo "FS:ERROR: ", flres.error() yield DbQueryResponse[K,V].failure flres.error() + continue let key = K.toKey($Key.init(keyPath).expect("valid key")) @@ -241,8 +242,8 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] if res.isErr(): # echo "FS:ERROR: ", res.error() yield DbQueryResponse[K,V].failure res.error() - else: - res.get() + continue + res.get() else: V.new() diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index b3fecbe6..e292e6ee 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -197,109 +197,110 @@ template queryTests*( res[2].key.get == key3 res[2].data == val3 - test "Should apply limit": - let - key = Key.init("/a").tryGet - q = dbQuery(key= key1, limit= 10, value= false) - - for i in 0..<100: + if extended: + test "Should apply limit": let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) - - ds.put(key, val).tryGet - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) + key = Key.init("/a").tryGet + q = dbQuery(key= key1, limit= 10, value= false) - check: - res.len == 10 + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) - test "Should not apply offset": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = dbQuery(key= keyId, offset= 90) + ds.put(key, val).tryGet - for i in 0..<100: + var + handle = ds.query(q).tryGet let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) - - ds.put(key, val).tryGet - - var - qr = ds.query(q) - # echo "RES: ", qr.repr - - var - handle = ds.query(q).tryGet - let - res = handle.iter().toSeq().mapIt(it.tryGet()) - - # echo "RES: ", res.mapIt(it.key) - check: - res.len == 10 + res = handle.iter().toSeq().mapIt(it.tryGet()) - test "Should not apply offset and limit": - let - key = Key.init("/a").tryGet - keyId = KeyId.new $key - q = dbQuery(key= keyId, offset= 95, limit= 5) + check: + res.len == 10 - for i in 0..<100: + test "Should not apply offset": let - key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet - val = DataBuffer.new("val " & $i) + key = Key.init("/a").tryGet + keyId = KeyId.new $key + q = dbQuery(key= keyId, offset= 90) - ds.put(key, val).tryGet + for i in 0..<100: + let + key = KeyId.new $Key.init(key, Key.init("/" & $i).tryGet).tryGet + val = DataBuffer.new("val " & $i) - var - handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) + ds.put(key, val).tryGet - check: - res.len == 5 + var + qr = ds.query(q) + # echo "RES: ", qr.repr - for i in 0.. int: - cmp($a.key.get, $b.key.get) + ds.put(key, val).tryGet - kvs = kvs.reversed var handle = ds.query(q).tryGet res = handle.iter().toSeq().mapIt(it.tryGet()) check: - res.len == 100 + res.len == 5 + + for i in 0.. int: + cmp($a.key.get, $b.key.get) + + kvs = kvs.reversed + var + handle = ds.query(q).tryGet + res = handle.iter().toSeq().mapIt(it.tryGet()) + + check: + res.len == 100 + + for i, r in res[1..^1]: + check: + res[i].key.get == kvs[i].key.get + res[i].data == kvs[i].data \ No newline at end of file diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 4eb4db30..8a0dd176 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -188,7 +188,7 @@ suite "queryTests": val2 = DataBuffer.new "value for 2" val3 = DataBuffer.new "value for 3" - queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=true) + queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=false) removeDir(basePathAbs) require(not dirExists(basePathAbs)) From dcb70ca429da9ac92c01e5889b2d35f598069b76 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:35:42 -0700 Subject: [PATCH 395/445] refactor - tests --- datastore/fsds.nim | 2 +- tests/datastore/backendCommonTests.nim | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 3e09210e..24874a0f 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -102,7 +102,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = file.close if not file.open(path): - return failure "unable to open file!" + return failure "unable to open file! path: " & path try: let diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index e292e6ee..9af3de74 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -160,7 +160,10 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.iter().toSeq().mapIt(block: + echo "RES: ", it.repr + it.tryGet() + ) check: res.len == 2 From 0f0e113f4faae58ea14720cd08f5e1adccc6fbb6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:49:59 -0700 Subject: [PATCH 396/445] refactor - tests --- datastore/fsds.nim | 5 +++-- tests/datastore/backendCommonTests.nim | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 24874a0f..9d15372f 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -190,6 +190,7 @@ proc query*[K,V]( let key = query.key without path =? self.findPath(key), error: return failure error + echo "query:key: ", key let basePath = # it there is a file in the directory @@ -211,10 +212,10 @@ proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv[K,V]]) = iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] ): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) - # echo "FS:root: ", root + echo "FS:root: ", root for path in root.dirIter(): - # echo "FS:path: ", path + echo "FS:path: ", path if handle.cancel: # echo "FS:CANCELLED!" break diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index 9af3de74..3e99ceea 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -161,9 +161,11 @@ template queryTests*( handle = ds.query(q).tryGet let res = handle.iter().toSeq().mapIt(block: - echo "RES: ", it.repr + echo "\nRES: ", it.repr + # quit(1) it.tryGet() - ) + ) + # ).filterIt(it.isOk).mapIt(it.tryGet()) check: res.len == 2 From fb73d6b5a48c612eb57d2552607d727a4c3b65b6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:57:50 -0700 Subject: [PATCH 397/445] refactor - tests --- datastore/fsds.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 9d15372f..87fb7ad8 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -212,9 +212,11 @@ proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv[K,V]]) = iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] ): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) + let basePath = $(handle.env.basePath) echo "FS:root: ", root + echo "FS:basePath: ", basePath - for path in root.dirIter(): + for path in basePath.dirIter(): echo "FS:path: ", path if handle.cancel: # echo "FS:CANCELLED!" From 1ef4825c324c06192291dd060697e2cefa36f091 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:58:42 -0700 Subject: [PATCH 398/445] refactor - tests --- datastore/fsds.nim | 7 ------- tests/datastore/backendCommonTests.nim | 6 +----- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 87fb7ad8..d0eacceb 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -213,13 +213,9 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] ): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) let basePath = $(handle.env.basePath) - echo "FS:root: ", root - echo "FS:basePath: ", basePath for path in basePath.dirIter(): - echo "FS:path: ", path if handle.cancel: - # echo "FS:CANCELLED!" break var @@ -233,7 +229,6 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] let flres = (basePath / path).absolutePath().catch if flres.isErr(): - # echo "FS:ERROR: ", flres.error() yield DbQueryResponse[K,V].failure flres.error() continue @@ -243,14 +238,12 @@ iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] if handle.query.value: let res = readFile[V](handle.env.self, flres.get) if res.isErr(): - # echo "FS:ERROR: ", res.error() yield DbQueryResponse[K,V].failure res.error() continue res.get() else: V.new() - # echo "FS:SUCCESS: ", key yield success (key.some, data) handle.close() diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index 3e99ceea..a7fd3145 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -160,11 +160,7 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(block: - echo "\nRES: ", it.repr - # quit(1) - it.tryGet() - ) + res = handle.iter().toSeq().mapIt(it.tryGet()) # ).filterIt(it.isOk).mapIt(it.tryGet()) check: From ea4790a6b01095a5b317eee846b803dd3232068e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 19:59:41 -0700 Subject: [PATCH 399/445] refactor - tests --- datastore.nimble | 2 +- datastore/fsds.nim | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datastore.nimble b/datastore.nimble index bcd31398..f108da78 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -9,7 +9,7 @@ license = "Apache License 2.0 or MIT" requires "nim >= 1.6.14", "asynctest >= 0.3.1 & < 0.4.0", "chronos#0277b65be2c7a365ac13df002fba6e172be55537", - "questionable", + "questionable#head", "sqlite3_abi", "stew", "unittest2", diff --git a/datastore/fsds.nim b/datastore/fsds.nim index d0eacceb..2877d7cc 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -190,7 +190,6 @@ proc query*[K,V]( let key = query.key without path =? self.findPath(key), error: return failure error - echo "query:key: ", key let basePath = # it there is a file in the directory From 58fcdd9460ccd34cb0f3a5c04a8c01761b65ed41 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:04:18 -0700 Subject: [PATCH 400/445] switch to two sqlites for now --- tests/datastore/testtieredds.nim | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/datastore/testtieredds.nim b/tests/datastore/testtieredds.nim index 4ea76a74..4186cd51 100644 --- a/tests/datastore/testtieredds.nim +++ b/tests/datastore/testtieredds.nim @@ -23,7 +23,8 @@ suite "Test Basic Tired Datastore": var ds1: SQLiteDatastore - ds2: FSDatastore + ds2: SQLiteDatastore + # ds2: FSDatastore tiredDs: TieredDatastore setupAll: @@ -32,8 +33,9 @@ suite "Test Basic Tired Datastore": createDir(rootAbs) ds1 = SQLiteDatastore.new(Memory).tryGet - ds2 = FSDatastore.new(rootAbs, depth = 5).tryGet - tiredDs = TieredDatastore.new(@[ds1, ds2]).tryGet + ds2 = SQLiteDatastore.new(Memory).tryGet + # ds2 = FSDatastore.new(rootAbs, depth = 5).tryGet + tiredDs = TieredDatastore.new(@[Datastore ds1, ds2]).tryGet teardownAll: removeDir(rootAbs) @@ -52,14 +54,16 @@ suite "TieredDatastore": var ds1: SQLiteDatastore - ds2: FSDatastore + ds2: SQLiteDatastore + # ds2: FSDatastore setup: removeDir(rootAbs) require(not dirExists(rootAbs)) createDir(rootAbs) ds1 = SQLiteDatastore.new(Memory).get - ds2 = FSDatastore.new(rootAbs, depth = 5).get + ds2 = SQLiteDatastore.new(Memory).get + # ds2 = FSDatastore.new(rootAbs, depth = 5).get teardown: if not ds1.isNil: @@ -76,17 +80,17 @@ suite "TieredDatastore": TieredDatastore.new([]).isErr TieredDatastore.new(@[]).isErr TieredDatastore.new(ds1, ds2).isOk - TieredDatastore.new([ds1, ds2]).isOk - TieredDatastore.new(@[ds1, ds2]).isOk + TieredDatastore.new([Datastore ds1, ds2]).isOk + TieredDatastore.new(@[Datastore ds1, ds2]).isOk test "accessors": let - stores = @[ds1, ds2] + stores = @[Datastore ds1, ds2] check: TieredDatastore.new(ds1, ds2).tryGet.stores == stores - TieredDatastore.new([ds1, ds2]).tryGet.stores == stores - TieredDatastore.new(@[ds1, ds2]).tryGet.stores == stores + TieredDatastore.new([Datastore ds1, ds2]).tryGet.stores == stores + TieredDatastore.new(@[Datastore ds1, ds2]).tryGet.stores == stores test "put": let From e761957fb654305601d49f3a2999f5dfc617a6a7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:06:01 -0700 Subject: [PATCH 401/445] switch to two sqlites for now --- tests/datastore/testmountedds.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/datastore/testmountedds.nim b/tests/datastore/testmountedds.nim index fbe0dfdc..4e6a7af1 100644 --- a/tests/datastore/testmountedds.nim +++ b/tests/datastore/testmountedds.nim @@ -28,7 +28,8 @@ suite "Test Basic Mounted Datastore": var sql: SQLiteDatastore - fs: FSDatastore + sql2: SQLiteDatastore + # fs: FSDatastore mountedDs: MountedDatastore setupAll: @@ -37,10 +38,11 @@ suite "Test Basic Mounted Datastore": createDir(rootAbs) sql = SQLiteDatastore.new(Memory).tryGet - fs = FSDatastore.new(rootAbs, depth = 5).tryGet + sql2 = SQLiteDatastore.new(Memory).tryGet + # fs = FSDatastore.new(rootAbs, depth = 5).tryGet mountedDs = MountedDatastore.new({ sqlKey: Datastore(sql), - fsKey: Datastore(fs)}.toTable) + fsKey: Datastore(sql2)}.toTable) .tryGet teardownAll: From b7c93083c24bf31ece8ca6ce44a6bc8a203d4203 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:09:53 -0700 Subject: [PATCH 402/445] switch to two sqlites for now --- tests/datastore/testfsds.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 8a0dd176..1d954140 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -14,6 +14,7 @@ import pkg/datastore/backend import ./backendCommonTests + suite "Test Basic FSDatastore": let path = currentSourcePath() # get this file's name From 8f2f06e39f11db4fef1beb3ab8749c08d617dc97 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:10:47 -0700 Subject: [PATCH 403/445] skip fsds for now --- tests/testall.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testall.nim b/tests/testall.nim index 83bdf381..f058de01 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -1,7 +1,7 @@ import ./datastore/testkey, ./datastore/testdatastore, - ./datastore/testfsds, + # ./datastore/testfsds, ./datastore/testsql, ./datastore/testtieredds, ./datastore/testmountedds, From 4b804d1c9f681a22d61168066d09e6ce1b029bb7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:13:33 -0700 Subject: [PATCH 404/445] change threding lib --- datastore.nimble | 2 +- tests/testall.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore.nimble b/datastore.nimble index f108da78..a6505981 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -14,7 +14,7 @@ requires "nim >= 1.6.14", "stew", "unittest2", "pretty", - "threading", + "https://github.com/elcritch/threading#test-smartptrsleak", "taskpools", "upraises >= 0.1.0 & < 0.2.0", "chronicles" diff --git a/tests/testall.nim b/tests/testall.nim index f058de01..47e2960a 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -1,8 +1,8 @@ import ./datastore/testkey, ./datastore/testdatastore, - # ./datastore/testfsds, ./datastore/testsql, + ./datastore/testfsds, ./datastore/testtieredds, ./datastore/testmountedds, ./datastore/testdatabuffer, From e6afc68caba26782afe9c85aba457b9429621ba7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 20:28:40 -0700 Subject: [PATCH 405/445] whats up with windows + questionable + generics --- datastore/threads/threadproxyds.nim | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8be8a8a5..e85038c8 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -141,8 +141,9 @@ proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = method has*[BT](self: ThreadDatastore[BT], key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() - without signal =? acquireSignal(), err: - return failure err + let signal = acquireSignal().get() + # without signal =? acquireSignal(), err: + # return failure err let ctx = newTaskCtx(bool, signal=signal) dispatchTask(self, signal): @@ -160,8 +161,9 @@ method delete*[BT](self: ThreadDatastore[BT], key: Key): Future[?!void] {.async.} = ## delete key await self.semaphore.acquire() - without signal =? acquireSignal(), err: - return failure err + let signal = acquireSignal().get() + # without signal =? acquireSignal(), err: + # return failure err let ctx = newTaskCtx(void, signal=signal) dispatchTask(self, signal): @@ -191,8 +193,9 @@ method put*[BT](self: ThreadDatastore[BT], data: seq[byte]): Future[?!void] {.async.} = ## put key with data await self.semaphore.acquire() - without signal =? acquireSignal(), err: - return failure err + let signal = acquireSignal().get() + # without signal =? acquireSignal(), err: + # return failure err let ctx = newTaskCtx(void, signal=signal) dispatchTask(self, signal): @@ -224,8 +227,9 @@ method get*[BT](self: ThreadDatastore[BT], key: Key, ): Future[?!seq[byte]] {.async.} = await self.semaphore.acquire() - without signal =? acquireSignal(), err: - return failure err + let signal = acquireSignal().get() + # without signal =? acquireSignal(), err: + # return failure err let ctx = newTaskCtx(DataBuffer, signal=signal) dispatchTask(self, signal): @@ -287,10 +291,12 @@ method query*[BT](self: ThreadDatastore[BT], ## keeps one thread running queryTask until finished ## await self.semaphore.acquire() - without signal =? acquireSignal(), err: - return failure err - without nextSignal =? acquireSignal(), err: - return failure err + let signal = acquireSignal().get() + # without signal =? acquireSignal(), err: + # return failure err + let nextSignal = acquireSignal().get() + # without nextSignal =? acquireSignal(), err: + # return failure err let ctx = newTaskCtx(QResult, signal=signal, nextSignal=nextSignal) proc iterDispose() {.async.} = From eba40334b8c53a65dae560e13a00a7b4d491270f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 21:03:35 -0700 Subject: [PATCH 406/445] whats up with windows + questionable + generics --- datastore/fsds.nim | 15 ++++++-- datastore/sql/sqliteds.nim | 17 +++++++-- datastore/threads/threadproxyds.nim | 10 ++++- tests/datastore/testthreadproxyds.nim | 55 +++++++++++++-------------- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 2877d7cc..ee18c566 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -178,6 +178,12 @@ proc close*[K,V](self: FSDatastore[K,V]): ?!void = return success() type + FsQueryHandle*[K, V] = object + query*: DbQuery[K] + cancel*: bool + closed*: bool + env*: FsQueryEnv[K,V] + FsQueryEnv*[K,V] = object self: FSDatastore[K,V] basePath: DataBuffer @@ -185,7 +191,7 @@ type proc query*[K,V]( self: FSDatastore[K,V], query: DbQuery[K], -): Result[DbQueryHandle[K, V, FsQueryEnv[K,V]], ref CatchableError] = +): Result[FsQueryHandle[K, V], ref CatchableError] = let key = query.key without path =? self.findPath(key), error: @@ -202,14 +208,15 @@ proc query*[K,V]( path.changeFileExt("") let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) - success DbQueryHandle[KeyId, V, FsQueryEnv[K,V]](query: query, env: env) + success FsQueryHandle[K, V](query: query, env: env) proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv[K,V]]) = if not handle.closed: handle.closed = true -iterator iter*[K, V](handle: var DbQueryHandle[K, V, FsQueryEnv[K,V]] - ): ?!DbQueryResponse[K, V] = +iterator queryIter*[K, V]( + handle: var FsQueryHandle[K, V] +): ?!DbQueryResponse[K, V] = let root = $(handle.env.self.root) let basePath = $(handle.env.basePath) diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index e86fb905..e642b070 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -103,10 +103,17 @@ proc close*[K,V](self: SQLiteBackend[K,V]): ?!void = return success() +type + SqQueryHandle*[K, V] = object + query*: DbQuery[K] + cancel*: bool + closed*: bool + env*: RawStmtPtr + proc query*[K,V]( self: SQLiteBackend[K,V], query: DbQuery[K] -): Result[DbQueryHandle[K,V,RawStmtPtr], ref CatchableError] = +): Result[SqQueryHandle[K,V], ref CatchableError] = var queryStr = if query.value: @@ -151,16 +158,18 @@ proc query*[K,V]( if not (v == SQLITE_OK): return failure newException(DatastoreError, $sqlite3_errstr(v)) - success DbQueryHandle[K,V,RawStmtPtr](query: query, env: s) + success SqQueryHandle[K,V](query: query, env: s) -proc close*[K,V](handle: var DbQueryHandle[K,V,RawStmtPtr]) = +proc close*[K,V](handle: var SqQueryHandle[K,V]) = if not handle.closed: handle.closed = true discard sqlite3_reset(handle.env) discard sqlite3_clear_bindings(handle.env) handle.env.dispose() -iterator iter*[K, V](handle: var DbQueryHandle[K, V, RawStmtPtr]): ?!DbQueryResponse[K, V] = +iterator queyIter*[K, V]( + handle: var SqQueryHandle[K, V] +): ?!DbQueryResponse[K, V] = while not handle.cancel: let v = sqlite3_step(handle.env) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e85038c8..1f4d6796 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -23,6 +23,7 @@ import ../key import ../query import ../datastore import ../backend +import ../fsds import ../sql/sqliteds import ./asyncsemaphore @@ -251,10 +252,13 @@ method queryTask[DB]( query: DbQuery[KeyId], ) {.gcsafe, nimcall.} = ## run query command + mixin queryIter executeTask(ctx): # we execute this all inside `executeTask` # so we need to return a final result - let handleRes = ds.query(query) + let handleRes = query(ds, query) + static: + echo "HANDLE_RES: ", typeof(handleRes) if handleRes.isErr(): # set error and exit executeTask, which will fire final signal (?!QResult).err(handleRes.error()) @@ -266,7 +270,9 @@ method queryTask[DB]( raise newException(DeadThreadDefect, "queryTask timed out") var handle = handleRes.get() - for item in handle.iter(): + static: + echo "HANDLE: ", typeof(handle) + for item in handle.queyIter(): # wait for next request from async thread if ctx[].cancelled: diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index d59992cc..801f987b 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -15,8 +15,8 @@ import pkg/questionable/results import pkg/chronicles import pkg/threading/smartptrs -import pkg/datastore/sql/sqliteds import pkg/datastore/fsds +import pkg/datastore/sql/sqliteds import pkg/datastore/threads/threadproxyds {.all.} import ./dscommontests @@ -82,40 +82,39 @@ for i in 1..N: queryTests(ds, true) GC_fullCollect() -# suite "Test Basic ThreadDatastore with fsds": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath -# key = Key.init("/a/b").tryGet() -# bytes = "some bytes".toBytes -# otherBytes = "some other bytes".toBytes +suite "Test Basic ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes -# var -# fsStore: FSDatastore -# ds: ThreadDatastore -# taskPool: Taskpool + var + fsStore: FSDatastore[KeyId, DataBuffer] + ds: ThreadDatastore[FSDatastore[KeyId, DataBuffer]] + taskPool: Taskpool -# setupAll: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) -# fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() + fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() + ds = ThreadDatastore.new(fsStore, tp = taskPool).tryGet() -# teardown: -# GC_fullCollect() + teardown: + GC_fullCollect() -# teardownAll: -# (await ds.close()).tryGet() -# taskPool.shutdown() + teardownAll: + (await ds.close()).tryGet() -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + basicStoreTests(ds, key, bytes, otherBytes) -# basicStoreTests(fsStore, key, bytes, otherBytes) # suite "Test Query ThreadDatastore with fsds": # let From e6533643115c689a278e6bc7023b2e09da174352 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 21:07:31 -0700 Subject: [PATCH 407/445] whats up with windows + questionable + generics --- datastore/fsds.nim | 2 +- datastore/sql/sqliteds.nim | 2 +- datastore/threads/threadproxyds.nim | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index ee18c566..e3a6464a 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -210,7 +210,7 @@ proc query*[K,V]( let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) success FsQueryHandle[K, V](query: query, env: env) -proc close*[K,V](handle: var DbQueryHandle[K,V,FsQueryEnv[K,V]]) = +proc close*[K,V](handle: var FsQueryHandle[K,V]) = if not handle.closed: handle.closed = true diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index e642b070..79d4a31c 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -167,7 +167,7 @@ proc close*[K,V](handle: var SqQueryHandle[K,V]) = discard sqlite3_clear_bindings(handle.env) handle.env.dispose() -iterator queyIter*[K, V]( +iterator queryIter*[K, V]( handle: var SqQueryHandle[K, V] ): ?!DbQueryResponse[K, V] = while not handle.cancel: diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 1f4d6796..8d85c843 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -152,7 +152,7 @@ method has*[BT](self: ThreadDatastore[BT], self.tp.spawn hasTask(ctx, ds, key) return ctx[].res.toRes(v => v) -method deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; +proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): @@ -217,7 +217,7 @@ method put*[DB]( return success() -method getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; +proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; key: KeyId) {.gcsafe, nimcall.} = ## run backend command executeTask(ctx): @@ -246,11 +246,11 @@ method close*[BT](self: ThreadDatastore[BT]): Future[?!void] {.async.} = type QResult = DbQueryResponse[KeyId, DataBuffer] -method queryTask[DB]( +proc queryTask[DB]( ctx: TaskCtx[QResult], ds: DB, query: DbQuery[KeyId], -) {.gcsafe, nimcall.} = +) = ## run query command mixin queryIter executeTask(ctx): @@ -272,7 +272,7 @@ method queryTask[DB]( var handle = handleRes.get() static: echo "HANDLE: ", typeof(handle) - for item in handle.queyIter(): + for item in handle.queryIter(): # wait for next request from async thread if ctx[].cancelled: From f98432da858ef7cc3074efd24d41a7be9d87f8d0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 21:08:40 -0700 Subject: [PATCH 408/445] fixed thing --- tests/datastore/testthreadproxyds.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 801f987b..78a637ea 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -94,7 +94,6 @@ suite "Test Basic ThreadDatastore with fsds": var fsStore: FSDatastore[KeyId, DataBuffer] ds: ThreadDatastore[FSDatastore[KeyId, DataBuffer]] - taskPool: Taskpool setupAll: removeDir(basePathAbs) From d2d051d51f80a251ff9d66b066e6ff9f3ac71ed9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 21:10:30 -0700 Subject: [PATCH 409/445] fixed thing --- datastore/threads/threadproxyds.nim | 1 - tests/datastore/testthreadproxyds.nim | 1 - 2 files changed, 2 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 8d85c843..6dc1ae5d 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -133,7 +133,6 @@ template dispatchTask[BT](self: ThreadDatastore[BT], discard ctx[].signal.close() self.semaphore.release() - proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command executeTask(ctx): diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 78a637ea..3a309484 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -114,7 +114,6 @@ suite "Test Basic ThreadDatastore with fsds": basicStoreTests(ds, key, bytes, otherBytes) - # suite "Test Query ThreadDatastore with fsds": # let # path = currentSourcePath() # get this file's name From 3822a97593b11922fb81cf8bbcf98ca3b760fcf8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 21:13:06 -0700 Subject: [PATCH 410/445] fixed thing --- datastore/sql.nim | 9 ++++----- tests/datastore/backendCommonTests.nim | 18 +++++++++--------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 28b1f3d0..cfe67ac6 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -60,10 +60,9 @@ method put*(self: SQLiteDatastore, method close*(self: SQLiteDatastore): Future[?!void] {.async.} = self.db.close() -method queryIter*( - self: SQLiteDatastore, - query: Query -): ?!(iterator(): ?!QueryResponse) = +method queryIter*(self: SQLiteDatastore, + query: Query + ): ?!(iterator(): ?!QueryResponse) = let dbquery = dbQuery( key= KeyId.new query.key.id(), @@ -75,7 +74,7 @@ method queryIter*( var qhandle = ? self.db.query(dbquery) let iter = iterator(): ?!QueryResponse = - for resp in qhandle.iter(): + for resp in qhandle.queryIter(): without qres =? resp, err: yield QueryResponse.failure err let k = qres.key.map() do(k: KeyId) -> Key: diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/backendCommonTests.nim index a7fd3145..f35fd328 100644 --- a/tests/datastore/backendCommonTests.nim +++ b/tests/datastore/backendCommonTests.nim @@ -81,7 +81,7 @@ template queryTests*( var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) check: res.len == 3 @@ -107,7 +107,7 @@ template queryTests*( var res: seq[DbQueryResponse[KeyId, DataBuffer]] var cnt = 0 - for item in handle.iter(): + for item in handle.queryIter(): cnt.inc res.insert(item.tryGet(), 0) if cnt > 1: @@ -135,7 +135,7 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) check: res.len == 3 @@ -160,7 +160,7 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) # ).filterIt(it.isOk).mapIt(it.tryGet()) check: @@ -182,7 +182,7 @@ template queryTests*( var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) res.sort do (a, b: DbQueryResponse[KeyId, DataBuffer]) -> int: cmp($a.key.get, $b.key.get) @@ -214,7 +214,7 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) check: res.len == 10 @@ -239,7 +239,7 @@ template queryTests*( var handle = ds.query(q).tryGet let - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) # echo "RES: ", res.mapIt(it.key) check: @@ -260,7 +260,7 @@ template queryTests*( var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) check: res.len == 5 @@ -296,7 +296,7 @@ template queryTests*( kvs = kvs.reversed var handle = ds.query(q).tryGet - res = handle.iter().toSeq().mapIt(it.tryGet()) + res = handle.queryIter().toSeq().mapIt(it.tryGet()) check: res.len == 100 From 46f22dc10e46ac592f1250d22cd57eb027a55e13 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 13:24:55 -0700 Subject: [PATCH 411/445] test queries --- tests/datastore/testthreadproxyds.nim | 43 +++++++++++++-------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 3a309484..2fd1bb24 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -114,35 +114,32 @@ suite "Test Basic ThreadDatastore with fsds": basicStoreTests(ds, key, bytes, otherBytes) -# suite "Test Query ThreadDatastore with fsds": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath +suite "Test Query ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath -# var -# fsStore: FSDatastore -# ds: ThreadDatastore -# taskPool: Taskpool + var + fsStore: FSDatastore[KeyId, DataBuffer] + ds: ThreadDatastore[FSDatastore[KeyId, DataBuffer]] -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) -# fsStore = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(fsStore, withLocks = true, tp = taskPool).tryGet() + fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 5).tryGet() + ds = ThreadDatastore.new(fsStore, tp = taskPool).tryGet() -# teardown: -# GC_fullCollect() -# (await ds.close()).tryGet() -# taskPool.shutdown() + teardown: + GC_fullCollect() + (await ds.close()).tryGet() -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) -# queryTests(ds, false) + queryTests(ds, false) # suite "Test ThreadDatastore cancelations": # var From bd79cb73578c9ced8b987c2b436091717bb39b4d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 13:47:48 -0700 Subject: [PATCH 412/445] test task cancel --- datastore/threads/threadproxyds.nim | 10 ++-- tests/datastore/testthreadproxyds.nim | 75 +++++++++++---------------- 2 files changed, 35 insertions(+), 50 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 6dc1ae5d..cbe5e455 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -38,10 +38,10 @@ logScope: type TaskCtxObj*[T: ThreadTypes] = object - res: ThreadResult[T] + res*: ThreadResult[T] signal: ThreadSignalPtr - running: bool ## used to mark when a task worker is running - cancelled: bool ## used to cancel a task before it's started + running*: bool ## used to mark when a task worker is running + cancelled*: bool ## used to cancel a task before it's started nextSignal: ThreadSignalPtr TaskCtx*[T] = SharedPtr[TaskCtxObj[T]] @@ -78,7 +78,7 @@ proc acquireSignal(): ?!ThreadSignalPtr = else: success signal.get() -template executeTask[T](ctx: TaskCtx[T], blk: untyped) = +template executeTask*[T](ctx: TaskCtx[T], blk: untyped) = ## executes a task on a thread work and handles cleanup after cancels/errors ## try: @@ -115,7 +115,7 @@ template dispatchTaskWrap[BT](self: ThreadDatastore[BT], runTask() await wait(ctx[].signal) -template dispatchTask[BT](self: ThreadDatastore[BT], +template dispatchTask*[BT](self: ThreadDatastore[BT], signal: ThreadSignalPtr, blk: untyped ): auto = diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 2fd1bb24..45ac6ad0 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -141,55 +141,40 @@ suite "Test Query ThreadDatastore with fsds": queryTests(ds, false) -# suite "Test ThreadDatastore cancelations": -# var -# sqlStore: SQLiteBackend[KeyId,DataBuffer] -# ds: ThreadDatastore -# taskPool: Taskpool - -# privateAccess(ThreadDatastore) # expose private fields -# privateAccess(TaskCtx) # expose private fields - -# setupAll: -# sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() -# taskPool = Taskpool.new(NumThreads) -# ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - -# teardown: -# GC_fullCollect() # run full collect after each test - -# teardownAll: -# (await ds.close()).tryGet() -# taskPool.shutdown() - - # test "Should monitor signal and cancel": - # var - # signal = ThreadSignalPtr.new().tryGet() - # res = ThreadResult[void]() - # ctx = newSharedPtr(TaskCtxObj[void](signal: signal)) - # fut = newFuture[void]("signalMonitor") - # threadArgs = (addr ctx, addr fut) - # thread: Thread[type threadArgs] - - # proc threadTask(args: type threadArgs) = - # var (ctx, fut) = args - # proc asyncTask() {.async.} = - # let - # monitor = signalMonitor(ctx, fut[]) - - # await monitor +suite "Test ThreadDatastore cancelations": + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + sds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] - # waitFor asyncTask() + privateAccess(ThreadDatastore) # expose private fields + privateAccess(TaskCtx) # expose private fields - # createThread(thread, threadTask, threadArgs) - # ctx.cancelled = true - # check: ctx.signal.fireSync.tryGet + setupAll: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + sds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - # joinThreads(thread) + teardown: + GC_fullCollect() # run full collect after each test - # check: fut.cancelled - # check: ctx.signal.close().isOk - # fut = nil + test "Should monitor signal and cancel": + var + signal = ThreadSignalPtr.new().tryGet() + res = ThreadResult[void]() + + proc cancelTestTask[T](ctx: TaskCtx[T]) {.gcsafe.} = + executeTask(ctx): + (?!bool).ok(true) + + let ctx = newTaskCtx(bool, signal=signal) + ctx[].cancelled = true + dispatchTask(sds, signal): + sds.tp.spawn cancelTestTask(ctx) + + echo "ctx: ", ctx[] + check: + ctx[].res.isErr == true + ctx[].cancelled == true + ctx[].running == false # test "Should monitor and not cancel": # var From 248174630143d9f7356160c8510fcd895b5ffaee Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 14:25:25 -0700 Subject: [PATCH 413/445] test task cancel --- datastore/threads/threadproxyds.nim | 2 - tests/datastore/testthreadproxyds.nim | 80 ++++++++++++++------------- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index cbe5e455..19b5def9 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -305,11 +305,9 @@ method query*[BT](self: ThreadDatastore[BT], let ctx = newTaskCtx(QResult, signal=signal, nextSignal=nextSignal) proc iterDispose() {.async.} = - # echo "signal:CLOSE!" ctx.setCancelled() await ctx[].nextSignal.fire() discard ctx[].signal.close() - # echo "nextSignal:CLOSE!" discard ctx[].nextSignal.close() self.semaphore.release() diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 45ac6ad0..97994113 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -44,7 +44,6 @@ for i in 1..N: setupAll: sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - # taskPool = Taskpool.new(NumThreads) ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() teardown: @@ -52,7 +51,6 @@ for i in 1..N: teardownAll: (await ds.close()).tryGet() - # taskPool.shutdown() for i in 1..M: basicStoreTests(ds, key, bytes, otherBytes) @@ -76,7 +74,6 @@ for i in 1..N: GC_fullCollect() (await ds.close()).tryGet() - # taskPool.shutdown() for i in 1..M: queryTests(ds, true) @@ -159,9 +156,8 @@ suite "Test ThreadDatastore cancelations": test "Should monitor signal and cancel": var signal = ThreadSignalPtr.new().tryGet() - res = ThreadResult[void]() - proc cancelTestTask[T](ctx: TaskCtx[T]) {.gcsafe.} = + proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = executeTask(ctx): (?!bool).ok(true) @@ -170,40 +166,50 @@ suite "Test ThreadDatastore cancelations": dispatchTask(sds, signal): sds.tp.spawn cancelTestTask(ctx) - echo "ctx: ", ctx[] check: ctx[].res.isErr == true ctx[].cancelled == true ctx[].running == false - # test "Should monitor and not cancel": - # var - # signal = ThreadSignalPtr.new().tryGet() - # res = ThreadResult[void]() - # ctx = TaskCtx[void]( - # ds: sqlStore, - # res: addr res, - # signal: signal) - # fut = newFuture[void]("signalMonitor") - # threadArgs = (addr ctx, addr fut) - # thread: Thread[type threadArgs] - - # proc threadTask(args: type threadArgs) = - # var (ctx, fut) = args - # proc asyncTask() {.async.} = - # let - # monitor = signalMonitor(ctx, fut[]) - - # await monitor - - # waitFor asyncTask() - - # createThread(thread, threadTask, threadArgs) - # ctx.cancelled = false - # check: ctx.signal.fireSync.tryGet - - # joinThreads(thread) - - # check: not fut.cancelled - # check: ctx.signal.close().isOk - # fut = nil + test "Should cancel future": + + var + signal = ThreadSignalPtr.new().tryGet() + ms {.global.}: MutexSignal + flag {.global.}: int = 0 + + ms.init() + + type + TestValue = object + ThreadTestInt = (TestValue, ) + + proc `=destroy`(obj: var TestValue) = + echo "destroy TestObj!" + flag = 10 + + proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = + executeTask(ctx): + discard ctx[].signal.fireSync() + ms.wait() + (?!ThreadTestInt).ok(default(ThreadTestInt)) + + proc runTestTask() {.async.} = + + let ctx = newTaskCtx(ThreadTestInt, signal=signal) + dispatchTask(sds, signal): + sds.tp.spawn errorTestTask(ctx) + + echo "raise error" + raise newException(ValueError, "fake error") + + try: + await runTestTask() + except CatchableError as exc: + echo "caught: ", $exc + finally: + echo "finish" + ms.fire() + os.sleep(10) + check flag == 10 + From 43520c360833744834fca91df7acee9586652068 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 14:51:27 -0700 Subject: [PATCH 414/445] test task cancel --- datastore/threads/threadproxyds.nim | 5 +++- tests/datastore/testthreadproxyds.nim | 35 +++++++++++++++++++++------ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 19b5def9..e1a75718 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -128,8 +128,10 @@ template dispatchTask*[BT](self: ThreadDatastore[BT], trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() raise exc + except CatchableError as exc: + ctx.setCancelled() + raise exc finally: - # echo "signal:CLOSE!" discard ctx[].signal.close() self.semaphore.release() @@ -359,6 +361,7 @@ method query*[BT](self: ThreadDatastore[BT], return success iter except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg + ctx.setCancelled() await iterDispose() raise exc diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 97994113..c7f825de 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -14,6 +14,7 @@ import pkg/taskpools import pkg/questionable/results import pkg/chronicles import pkg/threading/smartptrs +import pkg/threading/atomics import pkg/datastore/fsds import pkg/datastore/sql/sqliteds @@ -176,7 +177,8 @@ suite "Test ThreadDatastore cancelations": var signal = ThreadSignalPtr.new().tryGet() ms {.global.}: MutexSignal - flag {.global.}: int = 0 + flag {.global.}: Atomic[bool] + ready {.global.}: Atomic[bool] ms.init() @@ -186,12 +188,25 @@ suite "Test ThreadDatastore cancelations": proc `=destroy`(obj: var TestValue) = echo "destroy TestObj!" - flag = 10 + flag.store(true) + + proc wait(flag: var Atomic[bool]) = + echo "wait for task to be ready..." + defer: echo "" + for i in 1..100: + stdout.write(".") + if flag.load() == true: + return + os.sleep(10) + raise newException(Defect, "timeout") proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = executeTask(ctx): + echo "task:exec" discard ctx[].signal.fireSync() + ready.store(true) ms.wait() + echo "ctx:task: ", ctx[] (?!ThreadTestInt).ok(default(ThreadTestInt)) proc runTestTask() {.async.} = @@ -199,17 +214,21 @@ suite "Test ThreadDatastore cancelations": let ctx = newTaskCtx(ThreadTestInt, signal=signal) dispatchTask(sds, signal): sds.tp.spawn errorTestTask(ctx) - - echo "raise error" - raise newException(ValueError, "fake error") + ready.wait() + echo "raise error" + raise newException(ValueError, "fake error") try: - await runTestTask() + block: + await runTestTask() except CatchableError as exc: echo "caught: ", $exc finally: echo "finish" + check ready.load() == true + ms.fire() - os.sleep(10) - check flag == 10 + GC_fullCollect() + flag.wait() + check flag.load() == true From 47fd3ba61fbbbfe40c4a8e804b88e0665f877bae Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 14:57:20 -0700 Subject: [PATCH 415/445] test task cancel --- tests/datastore/testthreadproxyds.nim | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index c7f825de..8a20fd4b 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -178,11 +178,13 @@ suite "Test ThreadDatastore cancelations": signal = ThreadSignalPtr.new().tryGet() ms {.global.}: MutexSignal flag {.global.}: Atomic[bool] + futFreed {.global.}: Atomic[bool] ready {.global.}: Atomic[bool] ms.init() type + FutTestObj = object TestValue = object ThreadTestInt = (TestValue, ) @@ -190,6 +192,10 @@ suite "Test ThreadDatastore cancelations": echo "destroy TestObj!" flag.store(true) + proc `=destroy`(obj: var FutTestObj) = + echo "destroy FutTestObj!" + futFreed.store(true) + proc wait(flag: var Atomic[bool]) = echo "wait for task to be ready..." defer: echo "" @@ -211,6 +217,10 @@ suite "Test ThreadDatastore cancelations": proc runTestTask() {.async.} = + let obj = FutTestObj() + await sleepAsync(1.milliseconds) + defer: echo "fut FutTestObj: ", obj + let ctx = newTaskCtx(ThreadTestInt, signal=signal) dispatchTask(sds, signal): sds.tp.spawn errorTestTask(ctx) @@ -226,9 +236,12 @@ suite "Test ThreadDatastore cancelations": finally: echo "finish" check ready.load() == true + GC_fullCollect() + futFreed.wait() + echo "future freed it's mem!" + check futFreed.load() == true ms.fire() - GC_fullCollect() flag.wait() check flag.load() == true From 0df63d61023de9303fcfde018dd8f093e595aae3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 15:05:07 -0700 Subject: [PATCH 416/445] test task cancel --- tests/datastore/testthreadproxyds.nim | 199 +++++++++++++------------- 1 file changed, 101 insertions(+), 98 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 8a20fd4b..021e23f6 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -139,109 +139,112 @@ suite "Test Query ThreadDatastore with fsds": queryTests(ds, false) -suite "Test ThreadDatastore cancelations": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - sds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] - - privateAccess(ThreadDatastore) # expose private fields - privateAccess(TaskCtx) # expose private fields - - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - sds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() # run full collect after each test - - test "Should monitor signal and cancel": +for i in 1..N: + suite "Test ThreadDatastore cancelations": var - signal = ThreadSignalPtr.new().tryGet() + sqlStore: SQLiteBackend[KeyId,DataBuffer] + sds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] - proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = - executeTask(ctx): - (?!bool).ok(true) + privateAccess(ThreadDatastore) # expose private fields + privateAccess(TaskCtx) # expose private fields - let ctx = newTaskCtx(bool, signal=signal) - ctx[].cancelled = true - dispatchTask(sds, signal): - sds.tp.spawn cancelTestTask(ctx) + setupAll: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + sds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - check: - ctx[].res.isErr == true - ctx[].cancelled == true - ctx[].running == false + teardown: + GC_fullCollect() # run full collect after each test - test "Should cancel future": + test "Should monitor signal and cancel": + var + signal = ThreadSignalPtr.new().tryGet() - var - signal = ThreadSignalPtr.new().tryGet() - ms {.global.}: MutexSignal - flag {.global.}: Atomic[bool] - futFreed {.global.}: Atomic[bool] - ready {.global.}: Atomic[bool] - - ms.init() - - type - FutTestObj = object - TestValue = object - ThreadTestInt = (TestValue, ) - - proc `=destroy`(obj: var TestValue) = - echo "destroy TestObj!" - flag.store(true) - - proc `=destroy`(obj: var FutTestObj) = - echo "destroy FutTestObj!" - futFreed.store(true) - - proc wait(flag: var Atomic[bool]) = - echo "wait for task to be ready..." - defer: echo "" - for i in 1..100: - stdout.write(".") - if flag.load() == true: - return - os.sleep(10) - raise newException(Defect, "timeout") - - proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = - executeTask(ctx): - echo "task:exec" - discard ctx[].signal.fireSync() - ready.store(true) - ms.wait() - echo "ctx:task: ", ctx[] - (?!ThreadTestInt).ok(default(ThreadTestInt)) - - proc runTestTask() {.async.} = - - let obj = FutTestObj() - await sleepAsync(1.milliseconds) - defer: echo "fut FutTestObj: ", obj - - let ctx = newTaskCtx(ThreadTestInt, signal=signal) - dispatchTask(sds, signal): - sds.tp.spawn errorTestTask(ctx) - ready.wait() - echo "raise error" - raise newException(ValueError, "fake error") - - try: - block: - await runTestTask() - except CatchableError as exc: - echo "caught: ", $exc - finally: - echo "finish" - check ready.load() == true - GC_fullCollect() - futFreed.wait() - echo "future freed it's mem!" - check futFreed.load() == true + proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = + executeTask(ctx): + (?!bool).ok(true) - ms.fire() - flag.wait() - check flag.load() == true + let ctx = newTaskCtx(bool, signal=signal) + ctx[].cancelled = true + dispatchTask(sds, signal): + sds.tp.spawn cancelTestTask(ctx) + + check: + ctx[].res.isErr == true + ctx[].cancelled == true + ctx[].running == false + + test "Should cancel future": + + var + signal = ThreadSignalPtr.new().tryGet() + ms {.global.}: MutexSignal + flag {.global.}: Atomic[bool] + futFreed {.global.}: Atomic[bool] + ready {.global.}: Atomic[bool] + + ms.init() + + type + FutTestObj = object + val: int + TestValue = object + ThreadTestInt = (TestValue, ) + + proc `=destroy`(obj: var TestValue) = + # echo "destroy TestObj!" + flag.store(true) + + proc `=destroy`(obj: var FutTestObj) = + # echo "destroy FutTestObj!" + futFreed.store(true) + + proc wait(flag: var Atomic[bool], name = "task") = + # echo "wait for " & name & " to be ready..." + defer: echo "" + for i in 1..100: + # stdout.write(".") + if flag.load() == true: + return + os.sleep(10) + raise newException(Defect, "timeout") + + proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = + executeTask(ctx): + # echo "task:exec" + discard ctx[].signal.fireSync() + ready.store(true) + ms.wait() + # echo "ctx:task: ", ctx[] + (?!ThreadTestInt).ok(default(ThreadTestInt)) + + proc runTestTask() {.async.} = + let obj = FutTestObj(val: 42) + await sleepAsync(1.milliseconds) + try: + let ctx = newTaskCtx(ThreadTestInt, signal=signal) + dispatchTask(sds, signal): + sds.tp.spawn errorTestTask(ctx) + ready.wait() + # echo "raise error" + raise newException(ValueError, "fake error") + finally: + # echo "fut FutTestObj: ", obj + assert obj.val == 42 # need to force future to keep ref here + try: + block: + await runTestTask() + except CatchableError as exc: + # echo "caught: ", $exc + discard + finally: + # echo "finish" + check ready.load() == true + GC_fullCollect() + futFreed.wait("futFreed") + echo "future freed it's mem!" + check futFreed.load() == true + + ms.fire() + flag.wait("flag") + check flag.load() == true From 374e5a891f3625c03024e2749b27643cde46486d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 15:07:40 -0700 Subject: [PATCH 417/445] test task cancel --- tests/datastore/testthreadproxyds.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 021e23f6..db20c819 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -200,7 +200,7 @@ for i in 1..N: proc wait(flag: var Atomic[bool], name = "task") = # echo "wait for " & name & " to be ready..." - defer: echo "" + # defer: echo "" for i in 1..100: # stdout.write(".") if flag.load() == true: @@ -214,7 +214,7 @@ for i in 1..N: discard ctx[].signal.fireSync() ready.store(true) ms.wait() - # echo "ctx:task: ", ctx[] + echo "task context memory: ", ctx[] (?!ThreadTestInt).ok(default(ThreadTestInt)) proc runTestTask() {.async.} = @@ -247,4 +247,3 @@ for i in 1..N: ms.fire() flag.wait("flag") check flag.load() == true - From 4002ba9507f9df558cd76e8e3d775d2b46d6d598 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:14:27 -0700 Subject: [PATCH 418/445] reorg --- datastore/fsds.nim | 312 ++++-------------- datastore/query.nim | 2 +- datastore/sql.nim | 6 +- datastore/sql/sqlitedsdb.nim | 2 +- datastore/sql/sqliteutils.nim | 2 +- datastore/{ => threads}/backend.nim | 4 +- datastore/threads/fsbackend.nim | 273 +++++++++++++++ .../sqliteds.nim => threads/sqlbackend.nim} | 4 +- datastore/threads/threadproxyds.nim | 24 +- datastore/threads/threadresult.nim | 2 +- tests/datastore/sql/testsqliteds.nim | 2 +- tests/datastore/sql/testsqlitedsdb.nim | 2 +- tests/datastore/testfsds.nim | 2 +- tests/datastore/testthreadproxyds.nim | 6 +- 14 files changed, 371 insertions(+), 272 deletions(-) rename datastore/{ => threads}/backend.nim (97%) create mode 100644 datastore/threads/fsbackend.nim rename datastore/{sql/sqliteds.nim => threads/sqlbackend.nim} (99%) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index e3a6464a..178de6e9 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -7,267 +7,93 @@ import pkg/questionable/results from pkg/stew/results as stewResults import get, isErr import pkg/upraises -import ./backend +import ./threads/backend import ./datastore export datastore push: {.upraises: [].} -type - FSDatastore*[K, V] = object - root*: DataBuffer - ignoreProtected: bool - depth: int - -proc isRootSubdir*(root, path: string): bool = - path.startsWith(root) - -proc validDepth*(self: FSDatastore, key: Key): bool = - key.len <= self.depth - -proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = - ## Return filename corresponding to the key - ## or failure if the key doesn't correspond to a valid filename - ## - let root = $self.root - let key = Key.init($key).get() - if not self.validDepth(key): - return failure "Path has invalid depth!" - - var - segments: seq[string] - - for ns in key: - let basename = ns.value.extractFilename - if basename == "" or not basename.isValidFilename: - return failure "Filename contains invalid chars!" - - if ns.field == "": - segments.add(ns.value) - else: - let basename = ns.field.extractFilename - if basename == "" or not basename.isValidFilename: - return failure "Filename contains invalid chars!" - - # `:` are replaced with `/` - segments.add(ns.field / ns.value) - - let - fullname = (root / segments.joinPath()) - .absolutePath() - .catch() - .get() - .addFileExt(FileExt) - - if not root.isRootSubdir(fullname): - return failure "Path is outside of `root` directory!" - - return success fullname - -proc has*[K,V](self: FSDatastore[K,V], key: K): ?!bool = - without path =? self.findPath(key), error: - return failure error - success path.fileExists() - -proc contains*[K](self: FSDatastore, key: K): bool = - return self.has(key).get() - -proc delete*[K,V](self: FSDatastore[K,V], key: K): ?!void = - without path =? self.findPath(key), error: - return failure error - - if not path.fileExists(): - return success() - - try: - removeFile(path) - except OSError as e: - return failure e - - return success() - -proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[K]): ?!void = - for key in keys: - if err =? self.delete(key).errorOption: - return failure err - - return success() - -proc readFile[V](self: FSDatastore, path: string): ?!V = - var - file: File - - defer: - file.close - - if not file.open(path): - return failure "unable to open file! path: " & path - - try: - let - size = file.getFileSize().int - when V is seq[byte]: - var bytes = newSeq[byte](size) - elif V is V: - var bytes = V.new(size=size) - else: - {.error: "unhandled result type".} - var - read = 0 - - # echo "BYTES: ", bytes.repr - while read < size: - read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) - - if read < size: - return failure $read & " bytes were read from " & path & - " but " & $size & " bytes were expected" - - return success bytes - - except CatchableError as e: - return failure e - -proc get*[K,V](self: FSDatastore[K,V], key: K): ?!V = - without path =? self.findPath(key), error: - return failure error +type + SQLiteDatastore* = ref object of Datastore + db: SQLiteBackend[KeyId, DataBuffer] - if not path.fileExists(): - return failure( - newException(DatastoreKeyNotFound, "Key doesn't exist")) +proc path*(self: SQLiteDatastore): string = + self.db.path() - return readFile[V](self, path) +proc readOnly*(self: SQLiteDatastore): bool = + self.db.readOnly() -proc put*[K,V](self: FSDatastore[K,V], - key: K, - data: V - ): ?!void = +method has*(self: SQLiteDatastore, + key: Key): Future[?!bool] {.async.} = + return self.db.has(KeyId.new key.id()) - without path =? self.findPath(key), error: - return failure error +method delete*(self: SQLiteDatastore, + key: Key): Future[?!void] {.async.} = + return self.db.delete(KeyId.new key.id()) - try: - var data = data - createDir(parentDir(path)) - writeFile(path, data.toOpenArray(0, data.len()-1)) - except CatchableError as e: - return failure e +method delete*(self: SQLiteDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + let dkeys = keys.mapIt(KeyId.new it.id()) + return self.db.delete(dkeys) - return success() +method get*(self: SQLiteDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: + d.toSeq() -proc put*[K,V]( - self: FSDatastore, - batch: seq[DbBatchEntry[K, V]]): ?!void = +method put*(self: SQLiteDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + self.db.put(KeyId.new key.id(), DataBuffer.new data) +method put*(self: SQLiteDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + var dbatch: seq[tuple[key: KeyId, data: DataBuffer]] for entry in batch: - if err =? self.put(entry.key, entry.data).errorOption: - return failure err - - return success() - -iterator dirIter(path: string): string {.gcsafe.} = - try: - for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): - yield p - except CatchableError as exc: - raise newException(Defect, exc.msg) - -proc close*[K,V](self: FSDatastore[K,V]): ?!void = - return success() - -type - FsQueryHandle*[K, V] = object - query*: DbQuery[K] - cancel*: bool - closed*: bool - env*: FsQueryEnv[K,V] - - FsQueryEnv*[K,V] = object - self: FSDatastore[K,V] - basePath: DataBuffer - -proc query*[K,V]( - self: FSDatastore[K,V], - query: DbQuery[K], -): Result[FsQueryHandle[K, V], ref CatchableError] = - - let key = query.key - without path =? self.findPath(key), error: - return failure error - - let basePath = - # it there is a file in the directory - # with the same name then list the contents - # of the directory, otherwise recurse - # into subdirectories - if path.fileExists: - path.parentDir - else: - path.changeFileExt("") + dbatch.add((KeyId.new entry.key.id(), DataBuffer.new entry.data)) + self.db.put(dbatch) + +method close*(self: SQLiteDatastore): Future[?!void] {.async.} = + self.db.close() + +method queryIter*(self: SQLiteDatastore, + query: Query + ): ?!(iterator(): ?!QueryResponse) = + + let dbquery = dbQuery( + key= KeyId.new query.key.id(), + value= query.value, + limit= query.limit, + offset= query.offset, + sort= query.sort, + ) + var qhandle = ? self.db.query(dbquery) + + let iter = iterator(): ?!QueryResponse = + for resp in qhandle.queryIter(): + without qres =? resp, err: + yield QueryResponse.failure err + let k = qres.key.map() do(k: KeyId) -> Key: + Key.init($k).expect("valid key") + let v: seq[byte] = qres.data.toSeq() + yield success (k, v) - let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) - success FsQueryHandle[K, V](query: query, env: env) - -proc close*[K,V](handle: var FsQueryHandle[K,V]) = - if not handle.closed: - handle.closed = true - -iterator queryIter*[K, V]( - handle: var FsQueryHandle[K, V] -): ?!DbQueryResponse[K, V] = - let root = $(handle.env.self.root) - let basePath = $(handle.env.basePath) - - for path in basePath.dirIter(): - if handle.cancel: - break - - var - basePath = $handle.env.basePath - keyPath = basePath - - keyPath.removePrefix(root) - keyPath = keyPath / path.changeFileExt("") - keyPath = keyPath.replace("\\", "/") - - let - flres = (basePath / path).absolutePath().catch - if flres.isErr(): - yield DbQueryResponse[K,V].failure flres.error() - continue - - let - key = K.toKey($Key.init(keyPath).expect("valid key")) - data = - if handle.query.value: - let res = readFile[V](handle.env.self, flres.get) - if res.isErr(): - yield DbQueryResponse[K,V].failure res.error() - continue - res.get() - else: - V.new() - - yield success (key.some, data) - handle.close() + success iter -proc newFSDatastore*[K,V](root: string, - depth = 2, - caseSensitive = true, - ignoreProtected = false - ): ?!FSDatastore[K,V] = +proc new*( + T: type SQLiteDatastore, + path: string, + readOnly = false): ?!SQLiteDatastore = - let root = ? ( - block: - if root.isAbsolute: root - else: getCurrentDir() / root).catch + success SQLiteDatastore( + db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) - if not dirExists(root): - return failure "directory does not exist: " & root +proc new*( + T: type SQLiteDatastore, + db: SQLiteBackend[KeyId, DataBuffer]): ?!T = - success FSDatastore[K,V]( - root: DataBuffer.new root, - ignoreProtected: ignoreProtected, - depth: depth) + success T( + db: db, + readOnly: db.readOnly) diff --git a/datastore/query.nim b/datastore/query.nim index 256ad97e..f10376fe 100644 --- a/datastore/query.nim +++ b/datastore/query.nim @@ -6,7 +6,7 @@ import pkg/questionable/results import ./key import ./types -import ./backend +import ./threads/backend export types export options, SortOrder diff --git a/datastore/sql.nim b/datastore/sql.nim index cfe67ac6..712bd9a6 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -10,10 +10,10 @@ import pkg/upraises import std/sequtils import ../datastore -import ./backend -import ./sql/sqliteds +import ./threads/backend +import ./threads/sqlbackend -export datastore, sqliteds +export datastore, sqlbackend push: {.upraises: [].} diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index ae381e1e..06e1dcff 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -4,7 +4,7 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises -import ../backend +import ../threads/backend import ./sqliteutils export sqliteutils diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index 884527b7..ea9bbcb1 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -3,7 +3,7 @@ import pkg/questionable/results import pkg/sqlite3_abi import pkg/upraises -import ../backend +import ../threads/backend export sqlite3_abi diff --git a/datastore/backend.nim b/datastore/threads/backend.nim similarity index 97% rename from datastore/backend.nim rename to datastore/threads/backend.nim index 410f5b11..f6e3c81f 100644 --- a/datastore/backend.nim +++ b/datastore/threads/backend.nim @@ -3,8 +3,8 @@ import std/options import pkg/questionable/results -import ./threads/databuffer -import ./types +import ./databuffer +import ../types export databuffer, types, SortOrder diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim new file mode 100644 index 00000000..e3a6464a --- /dev/null +++ b/datastore/threads/fsbackend.nim @@ -0,0 +1,273 @@ +import std/os +import std/options +import std/strutils + +import pkg/questionable +import pkg/questionable/results +from pkg/stew/results as stewResults import get, isErr +import pkg/upraises + +import ./backend +import ./datastore + +export datastore + +push: {.upraises: [].} + +type + FSDatastore*[K, V] = object + root*: DataBuffer + ignoreProtected: bool + depth: int + +proc isRootSubdir*(root, path: string): bool = + path.startsWith(root) + +proc validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.depth + +proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = + ## Return filename corresponding to the key + ## or failure if the key doesn't correspond to a valid filename + ## + let root = $self.root + let key = Key.init($key).get() + if not self.validDepth(key): + return failure "Path has invalid depth!" + + var + segments: seq[string] + + for ns in key: + let basename = ns.value.extractFilename + if basename == "" or not basename.isValidFilename: + return failure "Filename contains invalid chars!" + + if ns.field == "": + segments.add(ns.value) + else: + let basename = ns.field.extractFilename + if basename == "" or not basename.isValidFilename: + return failure "Filename contains invalid chars!" + + # `:` are replaced with `/` + segments.add(ns.field / ns.value) + + let + fullname = (root / segments.joinPath()) + .absolutePath() + .catch() + .get() + .addFileExt(FileExt) + + if not root.isRootSubdir(fullname): + return failure "Path is outside of `root` directory!" + + return success fullname + +proc has*[K,V](self: FSDatastore[K,V], key: K): ?!bool = + without path =? self.findPath(key), error: + return failure error + success path.fileExists() + +proc contains*[K](self: FSDatastore, key: K): bool = + return self.has(key).get() + +proc delete*[K,V](self: FSDatastore[K,V], key: K): ?!void = + without path =? self.findPath(key), error: + return failure error + + if not path.fileExists(): + return success() + + try: + removeFile(path) + except OSError as e: + return failure e + + return success() + +proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[K]): ?!void = + for key in keys: + if err =? self.delete(key).errorOption: + return failure err + + return success() + +proc readFile[V](self: FSDatastore, path: string): ?!V = + var + file: File + + defer: + file.close + + if not file.open(path): + return failure "unable to open file! path: " & path + + try: + let + size = file.getFileSize().int + + when V is seq[byte]: + var bytes = newSeq[byte](size) + elif V is V: + var bytes = V.new(size=size) + else: + {.error: "unhandled result type".} + var + read = 0 + + # echo "BYTES: ", bytes.repr + while read < size: + read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) + + if read < size: + return failure $read & " bytes were read from " & path & + " but " & $size & " bytes were expected" + + return success bytes + + except CatchableError as e: + return failure e + +proc get*[K,V](self: FSDatastore[K,V], key: K): ?!V = + without path =? self.findPath(key), error: + return failure error + + if not path.fileExists(): + return failure( + newException(DatastoreKeyNotFound, "Key doesn't exist")) + + return readFile[V](self, path) + +proc put*[K,V](self: FSDatastore[K,V], + key: K, + data: V + ): ?!void = + + without path =? self.findPath(key), error: + return failure error + + try: + var data = data + createDir(parentDir(path)) + writeFile(path, data.toOpenArray(0, data.len()-1)) + except CatchableError as e: + return failure e + + return success() + +proc put*[K,V]( + self: FSDatastore, + batch: seq[DbBatchEntry[K, V]]): ?!void = + + for entry in batch: + if err =? self.put(entry.key, entry.data).errorOption: + return failure err + + return success() + +iterator dirIter(path: string): string {.gcsafe.} = + try: + for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): + yield p + except CatchableError as exc: + raise newException(Defect, exc.msg) + +proc close*[K,V](self: FSDatastore[K,V]): ?!void = + return success() + +type + FsQueryHandle*[K, V] = object + query*: DbQuery[K] + cancel*: bool + closed*: bool + env*: FsQueryEnv[K,V] + + FsQueryEnv*[K,V] = object + self: FSDatastore[K,V] + basePath: DataBuffer + +proc query*[K,V]( + self: FSDatastore[K,V], + query: DbQuery[K], +): Result[FsQueryHandle[K, V], ref CatchableError] = + + let key = query.key + without path =? self.findPath(key), error: + return failure error + + let basePath = + # it there is a file in the directory + # with the same name then list the contents + # of the directory, otherwise recurse + # into subdirectories + if path.fileExists: + path.parentDir + else: + path.changeFileExt("") + + let env = FsQueryEnv[K,V](self: self, basePath: DataBuffer.new(basePath)) + success FsQueryHandle[K, V](query: query, env: env) + +proc close*[K,V](handle: var FsQueryHandle[K,V]) = + if not handle.closed: + handle.closed = true + +iterator queryIter*[K, V]( + handle: var FsQueryHandle[K, V] +): ?!DbQueryResponse[K, V] = + let root = $(handle.env.self.root) + let basePath = $(handle.env.basePath) + + for path in basePath.dirIter(): + if handle.cancel: + break + + var + basePath = $handle.env.basePath + keyPath = basePath + + keyPath.removePrefix(root) + keyPath = keyPath / path.changeFileExt("") + keyPath = keyPath.replace("\\", "/") + + let + flres = (basePath / path).absolutePath().catch + if flres.isErr(): + yield DbQueryResponse[K,V].failure flres.error() + continue + + let + key = K.toKey($Key.init(keyPath).expect("valid key")) + data = + if handle.query.value: + let res = readFile[V](handle.env.self, flres.get) + if res.isErr(): + yield DbQueryResponse[K,V].failure res.error() + continue + res.get() + else: + V.new() + + yield success (key.some, data) + handle.close() + +proc newFSDatastore*[K,V](root: string, + depth = 2, + caseSensitive = true, + ignoreProtected = false + ): ?!FSDatastore[K,V] = + + let root = ? ( + block: + if root.isAbsolute: root + else: getCurrentDir() / root).catch + + if not dirExists(root): + return failure "directory does not exist: " & root + + success FSDatastore[K,V]( + root: DataBuffer.new root, + ignoreProtected: ignoreProtected, + depth: depth) diff --git a/datastore/sql/sqliteds.nim b/datastore/threads/sqlbackend.nim similarity index 99% rename from datastore/sql/sqliteds.nim rename to datastore/threads/sqlbackend.nim index 79d4a31c..fdbce970 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/threads/sqlbackend.nim @@ -7,8 +7,8 @@ import pkg/sqlite3_abi from pkg/stew/results as stewResults import isErr import pkg/upraises -import ../backend -import ./sqlitedsdb +import ./backend +import ../sql/sqlitedsdb export backend, sqlitedsdb diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index e1a75718..7c7dc036 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -22,9 +22,9 @@ import pkg/threading/smartptrs import ../key import ../query import ../datastore -import ../backend -import ../fsds -import ../sql/sqliteds +import ./backend +import ./fsbackend +import ./sqlbackend import ./asyncsemaphore import ./databuffer @@ -140,7 +140,7 @@ proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = executeTask(ctx): has(ds, key) -method has*[BT](self: ThreadDatastore[BT], +proc has*[BT](self: ThreadDatastore[BT], key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() let signal = acquireSignal().get() @@ -159,7 +159,7 @@ proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; executeTask(ctx): delete(ds, key) -method delete*[BT](self: ThreadDatastore[BT], +proc delete*[BT](self: ThreadDatastore[BT], key: Key): Future[?!void] {.async.} = ## delete key await self.semaphore.acquire() @@ -174,7 +174,7 @@ method delete*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes() -method delete*[BT](self: ThreadDatastore[BT], +proc delete*[BT](self: ThreadDatastore[BT], keys: seq[Key]): Future[?!void] {.async.} = ## delete batch for key in keys: @@ -190,7 +190,7 @@ proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; executeTask(ctx): put(ds, key, data) -method put*[BT](self: ThreadDatastore[BT], +proc put*[BT](self: ThreadDatastore[BT], key: Key, data: seq[byte]): Future[?!void] {.async.} = ## put key with data @@ -206,8 +206,8 @@ method put*[BT](self: ThreadDatastore[BT], self.tp.spawn putTask(ctx, ds, key, data) return ctx[].res.toRes() - -method put*[DB]( + +proc put*[DB]( self: ThreadDatastore[DB], batch: seq[BatchEntry]): Future[?!void] {.async.} = ## put batch data @@ -225,7 +225,7 @@ proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; let res = get(ds, key) res -method get*[BT](self: ThreadDatastore[BT], +proc get*[BT](self: ThreadDatastore[BT], key: Key, ): Future[?!seq[byte]] {.async.} = await self.semaphore.acquire() @@ -240,7 +240,7 @@ method get*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes(v => v.toSeq()) -method close*[BT](self: ThreadDatastore[BT]): Future[?!void] {.async.} = +proc close*[BT](self: ThreadDatastore[BT]): Future[?!void] {.async.} = await self.semaphore.closeAll() self.backend.close() @@ -291,7 +291,7 @@ proc queryTask[DB]( # set final result (?!QResult).ok((KeyId.none, DataBuffer())) -method query*[BT](self: ThreadDatastore[BT], +proc query*[BT](self: ThreadDatastore[BT], q: Query ): Future[?!QueryIter] {.async.} = ## performs async query diff --git a/datastore/threads/threadresult.nim b/datastore/threads/threadresult.nim index 3508e9ee..6a6cb0ae 100644 --- a/datastore/threads/threadresult.nim +++ b/datastore/threads/threadresult.nim @@ -9,7 +9,7 @@ import ../types import ../query import ../key -import ../backend +import ./backend import ./databuffer type diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index e52d90b1..290f4fa2 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -8,7 +8,7 @@ import pkg/chronos import pkg/stew/results import pkg/stew/byteutils -import pkg/datastore/sql/sqliteds +import pkg/datastore/threads/sqlbackend import pkg/datastore/key import ../backendCommonTests diff --git a/tests/datastore/sql/testsqlitedsdb.nim b/tests/datastore/sql/testsqlitedsdb.nim index 03fd98fe..2438d366 100644 --- a/tests/datastore/sql/testsqlitedsdb.nim +++ b/tests/datastore/sql/testsqlitedsdb.nim @@ -7,7 +7,7 @@ import pkg/stew/byteutils import pkg/sqlite3_abi import pkg/datastore/key import pkg/datastore/sql/sqlitedsdb -import pkg/datastore/sql/sqliteds +import pkg/datastore/threads/sqlbackend suite "Test Open SQLite Datastore DB": let diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index 1d954140..f55348d5 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -10,7 +10,7 @@ import pkg/stew/byteutils import pkg/datastore/fsds import pkg/datastore/key -import pkg/datastore/backend +import pkg/datastore/threads/backend import ./backendCommonTests diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index db20c819..d6a2b83c 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -16,9 +16,9 @@ import pkg/chronicles import pkg/threading/smartptrs import pkg/threading/atomics -import pkg/datastore/fsds -import pkg/datastore/sql/sqliteds -import pkg/datastore/threads/threadproxyds {.all.} +import pkg/datastore/threads/fsbackend +import pkg/datastore/threads/sqlbackend +import pkg/datastore/threads/threadproxyds import ./dscommontests import ./querycommontests From 01ccbb579886fd5aac34dd09ba8a2a8174823423 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:36:05 -0700 Subject: [PATCH 419/445] reorg --- datastore/fsds.nim | 80 +++++++++------------------ datastore/threads/threadproxyds.nim | 16 +++--- tests/datastore/testthreadproxyds.nim | 6 +- 3 files changed, 37 insertions(+), 65 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 178de6e9..ac9817a7 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -6,94 +6,68 @@ import pkg/questionable import pkg/questionable/results from pkg/stew/results as stewResults import get, isErr import pkg/upraises +import pkg/chronos +import pkg/taskpools -import ./threads/backend +import ./threads/sqlbackend +import ./threads/threadproxyds import ./datastore -export datastore +export datastore, Taskpool push: {.upraises: [].} type SQLiteDatastore* = ref object of Datastore - db: SQLiteBackend[KeyId, DataBuffer] + db: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] proc path*(self: SQLiteDatastore): string = - self.db.path() + self.db.backend.path() proc readOnly*(self: SQLiteDatastore): bool = - self.db.readOnly() + self.db.backend.readOnly() method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = - return self.db.has(KeyId.new key.id()) + await self.db.has(key) method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = - return self.db.delete(KeyId.new key.id()) + await self.db.delete(key) method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = - let dkeys = keys.mapIt(KeyId.new it.id()) - return self.db.delete(dkeys) + await self.db.delete(keys) method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: - d.toSeq() + await self.db.get(key) method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = - self.db.put(KeyId.new key.id(), DataBuffer.new data) + await self.db.put(key, data) method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = - var dbatch: seq[tuple[key: KeyId, data: DataBuffer]] - for entry in batch: - dbatch.add((KeyId.new entry.key.id(), DataBuffer.new entry.data)) - self.db.put(dbatch) + await self.db.put(batch) method close*(self: SQLiteDatastore): Future[?!void] {.async.} = - self.db.close() - -method queryIter*(self: SQLiteDatastore, - query: Query - ): ?!(iterator(): ?!QueryResponse) = - - let dbquery = dbQuery( - key= KeyId.new query.key.id(), - value= query.value, - limit= query.limit, - offset= query.offset, - sort= query.sort, - ) - var qhandle = ? self.db.query(dbquery) - - let iter = iterator(): ?!QueryResponse = - for resp in qhandle.queryIter(): - without qres =? resp, err: - yield QueryResponse.failure err - let k = qres.key.map() do(k: KeyId) -> Key: - Key.init($k).expect("valid key") - let v: seq[byte] = qres.data.toSeq() - yield success (k, v) - - success iter + await self.db.close() -proc new*( - T: type SQLiteDatastore, - path: string, - readOnly = false): ?!SQLiteDatastore = - - success SQLiteDatastore( - db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) +method query*(self: SQLiteDatastore, + q: Query): Future[?!QueryIter] {.async.} = + await self.db.query(q) proc new*( T: type SQLiteDatastore, - db: SQLiteBackend[KeyId, DataBuffer]): ?!T = - - success T( - db: db, - readOnly: db.readOnly) + path: string, + readOnly = false, + tp: Taskpool, +): ?!SQLiteDatastore = + + let + backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) + db = ? ThreadDatastore.new(backend, tp = tp) + success SQLiteDatastore(db: db) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 7c7dc036..41e5b883 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -16,12 +16,12 @@ import pkg/chronos/threadsync import pkg/questionable import pkg/questionable/results import pkg/taskpools +import std/isolation import pkg/chronicles import pkg/threading/smartptrs import ../key import ../query -import ../datastore import ./backend import ./fsbackend import ./sqlbackend @@ -30,7 +30,7 @@ import ./asyncsemaphore import ./databuffer import ./threadresult -export threadresult +export threadresult, smartptrs, isolation, chronicles logScope: topics = "datastore threadproxyds" @@ -48,10 +48,10 @@ type ## Task context object. ## This is a SharedPtr to make the query iter simpler - ThreadDatastore*[BT] = ref object of Datastore - tp: Taskpool - backend: BT - semaphore: AsyncSemaphore # semaphore is used for backpressure \ + ThreadDatastore*[BT] = object + tp*: Taskpool + backend*: BT + semaphore*: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors proc newTaskCtx*[T](tp: typedesc[T], @@ -207,9 +207,9 @@ proc put*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes() -proc put*[DB]( +proc put*[E, DB]( self: ThreadDatastore[DB], - batch: seq[BatchEntry]): Future[?!void] {.async.} = + batch: seq[E]): Future[?!void] {.async.} = ## put batch data for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index d6a2b83c..e3217a43 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -37,15 +37,13 @@ for i in 1..N: suite "Test Basic ThreadDatastore with SQLite " & $i: var - sqlStore: SQLiteBackend[KeyId, DataBuffer] - ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] + ds: SQLiteDatastore key = Key.init("/a/b").tryGet() bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + ds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() teardown: GC_fullCollect() From 82bf1f1a3c86a61f88fd60e5218efd44e69f10fa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:37:44 -0700 Subject: [PATCH 420/445] reorg --- datastore/fsds.nim | 28 +++++++++++++-------------- tests/datastore/testthreadproxyds.nim | 11 +++-------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index ac9817a7..8cd331de 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -19,55 +19,55 @@ push: {.upraises: [].} type - SQLiteDatastore* = ref object of Datastore + FSDatastore* = ref object of Datastore db: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] -proc path*(self: SQLiteDatastore): string = +proc path*(self: FSDatastore): string = self.db.backend.path() -proc readOnly*(self: SQLiteDatastore): bool = +proc readOnly*(self: FSDatastore): bool = self.db.backend.readOnly() -method has*(self: SQLiteDatastore, +method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = await self.db.has(key) -method delete*(self: SQLiteDatastore, +method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = await self.db.delete(key) -method delete*(self: SQLiteDatastore, +method delete*(self: FSDatastore, keys: seq[Key]): Future[?!void] {.async.} = await self.db.delete(keys) -method get*(self: SQLiteDatastore, +method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = await self.db.get(key) -method put*(self: SQLiteDatastore, +method put*(self: FSDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = await self.db.put(key, data) -method put*(self: SQLiteDatastore, +method put*(self: FSDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = await self.db.put(batch) -method close*(self: SQLiteDatastore): Future[?!void] {.async.} = +method close*(self: FSDatastore): Future[?!void] {.async.} = await self.db.close() -method query*(self: SQLiteDatastore, +method query*(self: FSDatastore, q: Query): Future[?!QueryIter] {.async.} = await self.db.query(q) proc new*( - T: type SQLiteDatastore, + T: type FSDatastore, path: string, readOnly = false, tp: Taskpool, -): ?!SQLiteDatastore = +): ?!FSDatastore = let backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) db = ? ThreadDatastore.new(backend, tp = tp) - success SQLiteDatastore(db: db) + success FSDatastore(db: db) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index e3217a43..dd883952 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -60,14 +60,10 @@ for i in 1..N: suite "Test Query ThreadDatastore with SQLite " & $i: var - sqlStore: SQLiteBackend[KeyId, DataBuffer] - # taskPool: Taskpool - ds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] + ds: SQLiteDatastore setup: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - # taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + ds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() teardown: GC_fullCollect() @@ -88,8 +84,7 @@ suite "Test Basic ThreadDatastore with fsds": otherBytes = "some other bytes".toBytes var - fsStore: FSDatastore[KeyId, DataBuffer] - ds: ThreadDatastore[FSDatastore[KeyId, DataBuffer]] + ds: SQLiteDatastore setupAll: removeDir(basePathAbs) From 3e477f3461235ea63f4d04eaa971feb4f620128e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:40:19 -0700 Subject: [PATCH 421/445] reorg --- datastore/fsds.nim | 5 ++--- datastore/sql.nim | 28 ++++++++------------------- datastore/threads/fsbackend.nim | 34 ++++++++++++++++----------------- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 8cd331de..a3e1638a 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -9,7 +9,7 @@ import pkg/upraises import pkg/chronos import pkg/taskpools -import ./threads/sqlbackend +import ./threads/fsbackend import ./threads/threadproxyds import ./datastore @@ -17,10 +17,9 @@ export datastore, Taskpool push: {.upraises: [].} - type FSDatastore* = ref object of Datastore - db: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] + db: ThreadDatastore[FSBackend[KeyId, DataBuffer]] proc path*(self: FSDatastore): string = self.db.backend.path() diff --git a/datastore/sql.nim b/datastore/sql.nim index 712bd9a6..4611d8e4 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -13,7 +13,14 @@ import ../datastore import ./threads/backend import ./threads/sqlbackend -export datastore, sqlbackend +import pkg/chronos +import pkg/taskpools + +import ./threads/sqlbackend +import ./threads/threadproxyds +import ./datastore + +export datastore, Taskpool push: {.upraises: [].} @@ -64,25 +71,6 @@ method queryIter*(self: SQLiteDatastore, query: Query ): ?!(iterator(): ?!QueryResponse) = - let dbquery = dbQuery( - key= KeyId.new query.key.id(), - value= query.value, - limit= query.limit, - offset= query.offset, - sort= query.sort, - ) - var qhandle = ? self.db.query(dbquery) - - let iter = iterator(): ?!QueryResponse = - for resp in qhandle.queryIter(): - without qres =? resp, err: - yield QueryResponse.failure err - let k = qres.key.map() do(k: KeyId) -> Key: - Key.init($k).expect("valid key") - let v: seq[byte] = qres.data.toSeq() - yield success (k, v) - - success iter proc new*( T: type SQLiteDatastore, diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index e3a6464a..6cff1784 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -15,7 +15,7 @@ export datastore push: {.upraises: [].} type - FSDatastore*[K, V] = object + FSBackend*[K, V] = object root*: DataBuffer ignoreProtected: bool depth: int @@ -23,10 +23,10 @@ type proc isRootSubdir*(root, path: string): bool = path.startsWith(root) -proc validDepth*(self: FSDatastore, key: Key): bool = +proc validDepth*(self: FSBackend, key: Key): bool = key.len <= self.depth -proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = +proc findPath*[K,V](self: FSBackend[K,V], key: K): ?!string = ## Return filename corresponding to the key ## or failure if the key doesn't correspond to a valid filename ## @@ -65,15 +65,15 @@ proc findPath*[K,V](self: FSDatastore[K,V], key: K): ?!string = return success fullname -proc has*[K,V](self: FSDatastore[K,V], key: K): ?!bool = +proc has*[K,V](self: FSBackend[K,V], key: K): ?!bool = without path =? self.findPath(key), error: return failure error success path.fileExists() -proc contains*[K](self: FSDatastore, key: K): bool = +proc contains*[K](self: FSBackend, key: K): bool = return self.has(key).get() -proc delete*[K,V](self: FSDatastore[K,V], key: K): ?!void = +proc delete*[K,V](self: FSBackend[K,V], key: K): ?!void = without path =? self.findPath(key), error: return failure error @@ -87,14 +87,14 @@ proc delete*[K,V](self: FSDatastore[K,V], key: K): ?!void = return success() -proc delete*[K,V](self: FSDatastore[K,V], keys: openArray[K]): ?!void = +proc delete*[K,V](self: FSBackend[K,V], keys: openArray[K]): ?!void = for key in keys: if err =? self.delete(key).errorOption: return failure err return success() -proc readFile[V](self: FSDatastore, path: string): ?!V = +proc readFile[V](self: FSBackend, path: string): ?!V = var file: File @@ -130,7 +130,7 @@ proc readFile[V](self: FSDatastore, path: string): ?!V = except CatchableError as e: return failure e -proc get*[K,V](self: FSDatastore[K,V], key: K): ?!V = +proc get*[K,V](self: FSBackend[K,V], key: K): ?!V = without path =? self.findPath(key), error: return failure error @@ -140,7 +140,7 @@ proc get*[K,V](self: FSDatastore[K,V], key: K): ?!V = return readFile[V](self, path) -proc put*[K,V](self: FSDatastore[K,V], +proc put*[K,V](self: FSBackend[K,V], key: K, data: V ): ?!void = @@ -158,7 +158,7 @@ proc put*[K,V](self: FSDatastore[K,V], return success() proc put*[K,V]( - self: FSDatastore, + self: FSBackend, batch: seq[DbBatchEntry[K, V]]): ?!void = for entry in batch: @@ -174,7 +174,7 @@ iterator dirIter(path: string): string {.gcsafe.} = except CatchableError as exc: raise newException(Defect, exc.msg) -proc close*[K,V](self: FSDatastore[K,V]): ?!void = +proc close*[K,V](self: FSBackend[K,V]): ?!void = return success() type @@ -185,11 +185,11 @@ type env*: FsQueryEnv[K,V] FsQueryEnv*[K,V] = object - self: FSDatastore[K,V] + self: FSBackend[K,V] basePath: DataBuffer proc query*[K,V]( - self: FSDatastore[K,V], + self: FSBackend[K,V], query: DbQuery[K], ): Result[FsQueryHandle[K, V], ref CatchableError] = @@ -253,11 +253,11 @@ iterator queryIter*[K, V]( yield success (key.some, data) handle.close() -proc newFSDatastore*[K,V](root: string, +proc newFSBackend*[K,V](root: string, depth = 2, caseSensitive = true, ignoreProtected = false - ): ?!FSDatastore[K,V] = + ): ?!FSBackend[K,V] = let root = ? ( block: @@ -267,7 +267,7 @@ proc newFSDatastore*[K,V](root: string, if not dirExists(root): return failure "directory does not exist: " & root - success FSDatastore[K,V]( + success FSBackend[K,V]( root: DataBuffer.new root, ignoreProtected: ignoreProtected, depth: depth) From f82ea14464f51a89ab629c7d36d65e37df742fff Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:43:29 -0700 Subject: [PATCH 422/445] reorg --- datastore/fsds.nim | 6 ----- datastore/sql.nim | 40 ++++++++++++--------------------- datastore/threads/fsbackend.nim | 2 +- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index a3e1638a..8f9c781c 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -21,12 +21,6 @@ type FSDatastore* = ref object of Datastore db: ThreadDatastore[FSBackend[KeyId, DataBuffer]] -proc path*(self: FSDatastore): string = - self.db.backend.path() - -proc readOnly*(self: FSDatastore): bool = - self.db.backend.readOnly() - method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = await self.db.has(key) diff --git a/datastore/sql.nim b/datastore/sql.nim index 4611d8e4..3650323d 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -1,18 +1,12 @@ -import std/times + +import std/os import std/options +import std/strutils -import pkg/chronos import pkg/questionable import pkg/questionable/results -import pkg/sqlite3_abi -from pkg/stew/results as stewResults import isErr +from pkg/stew/results as stewResults import get, isErr import pkg/upraises - -import std/sequtils -import ../datastore -import ./threads/backend -import ./threads/sqlbackend - import pkg/chronos import pkg/taskpools @@ -36,41 +30,35 @@ proc readOnly*(self: SQLiteDatastore): bool = method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = - return self.db.has(KeyId.new key.id()) + await self.db.has(key) method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = - return self.db.delete(KeyId.new key.id()) + await self.db.delete(key) method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = - let dkeys = keys.mapIt(KeyId.new it.id()) - return self.db.delete(dkeys) + await self.db.delete(keys) method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = - self.db.get(KeyId.new key.id()).map() do(d: DataBuffer) -> seq[byte]: - d.toSeq() + await self.db.get(key) method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = - self.db.put(KeyId.new key.id(), DataBuffer.new data) + await self.db.put(key, data) method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = - var dbatch: seq[tuple[key: KeyId, data: DataBuffer]] - for entry in batch: - dbatch.add((KeyId.new entry.key.id(), DataBuffer.new entry.data)) - self.db.put(dbatch) + await self.db.put(batch) method close*(self: SQLiteDatastore): Future[?!void] {.async.} = - self.db.close() - -method queryIter*(self: SQLiteDatastore, - query: Query - ): ?!(iterator(): ?!QueryResponse) = + await self.db.close() +method query*(self: SQLiteDatastore, + q: Query): Future[?!QueryIter] {.async.} = + await self.db.query(q) proc new*( T: type SQLiteDatastore, diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index 6cff1784..1ad607b6 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -10,7 +10,7 @@ import pkg/upraises import ./backend import ./datastore -export datastore +export datastore, backend push: {.upraises: [].} From 96c54f6412dccc9ff81907b337ef0c2967c5956a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:47:13 -0700 Subject: [PATCH 423/445] reog --- datastore/sql.nim | 6 +++--- datastore/threads/threadproxyds.nim | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 3650323d..16ed24ad 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -20,13 +20,13 @@ push: {.upraises: [].} type SQLiteDatastore* = ref object of Datastore - db: SQLiteBackend[KeyId, DataBuffer] + db: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] proc path*(self: SQLiteDatastore): string = - self.db.path() + self.db.backend.path() proc readOnly*(self: SQLiteDatastore): bool = - self.db.readOnly() + self.db.backend.readOnly() method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 41e5b883..2181b6ae 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -23,8 +23,8 @@ import pkg/threading/smartptrs import ../key import ../query import ./backend -import ./fsbackend -import ./sqlbackend +# import ./fsbackend +# import ./sqlbackend import ./asyncsemaphore import ./databuffer @@ -137,6 +137,7 @@ template dispatchTask*[BT](self: ThreadDatastore[BT], proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = ## run backend command + mixin has executeTask(ctx): has(ds, key) @@ -156,6 +157,7 @@ proc has*[BT](self: ThreadDatastore[BT], proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId) {.gcsafe.} = ## run backend command + mixin delete executeTask(ctx): delete(ds, key) @@ -187,6 +189,7 @@ proc delete*[BT](self: ThreadDatastore[BT], proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; key: KeyId, data: DataBuffer) {.gcsafe, nimcall.} = + mixin put executeTask(ctx): put(ds, key, data) @@ -221,6 +224,7 @@ proc put*[E, DB]( proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; key: KeyId) {.gcsafe, nimcall.} = ## run backend command + mixin get executeTask(ctx): let res = get(ds, key) res From 47391672130eb65fb8627328626450468d1c6992 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 17:51:25 -0700 Subject: [PATCH 424/445] reorg --- datastore.nim | 4 +- datastore/sql.nim | 2 +- tests/datastore/sql/testsqlite.nim | 90 ------------------ tests/datastore/testsql.nim | 92 ++++++++++++++++++- .../{ => threads}/backendCommonTests.nim | 0 .../testsqlbackend.nim} | 3 +- 6 files changed, 93 insertions(+), 98 deletions(-) delete mode 100644 tests/datastore/sql/testsqlite.nim rename tests/datastore/{ => threads}/backendCommonTests.nim (100%) rename tests/datastore/{sql/testsqliteds.nim => threads/testsqlbackend.nim} (98%) diff --git a/datastore.nim b/datastore.nim index 5a971a5c..47c80bf3 100644 --- a/datastore.nim +++ b/datastore.nim @@ -1,7 +1,7 @@ import ./datastore/datastore import ./datastore/fsds -# import ./datastore/sql +import ./datastore/sql import ./datastore/mountedds import ./datastore/tieredds -export datastore, fsds, mountedds, tieredds +export datastore, fsds, sql, mountedds, tieredds diff --git a/datastore/sql.nim b/datastore/sql.nim index 16ed24ad..db6baed5 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -14,7 +14,7 @@ import ./threads/sqlbackend import ./threads/threadproxyds import ./datastore -export datastore, Taskpool +export datastore, keys, query, Taskpool push: {.upraises: [].} diff --git a/tests/datastore/sql/testsqlite.nim b/tests/datastore/sql/testsqlite.nim deleted file mode 100644 index b63bac7c..00000000 --- a/tests/datastore/sql/testsqlite.nim +++ /dev/null @@ -1,90 +0,0 @@ -import std/options -import std/os -import std/sequtils -from std/algorithm import sort, reversed - -import pkg/asynctest -import pkg/chronos -import pkg/stew/results -import pkg/stew/byteutils - -import pkg/datastore/sql - -import ../dscommontests -import ../querycommontests - -suite "Test Basic SQLiteDatastore": - let - ds = SQLiteDatastore.new(Memory).tryGet() - key = Key.init("a:b/c/d:e").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - teardownAll: - (await ds.close()).tryGet() - - basicStoreTests(ds, key, bytes, otherBytes) - - -suite "Test Read Only SQLiteDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - filename = "test_store" & DbExt - dbPathAbs = basePathAbs / filename - key = Key.init("a:b/c/d:e").tryGet() - bytes = "some bytes".toBytes - - var - dsDb: SQLiteDatastore - readOnlyDb: SQLiteDatastore - - setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - dsDb = SQLiteDatastore.new(path = dbPathAbs).tryGet() - readOnlyDb = SQLiteDatastore.new(path = dbPathAbs, readOnly = true).tryGet() - - teardownAll: - (await dsDb.close()).tryGet() - (await readOnlyDb.close()).tryGet() - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - test "put": - check: - (await readOnlyDb.put(key, bytes)).isErr - - (await dsDb.put(key, bytes)).tryGet() - - test "get": - check: - (await readOnlyDb.get(key)).tryGet() == bytes - (await dsDb.get(key)).tryGet() == bytes - - test "delete": - check: - (await readOnlyDb.delete(key)).isErr - - (await dsDb.delete(key)).tryGet() - - test "contains": - check: - not (await readOnlyDb.has(key)).tryGet() - not (await dsDb.has(key)).tryGet() - -# suite "Test Query": -# var -# ds: SQLiteDatastore - -# setup: -# ds = SQLiteDatastore.new(Memory).tryGet() - -# teardown: -# (await ds.close()).tryGet - -# queryTests(ds) diff --git a/tests/datastore/testsql.nim b/tests/datastore/testsql.nim index dcf6259a..a8c8ef8f 100644 --- a/tests/datastore/testsql.nim +++ b/tests/datastore/testsql.nim @@ -1,4 +1,90 @@ -import ./sql/testsqlitedsdb -import ./sql/testsqliteds +import std/options +import std/os +import std/sequtils +from std/algorithm import sort, reversed -{.warning[UnusedImport]: off.} +import pkg/asynctest +import pkg/chronos +import pkg/stew/results +import pkg/stew/byteutils + +import pkg/datastore/sql + +import ./dscommontests +import ./querycommontests + +suite "Test Basic SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + key = Key.init("a:b/c/d:e").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + teardownAll: + (await ds.close()).tryGet() + + basicStoreTests(ds, key, bytes, otherBytes) + + +suite "Test Read Only SQLiteDatastore": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + filename = "test_store" & DbExt + dbPathAbs = basePathAbs / filename + key = Key.init("a:b/c/d:e").tryGet() + bytes = "some bytes".toBytes + + var + dsDb: SQLiteDatastore + readOnlyDb: SQLiteDatastore + + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + dsDb = SQLiteDatastore.new(path = dbPathAbs).tryGet() + readOnlyDb = SQLiteDatastore.new(path = dbPathAbs, readOnly = true).tryGet() + + teardownAll: + (await dsDb.close()).tryGet() + (await readOnlyDb.close()).tryGet() + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + test "put": + check: + (await readOnlyDb.put(key, bytes)).isErr + + (await dsDb.put(key, bytes)).tryGet() + + test "get": + check: + (await readOnlyDb.get(key)).tryGet() == bytes + (await dsDb.get(key)).tryGet() == bytes + + test "delete": + check: + (await readOnlyDb.delete(key)).isErr + + (await dsDb.delete(key)).tryGet() + + test "contains": + check: + not (await readOnlyDb.has(key)).tryGet() + not (await dsDb.has(key)).tryGet() + +# suite "Test Query": +# var +# ds: SQLiteDatastore + +# setup: +# ds = SQLiteDatastore.new(Memory).tryGet() + +# teardown: +# (await ds.close()).tryGet + +# queryTests(ds) diff --git a/tests/datastore/backendCommonTests.nim b/tests/datastore/threads/backendCommonTests.nim similarity index 100% rename from tests/datastore/backendCommonTests.nim rename to tests/datastore/threads/backendCommonTests.nim diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/threads/testsqlbackend.nim similarity index 98% rename from tests/datastore/sql/testsqliteds.nim rename to tests/datastore/threads/testsqlbackend.nim index 290f4fa2..03a20cff 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/threads/testsqlbackend.nim @@ -11,8 +11,7 @@ import pkg/stew/byteutils import pkg/datastore/threads/sqlbackend import pkg/datastore/key -import ../backendCommonTests - +import ./backendCommonTests suite "Test Basic SQLiteDatastore": let From 8ce566a99b305024d0d5248cdb5d57bfddee0f20 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:02:21 -0700 Subject: [PATCH 425/445] reorg --- datastore/fsds.nim | 9 +- datastore/sql.nim | 16 +- tests/datastore/testthreadproxyds.nim | 242 -------------------------- 3 files changed, 12 insertions(+), 255 deletions(-) delete mode 100644 tests/datastore/testthreadproxyds.nim diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 8f9c781c..9ef5bce0 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -55,12 +55,15 @@ method query*(self: FSDatastore, proc new*( T: type FSDatastore, - path: string, - readOnly = false, + root: string, tp: Taskpool, + depth = 2, + caseSensitive = true, + ignoreProtected = false ): ?!FSDatastore = let - backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) + backend = ? newFSBackend[KeyId, DataBuffer]( + root=root, depth=depth, caseSensitive=caseSensitive, ignoreProtected=ignoreProtected) db = ? ThreadDatastore.new(backend, tp = tp) success FSDatastore(db: db) diff --git a/datastore/sql.nim b/datastore/sql.nim index db6baed5..66e9ce58 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -14,7 +14,7 @@ import ./threads/sqlbackend import ./threads/threadproxyds import ./datastore -export datastore, keys, query, Taskpool +export datastore, keys, query, Taskpool, Memory push: {.upraises: [].} @@ -63,15 +63,11 @@ method query*(self: SQLiteDatastore, proc new*( T: type SQLiteDatastore, path: string, + tp: Taskpool, readOnly = false): ?!SQLiteDatastore = - success SQLiteDatastore( - db: ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly)) + let + backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) + db = ? ThreadDatastore.new(backend, tp = tp) + success SQLiteDatastore(db: db) -proc new*( - T: type SQLiteDatastore, - db: SQLiteBackend[KeyId, DataBuffer]): ?!T = - - success T( - db: db, - readOnly: db.readOnly) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim deleted file mode 100644 index dd883952..00000000 --- a/tests/datastore/testthreadproxyds.nim +++ /dev/null @@ -1,242 +0,0 @@ -import std/options -import std/sequtils -import std/os -import std/cpuinfo -import std/algorithm -import std/importutils - -import pkg/asynctest -import pkg/chronos -import pkg/chronos/threadsync -import pkg/stew/results -import pkg/stew/byteutils -import pkg/taskpools -import pkg/questionable/results -import pkg/chronicles -import pkg/threading/smartptrs -import pkg/threading/atomics - -import pkg/datastore/threads/fsbackend -import pkg/datastore/threads/sqlbackend -import pkg/datastore/threads/threadproxyds - -import ./dscommontests -import ./querycommontests - -const - NumThreads = 20 # IO threads aren't attached to CPU count - ThreadTestLoops {.intdefine.} = 1 - N = ThreadTestLoops - ThreadTestInnerLoops {.intdefine.} = 1 - M = ThreadTestInnerLoops - -var - taskPool: Taskpool = Taskpool.new(NumThreads) - -for i in 1..N: - suite "Test Basic ThreadDatastore with SQLite " & $i: - - var - ds: SQLiteDatastore - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setupAll: - ds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - teardownAll: - (await ds.close()).tryGet() - - for i in 1..M: - basicStoreTests(ds, key, bytes, otherBytes) - GC_fullCollect() - - -for i in 1..N: - suite "Test Query ThreadDatastore with SQLite " & $i: - - var - ds: SQLiteDatastore - - setup: - ds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - (await ds.close()).tryGet() - - for i in 1..M: - queryTests(ds, true) - GC_fullCollect() - -suite "Test Basic ThreadDatastore with fsds": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - var - ds: SQLiteDatastore - - setupAll: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() - ds = ThreadDatastore.new(fsStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - teardownAll: - (await ds.close()).tryGet() - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - basicStoreTests(ds, key, bytes, otherBytes) - -suite "Test Query ThreadDatastore with fsds": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - - var - fsStore: FSDatastore[KeyId, DataBuffer] - ds: ThreadDatastore[FSDatastore[KeyId, DataBuffer]] - - setup: - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 5).tryGet() - ds = ThreadDatastore.new(fsStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - (await ds.close()).tryGet() - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - - queryTests(ds, false) - -for i in 1..N: - suite "Test ThreadDatastore cancelations": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - sds: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] - - privateAccess(ThreadDatastore) # expose private fields - privateAccess(TaskCtx) # expose private fields - - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - sds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() # run full collect after each test - - test "Should monitor signal and cancel": - var - signal = ThreadSignalPtr.new().tryGet() - - proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = - executeTask(ctx): - (?!bool).ok(true) - - let ctx = newTaskCtx(bool, signal=signal) - ctx[].cancelled = true - dispatchTask(sds, signal): - sds.tp.spawn cancelTestTask(ctx) - - check: - ctx[].res.isErr == true - ctx[].cancelled == true - ctx[].running == false - - test "Should cancel future": - - var - signal = ThreadSignalPtr.new().tryGet() - ms {.global.}: MutexSignal - flag {.global.}: Atomic[bool] - futFreed {.global.}: Atomic[bool] - ready {.global.}: Atomic[bool] - - ms.init() - - type - FutTestObj = object - val: int - TestValue = object - ThreadTestInt = (TestValue, ) - - proc `=destroy`(obj: var TestValue) = - # echo "destroy TestObj!" - flag.store(true) - - proc `=destroy`(obj: var FutTestObj) = - # echo "destroy FutTestObj!" - futFreed.store(true) - - proc wait(flag: var Atomic[bool], name = "task") = - # echo "wait for " & name & " to be ready..." - # defer: echo "" - for i in 1..100: - # stdout.write(".") - if flag.load() == true: - return - os.sleep(10) - raise newException(Defect, "timeout") - - proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = - executeTask(ctx): - # echo "task:exec" - discard ctx[].signal.fireSync() - ready.store(true) - ms.wait() - echo "task context memory: ", ctx[] - (?!ThreadTestInt).ok(default(ThreadTestInt)) - - proc runTestTask() {.async.} = - let obj = FutTestObj(val: 42) - await sleepAsync(1.milliseconds) - try: - let ctx = newTaskCtx(ThreadTestInt, signal=signal) - dispatchTask(sds, signal): - sds.tp.spawn errorTestTask(ctx) - ready.wait() - # echo "raise error" - raise newException(ValueError, "fake error") - finally: - # echo "fut FutTestObj: ", obj - assert obj.val == 42 # need to force future to keep ref here - try: - block: - await runTestTask() - except CatchableError as exc: - # echo "caught: ", $exc - discard - finally: - # echo "finish" - check ready.load() == true - GC_fullCollect() - futFreed.wait("futFreed") - echo "future freed it's mem!" - check futFreed.load() == true - - ms.fire() - flag.wait("flag") - check flag.load() == true From 5f84ed4bda4787f7183d939389ff57759a469539 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:03:41 -0700 Subject: [PATCH 426/445] reorg --- datastore/fsds.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 9ef5bce0..44b87b0b 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -46,13 +46,13 @@ method put*(self: FSDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = await self.db.put(batch) -method close*(self: FSDatastore): Future[?!void] {.async.} = - await self.db.close() - method query*(self: FSDatastore, q: Query): Future[?!QueryIter] {.async.} = await self.db.query(q) +method close*(self: FSDatastore): Future[?!void] {.async.} = + await self.db.close() + proc new*( T: type FSDatastore, root: string, From d24d3e5f2d1c324aa285d6146520c06b05e421ae Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:13:37 -0700 Subject: [PATCH 427/445] reorg --- datastore/fsds.nim | 2 +- datastore/sql.nim | 2 +- .../{threadproxyds.nim => threadproxy.nim} | 0 tests/datastore/threads/testthreadproxyds.nim | 236 ++++++++++++++++++ 4 files changed, 238 insertions(+), 2 deletions(-) rename datastore/threads/{threadproxyds.nim => threadproxy.nim} (100%) create mode 100644 tests/datastore/threads/testthreadproxyds.nim diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 44b87b0b..333e6535 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -10,7 +10,7 @@ import pkg/chronos import pkg/taskpools import ./threads/fsbackend -import ./threads/threadproxyds +import ./threads/threadproxy import ./datastore export datastore, Taskpool diff --git a/datastore/sql.nim b/datastore/sql.nim index 66e9ce58..0a860b6f 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -11,7 +11,7 @@ import pkg/chronos import pkg/taskpools import ./threads/sqlbackend -import ./threads/threadproxyds +import ./threads/threadproxy import ./datastore export datastore, keys, query, Taskpool, Memory diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxy.nim similarity index 100% rename from datastore/threads/threadproxyds.nim rename to datastore/threads/threadproxy.nim diff --git a/tests/datastore/threads/testthreadproxyds.nim b/tests/datastore/threads/testthreadproxyds.nim new file mode 100644 index 00000000..000a3e1f --- /dev/null +++ b/tests/datastore/threads/testthreadproxyds.nim @@ -0,0 +1,236 @@ +import std/options +import std/sequtils +import std/os +import std/cpuinfo +import std/algorithm +import std/importutils + +import pkg/asynctest +import pkg/chronos +import pkg/chronos/threadsync +import pkg/stew/results +import pkg/stew/byteutils +import pkg/taskpools +import pkg/questionable/results +import pkg/threading/atomics + +import pkg/datastore/sql +import pkg/datastore/fsds +import pkg/datastore/threads/threadproxy + +import ../dscommontests +import ../querycommontests + +const + NumThreads = 20 # IO threads aren't attached to CPU count + ThreadTestLoops {.intdefine.} = 1 + N = ThreadTestLoops + ThreadTestInnerLoops {.intdefine.} = 1 + M = ThreadTestInnerLoops + +var + taskPool: Taskpool = Taskpool.new(NumThreads) + +for i in 1..N: + suite "Test Basic ThreadDatastore with SQLite " & $i: + + var + ds: SQLiteDatastore + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + setupAll: + ds = SQLiteDatastore.new(Memory, tp=taskPool).tryGet() + + teardown: + GC_fullCollect() + + teardownAll: + (await ds.close()).tryGet() + + for i in 1..M: + basicStoreTests(ds, key, bytes, otherBytes) + GC_fullCollect() + + +for i in 1..N: + suite "Test Query ThreadDatastore with SQLite " & $i: + + var + ds: SQLiteDatastore + + setup: + ds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() + + teardown: + GC_fullCollect() + + (await ds.close()).tryGet() + + for i in 1..M: + queryTests(ds, true) + GC_fullCollect() + +suite "Test Basic ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes + + var + ds: FSDatastore + + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + ds = FSDatastore.new(root=basePathAbs, tp=taskPool, depth=5).tryGet() + + teardown: + GC_fullCollect() + + teardownAll: + (await ds.close()).tryGet() + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + basicStoreTests(ds, key, bytes, otherBytes) + +suite "Test Query ThreadDatastore with fsds": + let + path = currentSourcePath() # get this file's name + basePath = "tests_data" + basePathAbs = path.parentDir / basePath + + var + ds: FSDatastore + + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + ds = FSDatastore.new(root=basePathAbs, tp = taskPool, depth=5).tryGet() + + teardown: + GC_fullCollect() + (await ds.close()).tryGet() + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + queryTests(ds, false) + +for i in 1..N: + suite "Test ThreadDatastore cancelations": + + privateAccess(SQLiteDatastore) # expose private fields + privateAccess(ThreadDatastore) # expose private fields + # privateAccess(TaskCtx) # expose private fields + + var sds: SQLiteDatastore + + setupAll: + sds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() + + teardown: + GC_fullCollect() # run full collect after each test + + test "Should monitor signal and cancel": + var + signal = ThreadSignalPtr.new().tryGet() + + proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = + executeTask(ctx): + (?!bool).ok(true) + + let ctx = newTaskCtx(bool, signal=signal) + ctx[].cancelled = true + dispatchTask(sds.db, signal): + sds.db.tp.spawn cancelTestTask(ctx) + + check: + ctx[].res.isErr == true + ctx[].cancelled == true + ctx[].running == false + + test "Should cancel future": + + var + signal = ThreadSignalPtr.new().tryGet() + ms {.global.}: MutexSignal + flag {.global.}: Atomic[bool] + futFreed {.global.}: Atomic[bool] + ready {.global.}: Atomic[bool] + + ms.init() + + type + FutTestObj = object + val: int + TestValue = object + ThreadTestInt = (TestValue, ) + + proc `=destroy`(obj: var TestValue) = + # echo "destroy TestObj!" + flag.store(true) + + proc `=destroy`(obj: var FutTestObj) = + # echo "destroy FutTestObj!" + futFreed.store(true) + + proc wait(flag: var Atomic[bool], name = "task") = + # echo "wait for " & name & " to be ready..." + # defer: echo "" + for i in 1..100: + # stdout.write(".") + if flag.load() == true: + return + os.sleep(10) + raise newException(Defect, "timeout") + + proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = + executeTask(ctx): + # echo "task:exec" + discard ctx[].signal.fireSync() + ready.store(true) + ms.wait() + echo "task context memory: ", ctx[] + (?!ThreadTestInt).ok(default(ThreadTestInt)) + + proc runTestTask() {.async.} = + let obj = FutTestObj(val: 42) + await sleepAsync(1.milliseconds) + try: + let ctx = newTaskCtx(ThreadTestInt, signal=signal) + dispatchTask(sds.db, signal): + sds.db.tp.spawn errorTestTask(ctx) + ready.wait() + # echo "raise error" + raise newException(ValueError, "fake error") + finally: + # echo "fut FutTestObj: ", obj + assert obj.val == 42 # need to force future to keep ref here + try: + block: + await runTestTask() + except CatchableError as exc: + # echo "caught: ", $exc + discard + finally: + # echo "finish" + check ready.load() == true + GC_fullCollect() + futFreed.wait("futFreed") + echo "future freed it's mem!" + check futFreed.load() == true + + ms.fire() + flag.wait("flag") + check flag.load() == true From e945e3062641bd339b5474509234d4ab640e0cae Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:19:05 -0700 Subject: [PATCH 428/445] reorg --- datastore/fsds.nim | 4 +- datastore/sql.nim | 4 +- datastore/threads/threadproxy.nim | 37 +++++++++---------- tests/datastore/threads/testthreadproxyds.nim | 4 +- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 333e6535..8e9703aa 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -19,7 +19,7 @@ push: {.upraises: [].} type FSDatastore* = ref object of Datastore - db: ThreadDatastore[FSBackend[KeyId, DataBuffer]] + db: ThreadProxy[FSBackend[KeyId, DataBuffer]] method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = @@ -65,5 +65,5 @@ proc new*( let backend = ? newFSBackend[KeyId, DataBuffer]( root=root, depth=depth, caseSensitive=caseSensitive, ignoreProtected=ignoreProtected) - db = ? ThreadDatastore.new(backend, tp = tp) + db = ? ThreadProxy.new(backend, tp = tp) success FSDatastore(db: db) diff --git a/datastore/sql.nim b/datastore/sql.nim index 0a860b6f..5946ba67 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -20,7 +20,7 @@ push: {.upraises: [].} type SQLiteDatastore* = ref object of Datastore - db: ThreadDatastore[SQLiteBackend[KeyId, DataBuffer]] + db: ThreadProxy[SQLiteBackend[KeyId, DataBuffer]] proc path*(self: SQLiteDatastore): string = self.db.backend.path() @@ -68,6 +68,6 @@ proc new*( let backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) - db = ? ThreadDatastore.new(backend, tp = tp) + db = ? ThreadProxy.new(backend, tp = tp) success SQLiteDatastore(db: db) diff --git a/datastore/threads/threadproxy.nim b/datastore/threads/threadproxy.nim index 2181b6ae..85be32e4 100644 --- a/datastore/threads/threadproxy.nim +++ b/datastore/threads/threadproxy.nim @@ -48,10 +48,10 @@ type ## Task context object. ## This is a SharedPtr to make the query iter simpler - ThreadDatastore*[BT] = object - tp*: Taskpool + ThreadProxy*[BT] = object + tp: Taskpool backend*: BT - semaphore*: AsyncSemaphore # semaphore is used for backpressure \ + semaphore: AsyncSemaphore # semaphore is used for backpressure \ # to avoid exhausting file descriptors proc newTaskCtx*[T](tp: typedesc[T], @@ -105,7 +105,7 @@ template executeTask*[T](ctx: TaskCtx[T], blk: untyped) = ctx.setDone() discard ctx[].signal.fireSync() -template dispatchTaskWrap[BT](self: ThreadDatastore[BT], +template dispatchTaskWrap[BT](self: ThreadProxy[BT], signal: ThreadSignalPtr, blk: untyped ): auto = @@ -115,7 +115,7 @@ template dispatchTaskWrap[BT](self: ThreadDatastore[BT], runTask() await wait(ctx[].signal) -template dispatchTask*[BT](self: ThreadDatastore[BT], +template dispatchTask*[BT](self: ThreadProxy[BT], signal: ThreadSignalPtr, blk: untyped ): auto = @@ -141,7 +141,7 @@ proc hasTask[T, DB](ctx: TaskCtx[T], ds: DB, key: KeyId) {.gcsafe.} = executeTask(ctx): has(ds, key) -proc has*[BT](self: ThreadDatastore[BT], +proc has*[BT](self: ThreadProxy[BT], key: Key): Future[?!bool] {.async.} = await self.semaphore.acquire() let signal = acquireSignal().get() @@ -161,7 +161,7 @@ proc deleteTask[T, DB](ctx: TaskCtx[T], ds: DB; executeTask(ctx): delete(ds, key) -proc delete*[BT](self: ThreadDatastore[BT], +proc delete*[BT](self: ThreadProxy[BT], key: Key): Future[?!void] {.async.} = ## delete key await self.semaphore.acquire() @@ -176,7 +176,7 @@ proc delete*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes() -proc delete*[BT](self: ThreadDatastore[BT], +proc delete*[BT](self: ThreadProxy[BT], keys: seq[Key]): Future[?!void] {.async.} = ## delete batch for key in keys: @@ -193,7 +193,7 @@ proc putTask[T, DB](ctx: TaskCtx[T], ds: DB; executeTask(ctx): put(ds, key, data) -proc put*[BT](self: ThreadDatastore[BT], +proc put*[BT](self: ThreadProxy[BT], key: Key, data: seq[byte]): Future[?!void] {.async.} = ## put key with data @@ -210,9 +210,8 @@ proc put*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes() -proc put*[E, DB]( - self: ThreadDatastore[DB], - batch: seq[E]): Future[?!void] {.async.} = +proc put*[E, DB](self: ThreadProxy[DB], + batch: seq[E]): Future[?!void] {.async.} = ## put batch data for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: @@ -229,7 +228,7 @@ proc getTask[DB](ctx: TaskCtx[DataBuffer], ds: DB; let res = get(ds, key) res -proc get*[BT](self: ThreadDatastore[BT], +proc get*[BT](self: ThreadProxy[BT], key: Key, ): Future[?!seq[byte]] {.async.} = await self.semaphore.acquire() @@ -244,7 +243,7 @@ proc get*[BT](self: ThreadDatastore[BT], return ctx[].res.toRes(v => v.toSeq()) -proc close*[BT](self: ThreadDatastore[BT]): Future[?!void] {.async.} = +proc close*[BT](self: ThreadProxy[BT]): Future[?!void] {.async.} = await self.semaphore.closeAll() self.backend.close() @@ -295,7 +294,7 @@ proc queryTask[DB]( # set final result (?!QResult).ok((KeyId.none, DataBuffer())) -proc query*[BT](self: ThreadDatastore[BT], +proc query*[BT](self: ThreadProxy[BT], q: Query ): Future[?!QueryIter] {.async.} = ## performs async query @@ -369,14 +368,14 @@ proc query*[BT](self: ThreadDatastore[BT], await iterDispose() raise exc -proc new*[DB](self: type ThreadDatastore, +proc new*[DB](self: type ThreadProxy, db: DB, withLocks = static false, tp: Taskpool - ): ?!ThreadDatastore[DB] = - doAssert tp.numThreads > 1, "ThreadDatastore requires at least 2 threads" + ): ?!ThreadProxy[DB] = + doAssert tp.numThreads > 1, "ThreadProxy requires at least 2 threads" - success ThreadDatastore[DB]( + success ThreadProxy[DB]( tp: tp, backend: db, semaphore: AsyncSemaphore.new(tp.numThreads - 1) diff --git a/tests/datastore/threads/testthreadproxyds.nim b/tests/datastore/threads/testthreadproxyds.nim index 000a3e1f..75e3d852 100644 --- a/tests/datastore/threads/testthreadproxyds.nim +++ b/tests/datastore/threads/testthreadproxyds.nim @@ -131,8 +131,8 @@ for i in 1..N: suite "Test ThreadDatastore cancelations": privateAccess(SQLiteDatastore) # expose private fields - privateAccess(ThreadDatastore) # expose private fields - # privateAccess(TaskCtx) # expose private fields + privateAccess(ThreadProxy) # expose private fields + privateAccess(TaskCtx) # expose private fields var sds: SQLiteDatastore From 61c14e487ca4de5acee92e0d8ee8983ee887420c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:24:31 -0700 Subject: [PATCH 429/445] reorg --- datastore/fsds.nim | 2 +- datastore/sql.nim | 5 ++--- datastore/threads/backend.nim | 3 ++- tests/datastore/testsql.nim | 1 - tests/datastore/threads/testthreadproxyds.nim | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 8e9703aa..40e7c034 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -56,7 +56,7 @@ method close*(self: FSDatastore): Future[?!void] {.async.} = proc new*( T: type FSDatastore, root: string, - tp: Taskpool, + tp: Taskpool = Taskpool.new(4), depth = 2, caseSensitive = true, ignoreProtected = false diff --git a/datastore/sql.nim b/datastore/sql.nim index 5946ba67..6f64df85 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -14,7 +14,7 @@ import ./threads/sqlbackend import ./threads/threadproxy import ./datastore -export datastore, keys, query, Taskpool, Memory +export datastore, keys, query, Taskpool, Memory, DbExt push: {.upraises: [].} @@ -63,11 +63,10 @@ method query*(self: SQLiteDatastore, proc new*( T: type SQLiteDatastore, path: string, - tp: Taskpool, + tp: Taskpool = Taskpool.new(4), readOnly = false): ?!SQLiteDatastore = let backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) db = ? ThreadProxy.new(backend, tp = tp) success SQLiteDatastore(db: db) - diff --git a/datastore/threads/backend.nim b/datastore/threads/backend.nim index f6e3c81f..2faf9b76 100644 --- a/datastore/threads/backend.nim +++ b/datastore/threads/backend.nim @@ -5,8 +5,9 @@ import pkg/questionable/results import ./databuffer import ../types +import ../key -export databuffer, types, SortOrder +export databuffer, types, key, SortOrder ## Types for datastore backends. ## diff --git a/tests/datastore/testsql.nim b/tests/datastore/testsql.nim index a8c8ef8f..d17b55f0 100644 --- a/tests/datastore/testsql.nim +++ b/tests/datastore/testsql.nim @@ -25,7 +25,6 @@ suite "Test Basic SQLiteDatastore": basicStoreTests(ds, key, bytes, otherBytes) - suite "Test Read Only SQLiteDatastore": let path = currentSourcePath() # get this file's name diff --git a/tests/datastore/threads/testthreadproxyds.nim b/tests/datastore/threads/testthreadproxyds.nim index 75e3d852..c3ef153b 100644 --- a/tests/datastore/threads/testthreadproxyds.nim +++ b/tests/datastore/threads/testthreadproxyds.nim @@ -33,7 +33,6 @@ var for i in 1..N: suite "Test Basic ThreadDatastore with SQLite " & $i: - var ds: SQLiteDatastore key = Key.init("/a/b").tryGet() From de20fa4c1e2a22ed7ebc778a5c3cabf5d6dfd33e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:32:47 -0700 Subject: [PATCH 430/445] reorg --- datastore/fsds.nim | 5 +- datastore/threads/fsbackend.nim | 5 +- tests/datastore/testfsds.nim | 159 ++++++++++---------------------- 3 files changed, 57 insertions(+), 112 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 40e7c034..7e1ba33f 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -13,7 +13,7 @@ import ./threads/fsbackend import ./threads/threadproxy import ./datastore -export datastore, Taskpool +export datastore, threadproxy, fsbackend, Taskpool push: {.upraises: [].} @@ -21,6 +21,9 @@ type FSDatastore* = ref object of Datastore db: ThreadProxy[FSBackend[KeyId, DataBuffer]] +proc validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.db.backend.depth + method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = await self.db.has(key) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index 1ad607b6..b94c9d5f 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -8,9 +8,8 @@ from pkg/stew/results as stewResults import get, isErr import pkg/upraises import ./backend -import ./datastore -export datastore, backend +export backend push: {.upraises: [].} @@ -18,7 +17,7 @@ type FSBackend*[K, V] = object root*: DataBuffer ignoreProtected: bool - depth: int + depth*: int proc isRootSubdir*(root, path: string): bool = path.startsWith(root) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index f55348d5..8f12ea81 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -3,44 +3,15 @@ import std/sequtils import std/os from std/algorithm import sort, reversed -import pkg/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils import pkg/datastore/fsds -import pkg/datastore/key -import pkg/datastore/threads/backend - -import ./backendCommonTests - -suite "Test Basic FSDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - keyFull = Key.init("/a/b").tryGet() - key = KeyId.new keyFull.id() - bytes = DataBuffer.new "some bytes" - otherBytes = DataBuffer.new "some other bytes".toBytes - - var batch: seq[tuple[key: KeyId, data: DataBuffer]] - for k in 0..<100: - let kk = Key.init($keyFull, $k).tryGet().id() - batch.add( (KeyId.new kk, DataBuffer.new @[k.byte]) ) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - var - fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() - - testBasicBackend(fsStore, key, bytes, otherBytes, batch) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) +import ./dscommontests +import ./querycommontests suite "Test Basic FSDatastore": let @@ -51,22 +22,21 @@ suite "Test Basic FSDatastore": bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - var batch: seq[tuple[key: Key, data: seq[byte]]] - for k in 0..<100: - let kk = Key.init($key, $k).tryGet() - batch.add( (kk, @[k.byte]) ) + var + fsStore: FSDatastore - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) - var - fsStore = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() - testBasicBackend(fsStore, key, bytes, otherBytes, batch) + teardownAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + basicStoreTests(fsStore, key, bytes, otherBytes) suite "Test Misc FSDatastore": let @@ -86,7 +56,7 @@ suite "Test Misc FSDatastore": test "Test validDepth()": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = "/", depth = 3).tryGet() invalid = Key.init("/a/b/c/d").tryGet() valid = Key.init("/a/b/c").tryGet() @@ -96,40 +66,40 @@ suite "Test Misc FSDatastore": test "Test invalid key (path) depth": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/b/c/d").tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr test "Test valid key (path) depth": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/b/c").tryGet() check: - (fs.put(key, bytes)).isOk - (fs.get(key)).isOk - (fs.delete(key)).isOk - (fs.has(key)).isOk + (await fs.put(key, bytes)).isOk + (await fs.get(key)).isOk + (await fs.delete(key)).isOk + (await fs.has(key)).isOk test "Test key cannot write outside of root": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/../../c").tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr test "Test key cannot convert to invalid path": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs).tryGet() + fs = FSDatastore.new(root = basePathAbs).tryGet() for c in invalidFilenameChars: if c == ':': continue @@ -139,57 +109,30 @@ suite "Test Misc FSDatastore": key = Key.init("/" & c).tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr - - -# suite "Test Query": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath - -# var -# ds: FSDatastore - -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) - -# ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - -# teardown: - -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) - -# queryTests(ds, false) - -suite "queryTests": + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr +suite "Test Query": let path = currentSourcePath() # get this file's name basePath = "tests_data" basePathAbs = path.parentDir / basePath - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) + var + ds: FSDatastore - let - fsNew = proc(): FSDatastore[KeyId, DataBuffer] = - newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() - key1 = KeyId.new "/a" - key2 = KeyId.new "/a/b" - key3 = KeyId.new "/a/b/c" - val1 = DataBuffer.new "value for 1" - val2 = DataBuffer.new "value for 2" - val3 = DataBuffer.new "value for 3" - - queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=false) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) + + ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() + + teardown: + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + queryTests(ds, false) \ No newline at end of file From 9cc20e9e405d36ade9b7ae187b56f1394960ba9b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:34:59 -0700 Subject: [PATCH 431/445] reorg --- datastore/threads/threadproxy.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/threads/threadproxy.nim b/datastore/threads/threadproxy.nim index 85be32e4..a3f9a2d8 100644 --- a/datastore/threads/threadproxy.nim +++ b/datastore/threads/threadproxy.nim @@ -33,7 +33,7 @@ import ./threadresult export threadresult, smartptrs, isolation, chronicles logScope: - topics = "datastore threadproxyds" + topics = "datastore threadproxy" type From 2a4d96e10992ebd6cdb151a206cc4917e5b8daf2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:37:05 -0700 Subject: [PATCH 432/445] reorg --- .../threads/{testthreadproxyds.nim => testthreadproxy.nim} | 0 tests/testall.nim | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename tests/datastore/threads/{testthreadproxyds.nim => testthreadproxy.nim} (100%) diff --git a/tests/datastore/threads/testthreadproxyds.nim b/tests/datastore/threads/testthreadproxy.nim similarity index 100% rename from tests/datastore/threads/testthreadproxyds.nim rename to tests/datastore/threads/testthreadproxy.nim diff --git a/tests/testall.nim b/tests/testall.nim index 47e2960a..0ac781eb 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -6,7 +6,8 @@ import ./datastore/testtieredds, ./datastore/testmountedds, ./datastore/testdatabuffer, - ./datastore/testthreadproxyds, + ./datastore/threads/testsqlbackend, + ./datastore/threads/testthreadproxy, ./datastore/testasyncsemaphore, ./datastore/testsemaphore From ebc93e89247103b93d4caafc7028a498d436bf9b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 19:00:22 -0700 Subject: [PATCH 433/445] add files lock --- datastore/threads/fsbackend.nim | 71 +++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index b94c9d5f..1de58eff 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -1,6 +1,7 @@ import std/os import std/options import std/strutils +import std/tempfiles import pkg/questionable import pkg/questionable/results @@ -13,6 +14,15 @@ export backend push: {.upraises: [].} +import std/sharedtables + +var keyTable: SharedTable[KeyId, int] + +template withReadLock(key: KeyId, blk: untyped) = + `blk` +template withWriteLock(key: KeyId, blk: untyped) = + `blk` + type FSBackend*[K, V] = object root*: DataBuffer @@ -67,7 +77,8 @@ proc findPath*[K,V](self: FSBackend[K,V], key: K): ?!string = proc has*[K,V](self: FSBackend[K,V], key: K): ?!bool = without path =? self.findPath(key), error: return failure error - success path.fileExists() + withReadLock(key): + success path.fileExists() proc contains*[K](self: FSBackend, key: K): bool = return self.has(key).get() @@ -80,7 +91,8 @@ proc delete*[K,V](self: FSBackend[K,V], key: K): ?!void = return success() try: - removeFile(path) + withWriteLock(key): + removeFile(path) except OSError as e: return failure e @@ -100,34 +112,35 @@ proc readFile[V](self: FSBackend, path: string): ?!V = defer: file.close - if not file.open(path): - return failure "unable to open file! path: " & path + withReadLock(key): + if not file.open(path): + return failure "unable to open file! path: " & path - try: - let - size = file.getFileSize().int + try: + let + size = file.getFileSize().int - when V is seq[byte]: - var bytes = newSeq[byte](size) - elif V is V: - var bytes = V.new(size=size) - else: - {.error: "unhandled result type".} - var - read = 0 + when V is seq[byte]: + var bytes = newSeq[byte](size) + elif V is V: + var bytes = V.new(size=size) + else: + {.error: "unhandled result type".} + var + read = 0 - # echo "BYTES: ", bytes.repr - while read < size: - read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) + # echo "BYTES: ", bytes.repr + while read < size: + read += file.readBytes(bytes.toOpenArray(0, size-1), read, size) - if read < size: - return failure $read & " bytes were read from " & path & - " but " & $size & " bytes were expected" + if read < size: + return failure $read & " bytes were read from " & path & + " but " & $size & " bytes were expected" - return success bytes + return success bytes - except CatchableError as e: - return failure e + except CatchableError as e: + return failure e proc get*[K,V](self: FSBackend[K,V], key: K): ?!V = without path =? self.findPath(key), error: @@ -149,8 +162,14 @@ proc put*[K,V](self: FSBackend[K,V], try: var data = data - createDir(parentDir(path)) - writeFile(path, data.toOpenArray(0, data.len()-1)) + withWriteLock(KeyId.new path): + createDir(parentDir(path)) + + let tmpPath = genTempPath("temp", "", path.splitPath.tail) + writeFile(tmpPath, data.toOpenArray(0, data.len()-1)) + + withWriteLock(key): + moveFile(tmpPath, path) except CatchableError as e: return failure e From c85cf18af7eafaf7492b9116be618ba7c3c5ea3c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 19:17:43 -0700 Subject: [PATCH 434/445] add files lock --- datastore/threads/fsbackend.nim | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index 1de58eff..57358e6f 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -16,12 +16,36 @@ push: {.upraises: [].} import std/sharedtables -var keyTable: SharedTable[KeyId, int] +type + KeyLock = tuple[locked: bool] + +var keyTable: SharedTable[KeyId, KeyLock] +keyTable.init() + +template lockKeyImpl(key: KeyId, blk: untyped) = + var hasLock = false + try: + while not hasLock: + keyTable.withKey(key) do (k: KeyId, klock: var KeyLock, exists: var bool): + if not exists or not klock.locked: + klock.locked = true + exists = true + hasLock = klock.locked + os.sleep(1) + + `blk` + finally: + if hasLock: + keyTable.withKey(key) do (k: KeyId, klock: var KeyLock, exists: var bool): + assert exists and klock.locked + klock.locked = false + exists = false template withReadLock(key: KeyId, blk: untyped) = - `blk` + lockKeyImpl(key, blk) + template withWriteLock(key: KeyId, blk: untyped) = - `blk` + lockKeyImpl(key, blk) type FSBackend*[K, V] = object From 9184077262c4e4738ce65d57566c8b9b8405fc55 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 19:25:07 -0700 Subject: [PATCH 435/445] add files lock --- datastore/threads/fsbackend.nim | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index 57358e6f..d6cb0ef4 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -22,7 +22,7 @@ type var keyTable: SharedTable[KeyId, KeyLock] keyTable.init() -template lockKeyImpl(key: KeyId, blk: untyped) = +template lockKeyImpl(key: KeyId, blk: untyped): untyped = var hasLock = false try: while not hasLock: @@ -41,10 +41,10 @@ template lockKeyImpl(key: KeyId, blk: untyped) = klock.locked = false exists = false -template withReadLock(key: KeyId, blk: untyped) = +template withReadLock(key: KeyId, blk: untyped): untyped = lockKeyImpl(key, blk) -template withWriteLock(key: KeyId, blk: untyped) = +template withWriteLock(key: KeyId, blk: untyped): untyped = lockKeyImpl(key, blk) type @@ -129,7 +129,7 @@ proc delete*[K,V](self: FSBackend[K,V], keys: openArray[K]): ?!void = return success() -proc readFile[V](self: FSBackend, path: string): ?!V = +proc readFile[K, V](self: FSBackend, key: K, path: string): ?!V = var file: File @@ -174,7 +174,7 @@ proc get*[K,V](self: FSBackend[K,V], key: K): ?!V = return failure( newException(DatastoreKeyNotFound, "Key doesn't exist")) - return readFile[V](self, path) + return readFile[K, V](self, key, path) proc put*[K,V](self: FSBackend[K,V], key: K, @@ -189,11 +189,14 @@ proc put*[K,V](self: FSBackend[K,V], withWriteLock(KeyId.new path): createDir(parentDir(path)) - let tmpPath = genTempPath("temp", "", path.splitPath.tail) + let tmpPath = genTempPath("temp", path.splitPath.tail) writeFile(tmpPath, data.toOpenArray(0, data.len()-1)) withWriteLock(key): - moveFile(tmpPath, path) + try: + moveFile(tmpPath, path) + except Exception as e: + return failure e.msg except CatchableError as e: return failure e @@ -284,7 +287,7 @@ iterator queryIter*[K, V]( key = K.toKey($Key.init(keyPath).expect("valid key")) data = if handle.query.value: - let res = readFile[V](handle.env.self, flres.get) + let res = readFile[K, V](handle.env.self, key, flres.get) if res.isErr(): yield DbQueryResponse[K,V].failure res.error() continue From 61d01f89af95b3f159374eaab2e65b8ba9e00079 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 19:28:12 -0700 Subject: [PATCH 436/445] add files lock --- datastore/threads/fsbackend.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index d6cb0ef4..9a22f25b 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -31,7 +31,6 @@ template lockKeyImpl(key: KeyId, blk: untyped): untyped = klock.locked = true exists = true hasLock = klock.locked - os.sleep(1) `blk` finally: From 7d98f5ed9ddd7858ee93b1118b368342bf612cc5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 2 Oct 2023 17:22:55 -0700 Subject: [PATCH 437/445] switch back to upstream threading lib --- datastore.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore.nimble b/datastore.nimble index a6505981..4a75697f 100644 --- a/datastore.nimble +++ b/datastore.nimble @@ -14,7 +14,7 @@ requires "nim >= 1.6.14", "stew", "unittest2", "pretty", - "https://github.com/elcritch/threading#test-smartptrsleak", + "threading#7c033b9135b4baae84e6c89af7548848bba7dc02", # or more recent "taskpools", "upraises >= 0.1.0 & < 0.2.0", "chronicles" From a626a03a8cff4080f7a71d09a5902007119cccc3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 15:56:22 -0700 Subject: [PATCH 438/445] cleanup --- datastore/datastore.nim | 2 ++ datastore/fsds.nim | 3 +- datastore/threads/threadproxy.nim | 51 +++++++++++++++---------------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index c0f1f0d1..4be25986 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -8,6 +8,8 @@ import ./types export key, query, types +const datastoreUseSync* {.booldefine.} = false + push: {.upraises: [].} type diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 7e1ba33f..7420eff0 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -67,6 +67,7 @@ proc new*( let backend = ? newFSBackend[KeyId, DataBuffer]( - root=root, depth=depth, caseSensitive=caseSensitive, ignoreProtected=ignoreProtected) + root = root, depth = depth, caseSensitive = caseSensitive, + ignoreProtected = ignoreProtected) db = ? ThreadProxy.new(backend, tp = tp) success FSDatastore(db: db) diff --git a/datastore/threads/threadproxy.nim b/datastore/threads/threadproxy.nim index a3f9a2d8..a5538949 100644 --- a/datastore/threads/threadproxy.nim +++ b/datastore/threads/threadproxy.nim @@ -40,7 +40,7 @@ type TaskCtxObj*[T: ThreadTypes] = object res*: ThreadResult[T] signal: ThreadSignalPtr - running*: bool ## used to mark when a task worker is running + running*: bool ## used to mark when a task worker is running cancelled*: bool ## used to cancel a task before it's started nextSignal: ThreadSignalPtr @@ -60,31 +60,32 @@ proc newTaskCtx*[T](tp: typedesc[T], newSharedPtr(TaskCtxObj[T](signal: signal, nextSignal: nextSignal)) proc setCancelled[T](ctx: TaskCtx[T]) = - ctx[].cancelled = true + ctx[].cancelled = true proc setRunning[T](ctx: TaskCtx[T]): bool = - if ctx[].cancelled: - return false - ctx[].running = true - return true + if ctx[].cancelled: + return false + ctx[].running = true + return true proc setDone[T](ctx: TaskCtx[T]) = - ctx[].running = false + ctx[].running = false proc acquireSignal(): ?!ThreadSignalPtr = # echo "signal:OPEN!" let signal = ThreadSignalPtr.new() if signal.isErr(): - failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & signal.error()) + failure (ref CatchableError)(msg: "failed to aquire ThreadSignalPtr: " & + signal.error()) else: success signal.get() template executeTask*[T](ctx: TaskCtx[T], blk: untyped) = ## executes a task on a thread work and handles cleanup after cancels/errors - ## + ## try: if not ctx.setRunning(): return - + ## run backend command let res = `blk` if res.isOk(): @@ -148,7 +149,7 @@ proc has*[BT](self: ThreadProxy[BT], # without signal =? acquireSignal(), err: # return failure err - let ctx = newTaskCtx(bool, signal=signal) + let ctx = newTaskCtx(bool, signal = signal) dispatchTask(self, signal): let key = KeyId.new key.id() self.tp.spawn hasTask(ctx, ds, key) @@ -169,7 +170,7 @@ proc delete*[BT](self: ThreadProxy[BT], # without signal =? acquireSignal(), err: # return failure err - let ctx = newTaskCtx(void, signal=signal) + let ctx = newTaskCtx(void, signal = signal) dispatchTask(self, signal): let key = KeyId.new key.id() self.tp.spawn deleteTask(ctx, ds, key) @@ -202,7 +203,7 @@ proc put*[BT](self: ThreadProxy[BT], # without signal =? acquireSignal(), err: # return failure err - let ctx = newTaskCtx(void, signal=signal) + let ctx = newTaskCtx(void, signal = signal) dispatchTask(self, signal): let key = KeyId.new key.id() let data = DataBuffer.new data @@ -216,7 +217,7 @@ proc put*[E, DB](self: ThreadProxy[DB], for entry in batch: if err =? (await self.put(entry.key, entry.data)).errorOption: return failure err - + return success() @@ -236,7 +237,7 @@ proc get*[BT](self: ThreadProxy[BT], # without signal =? acquireSignal(), err: # return failure err - let ctx = newTaskCtx(DataBuffer, signal=signal) + let ctx = newTaskCtx(DataBuffer, signal = signal) dispatchTask(self, signal): let key = KeyId.new key.id() self.tp.spawn getTask(ctx, ds, key) @@ -261,8 +262,6 @@ proc queryTask[DB]( # we execute this all inside `executeTask` # so we need to return a final result let handleRes = query(ds, query) - static: - echo "HANDLE_RES: ", typeof(handleRes) if handleRes.isErr(): # set error and exit executeTask, which will fire final signal (?!QResult).err(handleRes.error()) @@ -274,8 +273,6 @@ proc queryTask[DB]( raise newException(DeadThreadDefect, "queryTask timed out") var handle = handleRes.get() - static: - echo "HANDLE: ", typeof(handle) for item in handle.queryIter(): # wait for next request from async thread @@ -298,8 +295,8 @@ proc query*[BT](self: ThreadProxy[BT], q: Query ): Future[?!QueryIter] {.async.} = ## performs async query - ## keeps one thread running queryTask until finished - ## + ## keeps one thread running queryTask until finished + ## await self.semaphore.acquire() let signal = acquireSignal().get() # without signal =? acquireSignal(), err: @@ -307,7 +304,7 @@ proc query*[BT](self: ThreadProxy[BT], let nextSignal = acquireSignal().get() # without nextSignal =? acquireSignal(), err: # return failure err - let ctx = newTaskCtx(QResult, signal=signal, nextSignal=nextSignal) + let ctx = newTaskCtx(QResult, signal = signal, nextSignal = nextSignal) proc iterDispose() {.async.} = ctx.setCancelled() @@ -318,8 +315,8 @@ proc query*[BT](self: ThreadProxy[BT], try: let query = dbQuery( - key= KeyId.new q.key.id(), - value=q.value, limit=q.limit, offset=q.offset, sort=q.sort) + key = KeyId.new q.key.id(), + value = q.value, limit = q.limit, offset = q.offset, sort = q.sort) # setup initial queryTask dispatchTaskWrap(self, signal): @@ -337,9 +334,11 @@ proc query*[BT](self: ThreadProxy[BT], try: trace "About to query" if lock.locked: - return failure (ref DatastoreError)(msg: "Should always await query features") + return failure (ref DatastoreError)( + msg: "Should always await query features") if iter.finished == true: - return failure (ref QueryEndedError)(msg: "Calling next on a finished query!") + return failure (ref QueryEndedError)( + msg: "Calling next on a finished query!") await wait(ctx[].signal) if not ctx[].running: From 235bddb1c415a0281f35b8bae17360bea10de25a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:05:02 -0700 Subject: [PATCH 439/445] add compile option to use old sync setup --- datastore/datastore.nim | 2 +- datastore/fsds.nim | 333 +++++++++++++++++++++++++++++++++------- 2 files changed, 280 insertions(+), 55 deletions(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 4be25986..d570be1e 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -8,7 +8,7 @@ import ./types export key, query, types -const datastoreUseSync* {.booldefine.} = false +const datastoreUseAsync* {.booldefine.} = true push: {.upraises: [].} diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 7420eff0..9d0c50f8 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -17,57 +17,282 @@ export datastore, threadproxy, fsbackend, Taskpool push: {.upraises: [].} -type - FSDatastore* = ref object of Datastore - db: ThreadProxy[FSBackend[KeyId, DataBuffer]] - -proc validDepth*(self: FSDatastore, key: Key): bool = - key.len <= self.db.backend.depth - -method has*(self: FSDatastore, - key: Key): Future[?!bool] {.async.} = - await self.db.has(key) - -method delete*(self: FSDatastore, - key: Key): Future[?!void] {.async.} = - await self.db.delete(key) - -method delete*(self: FSDatastore, - keys: seq[Key]): Future[?!void] {.async.} = - await self.db.delete(keys) - -method get*(self: FSDatastore, - key: Key): Future[?!seq[byte]] {.async.} = - await self.db.get(key) - -method put*(self: FSDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async.} = - await self.db.put(key, data) - -method put*(self: FSDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = - await self.db.put(batch) - -method query*(self: FSDatastore, - q: Query): Future[?!QueryIter] {.async.} = - await self.db.query(q) - -method close*(self: FSDatastore): Future[?!void] {.async.} = - await self.db.close() - -proc new*( - T: type FSDatastore, - root: string, - tp: Taskpool = Taskpool.new(4), - depth = 2, - caseSensitive = true, - ignoreProtected = false -): ?!FSDatastore = - - let - backend = ? newFSBackend[KeyId, DataBuffer]( - root = root, depth = depth, caseSensitive = caseSensitive, - ignoreProtected = ignoreProtected) - db = ? ThreadProxy.new(backend, tp = tp) - success FSDatastore(db: db) +when datastoreUseAsync: + type + FSDatastore* = ref object of Datastore + db: ThreadProxy[FSBackend[KeyId, DataBuffer]] + + proc validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.db.backend.depth + + method has*(self: FSDatastore, + key: Key): Future[?!bool] {.async.} = + await self.db.has(key) + + method delete*(self: FSDatastore, + key: Key): Future[?!void] {.async.} = + await self.db.delete(key) + + method delete*(self: FSDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + await self.db.delete(keys) + + method get*(self: FSDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + await self.db.get(key) + + method put*(self: FSDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + await self.db.put(key, data) + + method put*(self: FSDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + await self.db.put(batch) + + method query*(self: FSDatastore, + q: Query): Future[?!QueryIter] {.async.} = + await self.db.query(q) + + method close*(self: FSDatastore): Future[?!void] {.async.} = + await self.db.close() + + proc new*( + T: type FSDatastore, + root: string, + tp: Taskpool = Taskpool.new(4), + depth = 2, + caseSensitive = true, + ignoreProtected = false + ): ?!FSDatastore = + + let + backend = ? newFSBackend[KeyId, DataBuffer]( + root = root, depth = depth, caseSensitive = caseSensitive, + ignoreProtected = ignoreProtected) + db = ? ThreadProxy.new(backend, tp = tp) + success FSDatastore(db: db) + +else: + type + FSDatastore* = ref object of Datastore + root*: string + ignoreProtected: bool + depth: int + + proc validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.depth + + proc isRootSubdir*(self: FSDatastore, path: string): bool = + path.startsWith(self.root) + + proc path*(self: FSDatastore, key: Key): ?!string = + ## Return filename corresponding to the key + ## or failure if the key doesn't correspond to a valid filename + ## + + if not self.validDepth(key): + return failure "Path has invalid depth!" + + var + segments: seq[string] + + for ns in key: + let basename = ns.value.extractFilename + if basename == "" or not basename.isValidFilename: + return failure "Filename contains invalid chars!" + + if ns.field == "": + segments.add(ns.value) + else: + let basename = ns.field.extractFilename + if basename == "" or not basename.isValidFilename: + return failure "Filename contains invalid chars!" + + # `:` are replaced with `/` + segments.add(ns.field / ns.value) + + let + fullname = (self.root / segments.joinPath()) + .absolutePath() + .catch() + .get() + .addFileExt(FileExt) + + if not self.isRootSubdir(fullname): + return failure "Path is outside of `root` directory!" + + return success fullname + + method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = + return self.path(key).?fileExists() + + method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = + without path =? self.path(key), error: + return failure error + + if not path.fileExists(): + return success() + + try: + removeFile(path) + except OSError as e: + return failure e + + return success() + + method delete*(self: FSDatastore, keys: seq[Key]): Future[?!void] {.async.} = + for key in keys: + if err =? (await self.delete(key)).errorOption: + return failure err + + return success() + + proc readFile*(self: FSDatastore, path: string): ?!seq[byte] = + var + file: File + + defer: + file.close + + if not file.open(path): + return failure "unable to open file!" + + try: + let + size = file.getFileSize + + var + bytes = newSeq[byte](size) + read = 0 + + while read < size: + read += file.readBytes(bytes, read, size) + + if read < size: + return failure $read & " bytes were read from " & path & + " but " & $size & " bytes were expected" + + return success bytes + + except CatchableError as e: + return failure e + + method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = + without path =? self.path(key), error: + return failure error + + if not path.fileExists(): + return failure( + newException(DatastoreKeyNotFound, "Key doesn't exist")) + + return self.readFile(path) + + method put*( + self: FSDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + + without path =? self.path(key), error: + return failure error + + try: + createDir(parentDir(path)) + writeFile(path, data) + except CatchableError as e: + return failure e + + return success() + + method put*( + self: FSDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + + for entry in batch: + if err =? (await self.put(entry.key, entry.data)).errorOption: + return failure err + + return success() + + proc dirWalker(path: string): iterator: string {.gcsafe.} = + return iterator(): string = + try: + for p in path.walkDirRec(yieldFilter = {pcFile}, relative = true): + yield p + except CatchableError as exc: + raise newException(Defect, exc.msg) + + method close*(self: FSDatastore): Future[?!void] {.async.} = + return success() + + method query*( + self: FSDatastore, + query: Query): Future[?!QueryIter] {.async.} = + + without path =? self.path(query.key), error: + return failure error + + let basePath = + # it there is a file in the directory + # with the same name then list the contents + # of the directory, otherwise recurse + # into subdirectories + if path.fileExists: + path.parentDir + else: + path.changeFileExt("") + + let + walker = dirWalker(basePath) + + var + iter = QueryIter.new() + + proc next(): Future[?!QueryResponse] {.async.} = + let + path = walker() + + if finished(walker): + iter.finished = true + return success (Key.none, EmptyBytes) + + var + keyPath = basePath + + keyPath.removePrefix(self.root) + keyPath = keyPath / path.changeFileExt("") + keyPath = keyPath.replace("\\", "/") + + let + key = Key.init(keyPath).expect("should not fail") + data = + if query.value: + self.readFile((basePath / path).absolutePath) + .expect("Should read file") + else: + @[] + + return success (key.some, data) + + iter.next = next + return success iter + + proc new*( + T: type FSDatastore, + root: string, + depth = 2, + caseSensitive = true, + ignoreProtected = false): ?!FSDatastore = + + let root = ? ( + block: + if root.isAbsolute: root + else: getCurrentDir() / root).catch + + if not dirExists(root): + return failure "directory does not exist: " & root + + success FSDatastore( + root: root, + ignoreProtected: ignoreProtected, + depth: depth) From a8da4c4f389160209771fbbc7ec06ae3d975d623 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:07:32 -0700 Subject: [PATCH 440/445] copy paste sync-sqlite setup too --- datastore/sql.nim | 329 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 277 insertions(+), 52 deletions(-) diff --git a/datastore/sql.nim b/datastore/sql.nim index 6f64df85..ab0490a7 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -18,55 +18,280 @@ export datastore, keys, query, Taskpool, Memory, DbExt push: {.upraises: [].} -type - SQLiteDatastore* = ref object of Datastore - db: ThreadProxy[SQLiteBackend[KeyId, DataBuffer]] - -proc path*(self: SQLiteDatastore): string = - self.db.backend.path() - -proc readOnly*(self: SQLiteDatastore): bool = - self.db.backend.readOnly() - -method has*(self: SQLiteDatastore, - key: Key): Future[?!bool] {.async.} = - await self.db.has(key) - -method delete*(self: SQLiteDatastore, - key: Key): Future[?!void] {.async.} = - await self.db.delete(key) - -method delete*(self: SQLiteDatastore, - keys: seq[Key]): Future[?!void] {.async.} = - await self.db.delete(keys) - -method get*(self: SQLiteDatastore, - key: Key): Future[?!seq[byte]] {.async.} = - await self.db.get(key) - -method put*(self: SQLiteDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async.} = - await self.db.put(key, data) - -method put*(self: SQLiteDatastore, - batch: seq[BatchEntry]): Future[?!void] {.async.} = - await self.db.put(batch) - -method close*(self: SQLiteDatastore): Future[?!void] {.async.} = - await self.db.close() - -method query*(self: SQLiteDatastore, - q: Query): Future[?!QueryIter] {.async.} = - await self.db.query(q) - -proc new*( - T: type SQLiteDatastore, - path: string, - tp: Taskpool = Taskpool.new(4), - readOnly = false): ?!SQLiteDatastore = - - let - backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) - db = ? ThreadProxy.new(backend, tp = tp) - success SQLiteDatastore(db: db) +when datastoreUseAsync: + + type + SQLiteDatastore* = ref object of Datastore + db: ThreadProxy[SQLiteBackend[KeyId, DataBuffer]] + + proc path*(self: SQLiteDatastore): string = + self.db.backend.path() + + proc readOnly*(self: SQLiteDatastore): bool = + self.db.backend.readOnly() + + method has*(self: SQLiteDatastore, + key: Key): Future[?!bool] {.async.} = + await self.db.has(key) + + method delete*(self: SQLiteDatastore, + key: Key): Future[?!void] {.async.} = + await self.db.delete(key) + + method delete*(self: SQLiteDatastore, + keys: seq[Key]): Future[?!void] {.async.} = + await self.db.delete(keys) + + method get*(self: SQLiteDatastore, + key: Key): Future[?!seq[byte]] {.async.} = + await self.db.get(key) + + method put*(self: SQLiteDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async.} = + await self.db.put(key, data) + + method put*(self: SQLiteDatastore, + batch: seq[BatchEntry]): Future[?!void] {.async.} = + await self.db.put(batch) + + method close*(self: SQLiteDatastore): Future[?!void] {.async.} = + await self.db.close() + + method query*(self: SQLiteDatastore, + q: Query): Future[?!QueryIter] {.async.} = + await self.db.query(q) + + proc new*( + T: type SQLiteDatastore, + path: string, + tp: Taskpool = Taskpool.new(4), + readOnly = false): ?!SQLiteDatastore = + + let + backend = ? newSQLiteBackend[KeyId, DataBuffer](path, readOnly) + db = ? ThreadProxy.new(backend, tp = tp) + success SQLiteDatastore(db: db) + +else: + + type + SQLiteDatastore* = ref object of Datastore + readOnly: bool + db: SQLiteDsDb + + proc path*(self: SQLiteDatastore): string = + self.db.dbPath + + proc `readOnly=`*(self: SQLiteDatastore): bool + {.error: "readOnly should not be assigned".} + + proc timestamp*(t = epochTime()): int64 = + (t * 1_000_000).int64 + + method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = + var + exists = false + + proc onData(s: RawStmtPtr) = + exists = sqlite3_column_int64(s, ContainsStmtExistsCol.cint).bool + + if err =? self.db.containsStmt.query((key.id), onData).errorOption: + return failure err + + return success exists + + method delete*(self: SQLiteDatastore, key: Key): Future[?!void] {.async.} = + return self.db.deleteStmt.exec((key.id)) + + method delete*(self: SQLiteDatastore, keys: seq[Key]): Future[?!void] {.async.} = + if err =? self.db.beginStmt.exec().errorOption: + return failure(err) + + for key in keys: + if err =? self.db.deleteStmt.exec((key.id)).errorOption: + if err =? self.db.rollbackStmt.exec().errorOption: + return failure err.msg + + return failure err.msg + + if err =? self.db.endStmt.exec().errorOption: + return failure err.msg + + return success() + + method get*(self: SQLiteDatastore, key: Key): Future[?!seq[byte]] {.async.} = + # see comment in ./filesystem_datastore re: finer control of memory + # allocation in `method get`, could apply here as well if bytes were read + # incrementally with `sqlite3_blob_read` + + var + bytes: seq[byte] + + proc onData(s: RawStmtPtr) = + bytes = self.db.getDataCol() + + if err =? self.db.getStmt.query((key.id), onData).errorOption: + return failure(err) + + if bytes.len <= 0: + return failure( + newException(DatastoreKeyNotFound, "Key doesn't exist")) + + return success bytes + + method put*(self: SQLiteDatastore, key: Key, data: seq[byte]): Future[?!void] {.async.} = + return self.db.putStmt.exec((key.id, data, timestamp())) + + method put*(self: SQLiteDatastore, batch: seq[BatchEntry]): Future[?!void] {.async.} = + if err =? self.db.beginStmt.exec().errorOption: + return failure err + + for entry in batch: + if err =? self.db.putStmt.exec((entry.key.id, entry.data, timestamp())).errorOption: + if err =? self.db.rollbackStmt.exec().errorOption: + return failure err + + return failure err + + if err =? self.db.endStmt.exec().errorOption: + return failure err + + return success() + + method close*(self: SQLiteDatastore): Future[?!void] {.async.} = + self.db.close() + + return success() + + method query*( + self: SQLiteDatastore, + query: Query): Future[?!QueryIter] {.async.} = + + var + iter = QueryIter() + queryStr = if query.value: + QueryStmtDataIdStr + else: + QueryStmtIdStr + + if query.sort == SortOrder.Descending: + queryStr &= QueryStmtOrderDescending + else: + queryStr &= QueryStmtOrderAscending + + if query.limit != 0: + queryStr &= QueryStmtLimit + + if query.offset != 0: + queryStr &= QueryStmtOffset + + let + queryStmt = QueryStmt.prepare( + self.db.env, queryStr).expect("should not fail") + + s = RawStmtPtr(queryStmt) + + var + v = sqlite3_bind_text( + s, 1.cint, (query.key.id & "*").cstring, -1.cint, SQLITE_TRANSIENT_GCSAFE) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + if query.limit != 0: + v = sqlite3_bind_int(s, 2.cint, query.limit.cint) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + if query.offset != 0: + v = sqlite3_bind_int(s, 3.cint, query.offset.cint) + + if not (v == SQLITE_OK): + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + proc next(): Future[?!QueryResponse] {.async.} = + if iter.finished: + return failure(newException(QueryEndedError, "Calling next on a finished query!")) + + let + v = sqlite3_step(s) + + case v + of SQLITE_ROW: + let + key = Key.init( + $sqlite3_column_text_not_null(s, QueryStmtIdCol)) + .expect("should not fail") + + blob: ?pointer = + if query.value: + sqlite3_column_blob(s, QueryStmtDataCol).some + else: + pointer.none + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isSome and blob.get().isNil: + let + v = sqlite3_errcode(sqlite3_db_handle(s)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + iter.finished = true + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(s, QueryStmtDataCol) + data = if blob.isSome: + @( + toOpenArray(cast[ptr UncheckedArray[byte]](blob.get), + 0, + dataLen - 1)) + else: + @[] + + return success (key.some, data) + of SQLITE_DONE: + iter.finished = true + return success (Key.none, EmptyBytes) + else: + iter.finished = true + return failure newException(DatastoreError, $sqlite3_errstr(v)) + + iter.dispose = proc(): Future[?!void] {.async.} = + discard sqlite3_reset(s) + discard sqlite3_clear_bindings(s) + s.dispose + return success() + + iter.next = next + return success iter + + proc new*( + T: type SQLiteDatastore, + path: string, + readOnly = false): ?!T = + + let + flags = + if readOnly: SQLITE_OPEN_READONLY + else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE + + success T( + db: ? SQLiteDsDb.open(path, flags), + readOnly: readOnly) + + proc new*( + T: type SQLiteDatastore, + db: SQLiteDsDb): ?!T = + + success T( + db: db, + readOnly: db.readOnly) \ No newline at end of file From 4844f7dad0275800916e3f2c7a7b6373b065087b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:18:18 -0700 Subject: [PATCH 441/445] updates --- config.nims | 2 ++ datastore/sql.nim | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.nims b/config.nims index ed3a5315..c9ad1562 100644 --- a/config.nims +++ b/config.nims @@ -3,6 +3,8 @@ --styleCheck:usages --styleCheck:error +--d:"datastoreUseAsync=false" + when (NimMajor, NimMinor) == (1, 2): switch("hint", "Processing:off") switch("hint", "XDeclaredButNotUsed:off") diff --git a/datastore/sql.nim b/datastore/sql.nim index ab0490a7..6ecf7d70 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -78,17 +78,14 @@ else: type SQLiteDatastore* = ref object of Datastore readOnly: bool - db: SQLiteDsDb + db: SQLiteDsDb[string, seq[byte]] proc path*(self: SQLiteDatastore): string = - self.db.dbPath + $self.db.dbPath proc `readOnly=`*(self: SQLiteDatastore): bool {.error: "readOnly should not be assigned".} - proc timestamp*(t = epochTime()): int64 = - (t * 1_000_000).int64 - method has*(self: SQLiteDatastore, key: Key): Future[?!bool] {.async.} = var exists = false @@ -129,7 +126,7 @@ else: bytes: seq[byte] proc onData(s: RawStmtPtr) = - bytes = self.db.getDataCol() + bytes = dataCol[seq[byte]](self.db.getDataCol) if err =? self.db.getStmt.query((key.id), onData).errorOption: return failure(err) From ab2e7b53ce2ea75f0d2f5e67076ae6ed2d591a7a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:22:00 -0700 Subject: [PATCH 442/445] updates --- datastore/fsds.nim | 1 + datastore/sql.nim | 9 +- tests/datastore/threads/testthreadproxy.nim | 211 ++++++++++---------- 3 files changed, 112 insertions(+), 109 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 9d0c50f8..e22788ff 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -280,6 +280,7 @@ else: proc new*( T: type FSDatastore, root: string, + tp: Taskpool = nil, depth = 2, caseSensitive = true, ignoreProtected = false): ?!FSDatastore = diff --git a/datastore/sql.nim b/datastore/sql.nim index 6ecf7d70..fa71d848 100644 --- a/datastore/sql.nim +++ b/datastore/sql.nim @@ -274,6 +274,7 @@ else: proc new*( T: type SQLiteDatastore, path: string, + tp: Taskpool = nil, readOnly = false): ?!T = let @@ -281,14 +282,14 @@ else: if readOnly: SQLITE_OPEN_READONLY else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE - success T( - db: ? SQLiteDsDb.open(path, flags), + success SQLiteDatastore( + db: ? SQLiteDsDb[string, seq[byte]].open(path, flags), readOnly: readOnly) proc new*( T: type SQLiteDatastore, - db: SQLiteDsDb): ?!T = + db: SQLiteDsDb): ?!SQLiteDatastore = - success T( + success SQLiteDatastore( db: db, readOnly: db.readOnly) \ No newline at end of file diff --git a/tests/datastore/threads/testthreadproxy.nim b/tests/datastore/threads/testthreadproxy.nim index c3ef153b..6851566f 100644 --- a/tests/datastore/threads/testthreadproxy.nim +++ b/tests/datastore/threads/testthreadproxy.nim @@ -126,110 +126,111 @@ suite "Test Query ThreadDatastore with fsds": queryTests(ds, false) -for i in 1..N: - suite "Test ThreadDatastore cancelations": - - privateAccess(SQLiteDatastore) # expose private fields - privateAccess(ThreadProxy) # expose private fields - privateAccess(TaskCtx) # expose private fields - - var sds: SQLiteDatastore - - setupAll: - sds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() # run full collect after each test - - test "Should monitor signal and cancel": - var - signal = ThreadSignalPtr.new().tryGet() - - proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = - executeTask(ctx): - (?!bool).ok(true) - - let ctx = newTaskCtx(bool, signal=signal) - ctx[].cancelled = true - dispatchTask(sds.db, signal): - sds.db.tp.spawn cancelTestTask(ctx) - - check: - ctx[].res.isErr == true - ctx[].cancelled == true - ctx[].running == false - - test "Should cancel future": - - var - signal = ThreadSignalPtr.new().tryGet() - ms {.global.}: MutexSignal - flag {.global.}: Atomic[bool] - futFreed {.global.}: Atomic[bool] - ready {.global.}: Atomic[bool] - - ms.init() - - type - FutTestObj = object - val: int - TestValue = object - ThreadTestInt = (TestValue, ) - - proc `=destroy`(obj: var TestValue) = - # echo "destroy TestObj!" - flag.store(true) - - proc `=destroy`(obj: var FutTestObj) = - # echo "destroy FutTestObj!" - futFreed.store(true) - - proc wait(flag: var Atomic[bool], name = "task") = - # echo "wait for " & name & " to be ready..." - # defer: echo "" - for i in 1..100: - # stdout.write(".") - if flag.load() == true: - return - os.sleep(10) - raise newException(Defect, "timeout") - - proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = - executeTask(ctx): - # echo "task:exec" - discard ctx[].signal.fireSync() - ready.store(true) - ms.wait() - echo "task context memory: ", ctx[] - (?!ThreadTestInt).ok(default(ThreadTestInt)) - - proc runTestTask() {.async.} = - let obj = FutTestObj(val: 42) - await sleepAsync(1.milliseconds) +when datastoreUseAsync: + for i in 1..N: + suite "Test ThreadDatastore cancelations": + + privateAccess(SQLiteDatastore) # expose private fields + privateAccess(ThreadProxy) # expose private fields + privateAccess(TaskCtx) # expose private fields + + var sds: SQLiteDatastore + + setupAll: + sds = SQLiteDatastore.new(Memory, tp = taskPool).tryGet() + + teardown: + GC_fullCollect() # run full collect after each test + + test "Should monitor signal and cancel": + var + signal = ThreadSignalPtr.new().tryGet() + + proc cancelTestTask(ctx: TaskCtx[bool]) {.gcsafe.} = + executeTask(ctx): + (?!bool).ok(true) + + let ctx = newTaskCtx(bool, signal=signal) + ctx[].cancelled = true + dispatchTask(sds.db, signal): + sds.db.tp.spawn cancelTestTask(ctx) + + check: + ctx[].res.isErr == true + ctx[].cancelled == true + ctx[].running == false + + test "Should cancel future": + + var + signal = ThreadSignalPtr.new().tryGet() + ms {.global.}: MutexSignal + flag {.global.}: Atomic[bool] + futFreed {.global.}: Atomic[bool] + ready {.global.}: Atomic[bool] + + ms.init() + + type + FutTestObj = object + val: int + TestValue = object + ThreadTestInt = (TestValue, ) + + proc `=destroy`(obj: var TestValue) = + # echo "destroy TestObj!" + flag.store(true) + + proc `=destroy`(obj: var FutTestObj) = + # echo "destroy FutTestObj!" + futFreed.store(true) + + proc wait(flag: var Atomic[bool], name = "task") = + # echo "wait for " & name & " to be ready..." + # defer: echo "" + for i in 1..100: + # stdout.write(".") + if flag.load() == true: + return + os.sleep(10) + raise newException(Defect, "timeout") + + proc errorTestTask(ctx: TaskCtx[ThreadTestInt]) {.gcsafe, nimcall.} = + executeTask(ctx): + # echo "task:exec" + discard ctx[].signal.fireSync() + ready.store(true) + ms.wait() + echo "task context memory: ", ctx[] + (?!ThreadTestInt).ok(default(ThreadTestInt)) + + proc runTestTask() {.async.} = + let obj = FutTestObj(val: 42) + await sleepAsync(1.milliseconds) + try: + let ctx = newTaskCtx(ThreadTestInt, signal=signal) + dispatchTask(sds.db, signal): + sds.db.tp.spawn errorTestTask(ctx) + ready.wait() + # echo "raise error" + raise newException(ValueError, "fake error") + finally: + # echo "fut FutTestObj: ", obj + assert obj.val == 42 # need to force future to keep ref here try: - let ctx = newTaskCtx(ThreadTestInt, signal=signal) - dispatchTask(sds.db, signal): - sds.db.tp.spawn errorTestTask(ctx) - ready.wait() - # echo "raise error" - raise newException(ValueError, "fake error") + block: + await runTestTask() + except CatchableError as exc: + # echo "caught: ", $exc + discard finally: - # echo "fut FutTestObj: ", obj - assert obj.val == 42 # need to force future to keep ref here - try: - block: - await runTestTask() - except CatchableError as exc: - # echo "caught: ", $exc - discard - finally: - # echo "finish" - check ready.load() == true - GC_fullCollect() - futFreed.wait("futFreed") - echo "future freed it's mem!" - check futFreed.load() == true - - ms.fire() - flag.wait("flag") - check flag.load() == true + # echo "finish" + check ready.load() == true + GC_fullCollect() + futFreed.wait("futFreed") + echo "future freed it's mem!" + check futFreed.load() == true + + ms.fire() + flag.wait("flag") + check flag.load() == true From e7aef8ea8573658aa7cdc94be6eb2f1446a2efa7 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:22:09 -0700 Subject: [PATCH 443/445] updates --- config.nims | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.nims b/config.nims index c9ad1562..83d02dcc 100644 --- a/config.nims +++ b/config.nims @@ -3,7 +3,7 @@ --styleCheck:usages --styleCheck:error ---d:"datastoreUseAsync=false" +# --d:"datastoreUseAsync=false" when (NimMajor, NimMinor) == (1, 2): switch("hint", "Processing:off") From 5dcd8ce6280aa4de9d412f5970a47135f9e4db35 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 20 Oct 2023 16:24:57 -0700 Subject: [PATCH 444/445] run and build sync tests --- .github/workflows/tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0e2e106d..3d85c89b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -134,3 +134,10 @@ jobs: nim --version echo nimble --verbose test + + - name: Build and run tests + run: | + source "${HOME}/.bash_env" + nim --version + echo + nimble --verbose -d:datastoreUseAsync=false test From 48157960f4836deb273101134ea31978dea6014f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 23 Oct 2023 13:07:49 -0700 Subject: [PATCH 445/445] rename toSeq to toSequence to avoid conflicting with system.toSeq --- datastore/threads/databuffer.nim | 4 ++-- datastore/threads/threadproxy.nim | 4 ++-- tests/datastore/testdatabuffer.nim | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 5f80cbfe..dc8b927d 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -87,7 +87,7 @@ proc setData*[T: byte | char](db: DataBuffer, data: openArray[T]) = copyMem(db[].buf, baseAddr data, data.len()) db[].size = data.len() -proc toSeq*(self: DataBuffer): seq[byte] = +proc toSequence*(self: DataBuffer): seq[byte] = ## convert buffer to a seq type using copy and either a byte or char ## @@ -100,7 +100,7 @@ proc `@`*(self: DataBuffer): seq[byte] = ## either a byte or char ## - self.toSeq() + self.toSequence() proc toString*(data: DataBuffer): string = ## convert buffer to string type using copy diff --git a/datastore/threads/threadproxy.nim b/datastore/threads/threadproxy.nim index a5538949..3e2df5b9 100644 --- a/datastore/threads/threadproxy.nim +++ b/datastore/threads/threadproxy.nim @@ -242,7 +242,7 @@ proc get*[BT](self: ThreadProxy[BT], let key = KeyId.new key.id() self.tp.spawn getTask(ctx, ds, key) - return ctx[].res.toRes(v => v.toSeq()) + return ctx[].res.toRes(v => v.toSequence()) proc close*[BT](self: ThreadProxy[BT]): Future[?!void] {.async.} = await self.semaphore.closeAll() @@ -352,7 +352,7 @@ proc query*[BT](self: ThreadProxy[BT], else: let qres = ctx[].res.get() let key = qres.key.map(proc (k: KeyId): Key = k.toKey()) - let data = qres.data.toSeq() + let data = qres.data.toSequence() return (?!QueryResponse).ok((key: key, data: data)) except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index eb979317..ee8bc117 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -45,7 +45,7 @@ proc thread2(val: int) {.thread.} = echo "thread2: receiving " let msg: DataBuffer = shareVal echo "thread2: received: ", msg - check string.fromBytes(msg.toSeq()) == "hello world" + check string.fromBytes(msg.toSequence()) == "hello world" # os.sleep(100) proc runBasicTest() = @@ -104,10 +104,10 @@ suite "Share buffer test": check Key.init($a).tryGet == k1 test "seq conversion": - check string.fromBytes(a.toSeq()) == "/a/b" + check string.fromBytes(a.toSequence()) == "/a/b" test "seq conversion": - check a.toSeq() == "/a/b".toBytes + check a.toSequence() == "/a/b".toBytes test "basic null terminate test": let cstr = DataBuffer.new("test", {dbNullTerminate})