diff --git a/ethers/providers/jsonrpc/json.nim b/ethers/providers/jsonrpc/json.nim index af29b0d..bff9344 100644 --- a/ethers/providers/jsonrpc/json.nim +++ b/ethers/providers/jsonrpc/json.nim @@ -1,12 +1,14 @@ -import std/json except `%`, `%*` +import std/json as stdjson except `%`, `%*` import std/macros import std/options +import std/sequtils +import std/sets import std/strutils # import std/strformat import std/tables import std/typetraits -import pkg/chronicles +import pkg/chronicles except toJson import pkg/contractabi import pkg/stew/byteutils import pkg/stint @@ -15,21 +17,52 @@ import pkg/questionable/results import ../../basics -export json except `%`, `%*` +export stdjson except `%`, `%*`, parseJson +export chronicles except toJson +export sets {.push raises: [].} logScope: - topics = "json serialization" + topics = "json de/serialization" type - SerializationError = object of EthersError - UnexpectedKindError = object of SerializationError - -template serialize* {.pragma.} - -proc mapErrTo[T, E1: CatchableError, E2: CatchableError](r: Result[T, E1], _: type E2): ?!T = - r.mapErr(proc (e: E1): E2 = E2(msg: e.msg)) + SerdeError* = object of EthersError + UnexpectedKindError* = object of SerdeError + DeserializeMode* = enum + Default, ## objects can have more or less fields than json + OptIn, ## json must have fields marked with {.serialize.} + Strict ## object fields and json fields must match exactly + +# template serializeAll* {.pragma.} +template serialize*(key = "", ignore = false) {.pragma.} +template deserialize*(key = "", mode = DeserializeMode.Default) {.pragma.} + +template expectEmptyPragma(value, pragma, msg) = + static: + when value.hasCustomPragma(pragma): + const params = value.getCustomPragmaVal(pragma) + for param in params.fields: + if param != typeof(param).default: + raiseAssert(msg) + +template expectMissingPragmaParam(value, pragma, name, msg) = + static: + when value.hasCustomPragma(pragma): + const params = value.getCustomPragmaVal(pragma) + for paramName, paramValue in params.fieldPairs: + if paramName == name and paramValue != typeof(paramValue).default: + raiseAssert(msg) + +proc mapErrTo[E1: ref CatchableError, E2: SerdeError]( + e1: E1, + _: type E2, + msg: string = e1.msg): ref E2 = + + return newException(E2, msg, e1) + +proc newSerdeError(msg: string): ref SerdeError = + newException(SerdeError, msg) proc newUnexpectedKindError( expectedType: type, @@ -40,7 +73,7 @@ proc newUnexpectedKindError( else: $json.kind newException(UnexpectedKindError, "deserialization to " & $expectedType & " failed: expected " & - expectedKinds & "but got " & $kind) + expectedKinds & " but got " & $kind) proc newUnexpectedKindError( expectedType: type, @@ -71,24 +104,34 @@ template expectJsonKind*( ) = expectJsonKind(expectedType, {expectedKind}, json) +proc fieldKeys[T](obj: T): seq[string] = + for name, _ in fieldPairs(when type(T) is ref: obj[] else: obj): + result.add name + +func keysNotIn[T](json: JsonNode, obj: T): HashSet[string] = + let jsonKeys = json.keys.toSeq.toHashSet + let objKeys = obj.fieldKeys.toHashSet + difference(jsonKeys, objKeys) + proc fromJson*( T: type enum, json: JsonNode ): ?!T = expectJsonKind(string, JString, json) - catch parseEnum[T](json.str) + without val =? parseEnum[T](json.str).catch, error: + return failure error.mapErrTo(SerdeError) + return success val proc fromJson*( _: type string, json: JsonNode ): ?!string = if json.isNil: - let err = newException(ValueError, "'json' expected, but was nil") - return failure(err) + return failure newSerdeError("'json' expected, but was nil") elif json.kind == JNull: return success("null") elif json.isNil or json.kind != JString: - return failure(newUnexpectedKindError(string, JString, json)) + return failure newUnexpectedKindError(string, JString, json) catch json.getStr proc fromJson*( @@ -113,7 +156,8 @@ proc fromJson*[T: SomeInteger]( expectJsonKind(T, {JInt, JString}, json) case json.kind of JString: - let x = parseBiggestUInt(json.str) + without x =? parseBiggestUInt(json.str).catch, error: + return failure newSerdeError(error.msg) return success cast[T](x) else: return success T(json.num) @@ -201,41 +245,82 @@ proc fromJson*[T]( arr.add(? T.fromJson(elem)) success arr +template getDeserializationKey(fieldName, fieldValue): string = + when fieldValue.hasCustomPragma(deserialize): + fieldValue.expectMissingPragmaParam(deserialize, "mode", + "Cannot set 'mode' on field defintion.") + let (key, mode) = fieldValue.getCustomPragmaVal(deserialize) + if key != "": key + else: fieldName + else: fieldName + +template getDeserializationMode(T): DeserializeMode = + when T.hasCustomPragma(deserialize): + T.expectMissingPragmaParam(deserialize, "key", + "Cannot set 'key' on object definition.") + T.getCustomPragmaVal(deserialize)[1] # mode = second pragma param + else: + DeserializeMode.Default + proc fromJson*[T: ref object or object]( _: type T, json: JsonNode ): ?!T = + when T is JsonNode: return success T(json) expectJsonKind(T, JObject, json) var res = when type(T) is ref: T.new() else: T.default + let mode = T.getDeserializationMode() - # Leave this in, it's good for debugging: - trace "deserializing object", to = $T, json for name, value in fieldPairs(when type(T) is ref: res[] else: res): - logScope: field = $T & "." & name + mode + + let key = getDeserializationKey(name, value) + var skip = false # workaround for 'continue' not supported in a 'fields' loop + + if mode == Strict and key notin json: + return failure newSerdeError("object field missing in json: " & key) + + if mode == OptIn: + if not value.hasCustomPragma(deserialize): + debug "object field not marked as 'deserialize', skipping", name = name + # use skip as workaround for 'continue' not supported in a 'fields' loop + skip = true + elif key notin json: + return failure newSerdeError("object field missing in json: " & key) - if name in json and - jsonVal =? json{name}.catch and - not jsonVal.isNil: + if key in json and + jsonVal =? json{key}.catch and + not jsonVal.isNil and + not skip: without parsed =? type(value).fromJson(jsonVal), e: - error "error deserializing field", + warn "failed to deserialize field", + `type` = $typeof(value), json = jsonVal, error = e.msg return failure(e) value = parsed - else: - debug "object field does not exist in json, skipping", json + elif mode == DeserializeMode.Default: + debug "object field missing in json, skipping", key, json + + # ensure there's no extra fields in json + if mode == DeserializeMode.Strict: + let extraFields = json.keysNotIn(res) + if extraFields.len > 0: + return failure newSerdeError("json field(s) missing in object: " & $extraFields) + success(res) -proc parse*(json: string): ?!JsonNode = +proc parseJson*(json: string): ?!JsonNode = + ## fix for nim raising Exception try: - return parseJson(json).catch + return stdjson.parseJson(json).catch except Exception as e: return err newException(CatchableError, e.msg) @@ -254,10 +339,10 @@ proc fromJson*( proc fromJson*[T: ref object or object]( _: type T, - json: string + jsn: string ): ?!T = - let json = ? parse(json) - T.fromJson(json) + let jsn = ? json.parseJson(jsn) # full qualification required in-module only + T.fromJson(jsn) func `%`*(s: string): JsonNode = newJString(s) @@ -301,18 +386,34 @@ func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode = func `%`*[T](opt: Option[T]): JsonNode = if opt.isSome: %(opt.get) else: newJNull() -func `%`*[T: object](obj: T): JsonNode = + +func `%`*[T: object or ref object](obj: T): JsonNode = + + # T.expectMissingPragma(serialize, "Invalid pragma on object definition.") + let jsonObj = newJObject() - for name, value in obj.fieldPairs: - when value.hasCustomPragma(serialize): + let o = when T is ref object: obj[] + else: obj + + T.expectEmptyPragma(serialize, "Cannot specify 'key' or 'ignore' on object defition") + + const serializeAllFields = T.hasCustomPragma(serialize) + + for name, value in o.fieldPairs: + # TODO: move to % + # value.expectMissingPragma(deserializeMode, "Invalid pragma on field definition.") + # static: + const serializeField = value.hasCustomPragma(serialize) + when serializeField: + let (keyOverride, ignore) = value.getCustomPragmaVal(serialize) + if not ignore: + let key = if keyOverride != "": keyOverride + else: name + jsonObj[key] = %value + + elif serializeAllFields: jsonObj[name] = %value - jsonObj -func `%`*[T: ref object](obj: T): JsonNode = - let jsonObj = newJObject() - for name, value in obj[].fieldPairs: - when value.hasCustomPragma(serialize): - jsonObj[name] = %(value) jsonObj proc `%`*(o: enum): JsonNode = % $o diff --git a/testmodule/providers/jsonrpc/testjson.nim b/testmodule/providers/jsonrpc/testjson.nim new file mode 100644 index 0000000..909be5b --- /dev/null +++ b/testmodule/providers/jsonrpc/testjson.nim @@ -0,0 +1,385 @@ +import std/math +import std/options +import std/strformat +import std/strutils +import std/unittest +import pkg/stew/byteutils +import pkg/stint +import pkg/ethers/providers/jsonrpc/json as utilsjson +import pkg/questionable +import pkg/questionable/results + + +func flatten(s: string): string = + s.replace(" ") + .replace("\n") + +suite "json serialization - serialize": + + test "serializes UInt256 to non-hex string representation": + check (% 100000.u256) == newJString("100000") + + test "serializes sequence to an array": + let json = % @[1, 2, 3] + let expected = "[1,2,3]" + check $json == expected + + test "serializes Option[T] when has a value": + let obj = %(some 1) + let expected = "1" + check $obj == expected + + test "serializes Option[T] when doesn't have a value": + let obj = %(none int) + let expected = "null" + check $obj == expected + + test "serializes uints int.high or smaller": + let largeUInt: uint = uint(int.high) + check %largeUInt == newJInt(BiggestInt(largeUInt)) + + test "serializes large uints": + let largeUInt: uint = uint(int.high) + 1'u + check %largeUInt == newJString($largeUInt) + + test "serializes Inf float": + check %Inf == newJString("inf") + + test "serializes -Inf float": + check %(-Inf) == newJString("-inf") + + test "can construct json objects with %*": + type MyObj = object + mystring {.serialize.}: string + myint {.serialize.}: int + myoption {.serialize.}: ?bool + + let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true) + let mystuint = 100000.u256 + + let json = %*{ + "myobj": myobj, + "mystuint": mystuint + } + + let expected = """{ + "myobj": { + "mystring": "abc", + "myint": 123, + "myoption": true + }, + "mystuint": "100000" + }""".flatten + + check $json == expected + + test "only serializes marked fields": + type MyObj = object + mystring {.serialize.}: string + myint {.serialize.}: int + mybool: bool + + let obj = % MyObj(mystring: "abc", myint: 1, mybool: true) + + let expected = """{ + "mystring": "abc", + "myint": 1 + }""".flatten + + check $obj == expected + + test "serializes ref objects": + type MyRef = ref object + mystring {.serialize.}: string + myint {.serialize.}: int + + let obj = % MyRef(mystring: "abc", myint: 1) + + let expected = """{ + "mystring": "abc", + "myint": 1 + }""".flatten + + check $obj == expected + +suite "json serialization - deserialize": + + test "deserializes NaN float": + check %NaN == newJString("nan") + + test "deserialize enum": + type MyEnum = enum + First, + Second + let json = newJString("Second") + check !MyEnum.fromJson(json) == Second + + test "deserializes UInt256 from non-hex string representation": + let json = newJString("100000") + check !UInt256.fromJson(json) == 100000.u256 + + test "deserializes Option[T] when has a value": + let json = newJInt(1) + check (!fromJson(?int, json) == some 1) + + test "deserializes Option[T] when doesn't have a value": + let json = newJNull() + check !fromJson(?int, json) == none int + + test "deserializes float": + let json = newJFloat(1.234) + check !float.fromJson(json) == 1.234 + + test "deserializes Inf float": + let json = newJString("inf") + check !float.fromJson(json) == Inf + + test "deserializes -Inf float": + let json = newJString("-inf") + check !float.fromJson(json) == -Inf + + test "deserializes NaN float": + let json = newJString("nan") + check (!float.fromJson(json)).isNaN + + test "deserializes array to sequence": + let expected = @[1, 2, 3] + let json = !"[1,2,3]".parseJson + check !seq[int].fromJson(json) == expected + + test "deserializes uints int.high or smaller": + let largeUInt: uint = uint(int.high) + let json = newJInt(BiggestInt(largeUInt)) + check !uint.fromJson(json) == largeUInt + + test "deserializes large uints": + let largeUInt: uint = uint(int.high) + 1'u + let json = newJString($BiggestUInt(largeUInt)) + check !uint.fromJson(json) == largeUInt + + test "can deserialize json objects": + type MyObj = object + mystring: string + myint: int + myoption: ?bool + + let expected = MyObj(mystring: "abc", myint: 123, myoption: some true) + + let json = !parseJson("""{ + "mystring": "abc", + "myint": 123, + "myoption": true + }""") + check !MyObj.fromJson(json) == expected + + test "ignores serialize pragma when deserializing": + type MyObj = object + mystring {.serialize.}: string + mybool: bool + + let expected = MyObj(mystring: "abc", mybool: true) + + let json = !parseJson("""{ + "mystring": "abc", + "mybool": true + }""") + + check !MyObj.fromJson(json) == expected + + test "deserializes objects with extra fields": + type MyObj = object + mystring: string + mybool: bool + + let expected = MyObj(mystring: "abc", mybool: true) + + let json = !"""{ + "mystring": "abc", + "mybool": true, + "extra": "extra" + }""".parseJson + check !MyObj.fromJson(json) == expected + + test "deserializes objects with less fields": + type MyObj = object + mystring: string + mybool: bool + + let expected = MyObj(mystring: "abc", mybool: false) + + let json = !"""{ + "mystring": "abc" + }""".parseJson + check !MyObj.fromJson(json) == expected + + test "deserializes ref objects": + type MyRef = ref object + mystring: string + myint: int + + let expected = MyRef(mystring: "abc", myint: 1) + + let json = !"""{ + "mystring": "abc", + "myint": 1 + }""".parseJson + + let deserialized = !MyRef.fromJson(json) + check deserialized.mystring == expected.mystring + check deserialized.myint == expected.myint + +suite "json serialization pragmas": + + test "fails to compile when object marked with 'serialize' specifies options": + type + MyObj {.serialize(key="test", ignore=true).} = object + + check not compiles(%MyObj()) + + test "compiles when object marked with 'serialize' only": + type + MyObj {.serialize.} = object + + check compiles(%MyObj()) + + test "fails to compile when field marked with 'deserialize' specifies mode": + type + MyObj = object + field {.deserialize(mode=OptIn).}: bool + + check not compiles(MyObj.fromJson("""{"field":true}""")) + + test "compiles when object marked with 'deserialize' specifies mode": + type + MyObj {.deserialize(mode=OptIn).} = object + field: bool + + check compiles(MyObj.fromJson("""{"field":true}""")) + + test "fails to compile when object marked with 'deserialize' specifies key": + type + MyObj {.deserialize("test").} = object + field: bool + + check not compiles(MyObj.fromJson("""{"field":true}""")) + + test "compiles when field marked with 'deserialize' specifies key": + type + MyObj = object + field {.deserialize("test").}: bool + + check compiles(MyObj.fromJson("""{"field":true}""")) + + test "compiles when field marked with empty 'deserialize'": + type + MyObj = object + field {.deserialize.}: bool + + check compiles(MyObj.fromJson("""{"field":true}""")) + + test "compiles when field marked with 'serialize'": + type + MyObj = object + field {.serialize.}: bool + + check compiles(%MyObj()) + + test "serializes field with key when specified": + type MyObj = object + field {.serialize("test").}: bool + + let obj = MyObj(field: true) + check obj.toJson == """{"test":true}""" + + test "does not serialize ignored field": + type MyObj = object + field1 {.serialize.}: bool + field2 {.serialize(ignore=true).}: bool + + let obj = MyObj(field1: true, field2: true) + check obj.toJson == """{"field1":true}""" + + test "serialize on object definition serializes all fields": + type MyObj {.serialize.} = object + field1: bool + field2: bool + + let obj = MyObj(field1: true, field2: true) + check obj.toJson == """{"field1":true,"field2":true}""" + + test "ignores field when object has serialize": + type MyObj {.serialize.} = object + field1 {.serialize(ignore=true).}: bool + field2: bool + + let obj = MyObj(field1: true, field2: true) + check obj.toJson == """{"field2":true}""" + + test "serializes field with key when object has serialize": + type MyObj {.serialize.} = object + field1 {.serialize("test").}: bool + field2: bool + + let obj = MyObj(field1: true, field2: true) + check obj.toJson == """{"test":true,"field2":true}""" + + test "deserializes matching object and json fields when mode is Strict": + type MyObj {.deserialize(mode=Strict).} = object + field1: bool + field2: bool + + let val = !MyObj.fromJson("""{"field1":true,"field2":true}""") + check val == MyObj(field1: true, field2: true) + + test "fails to deserialize with missing json field when mode is Strict": + type MyObj {.deserialize(mode=Strict).} = object + field1: bool + field2: bool + + let r = MyObj.fromJson("""{"field2":true}""") + check r.isFailure + check r.error of SerdeError + check r.error.msg == "object field missing in json: field1" + + test "fails to deserialize with missing object field when mode is Strict": + type MyObj {.deserialize(mode=Strict).} = object + field2: bool + + let r = MyObj.fromJson("""{"field1":true,"field2":true}""") + check r.isFailure + check r.error of SerdeError + check r.error.msg == "json field(s) missing in object: {\"field1\"}" + + test "deserializes only fields marked as deserialize when mode is OptIn": + type MyObj {.deserialize(mode=OptIn).} = object + field1: int + field2 {.deserialize.}: bool + + let val = !MyObj.fromJson("""{"field1":true,"field2":true}""") + check val == MyObj(field1: 0, field2: true) + + test "can deserialize object in default mode when not marked with deserialize": + type MyObj = object + field1: bool + field2: bool + + let val = !MyObj.fromJson("""{"field1":true,"field3":true}""") + check val == MyObj(field1: true, field2: false) + + test "deserializes object field with marked json key": + type MyObj = object + field1 {.deserialize("test").}: bool + field2: bool + + let val = !MyObj.fromJson("""{"test":true,"field2":true}""") + check val == MyObj(field1: true, field2: true) + + test "fails to deserialize object field with wrong type": + type MyObj = object + field1: int + field2: bool + + let r = MyObj.fromJson("""{"field1":true,"field2":true}""") + check r.isFailure + check r.error of UnexpectedKindError + check r.error.msg == "deserialization to int failed: expected {JInt} but got JBool"