diff --git a/javascript/package.json b/javascript/package.json index e48c30d2c4..3afe903167 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -9,5 +9,8 @@ "workspaces": [ "packages/hps", "packages/fury" - ] + ], + "devDependencies": { + "prettier": "^3.1.0" + } } diff --git a/javascript/packages/fury/lib/codeGen.ts b/javascript/packages/fury/lib/codeGen.ts index 6703f6f2d7..09e5dcec34 100644 --- a/javascript/packages/fury/lib/codeGen.ts +++ b/javascript/packages/fury/lib/codeGen.ts @@ -19,7 +19,8 @@ import { replaceBackslashAndQuote, safePropAccessor, safePropName } from './util import mapSerializer from './internalSerializer/map'; import setSerializer from './internalSerializer/set'; import { arraySerializer } from './internalSerializer/array'; -import { ArrayTypeDescription, Cast, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TypeDescription } from './description'; +import { tupleSerializer } from './internalSerializer/tuple'; +import { ArrayTypeDescription, Cast, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from './description'; function computeFieldHash(hash: number, id: number): number { let newHash = (hash) * 31 + (id); @@ -72,13 +73,13 @@ function typeHandlerDeclaration(fury: Fury) { const genBuiltinDeclaration = (type: number) => { const name = `type_${type}`.replace('-', '_'); return addDeclar(name, ` - const ${name} = classResolver.getSerializerById(${type})`); + const ${name} = classResolver.getSerializerById(${type})`); } const genTagDeclaration = (tag: string) => { const name = `tag_${count++}`; return addDeclar(name, ` - const ${name} = classResolver.getSerializerByTag("${replaceBackslashAndQuote(tag)}")`, tag); + const ${name} = classResolver.getSerializerByTag("${replaceBackslashAndQuote(tag)}")`, tag); } const genDeclaration = (description: TypeDescription): string => { @@ -87,17 +88,30 @@ function typeHandlerDeclaration(fury: Fury) { return genTagDeclaration(Cast(description).options.tag); } if (description.type === InternalSerializerType.ARRAY) { + const tupleOptions = Cast(description).options; + if (tupleOptions && tupleOptions.isTuple) { + const names = [] as string[]; + Cast(description).options.inner.forEach(v => { + names.push(genDeclaration(v)); + }) + + const name = `tuple_${names.join('_')}`; + return addDeclar(name, ` + const ${name} = tupleSerializer(fury, [${names.join(', ')}])` + ) + } + const inner = genDeclaration(Cast(description).options.inner); const name = `array_${inner}`; return addDeclar(name, ` - const ${name} = arraySerializer(fury, ${inner})` + const ${name} = arraySerializer(fury, ${inner})` ) } if (description.type === InternalSerializerType.FURY_SET) { const inner = genDeclaration(Cast(description).options.key); const name = `set_${inner}`; return addDeclar(name, ` - const ${name} = setSerializer(fury, ${inner})` + const ${name} = setSerializer(fury, ${inner})` ) } if (description.type === InternalSerializerType.MAP) { @@ -106,7 +120,7 @@ function typeHandlerDeclaration(fury: Fury) { const name = `map_${key}_${value}`; return addDeclar(name, ` - const ${name} = mapSerializer(fury, ${key}, ${value})` + const ${name} = mapSerializer(fury, ${key}, ${value})` ) } return genBuiltinDeclaration(description.type); @@ -125,41 +139,37 @@ function typeHandlerDeclaration(fury: Fury) { } } -export const genSerializer = (fury: Fury, description: TypeDescription) => { +export const generateInlineCode = (fury: Fury, description: TypeDescription) => { + const options = Cast(description).options; + const tag = options?.tag; const { genDeclaration, finish } = typeHandlerDeclaration(fury); - const tag = Cast(description).options?.tag; - if (fury.classResolver.getSerializerByTag(tag)) { - return fury.classResolver.getSerializerByTag(tag); - } - - fury.classResolver.registerSerializerByTag(tag, fury.classResolver.getSerializerById(InternalSerializerType.ANY)); const expectHash = computeStructHash(description); const read = ` - // relation tag: ${Cast(description).options?.tag} + // relation tag: ${tag} const result = { - ${Object.entries(Cast(description).options.props).sort().map(([key]) => { + ${Object.entries(options.props).sort().map(([key]) => { return `${safePropName(key)}: null` }).join(',\n')} }; pushReadObject(result); - ${Object.entries(Cast(description).options.props).sort().map(([key, value]) => { + ${Object.entries(options.props).sort().map(([key, value]) => { return `result${safePropAccessor(key)} = ${genDeclaration(value)}.read()`; }).join(';\n') } return result; `; - const write = Object.entries(Cast(description).options.props).sort().map(([key, value]) => { + const write = Object.entries(options.props).sort().map(([key, value]) => { return `${genDeclaration(value)}.write(v${safePropAccessor(key)})`; }).join(';\n'); const { names, declarations} = finish(); const validTag = replaceBackslashAndQuote(tag); - return fury.classResolver.registerSerializerByTag(tag, new Function( + return new Function( ` return function (fury, scope) { const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, mapSerializer, setSerializer } = scope; - ${declarations.join('')} + const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; + ${declarations.join('')} const tagBuffer = classResolver.tagToBuffer("${validTag}"); const bufferLen = tagBuffer.byteLength; @@ -188,10 +198,23 @@ return function (fury, scope) { } } ` - )()(fury, { + ) +} + +export const genSerializer = (fury: Fury, description: TypeDescription) => { + const tag = Cast(description).options?.tag; + if (fury.classResolver.getSerializerByTag(tag)) { + return fury.classResolver.getSerializerByTag(tag); + } + + fury.classResolver.registerSerializerByTag(tag, fury.classResolver.getSerializerById(InternalSerializerType.ANY)); + + const func = generateInlineCode(fury, description); + return fury.classResolver.registerSerializerByTag(tag, func()(fury, { InternalSerializerType, RefFlags, arraySerializer, + tupleSerializer, mapSerializer, setSerializer, })); diff --git a/javascript/packages/fury/lib/description.ts b/javascript/packages/fury/lib/description.ts index e5a8bc98fd..a11efb3e81 100644 --- a/javascript/packages/fury/lib/description.ts +++ b/javascript/packages/fury/lib/description.ts @@ -35,6 +35,14 @@ export interface ArrayTypeDescription extends TypeDescription { } } +export interface TupleTypeDescription extends TypeDescription { + options: { + isTuple: true, + inner: TypeDescription[]; + } +} + + export interface SetTypeDescription extends TypeDescription { options: { key: TypeDescription; @@ -80,6 +88,14 @@ type MapProps = T extends { ? Map, ToRecordType | null> : unknown; +type TupleProps = T extends { + options: { + inner: infer T2 extends readonly[...TypeDescription[]]; + }; + } + ? { [K in keyof T2]: ToRecordType } + : unknown; + type SetProps = T extends { options: { key: infer T2 extends TypeDescription; @@ -96,6 +112,10 @@ export type ToRecordType = T extends { type: InternalSerializerType.STRING; } ? string + : T extends { + type: InternalSerializerType.TUPLE; + } + ? TupleProps : T extends { type: | InternalSerializerType.UINT8 @@ -164,6 +184,15 @@ export const Type = { }, }; }, + tuple(t1: T1) { + return { + type: InternalSerializerType.TUPLE as const, + options: { + isTuple: true, + inner: t1, + }, + }; + }, map( key: T1, value: T2 diff --git a/javascript/packages/fury/lib/internalSerializer/tuple.ts b/javascript/packages/fury/lib/internalSerializer/tuple.ts new file mode 100644 index 0000000000..285a9aae0a --- /dev/null +++ b/javascript/packages/fury/lib/internalSerializer/tuple.ts @@ -0,0 +1,54 @@ +/* + * Copyright 2023 The Fury Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InternalSerializerType, Serializer } from "../type"; +import { Fury } from "../type"; + + +export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => { + const { binaryReader, binaryWriter, referenceResolver } = fury; + + const { pushReadObject } = referenceResolver; + const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter; + const { varInt32: readVarInt32 } = binaryReader; + + return { + ...referenceResolver.deref(() => { + const len = readVarInt32(); + const result = new Array(len); + pushReadObject(result); + for (let i = 0; i < len; i++) { + const item = serializers[i]; + result[i] = item.read(); + } + return result; + }), + write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.TUPLE, (v: any[]) => { + writeVarInt32(serializers.length); + + for (let i = 0; i < serializers.length; i++) { + const item = serializers[i]; + reserves(item.config().reserve); + item.write(v[i]); + } + }), + config: () => { + return { + reserve: 7, + } + } + } +} diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 13cdfa8577..2d23f82a93 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -23,9 +23,10 @@ export type BinaryWriter = ReturnType export type BinaryReader = ReturnType -export enum InternalSerializerType{ - STRING = 13, +export enum InternalSerializerType { + STRING = 13, ARRAY = 25, + TUPLE = 25, MAP = 30, BOOL = 1, UINT8 = 2, @@ -41,7 +42,7 @@ export enum InternalSerializerType{ BINARY = 14, DATE = 16, TIMESTAMP = 18, - FURY_TYPE_TAG = 256, + FURY_TYPE_TAG = 256, FURY_SET = 257, FURY_PRIMITIVE_BOOL_ARRAY = 258, FURY_PRIMITIVE_SHORT_ARRAY = 259, diff --git a/javascript/test/__snapshots__/codeGen.test.ts.snap b/javascript/test/__snapshots__/codeGen.test.ts.snap new file mode 100644 index 0000000000..5af2c6e3b6 --- /dev/null +++ b/javascript/test/__snapshots__/codeGen.test.ts.snap @@ -0,0 +1,118 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`codeGen can generate tuple declaration code 1`] = ` +"function anonymous( +) { + +return function (fury, scope) { + const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; + const { writeNullOrRef, pushReadObject } = referenceResolver; + const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; + + const tag_0 = classResolver.getSerializerByTag("example.foo.1") + const tag_1 = classResolver.getSerializerByTag("example.foo.2") + const tuple_tag_0_tag_1 = tupleSerializer(fury, [tag_0, tag_1]) + const tag_5 = classResolver.getSerializerByTag("example.bar.1") + const tag_6 = classResolver.getSerializerByTag("example.bar.2") + const tuple_tag_0_tag_5_tag_6 = tupleSerializer(fury, [tag_0, tag_5, tag_6]) + const tagBuffer = classResolver.tagToBuffer("tuple-object-wrapper"); + const bufferLen = tagBuffer.byteLength; + + const reserves = tag_0.config().reserve + tag_1.config().reserve + tuple_tag_0_tag_1.config().reserve + tag_5.config().reserve + tag_6.config().reserve + tuple_tag_0_tag_5_tag_6.config().reserve; + return { + ...referenceResolver.deref(() => { + const hash = binaryReader.int32(); + if (hash !== 16469457) { + throw new Error("validate hash failed: tuple-object-wrapper. expect 16469457, but got" + hash); + } + { + + // relation tag: tuple-object-wrapper + const result = { + tuple1: null, +tuple1_: null, +tuple2: null, +tuple2_: null + }; + pushReadObject(result); + result.tuple1 = tuple_tag_0_tag_1.read(); +result.tuple1_ = tuple_tag_0_tag_1.read(); +result.tuple2 = tuple_tag_0_tag_5_tag_6.read(); +result.tuple2_ = tuple_tag_0_tag_5_tag_6.read() + return result; + + } + }), + write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { + classResolver.writeTag(binaryWriter, "tuple-object-wrapper", tagBuffer, bufferLen); + binaryWriter.int32(16469457); + binaryWriter.reserve(reserves); + tuple_tag_0_tag_1.write(v.tuple1); +tuple_tag_0_tag_1.write(v.tuple1_); +tuple_tag_0_tag_5_tag_6.write(v.tuple2); +tuple_tag_0_tag_5_tag_6.write(v.tuple2_) + }), + config() { + return { + reserve: bufferLen + 8, + } + } + } +} + +}" +`; + +exports[`codeGen can generate tuple declaration code 2`] = ` +"function anonymous( +) { + +return function (fury, scope) { + const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; + const { writeNullOrRef, pushReadObject } = referenceResolver; + const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; + + const type_13 = classResolver.getSerializerById(13) + const type_1 = classResolver.getSerializerById(1) + const type_6 = classResolver.getSerializerById(6) + const type_14 = classResolver.getSerializerById(14) + const tuple_type_14 = tupleSerializer(fury, [type_14]) + const tuple_type_13_type_1_type_6_tuple_type_14 = tupleSerializer(fury, [type_13, type_1, type_6, tuple_type_14]) + const tagBuffer = classResolver.tagToBuffer("tuple-object-type3-tag"); + const bufferLen = tagBuffer.byteLength; + + const reserves = type_13.config().reserve + type_1.config().reserve + type_6.config().reserve + type_14.config().reserve + tuple_type_14.config().reserve + tuple_type_13_type_1_type_6_tuple_type_14.config().reserve; + return { + ...referenceResolver.deref(() => { + const hash = binaryReader.int32(); + if (hash !== 552) { + throw new Error("validate hash failed: tuple-object-type3-tag. expect 552, but got" + hash); + } + { + + // relation tag: tuple-object-type3-tag + const result = { + tuple: null + }; + pushReadObject(result); + result.tuple = tuple_type_13_type_1_type_6_tuple_type_14.read() + return result; + + } + }), + write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { + classResolver.writeTag(binaryWriter, "tuple-object-type3-tag", tagBuffer, bufferLen); + binaryWriter.int32(552); + binaryWriter.reserve(reserves); + tuple_type_13_type_1_type_6_tuple_type_14.write(v.tuple) + }), + config() { + return { + reserve: bufferLen + 8, + } + } + } +} + +}" +`; diff --git a/javascript/test/codeGen.test.ts b/javascript/test/codeGen.test.ts new file mode 100644 index 0000000000..6cb7ce4a6d --- /dev/null +++ b/javascript/test/codeGen.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The Fury Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, test } from '@jest/globals'; +import { tupleObjectDescription, tupleObjectType3Description } from './fixtures/tuple'; +import { generateInlineCode } from '../packages/fury/lib/codeGen'; +import FuryInternal from '../packages/fury/lib/fury'; + +describe('codeGen', () => { + test('can generate tuple declaration code', () => { + const fury = FuryInternal({ refTracking: true }); + const fn = generateInlineCode(fury, tupleObjectDescription); + expect(fn.toString()).toMatchSnapshot(); + + const fn2 = generateInlineCode(fury, tupleObjectType3Description); + expect(fn2.toString()).toMatchSnapshot(); + }) +}) diff --git a/javascript/test/fixtures/tuple.ts b/javascript/test/fixtures/tuple.ts new file mode 100644 index 0000000000..2e99827e7b --- /dev/null +++ b/javascript/test/fixtures/tuple.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2023 The Fury Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Type } from "../../packages/fury"; + +export const tupleType1 = Type.tuple( [ + Type.object('example.foo.1',{ + a: Type.object('example.foo.1.1',{ + b: Type.string() + }) + }), + Type.object('example.foo.2',{ + a: Type.object('example.foo.2.1',{ + c: Type.string() + }) + }) +]); + +export const tupleType2 = Type.tuple( [ + Type.object('example.foo.1',{ + a: Type.object('example.foo.1.1',{ + b: Type.string() + }) + }), + Type.object('example.bar.1',{ + a: Type.object('example.bar.1.1',{ + b: Type.string() + }) + }), + Type.object('example.bar.2',{ + a: Type.object('example.bar.2.1',{ + c: Type.string() + }) + }) +]); + +export const tupleType3 = Type.tuple([ + Type.string(), + Type.bool(), + Type.uint32(), + Type.tuple([ + Type.binary() + ]) +]) + +export const tupleObjectTag = 'tuple-object-wrapper'; + +export const tupleObjectDescription = Type.object(tupleObjectTag, { + tuple1: tupleType1, + tuple1_: tupleType1, + tuple2: tupleType2, + tuple2_: tupleType2, +}); + +export const tupleObjectType3Tag = 'tuple-object-type3-tag'; +export const tupleObjectType3Description = Type.object(tupleObjectType3Tag, { + tuple: tupleType3 +}) diff --git a/javascript/test/tuple.test.ts b/javascript/test/tuple.test.ts new file mode 100644 index 0000000000..bbc3e023cf --- /dev/null +++ b/javascript/test/tuple.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2023 The Fury Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Fury from '../packages/fury/index'; +import { describe, expect, test } from '@jest/globals'; +import { tupleObjectDescription, tupleObjectType3Description } from './fixtures/tuple'; + +describe('tuple', () => { + test('should tuple work', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(tupleObjectDescription); + const tuple1 = [{a: {b:'1'}}, {a: {c: '2'}}] as [{a: {b: string}}, {a: {c: string}}]; + const tuple2 = [{a: {b:'1'}}, {a: {b:'1'}}, {a: {c: '2'}}] as [{a: {b: string}}, {a: {b: string}}, {a: {c: string}}]; + const raw = { + tuple1: tuple1, + tuple1_: tuple1, + tuple2: tuple2, + tuple2_: tuple2 + }; + + const input = serialize(raw); + const result = deserialize( + input + ); + expect(result).toEqual(raw) + }); + + const type3Raw = { + tuple: [ + "1234", + false, + 2333, + [ + Buffer.alloc(10) + ] + ] + }; + + test('tuple support other types', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(tupleObjectType3Description); + + const input = serialize(type3Raw); + const result = deserialize( + input + ); + expect(result).toEqual(type3Raw) + }); + test('tuple will ignore items which index out of bounds', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(tupleObjectType3Description); + const raw = { + tuple: [ + "1234", + false, + 2333, + [ + Buffer.alloc(10) + ], + 1234, + 12345, + false + ] + }; + + const input = serialize(raw); + const result = deserialize( + input + ); + expect(result).toEqual(type3Raw) + }); +})