diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js index eb74e30536..5dd96f0fb1 100644 --- a/javascript/benchmark/index.js +++ b/javascript/benchmark/index.js @@ -20,7 +20,7 @@ const Fury = require("@furyjs/fury"); const utils = require("../test/util"); const hps = require('@furyjs/hps'); -const fury = new Fury.default({ hps, refTracking: false, useLatin1: true, useSliceString: true }); +const fury = new Fury.default({ hps, refTracking: false, useSliceString: true }); const Benchmark = require("benchmark"); const protobuf = require("protobufjs"); const path = require('path'); diff --git a/javascript/benchmark/sample.jpg b/javascript/benchmark/sample.jpg new file mode 100644 index 0000000000..5c98c2d23e Binary files /dev/null and b/javascript/benchmark/sample.jpg differ diff --git a/javascript/packages/fury/index.ts b/javascript/packages/fury/index.ts index fd5d1fa4b5..b305c4574e 100644 --- a/javascript/packages/fury/index.ts +++ b/javascript/packages/fury/index.ts @@ -57,12 +57,19 @@ export default class { serialize: (data: ToRecordType) => { return this.fury.serialize(data, serializer); }, + serializeVolatile: (data: ToRecordType) => { + return this.fury.serializeVolatile(data, serializer); + }, deserialize: (bytes: Uint8Array) => { return this.fury.deserialize(bytes, serializer) as ToRecordType; }, }; } + serializeVolatile(v: any, serialize?: Serializer) { + return this.fury.serializeVolatile(v, serialize); + } + serialize(v: any, serialize?: Serializer) { return this.fury.serialize(v, serialize); } diff --git a/javascript/packages/fury/lib/classResolver.ts b/javascript/packages/fury/lib/classResolver.ts index e2553ad81c..d344e511b4 100644 --- a/javascript/packages/fury/lib/classResolver.ts +++ b/javascript/packages/fury/lib/classResolver.ts @@ -34,12 +34,39 @@ import { BinaryWriter } from "./writer"; const USESTRINGVALUE = 0; const USESTRINGID = 1; +class Lazystring { + private string: string | null = null; + private start: number | null = null; + private len: number | null = null; + + static fromPair(start: number, len: number) { + const result = new Lazystring(); + result.start = start; + result.len = len; + return result; + } + + static fromString(str: string) { + const result = new Lazystring(); + result.string = str; + return result; + } + + toString(binaryReader: BinaryReader) { + if (this.string == null) { + const str = binaryReader.stringUtf8At(this.start!, this.len!); + return str; + } + return this.string; + } +} + export default class SerializerResolver { private internalSerializer: Serializer[] = new Array(300); private customSerializer: { [key: string]: Serializer } = { }; - private readStringPool: string[] = []; + private readStringPool: Lazystring[] = []; private writeStringCount = 0; private writeStringIndex: number[] = []; @@ -139,10 +166,9 @@ export default class SerializerResolver { const flag = binaryReader.uint8(); if (flag === USESTRINGVALUE) { binaryReader.skip(8); // The tag hash is not needed at the moment. - const str = binaryReader.stringUtf8(binaryReader.int16()); - return str; + return binaryReader.stringUtf8(binaryReader.int16()); } else { - return this.readStringPool[binaryReader.int16()]; + return this.readStringPool[binaryReader.int16()].toString(binaryReader); } } @@ -150,11 +176,19 @@ export default class SerializerResolver { const flag = binaryReader.uint8(); if (flag === USESTRINGVALUE) { binaryReader.skip(8); // The tag hash is not needed at the moment. - const str = binaryReader.stringUtf8(binaryReader.int16()); - this.readStringPool.push(str); - return str; + const start = binaryReader.getCursor(); + const len = binaryReader.int16(); + binaryReader.skip(len); + this.readStringPool.push(Lazystring.fromPair(start, len)); + const idx = this.readStringPool.length; + return () => { + return this.readStringPool[idx - 1].toString(binaryReader); + }; } else { - return this.readStringPool[binaryReader.int16()]; + const idx = binaryReader.int16(); + return () => { + return this.readStringPool[idx].toString(binaryReader); + }; } } } diff --git a/javascript/packages/fury/lib/error.ts b/javascript/packages/fury/lib/error.ts new file mode 100644 index 0000000000..2c8d3ed914 --- /dev/null +++ b/javascript/packages/fury/lib/error.ts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class OwnershipError extends Error { + constructor(message: string) { + super(message); + this.name = this.constructor.name; + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor); + } + } +} diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index fc77993944..5c4c3cef4b 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -22,6 +22,7 @@ import { BinaryWriter } from "./writer"; import { BinaryReader } from "./reader"; import { ReferenceResolver } from "./referenceResolver"; import { ConfigFlags, Serializer, Config, InternalSerializerType, Language } from "./type"; +import { OwnershipError } from "./error"; export default (config: Config) => { const binaryReader = BinaryReader(config); @@ -38,6 +39,7 @@ export default (config: Config) => { classResolver, binaryReader, binaryWriter, + serializeVolatile, }; classResolver.init(fury); @@ -71,10 +73,17 @@ export default (config: Config) => { } } - function serialize(data: T, serializer?: Serializer) { + function serializeInternal(data: T, serializer?: Serializer) { + try { + binaryWriter.reset(); + } catch (e) { + if (e instanceof OwnershipError) { + throw new Error("Permission denied. To release the serialization ownership, you must call the dispose function returned by serializeVolatile."); + } + throw e; + } referenceResolver.reset(); classResolver.reset(); - binaryWriter.reset(); let bitmap = 0; if (data === null) { bitmap |= ConfigFlags.isNullFlag; @@ -92,7 +101,16 @@ export default (config: Config) => { classResolver.getSerializerById(InternalSerializerType.ANY).write(data); } binaryWriter.setUint32Position(cursor, binaryWriter.getCursor()); // nativeObjects start offsets; - return binaryWriter.dump(); + return binaryWriter; + } + + function serialize(data: T, serializer?: Serializer) { + return serializeInternal(data, serializer).dump(); + } + + function serializeVolatile(data: T, serializer?: Serializer) { + return serializeInternal(data, serializer).dumpAndOwn(); } + return fury; }; diff --git a/javascript/packages/fury/lib/internalSerializer/number.ts b/javascript/packages/fury/lib/internalSerializer/number.ts index 7d0789fd09..87cd6b148e 100644 --- a/javascript/packages/fury/lib/internalSerializer/number.ts +++ b/javascript/packages/fury/lib/internalSerializer/number.ts @@ -21,7 +21,7 @@ import { InternalSerializerType, RefFlags, Fury } from "../type"; export const uInt8Serializer = (fury: Fury) => { const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, uint8: writeUInt8 } = binaryWriter; + const { uint8: writeUInt8 } = binaryWriter; const { uint8: readUInt8 } = binaryReader; return { ...referenceResolver.deref(() => { @@ -107,7 +107,7 @@ export const int8Serializer = (fury: Fury) => { export const uInt16Serializer = (fury: Fury) => { const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, uint16: writeUInt16 } = binaryWriter; + const { uint16: writeUInt16 } = binaryWriter; const { uint16: readUInt16 } = binaryReader; return { @@ -151,7 +151,7 @@ export const int16Serializer = (fury: Fury) => { export const uInt32Serializer = (fury: Fury) => { const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, uint32: writeUInt32 } = binaryWriter; + const { uint32: writeUInt32 } = binaryWriter; const { uint32: readUInt32 } = binaryReader; return { @@ -195,7 +195,7 @@ export const int32Serializer = (fury: Fury) => { export const uInt64Serializer = (fury: Fury) => { const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, uint64: writeUInt64 } = binaryWriter; + const { uint64: writeUInt64 } = binaryWriter; const { uint64: readUInt64 } = binaryReader; return { diff --git a/javascript/packages/fury/lib/reader.ts b/javascript/packages/fury/lib/reader.ts index 80e479d909..6b275f1d32 100644 --- a/javascript/packages/fury/lib/reader.ts +++ b/javascript/packages/fury/lib/reader.ts @@ -99,6 +99,10 @@ export const BinaryReader = (config: Config) => { return result; } + function stringUtf8At(start: number, len: number) { + return buffer.utf8Slice(start, start + len); + } + function stringUtf8(len: number) { const result = buffer.utf8Slice(cursor, cursor + len); cursor += len; @@ -106,9 +110,9 @@ export const BinaryReader = (config: Config) => { } function stringOfVarInt32() { - const useLatin1 = config.useLatin1 ? uint8() === LATIN1 : false; + const isLatin1 = uint8() === LATIN1; const len = varInt32(); - return useLatin1 ? stringLatin1(len) : stringUtf8(len); + return isLatin1 ? stringLatin1(len) : stringUtf8(len); } function stringLatin1Fast(len: number) { @@ -167,6 +171,7 @@ export const BinaryReader = (config: Config) => { bufferRef, uint8, reset, + stringUtf8At, stringUtf8, stringLatin1, stringOfVarInt32, diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 2d6606b297..39cd50669f 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -109,7 +109,6 @@ export interface Hps { export interface Config { hps?: Hps refTracking?: boolean - useLatin1?: boolean useSliceString?: boolean } diff --git a/javascript/packages/fury/lib/writer.ts b/javascript/packages/fury/lib/writer.ts index 01f11f644b..57ea680f86 100644 --- a/javascript/packages/fury/lib/writer.ts +++ b/javascript/packages/fury/lib/writer.ts @@ -19,6 +19,7 @@ import { Config, LATIN1, UTF8 } from "./type"; import { PlatformBuffer, alloc, strByteLength } from "./platformBuffer"; +import { OwnershipError } from "./error"; const MAX_POOL_SIZE = 1024 * 1024 * 3; // 3MB @@ -28,6 +29,7 @@ export const BinaryWriter = (config: Config) => { let arrayBuffer: PlatformBuffer; let dataView: DataView; let reserved = 0; + let locked = false; function initPoll() { byteLength = 1024 * 100; @@ -49,6 +51,9 @@ export const BinaryWriter = (config: Config) => { } function reset() { + if (locked) { + throw new OwnershipError("Ownership of writer was held by dumpAndOwn, but not released"); + } cursor = 0; reserved = 0; } @@ -170,9 +175,7 @@ export const BinaryWriter = (config: Config) => { return function (v: string) { const isLatin1 = detectIsLatin1(v); const len = isLatin1 ? v.length : strByteLength(v); - if (config.useLatin1) { - dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); - } + dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); varInt32(len); reserve(len); if (isLatin1) { @@ -191,9 +194,7 @@ export const BinaryWriter = (config: Config) => { function stringOfVarInt32Slow(v: string) { const len = strByteLength(v); const isLatin1 = len === v.length; - if (config.useLatin1) { - dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); - } + dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); varInt32(len); reserve(len); if (isLatin1) { @@ -236,6 +237,18 @@ export const BinaryWriter = (config: Config) => { return result; } + function dumpAndOwn() { + locked = true; + return { + get() { + return arrayBuffer.subarray(0, cursor); + }, + dispose() { + locked = false; + }, + }; + } + function getCursor() { return cursor; } @@ -278,5 +291,6 @@ export const BinaryWriter = (config: Config) => { int32, getCursor, setUint32Position, + dumpAndOwn, }; }; diff --git a/javascript/test/array.test.ts b/javascript/test/array.test.ts index 98c36af159..ea9c7d2217 100644 --- a/javascript/test/array.test.ts +++ b/javascript/test/array.test.ts @@ -122,7 +122,7 @@ describe('array', () => { } }; - const fury = new Fury({ refTracking: true, useLatin1: true }); + const fury = new Fury({ refTracking: true }); const serializer = fury.registerSerializer(description).serializer; const input = fury.serialize({ a7: ["hello", "world", null] diff --git a/javascript/test/io.test.ts b/javascript/test/io.test.ts index 03487315f2..040c9c7e2c 100644 --- a/javascript/test/io.test.ts +++ b/javascript/test/io.test.ts @@ -30,11 +30,7 @@ function num2Bin(num: number) { [ { - useLatin1: true, useSliceString: true, - }, - { - useLatin1: false, } ].forEach((config: Config) => { describe('writer', () => { @@ -186,9 +182,7 @@ function num2Bin(num: number) { const ab = writer.dump(); const reader = BinaryReader(config); reader.reset(ab); - if (config.useLatin1) { - expect(reader.uint8()).toBe(0); - } + expect(reader.uint8()).toBe(0); const len = reader.varInt32(); expect(len).toBe(11); const str = reader.stringLatin1(11); @@ -202,9 +196,7 @@ function num2Bin(num: number) { const ab = writer.dump(); const reader = BinaryReader(config); reader.reset(ab); - if (config.useLatin1) { - expect(reader.uint8()).toBe(0); - } + expect(reader.uint8()).toBe(0); const len = reader.varInt32(); expect(len).toBe(110); expect(reader.stringLatin1(len)).toBe(str); @@ -219,9 +211,7 @@ function num2Bin(num: number) { { reader.reset(ab); - if (config.useLatin1) { - expect(reader.uint8()).toBe(1); - } + expect(reader.uint8()).toBe(1); const len = reader.varInt32(); expect(len).toBe(17); expect(reader.stringUtf8(len)).toBe(str); @@ -240,9 +230,7 @@ function num2Bin(num: number) { const reader = BinaryReader(config); { reader.reset(ab); - if (config.useLatin1) { - expect(reader.uint8()).toBe(1); - } + expect(reader.uint8()).toBe(1); const len = reader.varInt32(); expect(len).toBe(170); expect(reader.stringUtf8(len)).toBe(str); diff --git a/javascript/test/protocol/struct.test.ts b/javascript/test/protocol/struct.test.ts index 4e5416330b..0dd3141919 100644 --- a/javascript/test/protocol/struct.test.ts +++ b/javascript/test/protocol/struct.test.ts @@ -43,58 +43,6 @@ describe('protocol', () => { const result = deserialize(bf); expect(result).toEqual(obj); }); - test('should py bin work', () => { - - const fury = new Fury({ refTracking: true }); - const { deserialize } = fury.registerSerializer({ - type: InternalSerializerType.FURY_TYPE_TAG, - options: { - tag: "example.ComplexObject", - props: { - f1: Type.string(), - f2: Type.map(Type.string(), Type.any()), - f3: Type.int8(), - f4: Type.int16(), - f5: Type.int32(), - f6: Type.int64(), - f7: Type.float(), - f8: Type.double(), - f9: Type.array(Type.int16()), - f10: Type.map(Type.int32(), Type.double()), - } - } - }); - - const obj = deserialize(new Uint8Array([ - 134, 2, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 159, 160, 124, 69, 240, 2, 120, 21, 0, - 101, 120, 97, 109, 112, 108, 101, 46, 67, 111, 109, 112, 108, 101, 120, 79, 98, 106, 101, - 99, 116, 71, 168, 32, 21, 0, 13, 0, 3, 115, 116, 114, 0, 30, 0, 2, 255, 7, 0, 1, 0, 0, 0, - 255, 12, 0, 85, 85, 85, 85, 85, 85, 213, 63, 255, 7, 0, 100, 0, 0, 0, 255, 12, 0, 146, 36, - 73, 146, 36, 73, 210, 63, 0, 30, 0, 2, 0, 13, 0, 2, 107, 49, 255, 3, 0, 255, 0, 13, 0, 2, - 107, 50, 255, 3, 0, 2, 255, 3, 0, 127, 255, 5, 0, 255, 127, 255, 7, 0, 255, 255, 255, 127, - 255, 9, 0, 255, 255, 255, 255, 255, 255, 255, 127, 255, 11, 0, 0, 0, 0, 63, 255, 12, 0, 85, - 85, 85, 85, 85, 85, 229, 63, 0, 25, 0, 2, 255, 5, 0, 1, 0, 255, 5, 0, 2, 0, 134, 2, 98, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 159, 160, 124, 69, 240, 2, 120, 21, 0, 101, 120, 97, 109, - 112, 108, 101, 46, 67, 111, 109, 112, 108, 101, 120, 79, 98, 106, 101, 99, 116, 71, 168, - 32, 21, 253, 253, 253, 255, 3, 0, 0, 255, 5, 0, 0, 0, 255, 7, 0, 0, 0, 0, 0, 255, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 255, 11, 0, 171, 170, 170, 62, 255, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, - ])); - obj.f6 = Number(obj.f6) - - expect(obj).toEqual({ - f1: "str", - f10: new Map([[1, 1 / 3], [100, 2 / 7]]), - f2: new Map([['k1', -1], ['k2', 2]]), - f3: 2**7 - 1, - f4: 2**15 - 1, - f5: 2**31 - 1, - f6: 2**63 - 1, - f7: 1 / 2, - f8: 2 / 3, - f9: [1, 2] - }) - }); }); diff --git a/javascript/test/reader.test.ts b/javascript/test/reader.test.ts index c627d04529..0e0e6dc699 100644 --- a/javascript/test/reader.test.ts +++ b/javascript/test/reader.test.ts @@ -28,11 +28,6 @@ const hps = process.env.enableHps ? require('@furyjs/hps') : null; [ { - useLatin1: true, - hps, - }, - { - useLatin1: false, hps, } ].forEach((config: Config) => { diff --git a/javascript/test/string.test.ts b/javascript/test/string.test.ts index 4aefb8f8bc..a736f7925c 100644 --- a/javascript/test/string.test.ts +++ b/javascript/test/string.test.ts @@ -23,11 +23,9 @@ const hps = require('@furyjs/hps'); [ { hps, - useLatin1: true }, { hps: null, - useLatin1: false } ].forEach(config => { describe('string', () => { diff --git a/javascript/test/writer.test.ts b/javascript/test/writer.test.ts new file mode 100644 index 0000000000..caf9e599ff --- /dev/null +++ b/javascript/test/writer.test.ts @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { OwnershipError } from '@furyjs/fury/lib/error'; +import { BinaryWriter } from '@furyjs/fury/lib/writer'; +import { describe, expect, test } from '@jest/globals'; + +describe('writer', () => { + test('should dumpOwn dispose work', () => { + const writer = BinaryWriter({}); + { + writer.uint8(256); + const { get, dispose } = writer.dumpAndOwn(); + const ab = get(); + expect(ab.byteLength).toBe(1); + expect(ab[0]).toBe(0); + expect(writer.getCursor()).toBe(1); + dispose(); + } + writer.reset(); + { + writer.uint8(256); + const { get, dispose } = writer.dumpAndOwn(); + const ab = get(); + expect(ab.byteLength).toBe(1); + expect(ab[0]).toBe(0); + expect(writer.getCursor()).toBe(1); + dispose(); + } + }); + + test('should dumpOwn work', () => { + const writer = BinaryWriter({}); + { + writer.uint8(256); + const { get } = writer.dumpAndOwn(); + const ab = get(); + expect(ab.byteLength).toBe(1); + expect(ab[0]).toBe(0); + expect(writer.getCursor()).toBe(1); + } + try { + writer.reset(); + } catch (error) { + expect(error instanceof OwnershipError).toBe(true); + return; + } + throw new Error("unreachable code") + }); +}); \ No newline at end of file