chore(sdk): Renames chunker package to seekable

I didn't rename chunker itself though. Seems unnecessary and will require more changes outside of the library. however, this is an excuse to move some code out of tdf3 into src, and unify it with the `DataSource` type, renamed to just `Source`, and add some helper/transformer methods.

lib/tdf3/src/utils/chunkers.ts → lib/src/seekable.ts

@@ -1,30 +1,52 @@
-import {
-  type DecoratedReadableStream,
-  isDecoratedReadableStream,
-} from '../client/DecoratedReadableStream.js';
-import { ConfigurationError, InvalidFileError, NetworkError } from '../../../src/errors.js';
+import { ConfigurationError, InvalidFileError, NetworkError } from './errors.js';
 
 /**
  * Read data from a seekable stream.
+ * This is an abstraction for URLs with range queries and local file objects.
  * @param byteStart First byte to read. If negative, reads from the end. If absent, reads everything
  * @param byteEnd Index after last byte to read (exclusive)
  */
 export type Chunker = (byteStart?: number, byteEnd?: number) => Promise<Uint8Array>;
 
+/**
+ * Type union for a variety of inputs.
+ */
+export type Source =
+  | { type: 'buffer'; location: Uint8Array }
+  | { type: 'chunker'; location: Chunker }
+  | { type: 'file-browser'; location: Blob }
+  | { type: 'remote'; location: string }
+  | { type: 'stream'; location: ReadableStream<Uint8Array> };
+
+/**
+ * Creates a seekable object from a browser file object.
+ * @param fileRef the browser file data
+ */
 export const fromBrowserFile = (fileRef: Blob): Chunker => {
   return async (byteStart?: number, byteEnd?: number): Promise<Uint8Array> => {
+    if (byteStart === undefined) {
+      return new Uint8Array(await fileRef.arrayBuffer());
+    }
     const chunkBlob = fileRef.slice(byteStart, byteEnd);
     const arrayBuffer = await new Response(chunkBlob).arrayBuffer();
     return new Uint8Array(arrayBuffer);
   };
 };
 
-export const fromBuffer = (source: Uint8Array | Buffer): Chunker => {
+export const fromBuffer = (source: Uint8Array): Chunker => {
   return (byteStart?: number, byteEnd?: number) => {
     return Promise.resolve(source.slice(byteStart, byteEnd));
   };
 };
 
+export const fromString = (source: string): Chunker => {
+  return fromBuffer(new TextEncoder().encode(source));
+};
+
+async function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function getRemoteChunk(url: string, range?: string): Promise<Uint8Array> {
   // loop with fetch for three times, with an exponential backoff
   // if the fetch fails with a network error
@@ -88,14 +110,7 @@ export const fromUrl = async (location: string): Promise<Chunker> => {
   };
 };
 
-export type DataSource =
-  | { type: 'buffer'; location: Uint8Array }
-  | { type: 'chunker'; location: Chunker }
-  | { type: 'file-browser'; location: Blob }
-  | { type: 'remote'; location: string }
-  | { type: 'stream'; location: DecoratedReadableStream };
-
-export const fromDataSource = async ({ type, location }: DataSource) => {
+export const fromSource = async ({ type, location }: Source): Promise<Chunker> => {
   switch (type) {
     case 'buffer':
       if (!(location instanceof Uint8Array)) {
@@ -118,14 +133,48 @@ export const fromDataSource = async ({ type, location }: DataSource) => {
       }
       return fromUrl(location);
     case 'stream':
-      if (!isDecoratedReadableStream(location)) {
-        throw new ConfigurationError('Invalid data source; must be DecoratedTdfStream');
-      }
-      return fromBuffer(await location.toBuffer());
+      return fromBuffer(new Uint8Array(await new Response(location).arrayBuffer()));
     default:
       throw new ConfigurationError(`Data source type not defined, or not supported: ${type}}`);
   }
 };
-async function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+
+export async function sourceToStream(source: Source): Promise<ReadableStream<Uint8Array>> {
+  switch (source.type) {
+    case 'stream':
+      return source.location;
+    case 'file-browser':
+      return source.location.stream();
+    case 'chunker': {
+      const chunkSize = 8 * 1024 * 1024; // 8 megabytes
+      let offset = 0;
+      return new ReadableStream({
+        async pull(controller) {
+          const chunk = await source.location(offset, offset + chunkSize);
+          if (chunk.length === 0) {
+            controller.close();
+            return;
+          }
+          controller.enqueue(chunk);
+          offset += chunk.length;
+        },
+      });
+    }
+    default: {
+      const chunker = await fromSource(source);
+      return new ReadableStream({
+        async start(controller) {
+          const chunk = await chunker();
+          controller.enqueue(chunk);
+          controller.close();
+        },
+      });
+    }
+  }
 }
+
+// Deprected name, prefer `fromSource`
+export const fromDataSource = fromSource;
+
+// Deprecated Name; prefer just `Source`
+export type DataSource = Source;

lib/src/utils.ts

@@ -1,7 +1,7 @@
 import { exportSPKI, importX509 } from 'jose';
 
 import { base64 } from './encodings/index.js';
-import { pemCertToCrypto, pemPublicToCrypto } from './nanotdf-crypto/index.js';
+import { pemCertToCrypto, pemPublicToCrypto } from './nanotdf-crypto/pemPublicToCrypto.js';
 import { ConfigurationError } from './errors.js';
 
 /**

lib/tdf3/index.ts

@@ -35,7 +35,7 @@ import {
   clientType,
 } from '../src/index.js';
 import { Algorithms, type AlgorithmName, type AlgorithmUrn } from './src/ciphers/algorithms.js';
-import { type Chunker } from './src/utils/chunkers.js';
+import { type Chunker } from '../src/seekable.js';
 
 export type {
   AlgorithmName,

lib/tdf3/src/client/builders.ts

@@ -4,9 +4,9 @@ import { type Metadata } from '../tdf.js';
 import { Binary } from '../binary.js';
 
 import { ConfigurationError } from '../../../src/errors.js';
+import { type Chunker } from '../../../src/seekable.js';
 import { PemKeyPair } from '../crypto/declarations.js';
 import { DecoratedReadableStream } from './DecoratedReadableStream.js';
-import { type Chunker } from '../utils/chunkers.js';
 import { AssertionConfig, AssertionVerificationKeys } from '../assertions.js';
 import { Value } from '../../../src/policy/attributes.js';
 

lib/tdf3/src/client/index.ts

@@ -1,10 +1,7 @@
 import { v4 } from 'uuid';
 import {
   ZipReader,
-  fromBuffer,
-  fromDataSource,
   streamToBuffer,
-  type Chunker,
   keyMiddleware as defaultKeyMiddleware,
 } from '../utils/index.js';
 import { base64 } from '../../../src/encodings/index.js';
@@ -43,6 +40,7 @@ import {
 } from './builders.js';
 import { KasPublicKeyInfo, OriginAllowList } from '../../../src/access.js';
 import { ConfigurationError } from '../../../src/errors.js';
+import { type Chunker, fromBuffer, fromDataSource, fromSource } from '../../../src/seekable.js';
 import { Binary } from '../binary.js';
 import { AesGcmCipher } from '../ciphers/aes-gcm-cipher.js';
 import { toCryptoKeyPair } from '../crypto/crypto-utils.js';
@@ -80,7 +78,7 @@ const makeChunkable = async (source: DecryptSource) => {
       initialChunker = source.location;
       break;
     default:
-      initialChunker = await fromDataSource(source);
+      initialChunker = await fromSource(source);
   }
 
   const magic: string = await getFirstTwoBytes(initialChunker);
@@ -548,5 +546,6 @@ export {
   EncryptParamsBuilder,
   HttpRequest,
   fromDataSource,
+  fromSource,
   withHeaders,
 };

lib/tdf3/src/tdf.ts

@@ -20,7 +20,6 @@ import {
 } from './models/index.js';
 import { base64 } from '../../src/encodings/index.js';
 import {
-  type Chunker,
   ZipReader,
   ZipWriter,
   keyMerge,
@@ -29,6 +28,7 @@ import {
 } from './utils/index.js';
 import { Binary } from './binary.js';
 import { KasPublicKeyAlgorithm, KasPublicKeyInfo, OriginAllowList } from '../../src/access.js';
+import { allPool, anyPool } from '../../src/concurrency.js';
 import {
   ConfigurationError,
   DecryptError,
@@ -38,6 +38,7 @@ import {
   UnsafeUrlError,
   UnsupportedFeatureError as UnsupportedError,
 } from '../../src/errors.js';
+import { type Chunker } from '../../src/seekable.js';
 
 // configurable
 // TODO: remove dependencies from ciphers so that we can open-source instead of relying on other Virtru libs
@@ -47,7 +48,6 @@ import { PolicyObject } from '../../src/tdf/PolicyObject.js';
 import { type CryptoService, type DecryptResult } from './crypto/declarations.js';
 import { CentralDirectory } from './utils/zip-reader.js';
 import { SymmetricCipher } from './ciphers/symmetric-cipher-base.js';
-import { allPool, anyPool } from '../../src/concurrency.js';
 
 // TODO: input validation on manifest JSON
 const DEFAULT_SEGMENT_SIZE = 1024 * 1024;

lib/tdf3/src/utils/index.ts

@@ -9,7 +9,6 @@ export { ZipReader, readUInt64LE } from './zip-reader.js';
 export { ZipWriter } from './zip-writer.js';
 export { keySplit, keyMerge } from './keysplit.js';
 export { streamToBuffer } from '../client/DecoratedReadableStream.js';
-export * from './chunkers.js';
 
 export type SupportedEncoding = 'hex' | 'utf8' | 'utf-8' | 'binary' | 'latin1' | 'base64';
 

lib/tdf3/src/utils/zip-reader.ts

@@ -1,6 +1,7 @@
 import { InvalidFileError } from '../../../src/errors.js';
 import { Manifest } from '../models/index.js';
-import { Chunker } from './chunkers.js';
+import { type Chunker } from '../../../src/seekable.js';
+
 import { readUInt32LE, readUInt16LE, copyUint8Arr, buffToString } from './index.js';
 
 // TODO: Better document what these constants are

lib/tests/mocha/unit/crypto-di.spec.ts

@@ -6,7 +6,7 @@ import {
   type DecryptResult,
   type EncryptResult,
   type PemKeyPair,
-} from 'tdf3/index.js';
+} from '../../../tdf3/index.js';
 import { Client } from '../../../tdf3/src/client/index.js';
 
 describe('CryptoService DI', () => {