Merge branch 'main' into assertion-verification-cli

lib/tdf3/src/utils/chunkers.ts → lib/src/seekable.ts

@@ -1,30 +1,52 @@
-import {
-  type DecoratedReadableStream,
-  isDecoratedReadableStream,
-} from '../client/DecoratedReadableStream.js';
-import { ConfigurationError, InvalidFileError, NetworkError } from '../../../src/errors.js';
+import { ConfigurationError, InvalidFileError, NetworkError } from './errors.js';
 
 /**
  * Read data from a seekable stream.
+ * This is an abstraction for URLs with range queries and local file objects.
  * @param byteStart First byte to read. If negative, reads from the end. If absent, reads everything
  * @param byteEnd Index after last byte to read (exclusive)
  */
 export type Chunker = (byteStart?: number, byteEnd?: number) => Promise<Uint8Array>;
 
+/**
+ * Type union for a variety of inputs.
+ */
+export type Source =
+  | { type: 'buffer'; location: Uint8Array }
+  | { type: 'chunker'; location: Chunker }
+  | { type: 'file-browser'; location: Blob }
+  | { type: 'remote'; location: string }
+  | { type: 'stream'; location: ReadableStream<Uint8Array> };
+
+/**
+ * Creates a seekable object from a browser file object.
+ * @param fileRef the browser file data
+ */
 export const fromBrowserFile = (fileRef: Blob): Chunker => {
   return async (byteStart?: number, byteEnd?: number): Promise<Uint8Array> => {
+    if (byteStart === undefined) {
+      return new Uint8Array(await fileRef.arrayBuffer());
+    }
     const chunkBlob = fileRef.slice(byteStart, byteEnd);
     const arrayBuffer = await new Response(chunkBlob).arrayBuffer();
     return new Uint8Array(arrayBuffer);
   };
 };
 
-export const fromBuffer = (source: Uint8Array | Buffer): Chunker => {
+export const fromBuffer = (source: Uint8Array): Chunker => {
   return (byteStart?: number, byteEnd?: number) => {
     return Promise.resolve(source.slice(byteStart, byteEnd));
   };
 };
 
+export const fromString = (source: string): Chunker => {
+  return fromBuffer(new TextEncoder().encode(source));
+};
+
+async function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function getRemoteChunk(url: string, range?: string): Promise<Uint8Array> {
   // loop with fetch for three times, with an exponential backoff
   // if the fetch fails with a network error
@@ -88,14 +110,7 @@ export const fromUrl = async (location: string): Promise<Chunker> => {
   };
 };
 
-export type DataSource =
-  | { type: 'buffer'; location: Uint8Array }
-  | { type: 'chunker'; location: Chunker }
-  | { type: 'file-browser'; location: Blob }
-  | { type: 'remote'; location: string }
-  | { type: 'stream'; location: DecoratedReadableStream };
-
-export const fromDataSource = async ({ type, location }: DataSource) => {
+export const fromSource = async ({ type, location }: Source): Promise<Chunker> => {
   switch (type) {
     case 'buffer':
       if (!(location instanceof Uint8Array)) {
@@ -118,14 +133,48 @@ export const fromDataSource = async ({ type, location }: DataSource) => {
       }
       return fromUrl(location);
     case 'stream':
-      if (!isDecoratedReadableStream(location)) {
-        throw new ConfigurationError('Invalid data source; must be DecoratedTdfStream');
-      }
-      return fromBuffer(await location.toBuffer());
+      return fromBuffer(new Uint8Array(await new Response(location).arrayBuffer()));
     default:
       throw new ConfigurationError(`Data source type not defined, or not supported: ${type}}`);
   }
 };
-async function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+
+export async function sourceToStream(source: Source): Promise<ReadableStream<Uint8Array>> {
+  switch (source.type) {
+    case 'stream':
+      return source.location;
+    case 'file-browser':
+      return source.location.stream();
+    case 'chunker': {
+      const chunkSize = 8 * 1024 * 1024; // 8 megabytes
+      let offset = 0;
+      return new ReadableStream({
+        async pull(controller) {
+          const chunk = await source.location(offset, offset + chunkSize);
+          if (chunk.length === 0) {
+            controller.close();
+            return;
+          }
+          controller.enqueue(chunk);
+          offset += chunk.length;
+        },
+      });
+    }
+    default: {
+      const chunker = await fromSource(source);
+      return new ReadableStream({
+        async start(controller) {
+          const chunk = await chunker();
+          controller.enqueue(chunk);
+          controller.close();
+        },
+      });
+    }
+  }
 }
+
+// Deprected name, prefer `fromSource`
+export const fromDataSource = fromSource;
+
+// Deprecated Name; prefer just `Source`
+export type DataSource = Source;

lib/src/utils.ts

@@ -1,7 +1,7 @@
 import { exportSPKI, importX509 } from 'jose';
 
 import { base64 } from './encodings/index.js';
-import { pemCertToCrypto, pemPublicToCrypto } from './nanotdf-crypto/index.js';
+import { pemCertToCrypto, pemPublicToCrypto } from './nanotdf-crypto/pemPublicToCrypto.js';
 import { ConfigurationError } from './errors.js';
 
 /**

lib/tdf3/index.ts

@@ -35,7 +35,7 @@ import {
   clientType,
 } from '../src/index.js';
 import { Algorithms, type AlgorithmName, type AlgorithmUrn } from './src/ciphers/algorithms.js';
-import { type Chunker } from './src/utils/chunkers.js';
+import { type Chunker } from '../src/seekable.js';
 
 export type {
   AlgorithmName,

lib/tdf3/src/client/builders.ts

@@ -4,9 +4,9 @@ import { type Metadata } from '../tdf.js';
 import { Binary } from '../binary.js';
 
 import { ConfigurationError } from '../../../src/errors.js';
+import { type Chunker } from '../../../src/seekable.js';
 import { PemKeyPair } from '../crypto/declarations.js';
 import { DecoratedReadableStream } from './DecoratedReadableStream.js';
-import { type Chunker } from '../utils/chunkers.js';
 import { AssertionConfig, AssertionVerificationKeys } from '../assertions.js';
 import { Value } from '../../../src/policy/attributes.js';
 

lib/tdf3/src/client/index.ts

@@ -1,10 +1,7 @@
 import { v4 } from 'uuid';
 import {
   ZipReader,
-  fromBuffer,
-  fromDataSource,
   streamToBuffer,
-  type Chunker,
   keyMiddleware as defaultKeyMiddleware,
 } from '../utils/index.js';
 import { base64 } from '../../../src/encodings/index.js';
@@ -44,6 +41,7 @@ import {
 } from './builders.js';
 import { KasPublicKeyInfo, OriginAllowList } from '../../../src/access.js';
 import { ConfigurationError } from '../../../src/errors.js';
+import { type Chunker, fromBuffer, fromDataSource, fromSource } from '../../../src/seekable.js';
 import { Binary } from '../binary.js';
 import { AesGcmCipher } from '../ciphers/aes-gcm-cipher.js';
 import { toCryptoKeyPair } from '../crypto/crypto-utils.js';
@@ -82,7 +80,7 @@ const makeChunkable = async (source: DecryptSource) => {
       initialChunker = source.location;
       break;
     default:
-      initialChunker = await fromDataSource(source);
+      initialChunker = await fromSource(source);
   }
 
   const magic: string = await getFirstTwoBytes(initialChunker);
@@ -564,5 +562,6 @@ export {
   EncryptParamsBuilder,
   HttpRequest,
   fromDataSource,
+  fromSource,
   withHeaders,
 };

lib/tdf3/src/tdf.ts

@@ -20,7 +20,6 @@ import {
 } from './models/index.js';
 import { base64 } from '../../src/encodings/index.js';
 import {
-  type Chunker,
   ZipReader,
   ZipWriter,
   base64ToBuffer,
@@ -30,6 +29,7 @@ import {
 } from './utils/index.js';
 import { Binary } from './binary.js';
 import { KasPublicKeyAlgorithm, KasPublicKeyInfo, OriginAllowList } from '../../src/access.js';
+import { allPool, anyPool } from '../../src/concurrency.js';
 import {
   ConfigurationError,
   DecryptError,
@@ -39,6 +39,7 @@ import {
   UnsafeUrlError,
   UnsupportedFeatureError as UnsupportedError,
 } from '../../src/errors.js';
+import { type Chunker } from '../../src/seekable.js';
 import { htmlWrapperTemplate } from './templates/index.js';
 
 // configurable
@@ -49,7 +50,6 @@ import { PolicyObject } from '../../src/tdf/PolicyObject.js';
 import { type CryptoService, type DecryptResult } from './crypto/declarations.js';
 import { CentralDirectory } from './utils/zip-reader.js';
 import { SymmetricCipher } from './ciphers/symmetric-cipher-base.js';
-import { allPool, anyPool } from '../../src/concurrency.js';
 
 // TODO: input validation on manifest JSON
 const DEFAULT_SEGMENT_SIZE = 1024 * 1024;

lib/tdf3/src/utils/index.ts

@@ -9,7 +9,6 @@ export { ZipReader, readUInt64LE } from './zip-reader.js';
 export { ZipWriter } from './zip-writer.js';
 export { keySplit, keyMerge } from './keysplit.js';
 export { streamToBuffer } from '../client/DecoratedReadableStream.js';
-export * from './chunkers.js';
 
 export type SupportedEncoding = 'hex' | 'utf8' | 'utf-8' | 'binary' | 'latin1' | 'base64';
 

lib/tdf3/src/utils/zip-reader.ts

@@ -1,6 +1,7 @@
 import { InvalidFileError } from '../../../src/errors.js';
 import { Manifest } from '../models/index.js';
-import { Chunker } from './chunkers.js';
+import { type Chunker } from '../../../src/seekable.js';
+
 import { readUInt32LE, readUInt16LE, copyUint8Arr, buffToString } from './index.js';
 
 // TODO: Better document what these constants are

lib/tests/mocha/unit/crypto-di.spec.ts

@@ -6,7 +6,7 @@ import {
   type DecryptResult,
   type EncryptResult,
   type PemKeyPair,
-} from 'tdf3/index.js';
+} from '../../../tdf3/index.js';
 import { Client } from '../../../tdf3/src/client/index.js';
 
 describe('CryptoService DI', () => {