feat: add support for optional unixfs file link writer in create file

README.md

@@ -129,6 +129,88 @@ const demo = async blob => {
 }
 ```
 
+### Collecting UnixFS FileLinks
+
+You can optionally pass a unixFsFileLinkWriter stream to capture metadata for each link (useful for indexing or tracking layout information).
+
+```js
+import {
+  createWriter,
+  createFileWriter,
+} from '@ipld/unixfs'
+
+import { withMaxChunkSize } from '@ipld/unixfs/file/chunker/fixed'
+import { withWidth } from '@ipld/unixfs/file/layout/balanced'
+
+const defaultSettings = UnixFS.configure({
+  fileChunkEncoder: raw,
+  smallFileEncoder: raw,
+  chunker: withMaxChunkSize(1024 * 1024),
+  fileLayout: withWidth(1024),
+})
+
+/**
+ * @param {Blob} blob
+ * @returns {Promise<import('@vascosantos/unixfs').FileLink[]>}
+ */
+async function collectUnixFsFileLinks(blob) {
+  const fileLinks = []
+
+  // Create a stream to collect metadata (FileLinks)
+  const { readable, writable } = new TransformStream()
+
+  // Set up the main UnixFS writer (data goes nowhere here)
+  const unixfsWriter = createWriter({
+    writable: new WritableStream(), // Discard actual DAG output
+    settings: defaultSettings,
+  })
+
+  // Set up the file writer with link metadata writer
+  const unixFsFileLinkWriter = writable.getWriter()
+
+  const fileWriter = createFileWriter({
+    ...unixfsWriter,
+    initOptions: {
+      unixFsFileLinkWriter,
+    },
+  })
+
+  // Start concurrent reading of the metadata stream
+  const fileLinkReader = readable.getReader()
+  const readLinks = (async () => {
+    while (true) {
+      const { done, value } = await fileLinkReader.read()
+      if (done) break
+      fileLinks.push(value)
+    }
+  })()
+
+  // Pipe the blob to the file writer
+  await blob.stream().pipeTo(
+    new WritableStream({
+      async write(chunk) {
+        await fileWriter.write(chunk)
+      },
+    })
+  )
+
+  // Finalize everything
+  await fileWriter.close()
+  await unixfsWriter.close()
+  await unixFsFileLinkWriter.close()
+
+  // Wait for all links to be read
+  await readLinks
+
+  return fileLinks
+}
+
+// Usage
+const blob = new Blob(['Hello UnixFS links'])
+const links = await collectUnixFsFileLinks(blob)
+console.log(links)
+```
+
 ## License
 
 Licensed under either of
@@ -144,4 +226,4 @@ Unless you explicitly state otherwise, any contribution intentionally submitted
 [readablestream]: https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream
 [car]: https://ipld.io/specs/transport/car/carv1/
 [`transformstream`]: https://developer.mozilla.org/en-US/docs/Web/API/TransformStream
-[`writablestream`]: https://developer.mozilla.org/en-US/docs/Web/API/WritableStream
+[`writablestream`]: https://developer.mozilla.org/en-US/docs/Web/API/WritableStream

src/api.ts

@@ -23,7 +23,7 @@ import type {
   Options as DirectoryWriterOptions,
   State as DirectoryWriterState,
 } from "./directory.js"
-import { Metadata } from "./unixfs.js"
+import { Metadata, FileLink } from "./unixfs.js"
 
 export type {
   WriterOptions,
@@ -47,6 +47,7 @@ export type {
   MultihashHasher,
   MultihashDigest,
   Metadata,
+  FileLink,
 }
 
 /**

src/file.js

@@ -28,7 +28,7 @@ export const defaults = () => ({
  * @param {Partial<API.EncoderSettings<Layout>>} config
  * @returns {API.EncoderSettings<Layout>}
  */
-export const configure = config => ({
+export const configure = (config) => ({
   ...defaults(),
   ...config,
 })
@@ -50,8 +50,15 @@ export const UnixFSRawLeaf = {
  * @param {API.Options<Layout>} options
  * @returns {API.View<Layout>}
  */
-export const create = ({ writer, metadata = {}, settings = defaults() }) =>
-  new FileWriterView(Writer.init(writer, metadata, configure(settings)))
+export const create = ({
+  writer,
+  metadata = {},
+  settings = defaults(),
+  initOptions = {},
+}) =>
+  new FileWriterView(
+    Writer.init(writer, metadata, configure(settings), initOptions)
+  )
 
 /**
  * @template T
@@ -98,7 +105,7 @@ export const close = async (
  */
 const perform = (view, effect) =>
   Task.fork(
-    Task.loop(effect, message => {
+    Task.loop(effect, (message) => {
       const { state, effect } = Writer.update(message, view.state)
       view.state = state
       return effect

src/file/api.ts

@@ -72,10 +72,17 @@ export interface EncoderSettings<Layout extends unknown = unknown> {
   linker: Linker
 }
 
+export interface InitOptions {
+  unixFsFileLinkWriter?: UnixFsFileLinkWriter
+}
+
+export interface UnixFsFileLinkWriter extends StreamWriter<UnixFS.FileLink> {}
+
 export interface Options<Layout = unknown> {
   writer: BlockWriter
   metadata?: UnixFS.Metadata
   settings?: EncoderSettings<Layout>
+  initOptions?: InitOptions
 }
 
 export interface CloseOptions {

src/file/writer.js

@@ -13,6 +13,7 @@ import * as Queue from "./layout/queue.js"
  * readonly metadata: UnixFS.Metadata
  * readonly config: API.EncoderSettings<Layout>
  * readonly writer: API.BlockWriter
+ * readonly unixFsFileLinkWriter?: API.UnixFsFileLinkWriter
  * chunker: Chunker.Chunker
  * layout: Layout
  * nodeQueue: Queue.Queue
@@ -25,6 +26,7 @@ import * as Queue from "./layout/queue.js"
  * readonly metadata: UnixFS.Metadata
  * readonly config: API.EncoderSettings<Layout>
  * readonly writer: API.BlockWriter
+ * readonly unixFsFileLinkWriter?: API.UnixFsFileLinkWriter
  * readonly rootID: Layout.NodeID
  * readonly end?: Task.Fork<void, never>
  * chunker?: null
@@ -39,6 +41,7 @@ import * as Queue from "./layout/queue.js"
  * readonly metadata: UnixFS.Metadata
  * readonly config: API.EncoderSettings<Layout>
  * readonly writer: API.BlockWriter
+ * readonly unixFsFileLinkWriter?: API.UnixFsFileLinkWriter
  * readonly link: Layout.Link
  * chunker?: null
  * layout?: null
@@ -63,6 +66,7 @@ import * as Queue from "./layout/queue.js"
  * |{type:"write", bytes:Uint8Array}
  * |{type:"link", link:API.EncodedFile}
  * |{type:"block"}
+ * |{type:"fileLink"}
  * |{type: "close"}
  * |{type: "end"}
  * } Message
@@ -82,6 +86,9 @@ export const update = (message, state) => {
     /* c8 ignore next 2 */
     case "block":
       return { state, effect: Task.none() }
+    /* c8 ignore next 2 */
+    case "fileLink":
+      return { state, effect: Task.none() }
     case "close":
       return close(state)
     case "end":
@@ -96,9 +103,10 @@ export const update = (message, state) => {
  * @param {API.BlockWriter} writer
  * @param {UnixFS.Metadata} metadata
  * @param {API.EncoderSettings} config
+ * @param {API.InitOptions} [options]
  * @returns {State<Layout>}
  */
-export const init = (writer, metadata, config) => {
+export const init = (writer, metadata, config, options = {}) => {
   return {
     status: "open",
     metadata,
@@ -116,6 +124,7 @@ export const init = (writer, metadata, config) => {
     // overhead.
     // @see https://github.com/Gozala/vectrie
     nodeQueue: Queue.mutable(),
+    unixFsFileLinkWriter: options.unixFsFileLinkWriter,
   }
 }
 /**
@@ -188,11 +197,23 @@ export const link = (state, { id, link, block }) => {
       ? state.end.resume()
       : Task.none()
 
+  if (!state.unixFsFileLinkWriter) {
+    return {
+      state: newState,
+      effect: Task.listen({
+        link: Task.effects(tasks),
+        block: writeBlock(state.writer, block),
+        end,
+      }),
+    }
+  }
+
   return {
     state: newState,
     effect: Task.listen({
       link: Task.effects(tasks),
       block: writeBlock(state.writer, block),
+      fileLink: writeFileLink(state.unixFsFileLinkWriter, link),
       end,
     }),
   }
@@ -203,7 +224,7 @@ export const link = (state, { id, link, block }) => {
  * @param {State<Layout>} state
  * @returns {Update<Layout>}
  */
-export const close = state => {
+export const close = (state) => {
   if (state.status === "open") {
     const { chunks } = Chunker.close(state.chunker)
     const { layout, ...write } = state.config.fileLayout.write(
@@ -269,7 +290,7 @@ export const close = state => {
  * @param {API.EncoderSettings} config
  */
 const encodeLeaves = (leaves, config) =>
-  leaves.map(leaf => encodeLeaf(config, leaf, config.fileChunkEncoder))
+  leaves.map((leaf) => encodeLeaf(config, leaf, config.fileChunkEncoder))
 
 /**
  * @param {API.EncoderSettings} config
@@ -286,6 +307,7 @@ const encodeLeaf = function* ({ hasher, linker }, { id, content }, encoder) {
   const link = /** @type {UnixFS.FileLink} */ ({
     cid,
     contentByteLength: content ? content.byteLength : 0,
+    contentByteOffset: content ? content.byteOffset : 0,
     dagByteLength: bytes.byteLength,
   })
 
@@ -297,7 +319,7 @@ const encodeLeaf = function* ({ hasher, linker }, { id, content }, encoder) {
  * @param {API.EncoderSettings} config
  */
 const encodeBranches = (nodes, config) =>
-  nodes.map(node => encodeBranch(config, node))
+  nodes.map((node) => encodeBranch(config, node))
 
 /**
  * @template Layout
@@ -338,13 +360,30 @@ export const writeBlock = function* (writer, block) {
   writer.write(block)
 }
 
+/**
+ * @param {API.UnixFsFileLinkWriter} writer
+ * @param {Layout.Link} link
+ * @returns {Task.Task<void, never>}
+ */
+
+export const writeFileLink = function* (writer, link) {
+  /* c8 ignore next 3 */
+  if (!writer) {
+    return
+  }
+  if ((writer.desiredSize || 0) <= 0) {
+    yield* Task.wait(writer.ready)
+  }
+  writer.write(link)
+}
+
 /**
  *
  * @param {Uint8Array|Chunker.Chunk} buffer
  * @returns
  */
 
-const asUint8Array = buffer =>
+const asUint8Array = (buffer) =>
   buffer instanceof Uint8Array
     ? buffer
     : buffer.copyTo(new Uint8Array(buffer.byteLength), 0)
@@ -353,4 +392,4 @@ const asUint8Array = buffer =>
  * @param {Layout.Node} node
  * @returns {node is Layout.Leaf}
  */
-const isLeafNode = node => node.children == null
+const isLeafNode = (node) => node.children == null

src/unixfs.ts

@@ -6,10 +6,15 @@ import type {
   Link as IPLDLink,
   Version as LinkVersion,
   Block as IPLDBlock,
-  BlockView as IPLDBlockView
+  BlockView as IPLDBlockView,
 } from "multiformats"
 import { Data, type IData } from "../gen/unixfs.js"
-export type { MultihashHasher, MultibaseEncoder, MultihashDigest, BlockEncoder }
+export type {
+  MultihashHasher,
+  MultibaseEncoder,
+  MultihashDigest,
+  BlockEncoder,
+}
 export * as Layout from "./file/layout/api"
 
 import NodeType = Data.DataType
@@ -161,6 +166,11 @@ export interface ContentDAGLink<T> extends DAGLink<T> {
    * Total number of bytes in the file
    */
   readonly contentByteLength: number
+
+  /**
+   * Offset bytes in the file
+   */
+  readonly contentByteOffset?: number
 }
 
 /**