diff --git a/archive/_stream_common.ts b/archive/_stream_common.ts new file mode 100644 index 000000000000..38a8ff8c211e --- /dev/null +++ b/archive/_stream_common.ts @@ -0,0 +1,93 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +export const recordSize = 512; + +export const ustarStructure = [ + { + field: "fileName", + length: 100, + }, + { + field: "fileMode", + length: 8, + }, + { + field: "uid", + length: 8, + }, + { + field: "gid", + length: 8, + }, + { + field: "fileSize", + length: 12, + }, + { + field: "mtime", + length: 12, + }, + { + field: "checksum", + length: 8, + }, + { + field: "type", + length: 1, + }, + { + field: "linkName", + length: 100, + }, + { + field: "ustar", + length: 8, + }, + { + field: "owner", + length: 32, + }, + { + field: "group", + length: 32, + }, + { + field: "majorNumber", + length: 8, + }, + { + field: "minorNumber", + length: 8, + }, + { + field: "fileNamePrefix", + length: 155, + }, + { + field: "padding", + length: 12, + }, +] as const; + +export const FILE_TYPES = [ + "file", + "link", + "symlink", + "character-device", + "block-device", + "directory", + "fifo", + "contiguous-file", +] as const; + +export type FileType = typeof FILE_TYPES[number]; + +export interface TarInfo { + fileMode?: number; + mtime?: number; + uid?: number; + gid?: number; + owner?: string; + group?: string; + type?: FileType; +} diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts new file mode 100644 index 000000000000..77538ab084fa --- /dev/null +++ b/archive/tar_stream.ts @@ -0,0 +1,212 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +/** + * Ported and modified from: https://github.com/beatgammit/tar-js and + * licensed as: + * + * (The MIT License) + * + * Copyright (c) 2011 T. Jameson Little + * Copyright (c) 2019 Jun Kato + * Copyright (c) 2018-2022 the Deno authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +import { + FILE_TYPES, + recordSize, + TarInfo, + ustarStructure, +} from "./_stream_common.ts"; + +const ustar = "ustar\u000000"; + +function pad(num: number, bytes: number): string { + return num.toString(8).padStart(bytes, "0"); +} +/* +struct posix_header { // byte offset + char name[100]; // 0 + char mode[8]; // 100 + char uid[8]; // 108 + char gid[8]; // 116 + char size[12]; // 124 + char mtime[12]; // 136 + char chksum[8]; // 148 + char typeflag; // 156 + char linkname[100]; // 157 + char magic[6]; // 257 + char version[2]; // 263 + char uname[32]; // 265 + char gname[32]; // 297 + char devmajor[8]; // 329 + char devminor[8]; // 337 + char prefix[155]; // 345 + // 500 +}; +*/ + +/** + * Create header for a file in a tar archive + */ +function formatHeader(data: TarData): Uint8Array { + const encoder = new TextEncoder(); + const buffer = new Uint8Array(512); + let offset = 0; + for (const value of ustarStructure) { + const entry = encoder.encode(data[value.field as keyof TarData]); + buffer.set(entry, offset); + offset += value.length; // space it out with nulls + } + return buffer; +} + +export interface TarData { + fileName?: string; + fileNamePrefix?: string; + fileMode?: string; + uid?: string; + gid?: string; + fileSize: string; + mtime?: string; + checksum?: string; + type?: string; + ustar?: string; + owner?: string; + group?: string; +} + +export interface TarDataWithSource extends TarData { + /** + * buffer to read + */ + readable: ReadableStream; +} + +export interface TarOptions extends TarInfo { + /** + * file name + */ + name: string; + + /** + * append any arbitrary content + */ + readable: ReadableStream; + + /** + * size of the content to be appended + */ + contentSize: number; +} + +/** + * A class to create a tar archive + */ +export class TarStream extends TransformStream { + constructor() { + super({ + transform: async (chunk: TarOptions, controller) => { + // separate file name into two parts if needed + let fileNamePrefix: string | undefined; + let fileName = chunk.name; + if (fileName.length > 100) { + const i = fileName.lastIndexOf("/", 155); + if (i >= 0) { + fileNamePrefix = fileName.substring(0, i); + fileName = fileName.substring(i + 1); + } else { + throw new Error( + "ustar format does not allow a long file name (length of [file name prefix] + / + [file name] must be shorter than 256 bytes)", + ); + } + if (fileNamePrefix.length > 155) { + throw new Error( + "ustar format does not allow a long file name (length of [file name prefix] + / + [file name] must be shorter than 256 bytes)", + ); + } + } + const mode = chunk.fileMode || parseInt("777", 8) & 0xfff; + const mtime = Math.floor(chunk.mtime ?? new Date().valueOf() / 1000); + const uid = chunk.uid || 0; + const gid = chunk.gid || 0; + + if (typeof chunk.owner === "string" && chunk.owner.length >= 32) { + throw new Error( + "ustar format does not allow owner name length >= 32 bytes", + ); + } + if (typeof chunk.group === "string" && chunk.group.length >= 32) { + throw new Error( + "ustar format does not allow group name length >= 32 bytes", + ); + } + + const type = FILE_TYPES.indexOf(chunk.type ?? "file"); + + const tarData: TarDataWithSource = { + fileName, + fileNamePrefix, + fileMode: pad(mode, 7), + uid: pad(uid, 7), + gid: pad(gid, 7), + fileSize: pad(chunk.contentSize, 11), + mtime: pad(mtime, 11), + checksum: " ", + type: type.toString(), + ustar, + owner: chunk.owner || "", + group: chunk.group || "", + readable: chunk.readable, + }; + + // calculate the checksum + let checksum = 0; + const encoder = new TextEncoder(); + for (const key in tarData) { + if (key === "readable") { + continue; + } + checksum += encoder.encode(tarData[key as keyof TarData]).reduce( + (p, c) => p + c, + 0, + ); + } + + tarData.checksum = pad(checksum, 6) + "\u0000 "; + + controller.enqueue(formatHeader(tarData)); + + for await (const readableChunk of chunk.readable) { + controller.enqueue(readableChunk); + } + + controller.enqueue( + new Uint8Array( + recordSize - + (parseInt(tarData.fileSize, 8) % recordSize || recordSize), + ), + ); + }, + flush(controller) { + // append 2 empty records + controller.enqueue(new Uint8Array(recordSize * 2)); + }, + }); + } +} diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts new file mode 100644 index 000000000000..550ae8a5f781 --- /dev/null +++ b/archive/tar_stream_test.ts @@ -0,0 +1,93 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +/** + * Tar test + * + * **test summary** + * - create a tar archive in memory containing output.txt and dir/tar.ts. + * - read and deflate a tar archive containing output.txt + * + * **to run this test** + * deno run --allow-read archive/tar_test.ts + */ +import { assert, assertEquals } from "../assert/mod.ts"; +import { type TarOptions, TarStream } from "./tar_stream.ts"; +import { UntarStream } from "./untar_stream.ts"; +import { Buffer } from "../streams/buffer.ts"; +import { toText } from "../streams/to_text.ts"; + +Deno.test("createTarArchive", async () => { + const buf = new Buffer(); + + // put data on memory + const content = new TextEncoder().encode("hello tar world!"); + + // write tar data to a buffer + await ReadableStream.from([ + { + name: "output.txt", + readable: ReadableStream.from([content]), + contentSize: content.byteLength, + }, + ]).pipeThrough(new TarStream()) + .pipeTo(buf.writable); + + // 2048 = 512 (header) + 512 (content) + 1024 (footer) + assertEquals(buf.bytes().length, 2048); +}); + +Deno.test("appendFileWithLongNameToTarArchive", async () => { + // 9 * 15 + 13 = 148 bytes + const fileName = "long-file-name/".repeat(10) + "file-name.txt"; + const text = "hello tar world!"; + + // create a tar archive + const untar = new UntarStream(); + await ReadableStream.from([ + { + name: fileName, + readable: ReadableStream.from([ + new TextEncoder().encode(text), + ]), + contentSize: 16, + }, + ]).pipeThrough(new TarStream()) + .pipeTo(untar.writable); + + // read data from a tar archive + const untarReader = untar.readable.getReader(); + const result = await untarReader.read(); + assert(!result.done); + assert(!result.value.consumed); + + const untarText = await toText(result.value.readable); + assert(result.value.consumed); + + // tests + assertEquals(result.value.fileName, fileName); + assertEquals(untarText, text); +}); + +Deno.test("directoryEntryType", async () => { + const entries: TarOptions[] = [ + { + name: "directory/", + readable: ReadableStream.from([]), + contentSize: 0, + type: "directory", + }, + { + name: "archive/testdata/", + type: "directory", + readable: ReadableStream.from([]), + contentSize: 0, + }, + ]; + const untar = new UntarStream(); + await ReadableStream.from(entries) + .pipeThrough(new TarStream()) + .pipeTo(untar.writable); + + for await (const entry of untar.readable) { + assertEquals(entry.type, "directory"); + } +}); diff --git a/archive/testdata/test_stream.tar b/archive/testdata/test_stream.tar new file mode 100644 index 000000000000..8ccb56109fbb Binary files /dev/null and b/archive/testdata/test_stream.tar differ diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts new file mode 100644 index 000000000000..602c2465ba3d --- /dev/null +++ b/archive/untar_stream.ts @@ -0,0 +1,265 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +/** + * Ported and modified from: https://github.com/beatgammit/tar-js and + * licensed as: + * + * (The MIT License) + * + * Copyright (c) 2011 T. Jameson Little + * Copyright (c) 2019 Jun Kato + * Copyright (c) 2018-2022 the Deno authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +import { Buffer } from "../streams/buffer.ts"; +import { + FILE_TYPES, + recordSize, + TarInfo, + ustarStructure, +} from "./_stream_common.ts"; +import { assert } from "../assert/assert.ts"; + +const decoder = new TextDecoder(); + +// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 +// eight checksum bytes taken to be ascii spaces (decimal value 32) +const initialChecksum = 8 * 32; + +/** + * Remove the trailing null codes + * @param buffer + */ +function trim(buffer: Uint8Array): Uint8Array { + const index = buffer.findIndex((v) => v === 0); + return index < 0 ? buffer : buffer.subarray(0, index); +} + +/** + * Parse file header in a tar archive + * @param buffer + */ +function parseHeader(buffer: Uint8Array): TarHeader { + const data: Record = {}; + let offset = 0; + for (const value of ustarStructure) { + data[value.field] = buffer.subarray(offset, offset + value.length); + offset += value.length; + } + return data as TarHeader; +} + +export type TarHeader = Record< + (typeof ustarStructure)[number]["field"], + Uint8Array +>; + +export interface TarMeta extends TarInfo { + fileName: string; + fileSize?: number; + linkName?: string; +} + +// deno-lint-ignore no-empty-interface +export interface TarEntry extends TarMeta {} + +export class TarEntry { + #header: TarHeader; + #readableInner: ReadableStream; + #readable: ReadableStream; + #fileSize: number; + #read = 0; + #consumed = false; + #entrySize: number; + + constructor( + meta: TarMeta, + header: TarHeader, + readable: ReadableStream, + ) { + Object.assign(this, meta); + this.#header = header; + this.#readableInner = readable; + this.#readable = new ReadableStream({ + pull: async (controller) => { + const p = new Uint8Array(controller.byobRequest!.view!.buffer); + // Bytes left for entry + const entryBytesLeft = this.#entrySize - this.#read; + // bufSize can't be greater than p.length nor bytes left in the entry + const bufSize = Math.min(p.byteLength, entryBytesLeft); + + if (entryBytesLeft <= 0) { + this.#consumed = true; + controller.close(); + return; + } + + const reader = this.#readableInner.getReader({ mode: "byob" }); + const res = await reader.read(new Uint8Array(bufSize), { + min: bufSize, + }); + reader.releaseLock(); + + const bytesLeft = this.#fileSize - this.#read; + this.#read += bufSize; + if (res.done || bytesLeft <= 0) { + if (res.done) this.#consumed = true; + controller.close(); + return; + } + p.set(res.value, 0); + controller.byobRequest!.respond(Math.min(bytesLeft, bufSize)); + }, + autoAllocateChunkSize: 512, + type: "bytes", + }, { highWaterMark: 0 }); + + // File Size + this.#fileSize = this.fileSize || 0; + // Entry Size + const blocks = Math.ceil(this.#fileSize / recordSize); + this.#entrySize = blocks * recordSize; + } + + get readable(): ReadableStream { + return this.#readable; + } + + get consumed(): boolean { + return this.#consumed; + } + + async discard() { + // Discard current entry + if (this.#consumed) return; + this.#consumed = true; + + for await (const _ of this.#readable) { + // + } + } +} + +/** A class to extract a tar archive */ +export class UntarStream implements TransformStream { + readable: ReadableStream; + #buffer: Buffer; + #block = new Uint8Array(recordSize); + #entry: TarEntry | undefined; + + constructor() { + this.#buffer = new Buffer(); + this.readable = new ReadableStream({ + pull: async (controller) => { + if (this.#entry && !this.#entry.consumed) { + // If entry body was not read, discard the body + // so we can read the next entry. + await this.#entry.discard(); + } + + const header = await this.#getHeader(); + if (header === null) { + controller.close(); + return; + } + + const meta = getMetadata(header); + this.#entry = new TarEntry(meta, header, this.#buffer.readable); + controller.enqueue(this.#entry); + }, + }, { highWaterMark: 0 }); + } + + get writable() { + return this.#buffer.writable; + } + + async #getHeader(): Promise { + const reader = this.#buffer.readable.getReader({ mode: "byob" }); + const res = await reader.read(this.#block, { min: this.#block.byteLength }); + reader.releaseLock(); + assert(!res.done); + this.#block = res.value; + const header = parseHeader(this.#block); + + // calculate the checksum + const checksum = getChecksum(this.#block); + + if (parseInt(decoder.decode(header.checksum), 8) !== checksum) { + if (checksum === initialChecksum) { + // EOF + return null; + } + throw new Error("checksum error"); + } + + const magic = decoder.decode(header.ustar); + + if (magic.indexOf("ustar")) { + throw new Error(`unsupported archive format: ${magic}`); + } + + return header; + } +} + +function getMetadata(header: TarHeader): TarMeta { + // get meta data + const meta: TarMeta = { + fileName: decoder.decode(trim(header.fileName)), + }; + const fileNamePrefix = trim(header.fileNamePrefix); + if (fileNamePrefix.byteLength > 0) { + meta.fileName = decoder.decode(fileNamePrefix) + "/" + meta.fileName; + } + + for (const key of ["fileMode", "mtime", "uid", "gid"] as const) { + const arr = trim(header[key]); + if (arr.byteLength > 0) { + meta[key] = parseInt(decoder.decode(arr), 8); + } + } + + for (const key of ["owner", "group", "type"] as const) { + const arr = trim(header[key]); + if (arr.byteLength > 0) { + // deno-lint-ignore no-explicit-any + meta[key] = decoder.decode(arr) as any; + } + } + + meta.fileSize = parseInt(decoder.decode(header.fileSize), 8); + meta.type = FILE_TYPES[parseInt(meta.type!)] ?? meta.type; + + if (meta.type === "symlink") { + meta.linkName = decoder.decode(trim(header.linkName)); + } + + return meta; +} + +function getChecksum(header: Uint8Array): number { + let sum = initialChecksum; + for (let i = 0; i < 512; i++) { + if (i < 148 || i >= 156) { + sum += header[i]; + } + } + return sum; +} diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts new file mode 100644 index 000000000000..0d642b94ec49 --- /dev/null +++ b/archive/untar_stream_test.ts @@ -0,0 +1,347 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +/** + * Tar test + * + * **test summary** + * - create a tar archive in memory containing output.txt and dir/tar.ts. + * - read and deflate a tar archive containing output.txt + * + * **to run this test** + * deno run --allow-read archive/tar_test.ts + */ +import { assert, assertEquals } from "../assert/mod.ts"; +import { dirname, fromFileUrl, resolve } from "../path/mod.ts"; +import { TarMeta, UntarStream } from "./untar_stream.ts"; +import { Buffer } from "../streams/buffer.ts"; +import { type TarOptions, TarStream } from "./tar_stream.ts"; +import { toArrayBuffer } from "../streams/to_array_buffer.ts"; +import { toText } from "../streams/to_text.ts"; + +const moduleDir = dirname(fromFileUrl(import.meta.url)); +const testdataDir = resolve(moduleDir, "testdata"); +const filePath = resolve(testdataDir, "example.txt"); + +async function getTarOptions(): Promise { + const file = await Deno.open(filePath, { read: true }); + return [ + { + name: "output.txt", + readable: ReadableStream.from([ + new TextEncoder().encode("hello tar world!"), + ]), + contentSize: 16, + }, + { + name: "dir/tar.ts", + readable: file.readable, + contentSize: file.statSync().size, + }, + ]; +} + +Deno.test("deflateTarArchive", async () => { + const fileName = "output.txt"; + const text = "hello tar world!"; + const content = new TextEncoder().encode(text); + + const untar = new UntarStream(); + await ReadableStream.from([ + { + name: fileName, + readable: ReadableStream.from([content]), + contentSize: content.byteLength, + }, + ]).pipeThrough(new TarStream()) + .pipeTo(untar.writable); + + const reader = untar.readable.getReader(); + const result = await reader.read(); + assert(!result.done); + + const untarText = await toText(result.value.readable); + + assert((await reader.read()).done); // EOF + // tests + assertEquals(result.value.fileName, fileName); + assertEquals(untarText, text); +}); + +Deno.test("untarAsyncIterator", async () => { + const entries = await getTarOptions(); + + const untar = new UntarStream(); + await ReadableStream.from(entries) + .pipeThrough(new TarStream()) + .pipeTo(untar.writable); + + let lastEntry; + for await (const entry of untar.readable) { + const expected = entries.shift(); + assert(expected); + + const buffer = new Buffer(); + await entry.readable.pipeTo(buffer.writable); + assertEquals( + await toArrayBuffer(expected.readable), + await toArrayBuffer(buffer.readable), + ); + assertEquals(expected.name, entry.fileName); + + if (lastEntry) assert(lastEntry.consumed); + lastEntry = entry; + } + assert(lastEntry); + assert(lastEntry.consumed); + assertEquals(entries.length, 0); +}); + +Deno.test("untarAsyncIteratorWithoutReadingBody", async () => { + const entries: TarOptions[] = await getTarOptions(); + + const tar = ReadableStream.from(entries) + .pipeThrough(new TarStream()); + + const untar = new UntarStream(); + // read data from a tar archive + await tar.pipeTo(untar.writable); + + for await (const entry of untar.readable) { + const expected = entries.shift(); + assert(expected); + assertEquals(expected.name, entry.fileName); + } + + assertEquals(entries.length, 0); +}); + +Deno.test( + "untarAsyncIteratorWithoutReadingBodyFromFileReadable", + async () => { + const filePath = resolve(testdataDir, "test_stream.tar"); + const file = await Deno.open(filePath, { read: true }); + + const entries = await getTarOptions(); + + for await (const entry of file.readable.pipeThrough(new UntarStream())) { + const expected = entries.shift(); + assert(expected); + assertEquals(expected.name, entry.fileName); + } + + assertEquals(entries.length, 0); + }, +); + +Deno.test("untarAsyncIteratorFromFileReadable", async function () { + const filePath = resolve(testdataDir, "test_stream.tar"); + const file = await Deno.open(filePath, { read: true }); + + const entries = await getTarOptions(); + + for await (const entry of file.readable.pipeThrough(new UntarStream())) { + const expected = entries.shift(); + assert(expected); + + assertEquals( + await toArrayBuffer(expected.readable), + await toArrayBuffer(entry.readable), + ); + assertEquals(expected.name, entry.fileName); + } + + assertEquals(entries.length, 0); +}); + +function getEntries(): TarOptions[] { + return [ + { + name: "output.txt", + readable: ReadableStream.from([ + new TextEncoder().encode("hello tar world!".repeat(100)), + ]), + contentSize: 1600, + }, + // Need to test at least two files, to make sure the first entry doesn't over-read + // Causing the next to fail with: checksum error + { + name: "deni.txt", + readable: ReadableStream.from([ + new TextEncoder().encode("deno!".repeat(250)), + ]), + contentSize: 1250, + }, + ]; +} + +Deno.test( + "untarAsyncIteratorReadingLessThanRecordSize", + async function (t) { + // record size is 512 + const bufSizes = [1, 53, 256, 511]; + + for (const bufSize of bufSizes) { + await t.step(bufSize.toString(), async () => { + const untar = ReadableStream.from(getEntries()) + .pipeThrough(new TarStream()) + .pipeThrough(new UntarStream()); + + const assertEntries = getEntries(); + // read data from a tar archive + for await (const entry of untar) { + const expected = assertEntries.shift(); + assert(expected); + assertEquals(expected.name, entry.fileName); + assertEquals( + await toArrayBuffer(entry.readable), + await toArrayBuffer(expected.readable), + ); + } + + assertEquals(assertEntries.length, 0); + }); + } + }, +); + +Deno.test("untarLinuxGeneratedTar", async function () { + const filePath = resolve(testdataDir, "deno.tar"); + const file = await Deno.open(filePath, { read: true }); + + const expectedEntries = [ + { + fileName: "archive/", + fileSize: 0, + fileMode: 509, + mtime: 1591800767, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "directory", + }, + { + fileName: "archive/deno/", + fileSize: 0, + fileMode: 509, + mtime: 1591799635, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "directory", + }, + { + fileName: "archive/deno/land/", + fileSize: 0, + fileMode: 509, + mtime: 1591799660, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "directory", + }, + { + fileName: "archive/deno/land/land.txt", + fileMode: 436, + fileSize: 5, + mtime: 1591799660, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "file", + content: "land\n", + }, + { + fileName: "archive/file.txt", + fileMode: 436, + fileSize: 5, + mtime: 1591799626, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "file", + content: "file\n", + }, + { + fileName: "archive/deno.txt", + fileMode: 436, + fileSize: 5, + mtime: 1591799642, + uid: 1001, + gid: 1001, + owner: "deno", + group: "deno", + type: "file", + content: "deno\n", + }, + ]; + + const untar = new UntarStream(); + await file.readable.pipeTo(untar.writable); + + for await (const entry of untar.readable) { + const expected = expectedEntries.shift(); + assert(expected); + const content = expected.content; + delete expected.content; + + // @ts-ignore its fine + assertEquals({ ...entry }, expected); + + if (content) { + const buffer = new Buffer(); + await entry.readable.pipeTo(buffer.writable); + assertEquals(content, await toText(buffer.readable)); + } + } +}); + +Deno.test("untarArchiveWithLink", async function () { + const filePath = resolve(testdataDir, "with_link.tar"); + const file = await Deno.open(filePath, { read: true }); + + type ExpectedEntry = TarMeta & { content?: string }; + + const expectedEntries: ExpectedEntry[] = [ + { + fileName: "hello.txt", + fileMode: 436, + fileSize: 14, + mtime: 1696384910, + uid: 1000, + gid: 1000, + owner: "user", + group: "user", + type: "file", + content: "Hello World!\n\n", + }, + { + fileName: "link_to_hello.txt", + linkName: "./hello.txt", + fileMode: 511, + fileSize: 0, + mtime: 1696384945, + uid: 1000, + gid: 1000, + owner: "user", + group: "user", + type: "symlink", + }, + ]; + + for await (const entry of file.readable.pipeThrough(new UntarStream())) { + const expected = expectedEntries.shift(); + assert(expected); + const content = expected.content; + delete expected.content; + + assertEquals({ ...entry }, expected); + + if (content) { + assertEquals(content, await toText(entry.readable)); + } + } +}); diff --git a/streams/buffer.ts b/streams/buffer.ts index a84f452b06df..e71e0b509a46 100644 --- a/streams/buffer.ts +++ b/streams/buffer.ts @@ -24,10 +24,15 @@ const DEFAULT_CHUNK_SIZE = 16_640; export class Buffer { #buf: Uint8Array; // contents are the bytes buf[off : len(buf)] #off = 0; // read at buf[off], write at buf[buf.byteLength] + #startedPromise = Promise.withResolvers(); + #startedBool = false; #readable: ReadableStream = new ReadableStream({ type: "bytes", - pull: (controller) => { + pull: async (controller) => { const view = new Uint8Array(controller.byobRequest!.view!.buffer); + if (!this.#startedBool) { + await this.#startedPromise.promise; + } if (this.empty()) { // Buffer is empty, reset to recover space. this.reset(); @@ -37,7 +42,9 @@ export class Buffer { } const nread = copy(this.#buf.subarray(this.#off), view); this.#off += nread; - controller.byobRequest!.respond(nread); + if (nread !== 0) { + controller.byobRequest!.respond(nread); + } }, autoAllocateChunkSize: DEFAULT_CHUNK_SIZE, }); @@ -51,6 +58,7 @@ export class Buffer { write: (chunk) => { const m = this.#grow(chunk.byteLength); copy(chunk, this.#buf, m); + this.#startedPromise.resolve(undefined); }, });