From 11732fde9e1e4e18140e13212e7bb2e64e32968a Mon Sep 17 00:00:00 2001 From: Benjamin Fischer <61995275+c4spar@users.noreply.github.com> Date: Mon, 22 Nov 2021 23:23:01 +0100 Subject: [PATCH] feat: add createBucket & getBucket method (#32) --- Makefile | 3 ++ README.md | 19 +++++++- docker-compose.yml | 5 +- mod.ts | 4 ++ src/bucket.ts | 118 ++++++++++++++++++--------------------------- src/bucket_test.ts | 24 ++++----- src/client.ts | 105 +++++++++++++++++++++++++++++++++++++++- src/client_test.ts | 52 ++++++++++++++++++++ src/request.ts | 76 +++++++++++++++++++++++++++++ src/types.ts | 70 ++++++++++++++++++++++++++- 10 files changed, 386 insertions(+), 90 deletions(-) create mode 100644 src/client_test.ts create mode 100644 src/request.ts diff --git a/Makefile b/Makefile index e2f10ba..7a831cb 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ test: aws --endpoint-url=http://localhost:9000 s3 rb s3://test || true aws --endpoint-url=http://localhost:9000 s3 mb s3://test + aws --endpoint-url=http://localhost:9000 s3 rm --recursive s3://create-bucket-test || true + aws --endpoint-url=http://localhost:9000 s3 rb s3://create-bucket-test || true + deno test -A ${DENO_ARGS} cleanup: diff --git a/README.md b/README.md index 4d3e194..83f2643 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,24 @@ Amazon S3 for Deno ## Example ```ts -import { S3Bucket } from "https://deno.land/x/s3@0.4.1/mod.ts"; +import { S3, S3Bucket } from "https://deno.land/x/s3@0.4.1/mod.ts"; -const bucket = new S3Bucket({ +// Create a S3 instance. +const s3 = new S3({ + accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, + secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, + region: "us-east-1", + endpointURL: Deno.env.get("S3_ENDPOINT_URL"), +}); + +// Create a new bucket. +let bucket = await s3.createBucket("test", { acl: "private" }); + +// Get an existing bucket. +bucket = s3.getBucket("test"); + +// Create a bucket instance manuely. +bucket = new S3Bucket({ accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, bucket: "test", diff --git a/docker-compose.yml b/docker-compose.yml index a751266..f014e1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,9 +3,10 @@ version: "3" services: minio: image: minio/minio - command: server /data + command: server /data --console-address ":9001" environment: MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ports: - - "9000:9000" \ No newline at end of file + - "9000:9000" + - "9001:9001" \ No newline at end of file diff --git a/mod.ts b/mod.ts index 592eab1..0cf40ea 100644 --- a/mod.ts +++ b/mod.ts @@ -1,9 +1,12 @@ +export * from "./src/client.ts"; export * from "./src/bucket.ts"; export type { CommonPrefix, CopyDirective, CopyObjectOptions, CopyObjectResponse, + CreateBucketConfiguration, + CreateBucketOptions, DeleteObjectOptions, DeleteObjectResponse, GetObjectOptions, @@ -12,6 +15,7 @@ export type { ListAllObjectsOptions, ListObjectsOptions, ListObjectsResponse, + LocationConstraint, LockMode, PutObjectOptions, PutObjectResponse, diff --git a/src/bucket.ts b/src/bucket.ts index 506d501..12c8796 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -3,7 +3,6 @@ import { decodeXMLEntities, parseXML, pooledMap, - sha256Hex, } from "../deps.ts"; import type { S3Config } from "./client.ts"; import type { @@ -26,10 +25,8 @@ import type { } from "./types.ts"; import { S3Error } from "./error.ts"; import type { Signer } from "../deps.ts"; - -interface Params { - [key: string]: string; -} +import { doRequest, encodeURIS3 } from "./request.ts"; +import type { Params } from "./request.ts"; export interface S3BucketConfig extends S3Config { bucket: string; @@ -38,7 +35,6 @@ export interface S3BucketConfig extends S3Config { export class S3Bucket { #signer: Signer; #host: string; - #bucket: string; constructor(config: S3BucketConfig) { this.#signer = new AWSSignerV4(config.region, { @@ -46,7 +42,6 @@ export class S3Bucket { awsSecretKey: config.secretKey, sessionToken: config.sessionToken, }); - this.#bucket = config.bucket; this.#host = config.endpointURL ? new URL(`/${config.bucket}/`, config.endpointURL).toString() : config.bucket.indexOf(".") >= 0 @@ -54,33 +49,6 @@ export class S3Bucket { : `https://${config.bucket}.s3.${config.region}.amazonaws.com/`; } - private async _doRequest( - path: string, - params: Params, - method: string, - headers: Params, - body?: Uint8Array | undefined, - ): Promise { - const url = path == "/" - ? new URL(this.#host) - : new URL(encodeURIS3(path), this.#host); - for (const key in params) { - url.searchParams.set(key, params[key]); - } - const request = new Request(url.toString(), { - headers, - method, - body, - }); - - const signedRequest = await this.#signer.sign("s3", request); - signedRequest.headers.set("x-amz-content-sha256", sha256Hex(body ?? "")); - if (body) { - signedRequest.headers.set("content-length", body.length.toFixed(0)); - } - return fetch(signedRequest); - } - async headObject( key: string, options?: GetObjectOptions, @@ -120,7 +88,14 @@ export class S3Bucket { params["VersionId"] = options.versionId; } - const res = await this._doRequest(key, params, "HEAD", headers); + const res = await doRequest({ + host: this.#host, + signer: this.#signer, + path: key, + method: "HEAD", + params, + headers, + }); if (res.body) { await res.arrayBuffer(); } @@ -218,7 +193,14 @@ export class S3Bucket { params["VersionId"] = options.versionId; } - const res = await this._doRequest(key, params, "GET", headers); + const res = await doRequest({ + host: this.#host, + signer: this.#signer, + path: key, + method: "GET", + params, + headers, + }); if (res.status === 404) { // clean up http body await res.arrayBuffer(); @@ -299,7 +281,13 @@ export class S3Bucket { params["continuation-token"] = options.continuationToken; } - const res = await this._doRequest(`/`, params, "GET", headers); + const res = await doRequest({ + host: this.#host, + signer: this.#signer, + method: "GET", + params, + headers, + }); if (res.status === 404) { // clean up http body await res.arrayBuffer(); @@ -464,7 +452,14 @@ export class S3Bucket { } } - const resp = await this._doRequest(key, {}, "PUT", headers, body); + const resp = await doRequest({ + host: this.#host, + signer: this.#signer, + path: key, + method: "PUT", + headers, + body, + }); if (resp.status !== 200) { throw new S3Error( `Failed to put object: ${resp.status} ${resp.statusText}`, @@ -557,7 +552,13 @@ export class S3Bucket { headers["x-amz-tagging-directive"] = options.taggingDirective; } - const resp = await this._doRequest(destination, {}, "PUT", headers); + const resp = await doRequest({ + host: this.#host, + signer: this.#signer, + path: destination, + method: "PUT", + headers, + }); if (resp.status !== 200) { throw new S3Error( `Failed to copy object: ${resp.status} ${resp.statusText}`, @@ -580,7 +581,13 @@ export class S3Bucket { if (options?.versionId) { params.versionId = options.versionId; } - const resp = await this._doRequest(key, params, "DELETE", {}); + const resp = await doRequest({ + host: this.#host, + signer: this.#signer, + path: key, + method: "DELETE", + params, + }); if (resp.status !== 204) { throw new S3Error( `Failed to put object: ${resp.status} ${resp.statusText}`, @@ -618,37 +625,6 @@ export class S3Bucket { } } -function encodeURIS3(input: string): string { - let result = ""; - for (const ch of input) { - if ( - (ch >= "A" && ch <= "Z") || - (ch >= "a" && ch <= "z") || - (ch >= "0" && ch <= "9") || - ch == "_" || - ch == "-" || - ch == "~" || - ch == "." - ) { - result += ch; - } else if (ch == "/") { - result += "/"; - } else { - result += stringToHex(ch); - } - } - return result; -} - -const encoder = new TextEncoder(); - -function stringToHex(input: string) { - return [...encoder.encode(input)] - .map((s) => "%" + s.toString(16)) - .join("") - .toUpperCase(); -} - interface Document { declaration: { attributes: Record; diff --git a/src/bucket_test.ts b/src/bucket_test.ts index fa8d75a..875fb54 100644 --- a/src/bucket_test.ts +++ b/src/bucket_test.ts @@ -12,7 +12,7 @@ const bucket = new S3Bucket({ const encoder = new TextEncoder(); Deno.test({ - name: "put object", + name: "[bucket] put object", async fn() { await bucket.putObject("test", encoder.encode("Test1"), { contentType: "text/plain", @@ -24,7 +24,7 @@ Deno.test({ }); Deno.test({ - name: "put object with % in key", + name: "[bucket] put object with % in key", async fn() { await bucket.putObject( "ltest/versions/1.0.0/raw/fixtures/%", @@ -38,7 +38,7 @@ Deno.test({ }); Deno.test({ - name: "put object with @ in key", + name: "[bucket] put object with @ in key", async fn() { await bucket.putObject( "dex/versions/1.0.0/raw/lib/deps/interpret@2.0.0/README.md", @@ -54,7 +54,7 @@ Deno.test({ }); Deno.test({ - name: "put object with 日本語 in key", + name: "[bucket] put object with 日本語 in key", async fn() { await bucket.putObject( "servest/versions/1.0.0/raw/fixtures/日本語.txt", @@ -68,7 +68,7 @@ Deno.test({ }); Deno.test({ - name: "head object success", + name: "[bucket] head object success", async fn() { // setup await bucket.putObject("test", encoder.encode("Test1"), { @@ -92,14 +92,14 @@ Deno.test({ }); Deno.test({ - name: "head object not found", + name: "[bucket] head object not found", async fn() { assertEquals(await bucket.headObject("test2"), undefined); }, }); Deno.test({ - name: "get object success", + name: "[bucket] get object success", async fn() { // setup await bucket.putObject("test", encoder.encode("Test1"), { @@ -125,14 +125,14 @@ Deno.test({ }); Deno.test({ - name: "get object not found", + name: "[bucket] get object not found", async fn() { assertEquals(await bucket.getObject("test2"), undefined); }, }); Deno.test({ - name: "delete object", + name: "[bucket] delete object", async fn() { // setup await bucket.putObject("test", encoder.encode("test")); @@ -149,7 +149,7 @@ Deno.test({ }); Deno.test({ - name: "copy object", + name: "[bucket] copy object", async fn() { await bucket.putObject("test3", encoder.encode("Test1")); await bucket @@ -172,7 +172,7 @@ Deno.test({ }); Deno.test({ - name: "list objects", + name: "[bucket] list objects", async fn() { // setup const content = encoder.encode("Test1"); @@ -255,7 +255,7 @@ Deno.test({ }); Deno.test({ - name: "empty bucket", + name: "[bucket] empty bucket", async fn() { // setup const content = encoder.encode("Test1"); diff --git a/src/client.ts b/src/client.ts index 40d2c79..75f3c29 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,4 +1,9 @@ import { AWSSignerV4 } from "../deps.ts"; +import type { CreateBucketOptions } from "./types.ts"; +import { S3Error } from "./error.ts"; +import { S3Bucket } from "./bucket.ts"; +import { doRequest, encoder } from "./request.ts"; +import type { Params } from "./request.ts"; export interface S3Config { region: string; @@ -8,9 +13,24 @@ export interface S3Config { endpointURL?: string; } +/** + * A S3 instance can be used to manage multiple buckets. + * + * ``` + * const s3 = new S3({ + * accessKeyID: "", + * secretKey: "", + * region: "eu-south-1", + * }); + * + * const bucket1: S3Bucket = s3.getBucket("my-bucket"); + * const bucket2: S3Bucket = await s3.createBucket("my-second-bucket"); + * ``` + */ export class S3 { - #signer: AWSSignerV4; - #host: string; + readonly #signer: AWSSignerV4; + readonly #host: string; + readonly #config: S3Config; constructor(config: S3Config) { this.#signer = new AWSSignerV4(config.region, { @@ -19,5 +39,86 @@ export class S3 { }); this.#host = config.endpointURL ?? `https://s3.${config.region}.amazonaws.com/`; + this.#config = { ...config }; + } + + /** Creates a new S3Bucket instance with the same config passed to the S3 client. */ + getBucket(bucket: string): S3Bucket { + return new S3Bucket({ + ...this.#config, + bucket, + }); + } + + /** + * Creates a new S3 bucket. By default, the bucket is created in the region + * specified with the S3 options. If not specified the US East (N. Virginia) + * region is used. Optionally, you can specify a Region with the + * `locationConstraint` option. + * + * ``` + * const bucket: S3Bucket = await s3.createBucket("my-bucket", { + * locationConstraint: "EU", + * }); + * ``` + */ + async createBucket( + bucket: string, + options?: CreateBucketOptions, + ): Promise { + const headers: Params = {}; + + if (options?.acl) { + headers["x-amz-acl"] = options.acl; + } + if (options?.grantFullControl) { + headers["x-amz-grant-full-control"] = options.grantFullControl; + } + if (options?.grantRead) { + headers["x-amz-grant-read"] = options.grantRead; + } + if (options?.grantReadAcp) { + headers["x-amz-grant-read-acp"] = options.grantReadAcp; + } + if (options?.grantWrite) { + headers["x-amz-grant-write"] = options.grantWrite; + } + if (options?.grantWriteAcp) { + headers["x-amz-grant-write-acp"] = options.grantWriteAcp; + } + if (options?.bucketObjectLockEnabled) { + headers["x-amz-bucket-object-lock-enabled"] = + options.bucketObjectLockEnabled; + } + + const body = encoder.encode( + '' + + '' + + ` ${ + options?.locationConstraint ?? this.#config.region + }` + + "", + ); + + const resp = await doRequest({ + host: this.#host, + signer: this.#signer, + path: bucket, + method: "PUT", + headers, + body, + }); + + if (resp.status !== 200) { + throw new S3Error( + `Failed to create bucket "${bucket}": ${resp.status} ${resp.statusText}`, + await resp.text(), + ); + } + + // clean up http body + await resp.arrayBuffer(); + + return this.getBucket(bucket); } } diff --git a/src/client_test.ts b/src/client_test.ts new file mode 100644 index 0000000..b1e24a6 --- /dev/null +++ b/src/client_test.ts @@ -0,0 +1,52 @@ +import { assertEquals, assertThrowsAsync } from "../test_deps.ts"; +import { S3Error } from "./error.ts"; +import { S3 } from "./client.ts"; +import { encoder } from "./request.ts"; + +const s3 = new S3({ + accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, + secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, + region: "us-east-1", + endpointURL: Deno.env.get("S3_ENDPOINT_URL"), +}); + +Deno.test({ + name: "[client] should get an existing bucket", + async fn() { + const bucket = await s3.getBucket("test"); + + // Check if returned bucket instance is working. + await bucket.putObject("test", encoder.encode("test")); + const resp = await bucket.getObject("test"); + const body = await new Response(resp?.body).text(); + assertEquals(body, "test"); + + // teardown + await bucket.deleteObject("test"); + }, +}); + +Deno.test({ + name: "[client] should create a new bucket", + async fn() { + const bucket = await s3.createBucket("create-bucket-test", { + acl: "public-read-write", + }); + + // Check if returned bucket instance is working. + await bucket.putObject("test", encoder.encode("test")); + const resp = await bucket.getObject("test"); + const body = await new Response(resp?.body).text(); + assertEquals(body, "test"); + + await assertThrowsAsync( + () => s3.createBucket("create-bucket-test"), + S3Error, + 'Failed to create bucket "create-bucket-test": 409 Conflict', + ); + + // teardown + await bucket.deleteObject("test"); + // @TODO: delete also bucket once s3.deleteBucket is implemented. + }, +}); diff --git a/src/request.ts b/src/request.ts new file mode 100644 index 0000000..05ac574 --- /dev/null +++ b/src/request.ts @@ -0,0 +1,76 @@ +import { sha256Hex } from "../deps.ts"; +import type { Signer } from "../deps.ts"; + +export interface Params { + [key: string]: string; +} + +export const encoder = new TextEncoder(); + +interface S3RequestOptions { + host: string; + signer: Signer; + method: string; + path?: string; + params?: Params; + headers?: Params; + body?: Uint8Array | undefined; +} + +export async function doRequest({ + host, + signer, + path = "/", + params, + method, + headers, + body, +}: S3RequestOptions): Promise { + const url = path == "/" ? new URL(host) : new URL(encodeURIS3(path), host); + if (params) { + for (const key in params) { + url.searchParams.set(key, params[key]); + } + } + const request = new Request(url.toString(), { + headers, + method, + body, + }); + + const signedRequest = await signer.sign("s3", request); + signedRequest.headers.set("x-amz-content-sha256", sha256Hex(body ?? "")); + if (body) { + signedRequest.headers.set("content-length", body.length.toFixed(0)); + } + return fetch(signedRequest); +} + +export function encodeURIS3(input: string): string { + let result = ""; + for (const ch of input) { + if ( + (ch >= "A" && ch <= "Z") || + (ch >= "a" && ch <= "z") || + (ch >= "0" && ch <= "9") || + ch == "_" || + ch == "-" || + ch == "~" || + ch == "." + ) { + result += ch; + } else if (ch == "/") { + result += "/"; + } else { + result += stringToHex(ch); + } + } + return result; +} + +function stringToHex(input: string) { + return [...encoder.encode(input)] + .map((s) => "%" + s.toString(16)) + .join("") + .toUpperCase(); +} diff --git a/src/types.ts b/src/types.ts index 5c50161..9819225 100644 --- a/src/types.ts +++ b/src/types.ts @@ -175,7 +175,7 @@ export interface HeadObjectResponse { /** * Amazon S3 can return this if your request involves a bucket that is * either a source or destination in a replication rule. - * */ + */ replicationStatus?: ReplicationStatus; /** @@ -536,3 +536,71 @@ export interface DeleteObjectResponse { versionID?: string; deleteMarker: boolean; } + +export type LocationConstraint = + | "af-south-1" + | "ap-east-1" + | "ap-northeast-1" + | "ap-northeast-2" + | "ap-northeast-3" + | "ap-south-1" + | "ap-southeast-1" + | "ap-southeast-2" + | "ca-central-1" + | "cn-north-1" + | "cn-northwest-1" + | "EU" + | "Europe" + | "eu-central-1" + | "eu-north-1" + | "eu-south-1" + | "eu-west-1" + | "eu-west-2" + | "eu-west-3" + | "me-south-1" + | "sa-east-1" + | "us-east-1" + | "us-east-2" + | "us-gov-east-1" + | "us-gov-west-1" + | "us-west-1" + | "us-west-2"; + +export interface CreateBucketConfiguration { + /** + * Specifies the Region where the bucket will be created. If you don't + * specify a Region, the bucket is created in the US East (N. Virginia) + * Region (us-east-1). + */ + locationConstraint?: LocationConstraint; +} + +export interface CreateBucketOptions extends CreateBucketConfiguration { + /** The canned ACL to apply to the bucket */ + acl?: + | "private" + | "public-read" + | "public-read-write" + | "authenticated-read"; + + /** Specifies whether you want S3 Object Lock to be enabled for the new bucket. */ + bucketObjectLockEnabled?: string; + + /** Allows grantee the read, write, read ACP, and write ACP permissions on the bucket. */ + grantFullControl?: string; + + /** Allows grantee to list the objects in the bucket. */ + grantRead?: string; + + /** Allows grantee to read the bucket ACL. */ + grantReadAcp?: string; + + /** + * Allows grantee to create new objects in the bucket. + * For the bucket and object owners of existing objects, also allows deletions and overwrites of those objects. + */ + grantWrite?: string; + + /** Allows grantee to write the ACL for the applicable bucket. */ + grantWriteAcp?: string; +}