From 0a604969037571f57e4ec14831d3bf25ab258df3 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Jun 2025 11:18:25 -0700 Subject: [PATCH 1/6] feat: add pagination to getAllBuckets --- src/http/routes/bucket/getAllBuckets.ts | 22 ++++++++++++++++++++-- src/storage/database/adapter.ts | 9 ++++++++- src/storage/database/knex.ts | 19 +++++++++++++++++-- src/storage/storage.ts | 7 ++++--- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index ca79c2c6..8ffeff6a 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -1,4 +1,5 @@ import { FastifyInstance } from 'fastify' +import { FromSchema } from 'json-schema-to-ts' import { createDefaultSchema } from '../../routes-helper' import { AuthenticatedRequest } from '../../types' import { bucketSchema } from '@storage/schemas' @@ -23,14 +24,29 @@ const successResponseSchema = { ], } +const requestQuerySchema = { + type: 'object', + properties: { + limit: { type: 'integer', minimum: 1, examples: [10] }, + offset: { type: 'integer', minimum: 0, examples: [0] }, + sortColumn: { type: 'string', enum: ['id', 'name', 'created_at', 'updated_at'] }, + sortOrder: { type: 'string', enum: ['asc', 'desc'] }, + }, +} as const + +interface GetAllBucketsRequest extends AuthenticatedRequest { + Querystring: FromSchema +} + export default async function routes(fastify: FastifyInstance) { const summary = 'Gets all buckets' const schema = createDefaultSchema(successResponseSchema, { + querystring: requestQuerySchema, summary, tags: ['bucket'], }) - fastify.get( + fastify.get( '/', { schema, @@ -39,8 +55,10 @@ export default async function routes(fastify: FastifyInstance) { }, }, async (request, response) => { + const { limit, offset, sortColumn, sortOrder } = request.query const results = await request.storage.listBuckets( - 'id, name, public, owner, created_at, updated_at, file_size_limit, allowed_mime_types' + 'id, name, public, owner, created_at, updated_at, file_size_limit, allowed_mime_types', + { limit, offset, sortColumn, sortOrder } ) return response.send(results) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index a81d8a12..c581f4d1 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -44,6 +44,13 @@ export interface DatabaseOptions { parentConnection?: TenantConnection } +export interface ListBucketOptions { + limit?: number + offset?: number + sortColumn?: string + sortOrder?: string +} + export interface Database { tenantHost: string tenantId: string @@ -108,7 +115,7 @@ export interface Database { } ): Promise - listBuckets(columns: string): Promise + listBuckets(columns: string, options?: ListBucketOptions): Promise mustLockObject(bucketId: string, objectName: string, version?: string): Promise waitObjectLock( bucketId: string, diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index eb511b4c..a69effa4 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -15,6 +15,7 @@ import { FindBucketFilters, FindObjectFilters, SearchObjectOption, + ListBucketOptions, } from './adapter' import { DatabaseError } from 'pg' import { getTenantConfig, TenantConnection } from '@internal/database' @@ -283,9 +284,23 @@ export class StorageKnexDB implements Database { }) } - async listBuckets(columns = 'id') { + async listBuckets(columns = 'id', options?: ListBucketOptions) { const data = await this.runQuery('ListBuckets', (knex) => { - return knex.from('buckets').select(columns.split(',')) + const query = knex.from('buckets').select(columns.split(',')) + + if (options?.sortColumn) { + query.orderBy(options.sortColumn, options.sortOrder || 'asc') + } + + if (options?.limit) { + query.limit(options.limit) + } + + if (options?.offset) { + query.offset(options.offset) + } + + return query }) return data as Bucket[] diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 07de3018..e17c1485 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -1,5 +1,5 @@ import { StorageBackendAdapter, withOptionalVersion } from './backend' -import { Database, FindBucketFilters } from './database' +import { Database, FindBucketFilters, ListBucketOptions } from './database' import { ERRORS } from '@internal/errors' import { AssetRenderer, HeadRenderer, ImageRenderer } from './renderer' import { getFileSizeLimit, mustBeValidBucketName, parseFileSizeToBytes } from './limits' @@ -68,9 +68,10 @@ export class Storage { /** * List buckets * @param columns + * @param options */ - listBuckets(columns = 'id') { - return this.db.listBuckets(columns) + listBuckets(columns = 'id', options?: ListBucketOptions) { + return this.db.listBuckets(columns, options) } /** From 9b315531d9e93f28a9c930de2681b9f9fa309fec Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Jun 2025 17:05:56 -0700 Subject: [PATCH 2/6] test: added test for get buckets with limit, offset and sorting --- src/test/bucket.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts index 69c559d2..0f186dd8 100644 --- a/src/test/bucket.test.ts +++ b/src/test/bucket.test.ts @@ -141,7 +141,29 @@ describe('testing GET all buckets', () => { }) expect(response.statusCode).toBe(400) }) + + test('user is able to get buckets with limit, offset and sorting', async () => { + const response = await appInstance.inject({ + method: 'GET', + url: `/bucket?limit=1&offset=2&sortColumn=name&sortOrder=asc`, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + expect(response.statusCode).toBe(200) + const responseJSON = JSON.parse(response.body) + expect(responseJSON.length).toEqual(1) + expect(responseJSON[0]).toMatchObject({ + id: "bucket4", + name: "bucket4", + public: false, + file_size_limit: null, + allowed_mime_types: null, + }) + }) + }) + /* * POST /bucket */ From 0bb6e88f34e4a3aedaa47d230d21e82c41264f55 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Jun 2025 17:07:35 -0700 Subject: [PATCH 3/6] fix: code formatting --- src/test/bucket.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts index 0f186dd8..f585b031 100644 --- a/src/test/bucket.test.ts +++ b/src/test/bucket.test.ts @@ -154,14 +154,13 @@ describe('testing GET all buckets', () => { const responseJSON = JSON.parse(response.body) expect(responseJSON.length).toEqual(1) expect(responseJSON[0]).toMatchObject({ - id: "bucket4", - name: "bucket4", - public: false, + id: 'bucket4', + name: 'bucket4', + public: false, file_size_limit: null, allowed_mime_types: null, }) }) - }) /* From 728ffa762b2e14d0504bf7c9f19b778030fdb7e7 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Thu, 12 Jun 2025 15:08:05 -0700 Subject: [PATCH 4/6] feat: add serach param, explicit undefined checking, tests on limit & offset 0 --- src/http/routes/bucket/getAllBuckets.ts | 7 ++++--- src/storage/database/adapter.ts | 1 + src/storage/database/knex.ts | 10 +++++++--- src/test/bucket.test.ts | 26 +++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index 8ffeff6a..745b5ddc 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -28,9 +28,10 @@ const requestQuerySchema = { type: 'object', properties: { limit: { type: 'integer', minimum: 1, examples: [10] }, - offset: { type: 'integer', minimum: 0, examples: [0] }, + offset: { type: 'integer', minimum: 1, examples: [1] }, sortColumn: { type: 'string', enum: ['id', 'name', 'created_at', 'updated_at'] }, sortOrder: { type: 'string', enum: ['asc', 'desc'] }, + search: { type: 'string', examples: ["my-bucket"] } }, } as const @@ -55,10 +56,10 @@ export default async function routes(fastify: FastifyInstance) { }, }, async (request, response) => { - const { limit, offset, sortColumn, sortOrder } = request.query + const { limit, offset, sortColumn, sortOrder, search } = request.query const results = await request.storage.listBuckets( 'id, name, public, owner, created_at, updated_at, file_size_limit, allowed_mime_types', - { limit, offset, sortColumn, sortOrder } + { limit, offset, sortColumn, sortOrder, search } ) return response.send(results) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index c581f4d1..9382d05e 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -49,6 +49,7 @@ export interface ListBucketOptions { offset?: number sortColumn?: string sortOrder?: string + search?: string } export interface Database { diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index a69effa4..29dff23b 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -288,15 +288,19 @@ export class StorageKnexDB implements Database { const data = await this.runQuery('ListBuckets', (knex) => { const query = knex.from('buckets').select(columns.split(',')) - if (options?.sortColumn) { + if (options?.search !== undefined) { + query.where('name', 'like', `${options.search}%`) + } + + if (options?.sortColumn !== undefined) { query.orderBy(options.sortColumn, options.sortOrder || 'asc') } - if (options?.limit) { + if (options?.limit !== undefined) { query.limit(options.limit) } - if (options?.offset) { + if (options?.offset !== undefined) { query.offset(options.offset) } diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts index f585b031..e0b11ce3 100644 --- a/src/test/bucket.test.ts +++ b/src/test/bucket.test.ts @@ -142,10 +142,10 @@ describe('testing GET all buckets', () => { expect(response.statusCode).toBe(400) }) - test('user is able to get buckets with limit, offset and sorting', async () => { + test('user is able to get buckets with limit, offset, search and sorting', async () => { const response = await appInstance.inject({ method: 'GET', - url: `/bucket?limit=1&offset=2&sortColumn=name&sortOrder=asc`, + url: `/bucket?limit=1&offset=2&sortColumn=name&sortOrder=asc&search=bucket`, headers: { authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, }, @@ -161,6 +161,28 @@ describe('testing GET all buckets', () => { allowed_mime_types: null, }) }) + + test('limit=0 returns 400', async () => { + const response = await appInstance.inject({ + method: 'GET', + url: `/bucket?limit=0`, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + expect(response.statusCode).toBe(400) + }) + + test('offset=0 returns 400', async () => { + const response = await appInstance.inject({ + method: 'GET', + url: `/bucket?offset=0`, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + expect(response.statusCode).toBe(400) + }) }) /* From 6e43e243c4bef0b6059cd378f808401c0626c10d Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Wed, 25 Jun 2025 11:36:31 -0500 Subject: [PATCH 5/6] fix: offset min 0, exclude empty strings, ilike search and search anywhere in bucket name --- src/http/routes/bucket/getAllBuckets.ts | 2 +- src/storage/database/knex.ts | 12 ++++++------ src/test/bucket.test.ts | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index 745b5ddc..d3e8ed93 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -28,7 +28,7 @@ const requestQuerySchema = { type: 'object', properties: { limit: { type: 'integer', minimum: 1, examples: [10] }, - offset: { type: 'integer', minimum: 1, examples: [1] }, + offset: { type: 'integer', minimum: 0, examples: [0] }, sortColumn: { type: 'string', enum: ['id', 'name', 'created_at', 'updated_at'] }, sortOrder: { type: 'string', enum: ['asc', 'desc'] }, search: { type: 'string', examples: ["my-bucket"] } diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 29dff23b..6b02636f 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -288,9 +288,9 @@ export class StorageKnexDB implements Database { const data = await this.runQuery('ListBuckets', (knex) => { const query = knex.from('buckets').select(columns.split(',')) - if (options?.search !== undefined) { - query.where('name', 'like', `${options.search}%`) - } + if (options?.search !== undefined && options.search.length > 0) { + query.where('name', 'ilike', `%${options.search}%`) + } if (options?.sortColumn !== undefined) { query.orderBy(options.sortColumn, options.sortOrder || 'asc') @@ -593,8 +593,8 @@ export class StorageKnexDB implements Database { return object as typeof filters extends FindObjectFilters ? FindObjectFilters['dontErrorOnEmpty'] extends true - ? Obj | undefined - : Obj + ? Obj | undefined + : Obj : Obj } @@ -847,7 +847,7 @@ export class StorageKnexDB implements Database { const differentScopes = Boolean( this.options.parentConnection?.role && - this.connection.role !== this.options.parentConnection?.role + this.connection.role !== this.options.parentConnection?.role ) const needsNewTransaction = !tnx || differentScopes diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts index e0b11ce3..6ab9f32e 100644 --- a/src/test/bucket.test.ts +++ b/src/test/bucket.test.ts @@ -173,10 +173,10 @@ describe('testing GET all buckets', () => { expect(response.statusCode).toBe(400) }) - test('offset=0 returns 400', async () => { + test('offset=-1 returns 400', async () => { const response = await appInstance.inject({ method: 'GET', - url: `/bucket?offset=0`, + url: `/bucket?offset=-1`, headers: { authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, }, From 28b32519b450d4656e39867bcda41333160eadf3 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Wed, 25 Jun 2025 11:37:59 -0500 Subject: [PATCH 6/6] fix: formatting --- src/http/routes/bucket/getAllBuckets.ts | 2 +- src/storage/database/knex.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index d3e8ed93..3bcebb1d 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -31,7 +31,7 @@ const requestQuerySchema = { offset: { type: 'integer', minimum: 0, examples: [0] }, sortColumn: { type: 'string', enum: ['id', 'name', 'created_at', 'updated_at'] }, sortOrder: { type: 'string', enum: ['asc', 'desc'] }, - search: { type: 'string', examples: ["my-bucket"] } + search: { type: 'string', examples: ['my-bucket'] }, }, } as const diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 6b02636f..98dd7b8f 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -593,8 +593,8 @@ export class StorageKnexDB implements Database { return object as typeof filters extends FindObjectFilters ? FindObjectFilters['dontErrorOnEmpty'] extends true - ? Obj | undefined - : Obj + ? Obj | undefined + : Obj : Obj } @@ -847,7 +847,7 @@ export class StorageKnexDB implements Database { const differentScopes = Boolean( this.options.parentConnection?.role && - this.connection.role !== this.options.parentConnection?.role + this.connection.role !== this.options.parentConnection?.role ) const needsNewTransaction = !tnx || differentScopes