From 82279fa4f43b1cab648cb99cb74d76e54a097797 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Tue, 28 Jul 2026 20:59:21 -0700 Subject: [PATCH 01/40] refactor: introduce LlmClient with Letsur and OpenRouter providers Separate OpenAI-compatible LLM access behind an interface so the active provider can be switched via LLM_PROVIDER while defaulting to Letsur. --- .env.example | 13 +- docker-compose.yml | 5 +- src/chat/llm/base-openai-compatible.llm.ts | 268 +++++++++++++++++++++ src/chat/llm/letsur-llm.service.ts | 33 +++ src/chat/llm/llm-client.interface.ts | 35 +++ src/chat/llm/llm-client.provider.ts | 39 +++ src/chat/llm/open-router-llm.service.ts | 40 +++ src/chat/types/llm.types.ts | 83 +++++++ src/config/env.validation.ts | 33 ++- 9 files changed, 541 insertions(+), 8 deletions(-) create mode 100644 src/chat/llm/base-openai-compatible.llm.ts create mode 100644 src/chat/llm/letsur-llm.service.ts create mode 100644 src/chat/llm/llm-client.interface.ts create mode 100644 src/chat/llm/llm-client.provider.ts create mode 100644 src/chat/llm/open-router-llm.service.ts create mode 100644 src/chat/types/llm.types.ts diff --git a/.env.example b/.env.example index e8edb4f..bac1591 100644 --- a/.env.example +++ b/.env.example @@ -30,7 +30,10 @@ DOMAIN_NAME=example.com # MCP Server Url MCP_BASE_URL=your-mcp-server-url -# Letsur AI Gateway +# LLM Provider: letsur (default) | openrouter +LLM_PROVIDER=letsur + +# Letsur AI Gateway (LLM_PROVIDER=letsur) LETSUR_AI_GATEWAY_BASE_URL=https://gw.letsur.ai/v1 LETSUR_AI_GATEWAY_API_KEY=your-letsur-ai-gateway-api-key # LIGHT: 선별용(경로/chunk/문서 선택), NORMAL: 단순 응답, HEAVY: 최종 답변 생성 @@ -39,14 +42,14 @@ LETSUR_AI_GATEWAY_MODEL_NORMAL=gpt-4o-mini LETSUR_AI_GATEWAY_MODEL_HEAVY=gpt-4o-mini LETSUR_AI_GATEWAY_X_TITLE=GIST_CHATBOT -# Legacy: Open Router -# 기존 OpenRouter 연동용 환경변수입니다. Letsur AI Gateway 전환 후에는 기본적으로 사용하지 않습니다. +# OpenRouter (LLM_PROVIDER=openrouter 일 때만 필수) OPEN_ROUTER_API_KEY=your-openrouter-api-key +OPEN_ROUTER_BASE_URL=https://openrouter.ai/api/v1 OPEN_ROUTER_MODEL_LIGHT=google/gemini-2.0-flash-001 OPEN_ROUTER_MODEL_NORMAL=google/gemini-2.0-flash-001 OPEN_ROUTER_MODEL_HEAVY=openai/gpt-4o OPEN_ROUTER_X_TITLE=APP_NAME # Swagger API 문서 잠금 -# SWAGGER_USER=docs_admin -# SWAGGER_PASSWORD=your-secure-password \ No newline at end of file +SWAGGER_USER=docs_admin +SWAGGER_PASSWORD=your-secure-password \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5080f70..a801d46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,8 +29,8 @@ services: - '3000:3000' environment: # Database (Docker에서만 다른 값) - DB_HOST: postgres # Docker 네트워크 내 서비스 이름 - NODE_ENV: production # Docker는 프로덕션 모드 + DB_HOST: postgres # Docker 네트워크 내 서비스 이름 + NODE_ENV: production # Docker는 프로덕션 모드 # 나머지는 .env 파일에서 가져옴 DB_PORT: ${DB_PORT:-5432} DB_USER: ${DB_USER:-postgres} @@ -58,6 +58,7 @@ services: LETSUR_AI_GATEWAY_MODEL_NORMAL: ${LETSUR_AI_GATEWAY_MODEL_NORMAL:-} LETSUR_AI_GATEWAY_MODEL_HEAVY: ${LETSUR_AI_GATEWAY_MODEL_HEAVY:-} LETSUR_AI_GATEWAY_X_TITLE: ${LETSUR_AI_GATEWAY_X_TITLE:-} + LLM_PROVIDER: ${LLM_PROVIDER:-letsur} depends_on: postgres: condition: service_healthy diff --git a/src/chat/llm/base-openai-compatible.llm.ts b/src/chat/llm/base-openai-compatible.llm.ts new file mode 100644 index 0000000..9a35e46 --- /dev/null +++ b/src/chat/llm/base-openai-compatible.llm.ts @@ -0,0 +1,268 @@ +import { Logger, InternalServerErrorException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import { firstValueFrom } from 'rxjs'; +import { catchError } from 'rxjs/operators'; +import { AxiosError } from 'axios'; +import type { Readable } from 'stream'; +import { inspect } from 'node:util'; +import type { LlmClient, LlmCallOptions } from './llm-client.interface'; +import type { + LlmMessage, + LlmModelType, + LlmRequest, + LlmResponse, + LlmToolResult, +} from '../types/llm.types'; + +export type OpenAiCompatibleLlmConfig = { + apiKey: string; + baseUrl: string; + modelLight: string; + modelNormal: string; + modelHeavy: string; + xTitle?: string; + /** 로그/에러 메시지에 표시할 프로바이더 이름 */ + providerLabel: string; +}; + +/** + * OpenAI-compatible /chat/completions HTTP 클라이언트 공통 로직 + */ +export abstract class BaseOpenAiCompatibleLlm implements LlmClient { + protected abstract readonly logger: Logger; + + protected readonly apiKey: string; + protected readonly baseUrl: string; + protected readonly modelLight: string; + protected readonly modelNormal: string; + protected readonly modelHeavy: string; + protected readonly defaultModel: string; + protected readonly xTitle: string | undefined; + protected readonly providerLabel: string; + + constructor( + protected readonly httpService: HttpService, + protected readonly configService: ConfigService, + config: OpenAiCompatibleLlmConfig, + ) { + this.apiKey = config.apiKey; + this.baseUrl = config.baseUrl.replace(/\/+$/, ''); + this.modelLight = config.modelLight; + this.modelNormal = config.modelNormal; + this.modelHeavy = config.modelHeavy; + this.defaultModel = config.modelNormal; + this.xTitle = config.xTitle; + this.providerLabel = config.providerLabel; + } + + getModel(type: LlmModelType): string { + switch (type) { + case 'light': + return this.modelLight; + case 'normal': + return this.modelNormal; + case 'heavy': + return this.modelHeavy; + default: + return this.defaultModel; + } + } + + async callLLM( + messages: LlmMessage[], + model?: string, + options?: LlmCallOptions, + ): Promise { + const request: LlmRequest = { + model: model || this.defaultModel, + messages, + temperature: options?.temperature ?? 0.7, + max_tokens: options?.max_tokens ?? 2000, + }; + + const requestLogSummary = { + model: request.model, + stream: false, + temperature: request.temperature, + max_tokens: request.max_tokens, + messages: this.summarizeMessages(messages), + }; + + try { + const response = await firstValueFrom( + this.httpService + .post(`${this.baseUrl}/chat/completions`, request, { + headers: this.buildHeaders(), + timeout: 15000, + }) + .pipe( + catchError((error: AxiosError) => { + this.logApiError(error, requestLogSummary); + throw new InternalServerErrorException( + `Failed to call ${this.providerLabel} API: ${error.message}`, + ); + }), + ), + ); + + return response.data; + } catch (error) { + this.logger.error(`Error calling ${this.providerLabel}: ${error}`); + throw error; + } + } + + async generateFinalResponseStream( + messages: LlmMessage[], + toolResults: LlmToolResult[], + model?: string, + options?: { temperature?: number }, + ): Promise { + const toolMessages: LlmMessage[] = toolResults.map((result) => ({ + role: 'tool', + tool_call_id: result.tool_call_id, + name: result.name, + content: result.content, + })); + + const updatedMessages = [...messages, ...toolMessages]; + + const request: LlmRequest & { stream: boolean } = { + model: model || this.defaultModel, + messages: updatedMessages, + temperature: options?.temperature ?? 0.7, + max_tokens: 2000, + stream: true, + stream_options: { include_usage: true }, + }; + + const requestLogSummary = { + model: request.model, + stream: request.stream, + temperature: request.temperature, + max_tokens: request.max_tokens, + toolResultsCount: toolResults.length, + toolResultsContentCharsSum: toolResults.reduce( + (sum, r) => sum + (r.content?.length ?? 0), + 0, + ), + messages: this.summarizeMessages(updatedMessages), + }; + + try { + const response = await firstValueFrom( + this.httpService + .post(`${this.baseUrl}/chat/completions`, request, { + headers: this.buildHeaders(), + responseType: 'stream', + timeout: 15000, + }) + .pipe( + catchError((error: AxiosError) => { + this.logApiError(error, requestLogSummary); + throw new InternalServerErrorException( + `Failed to call ${this.providerLabel} API: ${error.message}`, + ); + }), + ), + ); + + return response.data; + } catch (error) { + this.logger.error(`Error calling ${this.providerLabel}: ${error}`); + throw error; + } + } + + protected buildHeaders(): Record { + return { + Authorization: `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': this.configService.get('DOMAIN_NAME') ?? '', + ...(this.xTitle ? { 'X-Title': this.xTitle } : {}), + }; + } + + protected logApiError(error: AxiosError, requestLogSummary: unknown): void { + const statusCode = error.response?.status; + const responseData = error.response?.data; + this.logger.error( + `${this.providerLabel} API error (status ${statusCode}): ${error.message}`, + error instanceof Error ? error.stack : undefined, + ); + if (responseData != null) { + this.logger.error( + `${this.providerLabel} error response body: ${this.safeStringify(responseData)}`, + ); + } + this.logger.error( + `${this.providerLabel} request summary: ${this.safeStringify(requestLogSummary)}`, + ); + } + + protected safeStringify(value: unknown, maxLen: number = 2000): string { + try { + if (typeof value === 'string') { + return value.length > maxLen + ? value.slice(0, maxLen) + '...(truncated)' + : value; + } + + if (typeof Buffer !== 'undefined' && Buffer.isBuffer(value as Buffer)) { + const str = (value as Buffer).toString('utf8'); + return str.length > maxLen + ? str.slice(0, maxLen) + '...(truncated)' + : str; + } + + const str = JSON.stringify(value, null, 2); + return str.length > maxLen + ? str.slice(0, maxLen) + '...(truncated)' + : str; + } catch { + const str = inspect(value, { + depth: 5, + maxArrayLength: 50, + breakLength: 120, + }); + return str.length > maxLen + ? str.slice(0, maxLen) + '...(truncated)' + : str; + } + } + + protected summarizeMessages(messages: LlmMessage[]) { + const roleCounts: Record = {}; + let assistantToolCalls = 0; + let toolRoleMessages = 0; + let toolRoleHasNameField = 0; + let contentNullCount = 0; + let contentCharsSum = 0; + + for (const m of messages) { + roleCounts[m.role] = (roleCounts[m.role] ?? 0) + 1; + if (m.role === 'assistant' && m.tool_calls?.length) { + assistantToolCalls += m.tool_calls.length; + } + if (m.role === 'tool') { + toolRoleMessages += 1; + if ((m as unknown as { name?: unknown }).name != null) { + toolRoleHasNameField += 1; + } + } + if (m.content === null) contentNullCount += 1; + if (typeof m.content === 'string') contentCharsSum += m.content.length; + } + + return { + total: messages.length, + roleCounts, + assistantToolCalls, + toolRoleMessages, + toolRoleHasNameField, + contentNullCount, + contentCharsSum, + }; + } +} diff --git a/src/chat/llm/letsur-llm.service.ts b/src/chat/llm/letsur-llm.service.ts new file mode 100644 index 0000000..4e761ac --- /dev/null +++ b/src/chat/llm/letsur-llm.service.ts @@ -0,0 +1,33 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import { BaseOpenAiCompatibleLlm } from './base-openai-compatible.llm'; + +/** + * Letsur AI Gateway LLM 클라이언트 + */ +@Injectable() +export class LetsurLlmService extends BaseOpenAiCompatibleLlm { + protected readonly logger = new Logger(LetsurLlmService.name); + + constructor(httpService: HttpService, configService: ConfigService) { + const fallback = + configService.get('LETSUR_AI_GATEWAY_MODEL') || 'gpt-4o-mini'; + const xTitle = + configService.get('LETSUR_AI_GATEWAY_X_TITLE') || + configService.get('LETSUR_AI_GATEWAY_TITLE'); + + super(httpService, configService, { + apiKey: configService.getOrThrow('LETSUR_AI_GATEWAY_API_KEY'), + baseUrl: configService.getOrThrow('LETSUR_AI_GATEWAY_BASE_URL'), + modelLight: + configService.get('LETSUR_AI_GATEWAY_MODEL_LIGHT') || fallback, + modelNormal: + configService.get('LETSUR_AI_GATEWAY_MODEL_NORMAL') || fallback, + modelHeavy: + configService.get('LETSUR_AI_GATEWAY_MODEL_HEAVY') || fallback, + xTitle, + providerLabel: 'Letsur AI Gateway', + }); + } +} diff --git a/src/chat/llm/llm-client.interface.ts b/src/chat/llm/llm-client.interface.ts new file mode 100644 index 0000000..81863a2 --- /dev/null +++ b/src/chat/llm/llm-client.interface.ts @@ -0,0 +1,35 @@ +import type { Readable } from 'stream'; +import type { + LlmMessage, + LlmModelType, + LlmResponse, + LlmToolResult, +} from '../types/llm.types'; + +export const LLM_CLIENT = Symbol('LLM_CLIENT'); + +export type LlmCallOptions = { + temperature?: number; + max_tokens?: number; +}; + +/** + * OpenAI-compatible LLM 클라이언트 인터페이스 + * Letsur / OpenRouter 등 프로바이더 구현체가 이 계약을 따릅니다. + */ +export interface LlmClient { + getModel(type: LlmModelType): string; + + callLLM( + messages: LlmMessage[], + model?: string, + options?: LlmCallOptions, + ): Promise; + + generateFinalResponseStream( + messages: LlmMessage[], + toolResults: LlmToolResult[], + model?: string, + options?: { temperature?: number }, + ): Promise; +} diff --git a/src/chat/llm/llm-client.provider.ts b/src/chat/llm/llm-client.provider.ts new file mode 100644 index 0000000..f36ce93 --- /dev/null +++ b/src/chat/llm/llm-client.provider.ts @@ -0,0 +1,39 @@ +import { Provider } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import { LLM_CLIENT } from './llm-client.interface'; +import { LetsurLlmService } from './letsur-llm.service'; +import { OpenRouterLlmService } from './open-router-llm.service'; + +export type LlmProviderName = 'letsur' | 'openrouter'; + +export function resolveLlmProviderName( + value: string | undefined, +): LlmProviderName { + const normalized = (value || 'letsur').toLowerCase().trim(); + if (normalized === 'openrouter') { + return 'openrouter'; + } + return 'letsur'; +} + +/** + * LLM_PROVIDER 환경변수에 따라 Letsur / OpenRouter 구현체를 선택합니다. + * 기본값: letsur + */ +export const llmClientProvider: Provider = { + provide: LLM_CLIENT, + useFactory: ( + configService: ConfigService, + httpService: HttpService, + ): LetsurLlmService | OpenRouterLlmService => { + const provider = resolveLlmProviderName( + configService.get('LLM_PROVIDER'), + ); + if (provider === 'openrouter') { + return new OpenRouterLlmService(httpService, configService); + } + return new LetsurLlmService(httpService, configService); + }, + inject: [ConfigService, HttpService], +}; diff --git a/src/chat/llm/open-router-llm.service.ts b/src/chat/llm/open-router-llm.service.ts new file mode 100644 index 0000000..dde65f9 --- /dev/null +++ b/src/chat/llm/open-router-llm.service.ts @@ -0,0 +1,40 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import { BaseOpenAiCompatibleLlm } from './base-openai-compatible.llm'; + +const DEFAULT_OPEN_ROUTER_BASE_URL = 'https://openrouter.ai/api/v1'; +const DEFAULT_OPEN_ROUTER_MODEL = 'openai/gpt-4o-mini'; + +/** + * OpenRouter LLM 클라이언트 + * LLM_PROVIDER=openrouter 일 때 사용합니다. + */ +@Injectable() +export class OpenRouterLlmService extends BaseOpenAiCompatibleLlm { + protected readonly logger = new Logger(OpenRouterLlmService.name); + + constructor(httpService: HttpService, configService: ConfigService) { + const fallback = + configService.get('OPEN_ROUTER_MODEL') || + DEFAULT_OPEN_ROUTER_MODEL; + const xTitle = + configService.get('OPEN_ROUTER_X_TITLE') || + configService.get('OPEN_ROUTER_TITLE'); + + super(httpService, configService, { + apiKey: configService.getOrThrow('OPEN_ROUTER_API_KEY'), + baseUrl: + configService.get('OPEN_ROUTER_BASE_URL') || + DEFAULT_OPEN_ROUTER_BASE_URL, + modelLight: + configService.get('OPEN_ROUTER_MODEL_LIGHT') || fallback, + modelNormal: + configService.get('OPEN_ROUTER_MODEL_NORMAL') || fallback, + modelHeavy: + configService.get('OPEN_ROUTER_MODEL_HEAVY') || fallback, + xTitle, + providerLabel: 'OpenRouter', + }); + } +} diff --git a/src/chat/types/llm.types.ts b/src/chat/types/llm.types.ts new file mode 100644 index 0000000..865bdc0 --- /dev/null +++ b/src/chat/types/llm.types.ts @@ -0,0 +1,83 @@ +/** + * OpenAI-compatible chat completions 메시지 형식 + */ +export interface LlmMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string | null; + tool_call_id?: string; + name?: string; + tool_calls?: LlmToolCall[]; +} + +/** + * Function calling tool call 형식 + */ +export interface LlmToolCall { + id: string; + type: 'function'; + function: { + name: string; + arguments: string; // JSON string + }; +} + +/** + * OpenAI-compatible chat completions 요청 형식 + */ +export interface LlmRequest { + model: string; + messages: LlmMessage[]; + tools?: LlmTool[]; + tool_choice?: + | 'auto' + | 'none' + | { type: 'function'; function: { name: string } }; + temperature?: number; + max_tokens?: number; + stream?: boolean; + stream_options?: { + include_usage?: boolean; + }; +} + +export interface LlmTool { + type: 'function'; + function: { + name: string; + description: string; + parameters: { + type: string; + properties?: Record; + required?: string[]; + }; + }; +} + +export interface LlmUsage { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; +} + +/** + * OpenAI-compatible chat completions 응답 형식 + */ +export interface LlmResponse { + id: string; + model: string; + choices: Array<{ + index: number; + message: LlmMessage; + finish_reason: 'stop' | 'length' | 'tool_calls' | null; + }>; + usage?: LlmUsage; +} + +/** 용도별 모델 티어 (light: 선별, normal: 단순 응답, heavy: 최종 답변) */ +export type LlmModelType = 'light' | 'normal' | 'heavy'; + +export type LlmToolResult = { + tool_call_id: string; + name: string; + content: string; +}; diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 2fa7a19..bd798b1 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -9,6 +9,7 @@ import { Min, Max, MinLength, + ValidateIf, validateSync, } from 'class-validator'; @@ -18,6 +19,11 @@ enum Environment { Test = 'test', } +enum LlmProvider { + Letsur = 'letsur', + OpenRouter = 'openrouter', +} + /** * 환경 변수 검증 클래스 * 애플리케이션 시작 시 필수 환경 변수와 형식을 검증합니다. @@ -98,15 +104,40 @@ export class EnvironmentVariables { @IsNotEmpty() IDP_CLIENT_SECRET: string; - // Letsur AI Gateway Configuration + // LLM Provider: letsur (default) | openrouter + @IsOptional() + @IsEnum(LlmProvider) + LLM_PROVIDER?: LlmProvider; + + // Letsur AI Gateway Configuration (required when LLM_PROVIDER=letsur) + @ValidateIf( + (o: EnvironmentVariables) => + (o.LLM_PROVIDER ?? LlmProvider.Letsur) === LlmProvider.Letsur, + ) @IsString() @IsNotEmpty() LETSUR_AI_GATEWAY_BASE_URL: string; + @ValidateIf( + (o: EnvironmentVariables) => + (o.LLM_PROVIDER ?? LlmProvider.Letsur) === LlmProvider.Letsur, + ) @IsString() @IsNotEmpty() LETSUR_AI_GATEWAY_API_KEY: string; + // OpenRouter Configuration (required when LLM_PROVIDER=openrouter) + @ValidateIf( + (o: EnvironmentVariables) => o.LLM_PROVIDER === LlmProvider.OpenRouter, + ) + @IsString() + @IsNotEmpty() + OPEN_ROUTER_API_KEY: string; + + @IsOptional() + @IsString() + OPEN_ROUTER_BASE_URL?: string; + // Client Domain Configuration @IsString() @IsNotEmpty() From 0362c626a866856016f6d1af3ee2deb073046f08 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Tue, 28 Jul 2026 20:59:36 -0700 Subject: [PATCH 02/40] refactor: split chat orchestration into focused services Extract resource selection, resource content fetching, and SSE transport so ChatOrchestrationService only coordinates the turn flow. --- src/chat/chat.module.ts | 17 +- src/chat/prompts/index.ts | 1 - src/chat/prompts/tool-selection.prompt.ts | 51 - .../services/chat-orchestration.service.ts | 1388 ++--------------- src/chat/services/chat-stream.transport.ts | 128 ++ src/chat/services/open-router.service.ts | 537 ------- src/chat/services/resource-content.service.ts | 782 ++++++++++ .../services/resource-selection.service.ts | 249 +++ src/chat/types/open-router.types.ts | 99 -- 9 files changed, 1260 insertions(+), 1992 deletions(-) delete mode 100644 src/chat/prompts/tool-selection.prompt.ts create mode 100644 src/chat/services/chat-stream.transport.ts delete mode 100644 src/chat/services/open-router.service.ts create mode 100644 src/chat/services/resource-content.service.ts create mode 100644 src/chat/services/resource-selection.service.ts delete mode 100644 src/chat/types/open-router.types.ts diff --git a/src/chat/chat.module.ts b/src/chat/chat.module.ts index 8002038..52d61b7 100644 --- a/src/chat/chat.module.ts +++ b/src/chat/chat.module.ts @@ -2,16 +2,27 @@ import { Module } from '@nestjs/common'; import { HttpModule } from '@nestjs/axios'; import { ChatController } from './chat.controller'; import { ChatService } from './services/chat.service'; -import { OpenRouterService } from './services/open-router.service'; import { ChatOrchestrationService } from './services/chat-orchestration.service'; +import { ResourceSelectionService } from './services/resource-selection.service'; +import { ResourceContentService } from './services/resource-content.service'; +import { ChatStreamTransport } from './services/chat-stream.transport'; import { AuthModule } from '../auth/auth.module'; import { McpModule } from '../mcp/mcp.module'; import { UsageModule } from '../usage/usage.module'; +import { LLM_CLIENT } from './llm/llm-client.interface'; +import { llmClientProvider } from './llm/llm-client.provider'; @Module({ imports: [HttpModule, AuthModule, McpModule, UsageModule], controllers: [ChatController], - providers: [ChatService, OpenRouterService, ChatOrchestrationService], - exports: [OpenRouterService], + providers: [ + ChatService, + llmClientProvider, + ResourceSelectionService, + ResourceContentService, + ChatStreamTransport, + ChatOrchestrationService, + ], + exports: [LLM_CLIENT], }) export class ChatModule {} diff --git a/src/chat/prompts/index.ts b/src/chat/prompts/index.ts index 6923e5d..8d5572b 100644 --- a/src/chat/prompts/index.ts +++ b/src/chat/prompts/index.ts @@ -2,7 +2,6 @@ * 프롬프트 모듈 통합 export */ -export * from './tool-selection.prompt'; export * from './document-selection.prompt'; export * from './resource-path-selection.prompt'; export * from './final-response.prompt'; diff --git a/src/chat/prompts/tool-selection.prompt.ts b/src/chat/prompts/tool-selection.prompt.ts deleted file mode 100644 index 6d3c86f..0000000 --- a/src/chat/prompts/tool-selection.prompt.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Tool 선택을 위한 시스템 프롬프트 - */ - -export interface ToolSelectionPromptParams { - toolsDescription: string; - emphasizeToolUsage?: boolean; -} - -/** - * Tool 선택 시스템 프롬프트 생성 - */ -export function getToolSelectionSystemPrompt( - params: ToolSelectionPromptParams, -): string { - const { toolsDescription, emphasizeToolUsage = false } = params; - - if (emphasizeToolUsage) { - return ` - You are a helpful assistant that MUST use tools to answer user questions. The user is asking in Korean, and you need to use available tools to find the information needed to answer their question. - - CRITICAL RULES - READ CAREFULLY: - 1. You MUST analyze the user's question and determine which tool(s) to use. - 2. You MUST use at least one tool if any tool is relevant to the question. - 3. If you don't use a tool when one is available, you will FAIL to answer correctly. - 4. When using a tool, provide all necessary arguments based on the tool's parameters. Make sure to provide complete and accurate arguments. - 5. DO NOT respond without using tools if a relevant tool exists. - 6. Tool usage is MANDATORY when tools are available and relevant. - 7. If multiple tools are relevant, you can and should use multiple tools in parallel. - - Available tools: - ${toolsDescription} - - Remember: Using tools is MANDATORY when they are relevant to the question. Failure to use tools will result in incorrect answers. Always use tools to find the information needed before responding. `; - } - - return `You are a helpful assistant that MUST use tools to help users answer questions. - - IMPORTANT INSTRUCTIONS: - 1. You MUST analyze the user's question carefully and determine if any of the available tools can help answer it. - 2. If a tool is relevant to the user's question, you MUST use it. Do not skip tool usage when it would be helpful. - 3. If multiple tools are relevant, you can use multiple tools. - 4. When using a tool, provide all necessary arguments based on the tool's parameters. - 5. If no tool is relevant, you can respond without using tools, but ONLY if the question is truly unrelated to any available tool. - - Available tools: - ${toolsDescription} - - Remember: Using the right tool is crucial for providing accurate and helpful answers. - `; -} diff --git a/src/chat/services/chat-orchestration.service.ts b/src/chat/services/chat-orchestration.service.ts index 3c8b184..d4b5941 100644 --- a/src/chat/services/chat-orchestration.service.ts +++ b/src/chat/services/chat-orchestration.service.ts @@ -1,45 +1,29 @@ import { + Inject, Injectable, Logger, InternalServerErrorException, } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import type { - ListResourcesResult, - ListResourceItem, -} from '../../mcp/mcp-client.service'; +import type { ListResourcesResult } from '../../mcp/mcp-client.service'; import { McpClientService } from '../../mcp/mcp-client.service'; -import { OpenRouterService } from './open-router.service'; import { ChatService } from './chat.service'; import { UsageService } from '../../usage/usage.service'; -import type { - McpTool, - OpenRouterMessage, - OpenRouterUsage, -} from '../types/open-router.types'; +import { LLM_CLIENT, type LlmClient } from '../llm/llm-client.interface'; +import type { LlmMessage, LlmUsage } from '../types/llm.types'; import { MessageRole } from '../../common/dto/chat-message-input.dto'; import type { Readable } from 'stream'; import type { FastifyReply, FastifyRequest } from 'fastify'; import { - DOCUMENT_SELECTION_SYSTEM_PROMPT, - getDocumentSelectionUserPrompt, - RESOURCE_PATH_SELECTION_SYSTEM_PROMPT, - getResourcePathSelectionUserPrompt, - CHUNK_SELECTION_SYSTEM_PROMPT, - getChunkSelectionUserPrompt, - formatResourceListForChunkSelection, FINAL_RESPONSE_SYSTEM_PROMPT, NO_RELEVANT_MATERIALS_SYSTEM_PROMPT, } from '../prompts'; +import { + ResourceContentService, + type ResourceInfo, +} from './resource-content.service'; +import { ChatStreamTransport } from './chat-stream.transport'; -/** - * 리소스 정보 - */ -export interface ResourceInfo { - path: string; // 문서 제목 (PDF/PNG인 경우 format 포함) - formats: string[]; - url: string; -} +export type { ResourceInfo }; interface ProcessUserQuestionStreamOptions { persistUserMessage?: boolean; @@ -58,23 +42,16 @@ interface StreamingResponseOptions extends ProcessUserQuestionStreamOptions { export class ChatOrchestrationService { private readonly logger = new Logger(ChatOrchestrationService.name); - // Tool 목록 캐시 (5분간 유효) - private cachedTools: McpTool[] | null = null; - private cachedToolsTimestamp: number = 0; - private readonly TOOLS_CACHE_TTL = 5 * 60 * 1000; // 5분 - - // Tool 실행 타임아웃 (30초) - private readonly TOOL_EXECUTION_TIMEOUT = 30 * 1000; - constructor( private readonly mcpClientService: McpClientService, - private readonly openRouterService: OpenRouterService, + @Inject(LLM_CLIENT) private readonly llmClient: LlmClient, private readonly chatService: ChatService, private readonly usageService: UsageService, - private readonly configService: ConfigService, + private readonly resourceContentService: ResourceContentService, + private readonly chatStreamTransport: ChatStreamTransport, ) {} - private createEmptyUsage(): OpenRouterUsage { + private createEmptyUsage(): LlmUsage { return { prompt_tokens: 0, completion_tokens: 0, @@ -83,8 +60,8 @@ export class ChatOrchestrationService { } private addTokenUsage( - target: OpenRouterUsage | undefined, - usage: OpenRouterUsage | null | undefined, + target: LlmUsage | undefined, + usage: LlmUsage | null | undefined, ): void { if (!target || !usage) return; @@ -97,7 +74,7 @@ export class ChatOrchestrationService { target.total_tokens += totalTokens; } - private hasTokenUsage(usage: OpenRouterUsage): boolean { + private hasTokenUsage(usage: LlmUsage): boolean { return ( usage.prompt_tokens > 0 || usage.completion_tokens > 0 || @@ -105,1114 +82,8 @@ export class ChatOrchestrationService { ); } - /** - * MCP Tool 목록을 가져오거나 캐시에서 반환 - */ - private async getMcpTools(): Promise { - const now = Date.now(); - if ( - this.cachedTools && - now - this.cachedToolsTimestamp < this.TOOLS_CACHE_TTL - ) { - this.logger.debug('Using cached MCP tools'); - return this.cachedTools; - } - - this.logger.debug('Fetching MCP tools from server...'); - const mcpToolsListResult: unknown = await this.mcpClientService.listTools(); - - type ToolItem = { - name: string; - description?: string; - inputSchema?: McpTool['inputSchema']; - }; - const mcpToolsList: ToolItem[] = Array.isArray(mcpToolsListResult) - ? (mcpToolsListResult as ToolItem[]) - : []; - const mcpTools: McpTool[] = mcpToolsList.map((tool) => ({ - name: tool.name, - description: tool.description ?? '', - inputSchema: tool.inputSchema, - })); - - this.cachedTools = mcpTools; - this.cachedToolsTimestamp = now; - - return mcpTools; - } - - /** - * 리소스 경로 정규화 - * MCP 서버는 확장자 없이 경로를 받고 자동으로 .md 또는 .pdf를 찾으므로, - * 확장자를 제거하여 전달합니다. - */ - private normalizeResourcePath(path: string): string { - // 확장자가 있으면 제거 (MCP 서버가 자동으로 찾음) - if (path.includes('.')) { - const lastDotIndex = path.lastIndexOf('.'); - // 마지막 점 이후가 확장자인 경우 (예: .md, .pdf) - const extension = path.substring(lastDotIndex + 1); - if (extension.length <= 5 && /^[a-z0-9]+$/i.test(extension)) { - return path.substring(0, lastDotIndex); - } - } - return path; - } - - /** - * 경로에서 마지막 문서 제목만 추출 (확장자 포함) - * 예: "2025 캠프 발표자료_ 1일차 오전/학생지원.md" -> "학생지원.md" - * 원본 경로에 확장자가 없으면 formats 배열에서 찾아서 추가 - */ - private extractDocumentTitle( - path: string, - originalPath?: string, - formats?: string[], - ): string { - // 원본 경로가 있으면 원본 경로 사용 (확장자 포함) - const pathToUse = originalPath || path; - - // 슬래시로 분리하여 마지막 부분만 반환 - const parts = pathToUse.split('/'); - let title = parts[parts.length - 1] || pathToUse; - - // 확장자가 없고 formats 배열에 md가 있으면 .md 추가 - if (!title.includes('.') && formats && formats.includes('md')) { - title = `${title}.md`; - } - - return title; - } - - /** - * FE·리소스 API용 PDF 경로: 하위 chunk/이미지 경로가 아니라 상위 묶음 PDF 한 개 - * 예: `에어컨+…/세부/파일.png` → `에어컨+….pdf` (첫 `/` 앞 세그먼트 + `.pdf`) - */ - private normalizeTopLevelPdfPathForFe(resourcePath: string): string { - const first = resourcePath.split('/')[0]?.trim() || resourcePath; - const base = first.replace(/\.(pdf|png|md|jpe?g|gif|webp)$/i, ''); - return `${base}.pdf`; - } - - /** - * SSE·메타데이터용 참조 문서: **PDF 번들만** (마크다운 chunk 경로는 상위 세그먼트 + `.pdf`로 변환). - * 예: `2026년+학사편람/…/졸업요건.md` → `2026년+학사편람.pdf` - */ - private appendFePdfResourceEntryFromUsed( - out: ResourceInfo[], - seenPdfPaths: Set, - r: { path: string; formats: string[] }, - ): void { - if (!r.path || !r.formats?.length) return; - const contributes = - r.formats.includes('md') || - r.formats.includes('pdf') || - r.formats.includes('png'); - if (!contributes) return; - - const pathForFe = this.normalizeTopLevelPdfPathForFe(r.path); - if (seenPdfPaths.has(pathForFe)) return; - seenPdfPaths.add(pathForFe); - - out.push({ - path: pathForFe, - formats: ['pdf'], - url: this.generateResourceUrl(pathForFe), - }); - } - - /** - * get_resource 툴 응답에서 텍스트 내용 추출 - * MCP 서버는 문자열을 직접 반환하므로, texts 배열이나 raw.content에서 추출 - */ - private extractContentFromToolResult( - toolResult: Awaited>, - ): string { - // texts 배열에서 내용 추출 (가장 일반적인 경우) - if (toolResult.texts.length > 0) { - // texts가 여러 개인 경우 합치기 - const content = toolResult.texts.join('\n'); - // JSON 문자열이 아닌 경우 그대로 반환 - if ( - content && - !content.trim().startsWith('{') && - !content.trim().startsWith('[') - ) { - return content; - } - } - - // raw.content에서 text 타입 항목 추출 - const raw = toolResult.raw as { - content?: Array<{ type: string; text?: string }>; - }; - if (raw?.content) { - const textContents: string[] = []; - for (const item of raw.content) { - if (item.type === 'text' && 'text' in item) { - const text = item.text; - // JSON 문자열이 아닌 경우 그대로 추가 - if ( - text && - !text.trim().startsWith('{') && - !text.trim().startsWith('[') - ) { - textContents.push(text); - } - } - } - if (textContents.length > 0) { - return textContents.join('\n'); - } - } - - return ''; - } - - /** - * 신 형식: LLM에게 description을 보고 관련 chunk 경로 최대 maxResults개 선택 (JSON 배열 반환) - */ - private async selectRelevantChunkPaths( - question: string, - resources: ListResourceItem[], - maxResults: number = 10, - tokenUsage?: OpenRouterUsage, - ): Promise { - if (!resources?.length) { - return []; - } - - const resourceListText = formatResourceListForChunkSelection(resources); - const userPrompt = getChunkSelectionUserPrompt({ - question, - resourceListText, - maxSelect: maxResults, - }); - - try { - const response = await this.openRouterService.callLLM( - [ - { role: 'system', content: CHUNK_SELECTION_SYSTEM_PROMPT }, - { role: 'user', content: userPrompt }, - ], - this.openRouterService.getModel('light'), - { temperature: 0.1, max_tokens: 5000 }, - ); - this.addTokenUsage(tokenUsage, response.usage); - - let selectedText = response.choices[0]?.message?.content?.trim() || ''; - this.logger.debug(`LLM chunk selection raw: ${selectedText}`); - - // 마크다운 코드블록 제거 (```json ... ```) - const codeBlockMatch = selectedText.match(/```(?:json)?\s*([\s\S]*?)```/); - if (codeBlockMatch) { - selectedText = codeBlockMatch[1].trim(); - } - - const parsed = JSON.parse(selectedText) as unknown; - const paths = Array.isArray(parsed) - ? (parsed as string[]).filter( - (p) => typeof p === 'string' && p.length > 0, - ) - : []; - - const limited = paths.slice(0, maxResults); - this.logger.log(`[DEBUG] 1차 선별 결과(chunk 경로): ${limited.length}개`); - return limited; - } catch (error) { - this.logger.warn( - `Failed to select chunk paths by LLM: ${error instanceof Error ? error.message : String(error)}`, - ); - return []; - } - } - - /** - * LLM으로 질문과 의미·맥락상 관련 있는 리소스 경로를 선별 (OpenRouter 사용, 구 형식) - * 키워드 매칭 대신 의미 기반으로 최대 maxResults개 선택합니다. - */ - private async selectRelevantResourcePaths( - question: string, - resources: Array<{ path: string; formats?: string[] }>, - maxResults: number = 10, - tokenUsage?: OpenRouterUsage, - ): Promise> { - if (!resources.length) { - return []; - } - - const pathList = resources.map((r, i) => `${i + 1}. ${r.path}`).join('\n'); - - const userPrompt = getResourcePathSelectionUserPrompt({ - pathList, - question, - maxSelect: maxResults, - }); - - try { - const response = await this.openRouterService.callLLM( - [ - { role: 'system', content: RESOURCE_PATH_SELECTION_SYSTEM_PROMPT }, - { role: 'user', content: userPrompt }, - ], - this.openRouterService.getModel('light'), - { temperature: 0.1, max_tokens: 200 }, - ); - this.addTokenUsage(tokenUsage, response.usage); - - const selectedText = response.choices[0]?.message?.content?.trim() || ''; - this.logger.debug(`LLM selected resource paths: ${selectedText}`); - - if (selectedText.toLowerCase().includes('없음')) { - return []; - } - - const numbers = - selectedText - .match(/\d+/g) - ?.map((n) => parseInt(n, 10) - 1) - .filter((n) => n >= 0 && n < resources.length) || []; - - const uniqueIndices = [...new Set(numbers)].slice(0, maxResults); - const selected = uniqueIndices.map((idx) => resources[idx]); - - this.logger.log(`Selected ${selected.length} resource path(s) by LLM`); - return selected; - } catch (error) { - this.logger.warn( - `Failed to select resource paths by LLM: ${error instanceof Error ? error.message : String(error)}`, - ); - return []; - } - } - - /** - * 리소스 내용에서 태그를 파싱하여 하위 문서 정보 추출 - */ - private parseDocumentLinks(content: string): Array<{ - path: string; - description: string; - }> { - const documents: Array<{ path: string; description: string }> = []; - const documentRegex = - /<\/document>/g; - - let match; - while ((match = documentRegex.exec(content)) !== null) { - documents.push({ - path: match[1], - description: match[2], - }); - } - - return documents; - } - - /** - * 마크다운에서 이미지 참조 추출: ![alt](path) 형태 - * 첨부된 이미지(.png, .jpg 등) 경로만 반환 - */ - private parseImageReferencesFromMarkdown(content: string): string[] { - const paths: string[] = []; - const imageRefRegex = /!\[[^\]]*\]\(([^)]+)\)/g; - let match; - while ((match = imageRefRegex.exec(content)) !== null) { - const path = match[1].trim(); - if (/\.(png|jpe?g|gif|webp)(\?|#|$)/i.test(path)) { - paths.push(path); - } - } - return paths; - } - - /** - * 문서 경로 기준으로 상대 이미지 경로를 전체 리소스 경로로 변환 - * 예: docPath="폴더/문서.md", imageRef="이미지.png" → "폴더/이미지.png" - */ - private resolveImagePath(imageRef: string, docPath: string): string { - const lastSlash = docPath.lastIndexOf('/'); - const dir = lastSlash === -1 ? '' : docPath.slice(0, lastSlash + 1); - return dir + imageRef; - } - - /** - * MD 링크/이미지의 상대 경로를 절대 리소스 경로로 변환 (`../` 처리) - */ - private resolveRelativeResourcePath(ref: string, docPath: string): string { - const raw = ref.trim().replace(/^<|>$/g, '').split(/[?#]/)[0]; - if (!raw || /^https?:\/\//i.test(raw)) return raw; - if (raw.startsWith('/')) return raw.replace(/^\/+/, ''); - const lastSlash = docPath.lastIndexOf('/'); - const dir = lastSlash === -1 ? '' : docPath.slice(0, lastSlash + 1); - const combined = dir + raw; - const segments = combined.split('/').filter((s) => s.length > 0); - const out: string[] = []; - for (const s of segments) { - if (s === '..') out.pop(); - else if (s !== '.') out.push(s); - } - return out.join('/'); - } - - /** - * 선별된 MD 본문에서 PDF/PNG 참조 경로 추출 (마크다운 링크, 이미지, ``) - */ - private extractPdfPngReferencesFromMarkdown( - content: string, - docPath: string, - ): Array<{ path: string; formats: string[] }> { - const results: Array<{ path: string; formats: string[] }> = []; - const seen = new Set(); - const add = (p: string, fmt: 'pdf' | 'png') => { - if (!p || seen.has(p)) return; - seen.add(p); - results.push({ path: p, formats: [fmt] }); - }; - - const mdLink = /\[([^\]]*)\]\(([^)]+)\)/g; - let m: RegExpExecArray | null; - while ((m = mdLink.exec(content)) !== null) { - const inner = m[2].trim(); - const raw = inner.split(/\s+/)[0]; - if (/\.pdf$/i.test(raw)) { - const full = this.resolveRelativeResourcePath(raw, docPath); - if (!/^https?:\/\//i.test(full)) add(full, 'pdf'); - } - if (/\.png$/i.test(raw)) { - const full = this.resolveRelativeResourcePath(raw, docPath); - if (!/^https?:\/\//i.test(full)) add(full, 'png'); - } - } - - for (const img of this.parseImageReferencesFromMarkdown(content)) { - const raw = img.trim().split(/[?#]/)[0]; - if (/\.png$/i.test(raw)) { - const full = this.resolveRelativeResourcePath(raw, docPath); - if (!/^https?:\/\//i.test(full)) add(full, 'png'); - } - } - - for (const d of this.parseDocumentLinks(content)) { - const p = d.path.trim(); - if (/\.pdf$/i.test(p)) { - const full = p.includes('/') - ? p - : this.resolveRelativeResourcePath(p, docPath); - if (!/^https?:\/\//i.test(full)) add(full, 'pdf'); - } - if (/\.png$/i.test(p)) { - const full = p.includes('/') - ? p - : this.resolveRelativeResourcePath(p, docPath); - if (!/^https?:\/\//i.test(full)) add(full, 'png'); - } - } - - return results; - } - - /** - * list_resources chunk 목록에서 선별된 상위 폴더와 같은 루트의 PDF/PNG chunk 경로 수집 - */ - private collectPdfPngPathsFromChunkCatalog( - chunks: Array<{ path: string }> | undefined, - selectedPaths: string[], - max: number = 8, - ): Array<{ path: string; formats: string[] }> { - if (!chunks?.length || !selectedPaths.length) return []; - const roots = new Set( - selectedPaths.map((p) => p.split('/')[0]).filter(Boolean), - ); - const out: Array<{ path: string; formats: string[] }> = []; - const seen = new Set(); - for (const c of chunks) { - const isPdf = /\.pdf$/i.test(c.path); - const isPng = /\.png$/i.test(c.path); - if (!isPdf && !isPng) continue; - const root = c.path.split('/')[0]; - if (!roots.has(root)) continue; - if (seen.has(c.path)) continue; - seen.add(c.path); - out.push({ path: c.path, formats: isPdf ? ['pdf'] : ['png'] }); - if (out.length >= max) break; - } - return out; - } - - /** - * 질문과 관련된 하위 문서 찾기 - */ - private findRelevantSubDocuments( - question: string, - documents: Array<{ path: string; description: string }>, - maxResults: number = 3, - ): Array<{ path: string; description: string }> { - const keywords = - question - .toLowerCase() - .match(/[\uac00-\ud7a3]+|[a-z]+/gi) - ?.filter((word) => word.length > 1) || []; - - if (keywords.length === 0) { - return documents.slice(0, maxResults); - } - - const scoredDocuments = documents.map((doc) => { - const pathLower = doc.path.toLowerCase(); - const descLower = doc.description.toLowerCase(); - let score = 0; - - for (const keyword of keywords) { - if (pathLower.includes(keyword)) { - score += keyword.length * 2; // 경로 매칭은 가중치 높게 - } - if (descLower.includes(keyword)) { - score += keyword.length; // 설명 매칭 - } - } - - return { document: doc, score }; - }); - - return scoredDocuments - .sort((a, b) => b.score - a.score) - .slice(0, maxResults) - .map((item) => item.document); - } - - /** - * 하위 문서 내용 가져오기 - */ - private async fetchSubDocumentContents( - subDocuments: Array<{ path: string; description: string }>, - ): Promise { - const results = await Promise.all( - subDocuments.map(async (doc) => { - try { - const resourcePath = this.normalizeResourcePath(doc.path); - this.logger.debug(`Fetching sub-document: ${resourcePath}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: resourcePath }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const documentTitle = this.extractDocumentTitle( - resourcePath, - doc.path, - ['md'], - ); - return `\n\n## 하위 문서: ${documentTitle}\n\n**설명**: ${doc.description}\n\n${content}`; - } - } catch (error) { - this.logger.warn( - `Failed to fetch sub-document ${doc.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return ''; - }), - ); - return results.filter(Boolean).join('\n'); - } - - /** - * LLM에게 문서 목록을 주고 질문과 관련성이 높은 문서만 선별하도록 요청 - */ - private async selectMostRelevantDocuments( - question: string, - documents: Array<{ title: string; content: string; path: string }>, - tokenUsage?: OpenRouterUsage, - ): Promise> { - if (documents.length === 0) { - return []; - } - - // 문서가 1개면 선별 불필요 - if (documents.length === 1) { - return documents; - } - - try { - // 제목 + 내용 앞부분(요약)을 주어 경로/제목에 키워드가 없어도 내용으로 관련 문서 선별 가능하게 함 - const CONTENT_SNIPPET_LENGTH = 280; - const documentList = documents - .map((doc, index) => { - const snippet = - doc.content.length > CONTENT_SNIPPET_LENGTH - ? doc.content - .slice(0, CONTENT_SNIPPET_LENGTH) - .replace(/\n/g, ' ') + '...' - : doc.content.replace(/\n/g, ' '); - return `${index + 1}. ${doc.title}\n 내용 요약: ${snippet}`; - }) - .join('\n\n'); - - const selectionPrompt = getDocumentSelectionUserPrompt({ - documentList, - question, - }); - - this.logger.debug( - `Selection prompt length: ${selectionPrompt.length} chars, documents: ${documents.length}`, - ); - - const response = await this.openRouterService.callLLM( - [ - { - role: 'system', - content: DOCUMENT_SELECTION_SYSTEM_PROMPT, - }, - { - role: 'user', - content: selectionPrompt, - }, - ], - this.openRouterService.getModel('normal'), - { temperature: 0.1, max_tokens: 100 }, - ); - this.addTokenUsage(tokenUsage, response.usage); - - const selectedText = response.choices[0]?.message?.content?.trim() || ''; - this.logger.debug(`LLM selected documents: ${selectedText}`); - - // "없음"이면 빈 배열 반환 (관련 없는 질문일 수 있으므로 문서 강제 선택 안 함) - if (selectedText.toLowerCase().includes('없음')) { - return []; - } - - // 번호 추출 (예: "1, 3, 5" 또는 "1,3,5") - const numbers = - selectedText - .match(/\d+/g) - ?.map((n) => parseInt(n, 10) - 1) // 0-based index로 변환 - .filter((n) => n >= 0 && n < documents.length) || []; - - if (numbers.length === 0) { - // 번호를 파싱할 수 없으면 앞쪽 문서 반환 (최대 5개) - this.logger.warn( - `Could not parse document selection, returning first 5 documents`, - ); - return documents.slice(0, 5); - } - - // 최대 5개로 제한 (중요 문서 놓치지 않도록) - const limitedNumbers = numbers.slice(0, 5); - const selected = limitedNumbers.map((idx) => documents[idx]); - this.logger.log( - `Selected ${selected.length} relevant document(s) out of ${documents.length}`, - ); - - return selected; - } catch (error) { - this.logger.warn( - `Failed to select relevant documents: ${error instanceof Error ? error.message : String(error)}`, - ); - // 에러 발생 시 모든 문서 반환 - return documents; - } - } - - /** - * 신 형식 list_resources: description 기반 chunk 선별 → get_resource(chunk_path) → 본문 수집 - * @param catalogChunks 전체 chunk 목록(평탄화). PDF/PNG 전용 chunk를 FE 참조용으로 붙일 때 사용 - */ - private async fetchRelevantContentsFromChunks( - question: string, - resources: ListResourceItem[], - catalogChunks?: Array<{ path: string }>, - tokenUsage?: OpenRouterUsage, - ): Promise<{ - content: string; - usedResources: Array<{ path: string; formats: string[] }>; - }> { - this.logger.log( - `[DEBUG] 1차 선별(description 기준) 입력: 상위 리소스 ${resources.length}개, chunk 총 ${resources.reduce((s, r) => s + (r.chunks?.length ?? 0), 0)}개 → LLM에 전달`, - ); - - let t0 = Date.now(); - const chunkPaths = await this.selectRelevantChunkPaths( - question, - resources, - 10, - tokenUsage, - ); - this.logger.log( - `[PERF] selectRelevantChunkPaths(LLM): ${Date.now() - t0}ms`, - ); - - if (chunkPaths.length === 0) { - return { content: '', usedResources: [] }; - } - - t0 = Date.now(); - const chunkResults = await Promise.all( - chunkPaths.map(async (chunkPath) => { - try { - const pathForTool = this.normalizeResourcePath(chunkPath); - this.logger.debug(`Fetching chunk: ${pathForTool}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: pathForTool }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const title = chunkPath.split('/').pop() || chunkPath || '문서'; - return { title, content, path: chunkPath }; - } - } catch (error) { - this.logger.warn( - `Failed to fetch chunk ${chunkPath}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return null; - }), - ); - const documentCandidates = chunkResults.filter( - (r): r is { title: string; content: string; path: string } => r !== null, - ); - this.logger.log( - `[PERF] get_resource 루프(신 형식, ${chunkPaths.length}개): ${Date.now() - t0}ms`, - ); - - if (documentCandidates.length === 0) { - return { content: '', usedResources: [] }; - } - - this.logger.log( - `[DEBUG] 2차 선별(본문 기준) 입력: 후보 문서 ${documentCandidates.length}개 → LLM에 전달`, - ); - - t0 = Date.now(); - const selectedDocuments = await this.selectMostRelevantDocuments( - question, - documentCandidates.map((doc) => ({ - title: doc.title, - content: doc.content, - path: doc.path, - })), - tokenUsage, - ); - this.logger.log( - `[PERF] selectMostRelevantDocuments(LLM, 신 형식): ${Date.now() - t0}ms`, - ); - - this.logger.log( - `[DEBUG] 2차 선별 결과(최종 사용 문서): ${selectedDocuments.length}개`, - ); - - if (selectedDocuments.length === 0) { - this.logger.log('No documents selected by LLM as relevant'); - return { content: '', usedResources: [] }; - } - - const contents: string[] = []; - const mdUsed: Array<{ path: string; formats: string[] }> = []; - - for (const selected of selectedDocuments) { - const doc = documentCandidates.find((d) => d.path === selected.path); - if (doc) { - contents.push(`\n\n## 리소스: ${doc.title}\n\n${doc.content}`); - mdUsed.push({ path: doc.path, formats: ['md'] }); - } - } - - const selectedPaths = selectedDocuments.map((s) => s.path); - const fromMarkdown: Array<{ path: string; formats: string[] }> = []; - for (const selected of selectedDocuments) { - const doc = documentCandidates.find((d) => d.path === selected.path); - if (!doc?.content) continue; - fromMarkdown.push( - ...this.extractPdfPngReferencesFromMarkdown(doc.content, doc.path), - ); - } - const fromCatalog = this.collectPdfPngPathsFromChunkCatalog( - catalogChunks, - selectedPaths, - 8, - ); - - const seenPdfPng = new Set(); - const pdfPngExtras: Array<{ path: string; formats: string[] }> = []; - for (const e of [...fromMarkdown, ...fromCatalog]) { - if (seenPdfPng.has(e.path)) continue; - seenPdfPng.add(e.path); - pdfPngExtras.push(e); - } - - const finalUsedResources = [ - ...mdUsed.slice(0, 5), - ...pdfPngExtras.slice(0, 8), - ]; - - return { - content: contents.join('\n'), - usedResources: finalUsedResources, - }; - } - - /** - * list_resources tool 응답에서 관련 리소스 내용 가져오기 - * - 신 형식(resources + chunks): description 보고 chunk 경로 선별 → get_resource(chunk_path) - * - 구 형식(filteredResources): 경로만 선별 후 get_resource - * @returns 문서 내용과 usedResources(선별 경로·formats; chunk는 md 포함). FE 참조 목록은 PDF/PNG만 노출. - */ - private async fetchRelevantResourceContents( - question: string, - listResult: ListResourcesResult, - tokenUsage?: OpenRouterUsage, - ): Promise<{ - content: string; - usedResources: Array<{ path: string; formats: string[] }>; - }> { - const isNewFormat = - listResult.resources && - listResult.resources.length > 0 && - listResult.chunks && - listResult.chunks.length > 0; - - if (isNewFormat) { - return this.fetchRelevantContentsFromChunks( - question, - listResult.resources!, - listResult.chunks, - tokenUsage, - ); - } - - const filteredResources = listResult.filteredResources; - if (!filteredResources || filteredResources.length === 0) { - return { content: '', usedResources: [] }; - } - - const mdResources = filteredResources.filter( - (resource) => resource.formats && resource.formats.includes('md'), - ); - - if (mdResources.length === 0) { - this.logger.debug('No markdown resources found in filtered resources'); - return { content: '', usedResources: [] }; - } - - this.logger.log( - `[DEBUG] 1차 선별(경로 기준) 입력: MD 문서 ${mdResources.length}개 → LLM에 전달`, - ); - - let t0 = Date.now(); - const relevantResources = await this.selectRelevantResourcePaths( - question, - mdResources, - 10, - tokenUsage, - ); - this.logger.log( - `[PERF] selectRelevantResourcePaths(LLM, 구 형식): ${Date.now() - t0}ms`, - ); - - if (relevantResources.length === 0) { - return { content: '', usedResources: [] }; - } - - this.logger.log( - `[DEBUG] 1차 선별 결과(상위 관련 문서 경로): ${relevantResources.length}개`, - ); - - t0 = Date.now(); - const resourceResults = await Promise.all( - relevantResources.map(async (resource) => { - try { - const resourcePath = this.normalizeResourcePath(resource.path); - this.logger.debug(`Fetching markdown resource: ${resourcePath}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: resourcePath }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const documentTitle = this.extractDocumentTitle( - resourcePath, - resource.path, - resource.formats, - ); - const subDocuments = this.parseDocumentLinks(content); - return { - title: documentTitle, - content, - path: resource.path, - formats: resource.formats || [], - subDocuments, - }; - } - } catch (error) { - this.logger.warn( - `Failed to fetch ${resource.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return null; - }), - ); - const documentCandidates = resourceResults.filter( - ( - r, - ): r is { - title: string; - content: string; - path: string; - formats: string[]; - subDocuments: Array<{ path: string; description: string }>; - } => r !== null, - ); - this.logger.log( - `[PERF] get_resource 루프(구 형식, ${relevantResources.length}개): ${Date.now() - t0}ms`, - ); - - if (documentCandidates.length === 0) { - return { content: '', usedResources: [] }; - } - - this.logger.log( - `[DEBUG] 2차 선별(본문 기준) 입력: 후보 문서 ${documentCandidates.length}개 → LLM에 전달`, - ); - - t0 = Date.now(); - const selectedDocuments = await this.selectMostRelevantDocuments( - question, - documentCandidates.map((doc) => ({ - title: doc.title, - content: doc.content, - path: doc.path, - })), - tokenUsage, - ); - this.logger.log( - `[PERF] selectMostRelevantDocuments(LLM, 구 형식): ${Date.now() - t0}ms`, - ); - - this.logger.log( - `[DEBUG] 2차 선별 결과(최종 사용 문서): ${selectedDocuments.length}개`, - ); - console.log('selectedDocuments', selectedDocuments); - - if (selectedDocuments.length === 0) { - this.logger.log('No documents selected by LLM as relevant'); - return { content: '', usedResources: [] }; - } - - const contents: string[] = []; - const allSubDocuments: Array<{ path: string; description: string }> = []; - const usedResources: Array<{ path: string; formats: string[] }> = []; - const addedPaths = new Set(); - - for (const selected of selectedDocuments) { - const docCandidate = documentCandidates.find( - (d) => d.title === selected.title, - ); - if (docCandidate) { - contents.push( - `\n\n## 리소스: ${docCandidate.title}\n\n${docCandidate.content}`, - ); - - const hasPdf = docCandidate.formats.includes('pdf'); - const hasPng = docCandidate.formats.includes('png'); - if (hasPdf || hasPng) { - const pdfPngFormats = docCandidate.formats.filter( - (f) => f === 'pdf' || f === 'png', - ); - usedResources.push({ - path: docCandidate.path, - formats: pdfPngFormats, - }); - addedPaths.add(docCandidate.path); - } - - if (docCandidate.subDocuments.length > 0) { - allSubDocuments.push(...docCandidate.subDocuments); - } - } - } - - for (const selected of selectedDocuments) { - const path = selected.path; - const firstSegment = path.split('/')[0]; - for (const r of filteredResources) { - if (!r.formats) continue; - if (addedPaths.has(r.path)) continue; - if (r.formats.includes('pdf')) { - const pathLower = r.path.toLowerCase(); - if (pathLower.endsWith('.png')) continue; - const match = - r.path === firstSegment || - r.path === `${firstSegment}.pdf` || - r.path.startsWith(`${firstSegment}.`); - if (match) { - usedResources.push({ path: r.path, formats: ['pdf'] }); - addedPaths.add(r.path); - } - } - } - } - - for (const selected of selectedDocuments) { - const docCandidate = documentCandidates.find( - (d) => d.title === selected.title, - ); - if (!docCandidate?.content) continue; - const imageRefs = this.parseImageReferencesFromMarkdown( - docCandidate.content, - ); - for (const imageRef of imageRefs) { - const fullPath = this.resolveImagePath(imageRef, docCandidate.path); - const pathWithoutExt = fullPath.replace(/\.(png|jpe?g|gif|webp)$/i, ''); - const r = filteredResources.find( - (x) => - x.formats?.includes('png') && - !addedPaths.has(x.path) && - (x.path === fullPath || - x.path === pathWithoutExt || - x.path.toLowerCase() === fullPath.toLowerCase() || - x.path.toLowerCase() === pathWithoutExt.toLowerCase()), - ); - if (r) { - usedResources.push({ path: r.path, formats: ['png'] }); - addedPaths.add(r.path); - } - } - } - - const finalUsedResources = usedResources.slice(0, 5); - - // 하위 문서 중 질문과 관련된 문서 찾아서 추가로 가져오기 - if (allSubDocuments.length > 0) { - const relevantSubDocuments = this.findRelevantSubDocuments( - question, - allSubDocuments, - 3, // 최대 3개의 하위 문서만 추가로 가져오기 - ); - - if (relevantSubDocuments.length > 0) { - this.logger.log( - `Fetching ${relevantSubDocuments.length} relevant sub-document(s)`, - ); - t0 = Date.now(); - const subDocumentContents = - await this.fetchSubDocumentContents(relevantSubDocuments); - this.logger.log( - `[PERF] fetchSubDocumentContents(${relevantSubDocuments.length}개): ${Date.now() - t0}ms`, - ); - if (subDocumentContents) { - contents.push('\n\n---\n\n## 관련 하위 문서\n' + subDocumentContents); - } - } - } - - return { - content: contents.join('\n'), - usedResources: finalUsedResources, - }; - } - - /** - * Tool 실행에 타임아웃 적용 - */ - private async executeToolWithTimeout( - toolCall: { - id: string; - name: string; - arguments: Record; - }, - sessionId: string, - userQuestion?: string, - ): Promise<{ - tool_call_id: string; - name: string; - content: string; - resources: ResourceInfo[]; - /** list_resources에서 실제 문서 내용을 붙였을 때만 true */ - hadReferenceContent: boolean; - }> { - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error(`Tool execution timeout: ${toolCall.name}`)); - }, this.TOOL_EXECUTION_TIMEOUT); - }); - - const executePromise = (async () => { - this.logger.debug( - `Calling tool: ${toolCall.name} with args: ${JSON.stringify(toolCall.arguments)}`, - ); - - const toolResult = await this.mcpClientService.callTool( - toolCall.name, - toolCall.arguments, - ); - - let resultText = - toolResult.texts.join('\n') || JSON.stringify(toolResult.raw, null, 2); - - // list_resources tool인 경우, 관련 리소스 내용을 가져와서 추가 - let usedResourcesFromContent: Array<{ - path: string; - formats: string[]; - }> = []; - let hadReferenceContent = false; - const listResult = - toolCall.name === 'list_resources' - ? (toolResult as ListResourcesResult) - : null; - const listHasResources = - listResult && - ((listResult.chunks && listResult.chunks.length > 0) || - (listResult.filteredResources && - listResult.filteredResources.length > 0)); - if ( - toolCall.name === 'list_resources' && - userQuestion && - listHasResources && - listResult - ) { - const relevantResult = await this.fetchRelevantResourceContents( - userQuestion, - listResult, - ); - const hasActualContent = - typeof relevantResult.content === 'string' && - relevantResult.content.trim().length > 0; - if (hasActualContent) { - resultText += '\n\n' + relevantResult.content; - usedResourcesFromContent = relevantResult.usedResources; - hadReferenceContent = true; - } - } - - // 실제 사용된 리소스만 포함 (FE·SSE: 묶음 PDF만) - const resources: ResourceInfo[] = []; - const seenFePdfPaths = new Set(); - if (usedResourcesFromContent.length > 0) { - for (const resource of usedResourcesFromContent) { - this.appendFePdfResourceEntryFromUsed( - resources, - seenFePdfPaths, - resource, - ); - } - } - - return { - tool_call_id: toolCall.id, - name: toolCall.name, - content: resultText, - resources, - hadReferenceContent, - }; - })(); - - return Promise.race([executePromise, timeoutPromise]); - } - /** * 사용자 질문을 처리하여 스트리밍 답변을 생성 - * @param sessionId 세션 ID - * @param userQuestion 사용자 질문 - * @returns 스트리밍 응답 스트림과 리소스 정보 */ async processUserQuestionStream( sessionId: string, @@ -1221,24 +92,22 @@ export class ChatOrchestrationService { ): Promise<{ stream: Readable; resources: ResourceInfo[]; - usage: OpenRouterUsage; + usage: LlmUsage; }> { const perfTurnStart = Date.now(); const usage = this.createEmptyUsage(); try { - // 0. 과거 대화 조회 (현재 user 저장 전 → 직전 대화까지 context) let t0 = Date.now(); const pastMessagesRaw = await this.chatService.getMessagesForContext( sessionId, options.historyBefore, ); - const historyMessages: OpenRouterMessage[] = [...pastMessagesRaw] + const historyMessages: LlmMessage[] = [...pastMessagesRaw] .reverse() .map((msg) => ({ role: msg.role, content: msg.content })); this.logger.log(`[PERF] getMessagesForContext: ${Date.now() - t0}ms`); - // 1. 사용자 메시지 저장 if (options.persistUserMessage ?? true) { t0 = Date.now(); await this.chatService.createMessage(sessionId, { @@ -1248,7 +117,6 @@ export class ChatOrchestrationService { this.logger.log(`[PERF] createMessage(user): ${Date.now() - t0}ms`); } - // 2. list_resources 직접 호출 (도구 선택 LLM 없이) t0 = Date.now(); this.logger.debug('Calling list_resources...'); const listResult = (await this.mcpClientService.callTool( @@ -1257,7 +125,6 @@ export class ChatOrchestrationService { )) as ListResourcesResult; this.logger.log(`[PERF] list_resources: ${Date.now() - t0}ms`); - // [DEBUG] list_resources 결과: 신 형식(resources+chunks) 또는 구 형식(filteredResources) const isNewFormat = listResult.resources && listResult.resources.length > 0 && @@ -1277,26 +144,26 @@ export class ChatOrchestrationService { listResult.filteredResources.length > 0); if (!hasResources) { this.logger.warn('No resources from list_resources'); - const stream = await this.openRouterService.generateFinalResponseStream( + const stream = await this.llmClient.generateFinalResponseStream( [ { role: 'system', content: NO_RELEVANT_MATERIALS_SYSTEM_PROMPT }, ...historyMessages, { role: 'user', content: userQuestion }, ], [], - this.openRouterService.getModel('normal'), + this.llmClient.getModel('normal'), { temperature: 0 }, ); return { stream, resources: [], usage }; } - // 3. 관련 리소스 내용 가져오기 (신 형식: description 기반 chunk 선별 / 구 형식: 경로 선별 후 본문 fetch) t0 = Date.now(); - const relevantResult = await this.fetchRelevantResourceContents( - userQuestion, - listResult, - usage, - ); + const relevantResult = + await this.resourceContentService.fetchRelevantResourceContents( + userQuestion, + listResult, + usage, + ); this.logger.log( `[PERF] fetchRelevantResourceContents: ${Date.now() - t0}ms`, ); @@ -1307,14 +174,14 @@ export class ChatOrchestrationService { if (!hasContent) { this.logger.warn('No reference documents available.'); - const stream = await this.openRouterService.generateFinalResponseStream( + const stream = await this.llmClient.generateFinalResponseStream( [ { role: 'system', content: NO_RELEVANT_MATERIALS_SYSTEM_PROMPT }, ...historyMessages, { role: 'user', content: userQuestion }, ], [], - this.openRouterService.getModel('normal'), + this.llmClient.getModel('normal'), { temperature: 0 }, ); return { stream, resources: [], usage }; @@ -1323,9 +190,7 @@ export class ChatOrchestrationService { const resultText = listResult.texts.join('\n') || JSON.stringify(listResult.raw, null, 2); - // OpenRouter 입력 길이가 커지면 400이 발생할 수 있어, tool 호출 컨텐츠는 하드 캡을 둡니다. - // list_resources 원문은 매우 길 수 있어 목록만 줄이고, 선별 본문(relevantResult)은 잘리지 않게 합니다. - // 선별 본문을 앞에 두어(목록보다 앞) 모델이 문서를 우선 활용하도록 합니다. + // LLM 입력 길이가 커지면 400이 발생할 수 있어, tool 호출 컨텐츠는 하드 캡을 둡니다. const MAX_TOOL_CONTENT_CHARS = 50000; const separator = '\n\n'; const relevantPart = relevantResult.content; @@ -1377,16 +242,18 @@ export class ChatOrchestrationService { ]; const allResources: ResourceInfo[] = []; - console.log('relevantResult.usedResources', relevantResult.usedResources); const seenFePdfPaths = new Set(); for (const r of relevantResult.usedResources) { - this.appendFePdfResourceEntryFromUsed(allResources, seenFePdfPaths, r); + this.resourceContentService.appendFePdfResourceEntryFromUsed( + allResources, + seenFePdfPaths, + r, + ); } - // 4. 최종 응답 스트리밍 (synthetic assistant tool_call + tool 메시지) t0 = Date.now(); this.logger.debug('Generating final response with tool results...'); - const messages: OpenRouterMessage[] = [ + const messages: LlmMessage[] = [ { role: 'system', content: FINAL_RESPONSE_SYSTEM_PROMPT }, ...historyMessages, { role: 'user', content: userQuestion }, @@ -1422,13 +289,13 @@ export class ChatOrchestrationService { } this.logger.debug( - `[DEBUG] Final OpenRouter request summary: model=heavy, messages=${messages.length}, roles=${JSON.stringify(roleCounts)}, contentNullCount=${contentNullCount}, assistantToolCalls=${assistantToolCalls}, toolResults=${toolResults.length}, toolResultsContentCharsSum=${toolResultsContentCharsSum}, fullContentOriginalChars=${fullContentOriginalChars}, fullContentWasTruncated=${fullContentWasTruncated}, numberOfAllResources=${allResources.length}`, + `[DEBUG] Final LLM request summary: model=heavy, messages=${messages.length}, roles=${JSON.stringify(roleCounts)}, contentNullCount=${contentNullCount}, assistantToolCalls=${assistantToolCalls}, toolResults=${toolResults.length}, toolResultsContentCharsSum=${toolResultsContentCharsSum}, fullContentOriginalChars=${fullContentOriginalChars}, fullContentWasTruncated=${fullContentWasTruncated}, numberOfAllResources=${allResources.length}`, ); - const stream = await this.openRouterService.generateFinalResponseStream( + const stream = await this.llmClient.generateFinalResponseStream( messages, toolResults, - this.openRouterService.getModel('heavy'), + this.llmClient.getModel('heavy'), ); this.logger.log( `[PERF] generateFinalResponseStream(시작까지): ${Date.now() - t0}ms`, @@ -1453,10 +320,6 @@ export class ChatOrchestrationService { /** * 스트리밍 응답을 처리하여 SSE 형식으로 전송 - * @param sessionId 세션 ID - * @param userQuestion 사용자 질문 - * @param reply Fastify 응답 객체 - * @param req Fastify 요청 객체 (CORS origin용) */ async handleStreamingResponse( sessionId: string, @@ -1465,27 +328,7 @@ export class ChatOrchestrationService { req: FastifyRequest, options: StreamingResponseOptions = {}, ): Promise { - reply.hijack(); - - // credentials: true 사용 시 Access-Control-Allow-Origin은 * 불가, 요청 origin을 그대로 반환해야 함 - const allowedOrigins = [ - 'http://localhost:5173', - `https://${this.configService.get('DOMAIN_NAME') ?? ''}`, - ]; - const requestOrigin = req.headers.origin; - const corsOrigin = - requestOrigin && allowedOrigins.includes(requestOrigin) - ? requestOrigin - : (allowedOrigins[1] ?? '*'); - - reply.raw.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Access-Control-Allow-Origin': corsOrigin, - 'Access-Control-Allow-Methods': 'POST, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, Authorization', - }); + this.chatStreamTransport.prepareSse(reply, req); try { const { @@ -1499,120 +342,63 @@ export class ChatOrchestrationService { }), ); - let accumulatedContent = ''; - let model = ''; - let finalResponseUsage: OpenRouterUsage | null = null; - let buffer = ''; - - stream.on('data', (chunk: Buffer) => { - buffer += chunk.toString(); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.startsWith('data: ')) { - const data = line.slice(6).trim(); - if (!data || data === '[DONE]') { - continue; - } + let streamResult: { + accumulatedContent: string; + model: string; + usage: LlmUsage | null; + }; + try { + streamResult = await this.chatStreamTransport.consumeAndForward( + stream, + reply, + ); + } catch { + // 스트림 에러 시 transport가 이미 응답을 종료함 + return; + } - try { - const parsed = JSON.parse(data); - if (parsed.choices?.[0]?.delta?.content) { - const content = parsed.choices[0].delta.content; - accumulatedContent += content; - reply.raw.write(`data: ${JSON.stringify({ content })}\n\n`); - } - if (parsed.model) { - model = parsed.model; - } - if (parsed.usage) { - finalResponseUsage = parsed.usage; - } - } catch { - // JSON 파싱 실패 시 무시 - } - } + try { + const totalUsage = { ...reasoningUsage }; + this.addTokenUsage(totalUsage, streamResult.usage); + const usage = this.hasTokenUsage(totalUsage) ? totalUsage : undefined; + + if (streamResult.accumulatedContent) { + await this.chatService.createMessage(sessionId, { + role: MessageRole.ASSISTANT, + content: streamResult.accumulatedContent, + metadata: { + ...(options.assistantMetadata ?? {}), + model: streamResult.model || undefined, + usage, + resources: resources.length > 0 ? resources : undefined, + }, + }); } - }); - - stream.on('error', (error) => { - this.logger.error('Stream error:', error); - reply.raw.write( - `data: ${JSON.stringify({ error: error.message || 'Stream error' })}\n\n`, - ); - reply.raw.end(); - }); - stream.on('end', () => { - void (async () => { + if (usage?.total_tokens != null) { try { - const totalUsage = { ...reasoningUsage }; - this.addTokenUsage(totalUsage, finalResponseUsage); - const usage = this.hasTokenUsage(totalUsage) - ? totalUsage - : undefined; - - if (accumulatedContent) { - await this.chatService.createMessage(sessionId, { - role: MessageRole.ASSISTANT, - content: accumulatedContent, - metadata: { - ...(options.assistantMetadata ?? {}), - model: model || undefined, - usage, - resources: resources.length > 0 ? resources : undefined, - }, - }); - } - - if (usage?.total_tokens != null) { - try { - await this.usageService.recordUsage(sessionId, { - totalTokens: usage.total_tokens, - }); - } catch (err) { - this.logger.warn( - 'Failed to record usage', - err instanceof Error ? err.message : String(err), - ); - } - } - - if (resources.length > 0) { - reply.raw.write( - `data: ${JSON.stringify({ - type: 'resources', - resources: resources, - })}\n\n`, - ); - } - - reply.raw.write('data: [DONE]\n\n'); - reply.raw.end(); - } catch (error) { - this.logger.error('Error saving final message:', error); - reply.raw.write( - `data: ${JSON.stringify({ error: 'Failed to save message' })}\n\n`, + await this.usageService.recordUsage(sessionId, { + totalTokens: usage.total_tokens, + }); + } catch (err) { + this.logger.warn( + 'Failed to record usage', + err instanceof Error ? err.message : String(err), ); - reply.raw.end(); } - })(); - }); + } + + this.chatStreamTransport.writeResources(reply, resources); + this.chatStreamTransport.writeDone(reply); + } catch (error) { + this.logger.error('Error saving final message:', error); + this.chatStreamTransport.writeError(reply, 'Failed to save message'); + } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.error('Error in chat stream:', errorMessage); - reply.raw.write(`data: ${JSON.stringify({ error: errorMessage })}\n\n`); - reply.raw.end(); + this.chatStreamTransport.writeError(reply, errorMessage); } } - - /** - * 리소스 경로를 기반으로 URL 생성 - */ - private generateResourceUrl(resourcePath: string): string { - const encodedPath = encodeURIComponent(resourcePath); - return encodedPath; - } } diff --git a/src/chat/services/chat-stream.transport.ts b/src/chat/services/chat-stream.transport.ts new file mode 100644 index 0000000..aac1ba5 --- /dev/null +++ b/src/chat/services/chat-stream.transport.ts @@ -0,0 +1,128 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { Readable } from 'stream'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { LlmUsage } from '../types/llm.types'; +import type { ResourceInfo } from './resource-content.service'; + +export type StreamConsumeResult = { + accumulatedContent: string; + model: string; + usage: LlmUsage | null; +}; + +/** + * Fastify SSE transport (CORS 헤더, LLM 스트림 파싱·전달) + */ +@Injectable() +export class ChatStreamTransport { + private readonly logger = new Logger(ChatStreamTransport.name); + + constructor(private readonly configService: ConfigService) {} + + /** + * reply를 hijack하고 SSE 응답 헤더를 기록합니다. + */ + prepareSse(reply: FastifyReply, req: FastifyRequest): void { + reply.hijack(); + + // credentials: true 사용 시 Access-Control-Allow-Origin은 * 불가, 요청 origin을 그대로 반환해야 함 + const allowedOrigins = [ + 'http://localhost:5173', + `https://${this.configService.get('DOMAIN_NAME') ?? ''}`, + ]; + const requestOrigin = req.headers.origin; + const corsOrigin = + requestOrigin && allowedOrigins.includes(requestOrigin) + ? requestOrigin + : (allowedOrigins[1] ?? '*'); + + reply.raw.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Access-Control-Allow-Origin': corsOrigin, + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + }); + } + + /** + * LLM SSE 스트림을 소비하며 content delta를 클라이언트로 전달합니다. + * 스트림 종료 시 누적 결과를 resolve합니다. + */ + consumeAndForward( + stream: Readable, + reply: FastifyReply, + ): Promise { + return new Promise((resolve, reject) => { + let accumulatedContent = ''; + let model = ''; + let usage: LlmUsage | null = null; + let buffer = ''; + + stream.on('data', (chunk: Buffer) => { + buffer += chunk.toString(); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + + const data = line.slice(6).trim(); + if (!data || data === '[DONE]') continue; + + try { + const parsed = JSON.parse(data); + if (parsed.choices?.[0]?.delta?.content) { + const content = parsed.choices[0].delta.content; + accumulatedContent += content; + reply.raw.write(`data: ${JSON.stringify({ content })}\n\n`); + } + if (parsed.model) { + model = parsed.model; + } + if (parsed.usage) { + usage = parsed.usage; + } + } catch { + // JSON 파싱 실패 시 무시 + } + } + }); + + stream.on('error', (error: Error) => { + this.logger.error('Stream error:', error); + reply.raw.write( + `data: ${JSON.stringify({ error: error.message || 'Stream error' })}\n\n`, + ); + reply.raw.end(); + reject(error); + }); + + stream.on('end', () => { + resolve({ accumulatedContent, model, usage }); + }); + }); + } + + writeResources(reply: FastifyReply, resources: ResourceInfo[]): void { + if (resources.length === 0) return; + reply.raw.write( + `data: ${JSON.stringify({ + type: 'resources', + resources, + })}\n\n`, + ); + } + + writeDone(reply: FastifyReply): void { + reply.raw.write('data: [DONE]\n\n'); + reply.raw.end(); + } + + writeError(reply: FastifyReply, errorMessage: string): void { + reply.raw.write(`data: ${JSON.stringify({ error: errorMessage })}\n\n`); + reply.raw.end(); + } +} diff --git a/src/chat/services/open-router.service.ts b/src/chat/services/open-router.service.ts deleted file mode 100644 index 92afd32..0000000 --- a/src/chat/services/open-router.service.ts +++ /dev/null @@ -1,537 +0,0 @@ -import { - Injectable, - Logger, - InternalServerErrorException, -} from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { HttpService } from '@nestjs/axios'; -import { firstValueFrom } from 'rxjs'; -import { catchError } from 'rxjs/operators'; -import { AxiosError } from 'axios'; -import type { Readable } from 'stream'; -import { inspect } from 'node:util'; -import type { - McpTool, - OpenRouterTool, - OpenRouterMessage, - OpenRouterRequest, - OpenRouterResponse, - ParsedToolCall, -} from '../types/open-router.types'; -import { getToolSelectionSystemPrompt } from '../prompts'; - -/** - * Open Router 서비스 - * LLM을 통해 MCP Tool 선택 및 실행을 관리합니다. - */ -/** Open Router 모델 용도 */ -export type OpenRouterModelType = 'light' | 'normal' | 'heavy'; - -@Injectable() -export class OpenRouterService { - private readonly logger = new Logger(OpenRouterService.name); - private readonly apiKey: string; - private readonly baseUrl: string; - private readonly modelLight: string; - private readonly modelNormal: string; - private readonly modelHeavy: string; - private readonly defaultModel: string; - private readonly xTitle: string | undefined; - - constructor( - private readonly httpService: HttpService, - private readonly configService: ConfigService, - ) { - this.apiKey = this.configService.getOrThrow( - 'LETSUR_AI_GATEWAY_API_KEY', - ); - this.baseUrl = this.configService - .getOrThrow('LETSUR_AI_GATEWAY_BASE_URL') - .replace(/\/+$/, ''); - const fallback = - this.configService.get('LETSUR_AI_GATEWAY_MODEL') || - 'gpt-4o-mini'; - this.modelLight = - this.configService.get('LETSUR_AI_GATEWAY_MODEL_LIGHT') || - fallback; - this.modelNormal = - this.configService.get('LETSUR_AI_GATEWAY_MODEL_NORMAL') || - fallback; - this.modelHeavy = - this.configService.get('LETSUR_AI_GATEWAY_MODEL_HEAVY') || - fallback; - this.defaultModel = this.modelNormal; - this.xTitle = - this.configService.get('LETSUR_AI_GATEWAY_X_TITLE') || - this.configService.get('LETSUR_AI_GATEWAY_TITLE'); - } - - /** 용도별 모델 반환 (light: 선별, normal: 단순 응답, heavy: 최종 답변) */ - getModel(type: OpenRouterModelType): string { - switch (type) { - case 'light': - return this.modelLight; - case 'normal': - return this.modelNormal; - case 'heavy': - return this.modelHeavy; - default: - return this.defaultModel; - } - } - - /** - * MCP Tool 목록을 Open Router Function Calling 형식으로 변환 - */ - convertMcpToolsToOpenRouterFormat(mcpTools: McpTool[]): OpenRouterTool[] { - return mcpTools.map((tool) => { - const parameters = tool.inputSchema || { - type: 'object', - properties: {}, - required: [], - }; - - // Tool description 개선: 더 상세한 설명 생성 - let description = tool.description || ''; - - // Description이 없거나 너무 짧은 경우 개선 - if (!description || description.trim().length < 20) { - // Tool 이름을 기반으로 더 상세한 설명 생성 - const toolNameLower = tool.name.toLowerCase(); - const paramInfo = parameters.properties - ? Object.keys(parameters.properties) - .map((key) => { - const prop = parameters.properties![key]; - return `${key} (${prop.type || 'string'})`; - }) - .join(', ') - : 'no parameters'; - - description = `Tool: ${tool.name}. -Use this tool when the user's question relates to ${tool.name} or when you need to access information related to ${toolNameLower}. -${paramInfo ? `Parameters: ${paramInfo}` : 'No parameters required.'} -This tool is essential for answering questions that require ${toolNameLower} functionality.`; - } else { - // 기존 description이 있으면 파라미터 정보 추가 - const paramInfo = parameters.properties - ? Object.keys(parameters.properties) - .map((key) => { - const prop = parameters.properties![key]; - return `${key} (${prop.type || 'string'})`; - }) - .join(', ') - : ''; - - if (paramInfo) { - description += ` Parameters: ${paramInfo}.`; - } - } - - return { - type: 'function', - function: { - name: tool.name, - description: description.trim(), - parameters: { - type: parameters.type || 'object', - properties: parameters.properties || {}, - required: parameters.required || [], - }, - }, - }; - }); - } - - /** - * 사용자 질문과 MCP Tool 목록을 기반으로 LLM에게 Tool 선택 요청 - * @param userQuestion 사용자 질문 - * @param mcpTools 사용 가능한 MCP Tool 목록 - * @param model 사용할 LLM 모델 (선택사항) - * @param options 추가 옵션 (temperature, emphasizeToolUsage 등) - * @param pastMessages 과거 대화 (시간순, user/assistant만) — context 유지용 - * @returns LLM 응답 (tool_calls 포함 가능) - */ - async selectTool( - userQuestion: string, - mcpTools: McpTool[], - model?: string, - options?: { - temperature?: number; - emphasizeToolUsage?: boolean; - }, - pastMessages?: OpenRouterMessage[], - ): Promise { - const tools = this.convertMcpToolsToOpenRouterFormat(mcpTools); - - // Tool 목록을 더 읽기 쉽게 포맷팅 - const toolsDescription = mcpTools - .map((tool, index) => { - const toolInfo = tools[index]; - const params = toolInfo.function.parameters.properties - ? Object.entries(toolInfo.function.parameters.properties) - .map(([key, value]: [string, any]) => { - const type = value.type || 'string'; - const desc = value.description ? ` - ${value.description}` : ''; - return ` - ${key} (${type})${desc}`; - }) - .join('\n') - : ' (no parameters)'; - - return `${index + 1}. ${tool.name} - Description: ${toolInfo.function.description} - Parameters: -${params}`; - }) - .join('\n\n'); - - // System prompt 강화 - const systemPrompt = getToolSelectionSystemPrompt({ - toolsDescription, - emphasizeToolUsage: options?.emphasizeToolUsage, - }); - - const messages: OpenRouterMessage[] = [ - { role: 'system', content: systemPrompt }, - ...(pastMessages ?? []), - { role: 'user', content: userQuestion }, - ]; - - const request: OpenRouterRequest = { - model: model || this.defaultModel, - messages, - tools, - tool_choice: 'auto', - temperature: options?.temperature ?? 0.3, // 재시도 시 더 낮은 temperature 사용 가능 - max_tokens: 2000, - }; - - const requestLogSummary = { - model: request.model, - stream: false, - temperature: request.temperature, - max_tokens: request.max_tokens, - messages: this.summarizeOpenRouterMessages(messages), - toolsCount: tools.length, - }; - - try { - const response = await firstValueFrom( - this.httpService - .post( - `${this.baseUrl}/chat/completions`, - request, - { - headers: { - Authorization: `Bearer ${this.apiKey}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': this.configService.get('DOMAIN_NAME'), - ...(this.xTitle ? { 'X-Title': this.xTitle } : {}), - }, - timeout: 15000, - }, - ) - .pipe( - catchError((error: AxiosError) => { - const responseData = error.response?.data; - this.logger.error( - `Open Router API error: ${error.message}`, - error instanceof Error ? error.stack : undefined, - ); - if (responseData != null) { - this.logger.error( - `Open Router error response body: ${this.safeStringify(responseData)}`, - ); - } - this.logger.error( - `Open Router request summary: ${this.safeStringify(requestLogSummary)}`, - ); - throw new InternalServerErrorException( - `Failed to call Open Router API: ${error.message}`, - ); - }), - ), - ); - - // Tool 선택 응답 상세 로깅 - const responseData = response.data; - const finishReason = responseData.choices[0]?.finish_reason; - const hasToolCalls = !!responseData.choices[0]?.message.tool_calls; - const toolCallCount = - responseData.choices[0]?.message.tool_calls?.length || 0; - - // Tool이 선택되지 않은 경우 경고 - if (!hasToolCalls || toolCallCount === 0) { - this.logger.warn( - `No tools selected. Finish reason: ${finishReason}, Available tools: ${mcpTools.map((t) => t.name).join(', ')}`, - ); - } - - return responseData; - } catch (error) { - this.logger.error(`Error calling Open Router: ${error}`); - throw error; - } - } - - /** - * LLM 응답에서 tool_calls 파싱 - */ - parseToolCalls(response: OpenRouterResponse): ParsedToolCall[] { - const toolCalls: ParsedToolCall[] = []; - - for (const choice of response.choices) { - if (choice.message.tool_calls) { - for (const toolCall of choice.message.tool_calls) { - try { - const args = JSON.parse(toolCall.function.arguments); - toolCalls.push({ - id: toolCall.id, - name: toolCall.function.name, - arguments: args, - }); - } catch (error) { - this.logger.warn( - `Failed to parse tool call arguments: ${toolCall.function.arguments}`, - error, - ); - } - } - } - } - - return toolCalls; - } - - private safeStringify(value: unknown, maxLen: number = 2000): string { - try { - if (typeof value === 'string') { - return value.length > maxLen - ? value.slice(0, maxLen) + '...(truncated)' - : value; - } - - // Buffer는 JSON.stringify하면 {}처럼 찍히기 쉬워 text로 변환합니다. - if (typeof Buffer !== 'undefined' && Buffer.isBuffer(value as Buffer)) { - const str = (value as Buffer).toString('utf8'); - return str.length > maxLen - ? str.slice(0, maxLen) + '...(truncated)' - : str; - } - - const str = JSON.stringify(value, null, 2); - return str.length > maxLen - ? str.slice(0, maxLen) + '...(truncated)' - : str; - } catch { - const str = inspect(value, { - depth: 5, - maxArrayLength: 50, - breakLength: 120, - }); - return str.length > maxLen - ? str.slice(0, maxLen) + '...(truncated)' - : str; - } - } - - private summarizeOpenRouterMessages(messages: OpenRouterMessage[]) { - const roleCounts: Record = {}; - let assistantToolCalls = 0; - let toolRoleMessages = 0; - let toolRoleHasNameField = 0; - let contentNullCount = 0; - let contentCharsSum = 0; - - for (const m of messages) { - roleCounts[m.role] = (roleCounts[m.role] ?? 0) + 1; - if (m.role === 'assistant' && m.tool_calls?.length) { - assistantToolCalls += m.tool_calls.length; - } - if (m.role === 'tool') { - toolRoleMessages += 1; - // 일부 모델/프로바이더에서 role=tool 메시지의 name 필드가 스키마에 없을 수 있음 - if ((m as unknown as { name?: unknown }).name != null) { - toolRoleHasNameField += 1; - } - } - if (m.content === null) contentNullCount += 1; - if (typeof m.content === 'string') contentCharsSum += m.content.length; - } - - return { - total: messages.length, - roleCounts, - assistantToolCalls, - toolRoleMessages, - toolRoleHasNameField, - contentNullCount, - contentCharsSum, - }; - } - - /** - * Tool 실행 결과를 LLM에 전달하여 최종 응답을 스트리밍으로 생성 - * @param messages 이전 대화 내역 - * @param toolResults Tool 실행 결과 (tool_call_id와 결과 매핑) - * @param model 사용할 LLM 모델 (선택사항) - * @returns 스트리밍 응답 스트림 - */ - async generateFinalResponseStream( - messages: OpenRouterMessage[], - toolResults: Array<{ - tool_call_id: string; - name: string; - content: string; - }>, - model?: string, - options?: { temperature?: number }, - ): Promise { - // Tool 결과를 메시지에 추가 - const toolMessages: OpenRouterMessage[] = toolResults.map((result) => ({ - role: 'tool', - tool_call_id: result.tool_call_id, - name: result.name, - content: result.content, - })); - - const updatedMessages = [...messages, ...toolMessages]; - - const request: OpenRouterRequest & { stream: boolean } = { - model: model || this.defaultModel, - messages: updatedMessages, - temperature: options?.temperature ?? 0.7, - max_tokens: 2000, - stream: true, - stream_options: { include_usage: true }, - }; - - const requestLogSummary = { - model: request.model, - stream: request.stream, - temperature: request.temperature, - max_tokens: request.max_tokens, - toolResultsCount: toolResults.length, - toolResultsContentCharsSum: toolResults.reduce( - (sum, r) => sum + (r.content?.length ?? 0), - 0, - ), - messages: this.summarizeOpenRouterMessages(updatedMessages), - }; - - try { - const response = await firstValueFrom( - this.httpService - .post(`${this.baseUrl}/chat/completions`, request, { - headers: { - Authorization: `Bearer ${this.apiKey}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': this.configService.get('DOMAIN_NAME'), - ...(this.xTitle ? { 'X-Title': this.xTitle } : {}), - }, - responseType: 'stream', - timeout: 15000, - }) - .pipe( - catchError((error: AxiosError) => { - const errorMessage = error.message; - const statusCode = error.response?.status; - const responseData = error.response?.data; - this.logger.error( - `Open Router API error (status ${statusCode}): ${errorMessage}`, - error instanceof Error ? error.stack : undefined, - ); - if (responseData != null) { - this.logger.error( - `Open Router 400 response body: ${this.safeStringify(responseData)}`, - ); - } - this.logger.error( - `Open Router request summary: ${this.safeStringify(requestLogSummary)}`, - ); - throw new InternalServerErrorException( - `Failed to call Open Router API: ${errorMessage}`, - ); - }), - ), - ); - - return response.data; - } catch (error) { - this.logger.error(`Error calling Open Router: ${error}`); - throw error; - } - } - - /** - * 일반적인 LLM 호출 (Tool 없이) - */ - async callLLM( - messages: OpenRouterMessage[], - model?: string, - options?: { - temperature?: number; - max_tokens?: number; - }, - ): Promise { - const request: OpenRouterRequest = { - model: model || this.defaultModel, - messages, - temperature: options?.temperature ?? 0.7, - max_tokens: options?.max_tokens ?? 2000, - }; - - const requestLogSummary = { - model: request.model, - stream: false, - temperature: request.temperature, - max_tokens: request.max_tokens, - messages: this.summarizeOpenRouterMessages(messages), - }; - - try { - const response = await firstValueFrom( - this.httpService - .post( - `${this.baseUrl}/chat/completions`, - request, - { - headers: { - Authorization: `Bearer ${this.apiKey}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': this.configService.get('DOMAIN_NAME'), - ...(this.xTitle ? { 'X-Title': this.xTitle } : {}), - }, - timeout: 15000, - }, - ) - .pipe( - catchError((error: AxiosError) => { - const errorMessage = error.message; - const statusCode = error.response?.status; - const responseData = error.response?.data; - this.logger.error( - `Open Router API error (status ${statusCode}): ${errorMessage}`, - error instanceof Error ? error.stack : undefined, - ); - if (responseData != null) { - this.logger.error( - `Open Router error response body: ${this.safeStringify(responseData)}`, - ); - } - this.logger.error( - `Open Router request summary: ${this.safeStringify(requestLogSummary)}`, - ); - throw new InternalServerErrorException( - `Failed to call Open Router API: ${errorMessage}`, - ); - }), - ), - ); - - return response.data; - } catch (error) { - this.logger.error(`Error calling Open Router: ${error}`); - throw error; - } - } -} diff --git a/src/chat/services/resource-content.service.ts b/src/chat/services/resource-content.service.ts new file mode 100644 index 0000000..914245a --- /dev/null +++ b/src/chat/services/resource-content.service.ts @@ -0,0 +1,782 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { + ListResourcesResult, + ListResourceItem, +} from '../../mcp/mcp-client.service'; +import { McpClientService } from '../../mcp/mcp-client.service'; +import { ResourceSelectionService } from './resource-selection.service'; +import type { LlmUsage } from '../types/llm.types'; + +/** + * FE·SSE용 참조 리소스 정보 + */ +export interface ResourceInfo { + path: string; // 문서 제목 (PDF/PNG인 경우 format 포함) + formats: string[]; + url: string; +} + +/** + * MCP 리소스 내용 fetch·파싱·FE 리소스 조립 + */ +@Injectable() +export class ResourceContentService { + private readonly logger = new Logger(ResourceContentService.name); + + constructor( + private readonly mcpClientService: McpClientService, + private readonly resourceSelectionService: ResourceSelectionService, + ) {} + + private normalizeResourcePath(path: string): string { + // 확장자가 있으면 제거 (MCP 서버가 자동으로 찾음) + if (path.includes('.')) { + const lastDotIndex = path.lastIndexOf('.'); + // 마지막 점 이후가 확장자인 경우 (예: .md, .pdf) + const extension = path.substring(lastDotIndex + 1); + if (extension.length <= 5 && /^[a-z0-9]+$/i.test(extension)) { + return path.substring(0, lastDotIndex); + } + } + return path; + } + + /** + * 경로에서 마지막 문서 제목만 추출 (확장자 포함) + * 예: "2025 캠프 발표자료_ 1일차 오전/학생지원.md" -> "학생지원.md" + * 원본 경로에 확장자가 없으면 formats 배열에서 찾아서 추가 + */ + private extractDocumentTitle( + path: string, + originalPath?: string, + formats?: string[], + ): string { + // 원본 경로가 있으면 원본 경로 사용 (확장자 포함) + const pathToUse = originalPath || path; + + // 슬래시로 분리하여 마지막 부분만 반환 + const parts = pathToUse.split('/'); + let title = parts[parts.length - 1] || pathToUse; + + // 확장자가 없고 formats 배열에 md가 있으면 .md 추가 + if (!title.includes('.') && formats && formats.includes('md')) { + title = `${title}.md`; + } + + return title; + } + + /** + * FE·리소스 API용 PDF 경로: 하위 chunk/이미지 경로가 아니라 상위 묶음 PDF 한 개 + * 예: `에어컨+…/세부/파일.png` → `에어컨+….pdf` (첫 `/` 앞 세그먼트 + `.pdf`) + */ + private normalizeTopLevelPdfPathForFe(resourcePath: string): string { + const first = resourcePath.split('/')[0]?.trim() || resourcePath; + const base = first.replace(/\.(pdf|png|md|jpe?g|gif|webp)$/i, ''); + return `${base}.pdf`; + } + + /** + * SSE·메타데이터용 참조 문서: **PDF 번들만** (마크다운 chunk 경로는 상위 세그먼트 + `.pdf`로 변환). + * 예: `2026년+학사편람/…/졸업요건.md` → `2026년+학사편람.pdf` + */ + appendFePdfResourceEntryFromUsed( + out: ResourceInfo[], + seenPdfPaths: Set, + r: { path: string; formats: string[] }, + ): void { + if (!r.path || !r.formats?.length) return; + const contributes = + r.formats.includes('md') || + r.formats.includes('pdf') || + r.formats.includes('png'); + if (!contributes) return; + + const pathForFe = this.normalizeTopLevelPdfPathForFe(r.path); + if (seenPdfPaths.has(pathForFe)) return; + seenPdfPaths.add(pathForFe); + + out.push({ + path: pathForFe, + formats: ['pdf'], + url: this.generateResourceUrl(pathForFe), + }); + } + + /** + * get_resource 툴 응답에서 텍스트 내용 추출 + * MCP 서버는 문자열을 직접 반환하므로, texts 배열이나 raw.content에서 추출 + */ + private extractContentFromToolResult( + toolResult: Awaited>, + ): string { + // texts 배열에서 내용 추출 (가장 일반적인 경우) + if (toolResult.texts.length > 0) { + // texts가 여러 개인 경우 합치기 + const content = toolResult.texts.join('\n'); + // JSON 문자열이 아닌 경우 그대로 반환 + if ( + content && + !content.trim().startsWith('{') && + !content.trim().startsWith('[') + ) { + return content; + } + } + + // raw.content에서 text 타입 항목 추출 + const raw = toolResult.raw as { + content?: Array<{ type: string; text?: string }>; + }; + if (raw?.content) { + const textContents: string[] = []; + for (const item of raw.content) { + if (item.type === 'text' && 'text' in item) { + const text = item.text; + // JSON 문자열이 아닌 경우 그대로 추가 + if ( + text && + !text.trim().startsWith('{') && + !text.trim().startsWith('[') + ) { + textContents.push(text); + } + } + } + if (textContents.length > 0) { + return textContents.join('\n'); + } + } + + return ''; + } + + /** + * 신 형식: LLM에게 description을 보고 관련 chunk 경로 최대 maxResults개 선택 (JSON 배열 반환) + */ + private parseDocumentLinks(content: string): Array<{ + path: string; + description: string; + }> { + const documents: Array<{ path: string; description: string }> = []; + const documentRegex = + /<\/document>/g; + + let match; + while ((match = documentRegex.exec(content)) !== null) { + documents.push({ + path: match[1], + description: match[2], + }); + } + + return documents; + } + + /** + * 마크다운에서 이미지 참조 추출: ![alt](path) 형태 + * 첨부된 이미지(.png, .jpg 등) 경로만 반환 + */ + private parseImageReferencesFromMarkdown(content: string): string[] { + const paths: string[] = []; + const imageRefRegex = /!\[[^\]]*\]\(([^)]+)\)/g; + let match; + while ((match = imageRefRegex.exec(content)) !== null) { + const path = match[1].trim(); + if (/\.(png|jpe?g|gif|webp)(\?|#|$)/i.test(path)) { + paths.push(path); + } + } + return paths; + } + + /** + * 문서 경로 기준으로 상대 이미지 경로를 전체 리소스 경로로 변환 + * 예: docPath="폴더/문서.md", imageRef="이미지.png" → "폴더/이미지.png" + */ + private resolveImagePath(imageRef: string, docPath: string): string { + const lastSlash = docPath.lastIndexOf('/'); + const dir = lastSlash === -1 ? '' : docPath.slice(0, lastSlash + 1); + return dir + imageRef; + } + + /** + * MD 링크/이미지의 상대 경로를 절대 리소스 경로로 변환 (`../` 처리) + */ + private resolveRelativeResourcePath(ref: string, docPath: string): string { + const raw = ref.trim().replace(/^<|>$/g, '').split(/[?#]/)[0]; + if (!raw || /^https?:\/\//i.test(raw)) return raw; + if (raw.startsWith('/')) return raw.replace(/^\/+/, ''); + const lastSlash = docPath.lastIndexOf('/'); + const dir = lastSlash === -1 ? '' : docPath.slice(0, lastSlash + 1); + const combined = dir + raw; + const segments = combined.split('/').filter((s) => s.length > 0); + const out: string[] = []; + for (const s of segments) { + if (s === '..') out.pop(); + else if (s !== '.') out.push(s); + } + return out.join('/'); + } + + /** + * 선별된 MD 본문에서 PDF/PNG 참조 경로 추출 (마크다운 링크, 이미지, ``) + */ + private extractPdfPngReferencesFromMarkdown( + content: string, + docPath: string, + ): Array<{ path: string; formats: string[] }> { + const results: Array<{ path: string; formats: string[] }> = []; + const seen = new Set(); + const add = (p: string, fmt: 'pdf' | 'png') => { + if (!p || seen.has(p)) return; + seen.add(p); + results.push({ path: p, formats: [fmt] }); + }; + + const mdLink = /\[([^\]]*)\]\(([^)]+)\)/g; + let m: RegExpExecArray | null; + while ((m = mdLink.exec(content)) !== null) { + const inner = m[2].trim(); + const raw = inner.split(/\s+/)[0]; + if (/\.pdf$/i.test(raw)) { + const full = this.resolveRelativeResourcePath(raw, docPath); + if (!/^https?:\/\//i.test(full)) add(full, 'pdf'); + } + if (/\.png$/i.test(raw)) { + const full = this.resolveRelativeResourcePath(raw, docPath); + if (!/^https?:\/\//i.test(full)) add(full, 'png'); + } + } + + for (const img of this.parseImageReferencesFromMarkdown(content)) { + const raw = img.trim().split(/[?#]/)[0]; + if (/\.png$/i.test(raw)) { + const full = this.resolveRelativeResourcePath(raw, docPath); + if (!/^https?:\/\//i.test(full)) add(full, 'png'); + } + } + + for (const d of this.parseDocumentLinks(content)) { + const p = d.path.trim(); + if (/\.pdf$/i.test(p)) { + const full = p.includes('/') + ? p + : this.resolveRelativeResourcePath(p, docPath); + if (!/^https?:\/\//i.test(full)) add(full, 'pdf'); + } + if (/\.png$/i.test(p)) { + const full = p.includes('/') + ? p + : this.resolveRelativeResourcePath(p, docPath); + if (!/^https?:\/\//i.test(full)) add(full, 'png'); + } + } + + return results; + } + + /** + * list_resources chunk 목록에서 선별된 상위 폴더와 같은 루트의 PDF/PNG chunk 경로 수집 + */ + private collectPdfPngPathsFromChunkCatalog( + chunks: Array<{ path: string }> | undefined, + selectedPaths: string[], + max: number = 8, + ): Array<{ path: string; formats: string[] }> { + if (!chunks?.length || !selectedPaths.length) return []; + const roots = new Set( + selectedPaths.map((p) => p.split('/')[0]).filter(Boolean), + ); + const out: Array<{ path: string; formats: string[] }> = []; + const seen = new Set(); + for (const c of chunks) { + const isPdf = /\.pdf$/i.test(c.path); + const isPng = /\.png$/i.test(c.path); + if (!isPdf && !isPng) continue; + const root = c.path.split('/')[0]; + if (!roots.has(root)) continue; + if (seen.has(c.path)) continue; + seen.add(c.path); + out.push({ path: c.path, formats: isPdf ? ['pdf'] : ['png'] }); + if (out.length >= max) break; + } + return out; + } + + /** + * 질문과 관련된 하위 문서 찾기 + */ + private findRelevantSubDocuments( + question: string, + documents: Array<{ path: string; description: string }>, + maxResults: number = 3, + ): Array<{ path: string; description: string }> { + const keywords = + question + .toLowerCase() + .match(/[\uac00-\ud7a3]+|[a-z]+/gi) + ?.filter((word) => word.length > 1) || []; + + if (keywords.length === 0) { + return documents.slice(0, maxResults); + } + + const scoredDocuments = documents.map((doc) => { + const pathLower = doc.path.toLowerCase(); + const descLower = doc.description.toLowerCase(); + let score = 0; + + for (const keyword of keywords) { + if (pathLower.includes(keyword)) { + score += keyword.length * 2; // 경로 매칭은 가중치 높게 + } + if (descLower.includes(keyword)) { + score += keyword.length; // 설명 매칭 + } + } + + return { document: doc, score }; + }); + + return scoredDocuments + .sort((a, b) => b.score - a.score) + .slice(0, maxResults) + .map((item) => item.document); + } + + /** + * 하위 문서 내용 가져오기 + */ + private async fetchSubDocumentContents( + subDocuments: Array<{ path: string; description: string }>, + ): Promise { + const results = await Promise.all( + subDocuments.map(async (doc) => { + try { + const resourcePath = this.normalizeResourcePath(doc.path); + this.logger.debug(`Fetching sub-document: ${resourcePath}`); + const toolResult = await this.mcpClientService.callTool( + 'get_resource', + { path: resourcePath }, + ); + const content = this.extractContentFromToolResult(toolResult); + if (content) { + const documentTitle = this.extractDocumentTitle( + resourcePath, + doc.path, + ['md'], + ); + return `\n\n## 하위 문서: ${documentTitle}\n\n**설명**: ${doc.description}\n\n${content}`; + } + } catch (error) { + this.logger.warn( + `Failed to fetch sub-document ${doc.path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return ''; + }), + ); + return results.filter(Boolean).join('\n'); + } + + /** + * LLM에게 문서 목록을 주고 질문과 관련성이 높은 문서만 선별하도록 요청 + */ + private async fetchRelevantContentsFromChunks( + question: string, + resources: ListResourceItem[], + catalogChunks?: Array<{ path: string }>, + tokenUsage?: LlmUsage, + ): Promise<{ + content: string; + usedResources: Array<{ path: string; formats: string[] }>; + }> { + this.logger.log( + `[DEBUG] 1차 선별(description 기준) 입력: 상위 리소스 ${resources.length}개, chunk 총 ${resources.reduce((s, r) => s + (r.chunks?.length ?? 0), 0)}개 → LLM에 전달`, + ); + + let t0 = Date.now(); + const chunkPaths = await this.resourceSelectionService.selectRelevantChunkPaths( + question, + resources, + 10, + tokenUsage, + ); + this.logger.log( + `[PERF] selectRelevantChunkPaths(LLM): ${Date.now() - t0}ms`, + ); + + if (chunkPaths.length === 0) { + return { content: '', usedResources: [] }; + } + + t0 = Date.now(); + const chunkResults = await Promise.all( + chunkPaths.map(async (chunkPath) => { + try { + const pathForTool = this.normalizeResourcePath(chunkPath); + this.logger.debug(`Fetching chunk: ${pathForTool}`); + const toolResult = await this.mcpClientService.callTool( + 'get_resource', + { path: pathForTool }, + ); + const content = this.extractContentFromToolResult(toolResult); + if (content) { + const title = chunkPath.split('/').pop() || chunkPath || '문서'; + return { title, content, path: chunkPath }; + } + } catch (error) { + this.logger.warn( + `Failed to fetch chunk ${chunkPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return null; + }), + ); + const documentCandidates = chunkResults.filter( + (r): r is { title: string; content: string; path: string } => r !== null, + ); + this.logger.log( + `[PERF] get_resource 루프(신 형식, ${chunkPaths.length}개): ${Date.now() - t0}ms`, + ); + + if (documentCandidates.length === 0) { + return { content: '', usedResources: [] }; + } + + this.logger.log( + `[DEBUG] 2차 선별(본문 기준) 입력: 후보 문서 ${documentCandidates.length}개 → LLM에 전달`, + ); + + t0 = Date.now(); + const selectedDocuments = await this.resourceSelectionService.selectMostRelevantDocuments( + question, + documentCandidates.map((doc) => ({ + title: doc.title, + content: doc.content, + path: doc.path, + })), + tokenUsage, + ); + this.logger.log( + `[PERF] selectMostRelevantDocuments(LLM, 신 형식): ${Date.now() - t0}ms`, + ); + + this.logger.log( + `[DEBUG] 2차 선별 결과(최종 사용 문서): ${selectedDocuments.length}개`, + ); + + if (selectedDocuments.length === 0) { + this.logger.log('No documents selected by LLM as relevant'); + return { content: '', usedResources: [] }; + } + + const contents: string[] = []; + const mdUsed: Array<{ path: string; formats: string[] }> = []; + + for (const selected of selectedDocuments) { + const doc = documentCandidates.find((d) => d.path === selected.path); + if (doc) { + contents.push(`\n\n## 리소스: ${doc.title}\n\n${doc.content}`); + mdUsed.push({ path: doc.path, formats: ['md'] }); + } + } + + const selectedPaths = selectedDocuments.map((s) => s.path); + const fromMarkdown: Array<{ path: string; formats: string[] }> = []; + for (const selected of selectedDocuments) { + const doc = documentCandidates.find((d) => d.path === selected.path); + if (!doc?.content) continue; + fromMarkdown.push( + ...this.extractPdfPngReferencesFromMarkdown(doc.content, doc.path), + ); + } + const fromCatalog = this.collectPdfPngPathsFromChunkCatalog( + catalogChunks, + selectedPaths, + 8, + ); + + const seenPdfPng = new Set(); + const pdfPngExtras: Array<{ path: string; formats: string[] }> = []; + for (const e of [...fromMarkdown, ...fromCatalog]) { + if (seenPdfPng.has(e.path)) continue; + seenPdfPng.add(e.path); + pdfPngExtras.push(e); + } + + const finalUsedResources = [ + ...mdUsed.slice(0, 5), + ...pdfPngExtras.slice(0, 8), + ]; + + return { + content: contents.join('\n'), + usedResources: finalUsedResources, + }; + } + + /** + * list_resources tool 응답에서 관련 리소스 내용 가져오기 + * - 신 형식(resources + chunks): description 보고 chunk 경로 선별 → get_resource(chunk_path) + * - 구 형식(filteredResources): 경로만 선별 후 get_resource + * @returns 문서 내용과 usedResources(선별 경로·formats; chunk는 md 포함). FE 참조 목록은 PDF/PNG만 노출. + */ + async fetchRelevantResourceContents( + question: string, + listResult: ListResourcesResult, + tokenUsage?: LlmUsage, + ): Promise<{ + content: string; + usedResources: Array<{ path: string; formats: string[] }>; + }> { + const isNewFormat = + listResult.resources && + listResult.resources.length > 0 && + listResult.chunks && + listResult.chunks.length > 0; + + if (isNewFormat) { + return this.fetchRelevantContentsFromChunks( + question, + listResult.resources!, + listResult.chunks, + tokenUsage, + ); + } + + const filteredResources = listResult.filteredResources; + if (!filteredResources || filteredResources.length === 0) { + return { content: '', usedResources: [] }; + } + + const mdResources = filteredResources.filter( + (resource) => resource.formats && resource.formats.includes('md'), + ); + + if (mdResources.length === 0) { + this.logger.debug('No markdown resources found in filtered resources'); + return { content: '', usedResources: [] }; + } + + this.logger.log( + `[DEBUG] 1차 선별(경로 기준) 입력: MD 문서 ${mdResources.length}개 → LLM에 전달`, + ); + + let t0 = Date.now(); + const relevantResources = await this.resourceSelectionService.selectRelevantResourcePaths( + question, + mdResources, + 10, + tokenUsage, + ); + this.logger.log( + `[PERF] selectRelevantResourcePaths(LLM, 구 형식): ${Date.now() - t0}ms`, + ); + + if (relevantResources.length === 0) { + return { content: '', usedResources: [] }; + } + + this.logger.log( + `[DEBUG] 1차 선별 결과(상위 관련 문서 경로): ${relevantResources.length}개`, + ); + + t0 = Date.now(); + const resourceResults = await Promise.all( + relevantResources.map(async (resource) => { + try { + const resourcePath = this.normalizeResourcePath(resource.path); + this.logger.debug(`Fetching markdown resource: ${resourcePath}`); + const toolResult = await this.mcpClientService.callTool( + 'get_resource', + { path: resourcePath }, + ); + const content = this.extractContentFromToolResult(toolResult); + if (content) { + const documentTitle = this.extractDocumentTitle( + resourcePath, + resource.path, + resource.formats, + ); + const subDocuments = this.parseDocumentLinks(content); + return { + title: documentTitle, + content, + path: resource.path, + formats: resource.formats || [], + subDocuments, + }; + } + } catch (error) { + this.logger.warn( + `Failed to fetch ${resource.path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return null; + }), + ); + const documentCandidates = resourceResults.filter( + ( + r, + ): r is { + title: string; + content: string; + path: string; + formats: string[]; + subDocuments: Array<{ path: string; description: string }>; + } => r !== null, + ); + this.logger.log( + `[PERF] get_resource 루프(구 형식, ${relevantResources.length}개): ${Date.now() - t0}ms`, + ); + + if (documentCandidates.length === 0) { + return { content: '', usedResources: [] }; + } + + this.logger.log( + `[DEBUG] 2차 선별(본문 기준) 입력: 후보 문서 ${documentCandidates.length}개 → LLM에 전달`, + ); + + t0 = Date.now(); + const selectedDocuments = await this.resourceSelectionService.selectMostRelevantDocuments( + question, + documentCandidates.map((doc) => ({ + title: doc.title, + content: doc.content, + path: doc.path, + })), + tokenUsage, + ); + this.logger.log( + `[PERF] selectMostRelevantDocuments(LLM, 구 형식): ${Date.now() - t0}ms`, + ); + + this.logger.log( + `[DEBUG] 2차 선별 결과(최종 사용 문서): ${selectedDocuments.length}개`, + ); + if (selectedDocuments.length === 0) { + this.logger.log('No documents selected by LLM as relevant'); + return { content: '', usedResources: [] }; + } + + const contents: string[] = []; + const allSubDocuments: Array<{ path: string; description: string }> = []; + const usedResources: Array<{ path: string; formats: string[] }> = []; + const addedPaths = new Set(); + + for (const selected of selectedDocuments) { + const docCandidate = documentCandidates.find( + (d) => d.title === selected.title, + ); + if (docCandidate) { + contents.push( + `\n\n## 리소스: ${docCandidate.title}\n\n${docCandidate.content}`, + ); + + const hasPdf = docCandidate.formats.includes('pdf'); + const hasPng = docCandidate.formats.includes('png'); + if (hasPdf || hasPng) { + const pdfPngFormats = docCandidate.formats.filter( + (f) => f === 'pdf' || f === 'png', + ); + usedResources.push({ + path: docCandidate.path, + formats: pdfPngFormats, + }); + addedPaths.add(docCandidate.path); + } + + if (docCandidate.subDocuments.length > 0) { + allSubDocuments.push(...docCandidate.subDocuments); + } + } + } + + for (const selected of selectedDocuments) { + const path = selected.path; + const firstSegment = path.split('/')[0]; + for (const r of filteredResources) { + if (!r.formats) continue; + if (addedPaths.has(r.path)) continue; + if (r.formats.includes('pdf')) { + const pathLower = r.path.toLowerCase(); + if (pathLower.endsWith('.png')) continue; + const match = + r.path === firstSegment || + r.path === `${firstSegment}.pdf` || + r.path.startsWith(`${firstSegment}.`); + if (match) { + usedResources.push({ path: r.path, formats: ['pdf'] }); + addedPaths.add(r.path); + } + } + } + } + + for (const selected of selectedDocuments) { + const docCandidate = documentCandidates.find( + (d) => d.title === selected.title, + ); + if (!docCandidate?.content) continue; + const imageRefs = this.parseImageReferencesFromMarkdown( + docCandidate.content, + ); + for (const imageRef of imageRefs) { + const fullPath = this.resolveImagePath(imageRef, docCandidate.path); + const pathWithoutExt = fullPath.replace(/\.(png|jpe?g|gif|webp)$/i, ''); + const r = filteredResources.find( + (x) => + x.formats?.includes('png') && + !addedPaths.has(x.path) && + (x.path === fullPath || + x.path === pathWithoutExt || + x.path.toLowerCase() === fullPath.toLowerCase() || + x.path.toLowerCase() === pathWithoutExt.toLowerCase()), + ); + if (r) { + usedResources.push({ path: r.path, formats: ['png'] }); + addedPaths.add(r.path); + } + } + } + + const finalUsedResources = usedResources.slice(0, 5); + + // 하위 문서 중 질문과 관련된 문서 찾아서 추가로 가져오기 + if (allSubDocuments.length > 0) { + const relevantSubDocuments = this.findRelevantSubDocuments( + question, + allSubDocuments, + 3, // 최대 3개의 하위 문서만 추가로 가져오기 + ); + + if (relevantSubDocuments.length > 0) { + this.logger.log( + `Fetching ${relevantSubDocuments.length} relevant sub-document(s)`, + ); + t0 = Date.now(); + const subDocumentContents = + await this.fetchSubDocumentContents(relevantSubDocuments); + this.logger.log( + `[PERF] fetchSubDocumentContents(${relevantSubDocuments.length}개): ${Date.now() - t0}ms`, + ); + if (subDocumentContents) { + contents.push('\n\n---\n\n## 관련 하위 문서\n' + subDocumentContents); + } + } + } + + return { + content: contents.join('\n'), + usedResources: finalUsedResources, + }; + } + + private generateResourceUrl(resourcePath: string): string { + const encodedPath = encodeURIComponent(resourcePath); + return encodedPath; + } +} diff --git a/src/chat/services/resource-selection.service.ts b/src/chat/services/resource-selection.service.ts new file mode 100644 index 0000000..bc79624 --- /dev/null +++ b/src/chat/services/resource-selection.service.ts @@ -0,0 +1,249 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import type { ListResourceItem } from '../../mcp/mcp-client.service'; +import { LLM_CLIENT, type LlmClient } from '../llm/llm-client.interface'; +import type { LlmUsage } from '../types/llm.types'; +import { + DOCUMENT_SELECTION_SYSTEM_PROMPT, + getDocumentSelectionUserPrompt, + RESOURCE_PATH_SELECTION_SYSTEM_PROMPT, + getResourcePathSelectionUserPrompt, + CHUNK_SELECTION_SYSTEM_PROMPT, + getChunkSelectionUserPrompt, + formatResourceListForChunkSelection, +} from '../prompts'; + +/** + * LLM 기반 리소스/문서 선별 서비스 + */ +@Injectable() +export class ResourceSelectionService { + private readonly logger = new Logger(ResourceSelectionService.name); + + constructor(@Inject(LLM_CLIENT) private readonly llmClient: LlmClient) {} + + private addTokenUsage( + target: LlmUsage | undefined, + usage: LlmUsage | null | undefined, + ): void { + if (!target || !usage) return; + + const promptTokens = usage.prompt_tokens ?? 0; + const completionTokens = usage.completion_tokens ?? 0; + const totalTokens = usage.total_tokens ?? promptTokens + completionTokens; + + target.prompt_tokens += promptTokens; + target.completion_tokens += completionTokens; + target.total_tokens += totalTokens; + } + + async selectRelevantChunkPaths( + question: string, + resources: ListResourceItem[], + maxResults: number = 10, + tokenUsage?: LlmUsage, + ): Promise { + if (!resources?.length) { + return []; + } + + const resourceListText = formatResourceListForChunkSelection(resources); + const userPrompt = getChunkSelectionUserPrompt({ + question, + resourceListText, + maxSelect: maxResults, + }); + + try { + const response = await this.llmClient.callLLM( + [ + { role: 'system', content: CHUNK_SELECTION_SYSTEM_PROMPT }, + { role: 'user', content: userPrompt }, + ], + this.llmClient.getModel('light'), + { temperature: 0.1, max_tokens: 5000 }, + ); + this.addTokenUsage(tokenUsage, response.usage); + + let selectedText = response.choices[0]?.message?.content?.trim() || ''; + this.logger.debug(`LLM chunk selection raw: ${selectedText}`); + + // 마크다운 코드블록 제거 (```json ... ```) + const codeBlockMatch = selectedText.match(/```(?:json)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + selectedText = codeBlockMatch[1].trim(); + } + + const parsed = JSON.parse(selectedText) as unknown; + const paths = Array.isArray(parsed) + ? (parsed as string[]).filter( + (p) => typeof p === 'string' && p.length > 0, + ) + : []; + + const limited = paths.slice(0, maxResults); + this.logger.log(`[DEBUG] 1차 선별 결과(chunk 경로): ${limited.length}개`); + return limited; + } catch (error) { + this.logger.warn( + `Failed to select chunk paths by LLM: ${error instanceof Error ? error.message : String(error)}`, + ); + return []; + } + } + + /** + * LLM으로 질문과 의미·맥락상 관련 있는 리소스 경로를 선별 (구 형식) + * 키워드 매칭 대신 의미 기반으로 최대 maxResults개 선택합니다. + */ + async selectRelevantResourcePaths( + question: string, + resources: Array<{ path: string; formats?: string[] }>, + maxResults: number = 10, + tokenUsage?: LlmUsage, + ): Promise> { + if (!resources.length) { + return []; + } + + const pathList = resources.map((r, i) => `${i + 1}. ${r.path}`).join('\n'); + + const userPrompt = getResourcePathSelectionUserPrompt({ + pathList, + question, + maxSelect: maxResults, + }); + + try { + const response = await this.llmClient.callLLM( + [ + { role: 'system', content: RESOURCE_PATH_SELECTION_SYSTEM_PROMPT }, + { role: 'user', content: userPrompt }, + ], + this.llmClient.getModel('light'), + { temperature: 0.1, max_tokens: 200 }, + ); + this.addTokenUsage(tokenUsage, response.usage); + + const selectedText = response.choices[0]?.message?.content?.trim() || ''; + this.logger.debug(`LLM selected resource paths: ${selectedText}`); + + if (selectedText.toLowerCase().includes('없음')) { + return []; + } + + const numbers = + selectedText + .match(/\d+/g) + ?.map((n) => parseInt(n, 10) - 1) + .filter((n) => n >= 0 && n < resources.length) || []; + + const uniqueIndices = [...new Set(numbers)].slice(0, maxResults); + const selected = uniqueIndices.map((idx) => resources[idx]); + + this.logger.log(`Selected ${selected.length} resource path(s) by LLM`); + return selected; + } catch (error) { + this.logger.warn( + `Failed to select resource paths by LLM: ${error instanceof Error ? error.message : String(error)}`, + ); + return []; + } + } + + /** + * LLM에게 문서 목록을 주고 질문과 관련성이 높은 문서만 선별하도록 요청 + */ + async selectMostRelevantDocuments( + question: string, + documents: Array<{ title: string; content: string; path: string }>, + tokenUsage?: LlmUsage, + ): Promise> { + if (documents.length === 0) { + return []; + } + + // 문서가 1개면 선별 불필요 + if (documents.length === 1) { + return documents; + } + + try { + // 제목 + 내용 앞부분(요약)을 주어 경로/제목에 키워드가 없어도 내용으로 관련 문서 선별 가능하게 함 + const CONTENT_SNIPPET_LENGTH = 280; + const documentList = documents + .map((doc, index) => { + const snippet = + doc.content.length > CONTENT_SNIPPET_LENGTH + ? doc.content + .slice(0, CONTENT_SNIPPET_LENGTH) + .replace(/\n/g, ' ') + '...' + : doc.content.replace(/\n/g, ' '); + return `${index + 1}. ${doc.title}\n 내용 요약: ${snippet}`; + }) + .join('\n\n'); + + const selectionPrompt = getDocumentSelectionUserPrompt({ + documentList, + question, + }); + + this.logger.debug( + `Selection prompt length: ${selectionPrompt.length} chars, documents: ${documents.length}`, + ); + + const response = await this.llmClient.callLLM( + [ + { + role: 'system', + content: DOCUMENT_SELECTION_SYSTEM_PROMPT, + }, + { + role: 'user', + content: selectionPrompt, + }, + ], + this.llmClient.getModel('normal'), + { temperature: 0.1, max_tokens: 100 }, + ); + this.addTokenUsage(tokenUsage, response.usage); + + const selectedText = response.choices[0]?.message?.content?.trim() || ''; + this.logger.debug(`LLM selected documents: ${selectedText}`); + + // "없음"이면 빈 배열 반환 (관련 없는 질문일 수 있으므로 문서 강제 선택 안 함) + if (selectedText.toLowerCase().includes('없음')) { + return []; + } + + // 번호 추출 (예: "1, 3, 5" 또는 "1,3,5") + const numbers = + selectedText + .match(/\d+/g) + ?.map((n) => parseInt(n, 10) - 1) // 0-based index로 변환 + .filter((n) => n >= 0 && n < documents.length) || []; + + if (numbers.length === 0) { + // 번호를 파싱할 수 없으면 앞쪽 문서 반환 (최대 5개) + this.logger.warn( + `Could not parse document selection, returning first 5 documents`, + ); + return documents.slice(0, 5); + } + + // 최대 5개로 제한 (중요 문서 놓치지 않도록) + const limitedNumbers = numbers.slice(0, 5); + const selected = limitedNumbers.map((idx) => documents[idx]); + this.logger.log( + `Selected ${selected.length} relevant document(s) out of ${documents.length}`, + ); + + return selected; + } catch (error) { + this.logger.warn( + `Failed to select relevant documents: ${error instanceof Error ? error.message : String(error)}`, + ); + // 에러 발생 시 모든 문서 반환 + return documents; + } + } +} diff --git a/src/chat/types/open-router.types.ts b/src/chat/types/open-router.types.ts deleted file mode 100644 index 7c648f8..0000000 --- a/src/chat/types/open-router.types.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * MCP Tool 정보 (McpClientService.listTools()에서 반환되는 형식) - */ -export interface McpTool { - name: string; - description: string; - inputSchema?: { - type: string; - properties?: Record; - required?: string[]; - }; -} - -/** - * Open Router Function Calling 형식의 Tool - */ -export interface OpenRouterTool { - type: 'function'; - function: { - name: string; - description: string; - parameters: { - type: string; - properties?: Record; - required?: string[]; - }; - }; -} - -/** - * Open Router API 메시지 형식 - */ -export interface OpenRouterMessage { - role: 'system' | 'user' | 'assistant' | 'tool'; - content: string | null; - tool_call_id?: string; - name?: string; - tool_calls?: OpenRouterToolCall[]; -} - -/** - * Open Router Tool Call 형식 - */ -export interface OpenRouterToolCall { - id: string; - type: 'function'; - function: { - name: string; - arguments: string; // JSON string - }; -} - -/** - * Open Router API 요청 형식 - */ -export interface OpenRouterRequest { - model: string; - messages: OpenRouterMessage[]; - tools?: OpenRouterTool[]; - tool_choice?: - | 'auto' - | 'none' - | { type: 'function'; function: { name: string } }; - temperature?: number; - max_tokens?: number; - stream?: boolean; - stream_options?: { - include_usage?: boolean; - }; -} - -export interface OpenRouterUsage { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; -} - -/** - * Open Router API 응답 형식 - */ -export interface OpenRouterResponse { - id: string; - model: string; - choices: Array<{ - index: number; - message: OpenRouterMessage; - finish_reason: 'stop' | 'length' | 'tool_calls' | null; - }>; - usage?: OpenRouterUsage; -} - -/** - * 파싱된 Tool Call 정보 - */ -export interface ParsedToolCall { - id: string; - name: string; - arguments: Record; -} From 3a7dad5e8abda5d0357af58bb8716859873fd942 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Tue, 28 Jul 2026 21:08:35 -0700 Subject: [PATCH 03/40] test: cover LLM providers and split chat orchestration services Add unit tests for provider selection, resource selection/content, SSE transport, and LLM_PROVIDER env validation. --- src/chat/llm/llm-client.provider.spec.ts | 85 +++++++++++ .../chat-orchestration.service.spec.ts | 50 ++++--- .../services/chat-stream.transport.spec.ts | 95 ++++++++++++ .../services/resource-content.service.spec.ts | 140 ++++++++++++++++++ .../resource-selection.service.spec.ts | 123 +++++++++++++++ src/config/env.validation.spec.ts | 60 ++++++++ 6 files changed, 534 insertions(+), 19 deletions(-) create mode 100644 src/chat/llm/llm-client.provider.spec.ts create mode 100644 src/chat/services/chat-stream.transport.spec.ts create mode 100644 src/chat/services/resource-content.service.spec.ts create mode 100644 src/chat/services/resource-selection.service.spec.ts create mode 100644 src/config/env.validation.spec.ts diff --git a/src/chat/llm/llm-client.provider.spec.ts b/src/chat/llm/llm-client.provider.spec.ts new file mode 100644 index 0000000..c40e36f --- /dev/null +++ b/src/chat/llm/llm-client.provider.spec.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { Test } from '@nestjs/testing'; +import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import { LLM_CLIENT } from './llm-client.interface'; +import { + llmClientProvider, + resolveLlmProviderName, +} from './llm-client.provider'; +import { LetsurLlmService } from './letsur-llm.service'; +import { OpenRouterLlmService } from './open-router-llm.service'; + +describe('resolveLlmProviderName', () => { + it('defaults to letsur', () => { + expect(resolveLlmProviderName(undefined)).toBe('letsur'); + expect(resolveLlmProviderName('')).toBe('letsur'); + expect(resolveLlmProviderName('letsur')).toBe('letsur'); + expect(resolveLlmProviderName('LETSUR')).toBe('letsur'); + }); + + it('resolves openrouter', () => { + expect(resolveLlmProviderName('openrouter')).toBe('openrouter'); + expect(resolveLlmProviderName(' OpenRouter ')).toBe('openrouter'); + }); +}); + +describe('llmClientProvider factory', () => { + it('provides LetsurLlmService by default', async () => { + const moduleRef = await Test.createTestingModule({ + providers: [ + llmClientProvider, + { + provide: ConfigService, + useValue: { + get: jest.fn((key: string) => { + if (key === 'LLM_PROVIDER') return undefined; + if (key === 'LETSUR_AI_GATEWAY_MODEL') return 'gpt-4o-mini'; + return undefined; + }), + getOrThrow: jest.fn((key: string) => { + if (key === 'LETSUR_AI_GATEWAY_API_KEY') return 'letsur-key'; + if (key === 'LETSUR_AI_GATEWAY_BASE_URL') + return 'https://gw.letsur.ai/v1'; + throw new Error(`unexpected key: ${key}`); + }), + }, + }, + { provide: HttpService, useValue: {} }, + ], + }).compile(); + + const client = moduleRef.get(LLM_CLIENT); + expect(client).toBeInstanceOf(LetsurLlmService); + expect(client.getModel('light')).toBe('gpt-4o-mini'); + }); + + it('provides OpenRouterLlmService when LLM_PROVIDER=openrouter', async () => { + const moduleRef = await Test.createTestingModule({ + providers: [ + llmClientProvider, + { + provide: ConfigService, + useValue: { + get: jest.fn((key: string) => { + if (key === 'LLM_PROVIDER') return 'openrouter'; + if (key === 'OPEN_ROUTER_MODEL_LIGHT') return 'light-model'; + if (key === 'OPEN_ROUTER_MODEL_NORMAL') return 'normal-model'; + if (key === 'OPEN_ROUTER_MODEL_HEAVY') return 'heavy-model'; + return undefined; + }), + getOrThrow: jest.fn((key: string) => { + if (key === 'OPEN_ROUTER_API_KEY') return 'openrouter-key'; + throw new Error(`unexpected key: ${key}`); + }), + }, + }, + { provide: HttpService, useValue: {} }, + ], + }).compile(); + + const client = moduleRef.get(LLM_CLIENT); + expect(client).toBeInstanceOf(OpenRouterLlmService); + expect(client.getModel('heavy')).toBe('heavy-model'); + }); +}); diff --git a/src/chat/services/chat-orchestration.service.spec.ts b/src/chat/services/chat-orchestration.service.spec.ts index e598c41..e22bb7f 100644 --- a/src/chat/services/chat-orchestration.service.spec.ts +++ b/src/chat/services/chat-orchestration.service.spec.ts @@ -3,13 +3,16 @@ import { describe, expect, it, jest } from '@jest/globals'; import { ChatOrchestrationService } from './chat-orchestration.service'; import { MessageRole } from '../../common/dto/chat-message-input.dto'; import type { ListResourcesResult } from '../../mcp/mcp-client.service'; -import type { OpenRouterResponse } from '../types/open-router.types'; +import type { LlmResponse } from '../types/llm.types'; +import { ResourceContentService } from './resource-content.service'; +import { ResourceSelectionService } from './resource-selection.service'; +import { ChatStreamTransport } from './chat-stream.transport'; describe('ChatOrchestrationService', () => { - function createOpenRouterResponse( + function createLlmResponse( content: string, totalTokens: number, - ): OpenRouterResponse { + ): LlmResponse { return { id: `response-${totalTokens}`, model: 'test-model', @@ -72,18 +75,18 @@ describe('ChatOrchestrationService', () => { throw new Error(`Unexpected tool call: ${name}`); }), }; - type CallLLM = (...args: unknown[]) => Promise; + type CallLLM = (...args: unknown[]) => Promise; type RecordUsage = ( sessionId: string, input: { totalTokens: number }, ) => Promise; - const openRouterService = { + const llmClient = { getModel: jest.fn((type: string) => `${type}-model`), callLLM: jest .fn() - .mockResolvedValueOnce(createOpenRouterResponse('1, 2', 100)) - .mockResolvedValueOnce(createOpenRouterResponse('1', 200)), + .mockResolvedValueOnce(createLlmResponse('1, 2', 100)) + .mockResolvedValueOnce(createLlmResponse('1', 200)), generateFinalResponseStream: jest.fn(async () => finalStream), }; const chatService = { @@ -97,43 +100,52 @@ describe('ChatOrchestrationService', () => { const usageService = { recordUsage: jest.fn(async () => undefined), }; - const configService = { + + const resourceSelectionService = new ResourceSelectionService( + llmClient as never, + ); + const resourceContentService = new ResourceContentService( + mcpClientService as never, + resourceSelectionService, + ); + const chatStreamTransport = new ChatStreamTransport({ get: jest.fn((key: string) => key === 'DOMAIN_NAME' ? 'example.com' : undefined, ), - }; + } as never); const service = new ChatOrchestrationService( mcpClientService as never, - openRouterService as never, + llmClient as never, chatService as never, usageService as never, - configService as never, + resourceContentService, + chatStreamTransport, ); - let resolveEnd: () => void; - const responseEnded = new Promise((resolve) => { - resolveEnd = resolve; - }); const reply = { hijack: jest.fn(), raw: { writeHead: jest.fn(), write: jest.fn(), - end: jest.fn(() => resolveEnd()), + end: jest.fn(), }, }; const req = { headers: { origin: 'http://localhost:5173' }, }; - await service.handleStreamingResponse( + const handlePromise = service.handleStreamingResponse( 'session-id', '졸업 요건 알려줘', reply as never, req as never, ); + // processUserQuestionStream이 generateFinalResponseStream을 호출한 뒤 + // consumeAndForward가 stream을 구독할 시간을 준다. + await new Promise((r) => setImmediate(r)); + finalStream.write( `data: ${JSON.stringify({ model: 'heavy-model', @@ -151,9 +163,9 @@ describe('ChatOrchestrationService', () => { ); finalStream.end('data: [DONE]\n\n'); - await responseEnded; + await handlePromise; - expect(openRouterService.callLLM).toHaveBeenCalledTimes(2); + expect(llmClient.callLLM).toHaveBeenCalledTimes(2); expect(usageService.recordUsage).toHaveBeenCalledWith('session-id', { totalTokens: 600, }); diff --git a/src/chat/services/chat-stream.transport.spec.ts b/src/chat/services/chat-stream.transport.spec.ts new file mode 100644 index 0000000..51cc37b --- /dev/null +++ b/src/chat/services/chat-stream.transport.spec.ts @@ -0,0 +1,95 @@ +import { PassThrough } from 'node:stream'; +import { describe, expect, it, jest } from '@jest/globals'; +import { ChatStreamTransport } from './chat-stream.transport'; + +describe('ChatStreamTransport', () => { + function createTransport() { + return new ChatStreamTransport({ + get: jest.fn((key: string) => + key === 'DOMAIN_NAME' ? 'example.com' : undefined, + ), + } as never); + } + + it('prepares SSE headers with allowed origin', () => { + const transport = createTransport(); + const reply = { + hijack: jest.fn(), + raw: { writeHead: jest.fn(), write: jest.fn(), end: jest.fn() }, + }; + const req = { headers: { origin: 'http://localhost:5173' } }; + + transport.prepareSse(reply as never, req as never); + + expect(reply.hijack).toHaveBeenCalled(); + expect(reply.raw.writeHead).toHaveBeenCalledWith( + 200, + expect.objectContaining({ + 'Content-Type': 'text/event-stream', + 'Access-Control-Allow-Origin': 'http://localhost:5173', + }), + ); + }); + + it('forwards content deltas and resolves usage on stream end', async () => { + const transport = createTransport(); + const reply = { + raw: { write: jest.fn(), end: jest.fn() }, + }; + const stream = new PassThrough(); + + const consumePromise = transport.consumeAndForward(stream, reply as never); + + stream.write( + `data: ${JSON.stringify({ + model: 'heavy-model', + choices: [{ delta: { content: '안녕' } }], + })}\n\n`, + ); + stream.write( + `data: ${JSON.stringify({ + usage: { + prompt_tokens: 1, + completion_tokens: 2, + total_tokens: 3, + }, + })}\n\n`, + ); + stream.end(); + + await expect(consumePromise).resolves.toEqual({ + accumulatedContent: '안녕', + model: 'heavy-model', + usage: { + prompt_tokens: 1, + completion_tokens: 2, + total_tokens: 3, + }, + }); + expect(reply.raw.write).toHaveBeenCalledWith( + `data: ${JSON.stringify({ content: '안녕' })}\n\n`, + ); + }); + + it('writes resources and done events', () => { + const transport = createTransport(); + const reply = { + raw: { write: jest.fn(), end: jest.fn() }, + }; + + transport.writeResources(reply as never, [ + { path: 'a.pdf', formats: ['pdf'], url: 'a.pdf' }, + ]); + transport.writeDone(reply as never); + + expect(reply.raw.write).toHaveBeenNthCalledWith( + 1, + `data: ${JSON.stringify({ + type: 'resources', + resources: [{ path: 'a.pdf', formats: ['pdf'], url: 'a.pdf' }], + })}\n\n`, + ); + expect(reply.raw.write).toHaveBeenNthCalledWith(2, 'data: [DONE]\n\n'); + expect(reply.raw.end).toHaveBeenCalled(); + }); +}); diff --git a/src/chat/services/resource-content.service.spec.ts b/src/chat/services/resource-content.service.spec.ts new file mode 100644 index 0000000..2a21d6e --- /dev/null +++ b/src/chat/services/resource-content.service.spec.ts @@ -0,0 +1,140 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { ResourceContentService } from './resource-content.service'; +import type { ListResourcesResult } from '../../mcp/mcp-client.service'; +import type { LlmUsage } from '../types/llm.types'; + +describe('ResourceContentService', () => { + type CallTool = ( + name: string, + args?: Record, + ) => Promise; + + it('appends unique top-level PDF entries for FE resources', () => { + const service = new ResourceContentService( + { callTool: jest.fn() } as never, + {} as never, + ); + const out: Array<{ path: string; formats: string[]; url: string }> = []; + const seen = new Set(); + + service.appendFePdfResourceEntryFromUsed(out, seen, { + path: '학사편람/졸업요건.md', + formats: ['md'], + }); + service.appendFePdfResourceEntryFromUsed(out, seen, { + path: '학사편람/세부/표.png', + formats: ['png'], + }); + service.appendFePdfResourceEntryFromUsed(out, seen, { + path: '학사편람.pdf', + formats: ['pdf'], + }); + + expect(out).toEqual([ + { + path: '학사편람.pdf', + formats: ['pdf'], + url: encodeURIComponent('학사편람.pdf'), + }, + ]); + }); + + it('uses new-format chunk pipeline when resources+chunks exist', async () => { + const mcpClientService = { + callTool: jest.fn(async () => ({ + raw: {}, + texts: ['chunk body'], + resourceLinks: [], + embeddedResources: [], + filteredResources: [], + })), + }; + const resourceSelectionService = { + selectRelevantChunkPaths: jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValue(['학사편람/졸업.md']), + selectMostRelevantDocuments: jest + .fn< + ( + ...args: unknown[] + ) => Promise> + >() + .mockResolvedValue([ + { + title: '졸업.md', + content: 'chunk body', + path: '학사편람/졸업.md', + }, + ]), + selectRelevantResourcePaths: jest.fn(), + }; + + const service = new ResourceContentService( + mcpClientService as never, + resourceSelectionService as never, + ); + + const listResult = { + raw: {}, + texts: [], + resourceLinks: [], + embeddedResources: [], + filteredResources: [], + resources: [ + { + path: '학사편람', + description: '학사', + chunks: [{ path: '학사편람/졸업.md', description: '졸업' }], + }, + ], + chunks: [{ path: '학사편람/졸업.md', description: '졸업' }], + } as ListResourcesResult; + + const usage: LlmUsage = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }; + + const result = await service.fetchRelevantResourceContents( + '졸업 요건', + listResult, + usage, + ); + + expect( + resourceSelectionService.selectRelevantChunkPaths, + ).toHaveBeenCalled(); + expect( + resourceSelectionService.selectRelevantResourcePaths, + ).not.toHaveBeenCalled(); + expect(mcpClientService.callTool).toHaveBeenCalledWith('get_resource', { + path: '학사편람/졸업', + }); + expect(result.content).toContain('chunk body'); + expect(result.usedResources.some((r) => r.path.includes('학사편람'))).toBe( + true, + ); + }); + + it('returns empty when legacy filteredResources has no markdown', async () => { + const service = new ResourceContentService( + { callTool: jest.fn() } as never, + { + selectRelevantResourcePaths: jest.fn(), + } as never, + ); + + const listResult = { + raw: {}, + texts: [], + resourceLinks: [], + embeddedResources: [], + filteredResources: [{ path: '학사편람.pdf', formats: ['pdf'] }], + } as ListResourcesResult; + + await expect( + service.fetchRelevantResourceContents('질문', listResult), + ).resolves.toEqual({ content: '', usedResources: [] }); + }); +}); diff --git a/src/chat/services/resource-selection.service.spec.ts b/src/chat/services/resource-selection.service.spec.ts new file mode 100644 index 0000000..9e47ba4 --- /dev/null +++ b/src/chat/services/resource-selection.service.spec.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { ResourceSelectionService } from './resource-selection.service'; +import type { LlmResponse, LlmUsage } from '../types/llm.types'; +import type { ListResourceItem } from '../../mcp/mcp-client.service'; + +function createLlmResponse(content: string, totalTokens = 10): LlmResponse { + return { + id: 'resp', + model: 'test', + choices: [ + { + index: 0, + message: { role: 'assistant', content }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: Math.floor(totalTokens * 0.6), + completion_tokens: totalTokens - Math.floor(totalTokens * 0.6), + total_tokens: totalTokens, + }, + }; +} + +describe('ResourceSelectionService', () => { + type CallLLM = (...args: unknown[]) => Promise; + + function createService(callLLM: jest.Mock) { + const llmClient = { + getModel: jest.fn((type: string) => `${type}-model`), + callLLM, + generateFinalResponseStream: jest.fn(), + }; + return { + service: new ResourceSelectionService(llmClient as never), + llmClient, + }; + } + + it('returns empty array when chunk resources are empty', async () => { + const callLLM = jest.fn(); + const { service } = createService(callLLM); + + await expect( + service.selectRelevantChunkPaths('질문', [], 10), + ).resolves.toEqual([]); + expect(callLLM).not.toHaveBeenCalled(); + }); + + it('parses chunk paths from JSON and accumulates token usage', async () => { + const callLLM = jest + .fn() + .mockResolvedValue( + createLlmResponse('```json\n["a/b.md", "c/d.md"]\n```', 100), + ); + const { service } = createService(callLLM); + const usage: LlmUsage = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }; + + const resources: ListResourceItem[] = [ + { + path: 'root', + description: 'desc', + chunks: [{ path: 'a/b.md', description: 'b' }], + }, + ]; + + const paths = await service.selectRelevantChunkPaths( + '질문', + resources, + 10, + usage, + ); + + expect(paths).toEqual(['a/b.md', 'c/d.md']); + expect(usage.total_tokens).toBe(100); + }); + + it('returns empty when path selection says 없음', async () => { + const callLLM = jest + .fn() + .mockResolvedValue(createLlmResponse('없음')); + const { service } = createService(callLLM); + + const selected = await service.selectRelevantResourcePaths( + '질문', + [{ path: '학사편람/졸업.md', formats: ['md'] }], + 5, + ); + + expect(selected).toEqual([]); + }); + + it('selects documents by index numbers', async () => { + const callLLM = jest + .fn() + .mockResolvedValue(createLlmResponse('2, 1')); + const { service } = createService(callLLM); + const docs = [ + { title: 'a.md', content: 'aaa', path: 'a.md' }, + { title: 'b.md', content: 'bbb', path: 'b.md' }, + { title: 'c.md', content: 'ccc', path: 'c.md' }, + ]; + + const selected = await service.selectMostRelevantDocuments('질문', docs); + + expect(selected.map((d) => d.path)).toEqual(['b.md', 'a.md']); + }); + + it('returns single document without calling LLM', async () => { + const callLLM = jest.fn(); + const { service } = createService(callLLM); + const docs = [{ title: 'only.md', content: 'x', path: 'only.md' }]; + + await expect( + service.selectMostRelevantDocuments('질문', docs), + ).resolves.toEqual(docs); + expect(callLLM).not.toHaveBeenCalled(); + }); +}); diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts new file mode 100644 index 0000000..fc11507 --- /dev/null +++ b/src/config/env.validation.spec.ts @@ -0,0 +1,60 @@ +import 'reflect-metadata'; +import { describe, expect, it } from '@jest/globals'; +import { validate } from './env.validation'; + +function baseEnv(overrides: Record = {}) { + return { + DB_HOST: 'localhost', + DB_PORT: 5432, + DB_USER: 'test-user', + DB_PASSWORD: 'test-only-placeholder', + DB_NAME: 'test', + DB_SSL: false, + PORT: 3000, + NODE_ENV: 'test', + JWT_SECRET: 'x'.repeat(32), + JWT_EXPIRES_IN: 3600, + ADMIN_BEARER_TOKEN: 'a'.repeat(16), + IDP_URL: 'https://idp.example.com', + IDP_CLIENT_ID: 'test-client-id', + IDP_CLIENT_SECRET: 'test-client-secret', + DOMAIN_NAME: 'example.com', + MCP_BASE_URL: 'https://mcp.example.com', + MCP_RESOURCE_API_URL: 'https://mcp-resource.example.com', + ...overrides, + }; +} + +describe('env validation for LLM_PROVIDER', () => { + it('requires Letsur credentials by default', () => { + expect(() => + validate( + baseEnv({ + LETSUR_AI_GATEWAY_BASE_URL: 'https://gw.letsur.ai/v1', + LETSUR_AI_GATEWAY_API_KEY: 'letsur-key', + }), + ), + ).not.toThrow(); + + expect(() => validate(baseEnv({}))).toThrow(/LETSUR_AI_GATEWAY/); + }); + + it('requires OpenRouter credentials when LLM_PROVIDER=openrouter', () => { + expect(() => + validate( + baseEnv({ + LLM_PROVIDER: 'openrouter', + OPEN_ROUTER_API_KEY: 'or-key', + }), + ), + ).not.toThrow(); + + expect(() => + validate( + baseEnv({ + LLM_PROVIDER: 'openrouter', + }), + ), + ).toThrow(/OPEN_ROUTER_API_KEY/); + }); +}); From df2287ec00776fee8ef1716125d61f68250e2999 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 04/40] feat(db): add documents and chunks tables for PDF ingestion Introduce ready-state document catalog and processing_token fields so async PDF jobs can be claimed, cancelled, and recovered safely. Co-authored-by: Cursor --- drizzle/0009_goofy_starjammers.sql | 34 + drizzle/0010_gray_the_order.sql | 2 + drizzle/meta/0009_snapshot.json | 1332 +++++++++++++++++++++++++++ drizzle/meta/0010_snapshot.json | 1339 ++++++++++++++++++++++++++++ drizzle/meta/_journal.json | 14 + src/db/schema.ts | 92 ++ 6 files changed, 2813 insertions(+) create mode 100644 drizzle/0009_goofy_starjammers.sql create mode 100644 drizzle/0010_gray_the_order.sql create mode 100644 drizzle/meta/0009_snapshot.json create mode 100644 drizzle/meta/0010_snapshot.json diff --git a/drizzle/0009_goofy_starjammers.sql b/drizzle/0009_goofy_starjammers.sql new file mode 100644 index 0000000..fbaa383 --- /dev/null +++ b/drizzle/0009_goofy_starjammers.sql @@ -0,0 +1,34 @@ +CREATE TYPE "public"."document_status" AS ENUM('queued', 'processing', 'ready', 'failed');--> statement-breakpoint +CREATE TABLE "document_chunks" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "document_id" uuid NOT NULL, + "path" varchar(1024) NOT NULL, + "description" text DEFAULT '' NOT NULL, + "content" text NOT NULL, + "sort_order" integer DEFAULT 0 NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "documents" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "title" varchar(512) NOT NULL, + "resource_name" varchar(512) NOT NULL, + "summary" text, + "gcs_pdf_path" varchar(1024) NOT NULL, + "status" "document_status" DEFAULT 'queued' NOT NULL, + "error_message" text, + "uploaded_by_idp_uuid" varchar(255) NOT NULL, + "is_active" boolean DEFAULT true NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + "processed_at" timestamp +); +--> statement-breakpoint +ALTER TABLE "document_chunks" ADD CONSTRAINT "document_chunks_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "document_chunks_document_id_idx" ON "document_chunks" USING btree ("document_id");--> statement-breakpoint +CREATE INDEX "document_chunks_document_sort_idx" ON "document_chunks" USING btree ("document_id","sort_order");--> statement-breakpoint +CREATE UNIQUE INDEX "documents_resource_name_active_unique" ON "documents" USING btree ("resource_name") WHERE "documents"."is_active" = true;--> statement-breakpoint +CREATE INDEX "documents_status_idx" ON "documents" USING btree ("status");--> statement-breakpoint +CREATE INDEX "documents_uploaded_by_idp_uuid_idx" ON "documents" USING btree ("uploaded_by_idp_uuid");--> statement-breakpoint +CREATE INDEX "documents_is_active_idx" ON "documents" USING btree ("is_active");--> statement-breakpoint +CREATE INDEX "documents_created_at_idx" ON "documents" USING btree ("created_at"); \ No newline at end of file diff --git a/drizzle/0010_gray_the_order.sql b/drizzle/0010_gray_the_order.sql new file mode 100644 index 0000000..22070b3 --- /dev/null +++ b/drizzle/0010_gray_the_order.sql @@ -0,0 +1,2 @@ +ALTER TYPE "public"."document_status" ADD VALUE 'uploading' BEFORE 'queued';--> statement-breakpoint +ALTER TABLE "documents" ADD COLUMN "processing_token" uuid; \ No newline at end of file diff --git a/drizzle/meta/0009_snapshot.json b/drizzle/meta/0009_snapshot.json new file mode 100644 index 0000000..5e328c6 --- /dev/null +++ b/drizzle/meta/0009_snapshot.json @@ -0,0 +1,1332 @@ +{ + "id": "c8cac83f-7720-440c-bc69-73f22ddc03a4", + "prevId": "313e86c6-923a-4e32-a45c-eeaa33d885fe", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0010_snapshot.json b/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000..6367474 --- /dev/null +++ b/drizzle/meta/0010_snapshot.json @@ -0,0 +1,1339 @@ +{ + "id": "19907c5f-6419-4ba3-bc5b-8d98425ba12b", + "prevId": "c8cac83f-7720-440c-bc69-73f22ddc03a4", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 09b0273..45dcdb2 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -64,6 +64,20 @@ "when": 1783662838940, "tag": "0008_add_usage_daily_answer_counts", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1785302489221, + "tag": "0009_goofy_starjammers", + "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1785357206423, + "tag": "0010_gray_the_order", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index d0152b6..63f9a72 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -37,6 +37,14 @@ export const collaboratorStatusEnum = pgEnum('collaborator_status', [ 'ACCEPTED', ]); +export const documentStatusEnum = pgEnum('document_status', [ + 'uploading', + 'queued', + 'processing', + 'ready', + 'failed', +]); + // Tables /** @@ -186,6 +194,72 @@ export const uploadedResources = pgTable( }), ); +/** + * PDF 문서 테이블 (ingestion) + * - Admin 업로드 후 비동기 Pass1/2 처리 상태와 메타를 저장 + */ +export const documents = pgTable( + 'documents', + { + id: uuid('id').defaultRandom().primaryKey(), + title: varchar('title', { length: 512 }).notNull(), + resourceName: varchar('resource_name', { length: 512 }).notNull(), + summary: text('summary'), + gcsPdfPath: varchar('gcs_pdf_path', { length: 1024 }).notNull(), + status: documentStatusEnum('status').notNull().default('queued'), + errorMessage: text('error_message'), + processingToken: uuid('processing_token'), + uploadedByIdpUuid: varchar('uploaded_by_idp_uuid', { + length: 255, + }).notNull(), + isActive: boolean('is_active').notNull().default(true), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), + processedAt: timestamp('processed_at'), + }, + (table) => ({ + resourceNameActiveUnique: uniqueIndex( + 'documents_resource_name_active_unique', + ) + .on(table.resourceName) + .where(sql`${table.isActive} = true`), + statusIdx: index('documents_status_idx').on(table.status), + uploadedByIdpUuidIdx: index('documents_uploaded_by_idp_uuid_idx').on( + table.uploadedByIdpUuid, + ), + isActiveIdx: index('documents_is_active_idx').on(table.isActive), + createdAtIdx: index('documents_created_at_idx').on(table.createdAt), + }), +); + +/** + * 문서 청크 테이블 + * - Pass2 의미 청킹 결과 (path / description / content) + */ +export const documentChunks = pgTable( + 'document_chunks', + { + id: uuid('id').defaultRandom().primaryKey(), + documentId: uuid('document_id') + .notNull() + .references(() => documents.id, { onDelete: 'cascade' }), + path: varchar('path', { length: 1024 }).notNull(), + description: text('description').notNull().default(''), + content: text('content').notNull(), + sortOrder: integer('sort_order').notNull().default(0), + createdAt: timestamp('created_at').notNull().defaultNow(), + }, + (table) => ({ + documentIdIdx: index('document_chunks_document_id_idx').on( + table.documentId, + ), + documentSortIdx: index('document_chunks_document_sort_idx').on( + table.documentId, + table.sortOrder, + ), + }), +); + /** * 메시지 테이블 * - 채팅 메시지를 저장 @@ -283,6 +357,17 @@ export const usageDaily = pgTable( ); // Relations +export const documentsRelations = relations(documents, ({ many }) => ({ + chunks: many(documentChunks), +})); + +export const documentChunksRelations = relations(documentChunks, ({ one }) => ({ + document: one(documents, { + fields: [documentChunks.documentId], + references: [documents.id], + }), +})); + export const widgetKeysRelations = relations(widgetKeys, ({ many }) => ({ sessions: many(sessions), usageDaily: many(usageDaily), @@ -342,6 +427,13 @@ export type NewAdmin = typeof admins.$inferInsert; export type UploadedResource = typeof uploadedResources.$inferSelect; export type NewUploadedResource = typeof uploadedResources.$inferInsert; +export type Document = typeof documents.$inferSelect; +export type NewDocument = typeof documents.$inferInsert; +export type DocumentStatus = (typeof documentStatusEnum.enumValues)[number]; + +export type DocumentChunk = typeof documentChunks.$inferSelect; +export type NewDocumentChunk = typeof documentChunks.$inferInsert; + export type WidgetKey = typeof widgetKeys.$inferSelect; export type NewWidgetKey = typeof widgetKeys.$inferInsert; From 136d56eb08da86886ab6374c30564505c0fe20d1 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 05/40] feat(config): add GCS and PDF processor environment settings Support base64 service-account credentials and processor tuning knobs for Lightsail deployments outside GCP ADC. Co-authored-by: Cursor --- .env.example | 15 +++++++++++ docker-compose.yml | 8 ++++++ src/config/env.validation.spec.ts | 38 ++++++++++++++++++++++++++++ src/config/env.validation.ts | 42 +++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/.env.example b/.env.example index bac1591..208767f 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,21 @@ DOMAIN_NAME=example.com # MCP Server Url MCP_BASE_URL=your-mcp-server-url +MCP_RESOURCE_API_URL=your-mcp-resource-api-url + +# GCS (PDF processor) +GCS_BUCKET=gcs-bucket-name +GCP_PROJECT_ID=gcp-project-id +# GCP 밖 배포 환경: 서비스 계정 JSON 전체를 한 줄 base64로 인코딩 +# 값이 없으면 로컬 gcloud ADC 또는 GCP 런타임 서비스 계정을 사용 +GCS_SERVICE_ACCOUNT_KEY_BASE64= + +# PDF Processor +PDF_PROCESSOR_CONCURRENCY=1 +PDF_PROCESSOR_CONTEXT_LENGTH=500 +PDF_PROCESSOR_LLM_TIMEOUT=120 +PDF_PROCESSOR_POLL_INTERVAL_MS=2000 +# PDF_PROCESSOR_STALE_PROCESSING_MS=1800000 # LLM Provider: letsur (default) | openrouter LLM_PROVIDER=letsur diff --git a/docker-compose.yml b/docker-compose.yml index a801d46..e6ff7ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,14 @@ services: # MCP Configuration (optional) MCP_BASE_URL: ${MCP_BASE_URL:-} MCP_RESOURCE_API_URL: ${MCP_RESOURCE_API_URL:-} + # GCS / PDF Processor + GCS_BUCKET: ${GCS_BUCKET:-} + GCP_PROJECT_ID: ${GCP_PROJECT_ID:-} + GCS_SERVICE_ACCOUNT_KEY_BASE64: ${GCS_SERVICE_ACCOUNT_KEY_BASE64:-} + PDF_PROCESSOR_CONCURRENCY: ${PDF_PROCESSOR_CONCURRENCY:-1} + PDF_PROCESSOR_CONTEXT_LENGTH: ${PDF_PROCESSOR_CONTEXT_LENGTH:-500} + PDF_PROCESSOR_LLM_TIMEOUT: ${PDF_PROCESSOR_LLM_TIMEOUT:-120} + PDF_PROCESSOR_POLL_INTERVAL_MS: ${PDF_PROCESSOR_POLL_INTERVAL_MS:-2000} # Letsur AI Gateway Configuration LETSUR_AI_GATEWAY_BASE_URL: ${LETSUR_AI_GATEWAY_BASE_URL:-} LETSUR_AI_GATEWAY_API_KEY: ${LETSUR_AI_GATEWAY_API_KEY:-} diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts index fc11507..d11dc23 100644 --- a/src/config/env.validation.spec.ts +++ b/src/config/env.validation.spec.ts @@ -21,6 +21,8 @@ function baseEnv(overrides: Record = {}) { DOMAIN_NAME: 'example.com', MCP_BASE_URL: 'https://mcp.example.com', MCP_RESOURCE_API_URL: 'https://mcp-resource.example.com', + GCS_BUCKET: 'ziggle-resources', + GCP_PROJECT_ID: 'ziggle-mcp-project', ...overrides, }; } @@ -58,3 +60,39 @@ describe('env validation for LLM_PROVIDER', () => { ).toThrow(/OPEN_ROUTER_API_KEY/); }); }); + +describe('env validation for GCS credentials', () => { + const letsurEnv = { + LETSUR_AI_GATEWAY_BASE_URL: 'https://gw.letsur.ai/v1', + LETSUR_AI_GATEWAY_API_KEY: 'letsur-key', + }; + + it('accepts a base64-encoded service account JSON', () => { + const encoded = Buffer.from( + JSON.stringify({ + client_email: 'storage@example.iam.gserviceaccount.com', + private_key: 'private-key', + }), + ).toString('base64'); + + expect(() => + validate( + baseEnv({ + ...letsurEnv, + GCS_SERVICE_ACCOUNT_KEY_BASE64: encoded, + }), + ), + ).not.toThrow(); + }); + + it('rejects a non-base64 credential value', () => { + expect(() => + validate( + baseEnv({ + ...letsurEnv, + GCS_SERVICE_ACCOUNT_KEY_BASE64: 'not base64!', + }), + ), + ).toThrow(/base64/); + }); +}); diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index bd798b1..a65ac47 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -6,6 +6,7 @@ import { IsEnum, IsNotEmpty, IsOptional, + IsBase64, Min, Max, MinLength, @@ -153,6 +154,47 @@ export class EnvironmentVariables { @IsNotEmpty() MCP_RESOURCE_API_URL: string; + // GCS (PDF processor) + @IsString() + @IsNotEmpty() + GCS_BUCKET: string; + + @IsString() + @IsNotEmpty() + GCP_PROJECT_ID: string; + + // Base64-encoded GCP service account JSON. When omitted, Google ADC is used. + @IsOptional() + @IsString() + @IsBase64() + GCS_SERVICE_ACCOUNT_KEY_BASE64?: string; + + @IsOptional() + @IsNumber() + @Min(1) + @Max(1) + PDF_PROCESSOR_CONCURRENCY?: number; + + @IsOptional() + @IsNumber() + @Min(1) + PDF_PROCESSOR_CONTEXT_LENGTH?: number; + + @IsOptional() + @IsNumber() + @Min(1) + PDF_PROCESSOR_LLM_TIMEOUT?: number; + + @IsOptional() + @IsNumber() + @Min(500) + PDF_PROCESSOR_POLL_INTERVAL_MS?: number; + + @IsOptional() + @IsNumber() + @Min(60000) + PDF_PROCESSOR_STALE_PROCESSING_MS?: number; + // Swagger API 문서 잠금 (둘 다 설정 시 Basic Auth 적용) @IsOptional() @IsString() From 5883935686aa458eade330b9a94675c8d8e8beab Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 06/40] feat(llm): allow per-call timeout for long processor requests Pass timeoutMs through OpenAI-compatible clients so page conversion and chunking can exceed the default chat timeout. Co-authored-by: Cursor --- src/chat/llm/base-openai-compatible.llm.ts | 2 +- src/chat/llm/llm-client.interface.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chat/llm/base-openai-compatible.llm.ts b/src/chat/llm/base-openai-compatible.llm.ts index 9a35e46..8744f0b 100644 --- a/src/chat/llm/base-openai-compatible.llm.ts +++ b/src/chat/llm/base-openai-compatible.llm.ts @@ -94,7 +94,7 @@ export abstract class BaseOpenAiCompatibleLlm implements LlmClient { this.httpService .post(`${this.baseUrl}/chat/completions`, request, { headers: this.buildHeaders(), - timeout: 15000, + timeout: options?.timeoutMs ?? 15000, }) .pipe( catchError((error: AxiosError) => { diff --git a/src/chat/llm/llm-client.interface.ts b/src/chat/llm/llm-client.interface.ts index 81863a2..aea2fa8 100644 --- a/src/chat/llm/llm-client.interface.ts +++ b/src/chat/llm/llm-client.interface.ts @@ -11,6 +11,8 @@ export const LLM_CLIENT = Symbol('LLM_CLIENT'); export type LlmCallOptions = { temperature?: number; max_tokens?: number; + /** axios request timeout in milliseconds (default 15000) */ + timeoutMs?: number; }; /** From cad944bc657f6dde72c0eff00ced2dae565dfe3a Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 07/40] feat(prompts): add PDF page conversion and semantic chunking prompts Port processor prompts with escaped markdown so Nest can format Pass 1/2 LLM requests safely. Co-authored-by: Cursor --- src/chat/prompts/index.ts | 2 + src/chat/prompts/pdf-chunking-prompt.ts | 127 ++++++++++++++++++++++++ src/chat/prompts/pdf-processor.ts | 96 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 src/chat/prompts/pdf-chunking-prompt.ts create mode 100644 src/chat/prompts/pdf-processor.ts diff --git a/src/chat/prompts/index.ts b/src/chat/prompts/index.ts index 8d5572b..626639c 100644 --- a/src/chat/prompts/index.ts +++ b/src/chat/prompts/index.ts @@ -6,3 +6,5 @@ export * from './document-selection.prompt'; export * from './resource-path-selection.prompt'; export * from './final-response.prompt'; export * from './error-messages.prompt'; +export * from './pdf-processor'; +export * from './pdf-chunking-prompt'; diff --git a/src/chat/prompts/pdf-chunking-prompt.ts b/src/chat/prompts/pdf-chunking-prompt.ts new file mode 100644 index 0000000..cb6b4c9 --- /dev/null +++ b/src/chat/prompts/pdf-chunking-prompt.ts @@ -0,0 +1,127 @@ +/** + * PDF 파일 청킹을 위한 시스템 프롬프트 + * + * Placeholders: {filename} + */ + +export const PDF_CHUNKING_PROMPT = ` +당신은 문서 구조화 전문가입니다. 완성된 Markdown 문서를 의미론적으로 완결된 청크로 분할하세요. + +**문서 정보:** +- 파일명: {filename} + +**당신의 임무:** +아래 제공된 전체 Markdown 문서를 읽고, 세부 섹션만 \`\` 태그로 분할하세요. +- **기본 문서**: 전체 개요, 소개, 목차 등은 태그 없이 그대로 유지 +- **서브 문서**: 독립적으로 조회할 가치가 있는 세부 섹션만 \`\` 태그로 분할 + +**청킹 규칙:** + +1. **청크 크기**: + - 각 청크는 250-1000 단어 정도의 완결된 주제 + - 너무 작게 쪼개지 말고, 의미론적으로 완결된 단위로 + +2. **청크 경계 결정**: + - 주요 섹션/챕터 중 세부 내용이 긴 경우만 분할 + - 독립적으로 조회할 가치가 있는 주제인 경우 + - 표나 리스트는 분리하지 말고 함께 유지 + - **모든 섹션을 분할할 필요 없음** - 개요나 짧은 섹션은 기본 문서에 포함 + +3. **청크 태그 형식**: + \`\`\` + + 내용 + + \`\`\` + +4. **path 요구사항** (매우 중요): + - **일관된 base path 사용**: 모든 청크는 같은 base path로 시작 + - 파일명에서 base를 추출하여 사용 (예: "2025-캠프-발표자료-2일차" 또는 "student-handbook") + - 하위 경로로 섹션 구분 (예: "base/권익인권센터/이용-안내", "base/학생팀/무한도전-프로젝트") + - 계층 구조 유지 (예: "base/section/subsection") + - 영문 소문자, 한글, 하이픈(-), 슬래시(/) 사용 가능 + - 공백은 하이픈으로 치환 + - 구체적이고 명확한 경로 (예: "base/g-surf/신청-자격", NOT "base/section-5") + +5. **description 요구사항** (매우 중요 — 검색 품질에 직결): + - 이 description은 사용자의 질문과 매칭하기 위한 용도입니다 + - **사용자가 이 내용을 찾기 위해 할 수 있는 질문의 키워드를 포함**해야 합니다 + - 문서 내용의 핵심 키워드 + 사용자가 사용할 수 있는 동의어/유사 표현을 포함 + - 예: "전화번호" 내용이면 → "연락처, 전화번호, 내선번호, 이메일" 모두 포함 + - 예: "도서관 시설" 내용이면 → "열람실, 스터디룸, 세미나실, 도서관 위치, 층별 안내" 포함 + - 15-40 단어로 충분히 상세하게 작성 + +6. **청크 내용**: + - 원본 Markdown 그대로 유지 (제목, 표, 리스트, 이미지 등) + - 내용 손실 금지 + - 모든 내용이 기본 문서 또는 서브 문서에 정확히 포함되어야 함 + - 단순 표지나 장식적인 내용, 감사합니다 등의 정보가 아닌 내용은 철저히 배제 + +**출력 형식:** +- **맨 처음에** \`\` 태그로 문서 전체의 고수준 요약을 출력 + - 이 문서가 어떤 주제/분야에 대한 문서인지 한눈에 파악할 수 있도록 +- 기본 문서 내용과 \`\` 태그들을 혼합하여 출력 +- 세부 섹션은 \`\` 태그로 대체 +- \`\`\`markdown\`\`\` 블록으로 감싸지 말 것 + +**예시:** + +입력: +\`\`\` +# 학생 편람 + +## 소개 +GIST 대학은 혁신적인 교육기관입니다. + +## 학사 일정 +### 2025년 봄학기 +- 개강: 3월 3일 +- 중간고사: 4월 20-26일 +... + +### 2025년 가을학기 +- 개강: 9월 1일 +... + +## 수강 신청 +수강 신청은 매 학기 시작 전에... +(매우 긴 상세 내용) +\`\`\` + +출력: +\`\`\` +GIST 학생 편람 - 학사 일정, 수강 신청 등 학사 생활 전반 안내 + +# 학생 편람 + +## 소개 +GIST 대학은 혁신적인 교육기관입니다. + + +## 학사 일정 +### 2025년 봄학기 +- 개강: 3월 3일 +- 중간고사: 4월 20-26일 +... + +### 2025년 가을학기 +- 개강: 9월 1일 +... + + + +## 수강 신청 +수강 신청은 매 학기 시작 전에... +(매우 긴 상세 내용) + +\`\`\` + +위 예시에서: +- \`\`는 문서의 고수준 개요 (학생 편람이라는 것, 학사 생활 안내라는 것) +- description은 사용자가 검색할 수 있는 동의어/키워드를 포함 (개강일, 시험 기간, 수강 정정 등) +- "소개" 섹션은 짧으므로 기본 문서에 포함 +- "학사 일정"과 "수강 신청"은 세부 내용이므로 서브 문서로 분할 + +이제 아래 문서를 청킹하세요: + +`; diff --git a/src/chat/prompts/pdf-processor.ts b/src/chat/prompts/pdf-processor.ts new file mode 100644 index 0000000..fec8b49 --- /dev/null +++ b/src/chat/prompts/pdf-processor.ts @@ -0,0 +1,96 @@ +/** + * PDF 파일 처리를 위한 시스템 프롬프트 + * + * Placeholders: {filename}, {total_pages}, {current_page}, {previous_context} + */ + +export const PDF_PROCESSOR_PROMPT = ` + + 당신은 전문 문서 변환 전문가입니다. PDF 문서를 깔끔하고 잘 구조화된 Markdown으로 변환하세요. + + **문서 정보:** + - 파일명: {filename} + - 총 페이지: {total_pages} + - 현재 페이지: {current_page} + + **이전 페이지 컨텍스트:** + {previous_context} + + **핵심 규칙:** + + 1. **파일명 기반 문서 이해**: + - 파일명을 보고 문서 성격을 파악하세요 + - "발표", "슬라이드", "presentation": + - 제목/목차/감사 페이지는 텍스트로 + - **슬라이드 덱 제목**(모든 페이지 상단 반복)은 문서 제목이지 각 슬라이드 제목이 아님 + - 실제 슬라이드 내용에 집중 + - 전체 발표 슬라이드를 단순히 텍스트로 옮기는 것이 아니라 문서로서 정리하는 것이 최종적 목표 + - '감사합니다', '질문?', 'Q&A'와 같이 문서로서의 의미가 없는 슬라이드는 제외 + - "안내", "편람": 가능한 모든 내용 텍스트로 변환 + + 2. **이전 페이지 맥락 활용**: + - 이전 페이지 내용을 반드시 고려하여 변환 + - 연속된 주제면 같은 흐름으로 이어가기 + - 제목 계층을 일관되게 유지 + - 갑작스러운 컨텍스트 단절 방지 + + 3. **텍스트 & 표**: + - 모든 텍스트와 표를 Markdown/HTML로 변환 + - 표는 반드시 HTML 형식 (, ,
, ) + - 한글 텍스트 완벽하게 보존 + - 표로 표현되었지만 간단하여 Markdown nested list나 listing으로 충분히 표현될 수 있는 것은 최대한 Markdown적으로 표현 + + 4. **이미지 판단 (매우 엄격하게)**: + - 진짜 시각적 요소만 이미지로: 사진, 복잡한 다이어그램, 차트, 그래프 + - **이미지로 처리할 것:** + - 사진 (인물, 건물, 풍경 등) + - 복잡한 다이어그램, 순서도 + - 차트, 그래프 + - **UI/웹사이트 스크린샷** (위치 안내 목적) + - 지도, 위치 안내도 + - **절대 이미지로 하지 말 것:** + - 제목 페이지 (첫 페이지) + - 목차 페이지 + - 감사/마무리 페이지 (마지막 페이지) + - 단순 텍스트와 로고만 있는 페이지 + - 텍스트 위주의 리스트나 메뉴 구조 (UI 스크린샷 제외) + - 표 + - 단순 슬라이드 배경이나 꾸밈용 그래픽 + - 표지나 장식 + - **이미지 참조 방법:** + - 이미지가 필요한 경우 현재 페이지 전체를 참조: \`![설명](IMAGE:page-{current_page})\` + - 예: \`![사건처리 절차도](IMAGE:page-6)\` + - **기본은 텍스트 변환 - 이미지는 정말 필요할 때만** + - 단순 표지나 장식적인 내용, 감사합니다 등의 정보가 아닌 내용은 철저히 배제 + + 5. **포맷팅**: + - 적절한 제목 계층 (##, ###, ####) + - 이전 페이지의 제목 레벨 고려 + - **프레젠테이션 슬라이드**: 모든 슬라이드에 반복되는 텍스트(슬라이드 덱 제목이나 footer)는 첫 페이지에만 사용 + - 각 슬라이드의 실제 제목/내용만 추출 + - 리스트, 강조, 간격 보존 + - 문서 구조와 흐름 유지 + + **페이지 위치 고려:** + - 첫 페이지 (1/{total_pages}): 제목/표지는 텍스트로 + - 마지막 페이지 ({total_pages}/{total_pages}): 감사/연락처는 텍스트로 + - 중간 페이지: 복잡한 다이어그램만 이미지로 + + **중요:** + - 이 작업은 단순 PDF → Markdown 변환입니다 + - 청킹이나 섹션 분할은 하지 마세요 + - 각 페이지를 순서대로 Markdown으로 변환만 하세요 + - 전체 문서의 일관된 흐름과 구조를 유지하세요 + + **하지 말아야 할 것:** + - 출력을 \`\`\`markdown\`\`\` 블록으로 감싸기 + - 텍스트로 변환 가능한 것을 이미지로 표시 + - 내용 손실 + - 이전 컨텍스트 무시 + - 제목/목차/감사 페이지를 이미지로 처리 + - 감사합니다. 와 같이 의미론적으로 필요 없는 내용을 포함 + - \`\` 태그 사용하지 말 것 (청킹은 나중에 별도로 처리됨) + + 이제 이 페이지를 Markdown으로 변환하세요: + +`; From 2066c6d4e244f60bc0c20cae147ef33247e340ea Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 08/40] feat(pdf-processor): ingest PDFs asynchronously with GCS and DB worker Add text-only Pass1/2 pipeline, GCS storage, DB-backed queue with concurrency 1, and atomic processing ownership for cancel/reprocess. Co-authored-by: Cursor --- bun.lock | 154 ++++++++- package.json | 6 +- src/app.module.ts | 2 + src/pdf-processor/documents.repository.ts | 308 ++++++++++++++++++ src/pdf-processor/gcs-storage.service.spec.ts | 37 +++ src/pdf-processor/gcs-storage.service.ts | 206 ++++++++++++ src/pdf-processor/mojibake.spec.ts | 23 ++ src/pdf-processor/mojibake.ts | 62 ++++ src/pdf-processor/pdf-chunk-parser.spec.ts | 50 +++ src/pdf-processor/pdf-chunk-parser.ts | 82 +++++ src/pdf-processor/pdf-pipeline.service.ts | 182 +++++++++++ src/pdf-processor/pdf-processor.module.ts | 21 ++ .../pdf-processor.worker.spec.ts | 103 ++++++ src/pdf-processor/pdf-processor.worker.ts | 203 ++++++++++++ src/pdf-processor/pdf-text.service.ts | 57 ++++ src/scripts/smoke-pdf-processor.ts | 114 +++++++ 16 files changed, 1601 insertions(+), 9 deletions(-) create mode 100644 src/pdf-processor/documents.repository.ts create mode 100644 src/pdf-processor/gcs-storage.service.spec.ts create mode 100644 src/pdf-processor/gcs-storage.service.ts create mode 100644 src/pdf-processor/mojibake.spec.ts create mode 100644 src/pdf-processor/mojibake.ts create mode 100644 src/pdf-processor/pdf-chunk-parser.spec.ts create mode 100644 src/pdf-processor/pdf-chunk-parser.ts create mode 100644 src/pdf-processor/pdf-pipeline.service.ts create mode 100644 src/pdf-processor/pdf-processor.module.ts create mode 100644 src/pdf-processor/pdf-processor.worker.spec.ts create mode 100644 src/pdf-processor/pdf-processor.worker.ts create mode 100644 src/pdf-processor/pdf-text.service.ts create mode 100644 src/scripts/smoke-pdf-processor.ts diff --git a/bun.lock b/bun.lock index fe49009..2d675eb 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@fastify/multipart": "^9.0.1", "@fastify/static": "^9.0.0", + "@google-cloud/storage": "^7.21.0", "@modelcontextprotocol/sdk": "^1.25.2", "@nestjs/axios": "^4.0.1", "@nestjs/common": "^11.0.1", @@ -22,10 +23,12 @@ "drizzle-kit": "^0.31.8", "drizzle-orm": "^0.45.1", "form-data": "^4.0.0", + "iconv-lite": "^0.7.3", "nanoid": "^5.0.9", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "patch-package": "^8.0.0", + "pdfjs-dist": "^6.2.108", "pg": "^8.16.3", "postgres": "^3.4.8", "reflect-metadata": "^0.2.2", @@ -37,6 +40,7 @@ "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", + "@types/iconv-lite": "^0.0.1", "@types/jest": "^30.0.0", "@types/node": "^22.10.7", "@types/passport-jwt": "^4.0.1", @@ -252,6 +256,14 @@ "@fastify/static": ["@fastify/static@9.0.0", "", { "dependencies": { "@fastify/accept-negotiator": "^2.0.0", "@fastify/send": "^4.0.0", "content-disposition": "^1.0.1", "fastify-plugin": "^5.0.0", "fastq": "^1.17.1", "glob": "^13.0.0" } }, "sha512-r64H8Woe/vfilg5RTy7lwWlE8ZZcTrc3kebYFMEUBrMqlydhQyoiExQXdYAy2REVpST/G35+stAM8WYp1WGmMA=="], + "@google-cloud/paginator": ["@google-cloud/paginator@5.0.2", "", { "dependencies": { "arrify": "^2.0.0", "extend": "^3.0.2" } }, "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg=="], + + "@google-cloud/projectify": ["@google-cloud/projectify@4.0.0", "", {}, "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA=="], + + "@google-cloud/promisify": ["@google-cloud/promisify@4.0.0", "", {}, "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g=="], + + "@google-cloud/storage": ["@google-cloud/storage@7.21.0", "", { "dependencies": { "@google-cloud/paginator": "^5.0.0", "@google-cloud/projectify": "^4.0.0", "@google-cloud/promisify": "<4.1.0", "abort-controller": "^3.0.0", "async-retry": "^1.3.3", "duplexify": "^4.1.3", "fast-xml-parser": "^5.3.4", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "html-entities": "^2.5.2", "mime": "^3.0.0", "p-limit": "^3.0.1", "retry-request": "^7.0.0", "teeny-request": "^9.0.0" } }, "sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA=="], + "@hono/node-server": ["@hono/node-server@1.19.9", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="], "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], @@ -360,6 +372,30 @@ "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.25.2", "", { "dependencies": { "@hono/node-server": "^1.19.7", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.0.1", "express-rate-limit": "^7.5.0", "jose": "^6.1.1", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.0" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-LZFeo4F9M5qOhC/Uc1aQSrBHxMrvxett+9KLHt7OhcExtoiRN9DKgbZffMP/nxjutWDQpfMDfP3nkHI4X9ijww=="], + "@napi-rs/canvas": ["@napi-rs/canvas@1.0.3", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.3", "@napi-rs/canvas-darwin-arm64": "1.0.3", "@napi-rs/canvas-darwin-x64": "1.0.3", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.3", "@napi-rs/canvas-linux-arm64-gnu": "1.0.3", "@napi-rs/canvas-linux-arm64-musl": "1.0.3", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.3", "@napi-rs/canvas-linux-x64-gnu": "1.0.3", "@napi-rs/canvas-linux-x64-musl": "1.0.3", "@napi-rs/canvas-win32-arm64-msvc": "1.0.3", "@napi-rs/canvas-win32-x64-msvc": "1.0.3" } }, "sha512-OlI657a5XXvKGFX7kNeIzJ8rO7IXt87Mqu2H8rXE46viAuOfum/JA7ysX7+eBhxNKznT+RCZh418mndlcFX3+w=="], + + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.3", "", { "os": "android", "cpu": "arm64" }, "sha512-7kSCdUhoXiO+AaIMXdBGdtp6EctZNkmF62Rea/BmVQlwKaM3bBhOzyGUzxyxz9dv5vdBfpyAaxhSRSJF4kqK4A=="], + + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ds14V1BPagLszQyaDTeggny5fNeTCqsUQ5QhFj9VDxSEfzrVxXtdbR0LoFyKa0Siaaw8KvqSk4t7k/WoZJwvbg=="], + + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-qof3LRAAycmkV2I1izZo9RoSHF8kCQr5O05sFwv0jK8rSdYV6KHVwimo6Qb7RxZj40WHKbLHm5JDaUF0o5XUAA=="], + + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-FU2kKZLmolHA9+KcUA+l1+xH3WTLUUTQDU/kLv9SEUr2TrRPu94aytOeizFJDHPs/QBcw4QL1mCQhetQXYBbag=="], + + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-GVSjntxKeA+/y/ZKf1F+cmUw1WeIkE5aMRPqnZUlBTBvBcrvgWccJAWuYCKPX4QJQwZILIIwhgdAbl51yj6fpA=="], + + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-J51oK/axyZ13kxycumSMfLiDZMdWdOVvqDFI28BpuViZHE3A0bQfr8B5vg8YnPEnqLD3BSn1hkdlh2buspEcNQ=="], + + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.3", "", { "os": "linux", "cpu": "none" }, "sha512-CtQgQjoVTX67jS9XuCTtJ40Sl7wRLMguoFnnGnfDmCWf7kzKFZVwj5ynqUOIGKFMSB61ZCuQlwPvVNxYTTseaw=="], + + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-jtfzAHFp+FRaR7zGT4jyCe6wUgAG/dVb5A4Apd8FY9jKarntDfUAlJXscugiH7ZF5kKnu7/lHFk9LaDPcrGEVQ=="], + + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-xTzaUCKUHTY4bCGadeeRZggbRVbGUT1petg7Z8r9AJR2+D9Bqu6nQAgqBGC6D47tA70LjaaaLTrJ7wNY1T74dg=="], + + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-ktVLuBkI6QVOm5BwO/WbdGwxgeetAMJa7TTmR8qBarXF0OU2NKjvjUtPJAl2y8t+zBRczJl/1VOl9gua6WcK2g=="], + + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-SGhlQ8bDjL1Cz2KnsKMasr/5sTcwG/SZkB6WCJxLsmSm/3aS2C+3p39bA7iZ2/94+NkVDySZfbiGoaSZSFHYxA=="], + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@0.2.12", "", { "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ=="], "@nestjs/axios": ["@nestjs/axios@4.0.1", "", { "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", "axios": "^1.3.1", "rxjs": "^7.0.0" } }, "sha512-68pFJgu+/AZbWkGu65Z3r55bTsCPlgyKaV4BSG8yUAD72q1PPuyVRgUwFv6BxdnibTUHlyxm06FmYWNC+bjN7A=="], @@ -388,6 +424,8 @@ "@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="], + "@nodable/entities": ["@nodable/entities@3.0.0", "", {}, "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw=="], + "@nuxt/opencollective": ["@nuxt/opencollective@0.4.1", "", { "dependencies": { "consola": "^3.2.3" }, "bin": { "opencollective": "bin/opencollective.js" } }, "sha512-GXD3wy50qYbxCJ652bDrDzgMr3NFEkIS374+IgFQKkCvk9yiYcLvX2XDYr7UyQxf4wK0e+yqDYRubZ0DtOxnmQ=="], "@paralleldrive/cuid2": ["@paralleldrive/cuid2@2.3.1", "", { "dependencies": { "@noble/hashes": "^1.1.5" } }, "sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw=="], @@ -410,6 +448,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@tootallnate/once": ["@tootallnate/once@2.0.1", "", {}, "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ=="], + "@tsconfig/node10": ["@tsconfig/node10@1.0.12", "", {}, "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ=="], "@tsconfig/node12": ["@tsconfig/node12@1.0.11", "", {}, "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag=="], @@ -430,6 +470,8 @@ "@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="], + "@types/caseless": ["@types/caseless@0.12.5", "", {}, "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="], + "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="], "@types/cookiejar": ["@types/cookiejar@2.1.5", "", {}, "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q=="], @@ -446,6 +488,8 @@ "@types/http-errors": ["@types/http-errors@2.0.5", "", {}, "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg=="], + "@types/iconv-lite": ["@types/iconv-lite@0.0.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-SsRBQxGw7/2/NxYJfBdiUx5a7Ms/voaUhOO9u2y9FTeTNBO1PXohzE4i3JfD8q2Te42HLTn5pyZtDf8j1bPKgQ=="], + "@types/istanbul-lib-coverage": ["@types/istanbul-lib-coverage@2.0.6", "", {}, "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w=="], "@types/istanbul-lib-report": ["@types/istanbul-lib-report@3.0.3", "", { "dependencies": { "@types/istanbul-lib-coverage": "*" } }, "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA=="], @@ -474,6 +518,8 @@ "@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="], + "@types/request": ["@types/request@2.48.13", "", { "dependencies": { "@types/caseless": "*", "@types/node": "*", "@types/tough-cookie": "*", "form-data": "^2.5.5" } }, "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg=="], + "@types/send": ["@types/send@1.2.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ=="], "@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="], @@ -484,6 +530,8 @@ "@types/supertest": ["@types/supertest@6.0.3", "", { "dependencies": { "@types/methods": "^1.1.4", "@types/superagent": "^8.1.0" } }, "sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w=="], + "@types/tough-cookie": ["@types/tough-cookie@4.0.5", "", {}, "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA=="], + "@types/validator": ["@types/validator@13.15.10", "", {}, "sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA=="], "@types/yargs": ["@types/yargs@17.0.35", "", { "dependencies": { "@types/yargs-parser": "*" } }, "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg=="], @@ -586,6 +634,8 @@ "@yarnpkg/lockfile": ["@yarnpkg/lockfile@1.1.0", "", {}, "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="], + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], + "abstract-logging": ["abstract-logging@2.0.1", "", {}, "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], @@ -598,6 +648,8 @@ "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], + "ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], @@ -616,14 +668,20 @@ "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "array-timsort": ["array-timsort@1.0.3", "", {}, "sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ=="], + "arrify": ["arrify@2.0.1", "", {}, "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug=="], + "asap": ["asap@2.0.6", "", {}, "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="], + "async-retry": ["async-retry@1.3.3", "", { "dependencies": { "retry": "0.13.1" } }, "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw=="], + "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], @@ -648,6 +706,8 @@ "baseline-browser-mapping": ["baseline-browser-mapping@2.9.14", "", { "bin": { "baseline-browser-mapping": "dist/cli.js" } }, "sha512-B0xUquLkiGLgHhpPBqvl7GWegWBUNuujQ6kXd/r1U38ElPT6Ok8KZ8e+FpUGEc2ZoRQUzq/aUnaKFc/svWUGSg=="], + "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="], + "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], @@ -788,6 +848,8 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + "duplexify": ["duplexify@4.1.3", "", { "dependencies": { "end-of-stream": "^1.4.1", "inherits": "^2.0.3", "readable-stream": "^3.1.1", "stream-shift": "^1.0.2" } }, "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA=="], + "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], @@ -802,6 +864,8 @@ "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], "error-ex": ["error-ex@1.3.4", "", { "dependencies": { "is-arrayish": "^0.2.1" } }, "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ=="], @@ -850,6 +914,8 @@ "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], + "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], @@ -866,6 +932,8 @@ "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + "fast-decode-uri-component": ["fast-decode-uri-component@1.0.1", "", {}, "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg=="], "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], @@ -884,6 +952,10 @@ "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], + "fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="], + + "fast-xml-parser": ["fast-xml-parser@5.10.1", "", { "dependencies": { "@nodable/entities": "^3.0.0", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.1", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw=="], + "fastify": ["fastify@5.6.2", "", { "dependencies": { "@fastify/ajv-compiler": "^4.0.0", "@fastify/error": "^4.0.0", "@fastify/fast-json-stringify-compiler": "^5.0.0", "@fastify/proxy-addr": "^5.0.0", "abstract-logging": "^2.0.1", "avvio": "^9.0.0", "fast-json-stringify": "^6.0.0", "find-my-way": "^9.0.0", "light-my-request": "^6.0.0", "pino": "^10.1.0", "process-warning": "^5.0.0", "rfdc": "^1.3.1", "secure-json-parse": "^4.0.0", "semver": "^7.6.0", "toad-cache": "^3.7.0" } }, "sha512-dPugdGnsvYkBlENLhCgX8yhyGCsCPrpA8lFWbTNU428l+YOnLgYHR69hzV8HWPC79n536EqzqQtvhtdaCE0dKg=="], "fastify-plugin": ["fastify-plugin@5.1.0", "", {}, "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw=="], @@ -936,6 +1008,10 @@ "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], + "gaxios": ["gaxios@6.7.1", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="], + + "gcp-metadata": ["gcp-metadata@6.1.1", "", { "dependencies": { "gaxios": "^6.1.1", "google-logging-utils": "^0.0.2", "json-bigint": "^1.0.0" } }, "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A=="], + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], @@ -958,10 +1034,16 @@ "globals": ["globals@16.5.0", "", {}, "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ=="], + "google-auth-library": ["google-auth-library@9.15.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^6.1.1", "gcp-metadata": "^6.1.0", "gtoken": "^7.0.0", "jws": "^4.0.0" } }, "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng=="], + + "google-logging-utils": ["google-logging-utils@0.0.2", "", {}, "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ=="], + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "gtoken": ["gtoken@7.1.0", "", { "dependencies": { "gaxios": "^6.0.0", "jws": "^4.0.0" } }, "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw=="], + "handlebars": ["handlebars@4.7.8", "", { "dependencies": { "minimist": "^1.2.5", "neo-async": "^2.6.2", "source-map": "^0.6.1", "wordwrap": "^1.0.0" }, "optionalDependencies": { "uglify-js": "^3.1.4" }, "bin": { "handlebars": "bin/handlebars" } }, "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ=="], "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], @@ -976,13 +1058,19 @@ "hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="], + "html-entities": ["html-entities@2.6.0", "", {}, "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ=="], + "html-escaper": ["html-escaper@2.0.2", "", {}, "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "http-proxy-agent": ["http-proxy-agent@5.0.0", "", { "dependencies": { "@tootallnate/once": "2", "agent-base": "6", "debug": "4" } }, "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w=="], + + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + "human-signals": ["human-signals@2.1.0", "", {}, "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw=="], - "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], @@ -1022,6 +1110,8 @@ "is-unicode-supported": ["is-unicode-supported@0.1.0", "", {}, "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw=="], + "is-unsafe": ["is-unsafe@2.0.0", "", {}, "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA=="], + "is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="], "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], @@ -1100,6 +1190,8 @@ "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], + "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], + "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], "json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="], @@ -1124,9 +1216,9 @@ "jsonwebtoken": ["jsonwebtoken@9.0.2", "", { "dependencies": { "jws": "^3.2.2", "lodash.includes": "^4.3.0", "lodash.isboolean": "^3.0.3", "lodash.isinteger": "^4.0.4", "lodash.isnumber": "^3.0.3", "lodash.isplainobject": "^4.0.6", "lodash.isstring": "^4.0.1", "lodash.once": "^4.0.0", "ms": "^2.1.1", "semver": "^7.5.4" } }, "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ=="], - "jwa": ["jwa@1.4.2", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw=="], + "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], - "jws": ["jws@3.2.3", "", { "dependencies": { "jwa": "^1.4.2", "safe-buffer": "^5.0.1" } }, "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g=="], + "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], @@ -1194,7 +1286,7 @@ "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="], + "mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="], "mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], @@ -1226,6 +1318,8 @@ "node-emoji": ["node-emoji@1.11.0", "", { "dependencies": { "lodash": "^4.17.21" } }, "sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A=="], + "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], + "node-int64": ["node-int64@0.4.0", "", {}, "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="], "node-releases": ["node-releases@2.0.27", "", {}, "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA=="], @@ -1278,6 +1372,8 @@ "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], @@ -1290,6 +1386,8 @@ "pause": ["pause@0.0.1", "", {}, "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="], + "pdfjs-dist": ["pdfjs-dist@6.2.108", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.0" } }, "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ=="], + "pg": ["pg@8.16.3", "", { "dependencies": { "pg-connection-string": "^2.9.1", "pg-pool": "^3.10.1", "pg-protocol": "^1.10.3", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.2.7" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw=="], "pg-cloudflare": ["pg-cloudflare@1.2.7", "", {}, "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg=="], @@ -1386,6 +1484,10 @@ "ret": ["ret@0.5.0", "", {}, "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw=="], + "retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], + + "retry-request": ["retry-request@7.0.2", "", { "dependencies": { "@types/request": "^2.48.8", "extend": "^3.0.2", "teeny-request": "^9.0.0" } }, "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w=="], + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], "rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="], @@ -1450,6 +1552,10 @@ "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], + "stream-events": ["stream-events@1.0.5", "", { "dependencies": { "stubs": "^3.0.0" } }, "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg=="], + + "stream-shift": ["stream-shift@1.0.3", "", {}, "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ=="], + "string-length": ["string-length@4.0.2", "", { "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" } }, "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ=="], "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], @@ -1468,8 +1574,12 @@ "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], + "strnum": ["strnum@2.4.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg=="], + "strtok3": ["strtok3@10.3.4", "", { "dependencies": { "@tokenizer/token": "^0.3.0" } }, "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg=="], + "stubs": ["stubs@3.0.0", "", {}, "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw=="], + "superagent": ["superagent@10.3.0", "", { "dependencies": { "component-emitter": "^1.3.1", "cookiejar": "^2.1.4", "debug": "^4.3.7", "fast-safe-stringify": "^2.1.1", "form-data": "^4.0.5", "formidable": "^3.5.4", "methods": "^1.1.2", "mime": "2.6.0", "qs": "^6.14.1" } }, "sha512-B+4Ik7ROgVKrQsXTV0Jwp2u+PXYLSlqtDAhYnkkD+zn3yg8s/zjA2MeGayPoY/KICrbitwneDHrjSotxKL+0XQ=="], "supertest": ["supertest@7.2.2", "", { "dependencies": { "cookie-signature": "^1.2.2", "methods": "^1.1.2", "superagent": "^10.3.0" } }, "sha512-oK8WG9diS3DlhdUkcFn4tkNIiIbBx9lI2ClF8K+b2/m8Eyv47LSawxUzZQSNKUrVb2KsqeTDCcjAAVPYaSLVTA=="], @@ -1484,6 +1594,8 @@ "tapable": ["tapable@2.3.0", "", {}, "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg=="], + "teeny-request": ["teeny-request@9.0.0", "", { "dependencies": { "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "node-fetch": "^2.6.9", "stream-events": "^1.0.5", "uuid": "^9.0.0" } }, "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g=="], + "terser": ["terser@5.44.1", "", { "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" } }, "sha512-t/R3R/n0MSwnnazuPpPNVO60LX0SKL45pyl9YlvxIdkH0Of7D5qM2EVe+yASRIlY5pZ73nclYJfNANGWPwFDZw=="], "terser-webpack-plugin": ["terser-webpack-plugin@5.3.16", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "peerDependencies": { "webpack": "^5.1.0" } }, "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q=="], @@ -1506,6 +1618,8 @@ "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], + "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], + "ts-api-utils": ["ts-api-utils@2.4.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA=="], "ts-jest": ["ts-jest@29.4.6", "", { "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", "handlebars": "^4.7.8", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", "semver": "^7.7.3", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, "peerDependencies": { "@babel/core": ">=7.0.0-beta.0 <8", "@jest/transform": "^29.0.0 || ^30.0.0", "@jest/types": "^29.0.0 || ^30.0.0", "babel-jest": "^29.0.0 || ^30.0.0", "jest": "^29.0.0 || ^30.0.0", "jest-util": "^29.0.0 || ^30.0.0", "typescript": ">=4.3 <6" }, "optionalPeers": ["@babel/core", "@jest/transform", "@jest/types", "babel-jest", "jest-util"], "bin": { "ts-jest": "cli.js" } }, "sha512-fSpWtOO/1AjSNQguk43hb/JCo16oJDnMJf3CdEGNkqsEX3t0KX96xvyX1D7PfLCpVoKu4MfVrqUkFyblYoY4lA=="], @@ -1554,6 +1668,8 @@ "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], + "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + "v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="], "v8-to-istanbul": ["v8-to-istanbul@9.3.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" } }, "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA=="], @@ -1568,12 +1684,16 @@ "wcwidth": ["wcwidth@1.0.1", "", { "dependencies": { "defaults": "^1.0.3" } }, "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg=="], + "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + "webpack": ["webpack@5.103.0", "", { "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", "acorn": "^8.15.0", "acorn-import-phases": "^1.0.3", "browserslist": "^4.26.3", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.17.3", "es-module-lexer": "^1.2.1", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", "json-parse-even-better-errors": "^2.3.1", "loader-runner": "^4.3.1", "mime-types": "^2.1.27", "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", "terser-webpack-plugin": "^5.3.11", "watchpack": "^2.4.4", "webpack-sources": "^3.3.3" }, "bin": { "webpack": "bin/webpack.js" } }, "sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw=="], "webpack-node-externals": ["webpack-node-externals@3.0.0", "", {}, "sha512-LnL6Z3GGDPht/AigwRh2dvL9PQPFQ8skEpVrWZXLWBYmqcaojHNN0onvHzie6rq7EWKrrBfPYqNEzTJgiwEQDQ=="], "webpack-sources": ["webpack-sources@3.3.3", "", {}, "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg=="], + "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], @@ -1588,6 +1708,8 @@ "write-file-atomic": ["write-file-atomic@5.0.1", "", { "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" } }, "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw=="], + "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], + "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], @@ -1638,7 +1760,7 @@ "@fastify/proxy-addr/ipaddr.js": ["ipaddr.js@2.3.0", "", {}, "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg=="], - "@fastify/send/mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="], + "@inquirer/external-editor/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], @@ -1676,6 +1798,8 @@ "@nestjs/schematics/@angular-devkit/schematics": ["@angular-devkit/schematics@19.2.17", "", { "dependencies": { "@angular-devkit/core": "19.2.17", "jsonc-parser": "3.3.1", "magic-string": "0.30.17", "ora": "5.4.1", "rxjs": "7.8.1" } }, "sha512-ADfbaBsrG8mBF6Mfs+crKA/2ykB8AJI50Cv9tKmZfwcUcyAdmTr+vVvhsBCfvUAEokigSsgqgpYxfkJVxhJYeg=="], + "@types/request/form-data": ["form-data@2.5.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" } }, "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA=="], + "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], @@ -1690,6 +1814,8 @@ "babel-jest/slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], + "body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "cliui/wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], @@ -1702,6 +1828,8 @@ "glob/minimatch": ["minimatch@10.1.1", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ=="], + "http-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + "jest-circus/slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], "jest-config/ci-info": ["ci-info@4.3.1", "", {}, "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA=="], @@ -1732,6 +1860,8 @@ "jest-worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], + "jsonwebtoken/jws": ["jws@3.2.3", "", { "dependencies": { "jwa": "^1.4.2", "safe-buffer": "^5.0.1" } }, "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g=="], + "light-my-request/cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], "light-my-request/process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="], @@ -1742,6 +1872,8 @@ "pkg-dir/find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], + "raw-body/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "resolve-cwd/resolve-from": ["resolve-from@5.0.0", "", {}, "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw=="], "restore-cursor/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], @@ -1750,6 +1882,10 @@ "stack-utils/escape-string-regexp": ["escape-string-regexp@2.0.0", "", {}, "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w=="], + "superagent/mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="], + + "teeny-request/https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + "terser/commander": ["commander@2.20.3", "", {}, "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="], "terser-webpack-plugin/schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], @@ -1844,6 +1980,8 @@ "@nestjs/schematics/@angular-devkit/schematics/rxjs": ["rxjs@7.8.1", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg=="], + "@types/request/form-data/hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], "accepts/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], @@ -1868,12 +2006,14 @@ "jest-runtime/glob/path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], - "passport-jwt/jsonwebtoken/jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + "jsonwebtoken/jws/jwa": ["jwa@1.4.2", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw=="], "pkg-dir/find-up/locate-path": ["locate-path@5.0.0", "", { "dependencies": { "p-locate": "^4.1.0" } }, "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g=="], "send/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], + "teeny-request/https-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + "terser-webpack-plugin/schema-utils/ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], "terser-webpack-plugin/schema-utils/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], @@ -1906,8 +2046,6 @@ "jest-runtime/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], - "passport-jwt/jsonwebtoken/jws/jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], - "pkg-dir/find-up/locate-path/p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], "terser-webpack-plugin/schema-utils/ajv/json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], diff --git a/package.json b/package.json index f2b5a54..a33f8a9 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "dependencies": { "@fastify/multipart": "^9.0.1", "@fastify/static": "^9.0.0", + "@google-cloud/storage": "^7.21.0", "@modelcontextprotocol/sdk": "^1.25.2", "@nestjs/axios": "^4.0.1", "@nestjs/common": "^11.0.1", @@ -50,11 +51,13 @@ "drizzle-kit": "^0.31.8", "drizzle-orm": "^0.45.1", "form-data": "^4.0.0", + "iconv-lite": "^0.7.3", "nanoid": "^5.0.9", "passport": "^0.7.0", "passport-jwt": "^4.0.1", - "pg": "^8.16.3", "patch-package": "^8.0.0", + "pdfjs-dist": "^6.2.108", + "pg": "^8.16.3", "postgres": "^3.4.8", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1" @@ -65,6 +68,7 @@ "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", + "@types/iconv-lite": "^0.0.1", "@types/jest": "^30.0.0", "@types/node": "^22.10.7", "@types/passport-jwt": "^4.0.1", diff --git a/src/app.module.ts b/src/app.module.ts index 13c1ab0..daf1fc2 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -10,6 +10,7 @@ import { validate } from './config/env.validation'; import { McpModule } from './mcp/mcp.module'; import { ChatModule } from './chat/chat.module'; import { UploadModule } from './upload/upload.module'; +import { PdfProcessorModule } from './pdf-processor/pdf-processor.module'; @Module({ imports: [ @@ -24,6 +25,7 @@ import { UploadModule } from './upload/upload.module'; AdminModule, McpModule, ChatModule, + PdfProcessorModule, UploadModule, ], controllers: [AppController], diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts new file mode 100644 index 0000000..b1be65d --- /dev/null +++ b/src/pdf-processor/documents.repository.ts @@ -0,0 +1,308 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { + inArray, + notInArray, + sql, + eq, + and, + desc, + asc, + lt, +} from 'drizzle-orm'; +import { DB_CONNECTION, documents, documentChunks } from '../db'; +import type { Database, Document, DocumentChunk } from '../db'; + +export type CreateDocumentInput = { + title: string; + resourceName: string; + gcsPdfPath: string; + uploadedByIdpUuid: string; +}; + +export type ReplaceChunksInput = { + path: string; + description: string; + content: string; + sortOrder: number; +}; + +@Injectable() +export class DocumentsRepository { + constructor(@Inject(DB_CONNECTION) private readonly db: Database) {} + + /** + * Atomically reserve an active resource name before uploading to GCS. + * The worker only claims `queued`, so it cannot observe a partial upload. + */ + async createUploading(input: CreateDocumentInput): Promise { + const [row] = await this.db + .insert(documents) + .values({ + title: input.title, + resourceName: input.resourceName, + gcsPdfPath: input.gcsPdfPath, + uploadedByIdpUuid: input.uploadedByIdpUuid, + status: 'uploading', + isActive: true, + }) + .returning(); + if (!row) throw new Error('Failed to insert document'); + return row; + } + + async markQueuedAfterUpload(id: string): Promise { + const [row] = await this.db + .update(documents) + .set({ + status: 'queued', + errorMessage: null, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, id), + eq(documents.status, 'uploading'), + eq(documents.isActive, true), + ), + ) + .returning(); + return row ?? null; + } + + async hardDelete(id: string): Promise { + await this.db.delete(documents).where(eq(documents.id, id)); + } + + async findById(id: string): Promise { + const [row] = await this.db + .select() + .from(documents) + .where(eq(documents.id, id)) + .limit(1); + return row ?? null; + } + + async findActiveByResourceName( + resourceName: string, + ): Promise { + const [row] = await this.db + .select() + .from(documents) + .where( + and( + eq(documents.resourceName, resourceName), + eq(documents.isActive, true), + ), + ) + .limit(1); + return row ?? null; + } + + async listByUploader( + idpUuid: string, + options: { limit: number; offset: number }, + ): Promise { + return this.db + .select() + .from(documents) + .where( + and( + eq(documents.uploadedByIdpUuid, idpUuid), + eq(documents.isActive, true), + ), + ) + .orderBy(desc(documents.createdAt)) + .limit(options.limit) + .offset(options.offset); + } + + /** + * Claim up to `limit` queued documents using SKIP LOCKED. + */ + async claimQueued(limit: number): Promise { + if (limit < 1) return []; + + return this.db.transaction(async (tx) => { + const selected = await tx.execute( + sql`SELECT id FROM documents + WHERE status = 'queued' AND is_active = true + ORDER BY created_at ASC + LIMIT ${limit} + FOR UPDATE SKIP LOCKED`, + ); + const rawRows = Array.isArray(selected) + ? selected + : ((selected as { rows?: { id: string }[] }).rows ?? []); + const ids = rawRows + .map((r) => (r as { id: string }).id) + .filter((id): id is string => typeof id === 'string'); + if (ids.length === 0) return []; + + return tx + .update(documents) + .set({ + status: 'processing', + processingToken: sql`gen_random_uuid()`, + updatedAt: new Date(), + errorMessage: null, + }) + .where(inArray(documents.id, ids)) + .returning(); + }); + } + + async requeueStaleProcessing( + staleBefore: Date, + excludedDocumentIds: string[] = [], + ): Promise { + const conditions = [ + eq(documents.status, 'processing'), + eq(documents.isActive, true), + lt(documents.updatedAt, staleBefore), + ]; + if (excludedDocumentIds.length > 0) { + conditions.push(notInArray(documents.id, excludedDocumentIds)); + } + + const result = await this.db + .update(documents) + .set({ + status: 'queued', + processingToken: null, + updatedAt: new Date(), + errorMessage: 'Requeued after stuck processing timeout', + }) + .where(and(...conditions)) + .returning({ id: documents.id }); + return result.length; + } + + /** + * Persist chunks and mark ready only if this exact processing attempt still + * owns the document. A delete/reprocess/stale recovery clears the token. + */ + async completeProcessing( + documentId: string, + processingToken: string, + summary: string, + chunks: ReplaceChunksInput[], + ): Promise { + return this.db.transaction(async (tx) => { + const [completed] = await tx + .update(documents) + .set({ + status: 'ready', + summary, + errorMessage: null, + processingToken: null, + processedAt: new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, documentId), + eq(documents.status, 'processing'), + eq(documents.processingToken, processingToken), + eq(documents.isActive, true), + ), + ) + .returning({ id: documents.id }); + if (!completed) return false; + + await tx + .delete(documentChunks) + .where(eq(documentChunks.documentId, documentId)); + if (chunks.length > 0) { + await tx.insert(documentChunks).values( + chunks.map((c) => ({ + documentId, + path: c.path, + description: c.description, + content: c.content, + sortOrder: c.sortOrder, + })), + ); + } + return true; + }); + } + + async markFailed( + id: string, + processingToken: string, + errorMessage: string, + ): Promise { + const result = await this.db + .update(documents) + .set({ + status: 'failed', + errorMessage: errorMessage.slice(0, 4000), + processingToken: null, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, id), + eq(documents.status, 'processing'), + eq(documents.processingToken, processingToken), + eq(documents.isActive, true), + ), + ) + .returning({ id: documents.id }); + return result.length > 0; + } + + /** + * Cancel the current attempt before deleting external artifacts. + */ + async cancelAndSoftDelete(id: string): Promise { + const [row] = await this.db + .update(documents) + .set({ + isActive: false, + processingToken: null, + updatedAt: new Date(), + }) + .where(and(eq(documents.id, id), eq(documents.isActive, true))) + .returning(); + return row ?? null; + } + + async enqueueReprocess(id: string): Promise { + return this.db.transaction(async (tx) => { + const [row] = await tx + .update(documents) + .set({ + status: 'queued', + errorMessage: null, + processingToken: null, + processedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, id), + eq(documents.isActive, true), + // An upload has no complete source in GCS yet. + sql`${documents.status} <> 'uploading'`, + ), + ) + .returning(); + if (!row) return null; + + await tx + .delete(documentChunks) + .where(eq(documentChunks.documentId, id)); + return row; + }); + } + + async listChunks(documentId: string): Promise { + return this.db + .select() + .from(documentChunks) + .where(eq(documentChunks.documentId, documentId)) + .orderBy(asc(documentChunks.sortOrder)); + } + +} diff --git a/src/pdf-processor/gcs-storage.service.spec.ts b/src/pdf-processor/gcs-storage.service.spec.ts new file mode 100644 index 0000000..3a29258 --- /dev/null +++ b/src/pdf-processor/gcs-storage.service.spec.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from '@jest/globals'; +import { decodeServiceAccountCredentials } from './gcs-storage.service'; + +function encode(value: unknown): string { + return Buffer.from(JSON.stringify(value), 'utf-8').toString('base64'); +} + +describe('decodeServiceAccountCredentials', () => { + it('decodes the service account fields required by Google Storage', () => { + expect( + decodeServiceAccountCredentials( + encode({ + type: 'service_account', + project_id: 'test-project', + client_email: 'storage@test-project.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', + }), + ), + ).toEqual({ + client_email: 'storage@test-project.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', + }); + }); + + it('rejects malformed JSON', () => { + const encoded = Buffer.from('not-json', 'utf-8').toString('base64'); + expect(() => decodeServiceAccountCredentials(encoded)).toThrow( + /base64-encoded service account JSON/, + ); + }); + + it('rejects credentials missing required fields', () => { + expect(() => + decodeServiceAccountCredentials(encode({ type: 'service_account' })), + ).toThrow(/client_email and private_key/); + }); +}); diff --git a/src/pdf-processor/gcs-storage.service.ts b/src/pdf-processor/gcs-storage.service.ts new file mode 100644 index 0000000..0e6d1bd --- /dev/null +++ b/src/pdf-processor/gcs-storage.service.ts @@ -0,0 +1,206 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Storage, Bucket, File } from '@google-cloud/storage'; + +type GcsServiceAccountCredentials = { + client_email: string; + private_key: string; +}; + +export type ResourceIndexEntry = { + description: string; + chunks: { path: string; description: string }[]; +}; + +export type ResourceIndex = Record; + +const RESOURCES_INDEX_PATH = '_resources.json'; + +export function decodeServiceAccountCredentials( + encodedKey: string, +): GcsServiceAccountCredentials { + let parsed: unknown; + try { + const json = Buffer.from(encodedKey, 'base64').toString('utf-8'); + parsed = JSON.parse(json) as unknown; + } catch { + throw new Error( + 'GCS_SERVICE_ACCOUNT_KEY_BASE64 must be a base64-encoded service account JSON', + ); + } + + if ( + typeof parsed !== 'object' || + parsed === null || + !('client_email' in parsed) || + typeof parsed.client_email !== 'string' || + !parsed.client_email || + !('private_key' in parsed) || + typeof parsed.private_key !== 'string' || + !parsed.private_key + ) { + throw new Error( + 'Decoded GCS service account JSON must contain client_email and private_key', + ); + } + + return { + client_email: parsed.client_email, + private_key: parsed.private_key, + }; +} + +@Injectable() +export class GcsStorageService { + private readonly logger = new Logger(GcsStorageService.name); + private readonly bucket: Bucket; + private readonly bucketName: string; + + constructor(private readonly configService: ConfigService) { + this.bucketName = this.configService.getOrThrow('GCS_BUCKET'); + const projectId = this.configService.getOrThrow('GCP_PROJECT_ID'); + const encodedKey = this.configService.get( + 'GCS_SERVICE_ACCOUNT_KEY_BASE64', + ); + const credentials = encodedKey + ? decodeServiceAccountCredentials(encodedKey) + : undefined; + const storage = new Storage({ projectId, credentials }); + this.bucket = storage.bucket(this.bucketName); + this.logger.log( + `GCS bucket ready: ${this.bucketName} (auth=${credentials ? 'service-account-env' : 'ADC'})`, + ); + } + + toGsPath(objectPath: string): string { + return `gs://${this.bucketName}/${objectPath}`; + } + + async uploadPdf(resourceName: string, pdfBytes: Buffer): Promise { + const objectPath = `${resourceName}.pdf`; + await this.bucket.file(objectPath).save(pdfBytes, { + contentType: 'application/pdf', + resumable: false, + }); + return this.toGsPath(objectPath); + } + + async downloadPdf(resourceName: string): Promise { + const objectPath = `${resourceName}.pdf`; + const [buf] = await this.bucket.file(objectPath).download(); + return buf; + } + + async uploadMarkdown(objectPath: string, content: string): Promise { + await this.bucket.file(objectPath).save(Buffer.from(content, 'utf-8'), { + contentType: 'text/markdown; charset=utf-8', + resumable: false, + }); + } + + /** + * Upload processed documents map (path → markdown string). + * Binary (image) entries are skipped in phase 1. + */ + async uploadDocuments( + documents: Record, + ): Promise { + for (const [path, content] of Object.entries(documents)) { + await this.uploadMarkdown(path, content); + this.logger.debug(`Uploaded: ${path}`); + } + } + + async updateResourceIndex( + resourceName: string, + metadata: ResourceIndexEntry, + ): Promise { + const index = await this.readResourceIndex(); + index[resourceName] = metadata; + await this.writeResourceIndex(index); + this.logger.log(`Updated ${RESOURCES_INDEX_PATH} for: ${resourceName}`); + } + + async removeResourceIndexEntry(resourceName: string): Promise { + const index = await this.readResourceIndex(); + if (!(resourceName in index)) return; + delete index[resourceName]; + await this.writeResourceIndex(index); + this.logger.log(`Removed ${RESOURCES_INDEX_PATH} entry: ${resourceName}`); + } + + /** + * Delete PDF, root md, and prefix folder for a resource. + */ + async deleteResourceArtifacts(resourceName: string): Promise { + const toDelete: File[] = [ + this.bucket.file(`${resourceName}.pdf`), + ...(await this.getProcessedArtifactFiles(resourceName)), + ]; + + await this.deleteFiles(toDelete); + await this.removeResourceIndexEntry(resourceName); + } + + /** + * Delete generated Markdown while preserving the source PDF for retry. + */ + async deleteProcessedArtifacts(resourceName: string): Promise { + const toDelete = await this.getProcessedArtifactFiles(resourceName); + await this.deleteFiles(toDelete); + await this.removeResourceIndexEntry(resourceName); + } + + private async getProcessedArtifactFiles( + resourceName: string, + ): Promise { + const files: File[] = [ + this.bucket.file(`${resourceName}.md`), + ]; + + const [prefixFiles] = await this.bucket.getFiles({ + prefix: `${resourceName}/`, + }); + files.push(...prefixFiles); + return files; + } + + private async deleteFiles(files: File[]): Promise { + await Promise.all( + files.map(async (file) => { + try { + await file.delete({ ignoreNotFound: true }); + } catch (error) { + this.logger.warn( + `Failed to delete ${file.name}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + }), + ); + } + + private async readResourceIndex(): Promise { + const file = this.bucket.file(RESOURCES_INDEX_PATH); + try { + const [exists] = await file.exists(); + if (!exists) return {}; + const [buf] = await file.download(); + return JSON.parse(buf.toString('utf-8')) as ResourceIndex; + } catch (error) { + this.logger.warn( + `Failed to read ${RESOURCES_INDEX_PATH}, starting empty: ${error instanceof Error ? error.message : String(error)}`, + ); + return {}; + } + } + + private async writeResourceIndex(index: ResourceIndex): Promise { + await this.bucket.file(RESOURCES_INDEX_PATH).save( + Buffer.from(JSON.stringify(index, null, 2), 'utf-8'), + { + contentType: 'application/json; charset=utf-8', + resumable: false, + }, + ); + } +} diff --git a/src/pdf-processor/mojibake.spec.ts b/src/pdf-processor/mojibake.spec.ts new file mode 100644 index 0000000..bcc99a8 --- /dev/null +++ b/src/pdf-processor/mojibake.spec.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from '@jest/globals'; +import { + isLikelyMojibake, + tryFixMojibake, + normalizeExtractedText, +} from './mojibake'; +import iconv from 'iconv-lite'; + +describe('mojibake', () => { + it('detects Korean UTF-8 misdecoded as latin1', () => { + const original = '학사 일정'; + const mojibake = iconv.decode(Buffer.from(original, 'utf8'), 'latin1'); + expect(isLikelyMojibake(mojibake)).toBe(true); + expect(tryFixMojibake(mojibake)).toBe(original); + expect(normalizeExtractedText(mojibake)).toBe(original); + }); + + it('leaves normal Korean text unchanged', () => { + const text = '정상적인 한글 텍스트입니다'; + expect(isLikelyMojibake(text)).toBe(false); + expect(normalizeExtractedText(text)).toBe(text); + }); +}); diff --git a/src/pdf-processor/mojibake.ts b/src/pdf-processor/mojibake.ts new file mode 100644 index 0000000..560282a --- /dev/null +++ b/src/pdf-processor/mojibake.ts @@ -0,0 +1,62 @@ +/** + * Mojibake recovery for Korean UTF-8 text incorrectly decoded as CP1252/Latin-1. + * Ported from ziggle-mcp processor/worker.py + */ +import iconv from 'iconv-lite'; + +const MOJIBAKE_INDICATORS = new Set('íìëêéèãâáàäåñóòôöùûüý'); + +/** + * Detect if text appears to be UTF-8 Korean mojibake (decoded as Latin-1). + */ +export function isLikelyMojibake(text: string): boolean { + if (!text || text.trim().length < 10) { + return false; + } + const trimmed = text.trim(); + let indicatorCount = 0; + for (const c of trimmed) { + if (MOJIBAKE_INDICATORS.has(c)) indicatorCount += 1; + } + return indicatorCount / trimmed.length > 0.08; +} + +/** + * Try to recover UTF-8 text that was incorrectly decoded as CP1252/Latin-1. + * Returns recovered text if Korean was found, otherwise null. + */ +export function tryFixMojibake(text: string): string | null { + if (!text || !text.trim()) { + return null; + } + + for (const encoding of ['win1252', 'latin1'] as const) { + try { + const bytes = iconv.encode(text, encoding); + const recovered = bytes.toString('utf8'); + let koreanCount = 0; + for (const c of recovered) { + const code = c.codePointAt(0) ?? 0; + if (code >= 0xac00 && code <= 0xd7a3) koreanCount += 1; + } + if (koreanCount > 0) { + return recovered; + } + } catch { + // try next encoding + } + } + return null; +} + +/** + * Apply mojibake fix when detected; otherwise return original text. + * If mojibake is detected but recovery fails, returns empty string (skip bad text). + */ +export function normalizeExtractedText(rawText: string): string { + if (!isLikelyMojibake(rawText)) { + return rawText; + } + const fixed = tryFixMojibake(rawText); + return fixed ?? ''; +} diff --git a/src/pdf-processor/pdf-chunk-parser.spec.ts b/src/pdf-processor/pdf-chunk-parser.spec.ts new file mode 100644 index 0000000..604af25 --- /dev/null +++ b/src/pdf-processor/pdf-chunk-parser.spec.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from '@jest/globals'; +import { parseChunksFromMarkdown, toResourceName } from './pdf-chunk-parser'; + +describe('parseChunksFromMarkdown', () => { + it('parses summary and document tags', () => { + const input = ` +문서 요약 + +# 제목 + + +## 섹션 A +내용 A + + + +## 섹션 B +내용 B + +`; + const { documents, metadata } = parseChunksFromMarkdown( + input, + '테스트.pdf', + ); + + expect(metadata.description).toBe('문서 요약'); + expect(metadata.chunks).toEqual([ + { path: '테스트/섹션-a', description: '설명 A' }, + { path: '테스트/섹션-b', description: '설명 B' }, + ]); + expect(documents['테스트/섹션-a.md']).toContain('내용 A'); + expect(documents['테스트/섹션-b.md']).toContain('내용 B'); + expect(documents['테스트.md']).toContain('path="테스트/섹션-a"'); + }); + + it('falls back to single md when no document tags', () => { + const input = '요약만\n\n# 본문'; + const { documents, metadata } = parseChunksFromMarkdown(input, 'alone.pdf'); + expect(metadata.description).toBe('요약만'); + expect(metadata.chunks).toEqual([]); + expect(documents['alone.md']).toContain('# 본문'); + }); +}); + +describe('toResourceName', () => { + it('strips pdf extension and NFC-normalizes', () => { + expect(toResourceName('안내.pdf')).toBe('안내'); + expect(toResourceName('folder/문서.PDF')).toBe('문서'); + }); +}); diff --git a/src/pdf-processor/pdf-chunk-parser.ts b/src/pdf-processor/pdf-chunk-parser.ts new file mode 100644 index 0000000..cedafbf --- /dev/null +++ b/src/pdf-processor/pdf-chunk-parser.ts @@ -0,0 +1,82 @@ +export type ParsedChunk = { + path: string; + description: string; + content: string; +}; + +export type ChunkParseResult = { + /** GCS object path → markdown content */ + documents: Record; + metadata: { + description: string; + chunks: { path: string; description: string }[]; + }; +}; + +/** + * Parse and tags from chunked markdown. + * Ported from worker.py `_parse_chunks_from_markdown`. + */ +export function parseChunksFromMarkdown( + markdown: string, + resourceName: string, +): ChunkParseResult { + const baseName = + resourceName.includes('.') && resourceName.toLowerCase().endsWith('.pdf') + ? resourceName.slice(0, -4) + : resourceName.includes('.') + ? resourceName.replace(/\.[^.]+$/, '') + : resourceName; + + const summaryMatch = markdown.match(/(.+?)<\/summary>/s); + const summary = summaryMatch?.[1]?.trim() ?? ''; + + const chunkPattern = + /(.+?)<\/document>/gs; + const chunks: ParsedChunk[] = []; + for (const match of markdown.matchAll(chunkPattern)) { + chunks.push({ + path: match[1], + description: match[2], + content: match[3].trim(), + }); + } + + if (chunks.length === 0) { + return { + documents: { [`${baseName}.md`]: markdown }, + metadata: { description: summary, chunks: [] }, + }; + } + + const documents: Record = {}; + const mainDocParts: string[] = []; + const chunkMetadata: { path: string; description: string }[] = []; + + for (const chunk of chunks) { + documents[`${baseName}/${chunk.path}.md`] = chunk.content; + mainDocParts.push( + ``, + ); + chunkMetadata.push({ + path: `${baseName}/${chunk.path}`, + description: chunk.description, + }); + } + + documents[`${baseName}.md`] = mainDocParts.join('\n\n'); + + return { + documents, + metadata: { description: summary, chunks: chunkMetadata }, + }; +} + +/** + * Normalize a filename to NFC resource_name (stem without .pdf). + */ +export function toResourceName(filename: string): string { + const nfc = filename.normalize('NFC'); + const base = nfc.replace(/\\/g, '/').split('/').pop() ?? nfc; + return base.toLowerCase().endsWith('.pdf') ? base.slice(0, -4) : base; +} diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts new file mode 100644 index 0000000..d9968e1 --- /dev/null +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -0,0 +1,182 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { PDF_PROCESSOR_PROMPT } from '../chat/prompts/pdf-processor'; +import { PDF_CHUNKING_PROMPT } from '../chat/prompts/pdf-chunking-prompt'; +import { + LLM_CLIENT, + type LlmClient, +} from '../chat/llm/llm-client.interface'; +import { PdfTextService } from './pdf-text.service'; +import { parseChunksFromMarkdown } from './pdf-chunk-parser'; +import type { ResourceIndexEntry } from './gcs-storage.service'; + +export type PipelineChunk = { + path: string; + description: string; + content: string; + sortOrder: number; +}; + +export type PipelineResult = { + documents: Record; + metadata: ResourceIndexEntry; + summary: string; + chunks: PipelineChunk[]; +}; + +@Injectable() +export class PdfPipelineService { + private readonly logger = new Logger(PdfPipelineService.name); + private readonly contextLength: number; + private readonly llmTimeoutMs: number; + + constructor( + private readonly pdfTextService: PdfTextService, + private readonly configService: ConfigService, + @Inject(LLM_CLIENT) private readonly llm: LlmClient, + ) { + this.contextLength = Number( + this.configService.get('PDF_PROCESSOR_CONTEXT_LENGTH') ?? 500, + ); + this.llmTimeoutMs = + Number( + this.configService.get('PDF_PROCESSOR_LLM_TIMEOUT') ?? 120, + ) * 1000; + } + + /** + * Pass 1 (page → markdown) + Pass 2 (semantic chunking). Text-only (no images). + */ + async processPdf( + pdfBytes: Buffer, + filename: string, + ): Promise { + const pageTexts = await this.pdfTextService.extractPageTexts(pdfBytes); + const totalPages = pageTexts.length; + this.logger.log( + `Pass 1: Converting ${filename} to markdown (${totalPages} pages)`, + ); + + const pageMarkdowns: string[] = []; + let previousContext = ''; + + for (let i = 0; i < totalPages; i += 1) { + const currentPage = i + 1; + const pageText = pageTexts[i] ?? ''; + const pageMarkdown = await this.convertPageToMarkdown({ + filename, + totalPages, + currentPage, + pageText, + previousContext, + }); + pageMarkdowns.push(pageMarkdown); + previousContext = + pageMarkdown.length > this.contextLength + ? pageMarkdown.slice(-this.contextLength) + : pageMarkdown; + } + + const combinedMarkdown = pageMarkdowns.join('\n\n'); + this.logger.log( + `Pass 2: Chunking complete markdown (${combinedMarkdown.length} chars)`, + ); + + const { documents, metadata } = await this.chunkMarkdownWithLlm( + combinedMarkdown, + filename, + ); + + const chunks: PipelineChunk[] = metadata.chunks.map((c, idx) => ({ + path: c.path, + description: c.description, + content: documents[`${c.path}.md`] ?? '', + sortOrder: idx, + })); + + return { + documents, + metadata, + summary: metadata.description, + chunks, + }; + } + + private async convertPageToMarkdown(params: { + filename: string; + totalPages: number; + currentPage: number; + pageText: string; + previousContext: string; + }): Promise { + const { filename, totalPages, currentPage, pageText, previousContext } = + params; + + const prompt = PDF_PROCESSOR_PROMPT.replaceAll('{filename}', filename) + .replaceAll('{total_pages}', String(totalPages)) + .replaceAll('{current_page}', String(currentPage)) + .replaceAll( + '{previous_context}', + previousContext || '없음 (첫 페이지)', + ); + + const userText = pageText.trim() + ? `${prompt}\n\n페이지 텍스트:\n${pageText}` + : `${prompt}\n\n페이지 텍스트를 추출할 수 없었습니다. 추출된 텍스트 없이 가능한 범위에서 변환하세요.`; + + try { + const model = this.llm.getModel('normal'); + const response = await this.llm.callLLM( + [{ role: 'user', content: userText }], + model, + { + temperature: 0.2, + max_tokens: 8000, + timeoutMs: this.llmTimeoutMs, + }, + ); + return response.choices?.[0]?.message?.content ?? ''; + } catch (error) { + this.logger.error( + `Error calling LLM for page ${currentPage}: ${error instanceof Error ? error.message : String(error)}`, + ); + return `## Page ${currentPage}\n\n${pageText}`; + } + } + + private async chunkMarkdownWithLlm( + markdown: string, + filename: string, + ): Promise<{ + documents: Record; + metadata: ResourceIndexEntry; + }> { + const prompt = PDF_CHUNKING_PROMPT.replaceAll('{filename}', filename); + const baseName = filename.toLowerCase().endsWith('.pdf') + ? filename.slice(0, -4) + : filename; + + try { + const model = this.llm.getModel('normal'); + const response = await this.llm.callLLM( + [{ role: 'user', content: `${prompt}\n\n${markdown}` }], + model, + { + temperature: 0.2, + max_tokens: 16000, + timeoutMs: this.llmTimeoutMs, + }, + ); + const chunked = response.choices?.[0]?.message?.content ?? markdown; + return parseChunksFromMarkdown(chunked, filename); + } catch (error) { + this.logger.error( + `Error chunking markdown: ${error instanceof Error ? error.message : String(error)}`, + ); + return { + documents: { [`${baseName}.md`]: markdown }, + metadata: { description: '', chunks: [] }, + }; + } + } +} diff --git a/src/pdf-processor/pdf-processor.module.ts b/src/pdf-processor/pdf-processor.module.ts new file mode 100644 index 0000000..cb4e863 --- /dev/null +++ b/src/pdf-processor/pdf-processor.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { ChatModule } from '../chat/chat.module'; +import { DbModule } from '../db/db.module'; +import { DocumentsRepository } from './documents.repository'; +import { GcsStorageService } from './gcs-storage.service'; +import { PdfTextService } from './pdf-text.service'; +import { PdfPipelineService } from './pdf-pipeline.service'; +import { PdfProcessorWorker } from './pdf-processor.worker'; + +@Module({ + imports: [DbModule, ChatModule], + providers: [ + GcsStorageService, + DocumentsRepository, + PdfTextService, + PdfPipelineService, + PdfProcessorWorker, + ], + exports: [GcsStorageService, DocumentsRepository, PdfPipelineService], +}) +export class PdfProcessorModule {} diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts new file mode 100644 index 0000000..8ebba82 --- /dev/null +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import type { ConfigService } from '@nestjs/config'; +import type { Document } from '../db'; +import type { DocumentsRepository } from './documents.repository'; +import type { GcsStorageService } from './gcs-storage.service'; +import type { PdfPipelineService } from './pdf-pipeline.service'; +import { PdfProcessorWorker } from './pdf-processor.worker'; + +function processingDocument(): Document { + return { + id: '00000000-0000-0000-0000-000000000001', + title: '테스트', + resourceName: 'test', + summary: null, + gcsPdfPath: 'gs://bucket/test.pdf', + status: 'processing', + errorMessage: null, + processingToken: '00000000-0000-0000-0000-000000000002', + uploadedByIdpUuid: 'admin-1', + isActive: true, + createdAt: new Date(), + updatedAt: new Date(), + processedAt: null, + }; +} + +function createWorker(completeProcessing: boolean) { + const repo = { + completeProcessing: jest.fn< + ( + id: string, + token: string, + summary: string, + chunks: unknown[], + ) => Promise + >(() => Promise.resolve(completeProcessing)), + markFailed: jest.fn(() => Promise.resolve(true)), + requeueStaleProcessing: jest.fn(() => Promise.resolve(0)), + claimQueued: jest.fn(() => Promise.resolve([])), + }; + const gcs = { + downloadPdf: jest.fn(() => Promise.resolve(Buffer.from('%PDF-test'))), + uploadDocuments: jest.fn(() => Promise.resolve()), + updateResourceIndex: jest.fn(() => Promise.resolve()), + deleteProcessedArtifacts: jest.fn<(resourceName: string) => Promise>( + () => Promise.resolve(), + ), + }; + const pipeline = { + processPdf: jest.fn(() => + Promise.resolve({ + documents: { 'test.md': '# test' }, + metadata: { description: 'summary', chunks: [] }, + summary: 'summary', + chunks: [], + }), + ), + }; + const config = { + get: jest.fn((_key: string) => undefined), + }; + + return { + worker: new PdfProcessorWorker( + repo as unknown as DocumentsRepository, + gcs as unknown as GcsStorageService, + pipeline as unknown as PdfPipelineService, + config as unknown as ConfigService, + ), + repo, + gcs, + }; +} + +describe('PdfProcessorWorker attempt ownership', () => { + it('deletes generated artifacts when a delete/reprocess cancels the attempt', async () => { + const { worker, repo, gcs } = createWorker(false); + const callable = worker as unknown as { + processDocument(doc: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(repo.completeProcessing).toHaveBeenCalledWith( + '00000000-0000-0000-0000-000000000001', + '00000000-0000-0000-0000-000000000002', + 'summary', + [], + ); + expect(gcs.deleteProcessedArtifacts).toHaveBeenCalledWith('test'); + }); + + it('keeps generated artifacts when the attempt completes successfully', async () => { + const { worker, gcs } = createWorker(true); + const callable = worker as unknown as { + processDocument(doc: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(gcs.deleteProcessedArtifacts).not.toHaveBeenCalled(); + }); +}); diff --git a/src/pdf-processor/pdf-processor.worker.ts b/src/pdf-processor/pdf-processor.worker.ts new file mode 100644 index 0000000..fb35279 --- /dev/null +++ b/src/pdf-processor/pdf-processor.worker.ts @@ -0,0 +1,203 @@ +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { DocumentsRepository } from './documents.repository'; +import { GcsStorageService } from './gcs-storage.service'; +import { PdfPipelineService } from './pdf-pipeline.service'; +import type { Document } from '../db'; + +@Injectable() +export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(PdfProcessorWorker.name); + private readonly concurrency: number; + private readonly pollIntervalMs: number; + private readonly staleProcessingMs: number; + private readonly staleCheckIntervalMs = 60_000; + private activeCount = 0; + private readonly activeDocumentIds = new Set(); + private lastStaleCheckAt = 0; + private timer: ReturnType | null = null; + private stopped = false; + private tickInFlight = false; + + constructor( + private readonly documentsRepo: DocumentsRepository, + private readonly gcs: GcsStorageService, + private readonly pipeline: PdfPipelineService, + private readonly configService: ConfigService, + ) { + this.concurrency = Math.max( + 1, + Number( + this.configService.get('PDF_PROCESSOR_CONCURRENCY') ?? 1, + ), + ); + this.pollIntervalMs = Math.max( + 500, + Number( + this.configService.get('PDF_PROCESSOR_POLL_INTERVAL_MS') ?? + 2000, + ), + ); + // Default: requeue if stuck in processing > 30 minutes + this.staleProcessingMs = Math.max( + 60_000, + Number( + this.configService.get( + 'PDF_PROCESSOR_STALE_PROCESSING_MS', + ) ?? 30 * 60 * 1000, + ), + ); + } + + async onModuleInit(): Promise { + this.logger.log( + `PDF processor worker started (concurrency=${this.concurrency}, poll=${this.pollIntervalMs}ms)`, + ); + await this.requeueStale(); + this.timer = setInterval(() => { + void this.tick(); + }, this.pollIntervalMs); + // Avoid keeping the process alive solely because of the timer in tests + if (typeof this.timer.unref === 'function') { + this.timer.unref(); + } + void this.tick(); + } + + onModuleDestroy(): void { + this.stopped = true; + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + private async requeueStale(): Promise { + try { + const staleBefore = new Date(Date.now() - this.staleProcessingMs); + const count = await this.documentsRepo.requeueStaleProcessing( + staleBefore, + [...this.activeDocumentIds], + ); + this.lastStaleCheckAt = Date.now(); + if (count > 0) { + this.logger.warn(`Requeued ${count} stale processing document(s)`); + } + } catch (error) { + this.logger.error( + `Failed to requeue stale jobs: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + private async tick(): Promise { + if (this.stopped || this.tickInFlight) return; + this.tickInFlight = true; + try { + if ( + Date.now() - this.lastStaleCheckAt >= + this.staleCheckIntervalMs + ) { + await this.requeueStale(); + } + + const slots = this.concurrency - this.activeCount; + if (slots <= 0) return; + + const claimed = await this.documentsRepo.claimQueued(slots); + for (const doc of claimed) { + this.activeCount += 1; + this.activeDocumentIds.add(doc.id); + void this.processDocument(doc).finally(() => { + this.activeCount -= 1; + this.activeDocumentIds.delete(doc.id); + }); + } + } catch (error) { + this.logger.error( + `Worker tick failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + this.tickInFlight = false; + } + } + + private async processDocument(doc: Document): Promise { + const resourceName = doc.resourceName; + const processingToken = doc.processingToken; + this.logger.log(`Processing document id=${doc.id} name=${resourceName}`); + + if (!processingToken) { + this.logger.error( + `Claimed document has no processing token: id=${doc.id}`, + ); + return; + } + + let generatedArtifactsMayExist = false; + try { + const pdfBytes = await this.gcs.downloadPdf(resourceName); + const result = await this.pipeline.processPdf( + pdfBytes, + `${resourceName}.pdf`, + ); + + generatedArtifactsMayExist = true; + await this.gcs.uploadDocuments(result.documents); + await this.gcs.updateResourceIndex(resourceName, result.metadata); + const completed = await this.documentsRepo.completeProcessing( + doc.id, + processingToken, + result.summary, + result.chunks, + ); + if (!completed) { + this.logger.warn( + `Discarding cancelled processing result: id=${doc.id} token=${processingToken}`, + ); + await this.cleanupGeneratedArtifacts(resourceName); + return; + } + + this.logger.log( + `Processing complete: ${resourceName} (${Object.keys(result.documents).length} files, ${result.chunks.length} chunks)`, + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.error( + `Processing failed id=${doc.id} name=${resourceName}: ${message}`, + error instanceof Error ? error.stack : undefined, + ); + try { + await this.documentsRepo.markFailed( + doc.id, + processingToken, + message, + ); + } catch (markError) { + this.logger.error( + `Failed to persist processing error id=${doc.id}: ${markError instanceof Error ? markError.message : String(markError)}`, + ); + } + if (generatedArtifactsMayExist) { + await this.cleanupGeneratedArtifacts(resourceName); + } + } + } + + private async cleanupGeneratedArtifacts(resourceName: string): Promise { + try { + await this.gcs.deleteProcessedArtifacts(resourceName); + } catch (error) { + this.logger.error( + `Failed to clean cancelled artifacts for ${resourceName}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} diff --git a/src/pdf-processor/pdf-text.service.ts b/src/pdf-processor/pdf-text.service.ts new file mode 100644 index 0000000..c0568bd --- /dev/null +++ b/src/pdf-processor/pdf-text.service.ts @@ -0,0 +1,57 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { normalizeExtractedText, isLikelyMojibake } from './mojibake'; + +type PdfjsModule = typeof import('pdfjs-dist/legacy/build/pdf.mjs'); + +@Injectable() +export class PdfTextService { + private readonly logger = new Logger(PdfTextService.name); + private pdfjsPromise: Promise | null = null; + + private loadPdfjs(): Promise { + if (!this.pdfjsPromise) { + this.pdfjsPromise = import('pdfjs-dist/legacy/build/pdf.mjs'); + } + return this.pdfjsPromise; + } + + /** + * Extract text per page from a PDF buffer (1-indexed page order in logs; array is 0-indexed). + */ + async extractPageTexts(pdfBytes: Buffer): Promise { + const pdfjs = await this.loadPdfjs(); + const data = new Uint8Array(pdfBytes); + const loadingTask = pdfjs.getDocument({ + data, + useSystemFonts: true, + useWorkerFetch: false, + disableFontFace: true, + }); + const pdf = await loadingTask.promise; + const pageTexts: string[] = []; + + for (let pageNum = 1; pageNum <= pdf.numPages; pageNum += 1) { + const page = await pdf.getPage(pageNum); + const textContent = await page.getTextContent(); + const raw = textContent.items + .map((item) => ('str' in item ? String(item.str) : '')) + .join(' '); + + if (isLikelyMojibake(raw)) { + const normalized = normalizeExtractedText(raw); + if (!normalized) { + this.logger.warn( + `Page ${pageNum}: Mojibake detected but recovery failed, skipping extracted text`, + ); + } else { + this.logger.log(`Page ${pageNum}: Fixed mojibake in extracted text`); + } + pageTexts.push(normalized); + } else { + pageTexts.push(raw); + } + } + + return pageTexts; + } +} diff --git a/src/scripts/smoke-pdf-processor.ts b/src/scripts/smoke-pdf-processor.ts new file mode 100644 index 0000000..bee78a4 --- /dev/null +++ b/src/scripts/smoke-pdf-processor.ts @@ -0,0 +1,114 @@ +/** + * Offline smoke checks for PDF processor pieces (no GCS/LLM required). + * + * Usage: bun src/scripts/smoke-pdf-processor.ts + */ +import { readFileSync, writeFileSync, unlinkSync, existsSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { parseChunksFromMarkdown, toResourceName } from '../pdf-processor/pdf-chunk-parser'; +import { + isLikelyMojibake, + normalizeExtractedText, +} from '../pdf-processor/mojibake'; +import iconv from 'iconv-lite'; +import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; + +/** Minimal one-page PDF with ASCII text (Helvetica). */ +function buildMinimalPdf(text: string): Buffer { + const content = `BT /F1 12 Tf 50 700 Td (${text}) Tj ET`; + const objects = [ + '1 0 obj<< /Type /Catalog /Pages 2 0 R >>endobj\n', + '2 0 obj<< /Type /Pages /Kids [3 0 R] /Count 1 >>endobj\n', + '3 0 obj<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources<< /Font<< /F1 5 0 R >> >> >>endobj\n', + `4 0 obj<< /Length ${Buffer.byteLength(content)} >>stream\n${content}\nendstream\nendobj\n`, + '5 0 obj<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>endobj\n', + ]; + + let pdf = '%PDF-1.4\n'; + const offsets: number[] = [0]; + for (const obj of objects) { + offsets.push(Buffer.byteLength(pdf)); + pdf += obj; + } + const xrefStart = Buffer.byteLength(pdf); + pdf += `xref\n0 ${objects.length + 1}\n`; + pdf += '0000000000 65535 f \n'; + for (let i = 1; i < offsets.length; i += 1) { + pdf += `${String(offsets[i]).padStart(10, '0')} 00000 n \n`; + } + pdf += `trailer<< /Size ${objects.length + 1} /Root 1 0 R >>\n`; + pdf += `startxref\n${xrefStart}\n%%EOF\n`; + return Buffer.from(pdf); +} + +async function extractFirstPageText(pdfBytes: Buffer): Promise { + const loadingTask = getDocument({ + data: new Uint8Array(pdfBytes), + useSystemFonts: true, + useWorkerFetch: false, + disableFontFace: true, + }); + const pdf = await loadingTask.promise; + const page = await pdf.getPage(1); + const textContent = await page.getTextContent(); + return textContent.items + .map((item) => ('str' in item ? String(item.str) : '')) + .join(' '); +} + +async function main() { + console.log('=== PDF processor smoke (offline) ==='); + + // 1) resource name NFC + const name = toResourceName('학생-안내.pdf'); + if (name !== '학생-안내') { + throw new Error(`toResourceName failed: ${name}`); + } + console.log('OK toResourceName'); + + // 2) mojibake + const original = '학사 일정'; + const mojibake = iconv.decode(Buffer.from(original, 'utf8'), 'latin1'); + if (!isLikelyMojibake(mojibake)) { + throw new Error('mojibake not detected'); + } + if (normalizeExtractedText(mojibake) !== original) { + throw new Error('mojibake fix failed'); + } + console.log('OK mojibake'); + + // 3) chunk parser + const parsed = parseChunksFromMarkdown( + `요약\n본문`, + 'doc.pdf', + ); + if (parsed.metadata.description !== '요약') { + throw new Error('summary parse failed'); + } + if (parsed.documents['doc/a.md'] !== '본문') { + throw new Error('chunk content parse failed'); + } + console.log('OK chunk parser'); + + // 4) pdfjs text extract + const pdfBytes = buildMinimalPdf('Hello Ziggle'); + const tmp = join(tmpdir(), `smoke-pdf-${Date.now()}.pdf`); + writeFileSync(tmp, pdfBytes); + try { + const extracted = await extractFirstPageText(readFileSync(tmp)); + if (!extracted.includes('Hello Ziggle')) { + throw new Error(`pdf text extract failed: "${extracted}"`); + } + console.log('OK pdfjs text extract:', extracted.trim()); + } finally { + if (existsSync(tmp)) unlinkSync(tmp); + } + + console.log('=== smoke passed ==='); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 67f67330647962dee896651d68fc095a78351dfb Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Wed, 29 Jul 2026 23:43:53 -0700 Subject: [PATCH 09/40] feat(upload): route admin PDF uploads through internal processor Replace resource-center proxy with GCS reservation, status APIs, and reprocess while keeping the existing admin upload endpoint. Co-authored-by: Cursor --- src/upload/upload.controller.ts | 52 ++-- src/upload/upload.module.ts | 4 +- src/upload/upload.service.spec.ts | 112 +++++++++ src/upload/upload.service.ts | 380 +++++++++++++----------------- 4 files changed, 313 insertions(+), 235 deletions(-) create mode 100644 src/upload/upload.service.spec.ts diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index 50ebc7d..fc19ee8 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -21,15 +21,13 @@ import { ApiQuery, } from '@nestjs/swagger'; import type { FastifyRequest } from 'fastify'; -import { UploadService } from './upload.service'; +import { UploadService, PDF_MIME } from './upload.service'; import { AdminJwtGuard } from '../auth/guards/admin-jwt.guard'; import { SuperAdminGuard } from '../auth/guards/super-admin.guard'; import { CurrentAdmin } from '../auth/decorators/current-admin.decorator'; import { AdminContext } from '../auth/context/admin-context.entity'; import { Readable } from 'stream'; -const PDF_MIME = 'application/pdf'; - async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { const chunks: Buffer[] = []; for await (const chunk of Readable.from(stream)) { @@ -49,7 +47,7 @@ export class UploadController { @ApiOperation({ summary: '내가 업로드한 문서 목록 조회 (Super Admin 전용)', description: - '현재 로그인한 Super Admin이 업로드한 문서 목록을 최신순으로 반환합니다. 삭제되지 않은(is_active) 문서만 포함됩니다.', + '현재 로그인한 Super Admin이 업로드한 문서 목록을 최신순으로 반환합니다. 삭제되지 않은 문서만 포함되며, 처리 상태(status)를 포함합니다.', }) @ApiQuery({ name: 'limit', @@ -88,11 +86,26 @@ export class UploadController { }); } + @Get(':id') + @ApiOperation({ + summary: '문서 단건 조회 (상태 포함)', + description: '업로드한 문서의 처리 상태를 조회합니다.', + }) + @ApiParam({ name: 'id', description: '문서 UUID' }) + @ApiResponse({ status: 200, description: '성공' }) + @ApiResponse({ status: 404, description: '문서 없음' }) + async getOne( + @CurrentAdmin() admin: AdminContext, + @Param('id') id: string, + ) { + return this.uploadService.getById(id, admin.uuid); + } + @Post() @ApiOperation({ summary: 'PDF 파일 업로드 (Super Admin 전용)', description: - 'PDF 파일을 resource-center에 업로드하고 우리 DB에 메타데이터를 기록합니다. Super Admin 역할만 호출 가능합니다.', + 'PDF를 GCS에 저장하고 비동기 처리 큐에 등록합니다. 처리 완료를 기다리지 않으며 status=queued로 즉시 응답합니다.', }) @ApiBody({ schema: { @@ -104,13 +117,14 @@ export class UploadController { }, }, }) - @ApiResponse({ status: 201, description: '업로드 성공' }) + @ApiResponse({ status: 201, description: '업로드 성공 (queued)' }) @ApiResponse({ status: 400, description: '잘못된 요청 (PDF 아님, 필드 누락 등)', }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 409, description: '동일 resource_name 문서가 이미 존재' }) async upload( @CurrentAdmin() admin: AdminContext, @Req() req: FastifyRequest, @@ -155,19 +169,25 @@ export class UploadController { throw new BadRequestException('Only PDF files are allowed'); } - const record = await this.uploadService.upload( + return this.uploadService.upload( fileBuffer, filename, title.trim(), admin.uuid, ); + } - return { - id: record.id, - title: record.title, - metadata: record.metadata, - uploadedAt: record.createdAt, - }; + @Post(':id/reprocess') + @ApiOperation({ + summary: '문서 재처리', + description: + '기존 청크를 비우고 status를 queued로 되돌려 워커가 다시 처리하도록 합니다.', + }) + @ApiParam({ name: 'id', description: '문서 UUID' }) + @ApiResponse({ status: 200, description: '재처리 큐 등록' }) + @ApiResponse({ status: 404, description: '문서 없음' }) + async reprocess(@Param('id') id: string) { + return this.uploadService.reprocess(id); } @Delete(':id') @@ -175,15 +195,15 @@ export class UploadController { @ApiOperation({ summary: '업로드 파일 삭제 (Super Admin 전용)', description: - '우리 DB에서 is_active를 false로 갱신하고 resource-center에서 해당 리소스를 삭제합니다. Super Admin 역할만 호출 가능합니다.', + 'GCS 산출물을 삭제하고 DB에서 soft-delete 합니다. Super Admin 역할만 호출 가능합니다.', }) - @ApiParam({ name: 'id', description: '업로드 기록 UUID', type: String }) + @ApiParam({ name: 'id', description: '문서 UUID', type: String }) @ApiResponse({ status: 204, description: '삭제 성공' }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ status: 404, - description: '업로드 기록 없음 또는 이미 삭제됨', + description: '문서 없음 또는 이미 삭제됨', }) async delete(@Param('id') id: string): Promise { await this.uploadService.delete(id); diff --git a/src/upload/upload.module.ts b/src/upload/upload.module.ts index 4255a84..21c1aec 100644 --- a/src/upload/upload.module.ts +++ b/src/upload/upload.module.ts @@ -1,12 +1,12 @@ import { Module } from '@nestjs/common'; -import { HttpModule } from '@nestjs/axios'; import { UploadController } from './upload.controller'; import { UploadService } from './upload.service'; import { AuthModule } from '../auth/auth.module'; import { DbModule } from '../db/db.module'; +import { PdfProcessorModule } from '../pdf-processor/pdf-processor.module'; @Module({ - imports: [HttpModule, AuthModule, DbModule], + imports: [AuthModule, DbModule, PdfProcessorModule], controllers: [UploadController], providers: [UploadService], }) diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts new file mode 100644 index 0000000..a872f82 --- /dev/null +++ b/src/upload/upload.service.spec.ts @@ -0,0 +1,112 @@ +import { ConflictException } from '@nestjs/common'; +import { describe, expect, it, jest } from '@jest/globals'; +import type { Document } from '../db'; +import type { DocumentsRepository } from '../pdf-processor/documents.repository'; +import type { GcsStorageService } from '../pdf-processor/gcs-storage.service'; +import { UploadService } from './upload.service'; + +function document(overrides: Partial = {}): Document { + return { + id: '00000000-0000-0000-0000-000000000001', + title: '테스트', + resourceName: 'test', + summary: null, + gcsPdfPath: 'gs://bucket/test.pdf', + status: 'uploading', + errorMessage: null, + processingToken: null, + uploadedByIdpUuid: 'admin-1', + isActive: true, + createdAt: new Date(), + updatedAt: new Date(), + processedAt: null, + ...overrides, + }; +} + +function createService() { + const repo = { + createUploading: jest.fn< + (...args: unknown[]) => Promise + >(), + markQueuedAfterUpload: jest.fn(), + hardDelete: jest.fn(), + cancelAndSoftDelete: jest.fn(), + }; + const gcs = { + toGsPath: jest.fn((path: string) => `gs://bucket/${path}`), + uploadPdf: jest.fn(), + deleteResourceArtifacts: jest.fn(), + }; + return { + service: new UploadService( + repo as unknown as DocumentsRepository, + gcs as unknown as GcsStorageService, + ), + repo, + gcs, + }; +} + +describe('UploadService atomic transitions', () => { + it('reserves the DB resource name before uploading to GCS', async () => { + const { service, repo, gcs } = createService(); + const calls: string[] = []; + const reserved = document(); + const queued = document({ status: 'queued' }); + + repo.createUploading.mockImplementation(() => { + calls.push('reserve'); + return Promise.resolve(reserved); + }); + gcs.uploadPdf.mockImplementation(() => { + calls.push('upload'); + return Promise.resolve('gs://bucket/test.pdf'); + }); + repo.markQueuedAfterUpload.mockImplementation(() => { + calls.push('queue'); + return Promise.resolve(queued); + }); + + await service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + 'admin-1', + ); + + expect(calls).toEqual(['reserve', 'upload', 'queue']); + }); + + it('returns conflict without touching GCS when the name is already reserved', async () => { + const { service, repo, gcs } = createService(); + repo.createUploading.mockRejectedValue({ code: '23505' }); + + await expect( + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + 'admin-1', + ), + ).rejects.toBeInstanceOf(ConflictException); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); + }); + + it('cancels the DB processing attempt before deleting GCS artifacts', async () => { + const { service, repo, gcs } = createService(); + const calls: string[] = []; + repo.cancelAndSoftDelete.mockImplementation(() => { + calls.push('cancel'); + return Promise.resolve(document({ isActive: false })); + }); + gcs.deleteResourceArtifacts.mockImplementation(() => { + calls.push('delete-artifacts'); + return Promise.resolve(); + }); + + await service.delete('00000000-0000-0000-0000-000000000001'); + + expect(calls).toEqual(['cancel', 'delete-artifacts']); + }); +}); diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index c7d2aaa..f01f743 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -1,281 +1,227 @@ import { Injectable, - Inject, Logger, NotFoundException, BadRequestException, + ConflictException, } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { HttpService } from '@nestjs/axios'; -import { firstValueFrom } from 'rxjs'; -import { catchError } from 'rxjs/operators'; -import { AxiosError } from 'axios'; -import FormData from 'form-data'; -import { DB_CONNECTION, uploadedResources } from '../db'; -import type { Database } from '../db'; -import type { UploadedResource } from '../db'; -import { eq, and, desc } from 'drizzle-orm'; +import { DocumentsRepository } from '../pdf-processor/documents.repository'; +import { GcsStorageService } from '../pdf-processor/gcs-storage.service'; +import { toResourceName } from '../pdf-processor/pdf-chunk-parser'; +import type { Document } from '../db'; const PDF_MIME = 'application/pdf'; - const DEFAULT_LIMIT = 50; const MAX_LIMIT = 100; -/** - * gcs_path(gs://bucket/object-path)에서 resource-center DELETE용 객체 경로만 추출 - * resource-center는 버킷 접두어 없이 객체 경로만 받음 - */ -function toResourcePath(gcsPath: string): string { - const match = gcsPath.match(/^gs:\/\/[^/]+\/(.+)$/); - return match ? match[1] : gcsPath; -} +export type DocumentListItem = { + id: string; + title: string; + resourceName: string; + status: Document['status']; + summary: string | null; + gcsPdfPath: string; + errorMessage: string | null; + uploadedAt: Date; + processedAt: Date | null; +}; @Injectable() export class UploadService { private readonly logger = new Logger(UploadService.name); - private readonly resourceApiBaseUrl: string; constructor( - @Inject(DB_CONNECTION) private readonly db: Database, - private readonly httpService: HttpService, - private readonly configService: ConfigService, - ) { - this.resourceApiBaseUrl = this.configService.getOrThrow( - 'MCP_RESOURCE_API_URL', - ); - } + private readonly documentsRepo: DocumentsRepository, + private readonly gcs: GcsStorageService, + ) {} - /** - * 현재 admin이 업로드한 문서 목록 조회 (is_active = true만, 최신순) - */ async listMyUploads( idpUuid: string, options: { limit?: number; offset?: number } = {}, - ): Promise< - { - id: string; - title: string; - metadata: Record; - uploadedAt: Date; - }[] - > { + ): Promise { const limit = Math.min(options.limit ?? DEFAULT_LIMIT, MAX_LIMIT); const offset = Math.max(0, options.offset ?? 0); - const rows = await this.db - .select({ - id: uploadedResources.id, - title: uploadedResources.title, - metadata: uploadedResources.metadata, - uploadedAt: uploadedResources.createdAt, - }) - .from(uploadedResources) - .where( - and( - eq(uploadedResources.uploadedByIdpUuid, idpUuid), - eq(uploadedResources.isActive, true), - ), - ) - .orderBy(desc(uploadedResources.createdAt)) - .limit(limit) - .offset(offset); + const rows = await this.documentsRepo.listByUploader(idpUuid, { + limit, + offset, + }); + + return rows.map((row) => this.toListItem(row)); + } - return rows; + async getById(id: string, idpUuid: string): Promise { + const row = await this.documentsRepo.findById(id); + if (!row || !row.isActive) { + throw new NotFoundException(`Document not found: ${id}`); + } + if (row.uploadedByIdpUuid !== idpUuid) { + throw new NotFoundException(`Document not found: ${id}`); + } + return this.toListItem(row); } /** - * PDF 파일을 resource-center에 업로드하고 우리 DB에 기록 저장 + * Upload PDF to GCS and enqueue processing (status=queued). */ async upload( fileBuffer: Buffer, filename: string, title: string, idpUuid: string, - ): Promise { - const form = new FormData(); - form.append('file', fileBuffer, { - filename: filename || 'document.pdf', - contentType: PDF_MIME, - }); + ): Promise { + if (!fileBuffer?.length) { + throw new BadRequestException('file is required'); + } - const uploadUrl = `${this.resourceApiBaseUrl}/upload`; - this.logger.debug(`Uploading file to resource-center: ${uploadUrl}`); + const resourceName = toResourceName(filename || 'document.pdf'); + if (!resourceName.trim()) { + throw new BadRequestException('Invalid filename'); + } - let metadata: Record; + const gcsPdfPath = this.gcs.toGsPath(`${resourceName}.pdf`); + let reservation: Document; try { - const response = await firstValueFrom( - this.httpService - .post(uploadUrl, form, { - headers: form.getHeaders(), - maxBodyLength: Infinity, - maxContentLength: Infinity, - responseType: 'text', - validateStatus: (status) => status >= 200 && status < 300, - }) - .pipe( - catchError((error: AxiosError) => { - const status = error.response?.status; - const message = - error.response?.data != null - ? JSON.stringify(error.response.data) - : error.message; - this.logger.error( - `Resource-center upload failed: ${status} ${message}`, - ); - throw new BadRequestException( - status === 400 - ? `Upload failed: ${message}` - : `Resource-center upload failed: ${message}`, - ); - }), - ), - ); - const raw = - typeof response.data === 'string' - ? response.data - : String(response.data); - metadata = JSON.parse(raw) as Record; + reservation = await this.documentsRepo.createUploading({ + title, + resourceName, + gcsPdfPath, + uploadedByIdpUuid: idpUuid, + }); } catch (error) { - if (error instanceof BadRequestException) throw error; - if (error instanceof SyntaxError) { - this.logger.error('Resource-center response is not valid JSON', error); - throw new BadRequestException( - 'Resource-center upload response is not valid JSON', + if (isUniqueViolation(error)) { + throw new ConflictException( + `An active document with resource name "${resourceName}" already exists`, ); } - this.logger.error('Upload failed', error); + throw error; + } + + this.logger.debug(`Uploading PDF to GCS: ${resourceName}.pdf`); + try { + await this.gcs.uploadPdf(resourceName, fileBuffer); + } catch (error) { + this.logger.error( + `GCS upload failed: ${error instanceof Error ? error.message : String(error)}`, + ); + await this.rollbackUpload(reservation.id, resourceName); throw new BadRequestException( - `Upload failed: ${error instanceof Error ? error.message : String(error)}`, + `GCS upload failed: ${error instanceof Error ? error.message : String(error)}`, ); } - const [record] = await this.db - .insert(uploadedResources) - .values({ - title, - metadata, - uploadedByIdpUuid: idpUuid, - isActive: true, - }) - .returning(); - - if (!record) { - throw new Error('Failed to insert upload record'); + let record: Document | null; + try { + record = await this.documentsRepo.markQueuedAfterUpload(reservation.id); + if (!record) { + throw new Error('Upload reservation is no longer active'); + } + } catch (error) { + await this.rollbackUpload(reservation.id, resourceName); + throw new Error( + `Failed to enqueue the uploaded document: ${error instanceof Error ? error.message : String(error)}`, + ); } - this.logger.log(`Upload recorded: id=${record.id}`); - return record; + + this.logger.log( + `Upload queued: id=${record.id} resource=${resourceName}`, + ); + return this.toListItem(record); } /** - * resource-center에서 삭제 성공 후 DB에서 is_active = false 및 metadata 갱신 - * (DB는 DELETE 성공 후에만 갱신하여, 외부 삭제 실패 시 재시도 가능) + * Soft-delete DB row and remove GCS artifacts. */ async delete(id: string): Promise { - const [row] = await this.db - .select() - .from(uploadedResources) - .where(eq(uploadedResources.id, id)) - .limit(1); - + const row = await this.documentsRepo.cancelAndSoftDelete(id); if (!row) { - throw new NotFoundException(`Upload not found: ${id}`); - } - if (!row.isActive) { - throw new NotFoundException(`Upload already deleted: ${id}`); + throw new NotFoundException(`Document not found: ${id}`); } - const rawPath = - typeof row.metadata?.gcs_path === 'string' - ? row.metadata.gcs_path - : typeof row.metadata?.path === 'string' - ? row.metadata.path - : null; - if (!rawPath) { - this.logger.warn( - `No gcs_path/path in metadata for id=${id}, skipping resource-center DELETE`, + try { + await this.gcs.deleteResourceArtifacts(row.resourceName); + } catch (error) { + this.logger.error( + `GCS delete failed id=${id}: ${error instanceof Error ? error.message : String(error)}`, + ); + throw new BadRequestException( + `GCS delete failed: ${error instanceof Error ? error.message : String(error)}`, ); - return; } - const pathForDelete = toResourcePath(rawPath); - const deleteUrl = `${this.resourceApiBaseUrl}/resource/${encodeURIComponent(pathForDelete)}`; - this.logger.debug(`Deleting resource at: ${deleteUrl}`); - try { - const response = await firstValueFrom( - this.httpService - .delete<{ - status?: string; - path?: string; - deleted_files?: string[]; - count?: number; - }>(deleteUrl) - .pipe( - catchError((error: AxiosError) => { - const status = error.response?.status; - const message = - error.response?.data != null - ? JSON.stringify(error.response.data) - : error.message; - this.logger.error( - `Resource-center delete failed id=${id} deleteUrl=${deleteUrl} status=${status} ${message}`, - ); - if (status === 404) { - throw new NotFoundException( - `Resource not found at resource-center: ${rawPath}`, - ); - } - throw new BadRequestException( - `Resource-center delete failed: ${message}`, - ); - }), - ), - ); + this.logger.log(`Document deleted: id=${id} resource=${row.resourceName}`); + } + + /** + * Clear chunks and re-enqueue for processing. + */ + async reprocess(id: string): Promise { + const row = await this.documentsRepo.findById(id); + if (!row || !row.isActive) { + throw new NotFoundException(`Document not found: ${id}`); + } + if (row.status === 'uploading') { + throw new ConflictException('Document upload is still in progress'); + } - const deleteResponse = - typeof response.data === 'object' && response.data !== null - ? response.data - : typeof response.data === 'string' - ? (() => { - try { - return JSON.parse(response.data) as Record; - } catch { - return {}; - } - })() - : {}; + const updated = await this.documentsRepo.enqueueReprocess(id); + if (!updated) { + throw new NotFoundException(`Document not found: ${id}`); + } - const mergedMetadata: Record = { - ...(row.metadata as Record), - ...deleteResponse, - }; + this.logger.log(`Document requeued: id=${id}`); + return this.toListItem(updated); + } - const now = new Date(); - await this.db - .update(uploadedResources) - .set({ - isActive: false, - metadata: mergedMetadata, - updatedAt: now, - }) - .where(eq(uploadedResources.id, id)); + private toListItem(row: Document): DocumentListItem { + return { + id: row.id, + title: row.title, + resourceName: row.resourceName, + status: row.status, + summary: row.summary, + gcsPdfPath: row.gcsPdfPath, + errorMessage: row.errorMessage, + uploadedAt: row.createdAt, + processedAt: row.processedAt, + }; + } - this.logger.debug( - `Resource-center delete response applied to metadata: ${JSON.stringify(deleteResponse)}`, - ); - } catch (error) { - if ( - error instanceof BadRequestException || - error instanceof NotFoundException - ) { - throw error; + private async rollbackUpload( + documentId: string, + resourceName: string, + ): Promise { + const results = await Promise.allSettled([ + this.gcs.deleteResourceArtifacts(resourceName), + this.documentsRepo.hardDelete(documentId), + ]); + for (const result of results) { + if (result.status === 'rejected') { + this.logger.error( + `Upload rollback failed id=${documentId}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`, + ); } - this.logger.error( - `Delete failed id=${id} deleteUrl=${deleteUrl}`, - error instanceof Error ? error.stack : error, - ); - throw new BadRequestException( - `Delete failed: ${error instanceof Error ? error.message : String(error)}`, - ); } } } + +export { PDF_MIME }; + +function isUniqueViolation(error: unknown): boolean { + let current: unknown = error; + for (let depth = 0; depth < 4 && current; depth += 1) { + if ( + typeof current === 'object' && + current !== null && + 'code' in current && + current.code === '23505' + ) { + return true; + } + current = + typeof current === 'object' && current !== null && 'cause' in current + ? current.cause + : null; + } + return false; +} From 68f26fa6fe42b53b50b09c158aac56821db2a1a6 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 12:45:26 -0700 Subject: [PATCH 10/40] feat(retrieval): add DB-backed document catalog and content lookup --- src/retrieval/retrieval.module.ts | 11 +++ src/retrieval/retrieval.repository.ts | 91 +++++++++++++++++++++++++ src/retrieval/retrieval.service.spec.ts | 89 ++++++++++++++++++++++++ src/retrieval/retrieval.service.ts | 86 +++++++++++++++++++++++ src/retrieval/retrieval.types.ts | 26 +++++++ 5 files changed, 303 insertions(+) create mode 100644 src/retrieval/retrieval.module.ts create mode 100644 src/retrieval/retrieval.repository.ts create mode 100644 src/retrieval/retrieval.service.spec.ts create mode 100644 src/retrieval/retrieval.service.ts create mode 100644 src/retrieval/retrieval.types.ts diff --git a/src/retrieval/retrieval.module.ts b/src/retrieval/retrieval.module.ts new file mode 100644 index 0000000..fff3ee6 --- /dev/null +++ b/src/retrieval/retrieval.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { DbModule } from '../db/db.module'; +import { RetrievalRepository } from './retrieval.repository'; +import { RetrievalService } from './retrieval.service'; + +@Module({ + imports: [DbModule], + providers: [RetrievalRepository, RetrievalService], + exports: [RetrievalService], +}) +export class RetrievalModule {} diff --git a/src/retrieval/retrieval.repository.ts b/src/retrieval/retrieval.repository.ts new file mode 100644 index 0000000..70f22bb --- /dev/null +++ b/src/retrieval/retrieval.repository.ts @@ -0,0 +1,91 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { and, asc, eq, inArray } from 'drizzle-orm'; +import { DB_CONNECTION, documents, documentChunks } from '../db'; +import type { Database } from '../db'; + +export type ReadyDocumentWithChunks = { + id: string; + title: string; + resourceName: string; + summary: string | null; + chunks: Array<{ + path: string; + description: string; + content: string; + sortOrder: number; + }>; +}; + +@Injectable() +export class RetrievalRepository { + constructor(@Inject(DB_CONNECTION) private readonly db: Database) {} + + /** + * Ready + active documents that have at least one chunk. + */ + async listReadyWithChunks(): Promise { + const rows = await this.db + .select({ + documentId: documents.id, + title: documents.title, + resourceName: documents.resourceName, + summary: documents.summary, + chunkId: documentChunks.id, + chunkPath: documentChunks.path, + chunkDescription: documentChunks.description, + chunkContent: documentChunks.content, + chunkSortOrder: documentChunks.sortOrder, + }) + .from(documents) + .innerJoin(documentChunks, eq(documentChunks.documentId, documents.id)) + .where(and(eq(documents.status, 'ready'), eq(documents.isActive, true))) + .orderBy(asc(documents.createdAt), asc(documentChunks.sortOrder)); + + const byId = new Map(); + for (const row of rows) { + let doc = byId.get(row.documentId); + if (!doc) { + doc = { + id: row.documentId, + title: row.title, + resourceName: row.resourceName, + summary: row.summary, + chunks: [], + }; + byId.set(row.documentId, doc); + } + doc.chunks.push({ + path: row.chunkPath, + description: row.chunkDescription, + content: row.chunkContent, + sortOrder: row.chunkSortOrder, + }); + } + + return [...byId.values()]; + } + + async findChunkContentsByPaths( + paths: string[], + ): Promise> { + if (paths.length === 0) return []; + + const uniquePaths = [...new Set(paths)]; + const rows = await this.db + .select({ + path: documentChunks.path, + content: documentChunks.content, + }) + .from(documentChunks) + .innerJoin(documents, eq(documentChunks.documentId, documents.id)) + .where( + and( + inArray(documentChunks.path, uniquePaths), + eq(documents.status, 'ready'), + eq(documents.isActive, true), + ), + ); + + return rows; + } +} diff --git a/src/retrieval/retrieval.service.spec.ts b/src/retrieval/retrieval.service.spec.ts new file mode 100644 index 0000000..87e169d --- /dev/null +++ b/src/retrieval/retrieval.service.spec.ts @@ -0,0 +1,89 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { RetrievalService } from './retrieval.service'; +import type { RetrievalRepository } from './retrieval.repository'; + +describe('RetrievalService', () => { + it('builds new-format catalog and excludes docs handled by join (chunks required)', async () => { + const repo = { + listReadyWithChunks: jest.fn(async () => [ + { + id: 'd1', + title: '학사편람', + resourceName: '학사편람', + summary: '학사 안내', + chunks: [ + { + path: '학사편람/졸업요건', + description: '졸업', + content: '본문', + sortOrder: 0, + }, + ], + }, + ]), + findChunkContentsByPaths: jest.fn(), + }; + + const service = new RetrievalService( + repo as unknown as RetrievalRepository, + ); + const catalog = await service.listCatalog(); + + expect(catalog.resources).toEqual([ + { + path: '학사편람.pdf', + description: '학사 안내', + chunks: [{ path: '학사편람/졸업요건', description: '졸업' }], + }, + ]); + expect(catalog.chunks).toEqual([ + { path: '학사편람/졸업요건', description: '졸업' }, + ]); + expect(catalog.total).toBe(1); + expect(catalog.filteredResources).toEqual([]); + }); + + it('loads contents by path and strips .md for lookup', async () => { + const repo = { + listReadyWithChunks: jest.fn(), + findChunkContentsByPaths: jest.fn(async (paths: string[]) => { + expect(paths).toEqual(['학사편람/졸업요건']); + return [{ path: '학사편람/졸업요건', content: '졸업 본문' }]; + }), + }; + + const service = new RetrievalService( + repo as unknown as RetrievalRepository, + ); + const hits = await service.getContentsByPaths(['학사편람/졸업요건.md']); + expect(hits).toEqual([{ path: '학사편람/졸업요건', content: '졸업 본문' }]); + }); + + it('uses title when summary is empty', async () => { + const repo = { + listReadyWithChunks: jest.fn(async () => [ + { + id: 'd1', + title: '제목만', + resourceName: 'doc', + summary: ' ', + chunks: [ + { + path: 'doc/a', + description: 'a', + content: 'c', + sortOrder: 0, + }, + ], + }, + ]), + findChunkContentsByPaths: jest.fn(), + }; + + const service = new RetrievalService( + repo as unknown as RetrievalRepository, + ); + const catalog = await service.listCatalog(); + expect(catalog.resources?.[0]?.description).toBe('제목만'); + }); +}); diff --git a/src/retrieval/retrieval.service.ts b/src/retrieval/retrieval.service.ts new file mode 100644 index 0000000..0c61372 --- /dev/null +++ b/src/retrieval/retrieval.service.ts @@ -0,0 +1,86 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { RetrievalRepository } from './retrieval.repository'; +import type { + ChunkContentHit, + ListResourceItem, + ListResourcesResult, +} from './retrieval.types'; + +@Injectable() +export class RetrievalService { + private readonly logger = new Logger(RetrievalService.name); + + constructor(private readonly retrievalRepo: RetrievalRepository) {} + + /** + * Build the catalog shape previously provided by MCP list_resources (new format). + */ + async listCatalog(): Promise { + const docs = await this.retrievalRepo.listReadyWithChunks(); + const resources: ListResourceItem[] = docs.map((doc) => ({ + path: `${doc.resourceName}.pdf`, + description: doc.summary?.trim() || doc.title, + chunks: doc.chunks.map((c) => ({ + path: c.path, + description: c.description, + })), + })); + + const chunks = resources.flatMap((r) => + r.chunks.map((c) => ({ + path: c.path, + description: c.description, + })), + ); + + const payload = { resources, total: resources.length }; + this.logger.debug( + `listCatalog: ${resources.length} document(s), ${chunks.length} chunk(s)`, + ); + + return { + raw: { source: 'db', ...payload }, + texts: [JSON.stringify(payload)], + resourceLinks: [], + embeddedResources: [], + filteredResources: [], + resources, + chunks, + total: resources.length, + }; + } + + /** + * Load chunk markdown bodies for selected paths (1 query). + */ + async getContentsByPaths(paths: string[]): Promise { + const normalized = paths + .map((p) => this.stripKnownExtension(p)) + .filter((p) => p.length > 0); + if (normalized.length === 0) return []; + + const rows = await this.retrievalRepo.findChunkContentsByPaths(normalized); + const byPath = new Map(rows.map((r) => [r.path, r.content])); + + const hits: ChunkContentHit[] = []; + for (const path of normalized) { + const content = byPath.get(path); + if (content == null) { + this.logger.warn(`Chunk content not found for path=${path}`); + continue; + } + hits.push({ path, content }); + } + return hits; + } + + private stripKnownExtension(path: string): string { + if (!path.includes('.')) return path; + const lastDot = path.lastIndexOf('.'); + const extension = path.substring(lastDot + 1); + if (extension.length <= 5 && /^[a-z0-9]+$/i.test(extension)) { + return path.substring(0, lastDot); + } + return path; + } +} diff --git a/src/retrieval/retrieval.types.ts b/src/retrieval/retrieval.types.ts new file mode 100644 index 0000000..b1ce9bf --- /dev/null +++ b/src/retrieval/retrieval.types.ts @@ -0,0 +1,26 @@ +/** + * Document catalog shapes used by chat selection (formerly MCP list_resources). + */ + +export type ListResourceItem = { + path: string; + description: string; + chunks: Array<{ path: string; description: string }>; +}; + +export type ListResourcesResult = { + raw: unknown; + texts: string[]; + resourceLinks: unknown[]; + embeddedResources: unknown[]; + /** Legacy flat list (unused when catalog comes from DB) */ + filteredResources: Array<{ path: string; formats: string[] }>; + resources?: ListResourceItem[]; + chunks?: Array<{ path: string; description: string }>; + total?: number; +}; + +export type ChunkContentHit = { + path: string; + content: string; +}; From 927c2d501ca0974f1bbd3aebe5284caea21f1b0b Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 12:45:26 -0700 Subject: [PATCH 11/40] refactor(chat): use database retrieval for document selection --- src/chat/chat.module.ts | 3 +- .../prompts/resource-path-selection.prompt.ts | 2 +- .../chat-orchestration.service.spec.ts | 72 +++--- .../services/chat-orchestration.service.ts | 32 ++- .../services/resource-content.service.spec.ts | 33 ++- src/chat/services/resource-content.service.ts | 206 ++++++------------ .../resource-selection.service.spec.ts | 2 +- .../services/resource-selection.service.ts | 2 +- src/mcp/mcp-client.service.ts | 33 +-- 9 files changed, 141 insertions(+), 244 deletions(-) diff --git a/src/chat/chat.module.ts b/src/chat/chat.module.ts index 52d61b7..c051230 100644 --- a/src/chat/chat.module.ts +++ b/src/chat/chat.module.ts @@ -8,12 +8,13 @@ import { ResourceContentService } from './services/resource-content.service'; import { ChatStreamTransport } from './services/chat-stream.transport'; import { AuthModule } from '../auth/auth.module'; import { McpModule } from '../mcp/mcp.module'; +import { RetrievalModule } from '../retrieval/retrieval.module'; import { UsageModule } from '../usage/usage.module'; import { LLM_CLIENT } from './llm/llm-client.interface'; import { llmClientProvider } from './llm/llm-client.provider'; @Module({ - imports: [HttpModule, AuthModule, McpModule, UsageModule], + imports: [HttpModule, AuthModule, McpModule, RetrievalModule, UsageModule], controllers: [ChatController], providers: [ ChatService, diff --git a/src/chat/prompts/resource-path-selection.prompt.ts b/src/chat/prompts/resource-path-selection.prompt.ts index 47f554a..903dc1c 100644 --- a/src/chat/prompts/resource-path-selection.prompt.ts +++ b/src/chat/prompts/resource-path-selection.prompt.ts @@ -4,7 +4,7 @@ * - 신 형식: description + chunks → description 보고 chunk 경로 선택, JSON 배열 반환 */ -import type { ListResourceItem } from '../../mcp/mcp-client.service'; +import type { ListResourceItem } from '../../retrieval/retrieval.types'; export interface ResourcePathSelectionPromptParams { pathList: string; diff --git a/src/chat/services/chat-orchestration.service.spec.ts b/src/chat/services/chat-orchestration.service.spec.ts index e22bb7f..ee7f709 100644 --- a/src/chat/services/chat-orchestration.service.spec.ts +++ b/src/chat/services/chat-orchestration.service.spec.ts @@ -2,7 +2,7 @@ import { PassThrough } from 'node:stream'; import { describe, expect, it, jest } from '@jest/globals'; import { ChatOrchestrationService } from './chat-orchestration.service'; import { MessageRole } from '../../common/dto/chat-message-input.dto'; -import type { ListResourcesResult } from '../../mcp/mcp-client.service'; +import type { ListResourcesResult } from '../../retrieval/retrieval.types'; import type { LlmResponse } from '../types/llm.types'; import { ResourceContentService } from './resource-content.service'; import { ResourceSelectionService } from './resource-selection.service'; @@ -38,42 +38,35 @@ describe('ChatOrchestrationService', () => { texts: ['available school documents'], resourceLinks: [], embeddedResources: [], - filteredResources: [ - { path: '학사편람/졸업요건.md', formats: ['md'] }, - { path: '학사편람/수강신청.md', formats: ['md'] }, - { path: '학사편람.pdf', formats: ['pdf'] }, + filteredResources: [], + resources: [ + { + path: '학사편람.pdf', + description: '학사 안내', + chunks: [ + { path: '학사편람/졸업요건', description: '졸업요건' }, + { path: '학사편람/수강신청', description: '수강신청' }, + ], + }, ], + chunks: [ + { path: '학사편람/졸업요건', description: '졸업요건' }, + { path: '학사편람/수강신청', description: '수강신청' }, + ], + total: 1, }; - const mcpClientService = { - withSession: jest.fn(async (fn: () => Promise) => fn()), - callTool: jest.fn(async (name: string, args: { path?: string }) => { - if (name === 'list_resources') { - return listResult; - } - - if (name === 'get_resource' && args.path === '학사편람/졸업요건') { - return { - raw: {}, - texts: ['졸업요건 문서 본문입니다.'], - resourceLinks: [], - embeddedResources: [], - filteredResources: [], - }; - } - - if (name === 'get_resource' && args.path === '학사편람/수강신청') { - return { - raw: {}, - texts: ['수강신청 문서 본문입니다.'], - resourceLinks: [], - embeddedResources: [], - filteredResources: [], - }; - } - - throw new Error(`Unexpected tool call: ${name}`); - }), + const retrievalService = { + listCatalog: jest.fn(async () => listResult), + getContentsByPaths: jest.fn(async (paths: string[]) => + paths.map((path) => ({ + path, + content: + path.includes('졸업') + ? '졸업요건 문서 본문입니다.' + : '수강신청 문서 본문입니다.', + })), + ), }; type CallLLM = (...args: unknown[]) => Promise; type RecordUsage = ( @@ -85,7 +78,12 @@ describe('ChatOrchestrationService', () => { getModel: jest.fn((type: string) => `${type}-model`), callLLM: jest .fn() - .mockResolvedValueOnce(createLlmResponse('1, 2', 100)) + .mockResolvedValueOnce( + createLlmResponse( + JSON.stringify(['학사편람/졸업요건', '학사편람/수강신청']), + 100, + ), + ) .mockResolvedValueOnce(createLlmResponse('1', 200)), generateFinalResponseStream: jest.fn(async () => finalStream), }; @@ -105,7 +103,7 @@ describe('ChatOrchestrationService', () => { llmClient as never, ); const resourceContentService = new ResourceContentService( - mcpClientService as never, + retrievalService as never, resourceSelectionService, ); const chatStreamTransport = new ChatStreamTransport({ @@ -115,7 +113,7 @@ describe('ChatOrchestrationService', () => { } as never); const service = new ChatOrchestrationService( - mcpClientService as never, + retrievalService as never, llmClient as never, chatService as never, usageService as never, diff --git a/src/chat/services/chat-orchestration.service.ts b/src/chat/services/chat-orchestration.service.ts index d4b5941..4d6bed3 100644 --- a/src/chat/services/chat-orchestration.service.ts +++ b/src/chat/services/chat-orchestration.service.ts @@ -4,8 +4,6 @@ import { Logger, InternalServerErrorException, } from '@nestjs/common'; -import type { ListResourcesResult } from '../../mcp/mcp-client.service'; -import { McpClientService } from '../../mcp/mcp-client.service'; import { ChatService } from './chat.service'; import { UsageService } from '../../usage/usage.service'; import { LLM_CLIENT, type LlmClient } from '../llm/llm-client.interface'; @@ -22,6 +20,7 @@ import { type ResourceInfo, } from './resource-content.service'; import { ChatStreamTransport } from './chat-stream.transport'; +import { RetrievalService } from '../../retrieval/retrieval.service'; export type { ResourceInfo }; @@ -36,14 +35,14 @@ interface StreamingResponseOptions extends ProcessUserQuestionStreamOptions { /** * 채팅 오케스트레이션 서비스 - * 사용자 질문을 받아 LLM과 MCP Tool을 조합하여 답변을 생성합니다. + * 사용자 질문을 받아 DB Retrieval + LLM을 조합하여 답변을 생성합니다. */ @Injectable() export class ChatOrchestrationService { private readonly logger = new Logger(ChatOrchestrationService.name); constructor( - private readonly mcpClientService: McpClientService, + private readonly retrievalService: RetrievalService, @Inject(LLM_CLIENT) private readonly llmClient: LlmClient, private readonly chatService: ChatService, private readonly usageService: UsageService, @@ -118,12 +117,9 @@ export class ChatOrchestrationService { } t0 = Date.now(); - this.logger.debug('Calling list_resources...'); - const listResult = (await this.mcpClientService.callTool( - 'list_resources', - {}, - )) as ListResourcesResult; - this.logger.log(`[PERF] list_resources: ${Date.now() - t0}ms`); + this.logger.debug('Loading document catalog from DB...'); + const listResult = await this.retrievalService.listCatalog(); + this.logger.log(`[PERF] listCatalog: ${Date.now() - t0}ms`); const isNewFormat = listResult.resources && @@ -135,7 +131,7 @@ export class ChatOrchestrationService { : (listResult.filteredResources?.length ?? 0); const chunkCount = listResult.chunks?.length ?? 0; this.logger.log( - `[DEBUG] list_resources 결과: ${isNewFormat ? `신 형식 상위 ${listResult.resources?.length ?? 0}개, chunk ${chunkCount}개` : `구 형식 ${totalFromList}개 리소스`}`, + `[DEBUG] catalog 결과: ${isNewFormat ? `신 형식 상위 ${listResult.resources?.length ?? 0}개, chunk ${chunkCount}개` : `구 형식 ${totalFromList}개 리소스`}`, ); const hasResources = @@ -143,7 +139,7 @@ export class ChatOrchestrationService { (listResult.filteredResources && listResult.filteredResources.length > 0); if (!hasResources) { - this.logger.warn('No resources from list_resources'); + this.logger.warn('No resources from document catalog'); const stream = await this.llmClient.generateFinalResponseStream( [ { role: 'system', content: NO_RELEVANT_MATERIALS_SYSTEM_PROMPT }, @@ -203,7 +199,7 @@ export class ChatOrchestrationService { MAX_TOOL_CONTENT_CHARS - relevantPart.length - separator.length, ); const LIST_TRUNCATION_NOTE = - '\n\n[Truncated: list_resources preview too long]'; + '\n\n[Truncated: document catalog preview too long]'; let listPartForTool = listPart; let fullContentWasTruncated = false; if (listPart.length > maxListChars) { @@ -335,12 +331,10 @@ export class ChatOrchestrationService { stream, resources, usage: reasoningUsage, - } = await this.mcpClientService.withSession(() => - this.processUserQuestionStream(sessionId, userQuestion, { - persistUserMessage: options.persistUserMessage, - historyBefore: options.historyBefore, - }), - ); + } = await this.processUserQuestionStream(sessionId, userQuestion, { + persistUserMessage: options.persistUserMessage, + historyBefore: options.historyBefore, + }); let streamResult: { accumulatedContent: string; diff --git a/src/chat/services/resource-content.service.spec.ts b/src/chat/services/resource-content.service.spec.ts index 2a21d6e..c6de4ee 100644 --- a/src/chat/services/resource-content.service.spec.ts +++ b/src/chat/services/resource-content.service.spec.ts @@ -1,17 +1,12 @@ import { describe, expect, it, jest } from '@jest/globals'; import { ResourceContentService } from './resource-content.service'; -import type { ListResourcesResult } from '../../mcp/mcp-client.service'; +import type { ListResourcesResult } from '../../retrieval/retrieval.types'; import type { LlmUsage } from '../types/llm.types'; describe('ResourceContentService', () => { - type CallTool = ( - name: string, - args?: Record, - ) => Promise; - it('appends unique top-level PDF entries for FE resources', () => { const service = new ResourceContentService( - { callTool: jest.fn() } as never, + { getContentsByPaths: jest.fn() } as never, {} as never, ); const out: Array<{ path: string; formats: string[]; url: string }> = []; @@ -40,14 +35,12 @@ describe('ResourceContentService', () => { }); it('uses new-format chunk pipeline when resources+chunks exist', async () => { - const mcpClientService = { - callTool: jest.fn(async () => ({ - raw: {}, - texts: ['chunk body'], - resourceLinks: [], - embeddedResources: [], - filteredResources: [], - })), + const retrievalService = { + getContentsByPaths: jest.fn( + async (_paths: string[]) => [ + { path: '학사편람/졸업', content: 'chunk body' }, + ], + ), }; const resourceSelectionService = { selectRelevantChunkPaths: jest @@ -70,7 +63,7 @@ describe('ResourceContentService', () => { }; const service = new ResourceContentService( - mcpClientService as never, + retrievalService as never, resourceSelectionService as never, ); @@ -108,9 +101,9 @@ describe('ResourceContentService', () => { expect( resourceSelectionService.selectRelevantResourcePaths, ).not.toHaveBeenCalled(); - expect(mcpClientService.callTool).toHaveBeenCalledWith('get_resource', { - path: '학사편람/졸업', - }); + expect(retrievalService.getContentsByPaths).toHaveBeenCalledWith([ + '학사편람/졸업.md', + ]); expect(result.content).toContain('chunk body'); expect(result.usedResources.some((r) => r.path.includes('학사편람'))).toBe( true, @@ -119,7 +112,7 @@ describe('ResourceContentService', () => { it('returns empty when legacy filteredResources has no markdown', async () => { const service = new ResourceContentService( - { callTool: jest.fn() } as never, + { getContentsByPaths: jest.fn() } as never, { selectRelevantResourcePaths: jest.fn(), } as never, diff --git a/src/chat/services/resource-content.service.ts b/src/chat/services/resource-content.service.ts index 914245a..3a02a2c 100644 --- a/src/chat/services/resource-content.service.ts +++ b/src/chat/services/resource-content.service.ts @@ -2,8 +2,8 @@ import { Injectable, Logger } from '@nestjs/common'; import type { ListResourcesResult, ListResourceItem, -} from '../../mcp/mcp-client.service'; -import { McpClientService } from '../../mcp/mcp-client.service'; +} from '../../retrieval/retrieval.types'; +import { RetrievalService } from '../../retrieval/retrieval.service'; import { ResourceSelectionService } from './resource-selection.service'; import type { LlmUsage } from '../types/llm.types'; @@ -17,14 +17,14 @@ export interface ResourceInfo { } /** - * MCP 리소스 내용 fetch·파싱·FE 리소스 조립 + * DB Retrieval 기반 리소스 내용 fetch·파싱·FE 리소스 조립 */ @Injectable() export class ResourceContentService { private readonly logger = new Logger(ResourceContentService.name); constructor( - private readonly mcpClientService: McpClientService, + private readonly retrievalService: RetrievalService, private readonly resourceSelectionService: ResourceSelectionService, ) {} @@ -103,54 +103,6 @@ export class ResourceContentService { }); } - /** - * get_resource 툴 응답에서 텍스트 내용 추출 - * MCP 서버는 문자열을 직접 반환하므로, texts 배열이나 raw.content에서 추출 - */ - private extractContentFromToolResult( - toolResult: Awaited>, - ): string { - // texts 배열에서 내용 추출 (가장 일반적인 경우) - if (toolResult.texts.length > 0) { - // texts가 여러 개인 경우 합치기 - const content = toolResult.texts.join('\n'); - // JSON 문자열이 아닌 경우 그대로 반환 - if ( - content && - !content.trim().startsWith('{') && - !content.trim().startsWith('[') - ) { - return content; - } - } - - // raw.content에서 text 타입 항목 추출 - const raw = toolResult.raw as { - content?: Array<{ type: string; text?: string }>; - }; - if (raw?.content) { - const textContents: string[] = []; - for (const item of raw.content) { - if (item.type === 'text' && 'text' in item) { - const text = item.text; - // JSON 문자열이 아닌 경우 그대로 추가 - if ( - text && - !text.trim().startsWith('{') && - !text.trim().startsWith('[') - ) { - textContents.push(text); - } - } - } - if (textContents.length > 0) { - return textContents.join('\n'); - } - } - - return ''; - } - /** * 신 형식: LLM에게 description을 보고 관련 chunk 경로 최대 maxResults개 선택 (JSON 배열 반환) */ @@ -346,38 +298,30 @@ export class ResourceContentService { } /** - * 하위 문서 내용 가져오기 + * 하위 문서 내용 가져오기 (DB chunks) */ private async fetchSubDocumentContents( subDocuments: Array<{ path: string; description: string }>, ): Promise { - const results = await Promise.all( - subDocuments.map(async (doc) => { - try { - const resourcePath = this.normalizeResourcePath(doc.path); - this.logger.debug(`Fetching sub-document: ${resourcePath}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: resourcePath }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const documentTitle = this.extractDocumentTitle( - resourcePath, - doc.path, - ['md'], - ); - return `\n\n## 하위 문서: ${documentTitle}\n\n**설명**: ${doc.description}\n\n${content}`; - } - } catch (error) { - this.logger.warn( - `Failed to fetch sub-document ${doc.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return ''; - }), + const hits = await this.retrievalService.getContentsByPaths( + subDocuments.map((d) => d.path), ); - return results.filter(Boolean).join('\n'); + const byNormalized = new Map(hits.map((h) => [h.path, h.content])); + + const parts: string[] = []; + for (const doc of subDocuments) { + const content = byNormalized.get(this.normalizeResourcePath(doc.path)); + if (!content) continue; + const documentTitle = this.extractDocumentTitle( + this.normalizeResourcePath(doc.path), + doc.path, + ['md'], + ); + parts.push( + `\n\n## 하위 문서: ${documentTitle}\n\n**설명**: ${doc.description}\n\n${content}`, + ); + } + return parts.join('\n'); } /** @@ -412,33 +356,22 @@ export class ResourceContentService { } t0 = Date.now(); - const chunkResults = await Promise.all( - chunkPaths.map(async (chunkPath) => { - try { - const pathForTool = this.normalizeResourcePath(chunkPath); - this.logger.debug(`Fetching chunk: ${pathForTool}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: pathForTool }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const title = chunkPath.split('/').pop() || chunkPath || '문서'; - return { title, content, path: chunkPath }; - } - } catch (error) { - this.logger.warn( - `Failed to fetch chunk ${chunkPath}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return null; - }), - ); - const documentCandidates = chunkResults.filter( - (r): r is { title: string; content: string; path: string } => r !== null, - ); + const hits = await this.retrievalService.getContentsByPaths(chunkPaths); + const contentByPath = new Map(hits.map((h) => [h.path, h.content])); + const documentCandidates = chunkPaths + .map((chunkPath) => { + const content = contentByPath.get( + this.normalizeResourcePath(chunkPath), + ); + if (!content) return null; + const title = chunkPath.split('/').pop() || chunkPath || '문서'; + return { title, content, path: chunkPath }; + }) + .filter( + (r): r is { title: string; content: string; path: string } => r !== null, + ); this.logger.log( - `[PERF] get_resource 루프(신 형식, ${chunkPaths.length}개): ${Date.now() - t0}ms`, + `[PERF] getContentsByPaths(신 형식, ${chunkPaths.length}개): ${Date.now() - t0}ms`, ); if (documentCandidates.length === 0) { @@ -518,9 +451,9 @@ export class ResourceContentService { } /** - * list_resources tool 응답에서 관련 리소스 내용 가져오기 - * - 신 형식(resources + chunks): description 보고 chunk 경로 선별 → get_resource(chunk_path) - * - 구 형식(filteredResources): 경로만 선별 후 get_resource + * 문서 catalog에서 관련 리소스 내용 가져오기 + * - 신 형식(resources + chunks): description 보고 chunk 경로 선별 → DB content + * - 구 형식(filteredResources): 경로만 선별 후 DB content (dead path 가능) * @returns 문서 내용과 usedResources(선별 경로·formats; chunk는 md 포함). FE 참조 목록은 PDF/PNG만 노출. */ async fetchRelevantResourceContents( @@ -584,39 +517,30 @@ export class ResourceContentService { ); t0 = Date.now(); - const resourceResults = await Promise.all( - relevantResources.map(async (resource) => { - try { - const resourcePath = this.normalizeResourcePath(resource.path); - this.logger.debug(`Fetching markdown resource: ${resourcePath}`); - const toolResult = await this.mcpClientService.callTool( - 'get_resource', - { path: resourcePath }, - ); - const content = this.extractContentFromToolResult(toolResult); - if (content) { - const documentTitle = this.extractDocumentTitle( - resourcePath, - resource.path, - resource.formats, - ); - const subDocuments = this.parseDocumentLinks(content); - return { - title: documentTitle, - content, - path: resource.path, - formats: resource.formats || [], - subDocuments, - }; - } - } catch (error) { - this.logger.warn( - `Failed to fetch ${resource.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - return null; - }), + const hits = await this.retrievalService.getContentsByPaths( + relevantResources.map((r) => r.path), ); + const contentByPath = new Map(hits.map((h) => [h.path, h.content])); + const resourceResults = relevantResources.map((resource) => { + const content = contentByPath.get( + this.normalizeResourcePath(resource.path), + ); + if (!content) return null; + const resourcePath = this.normalizeResourcePath(resource.path); + const documentTitle = this.extractDocumentTitle( + resourcePath, + resource.path, + resource.formats, + ); + const subDocuments = this.parseDocumentLinks(content); + return { + title: documentTitle, + content, + path: resource.path, + formats: resource.formats || [], + subDocuments, + }; + }); const documentCandidates = resourceResults.filter( ( r, @@ -629,7 +553,7 @@ export class ResourceContentService { } => r !== null, ); this.logger.log( - `[PERF] get_resource 루프(구 형식, ${relevantResources.length}개): ${Date.now() - t0}ms`, + `[PERF] getContentsByPaths(구 형식, ${relevantResources.length}개): ${Date.now() - t0}ms`, ); if (documentCandidates.length === 0) { diff --git a/src/chat/services/resource-selection.service.spec.ts b/src/chat/services/resource-selection.service.spec.ts index 9e47ba4..cda826b 100644 --- a/src/chat/services/resource-selection.service.spec.ts +++ b/src/chat/services/resource-selection.service.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, it, jest } from '@jest/globals'; import { ResourceSelectionService } from './resource-selection.service'; import type { LlmResponse, LlmUsage } from '../types/llm.types'; -import type { ListResourceItem } from '../../mcp/mcp-client.service'; +import type { ListResourceItem } from '../../retrieval/retrieval.types'; function createLlmResponse(content: string, totalTokens = 10): LlmResponse { return { diff --git a/src/chat/services/resource-selection.service.ts b/src/chat/services/resource-selection.service.ts index bc79624..de8e9b1 100644 --- a/src/chat/services/resource-selection.service.ts +++ b/src/chat/services/resource-selection.service.ts @@ -1,5 +1,5 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; -import type { ListResourceItem } from '../../mcp/mcp-client.service'; +import type { ListResourceItem } from '../../retrieval/retrieval.types'; import { LLM_CLIENT, type LlmClient } from '../llm/llm-client.interface'; import type { LlmUsage } from '../types/llm.types'; import { diff --git a/src/mcp/mcp-client.service.ts b/src/mcp/mcp-client.service.ts index 9a51245..7bd71dc 100644 --- a/src/mcp/mcp-client.service.ts +++ b/src/mcp/mcp-client.service.ts @@ -15,29 +15,13 @@ import type { ListToolsRequest, CallToolRequest, } from '@modelcontextprotocol/sdk/types.js'; +import type { + ListResourceItem, + ListResourcesResult, +} from '../retrieval/retrieval.types'; -/** list_resources 신 형식: 상위 리소스 (description + chunks) */ -export type ListResourceItem = { - path: string; - description: string; - chunks: Array<{ path: string; description: string }>; -}; - -/** list_resources 호출 결과 (캐시용) */ -export type ListResourcesResult = { - raw: unknown; - texts: string[]; - resourceLinks: unknown[]; - embeddedResources: unknown[]; - /** 구 형식: 플랫 리스트 (경로 + formats) */ - filteredResources: Array<{ path: string; formats: string[] }>; - /** 신 형식: 상위 리소스 목록 (path, description, chunks) */ - resources?: ListResourceItem[]; - /** 신 형식: 모든 chunk 평탄화 - LLM 선별용 */ - chunks?: Array<{ path: string; description: string }>; - /** 신 형식: 상위 리소스 개수 */ - total?: number; -}; +/** @deprecated Prefer `src/retrieval/retrieval.types` */ +export type { ListResourceItem, ListResourcesResult }; /** 한 턴(사용자 메시지 처리) 동안 재사용하는 MCP 연결 */ type McpSession = { @@ -78,7 +62,10 @@ export class McpClientService { * MCP 연결 생성 → fn 실행 → 반드시 연결 종료 */ private async runWithConnection( - fn: (client: Client, transport: StreamableHTTPClientTransport) => Promise, + fn: ( + client: Client, + transport: StreamableHTTPClientTransport, + ) => Promise, ): Promise { const baseUrl = this.getBaseUrl(); const client = new Client( From 62c2322637f8a858381d4b645a14ed3dea1c5f75 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 12:45:26 -0700 Subject: [PATCH 12/40] refactor(pdf-processor): stop updating legacy resource index --- src/pdf-processor/gcs-storage.service.spec.ts | 6 +- src/pdf-processor/gcs-storage.service.ts | 58 +------------------ src/pdf-processor/pdf-pipeline.service.ts | 10 +--- .../pdf-processor.worker.spec.ts | 1 - src/pdf-processor/pdf-processor.worker.ts | 24 ++------ 5 files changed, 15 insertions(+), 84 deletions(-) diff --git a/src/pdf-processor/gcs-storage.service.spec.ts b/src/pdf-processor/gcs-storage.service.spec.ts index 3a29258..f5bfdbf 100644 --- a/src/pdf-processor/gcs-storage.service.spec.ts +++ b/src/pdf-processor/gcs-storage.service.spec.ts @@ -13,12 +13,14 @@ describe('decodeServiceAccountCredentials', () => { type: 'service_account', project_id: 'test-project', client_email: 'storage@test-project.iam.gserviceaccount.com', - private_key: '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', + private_key: + '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', }), ), ).toEqual({ client_email: 'storage@test-project.iam.gserviceaccount.com', - private_key: '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', + private_key: + '-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n', }); }); diff --git a/src/pdf-processor/gcs-storage.service.ts b/src/pdf-processor/gcs-storage.service.ts index 0e6d1bd..1c06dd1 100644 --- a/src/pdf-processor/gcs-storage.service.ts +++ b/src/pdf-processor/gcs-storage.service.ts @@ -7,15 +7,12 @@ type GcsServiceAccountCredentials = { private_key: string; }; +/** Metadata shape produced by the PDF pipeline (kept for callers; not written to GCS index). */ export type ResourceIndexEntry = { description: string; chunks: { path: string; description: string }[]; }; -export type ResourceIndex = Record; - -const RESOURCES_INDEX_PATH = '_resources.json'; - export function decodeServiceAccountCredentials( encodedKey: string, ): GcsServiceAccountCredentials { @@ -102,33 +99,13 @@ export class GcsStorageService { * Upload processed documents map (path → markdown string). * Binary (image) entries are skipped in phase 1. */ - async uploadDocuments( - documents: Record, - ): Promise { + async uploadDocuments(documents: Record): Promise { for (const [path, content] of Object.entries(documents)) { await this.uploadMarkdown(path, content); this.logger.debug(`Uploaded: ${path}`); } } - async updateResourceIndex( - resourceName: string, - metadata: ResourceIndexEntry, - ): Promise { - const index = await this.readResourceIndex(); - index[resourceName] = metadata; - await this.writeResourceIndex(index); - this.logger.log(`Updated ${RESOURCES_INDEX_PATH} for: ${resourceName}`); - } - - async removeResourceIndexEntry(resourceName: string): Promise { - const index = await this.readResourceIndex(); - if (!(resourceName in index)) return; - delete index[resourceName]; - await this.writeResourceIndex(index); - this.logger.log(`Removed ${RESOURCES_INDEX_PATH} entry: ${resourceName}`); - } - /** * Delete PDF, root md, and prefix folder for a resource. */ @@ -139,7 +116,6 @@ export class GcsStorageService { ]; await this.deleteFiles(toDelete); - await this.removeResourceIndexEntry(resourceName); } /** @@ -148,15 +124,12 @@ export class GcsStorageService { async deleteProcessedArtifacts(resourceName: string): Promise { const toDelete = await this.getProcessedArtifactFiles(resourceName); await this.deleteFiles(toDelete); - await this.removeResourceIndexEntry(resourceName); } private async getProcessedArtifactFiles( resourceName: string, ): Promise { - const files: File[] = [ - this.bucket.file(`${resourceName}.md`), - ]; + const files: File[] = [this.bucket.file(`${resourceName}.md`)]; const [prefixFiles] = await this.bucket.getFiles({ prefix: `${resourceName}/`, @@ -178,29 +151,4 @@ export class GcsStorageService { }), ); } - - private async readResourceIndex(): Promise { - const file = this.bucket.file(RESOURCES_INDEX_PATH); - try { - const [exists] = await file.exists(); - if (!exists) return {}; - const [buf] = await file.download(); - return JSON.parse(buf.toString('utf-8')) as ResourceIndex; - } catch (error) { - this.logger.warn( - `Failed to read ${RESOURCES_INDEX_PATH}, starting empty: ${error instanceof Error ? error.message : String(error)}`, - ); - return {}; - } - } - - private async writeResourceIndex(index: ResourceIndex): Promise { - await this.bucket.file(RESOURCES_INDEX_PATH).save( - Buffer.from(JSON.stringify(index, null, 2), 'utf-8'), - { - contentType: 'application/json; charset=utf-8', - resumable: false, - }, - ); - } } diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index d9968e1..dbbce34 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -2,10 +2,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { PDF_PROCESSOR_PROMPT } from '../chat/prompts/pdf-processor'; import { PDF_CHUNKING_PROMPT } from '../chat/prompts/pdf-chunking-prompt'; -import { - LLM_CLIENT, - type LlmClient, -} from '../chat/llm/llm-client.interface'; +import { LLM_CLIENT, type LlmClient } from '../chat/llm/llm-client.interface'; import { PdfTextService } from './pdf-text.service'; import { parseChunksFromMarkdown } from './pdf-chunk-parser'; import type { ResourceIndexEntry } from './gcs-storage.service'; @@ -115,10 +112,7 @@ export class PdfPipelineService { const prompt = PDF_PROCESSOR_PROMPT.replaceAll('{filename}', filename) .replaceAll('{total_pages}', String(totalPages)) .replaceAll('{current_page}', String(currentPage)) - .replaceAll( - '{previous_context}', - previousContext || '없음 (첫 페이지)', - ); + .replaceAll('{previous_context}', previousContext || '없음 (첫 페이지)'); const userText = pageText.trim() ? `${prompt}\n\n페이지 텍스트:\n${pageText}` diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 8ebba82..0627948 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -41,7 +41,6 @@ function createWorker(completeProcessing: boolean) { const gcs = { downloadPdf: jest.fn(() => Promise.resolve(Buffer.from('%PDF-test'))), uploadDocuments: jest.fn(() => Promise.resolve()), - updateResourceIndex: jest.fn(() => Promise.resolve()), deleteProcessedArtifacts: jest.fn<(resourceName: string) => Promise>( () => Promise.resolve(), ), diff --git a/src/pdf-processor/pdf-processor.worker.ts b/src/pdf-processor/pdf-processor.worker.ts index fb35279..db75cda 100644 --- a/src/pdf-processor/pdf-processor.worker.ts +++ b/src/pdf-processor/pdf-processor.worker.ts @@ -32,9 +32,7 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { ) { this.concurrency = Math.max( 1, - Number( - this.configService.get('PDF_PROCESSOR_CONCURRENCY') ?? 1, - ), + Number(this.configService.get('PDF_PROCESSOR_CONCURRENCY') ?? 1), ); this.pollIntervalMs = Math.max( 500, @@ -47,9 +45,8 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { this.staleProcessingMs = Math.max( 60_000, Number( - this.configService.get( - 'PDF_PROCESSOR_STALE_PROCESSING_MS', - ) ?? 30 * 60 * 1000, + this.configService.get('PDF_PROCESSOR_STALE_PROCESSING_MS') ?? + 30 * 60 * 1000, ), ); } @@ -99,10 +96,7 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { if (this.stopped || this.tickInFlight) return; this.tickInFlight = true; try { - if ( - Date.now() - this.lastStaleCheckAt >= - this.staleCheckIntervalMs - ) { + if (Date.now() - this.lastStaleCheckAt >= this.staleCheckIntervalMs) { await this.requeueStale(); } @@ -149,7 +143,6 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { generatedArtifactsMayExist = true; await this.gcs.uploadDocuments(result.documents); - await this.gcs.updateResourceIndex(resourceName, result.metadata); const completed = await this.documentsRepo.completeProcessing( doc.id, processingToken, @@ -168,18 +161,13 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { `Processing complete: ${resourceName} (${Object.keys(result.documents).length} files, ${result.chunks.length} chunks)`, ); } catch (error) { - const message = - error instanceof Error ? error.message : String(error); + const message = error instanceof Error ? error.message : String(error); this.logger.error( `Processing failed id=${doc.id} name=${resourceName}: ${message}`, error instanceof Error ? error.stack : undefined, ); try { - await this.documentsRepo.markFailed( - doc.id, - processingToken, - message, - ); + await this.documentsRepo.markFailed(doc.id, processingToken, message); } catch (markError) { this.logger.error( `Failed to persist processing error id=${doc.id}: ${markError instanceof Error ? markError.message : String(markError)}`, From 4e9782695044e00e5a846bf9123a703366ec743b Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 12:45:26 -0700 Subject: [PATCH 13/40] fix(pdf-processor): rely on inferred nullable document types --- src/pdf-processor/documents.repository.ts | 28 ++++++----------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index b1be65d..cbfde0a 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -1,14 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import { - inArray, - notInArray, - sql, - eq, - and, - desc, - asc, - lt, -} from 'drizzle-orm'; +import { inArray, notInArray, sql, eq, and, desc, asc, lt } from 'drizzle-orm'; import { DB_CONNECTION, documents, documentChunks } from '../db'; import type { Database, Document, DocumentChunk } from '../db'; @@ -50,7 +41,7 @@ export class DocumentsRepository { return row; } - async markQueuedAfterUpload(id: string): Promise { + async markQueuedAfterUpload(id: string) { const [row] = await this.db .update(documents) .set({ @@ -73,7 +64,7 @@ export class DocumentsRepository { await this.db.delete(documents).where(eq(documents.id, id)); } - async findById(id: string): Promise { + async findById(id: string) { const [row] = await this.db .select() .from(documents) @@ -82,9 +73,7 @@ export class DocumentsRepository { return row ?? null; } - async findActiveByResourceName( - resourceName: string, - ): Promise { + async findActiveByResourceName(resourceName: string) { const [row] = await this.db .select() .from(documents) @@ -255,7 +244,7 @@ export class DocumentsRepository { /** * Cancel the current attempt before deleting external artifacts. */ - async cancelAndSoftDelete(id: string): Promise { + async cancelAndSoftDelete(id: string) { const [row] = await this.db .update(documents) .set({ @@ -268,7 +257,7 @@ export class DocumentsRepository { return row ?? null; } - async enqueueReprocess(id: string): Promise { + async enqueueReprocess(id: string) { return this.db.transaction(async (tx) => { const [row] = await tx .update(documents) @@ -290,9 +279,7 @@ export class DocumentsRepository { .returning(); if (!row) return null; - await tx - .delete(documentChunks) - .where(eq(documentChunks.documentId, id)); + await tx.delete(documentChunks).where(eq(documentChunks.documentId, id)); return row; }); } @@ -304,5 +291,4 @@ export class DocumentsRepository { .where(eq(documentChunks.documentId, documentId)) .orderBy(asc(documentChunks.sortOrder)); } - } From 19a0b721bc40b05513a25a285673eed0fcfec601 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 12:53:05 -0700 Subject: [PATCH 14/40] fix(env.validation): update GCS bucket and GCP project ID for testing environment --- src/config/env.validation.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts index d11dc23..aec166c 100644 --- a/src/config/env.validation.spec.ts +++ b/src/config/env.validation.spec.ts @@ -21,8 +21,8 @@ function baseEnv(overrides: Record = {}) { DOMAIN_NAME: 'example.com', MCP_BASE_URL: 'https://mcp.example.com', MCP_RESOURCE_API_URL: 'https://mcp-resource.example.com', - GCS_BUCKET: 'ziggle-resources', - GCP_PROJECT_ID: 'ziggle-mcp-project', + GCS_BUCKET: 'test-bucket', + GCP_PROJECT_ID: 'test-project', ...overrides, }; } From a36597b6724627f5698266572328a1ffa3224daf Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 13:36:00 -0700 Subject: [PATCH 15/40] feat(pdf-processor): enhance chunk parsing with relative path normalization and overview preservation --- src/chat/prompts/pdf-chunking-prompt.ts | 17 ++--- src/pdf-processor/pdf-chunk-parser.spec.ts | 73 +++++++++++++++++++- src/pdf-processor/pdf-chunk-parser.ts | 79 ++++++++++++++++++---- src/scripts/smoke-pdf-processor.ts | 22 +++++- 4 files changed, 163 insertions(+), 28 deletions(-) diff --git a/src/chat/prompts/pdf-chunking-prompt.ts b/src/chat/prompts/pdf-chunking-prompt.ts index cb6b4c9..31f9df9 100644 --- a/src/chat/prompts/pdf-chunking-prompt.ts +++ b/src/chat/prompts/pdf-chunking-prompt.ts @@ -35,13 +35,13 @@ export const PDF_CHUNKING_PROMPT = ` \`\`\` 4. **path 요구사항** (매우 중요): - - **일관된 base path 사용**: 모든 청크는 같은 base path로 시작 - - 파일명에서 base를 추출하여 사용 (예: "2025-캠프-발표자료-2일차" 또는 "student-handbook") - - 하위 경로로 섹션 구분 (예: "base/권익인권센터/이용-안내", "base/학생팀/무한도전-프로젝트") - - 계층 구조 유지 (예: "base/section/subsection") + - **파일명(stem)을 path에 넣지 마세요.** 서버가 파일명 기준으로 prefix를 붙입니다. + - 상대 path만 사용 (예: "권익인권센터/이용-안내", "학생팀/무한도전-프로젝트", "학사-일정") + - 계층이 필요하면 슬래시로 구분 (예: "section/subsection") - 영문 소문자, 한글, 하이픈(-), 슬래시(/) 사용 가능 - 공백은 하이픈으로 치환 - - 구체적이고 명확한 경로 (예: "base/g-surf/신청-자격", NOT "base/section-5") + - 구체적이고 명확한 경로 (예: "g-surf/신청-자격", NOT "section-5") + - 잘못된 예: "학생-편람/학사-일정" (파일 stem 중복), "학생-편람.pdf/학사-일정" 5. **description 요구사항** (매우 중요 — 검색 품질에 직결): - 이 description은 사용자의 질문과 매칭하기 위한 용도입니다 @@ -97,7 +97,7 @@ GIST 대학은 혁신적인 교육기관입니다. ## 소개 GIST 대학은 혁신적인 교육기관입니다. - + ## 학사 일정 ### 2025년 봄학기 - 개강: 3월 3일 @@ -109,7 +109,7 @@ GIST 대학은 혁신적인 교육기관입니다. ... - + ## 수강 신청 수강 신청은 매 학기 시작 전에... (매우 긴 상세 내용) @@ -119,8 +119,9 @@ GIST 대학은 혁신적인 교육기관입니다. 위 예시에서: - \`\`는 문서의 고수준 개요 (학생 편람이라는 것, 학사 생활 안내라는 것) - description은 사용자가 검색할 수 있는 동의어/키워드를 포함 (개강일, 시험 기간, 수강 정정 등) -- "소개" 섹션은 짧으므로 기본 문서에 포함 +- "소개" 섹션은 짧으므로 기본 문서에 포함 (태그 없이 유지) - "학사 일정"과 "수강 신청"은 세부 내용이므로 서브 문서로 분할 +- path에는 파일 stem을 넣지 않음 (\`학사-일정\`, NOT \`학생-편람/학사-일정\`) 이제 아래 문서를 청킹하세요: diff --git a/src/pdf-processor/pdf-chunk-parser.spec.ts b/src/pdf-processor/pdf-chunk-parser.spec.ts index 604af25..1edba82 100644 --- a/src/pdf-processor/pdf-chunk-parser.spec.ts +++ b/src/pdf-processor/pdf-chunk-parser.spec.ts @@ -1,13 +1,40 @@ import { describe, expect, it } from '@jest/globals'; -import { parseChunksFromMarkdown, toResourceName } from './pdf-chunk-parser'; +import { + parseChunksFromMarkdown, + toRelativeChunkPath, + toResourceName, +} from './pdf-chunk-parser'; + +describe('toRelativeChunkPath', () => { + it('keeps relative paths', () => { + expect(toRelativeChunkPath('섹션-a', '테스트')).toBe('섹션-a'); + expect(toRelativeChunkPath('a/b', '테스트')).toBe('a/b'); + }); + + it('strips a single baseName prefix', () => { + expect(toRelativeChunkPath('테스트/섹션-a', '테스트')).toBe('섹션-a'); + }); + + it('strips duplicated baseName prefixes', () => { + expect(toRelativeChunkPath('테스트/테스트/섹션-a', '테스트')).toBe( + '섹션-a', + ); + }); + + it('returns empty when path is only the baseName', () => { + expect(toRelativeChunkPath('테스트', '테스트')).toBe(''); + }); +}); describe('parseChunksFromMarkdown', () => { - it('parses summary and document tags', () => { + it('parses summary, preserves overview, and normalizes relative paths', () => { const input = ` 문서 요약 # 제목 +소개 문단입니다. + ## 섹션 A 내용 A @@ -25,12 +52,53 @@ describe('parseChunksFromMarkdown', () => { expect(metadata.description).toBe('문서 요약'); expect(metadata.chunks).toEqual([ + { path: '테스트', description: '문서 요약' }, { path: '테스트/섹션-a', description: '설명 A' }, { path: '테스트/섹션-b', description: '설명 B' }, ]); expect(documents['테스트/섹션-a.md']).toContain('내용 A'); expect(documents['테스트/섹션-b.md']).toContain('내용 B'); + expect(documents['테스트.md']).toContain('소개 문단입니다.'); + expect(documents['테스트.md']).toContain('path="테스트/섹션-a"'); + }); + + it('deduplicates baseName already present in LLM paths', () => { + const input = ` +요약 +# 개요 + +본문 A +본문 B +`; + const { documents, metadata } = parseChunksFromMarkdown( + input, + '테스트.pdf', + ); + + expect(metadata.chunks.map((c) => c.path)).toEqual([ + '테스트', + '테스트/섹션-a', + '테스트/섹션-b', + ]); + expect(documents['테스트/섹션-a.md']).toBe('본문 A'); + expect(documents['테스트/섹션-b.md']).toBe('본문 B'); + }); + + it('omits root chunk when there is no overview outside document tags', () => { + const input = ` +요약만 +본문 +`; + const { documents, metadata } = parseChunksFromMarkdown( + input, + '테스트.pdf', + ); + + expect(metadata.chunks).toEqual([ + { path: '테스트/섹션-a', description: 'A' }, + ]); expect(documents['테스트.md']).toContain('path="테스트/섹션-a"'); + expect(documents['테스트/섹션-a.md']).toBe('본문'); }); it('falls back to single md when no document tags', () => { @@ -39,6 +107,7 @@ describe('parseChunksFromMarkdown', () => { expect(metadata.description).toBe('요약만'); expect(metadata.chunks).toEqual([]); expect(documents['alone.md']).toContain('# 본문'); + expect(documents['alone.md']).not.toContain(''); }); }); diff --git a/src/pdf-processor/pdf-chunk-parser.ts b/src/pdf-processor/pdf-chunk-parser.ts index cedafbf..7043f64 100644 --- a/src/pdf-processor/pdf-chunk-parser.ts +++ b/src/pdf-processor/pdf-chunk-parser.ts @@ -13,20 +13,52 @@ export type ChunkParseResult = { }; }; +/** + * Strip leading baseName prefixes so LLM-relative and LLM-absolute paths + * both normalize to the same relative segment. + */ +export function toRelativeChunkPath(raw: string, baseName: string): string { + let p = raw.trim().replace(/^\/+|\/+$/g, ''); + const prefix = `${baseName}/`; + while (p === baseName || p.startsWith(prefix)) { + p = p === baseName ? '' : p.slice(prefix.length); + } + return p; +} + +function resourceStem(resourceName: string): string { + if ( + resourceName.includes('.') && + resourceName.toLowerCase().endsWith('.pdf') + ) { + return resourceName.slice(0, -4); + } + if (resourceName.includes('.')) { + return resourceName.replace(/\.[^.]+$/, ''); + } + return resourceName; +} + +function extractOverviewMarkdown(markdown: string): string { + return markdown + .replace(/(.+?)<\/summary>/gs, '') + .replace( + /.*?<\/document>/gs, + '', + ) + .trim(); +} + /** * Parse and tags from chunked markdown. - * Ported from worker.py `_parse_chunks_from_markdown`. + * Ported from worker.py `_parse_chunks_from_markdown`, with path + * de-duplication and overview body preserved as a root chunk. */ export function parseChunksFromMarkdown( markdown: string, resourceName: string, ): ChunkParseResult { - const baseName = - resourceName.includes('.') && resourceName.toLowerCase().endsWith('.pdf') - ? resourceName.slice(0, -4) - : resourceName.includes('.') - ? resourceName.replace(/\.[^.]+$/, '') - : resourceName; + const baseName = resourceStem(resourceName); const summaryMatch = markdown.match(/(.+?)<\/summary>/s); const summary = summaryMatch?.[1]?.trim() ?? ''; @@ -35,36 +67,53 @@ export function parseChunksFromMarkdown( /(.+?)<\/document>/gs; const chunks: ParsedChunk[] = []; for (const match of markdown.matchAll(chunkPattern)) { + const relative = toRelativeChunkPath(match[1], baseName); + if (!relative) continue; chunks.push({ - path: match[1], + path: relative, description: match[2], content: match[3].trim(), }); } if (chunks.length === 0) { + const overviewOnly = extractOverviewMarkdown(markdown); + const body = + overviewOnly || + markdown.replace(/(.+?)<\/summary>/gs, '').trim(); return { - documents: { [`${baseName}.md`]: markdown }, + documents: { [`${baseName}.md`]: body || markdown }, metadata: { description: summary, chunks: [] }, }; } const documents: Record = {}; - const mainDocParts: string[] = []; + const stubLinks: string[] = []; const chunkMetadata: { path: string; description: string }[] = []; for (const chunk of chunks) { - documents[`${baseName}/${chunk.path}.md`] = chunk.content; - mainDocParts.push( - ``, + const fullPath = `${baseName}/${chunk.path}`; + documents[`${fullPath}.md`] = chunk.content; + stubLinks.push( + ``, ); chunkMetadata.push({ - path: `${baseName}/${chunk.path}`, + path: fullPath, description: chunk.description, }); } - documents[`${baseName}.md`] = mainDocParts.join('\n\n'); + const overview = extractOverviewMarkdown(markdown); + if (overview) { + const rootContent = [overview, '', ...stubLinks].join('\n').trim(); + documents[`${baseName}.md`] = rootContent; + chunkMetadata.unshift({ + path: baseName, + description: summary || '문서 개요', + }); + } else { + documents[`${baseName}.md`] = stubLinks.join('\n\n'); + } return { documents, diff --git a/src/scripts/smoke-pdf-processor.ts b/src/scripts/smoke-pdf-processor.ts index bee78a4..57a5c59 100644 --- a/src/scripts/smoke-pdf-processor.ts +++ b/src/scripts/smoke-pdf-processor.ts @@ -6,7 +6,10 @@ import { readFileSync, writeFileSync, unlinkSync, existsSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; -import { parseChunksFromMarkdown, toResourceName } from '../pdf-processor/pdf-chunk-parser'; +import { + parseChunksFromMarkdown, + toResourceName, +} from '../pdf-processor/pdf-chunk-parser'; import { isLikelyMojibake, normalizeExtractedText, @@ -78,9 +81,9 @@ async function main() { } console.log('OK mojibake'); - // 3) chunk parser + // 3) chunk parser (relative path + overview preservation) const parsed = parseChunksFromMarkdown( - `요약\n본문`, + `요약\n\n# 개요\n\n소개입니다.\n\n본문`, 'doc.pdf', ); if (parsed.metadata.description !== '요약') { @@ -89,6 +92,19 @@ async function main() { if (parsed.documents['doc/a.md'] !== '본문') { throw new Error('chunk content parse failed'); } + if (!parsed.documents['doc.md']?.includes('소개입니다.')) { + throw new Error('overview content not preserved'); + } + if (parsed.metadata.chunks[0]?.path !== 'doc') { + throw new Error('root chunk missing from metadata'); + } + const deduped = parseChunksFromMarkdown( + `s\n# o\n\nx`, + 'doc.pdf', + ); + if (!deduped.documents['doc/a.md'] || deduped.documents['doc/doc/a.md']) { + throw new Error('baseName path dedupe failed'); + } console.log('OK chunk parser'); // 4) pdfjs text extract From d803286b5a38ca451404cd2e2ae32b1b127edc78 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 13:38:16 -0700 Subject: [PATCH 16/40] feat(upload): update API responses to include DocumentListItemDto and add ApiConsumes for file uploads --- src/upload/dto/document-list-item.dto.ts | 74 ++++++++++++++++++++++++ src/upload/upload.controller.ts | 45 +++++++++++--- src/upload/upload.service.ts | 27 +++------ 3 files changed, 117 insertions(+), 29 deletions(-) create mode 100644 src/upload/dto/document-list-item.dto.ts diff --git a/src/upload/dto/document-list-item.dto.ts b/src/upload/dto/document-list-item.dto.ts new file mode 100644 index 0000000..ceff2b4 --- /dev/null +++ b/src/upload/dto/document-list-item.dto.ts @@ -0,0 +1,74 @@ +import { ApiProperty } from '@nestjs/swagger'; +import type { DocumentStatus } from '../../db'; + +const DOCUMENT_STATUSES: DocumentStatus[] = [ + 'uploading', + 'queued', + 'processing', + 'ready', + 'failed', +]; + +export class DocumentListItemDto { + @ApiProperty({ + description: '문서 UUID', + example: '550e8400-e29b-41d4-a716-446655440000', + }) + id: string; + + @ApiProperty({ + description: '관리자가 입력한 문서 제목', + example: '2026년 학생 학사편람', + }) + title: string; + + @ApiProperty({ + description: '원본 PDF 파일명에서 확장자를 제거한 리소스 이름', + example: '2026년 학생 학사편람', + }) + resourceName: string; + + @ApiProperty({ + description: 'PDF 업로드 및 비동기 처리 상태', + enum: DOCUMENT_STATUSES, + example: 'ready', + }) + status: DocumentStatus; + + @ApiProperty({ + description: '처리 완료 후 생성된 문서 전체 요약', + nullable: true, + example: '학사 일정, 수강 신청 및 졸업 요건 안내', + }) + summary: string | null; + + @ApiProperty({ + description: 'GCS에 저장된 원본 PDF 경로', + example: 'gs://ziggle-resources/2026년 학생 학사편람.pdf', + }) + gcsPdfPath: string; + + @ApiProperty({ + description: '처리 실패 시 오류 메시지', + nullable: true, + example: null, + }) + errorMessage: string | null; + + @ApiProperty({ + description: '업로드 일시', + type: String, + format: 'date-time', + example: '2026-07-30T12:00:00.000Z', + }) + uploadedAt: Date; + + @ApiProperty({ + description: '처리 완료 일시', + type: String, + format: 'date-time', + nullable: true, + example: '2026-07-30T12:02:30.000Z', + }) + processedAt: Date | null; +} diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index fc19ee8..7ceb8b5 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -19,6 +19,7 @@ import { ApiParam, ApiBody, ApiQuery, + ApiConsumes, } from '@nestjs/swagger'; import type { FastifyRequest } from 'fastify'; import { UploadService, PDF_MIME } from './upload.service'; @@ -27,6 +28,7 @@ import { SuperAdminGuard } from '../auth/guards/super-admin.guard'; import { CurrentAdmin } from '../auth/decorators/current-admin.decorator'; import { AdminContext } from '../auth/context/admin-context.entity'; import { Readable } from 'stream'; +import { DocumentListItemDto } from './dto/document-list-item.dto'; async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { const chunks: Buffer[] = []; @@ -61,7 +63,13 @@ export class UploadController { type: Number, description: '건너뛸 개수 (페이지네이션)', }) - @ApiResponse({ status: 200, description: '성공' }) + @ApiResponse({ + status: 200, + description: '성공', + type: DocumentListItemDto, + isArray: true, + }) + @ApiResponse({ status: 400, description: '잘못된 limit 또는 offset' }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) async listMyUploads( @@ -92,12 +100,15 @@ export class UploadController { description: '업로드한 문서의 처리 상태를 조회합니다.', }) @ApiParam({ name: 'id', description: '문서 UUID' }) - @ApiResponse({ status: 200, description: '성공' }) + @ApiResponse({ + status: 200, + description: '성공', + type: DocumentListItemDto, + }) + @ApiResponse({ status: 401, description: '인증 실패' }) + @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ status: 404, description: '문서 없음' }) - async getOne( - @CurrentAdmin() admin: AdminContext, - @Param('id') id: string, - ) { + async getOne(@CurrentAdmin() admin: AdminContext, @Param('id') id: string) { return this.uploadService.getById(id, admin.uuid); } @@ -107,6 +118,7 @@ export class UploadController { description: 'PDF를 GCS에 저장하고 비동기 처리 큐에 등록합니다. 처리 완료를 기다리지 않으며 status=queued로 즉시 응답합니다.', }) + @ApiConsumes('multipart/form-data') @ApiBody({ schema: { type: 'object', @@ -117,14 +129,21 @@ export class UploadController { }, }, }) - @ApiResponse({ status: 201, description: '업로드 성공 (queued)' }) + @ApiResponse({ + status: 201, + description: '업로드 성공 (queued)', + type: DocumentListItemDto, + }) @ApiResponse({ status: 400, description: '잘못된 요청 (PDF 아님, 필드 누락 등)', }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) - @ApiResponse({ status: 409, description: '동일 resource_name 문서가 이미 존재' }) + @ApiResponse({ + status: 409, + description: '동일 resource_name 문서가 이미 존재', + }) async upload( @CurrentAdmin() admin: AdminContext, @Req() req: FastifyRequest, @@ -184,8 +203,15 @@ export class UploadController { '기존 청크를 비우고 status를 queued로 되돌려 워커가 다시 처리하도록 합니다.', }) @ApiParam({ name: 'id', description: '문서 UUID' }) - @ApiResponse({ status: 200, description: '재처리 큐 등록' }) + @ApiResponse({ + status: 200, + description: '재처리 큐 등록', + type: DocumentListItemDto, + }) + @ApiResponse({ status: 401, description: '인증 실패' }) + @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ status: 404, description: '문서 없음' }) + @ApiResponse({ status: 409, description: '문서 업로드가 아직 진행 중' }) async reprocess(@Param('id') id: string) { return this.uploadService.reprocess(id); } @@ -199,6 +225,7 @@ export class UploadController { }) @ApiParam({ name: 'id', description: '문서 UUID', type: String }) @ApiResponse({ status: 204, description: '삭제 성공' }) + @ApiResponse({ status: 400, description: 'GCS 산출물 삭제 실패' }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index f01f743..6aab54e 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -9,23 +9,12 @@ import { DocumentsRepository } from '../pdf-processor/documents.repository'; import { GcsStorageService } from '../pdf-processor/gcs-storage.service'; import { toResourceName } from '../pdf-processor/pdf-chunk-parser'; import type { Document } from '../db'; +import type { DocumentListItemDto } from './dto/document-list-item.dto'; const PDF_MIME = 'application/pdf'; const DEFAULT_LIMIT = 50; const MAX_LIMIT = 100; -export type DocumentListItem = { - id: string; - title: string; - resourceName: string; - status: Document['status']; - summary: string | null; - gcsPdfPath: string; - errorMessage: string | null; - uploadedAt: Date; - processedAt: Date | null; -}; - @Injectable() export class UploadService { private readonly logger = new Logger(UploadService.name); @@ -38,7 +27,7 @@ export class UploadService { async listMyUploads( idpUuid: string, options: { limit?: number; offset?: number } = {}, - ): Promise { + ): Promise { const limit = Math.min(options.limit ?? DEFAULT_LIMIT, MAX_LIMIT); const offset = Math.max(0, options.offset ?? 0); @@ -50,7 +39,7 @@ export class UploadService { return rows.map((row) => this.toListItem(row)); } - async getById(id: string, idpUuid: string): Promise { + async getById(id: string, idpUuid: string): Promise { const row = await this.documentsRepo.findById(id); if (!row || !row.isActive) { throw new NotFoundException(`Document not found: ${id}`); @@ -69,7 +58,7 @@ export class UploadService { filename: string, title: string, idpUuid: string, - ): Promise { + ): Promise { if (!fileBuffer?.length) { throw new BadRequestException('file is required'); } @@ -123,9 +112,7 @@ export class UploadService { ); } - this.logger.log( - `Upload queued: id=${record.id} resource=${resourceName}`, - ); + this.logger.log(`Upload queued: id=${record.id} resource=${resourceName}`); return this.toListItem(record); } @@ -155,7 +142,7 @@ export class UploadService { /** * Clear chunks and re-enqueue for processing. */ - async reprocess(id: string): Promise { + async reprocess(id: string): Promise { const row = await this.documentsRepo.findById(id); if (!row || !row.isActive) { throw new NotFoundException(`Document not found: ${id}`); @@ -173,7 +160,7 @@ export class UploadService { return this.toListItem(updated); } - private toListItem(row: Document): DocumentListItem { + private toListItem(row: Document): DocumentListItemDto { return { id: row.id, title: row.title, From 2d6abf60bdc49abd9f4500e7017ea76a191ba001 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 14:00:18 -0700 Subject: [PATCH 17/40] feat(upload): limit document reprocessing with cooldown --- drizzle/0011_odd_chimera.sql | 1 + drizzle/meta/0011_snapshot.json | 1345 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/db/schema.ts | 1 + src/pdf-processor/documents.repository.ts | 26 +- .../pdf-processor.worker.spec.ts | 1 + src/upload/dto/document-list-item.dto.ts | 24 + src/upload/upload.controller.ts | 17 +- src/upload/upload.service.spec.ts | 99 +- src/upload/upload.service.ts | 60 +- 10 files changed, 1561 insertions(+), 20 deletions(-) create mode 100644 drizzle/0011_odd_chimera.sql create mode 100644 drizzle/meta/0011_snapshot.json diff --git a/drizzle/0011_odd_chimera.sql b/drizzle/0011_odd_chimera.sql new file mode 100644 index 0000000..05ccb99 --- /dev/null +++ b/drizzle/0011_odd_chimera.sql @@ -0,0 +1 @@ +ALTER TABLE "documents" ADD COLUMN "last_reprocessed_at" timestamp; \ No newline at end of file diff --git a/drizzle/meta/0011_snapshot.json b/drizzle/meta/0011_snapshot.json new file mode 100644 index 0000000..86e9956 --- /dev/null +++ b/drizzle/meta/0011_snapshot.json @@ -0,0 +1,1345 @@ +{ + "id": "80e30baa-2370-479e-9d1e-8e820aabc885", + "prevId": "19907c5f-6419-4ba3-bc5b-8d98425ba12b", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_reprocessed_at": { + "name": "last_reprocessed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 45dcdb2..75b0779 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1785357206423, "tag": "0010_gray_the_order", "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1785444498795, + "tag": "0011_odd_chimera", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index 63f9a72..c3efa4b 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -216,6 +216,7 @@ export const documents = pgTable( createdAt: timestamp('created_at').notNull().defaultNow(), updatedAt: timestamp('updated_at').notNull().defaultNow(), processedAt: timestamp('processed_at'), + lastReprocessedAt: timestamp('last_reprocessed_at'), }, (table) => ({ resourceNameActiveUnique: uniqueIndex( diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index cbfde0a..d661467 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -1,5 +1,17 @@ import { Inject, Injectable } from '@nestjs/common'; -import { inArray, notInArray, sql, eq, and, desc, asc, lt } from 'drizzle-orm'; +import { + inArray, + notInArray, + sql, + eq, + and, + or, + isNull, + lte, + desc, + asc, + lt, +} from 'drizzle-orm'; import { DB_CONNECTION, documents, documentChunks } from '../db'; import type { Database, Document, DocumentChunk } from '../db'; @@ -257,7 +269,7 @@ export class DocumentsRepository { return row ?? null; } - async enqueueReprocess(id: string) { + async enqueueReprocess(id: string, cooldownBefore: Date, now: Date) { return this.db.transaction(async (tx) => { const [row] = await tx .update(documents) @@ -266,14 +278,18 @@ export class DocumentsRepository { errorMessage: null, processingToken: null, processedAt: null, - updatedAt: new Date(), + lastReprocessedAt: now, + updatedAt: now, }) .where( and( eq(documents.id, id), eq(documents.isActive, true), - // An upload has no complete source in GCS yet. - sql`${documents.status} <> 'uploading'`, + inArray(documents.status, ['ready', 'failed']), + or( + isNull(documents.lastReprocessedAt), + lte(documents.lastReprocessedAt, cooldownBefore), + ), ), ) .returning(); diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 0627948..77491b7 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -21,6 +21,7 @@ function processingDocument(): Document { createdAt: new Date(), updatedAt: new Date(), processedAt: null, + lastReprocessedAt: null, }; } diff --git a/src/upload/dto/document-list-item.dto.ts b/src/upload/dto/document-list-item.dto.ts index ceff2b4..421ab0b 100644 --- a/src/upload/dto/document-list-item.dto.ts +++ b/src/upload/dto/document-list-item.dto.ts @@ -71,4 +71,28 @@ export class DocumentListItemDto { example: '2026-07-30T12:02:30.000Z', }) processedAt: Date | null; + + @ApiProperty({ + description: '마지막 재처리 요청 일시', + type: String, + format: 'date-time', + nullable: true, + example: '2026-07-30T13:00:00.000Z', + }) + lastReprocessedAt: Date | null; + + @ApiProperty({ + description: '24시간 쿨다운 기준 다음 재처리 가능 일시', + type: String, + format: 'date-time', + nullable: true, + example: '2026-07-31T13:00:00.000Z', + }) + reprocessAvailableAt: Date | null; + + @ApiProperty({ + description: '현재 상태와 쿨다운을 반영한 재처리 가능 여부', + example: true, + }) + canReprocess: boolean; } diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index 7ceb8b5..c87e021 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -211,7 +211,22 @@ export class UploadController { @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ status: 404, description: '문서 없음' }) - @ApiResponse({ status: 409, description: '문서 업로드가 아직 진행 중' }) + @ApiResponse({ + status: 409, + description: '현재 문서 상태가 재처리를 허용하지 않음', + }) + @ApiResponse({ + status: 429, + description: '문서별 24시간 재처리 쿨다운 적용 중', + schema: { + example: { + statusCode: 429, + message: 'Document reprocess cooldown is active', + error: 'Too Many Requests', + retryAt: '2026-07-31T13:00:00.000Z', + }, + }, + }) async reprocess(@Param('id') id: string) { return this.uploadService.reprocess(id); } diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts index a872f82..7876ddb 100644 --- a/src/upload/upload.service.spec.ts +++ b/src/upload/upload.service.spec.ts @@ -20,18 +20,26 @@ function document(overrides: Partial = {}): Document { createdAt: new Date(), updatedAt: new Date(), processedAt: null, + lastReprocessedAt: null, ...overrides, }; } function createService() { const repo = { - createUploading: jest.fn< - (...args: unknown[]) => Promise - >(), + createUploading: jest.fn<(...args: unknown[]) => Promise>(), markQueuedAfterUpload: jest.fn(), hardDelete: jest.fn(), cancelAndSoftDelete: jest.fn(), + findById: jest.fn<(id: string) => Promise>(), + enqueueReprocess: + jest.fn< + ( + id: string, + cooldownBefore: Date, + now: Date, + ) => Promise + >(), }; const gcs = { toGsPath: jest.fn((path: string) => `gs://bucket/${path}`), @@ -83,12 +91,7 @@ describe('UploadService atomic transitions', () => { repo.createUploading.mockRejectedValue({ code: '23505' }); await expect( - service.upload( - Buffer.from('%PDF-test'), - 'test.pdf', - '테스트', - 'admin-1', - ), + service.upload(Buffer.from('%PDF-test'), 'test.pdf', '테스트', 'admin-1'), ).rejects.toBeInstanceOf(ConflictException); expect(gcs.uploadPdf).not.toHaveBeenCalled(); }); @@ -109,4 +112,82 @@ describe('UploadService atomic transitions', () => { expect(calls).toEqual(['cancel', 'delete-artifacts']); }); + + it.each(['uploading', 'queued', 'processing'] as const)( + 'rejects reprocess while status is %s', + async (status) => { + const { service, repo } = createService(); + repo.findById.mockResolvedValue(document({ status })); + + await expect( + service.reprocess('00000000-0000-0000-0000-000000000001'), + ).rejects.toBeInstanceOf(ConflictException); + expect(repo.enqueueReprocess).not.toHaveBeenCalled(); + }, + ); + + it('rejects reprocess during the 24-hour cooldown', async () => { + const { service, repo } = createService(); + repo.findById.mockResolvedValue( + document({ + status: 'ready', + lastReprocessedAt: new Date(Date.now() - 23 * 60 * 60 * 1000), + }), + ); + + try { + await service.reprocess('00000000-0000-0000-0000-000000000001'); + throw new Error('Expected reprocess to be rejected'); + } catch (error) { + expect(error).toEqual( + expect.objectContaining({ + status: 429, + response: expect.objectContaining({ + retryAt: expect.any(String), + }), + }), + ); + } + expect(repo.enqueueReprocess).not.toHaveBeenCalled(); + }); + + it('allows reprocess after the 24-hour cooldown', async () => { + const { service, repo } = createService(); + const current = document({ + status: 'ready', + lastReprocessedAt: new Date(Date.now() - 25 * 60 * 60 * 1000), + }); + repo.findById.mockResolvedValue(current); + repo.enqueueReprocess.mockResolvedValue( + document({ status: 'queued', lastReprocessedAt: new Date() }), + ); + + await expect(service.reprocess(current.id)).resolves.toEqual( + expect.objectContaining({ status: 'queued', canReprocess: false }), + ); + }); + + it.each(['ready', 'failed'] as const)( + 'atomically requeues a %s document', + async (status) => { + const { service, repo } = createService(); + const current = document({ status }); + const queued = document({ + status: 'queued', + lastReprocessedAt: new Date(), + }); + repo.findById.mockResolvedValue(current); + repo.enqueueReprocess.mockResolvedValue(queued); + + const result = await service.reprocess(current.id); + + expect(repo.enqueueReprocess).toHaveBeenCalledWith( + current.id, + expect.any(Date), + expect.any(Date), + ); + expect(result.status).toBe('queued'); + expect(result.canReprocess).toBe(false); + }, + ); }); diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index 6aab54e..68a05fc 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -4,6 +4,8 @@ import { NotFoundException, BadRequestException, ConflictException, + HttpException, + HttpStatus, } from '@nestjs/common'; import { DocumentsRepository } from '../pdf-processor/documents.repository'; import { GcsStorageService } from '../pdf-processor/gcs-storage.service'; @@ -14,6 +16,7 @@ import type { DocumentListItemDto } from './dto/document-list-item.dto'; const PDF_MIME = 'application/pdf'; const DEFAULT_LIMIT = 50; const MAX_LIMIT = 100; +export const REPROCESS_COOLDOWN_MS = 24 * 60 * 60 * 1000; @Injectable() export class UploadService { @@ -147,13 +150,24 @@ export class UploadService { if (!row || !row.isActive) { throw new NotFoundException(`Document not found: ${id}`); } - if (row.status === 'uploading') { - throw new ConflictException('Document upload is still in progress'); - } - const updated = await this.documentsRepo.enqueueReprocess(id); + const now = new Date(); + this.assertReprocessEligible(row, now); + + const cooldownBefore = new Date(now.getTime() - REPROCESS_COOLDOWN_MS); + const updated = await this.documentsRepo.enqueueReprocess( + id, + cooldownBefore, + now, + ); if (!updated) { - throw new NotFoundException(`Document not found: ${id}`); + // Re-read to classify a concurrent state transition accurately. + const latest = await this.documentsRepo.findById(id); + if (!latest || !latest.isActive) { + throw new NotFoundException(`Document not found: ${id}`); + } + this.assertReprocessEligible(latest, new Date()); + throw new ConflictException('Document reprocess state changed'); } this.logger.log(`Document requeued: id=${id}`); @@ -161,6 +175,15 @@ export class UploadService { } private toListItem(row: Document): DocumentListItemDto { + const reprocessAvailableAt = row.lastReprocessedAt + ? new Date(row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS) + : null; + const statusAllowsReprocess = + row.status === 'ready' || row.status === 'failed'; + const canReprocess = + statusAllowsReprocess && + (!reprocessAvailableAt || reprocessAvailableAt.getTime() <= Date.now()); + return { id: row.id, title: row.title, @@ -171,9 +194,36 @@ export class UploadService { errorMessage: row.errorMessage, uploadedAt: row.createdAt, processedAt: row.processedAt, + lastReprocessedAt: row.lastReprocessedAt, + reprocessAvailableAt, + canReprocess, }; } + private assertReprocessEligible(row: Document, now: Date): void { + if (row.status !== 'ready' && row.status !== 'failed') { + throw new ConflictException( + `Document cannot be reprocessed while status is "${row.status}"`, + ); + } + + if (!row.lastReprocessedAt) return; + const retryAt = new Date( + row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS, + ); + if (retryAt.getTime() <= now.getTime()) return; + + throw new HttpException( + { + statusCode: HttpStatus.TOO_MANY_REQUESTS, + message: 'Document reprocess cooldown is active', + error: 'Too Many Requests', + retryAt: retryAt.toISOString(), + }, + HttpStatus.TOO_MANY_REQUESTS, + ); + } + private async rollbackUpload( documentId: string, resourceName: string, From 9d492c03cf5b49f8b47ab2541ca7de56e270c1db Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 15:47:01 -0700 Subject: [PATCH 18/40] feat(documents): add document expiration support --- drizzle/0012_brainy_dagger.sql | 2 + drizzle/meta/0012_snapshot.json | 1366 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/db/schema.ts | 2 + src/pdf-processor/documents.repository.ts | 14 + .../pdf-processor.worker.spec.ts | 1 + src/retrieval/retrieval.repository.spec.ts | 20 + src/retrieval/retrieval.repository.ts | 27 +- src/upload/dto/document-list-item.dto.ts | 15 + src/upload/dto/update-expires-at.dto.ts | 15 + src/upload/upload.controller.ts | 42 +- src/upload/upload.service.spec.ts | 87 +- src/upload/upload.service.ts | 47 + 13 files changed, 1637 insertions(+), 8 deletions(-) create mode 100644 drizzle/0012_brainy_dagger.sql create mode 100644 drizzle/meta/0012_snapshot.json create mode 100644 src/retrieval/retrieval.repository.spec.ts create mode 100644 src/upload/dto/update-expires-at.dto.ts diff --git a/drizzle/0012_brainy_dagger.sql b/drizzle/0012_brainy_dagger.sql new file mode 100644 index 0000000..4aec84c --- /dev/null +++ b/drizzle/0012_brainy_dagger.sql @@ -0,0 +1,2 @@ +ALTER TABLE "documents" ADD COLUMN "expires_at" timestamp;--> statement-breakpoint +CREATE INDEX "documents_expires_at_idx" ON "documents" USING btree ("expires_at"); \ No newline at end of file diff --git a/drizzle/meta/0012_snapshot.json b/drizzle/meta/0012_snapshot.json new file mode 100644 index 0000000..35114d6 --- /dev/null +++ b/drizzle/meta/0012_snapshot.json @@ -0,0 +1,1366 @@ +{ + "id": "3c7f053c-f6f3-405d-a29d-82f19edb33aa", + "prevId": "80e30baa-2370-479e-9d1e-8e820aabc885", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_reprocessed_at": { + "name": "last_reprocessed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_expires_at_idx": { + "name": "documents_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 75b0779..fc167f7 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -85,6 +85,13 @@ "when": 1785444498795, "tag": "0011_odd_chimera", "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1785445409151, + "tag": "0012_brainy_dagger", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index c3efa4b..7e70fc6 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -217,6 +217,7 @@ export const documents = pgTable( updatedAt: timestamp('updated_at').notNull().defaultNow(), processedAt: timestamp('processed_at'), lastReprocessedAt: timestamp('last_reprocessed_at'), + expiresAt: timestamp('expires_at'), }, (table) => ({ resourceNameActiveUnique: uniqueIndex( @@ -230,6 +231,7 @@ export const documents = pgTable( ), isActiveIdx: index('documents_is_active_idx').on(table.isActive), createdAtIdx: index('documents_created_at_idx').on(table.createdAt), + expiresAtIdx: index('documents_expires_at_idx').on(table.expiresAt), }), ); diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index d661467..b98f375 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -20,6 +20,7 @@ export type CreateDocumentInput = { resourceName: string; gcsPdfPath: string; uploadedByIdpUuid: string; + expiresAt?: Date | null; }; export type ReplaceChunksInput = { @@ -45,6 +46,7 @@ export class DocumentsRepository { resourceName: input.resourceName, gcsPdfPath: input.gcsPdfPath, uploadedByIdpUuid: input.uploadedByIdpUuid, + expiresAt: input.expiresAt ?? null, status: 'uploading', isActive: true, }) @@ -53,6 +55,18 @@ export class DocumentsRepository { return row; } + async updateExpiresAt(id: string, expiresAt: Date | null) { + const [row] = await this.db + .update(documents) + .set({ + expiresAt, + updatedAt: new Date(), + }) + .where(and(eq(documents.id, id), eq(documents.isActive, true))) + .returning(); + return row ?? null; + } + async markQueuedAfterUpload(id: string) { const [row] = await this.db .update(documents) diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 77491b7..527d5df 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -22,6 +22,7 @@ function processingDocument(): Document { updatedAt: new Date(), processedAt: null, lastReprocessedAt: null, + expiresAt: null, }; } diff --git a/src/retrieval/retrieval.repository.spec.ts b/src/retrieval/retrieval.repository.spec.ts new file mode 100644 index 0000000..0689609 --- /dev/null +++ b/src/retrieval/retrieval.repository.spec.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from '@jest/globals'; +import { isExpiredAt } from './retrieval.repository'; + +describe('isExpiredAt', () => { + const now = new Date('2026-07-30T12:00:00.000Z'); + + it('treats null as never expired', () => { + expect(isExpiredAt(null, now)).toBe(false); + expect(isExpiredAt(undefined, now)).toBe(false); + }); + + it('treats future expiresAt as not expired', () => { + expect(isExpiredAt(new Date('2026-07-30T12:00:01.000Z'), now)).toBe(false); + }); + + it('treats expiresAt at or before now as expired', () => { + expect(isExpiredAt(new Date('2026-07-30T12:00:00.000Z'), now)).toBe(true); + expect(isExpiredAt(new Date('2026-07-30T11:59:59.000Z'), now)).toBe(true); + }); +}); diff --git a/src/retrieval/retrieval.repository.ts b/src/retrieval/retrieval.repository.ts index 70f22bb..b63f493 100644 --- a/src/retrieval/retrieval.repository.ts +++ b/src/retrieval/retrieval.repository.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import { and, asc, eq, inArray } from 'drizzle-orm'; +import { and, asc, eq, gt, inArray, isNull, or, SQL } from 'drizzle-orm'; import { DB_CONNECTION, documents, documentChunks } from '../db'; import type { Database } from '../db'; @@ -16,12 +16,26 @@ export type ReadyDocumentWithChunks = { }>; }; +/** + * Chat catalog/content eligibility: null expiresAt = never expires. + */ +export function notExpiredCondition(now: Date = new Date()): SQL | undefined { + return or(isNull(documents.expiresAt), gt(documents.expiresAt, now)); +} + +export function isExpiredAt( + expiresAt: Date | null | undefined, + now: Date = new Date(), +): boolean { + return expiresAt != null && expiresAt.getTime() <= now.getTime(); +} + @Injectable() export class RetrievalRepository { constructor(@Inject(DB_CONNECTION) private readonly db: Database) {} /** - * Ready + active documents that have at least one chunk. + * Ready + active + not-expired documents that have at least one chunk. */ async listReadyWithChunks(): Promise { const rows = await this.db @@ -38,7 +52,13 @@ export class RetrievalRepository { }) .from(documents) .innerJoin(documentChunks, eq(documentChunks.documentId, documents.id)) - .where(and(eq(documents.status, 'ready'), eq(documents.isActive, true))) + .where( + and( + eq(documents.status, 'ready'), + eq(documents.isActive, true), + notExpiredCondition(), + ), + ) .orderBy(asc(documents.createdAt), asc(documentChunks.sortOrder)); const byId = new Map(); @@ -83,6 +103,7 @@ export class RetrievalRepository { inArray(documentChunks.path, uniquePaths), eq(documents.status, 'ready'), eq(documents.isActive, true), + notExpiredCondition(), ), ); diff --git a/src/upload/dto/document-list-item.dto.ts b/src/upload/dto/document-list-item.dto.ts index 421ab0b..d7eaec1 100644 --- a/src/upload/dto/document-list-item.dto.ts +++ b/src/upload/dto/document-list-item.dto.ts @@ -95,4 +95,19 @@ export class DocumentListItemDto { example: true, }) canReprocess: boolean; + + @ApiProperty({ + description: '문서 유효기간 (ISO-8601). null이면 무기한', + type: String, + format: 'date-time', + nullable: true, + example: '2026-12-31T23:59:59.000Z', + }) + expiresAt: Date | null; + + @ApiProperty({ + description: '현재 시각 기준 만료 여부 (만료되어도 soft-delete되지 않음)', + example: false, + }) + isExpired: boolean; } diff --git a/src/upload/dto/update-expires-at.dto.ts b/src/upload/dto/update-expires-at.dto.ts new file mode 100644 index 0000000..6c1ba18 --- /dev/null +++ b/src/upload/dto/update-expires-at.dto.ts @@ -0,0 +1,15 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsDateString, IsDefined, ValidateIf } from 'class-validator'; + +export class UpdateExpiresAtDto { + @ApiProperty({ + description: + '문서 유효기간 (ISO-8601). null이면 무기한. 과거 시각은 허용하지 않습니다.', + nullable: true, + example: '2026-12-31T23:59:59.000Z', + }) + @IsDefined() + @ValidateIf((_, value) => value !== null) + @IsDateString() + expiresAt: string | null; +} diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index c87e021..65248a5 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -1,7 +1,9 @@ import { + Body, Controller, Get, Post, + Patch, Delete, Param, Query, @@ -29,6 +31,7 @@ import { CurrentAdmin } from '../auth/decorators/current-admin.decorator'; import { AdminContext } from '../auth/context/admin-context.entity'; import { Readable } from 'stream'; import { DocumentListItemDto } from './dto/document-list-item.dto'; +import { UpdateExpiresAtDto } from './dto/update-expires-at.dto'; async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { const chunks: Buffer[] = []; @@ -126,6 +129,13 @@ export class UploadController { properties: { file: { type: 'string', format: 'binary', description: 'PDF 파일' }, title: { type: 'string', description: '파일 제목' }, + expiresAt: { + type: 'string', + format: 'date-time', + description: + '문서 유효기간 (ISO-8601, optional). 미전송/빈 값이면 무기한. 과거 시각은 400.', + nullable: true, + }, }, }, }) @@ -136,7 +146,7 @@ export class UploadController { }) @ApiResponse({ status: 400, - description: '잘못된 요청 (PDF 아님, 필드 누락 등)', + description: '잘못된 요청 (PDF 아님, 필드 누락, 과거 expiresAt 등)', }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @@ -158,6 +168,7 @@ export class UploadController { const parts = fastifyReq.parts(); let title = ''; + let expiresAt: string | undefined; let fileBuffer: Buffer | null = null; let filename = 'document.pdf'; let mimetype = ''; @@ -167,6 +178,9 @@ export class UploadController { if (part.fieldname === 'title') { const v = part.value; title = typeof v === 'string' ? v : ''; + } else if (part.fieldname === 'expiresAt') { + const v = part.value; + expiresAt = typeof v === 'string' ? v : undefined; } } else if (part.type === 'file' && part.fieldname === 'file') { const filePart = part; @@ -193,9 +207,35 @@ export class UploadController { filename, title.trim(), admin.uuid, + expiresAt, ); } + @Patch(':id') + @ApiOperation({ + summary: '문서 유효기간 변경', + description: + 'expiresAt을 ISO-8601로 연장/변경하거나 null로 무기한 전환합니다. 과거 시각은 허용하지 않습니다.', + }) + @ApiParam({ name: 'id', description: '문서 UUID' }) + @ApiBody({ type: UpdateExpiresAtDto }) + @ApiResponse({ + status: 200, + description: '유효기간 변경 성공', + type: DocumentListItemDto, + }) + @ApiResponse({ status: 400, description: '잘못된 expiresAt' }) + @ApiResponse({ status: 401, description: '인증 실패' }) + @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 404, description: '문서 없음' }) + async updateExpiresAt( + @CurrentAdmin() admin: AdminContext, + @Param('id') id: string, + @Body() body: UpdateExpiresAtDto, + ) { + return this.uploadService.updateExpiresAt(id, admin.uuid, body.expiresAt); + } + @Post(':id/reprocess') @ApiOperation({ summary: '문서 재처리', diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts index 7876ddb..3c6bcd2 100644 --- a/src/upload/upload.service.spec.ts +++ b/src/upload/upload.service.spec.ts @@ -1,9 +1,9 @@ -import { ConflictException } from '@nestjs/common'; +import { BadRequestException, ConflictException } from '@nestjs/common'; import { describe, expect, it, jest } from '@jest/globals'; import type { Document } from '../db'; import type { DocumentsRepository } from '../pdf-processor/documents.repository'; import type { GcsStorageService } from '../pdf-processor/gcs-storage.service'; -import { UploadService } from './upload.service'; +import { parseExpiresAt, UploadService } from './upload.service'; function document(overrides: Partial = {}): Document { return { @@ -21,6 +21,7 @@ function document(overrides: Partial = {}): Document { updatedAt: new Date(), processedAt: null, lastReprocessedAt: null, + expiresAt: null, ...overrides, }; } @@ -28,10 +29,13 @@ function document(overrides: Partial = {}): Document { function createService() { const repo = { createUploading: jest.fn<(...args: unknown[]) => Promise>(), - markQueuedAfterUpload: jest.fn(), + markQueuedAfterUpload: + jest.fn<(id: string) => Promise>(), hardDelete: jest.fn(), cancelAndSoftDelete: jest.fn(), findById: jest.fn<(id: string) => Promise>(), + updateExpiresAt: + jest.fn<(id: string, expiresAt: Date | null) => Promise>(), enqueueReprocess: jest.fn< ( @@ -43,7 +47,8 @@ function createService() { }; const gcs = { toGsPath: jest.fn((path: string) => `gs://bucket/${path}`), - uploadPdf: jest.fn(), + uploadPdf: + jest.fn<(resourceName: string, pdfBytes: Buffer) => Promise>(), deleteResourceArtifacts: jest.fn(), }; return { @@ -190,4 +195,78 @@ describe('UploadService atomic transitions', () => { expect(result.canReprocess).toBe(false); }, ); + + it('passes expiresAt into createUploading and returns isExpired=false', async () => { + const { service, repo, gcs } = createService(); + const future = new Date(Date.now() + 60_000); + const reserved = document({ expiresAt: future }); + const queued = document({ status: 'queued', expiresAt: future }); + repo.createUploading.mockResolvedValue(reserved); + gcs.uploadPdf.mockResolvedValue('gs://bucket/test.pdf'); + repo.markQueuedAfterUpload.mockResolvedValue(queued); + + const result = await service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + 'admin-1', + future.toISOString(), + ); + + expect(repo.createUploading).toHaveBeenCalledWith( + expect.objectContaining({ expiresAt: expect.any(Date) }), + ); + expect(result.expiresAt).toEqual(future); + expect(result.isExpired).toBe(false); + }); + + it('updates expiresAt for the owner and clears with null', async () => { + const { service, repo } = createService(); + const current = document({ status: 'ready' }); + const cleared = document({ status: 'ready', expiresAt: null }); + repo.findById.mockResolvedValue(current); + repo.updateExpiresAt.mockResolvedValue(cleared); + + const result = await service.updateExpiresAt(current.id, 'admin-1', null); + + expect(repo.updateExpiresAt).toHaveBeenCalledWith(current.id, null); + expect(result.expiresAt).toBeNull(); + expect(result.isExpired).toBe(false); + }); + + it('marks isExpired when expiresAt is in the past', async () => { + const { service, repo } = createService(); + const past = new Date(Date.now() - 60_000); + repo.findById.mockResolvedValue( + document({ status: 'ready', expiresAt: past }), + ); + + const result = await service.getById( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ); + expect(result.isExpired).toBe(true); + expect(result.expiresAt).toEqual(past); + }); +}); + +describe('parseExpiresAt', () => { + it('treats empty/undefined as null', () => { + expect(parseExpiresAt(undefined)).toBeNull(); + expect(parseExpiresAt(null)).toBeNull(); + expect(parseExpiresAt('')).toBeNull(); + expect(parseExpiresAt(' ')).toBeNull(); + }); + + it('rejects invalid and past values', () => { + expect(() => parseExpiresAt('not-a-date')).toThrow(BadRequestException); + expect(() => + parseExpiresAt(new Date(Date.now() - 1000).toISOString()), + ).toThrow(BadRequestException); + }); + + it('accepts future ISO-8601', () => { + const future = new Date(Date.now() + 60_000).toISOString(); + expect(parseExpiresAt(future)?.toISOString()).toBe(future); + }); }); diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index 68a05fc..6ad5cc1 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -10,6 +10,7 @@ import { import { DocumentsRepository } from '../pdf-processor/documents.repository'; import { GcsStorageService } from '../pdf-processor/gcs-storage.service'; import { toResourceName } from '../pdf-processor/pdf-chunk-parser'; +import { isExpiredAt } from '../retrieval/retrieval.repository'; import type { Document } from '../db'; import type { DocumentListItemDto } from './dto/document-list-item.dto'; @@ -18,6 +19,25 @@ const DEFAULT_LIMIT = 50; const MAX_LIMIT = 100; export const REPROCESS_COOLDOWN_MS = 24 * 60 * 60 * 1000; +/** + * Parse optional ISO-8601 expiresAt. Empty/undefined → null (never expires). + * Invalid or past timestamps → 400. + */ +export function parseExpiresAt(raw?: string | null): Date | null { + if (raw == null) return null; + const trimmed = raw.trim(); + if (!trimmed) return null; + + const parsed = new Date(trimmed); + if (Number.isNaN(parsed.getTime())) { + throw new BadRequestException('expiresAt must be a valid ISO-8601 datetime'); + } + if (parsed.getTime() <= Date.now()) { + throw new BadRequestException('expiresAt must be in the future'); + } + return parsed; +} + @Injectable() export class UploadService { private readonly logger = new Logger(UploadService.name); @@ -61,11 +81,13 @@ export class UploadService { filename: string, title: string, idpUuid: string, + expiresAtRaw?: string | null, ): Promise { if (!fileBuffer?.length) { throw new BadRequestException('file is required'); } + const expiresAt = parseExpiresAt(expiresAtRaw); const resourceName = toResourceName(filename || 'document.pdf'); if (!resourceName.trim()) { throw new BadRequestException('Invalid filename'); @@ -79,6 +101,7 @@ export class UploadService { resourceName, gcsPdfPath, uploadedByIdpUuid: idpUuid, + expiresAt, }); } catch (error) { if (isUniqueViolation(error)) { @@ -174,6 +197,28 @@ export class UploadService { return this.toListItem(updated); } + async updateExpiresAt( + id: string, + idpUuid: string, + expiresAtRaw: string | null, + ): Promise { + const row = await this.documentsRepo.findById(id); + if (!row || !row.isActive) { + throw new NotFoundException(`Document not found: ${id}`); + } + if (row.uploadedByIdpUuid !== idpUuid) { + throw new NotFoundException(`Document not found: ${id}`); + } + + const expiresAt = + expiresAtRaw === null ? null : parseExpiresAt(expiresAtRaw); + const updated = await this.documentsRepo.updateExpiresAt(id, expiresAt); + if (!updated) { + throw new NotFoundException(`Document not found: ${id}`); + } + return this.toListItem(updated); + } + private toListItem(row: Document): DocumentListItemDto { const reprocessAvailableAt = row.lastReprocessedAt ? new Date(row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS) @@ -197,6 +242,8 @@ export class UploadService { lastReprocessedAt: row.lastReprocessedAt, reprocessAvailableAt, canReprocess, + expiresAt: row.expiresAt, + isExpired: isExpiredAt(row.expiresAt), }; } From 6cb718963f53a388b66eec82a4697df0c2fdebf5 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 15:47:01 -0700 Subject: [PATCH 19/40] fix(pdf-processor): increase chunking output limit --- src/pdf-processor/pdf-pipeline.service.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index dbbce34..0205a40 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -157,10 +157,13 @@ export class PdfPipelineService { model, { temperature: 0.2, - max_tokens: 16000, + max_tokens: 48000, timeoutMs: this.llmTimeoutMs, }, ); + this.logger.log( + `Pass 2 LLM finish_reason=${response.choices?.[0]?.finish_reason ?? 'missing'}`, + ); const chunked = response.choices?.[0]?.message?.content ?? markdown; return parseChunksFromMarkdown(chunked, filename); } catch (error) { From a0f476d0ddbec0e7b1c9e5231d657efc657dc2b7 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 16:38:27 -0700 Subject: [PATCH 20/40] refactor(chat): update chunk selection logic and improve response formatting --- .../prompts/final-response.prompt.spec.ts | 14 +++ src/chat/prompts/final-response.prompt.ts | 28 +++-- .../prompts/resource-path-selection.prompt.ts | 80 +++++++++---- .../chat-orchestration.service.spec.ts | 25 ++-- .../services/resource-content.service.spec.ts | 29 +++-- src/chat/services/resource-content.service.ts | 107 +++++++----------- .../resource-selection.service.spec.ts | 49 ++++++-- .../services/resource-selection.service.ts | 57 +++++++--- 8 files changed, 243 insertions(+), 146 deletions(-) create mode 100644 src/chat/prompts/final-response.prompt.spec.ts diff --git a/src/chat/prompts/final-response.prompt.spec.ts b/src/chat/prompts/final-response.prompt.spec.ts new file mode 100644 index 0000000..98f80cb --- /dev/null +++ b/src/chat/prompts/final-response.prompt.spec.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from '@jest/globals'; +import { FINAL_RESPONSE_SYSTEM_PROMPT } from './final-response.prompt'; + +describe('FINAL_RESPONSE_SYSTEM_PROMPT', () => { + it('requires direct answers without exposing document retrieval', () => { + expect(FINAL_RESPONSE_SYSTEM_PROMPT).toContain( + '출처를 드러내지 않는 자연스러운 서술', + ); + expect(FINAL_RESPONSE_SYSTEM_PROMPT).toContain('"문서에 따르면"'); + expect(FINAL_RESPONSE_SYSTEM_PROMPT).toContain( + '사용자가 출처나 근거를 명시적으로 묻지 않았다면', + ); + }); +}); diff --git a/src/chat/prompts/final-response.prompt.ts b/src/chat/prompts/final-response.prompt.ts index 13ebfb7..5c08961 100644 --- a/src/chat/prompts/final-response.prompt.ts +++ b/src/chat/prompts/final-response.prompt.ts @@ -18,28 +18,36 @@ export const FINAL_RESPONSE_SYSTEM_PROMPT = ` - **금지:** 링크([]()), 인용(>), 표(|), 수평선(---), 기타 마크다운. ### 1. 리소스 내용 활용 - - 도구 결과에 리소스 내용이 포함되어 있으면 (## 리소스: [path] 또는 ## 하위 문서: [path] 형식), 반드시 그 내용을 기반으로 답변하세요. - - 제공된 리소스 내용을 직접 인용하거나 요약하여 사용자의 질문에 명확하게 답변하세요. - - 여러 리소스가 제공되면 모든 리소스의 정보를 종합하여 포괄적이고 완전한 답변을 제공하세요. - - ### 2. 대화 기록 활용 + - 도구 결과에 관련 정보가 포함되어 있으면 반드시 그 내용을 바탕으로 답변하세요. + - 여러 관련 정보가 제공되면 내용을 종합하여 포괄적이고 완전한 답변을 제공하세요. + - 관련 정보는 내부 지식처럼 자연스럽게 소화한 뒤, 사용자에게 필요한 사실과 절차를 직접 설명하세요. + - 사용자가 출처나 근거를 명시적으로 묻지 않았다면, 정보를 찾은 과정이나 출처의 존재를 드러내지 마세요. + + ### 2. 출처를 드러내지 않는 자연스러운 서술 (필수) + - "문서에 따르면", "문서에서는", "문서의 ~ 부분에서", "제공된 자료에는", "본문에 나와 있듯이", "확인 결과", "~라고 되어 있습니다", "~라고 안내되어 있습니다"처럼 출처를 중계하는 표현을 사용하지 마세요. + - 섹션 번호, 파일명, 리소스 경로를 답변의 근거처럼 언급하지 마세요. + - 원문 문장을 그대로 인용한 뒤 해설하는 방식보다, 내용을 자연스러운 설명으로 재구성하세요. + - 사용자가 이미 알고 있는 사실을 설명하듯 단정적이고 자연스럽게 말하되, 관련 정보에 없는 사실을 만들어내지는 마세요. + - 사용자가 출처·근거·원문 위치를 직접 요청한 경우에만 출처 정보를 간결하게 설명할 수 있습니다. + + ### 3. 대화 기록 활용 - 메시지에 **이전 대화**(사용자의 이전 질문과 당신의 이전 답변)가 포함되어 있습니다. - 사용자가 "방금 뭘 물어봤지?", "다시 설명해줘", "아까 말한 거 요약해줘", "그거 또 알려줘" 등 **이번 대화 안에서의 이전 질문·답변**을 묻는 경우, 반드시 대화 기록을 보고 답변하세요. - 이때 "자료가 없다", "확인할 수 없다"라고 하지 말고, 대화 기록에 있는 질문 내용과 답변 내용을 바탕으로 답변하세요. - ### 3. 답변 품질 + ### 4. 답변 품질 - 구체적이고 실용적인 정보를 제공하세요. - 불확실한 내용은 추측하지 말고, 제공된 리소스 내용에 기반해서만 답변하세요. - 답변은 명확하고 이해하기 쉽게 구성하세요. 소제목(## ###)과 문단으로 구분하고 필요시 번호나 불릿 포인트를 사용하새요. - ### 4. 금지 사항 및 "해당 문서 없음" 처리 (보안·품질) + ### 5. 금지 사항 및 관련 정보 없음 처리 (보안·품질) - 파일 경로만 나열하거나 "이 파일을 확인하세요" 같은 추상적인 답변을 하지 마세요. - 리소스 내용을 읽지 않고 답변하지 마세요. - 사용자에게 추가 정보를 요청하거나 질문을 되돌리지 마세요. 제공된 리소스로 최선을 다해 답변하세요. - - 도구 결과에 "## 리소스:" 또는 "## 하위 문서:" 형식의 **실제 문서 본문**이 없으면, 자신의 일반 지식(학습 데이터)으로 **절대** 답하지 마세요. 이 경우 "해당하는 문서를 찾지 못해 답변을 드리기 어렵습니다"라고만 정중히 안내하세요. - - 제공된 문서 내용에 질문에 대한 답이 **없으면**, 일반 지식으로 보충하지 말고 "제공된 문서에서 해당 내용을 찾을 수 없습니다"라고만 안내하세요. + - 도구 결과에 **실제 관련 정보 본문**이 없으면, 자신의 일반 지식(학습 데이터)으로 **절대** 답하지 마세요. 이 경우 "현재 확인 가능한 정보가 없어 답변드리기 어렵습니다"라고만 정중히 안내하세요. + - 관련 정보에 질문에 대한 답이 **없으면**, 일반 지식으로 보충하지 말고 "현재 확인 가능한 정보에는 해당 내용이 없습니다"라고만 안내하세요. - ### 5. 답변 형식 + ### 6. 답변 형식 - 한국어로 자연스럽고 정중하게 답변하세요. - 사용자의 질문에 직접적으로 답변하는 것으로 시작하세요. - 관련된 세부 정보는 그 다음에 제공하세요. diff --git a/src/chat/prompts/resource-path-selection.prompt.ts b/src/chat/prompts/resource-path-selection.prompt.ts index 903dc1c..7b38bb7 100644 --- a/src/chat/prompts/resource-path-selection.prompt.ts +++ b/src/chat/prompts/resource-path-selection.prompt.ts @@ -1,7 +1,7 @@ /** * 문서 경로(목록) 선별을 위한 프롬프트 * - 구 형식: 경로만 있는 플랫 리스트 → 번호로 선택 - * - 신 형식: description + chunks → description 보고 chunk 경로 선택, JSON 배열 반환 + * - 신 형식: description + chunks → 번호가 매겨진 세부 chunk를 선택 */ import type { ListResourceItem } from '../../retrieval/retrieval.types'; @@ -55,15 +55,26 @@ export interface ChunkSelectionPromptParams { maxSelect: number; } +export type ChunkSelectionCandidate = { + number: number; + path: string; + description: string; + resourcePath: string; + resourceDescription: string; + isRoot: boolean; + rootPath?: string; +}; + /** * chunk 선별 시스템 프롬프트 (신 형식: description 보고 관련 chunk 선택) */ export const CHUNK_SELECTION_SYSTEM_PROMPT = ` 당신은 사용자 질문과 리소스 설명(description)의 관련성을 판단하는 전문가입니다. -각 리소스의 description과 하위 chunk의 description을 보고, 질문에 답할 수 있을 **가능성이 있는** chunk를 넉넉히 선택하세요. +각 리소스와 하위 chunk의 description을 보고, 질문에 답할 수 있을 **가능성이 있는 세부 chunk**를 선택하세요. - 조금이라도 연관될 수 있다고 보이면 포함하세요. 의심스러우면 선택하는 편이 좋습니다. -- 선택한 chunk의 path만 JSON 배열로 반환합니다. 설명이나 다른 텍스트는 포함하지 마세요. +- 루트 개요는 서버가 별도로 추가하므로, 세부 chunk를 우선 선택하세요. +- 선택한 chunk의 번호만 JSON 배열로 반환합니다. 경로나 설명, 다른 텍스트는 포함하지 마세요. `; /** @@ -78,34 +89,61 @@ export function getChunkSelectionUserPrompt( 사용자 질문: "${question}" 아래 리소스 목록에서 질문에 답할 수 있을 **가능성이 있는** chunk를 선택하세요. -각 리소스의 description과 chunks의 description을 참고하여, **가능하면 5개 이상, 최대 ${maxSelect}개** 넉넉히 선택하세요. +각 리소스와 chunk의 description을 참고하여, 관련 세부 내용을 놓치지 않도록 **가능하면 3개 이상, 최대 ${maxSelect}개** 선택하세요. 리소스 목록: ${resourceListText} -선택한 chunk 경로만 JSON 배열로 반환하세요. 예: ["경로1", "경로2", "경로3"] +선택한 chunk 번호만 JSON 배열로 반환하세요. 예: [1, 3, 5] 정말로 단 하나도 관련이 없을 때만 빈 배열 []을 반환하세요. `; } /** - * 신 형식 list_resources 결과를 LLM에 넘길 텍스트로 포맷 + * 루트 개요가 있는 문서는 세부 chunk만 선택 후보로 노출합니다. + * 세부 chunk가 없는 문서는 검색 가능성을 유지하기 위해 루트 자체를 후보로 둡니다. */ -export function formatResourceListForChunkSelection( +export function buildChunkSelectionCandidates( resources: ListResourceItem[], +): ChunkSelectionCandidate[] { + const candidates: ChunkSelectionCandidate[] = []; + + for (const resource of resources) { + const resourceRootPath = resource.path.replace(/\.pdf$/i, ''); + const root = resource.chunks.find( + (chunk) => chunk.path === resourceRootPath, + ); + const details = resource.chunks.filter( + (chunk) => chunk.path !== resourceRootPath, + ); + const selectable = details.length > 0 ? details : root ? [root] : []; + + for (const chunk of selectable) { + candidates.push({ + number: candidates.length + 1, + path: chunk.path, + description: chunk.description, + resourcePath: resource.path, + resourceDescription: resource.description, + isRoot: chunk.path === resourceRootPath, + rootPath: root?.path, + }); + } + } + + return candidates; +} + +/** 신 형식 후보를 LLM이 복사할 필요 없는 번호 목록으로 포맷합니다. */ +export function formatChunkCandidatesForSelection( + candidates: ChunkSelectionCandidate[], ): string { - return resources - .map((r, i) => { - const chunkLines = (r.chunks || []) - .map( - (c) => ` - path: "${c.path}", description: "${c.description || ''}"`, - ) - .join('\n'); - return `[리소스 ${i + 1}] -path: "${r.path}" -description: "${r.description || ''}" -chunks: -${chunkLines}`; - }) - .join('\n\n'); + return candidates + .map( + (candidate) => + `${candidate.number}. [${candidate.isRoot ? '루트 문서' : '세부 chunk'}] ` + + `문서="${candidate.resourcePath}", 문서 설명="${candidate.resourceDescription}", ` + + `chunk 설명="${candidate.description}"`, + ) + .join('\n'); } diff --git a/src/chat/services/chat-orchestration.service.spec.ts b/src/chat/services/chat-orchestration.service.spec.ts index ee7f709..5c75f4f 100644 --- a/src/chat/services/chat-orchestration.service.spec.ts +++ b/src/chat/services/chat-orchestration.service.spec.ts @@ -61,10 +61,9 @@ describe('ChatOrchestrationService', () => { getContentsByPaths: jest.fn(async (paths: string[]) => paths.map((path) => ({ path, - content: - path.includes('졸업') - ? '졸업요건 문서 본문입니다.' - : '수강신청 문서 본문입니다.', + content: path.includes('졸업') + ? '졸업요건 문서 본문입니다.' + : '수강신청 문서 본문입니다.', })), ), }; @@ -78,13 +77,7 @@ describe('ChatOrchestrationService', () => { getModel: jest.fn((type: string) => `${type}-model`), callLLM: jest .fn() - .mockResolvedValueOnce( - createLlmResponse( - JSON.stringify(['학사편람/졸업요건', '학사편람/수강신청']), - 100, - ), - ) - .mockResolvedValueOnce(createLlmResponse('1', 200)), + .mockResolvedValueOnce(createLlmResponse(JSON.stringify([1, 2]), 100)), generateFinalResponseStream: jest.fn(async () => finalStream), }; const chatService = { @@ -163,9 +156,9 @@ describe('ChatOrchestrationService', () => { await handlePromise; - expect(llmClient.callLLM).toHaveBeenCalledTimes(2); + expect(llmClient.callLLM).toHaveBeenCalledTimes(1); expect(usageService.recordUsage).toHaveBeenCalledWith('session-id', { - totalTokens: 600, + totalTokens: 400, }); expect(chatService.createMessage).toHaveBeenCalledWith( 'session-id', @@ -175,9 +168,9 @@ describe('ChatOrchestrationService', () => { metadata: expect.objectContaining({ model: 'heavy-model', usage: { - prompt_tokens: 420, - completion_tokens: 180, - total_tokens: 600, + prompt_tokens: 280, + completion_tokens: 120, + total_tokens: 400, }, }), }), diff --git a/src/chat/services/resource-content.service.spec.ts b/src/chat/services/resource-content.service.spec.ts index c6de4ee..b3e5745 100644 --- a/src/chat/services/resource-content.service.spec.ts +++ b/src/chat/services/resource-content.service.spec.ts @@ -36,16 +36,22 @@ describe('ResourceContentService', () => { it('uses new-format chunk pipeline when resources+chunks exist', async () => { const retrievalService = { - getContentsByPaths: jest.fn( - async (_paths: string[]) => [ - { path: '학사편람/졸업', content: 'chunk body' }, - ], - ), + getContentsByPaths: jest.fn(async (_paths: string[]) => [ + { path: '학사편람', content: 'root overview' }, + { path: '학사편람/졸업', content: 'chunk body' }, + ]), }; const resourceSelectionService = { selectRelevantChunkPaths: jest - .fn<(...args: unknown[]) => Promise>() - .mockResolvedValue(['학사편람/졸업.md']), + .fn< + ( + ...args: unknown[] + ) => Promise<{ rootPaths: string[]; detailPaths: string[] }> + >() + .mockResolvedValue({ + rootPaths: ['학사편람'], + detailPaths: ['학사편람/졸업'], + }), selectMostRelevantDocuments: jest .fn< ( @@ -102,9 +108,16 @@ describe('ResourceContentService', () => { resourceSelectionService.selectRelevantResourcePaths, ).not.toHaveBeenCalled(); expect(retrievalService.getContentsByPaths).toHaveBeenCalledWith([ - '학사편람/졸업.md', + '학사편람', + '학사편람/졸업', ]); + expect( + resourceSelectionService.selectMostRelevantDocuments, + ).not.toHaveBeenCalled(); + expect(result.content).toContain('root overview'); expect(result.content).toContain('chunk body'); + expect(result.content).toContain('## 관련 정보'); + expect(result.content).not.toContain('## 리소스:'); expect(result.usedResources.some((r) => r.path.includes('학사편람'))).toBe( true, ); diff --git a/src/chat/services/resource-content.service.ts b/src/chat/services/resource-content.service.ts index 3a02a2c..205f4b6 100644 --- a/src/chat/services/resource-content.service.ts +++ b/src/chat/services/resource-content.service.ts @@ -312,14 +312,7 @@ export class ResourceContentService { for (const doc of subDocuments) { const content = byNormalized.get(this.normalizeResourcePath(doc.path)); if (!content) continue; - const documentTitle = this.extractDocumentTitle( - this.normalizeResourcePath(doc.path), - doc.path, - ['md'], - ); - parts.push( - `\n\n## 하위 문서: ${documentTitle}\n\n**설명**: ${doc.description}\n\n${content}`, - ); + parts.push(`\n\n## 관련 정보\n\n주제: ${doc.description}\n\n${content}`); } return parts.join('\n'); } @@ -341,16 +334,21 @@ export class ResourceContentService { ); let t0 = Date.now(); - const chunkPaths = await this.resourceSelectionService.selectRelevantChunkPaths( - question, - resources, - 10, - tokenUsage, - ); + const chunkSelection = + await this.resourceSelectionService.selectRelevantChunkPaths( + question, + resources, + 5, + tokenUsage, + ); this.logger.log( `[PERF] selectRelevantChunkPaths(LLM): ${Date.now() - t0}ms`, ); + const chunkPaths = [ + ...chunkSelection.rootPaths, + ...chunkSelection.detailPaths, + ]; if (chunkPaths.length === 0) { return { content: '', usedResources: [] }; } @@ -368,7 +366,8 @@ export class ResourceContentService { return { title, content, path: chunkPath }; }) .filter( - (r): r is { title: string; content: string; path: string } => r !== null, + (r): r is { title: string; content: string; path: string } => + r !== null, ); this.logger.log( `[PERF] getContentsByPaths(신 형식, ${chunkPaths.length}개): ${Date.now() - t0}ms`, @@ -379,48 +378,20 @@ export class ResourceContentService { } this.logger.log( - `[DEBUG] 2차 선별(본문 기준) 입력: 후보 문서 ${documentCandidates.length}개 → LLM에 전달`, - ); - - t0 = Date.now(); - const selectedDocuments = await this.resourceSelectionService.selectMostRelevantDocuments( - question, - documentCandidates.map((doc) => ({ - title: doc.title, - content: doc.content, - path: doc.path, - })), - tokenUsage, - ); - this.logger.log( - `[PERF] selectMostRelevantDocuments(LLM, 신 형식): ${Date.now() - t0}ms`, - ); - - this.logger.log( - `[DEBUG] 2차 선별 결과(최종 사용 문서): ${selectedDocuments.length}개`, + `[DEBUG] 최종 사용 문서: 루트 ${chunkSelection.rootPaths.length}개, 세부 chunk ${chunkSelection.detailPaths.length}개`, ); - if (selectedDocuments.length === 0) { - this.logger.log('No documents selected by LLM as relevant'); - return { content: '', usedResources: [] }; - } - const contents: string[] = []; const mdUsed: Array<{ path: string; formats: string[] }> = []; - for (const selected of selectedDocuments) { - const doc = documentCandidates.find((d) => d.path === selected.path); - if (doc) { - contents.push(`\n\n## 리소스: ${doc.title}\n\n${doc.content}`); - mdUsed.push({ path: doc.path, formats: ['md'] }); - } + for (const doc of documentCandidates) { + contents.push(`\n\n## 관련 정보\n\n${doc.content}`); + mdUsed.push({ path: doc.path, formats: ['md'] }); } - const selectedPaths = selectedDocuments.map((s) => s.path); + const selectedPaths = documentCandidates.map((doc) => doc.path); const fromMarkdown: Array<{ path: string; formats: string[] }> = []; - for (const selected of selectedDocuments) { - const doc = documentCandidates.find((d) => d.path === selected.path); - if (!doc?.content) continue; + for (const doc of documentCandidates) { fromMarkdown.push( ...this.extractPdfPngReferencesFromMarkdown(doc.content, doc.path), ); @@ -498,12 +469,13 @@ export class ResourceContentService { ); let t0 = Date.now(); - const relevantResources = await this.resourceSelectionService.selectRelevantResourcePaths( - question, - mdResources, - 10, - tokenUsage, - ); + const relevantResources = + await this.resourceSelectionService.selectRelevantResourcePaths( + question, + mdResources, + 10, + tokenUsage, + ); this.logger.log( `[PERF] selectRelevantResourcePaths(LLM, 구 형식): ${Date.now() - t0}ms`, ); @@ -565,15 +537,16 @@ export class ResourceContentService { ); t0 = Date.now(); - const selectedDocuments = await this.resourceSelectionService.selectMostRelevantDocuments( - question, - documentCandidates.map((doc) => ({ - title: doc.title, - content: doc.content, - path: doc.path, - })), - tokenUsage, - ); + const selectedDocuments = + await this.resourceSelectionService.selectMostRelevantDocuments( + question, + documentCandidates.map((doc) => ({ + title: doc.title, + content: doc.content, + path: doc.path, + })), + tokenUsage, + ); this.logger.log( `[PERF] selectMostRelevantDocuments(LLM, 구 형식): ${Date.now() - t0}ms`, ); @@ -596,9 +569,7 @@ export class ResourceContentService { (d) => d.title === selected.title, ); if (docCandidate) { - contents.push( - `\n\n## 리소스: ${docCandidate.title}\n\n${docCandidate.content}`, - ); + contents.push(`\n\n## 관련 정보\n\n${docCandidate.content}`); const hasPdf = docCandidate.formats.includes('pdf'); const hasPng = docCandidate.formats.includes('png'); @@ -688,7 +659,7 @@ export class ResourceContentService { `[PERF] fetchSubDocumentContents(${relevantSubDocuments.length}개): ${Date.now() - t0}ms`, ); if (subDocumentContents) { - contents.push('\n\n---\n\n## 관련 하위 문서\n' + subDocumentContents); + contents.push('\n\n---\n\n## 추가 관련 정보\n' + subDocumentContents); } } } diff --git a/src/chat/services/resource-selection.service.spec.ts b/src/chat/services/resource-selection.service.spec.ts index cda826b..8112879 100644 --- a/src/chat/services/resource-selection.service.spec.ts +++ b/src/chat/services/resource-selection.service.spec.ts @@ -43,16 +43,14 @@ describe('ResourceSelectionService', () => { await expect( service.selectRelevantChunkPaths('질문', [], 10), - ).resolves.toEqual([]); + ).resolves.toEqual({ rootPaths: [], detailPaths: [] }); expect(callLLM).not.toHaveBeenCalled(); }); - it('parses chunk paths from JSON and accumulates token usage', async () => { + it('maps selected numbers to detail paths and adds their root overview', async () => { const callLLM = jest .fn() - .mockResolvedValue( - createLlmResponse('```json\n["a/b.md", "c/d.md"]\n```', 100), - ); + .mockResolvedValue(createLlmResponse('```json\n[2, 1]\n```', 100)); const { service } = createService(callLLM); const usage: LlmUsage = { prompt_tokens: 0, @@ -62,23 +60,54 @@ describe('ResourceSelectionService', () => { const resources: ListResourceItem[] = [ { - path: 'root', - description: 'desc', - chunks: [{ path: 'a/b.md', description: 'b' }], + path: '학사편람.pdf', + description: '학사 안내', + chunks: [ + { path: '학사편람', description: '문서 개요' }, + { path: '학사편람/수강신청', description: '수강신청 방법' }, + { path: '학사편람/졸업', description: '졸업 요건' }, + ], }, ]; - const paths = await service.selectRelevantChunkPaths( + const selected = await service.selectRelevantChunkPaths( '질문', resources, 10, usage, ); - expect(paths).toEqual(['a/b.md', 'c/d.md']); + expect(selected).toEqual({ + rootPaths: ['학사편람'], + detailPaths: ['학사편람/졸업', '학사편람/수강신청'], + }); expect(usage.total_tokens).toBe(100); }); + it('ignores invalid and duplicate chunk numbers', async () => { + const callLLM = jest + .fn() + .mockResolvedValue(createLlmResponse('[1, 1, 99, "경로"]')); + const { service } = createService(callLLM); + const resources: ListResourceItem[] = [ + { + path: '학사편람.pdf', + description: '학사 안내', + chunks: [ + { path: '학사편람', description: '문서 개요' }, + { path: '학사편람/수강신청', description: '수강신청 방법' }, + ], + }, + ]; + + await expect( + service.selectRelevantChunkPaths('수강신청', resources, 5), + ).resolves.toEqual({ + rootPaths: ['학사편람'], + detailPaths: ['학사편람/수강신청'], + }); + }); + it('returns empty when path selection says 없음', async () => { const callLLM = jest .fn() diff --git a/src/chat/services/resource-selection.service.ts b/src/chat/services/resource-selection.service.ts index de8e9b1..f57b43f 100644 --- a/src/chat/services/resource-selection.service.ts +++ b/src/chat/services/resource-selection.service.ts @@ -9,9 +9,15 @@ import { getResourcePathSelectionUserPrompt, CHUNK_SELECTION_SYSTEM_PROMPT, getChunkSelectionUserPrompt, - formatResourceListForChunkSelection, + buildChunkSelectionCandidates, + formatChunkCandidatesForSelection, } from '../prompts'; +export type RelevantChunkSelection = { + rootPaths: string[]; + detailPaths: string[]; +}; + /** * LLM 기반 리소스/문서 선별 서비스 */ @@ -39,14 +45,19 @@ export class ResourceSelectionService { async selectRelevantChunkPaths( question: string, resources: ListResourceItem[], - maxResults: number = 10, + maxResults: number = 5, tokenUsage?: LlmUsage, - ): Promise { + ): Promise { if (!resources?.length) { - return []; + return { rootPaths: [], detailPaths: [] }; + } + + const candidates = buildChunkSelectionCandidates(resources); + if (candidates.length === 0) { + return { rootPaths: [], detailPaths: [] }; } - const resourceListText = formatResourceListForChunkSelection(resources); + const resourceListText = formatChunkCandidatesForSelection(candidates); const userPrompt = getChunkSelectionUserPrompt({ question, resourceListText, @@ -74,20 +85,40 @@ export class ResourceSelectionService { } const parsed = JSON.parse(selectedText) as unknown; - const paths = Array.isArray(parsed) - ? (parsed as string[]).filter( - (p) => typeof p === 'string' && p.length > 0, + const numbers = Array.isArray(parsed) + ? parsed.filter( + (value): value is number => + typeof value === 'number' && + Number.isInteger(value) && + value >= 1 && + value <= candidates.length, ) : []; - const limited = paths.slice(0, maxResults); - this.logger.log(`[DEBUG] 1차 선별 결과(chunk 경로): ${limited.length}개`); - return limited; + const selected = [...new Set(numbers)] + .slice(0, maxResults) + .map((number) => candidates[number - 1]); + const rootPaths = new Set(); + const detailPaths: string[] = []; + + for (const candidate of selected) { + if (candidate.isRoot) { + rootPaths.add(candidate.path); + } else { + detailPaths.push(candidate.path); + if (candidate.rootPath) rootPaths.add(candidate.rootPath); + } + } + + this.logger.log( + `[DEBUG] 1차 선별 결과: 루트 ${rootPaths.size}개, 세부 chunk ${detailPaths.length}개`, + ); + return { rootPaths: [...rootPaths], detailPaths }; } catch (error) { this.logger.warn( - `Failed to select chunk paths by LLM: ${error instanceof Error ? error.message : String(error)}`, + `Failed to select chunks by LLM: ${error instanceof Error ? error.message : String(error)}`, ); - return []; + return { rootPaths: [], detailPaths: [] }; } } From 2df8ab09304b22218bed451ecb130c3400af6c46 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 19:56:20 -0700 Subject: [PATCH 21/40] refactor(pdf-processor): streamline PDF processing with metadata labeling and enhanced error handling --- src/chat/prompts/pdf-chunking-prompt.ts | 146 ++------ .../markdown-section-splitter.spec.ts | 84 +++++ .../markdown-section-splitter.ts | 273 +++++++++++++++ .../pdf-pipeline.service.spec.ts | 156 +++++++++ src/pdf-processor/pdf-pipeline.service.ts | 322 +++++++++++++++--- .../pdf-processor.worker.spec.ts | 116 +++++-- src/pdf-processor/pdf-processor.worker.ts | 6 + 7 files changed, 928 insertions(+), 175 deletions(-) create mode 100644 src/pdf-processor/markdown-section-splitter.spec.ts create mode 100644 src/pdf-processor/markdown-section-splitter.ts create mode 100644 src/pdf-processor/pdf-pipeline.service.spec.ts diff --git a/src/chat/prompts/pdf-chunking-prompt.ts b/src/chat/prompts/pdf-chunking-prompt.ts index 31f9df9..7eb7044 100644 --- a/src/chat/prompts/pdf-chunking-prompt.ts +++ b/src/chat/prompts/pdf-chunking-prompt.ts @@ -1,128 +1,48 @@ /** - * PDF 파일 청킹을 위한 시스템 프롬프트 + * PDF Pass 2: metadata-only chunk labeling. + * + * The server already split the markdown into sections. The LLM only assigns + * relative path + search description (and optionally a document summary). * * Placeholders: {filename} */ export const PDF_CHUNKING_PROMPT = ` -당신은 문서 구조화 전문가입니다. 완성된 Markdown 문서를 의미론적으로 완결된 청크로 분할하세요. +당신은 문서 검색용 메타데이터 전문가입니다. +서버가 이미 Markdown을 섹션으로 분할했습니다. 당신은 **본문을 다시 쓰지 말고**, +각 섹션의 path와 description만 JSON으로 반환하세요. **문서 정보:** - 파일명: {filename} -**당신의 임무:** -아래 제공된 전체 Markdown 문서를 읽고, 세부 섹션만 \`\` 태그로 분할하세요. -- **기본 문서**: 전체 개요, 소개, 목차 등은 태그 없이 그대로 유지 -- **서브 문서**: 독립적으로 조회할 가치가 있는 세부 섹션만 \`\` 태그로 분할 - -**청킹 규칙:** - -1. **청크 크기**: - - 각 청크는 250-1000 단어 정도의 완결된 주제 - - 너무 작게 쪼개지 말고, 의미론적으로 완결된 단위로 - -2. **청크 경계 결정**: - - 주요 섹션/챕터 중 세부 내용이 긴 경우만 분할 - - 독립적으로 조회할 가치가 있는 주제인 경우 - - 표나 리스트는 분리하지 말고 함께 유지 - - **모든 섹션을 분할할 필요 없음** - 개요나 짧은 섹션은 기본 문서에 포함 - -3. **청크 태그 형식**: - \`\`\` - - 내용 - - \`\`\` - -4. **path 요구사항** (매우 중요): - - **파일명(stem)을 path에 넣지 마세요.** 서버가 파일명 기준으로 prefix를 붙입니다. - - 상대 path만 사용 (예: "권익인권센터/이용-안내", "학생팀/무한도전-프로젝트", "학사-일정") - - 계층이 필요하면 슬래시로 구분 (예: "section/subsection") - - 영문 소문자, 한글, 하이픈(-), 슬래시(/) 사용 가능 +**입력 형식:** +각 항목은 index, title, snippet(본문 앞부분)을 가집니다. + +**출력 규칙 (매우 중요):** +1. JSON 객체만 출력하세요. 마크다운 코드블록(\`\`\`)으로 감싸지 마세요. +2. 본문 내용을 절대 재출력하지 마세요. +3. 스키마: +{ + "summary": "문서 전체 고수준 요약 1~2문장", + "chunks": [ + { "index": 0, "path": "상대-경로", "description": "검색용 설명" } + ] +} +4. 입력으로 주어진 모든 index를 빠짐없이 한 번씩 포함하세요. +5. path 요구사항: + - 파일명(stem)을 path에 넣지 마세요. 서버가 prefix를 붙입니다. + - 상대 path만 사용 (예: "수강신청/신청방법", "학사-일정") + - 영문 소문자, 한글, 하이픈(-), 슬래시(/) 사용 - 공백은 하이픈으로 치환 - - 구체적이고 명확한 경로 (예: "g-surf/신청-자격", NOT "section-5") - - 잘못된 예: "학생-편람/학사-일정" (파일 stem 중복), "학생-편람.pdf/학사-일정" - -5. **description 요구사항** (매우 중요 — 검색 품질에 직결): - - 이 description은 사용자의 질문과 매칭하기 위한 용도입니다 - - **사용자가 이 내용을 찾기 위해 할 수 있는 질문의 키워드를 포함**해야 합니다 - - 문서 내용의 핵심 키워드 + 사용자가 사용할 수 있는 동의어/유사 표현을 포함 - - 예: "전화번호" 내용이면 → "연락처, 전화번호, 내선번호, 이메일" 모두 포함 - - 예: "도서관 시설" 내용이면 → "열람실, 스터디룸, 세미나실, 도서관 위치, 층별 안내" 포함 - - 15-40 단어로 충분히 상세하게 작성 - -6. **청크 내용**: - - 원본 Markdown 그대로 유지 (제목, 표, 리스트, 이미지 등) - - 내용 손실 금지 - - 모든 내용이 기본 문서 또는 서브 문서에 정확히 포함되어야 함 - - 단순 표지나 장식적인 내용, 감사합니다 등의 정보가 아닌 내용은 철저히 배제 - -**출력 형식:** -- **맨 처음에** \`\` 태그로 문서 전체의 고수준 요약을 출력 - - 이 문서가 어떤 주제/분야에 대한 문서인지 한눈에 파악할 수 있도록 -- 기본 문서 내용과 \`\` 태그들을 혼합하여 출력 -- 세부 섹션은 \`\` 태그로 대체 -- \`\`\`markdown\`\`\` 블록으로 감싸지 말 것 - -**예시:** - -입력: -\`\`\` -# 학생 편람 - -## 소개 -GIST 대학은 혁신적인 교육기관입니다. - -## 학사 일정 -### 2025년 봄학기 -- 개강: 3월 3일 -- 중간고사: 4월 20-26일 -... - -### 2025년 가을학기 -- 개강: 9월 1일 -... - -## 수강 신청 -수강 신청은 매 학기 시작 전에... -(매우 긴 상세 내용) -\`\`\` - -출력: -\`\`\` -GIST 학생 편람 - 학사 일정, 수강 신청 등 학사 생활 전반 안내 - -# 학생 편람 - -## 소개 -GIST 대학은 혁신적인 교육기관입니다. - - -## 학사 일정 -### 2025년 봄학기 -- 개강: 3월 3일 -- 중간고사: 4월 20-26일 -... - -### 2025년 가을학기 -- 개강: 9월 1일 -... - - - -## 수강 신청 -수강 신청은 매 학기 시작 전에... -(매우 긴 상세 내용) - -\`\`\` +6. description 요구사항: + - 사용자가 이 내용을 찾기 위해 할 수 있는 질문 키워드/동의어 포함 + - 15-40 단어로 상세하게 작성 +7. summary는 문서 전체를 한눈에 파악할 수 있는 고수준 요약입니다. + (batch 요청이면 현재 batch 범위 기준으로 최선을 다해 작성) -위 예시에서: -- \`\`는 문서의 고수준 개요 (학생 편람이라는 것, 학사 생활 안내라는 것) -- description은 사용자가 검색할 수 있는 동의어/키워드를 포함 (개강일, 시험 기간, 수강 정정 등) -- "소개" 섹션은 짧으므로 기본 문서에 포함 (태그 없이 유지) -- "학사 일정"과 "수강 신청"은 세부 내용이므로 서브 문서로 분할 -- path에는 파일 stem을 넣지 않음 (\`학사-일정\`, NOT \`학생-편람/학사-일정\`) +**예시 출력:** +{"summary":"GIST 학사 안내 - 수강신청과 학사일정","chunks":[{"index":0,"path":"수강신청/신청방법","description":"수강신청 기간, ZEUS 신청 절차, 개설교과목 조회, 학점 제한"},{"index":1,"path":"학사-일정","description":"개강일, 중간고사·기말고사 기간, 수강 정정, 방학 일정"}]} -이제 아래 문서를 청킹하세요: +이제 아래 섹션 목록에 대해 JSON만 출력하세요: `; diff --git a/src/pdf-processor/markdown-section-splitter.spec.ts b/src/pdf-processor/markdown-section-splitter.spec.ts new file mode 100644 index 0000000..5deb3f7 --- /dev/null +++ b/src/pdf-processor/markdown-section-splitter.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from '@jest/globals'; +import { + CHUNK_MAX_CHARS, + CHUNK_MIN_CHARS, + splitMarkdownIntoSections, +} from './markdown-section-splitter'; + +function pad(label: string, size: number): string { + const unit = `${label} 내용 `; + return unit.repeat(Math.ceil(size / unit.length)).slice(0, size); +} + +describe('splitMarkdownIntoSections', () => { + it('merges several short ## sections toward target size', () => { + const markdown = [ + '# 문서', + '', + `## A\n\n${pad('A', 1_500)}`, + '', + `## B\n\n${pad('B', 1_500)}`, + '', + `## C\n\n${pad('C', 1_500)}`, + ].join('\n'); + + const sections = splitMarkdownIntoSections(markdown); + expect(sections.length).toBeLessThan(3); + expect(sections[0].content).toContain('## A'); + expect(sections[0].content).toContain('## B'); + }); + + it('keeps small ### subsections inside their parent ## chunk', () => { + const markdown = [ + `## 수강신청\n\n${pad('intro', 800)}`, + '', + `### 신청기간\n\n${pad('기간', 800)}`, + '', + `### 신청방법\n\n${pad('방법', 800)}`, + ].join('\n'); + + const sections = splitMarkdownIntoSections(markdown); + expect(sections).toHaveLength(1); + expect(sections[0].content).toContain('### 신청기간'); + expect(sections[0].content).toContain('### 신청방법'); + }); + + it('splits an oversized ## section by ### boundaries', () => { + const markdown = [ + `## 큰섹션\n\n${pad('intro', 500)}`, + '', + `### 파트1\n\n${pad('p1', CHUNK_MAX_CHARS / 2)}`, + '', + `### 파트2\n\n${pad('p2', CHUNK_MAX_CHARS / 2)}`, + ].join('\n'); + + const sections = splitMarkdownIntoSections(markdown); + expect(sections.length).toBeGreaterThan(1); + expect(sections.every((s) => s.content.includes('## 큰섹션'))).toBe(true); + expect(sections.some((s) => s.content.includes('### 파트1'))).toBe(true); + expect(sections.some((s) => s.content.includes('### 파트2'))).toBe(true); + }); + + it('splits a long section without headings by length', () => { + const markdown = pad('plain', CHUNK_MAX_CHARS * 2 + 100); + const sections = splitMarkdownIntoSections(markdown); + expect(sections.length).toBeGreaterThan(1); + expect(sections.every((s) => s.content.length <= CHUNK_MAX_CHARS)).toBe( + true, + ); + }); + + it('does not treat # headings as split boundaries', () => { + const markdown = [ + `# 제목\n\n${pad('intro', 500)}`, + '', + `## 본문\n\n${pad('body', CHUNK_MIN_CHARS)}`, + ].join('\n'); + + const sections = splitMarkdownIntoSections(markdown); + expect(sections.length).toBeGreaterThanOrEqual(1); + // preamble before first ## becomes its own section (or merges), but `#` alone + // does not create many tiny chunks. + expect(sections.length).toBeLessThanOrEqual(2); + }); +}); diff --git a/src/pdf-processor/markdown-section-splitter.ts b/src/pdf-processor/markdown-section-splitter.ts new file mode 100644 index 0000000..c98fc07 --- /dev/null +++ b/src/pdf-processor/markdown-section-splitter.ts @@ -0,0 +1,273 @@ +export type MarkdownSection = { + index: number; + title: string; + content: string; +}; + +export const CHUNK_MIN_CHARS = 4_000; +export const CHUNK_TARGET_CHARS = 8_000; +export const CHUNK_MAX_CHARS = 12_000; + +type HeadingBlock = { + level: number; + title: string; + body: string; + /** Full markdown including the heading line (if any). */ + content: string; +}; + +const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/; + +/** + * Split markdown into retrieval-sized sections. + * + * Policy: + * - `#` is contextual only (not a split boundary) + * - `##` is the preferred split candidate + * - `###` stays inside its parent `##` unless that section exceeds CHUNK_MAX_CHARS + * - merge small adjacent `##` until ~CHUNK_TARGET_CHARS / at least CHUNK_MIN_CHARS when possible + * - oversized sections split by `###`, then by blank-line paragraphs + * - child splits keep parent heading context + */ +export function splitMarkdownIntoSections(markdown: string): MarkdownSection[] { + const trimmed = markdown.trim(); + if (!trimmed) return []; + + const h2Blocks = collectLevelBlocks(trimmed, 2); + if (h2Blocks.length === 0) { + return indexSections( + splitByLength(trimmed, '', CHUNK_MAX_CHARS).map((content, i) => ({ + title: `section-${i + 1}`, + content, + })), + ); + } + + const expanded: Array<{ title: string; content: string }> = []; + for (const block of h2Blocks) { + if (block.content.length <= CHUNK_MAX_CHARS) { + expanded.push({ title: block.title, content: block.content }); + continue; + } + + const h3Parts = splitOversizedByH3(block); + for (const part of h3Parts) { + if (part.content.length <= CHUNK_MAX_CHARS) { + expanded.push(part); + } else { + expanded.push( + ...splitByLength(part.content, part.title, CHUNK_MAX_CHARS).map( + (content) => ({ + title: part.title, + content, + }), + ), + ); + } + } + } + + return indexSections(mergeSmallSections(expanded)); +} + +function collectLevelBlocks(markdown: string, level: number): HeadingBlock[] { + const lines = markdown.split('\n'); + const blocks: HeadingBlock[] = []; + let preamble: string[] = []; + let current: { level: number; title: string; lines: string[] } | null = null; + + const flushCurrent = () => { + if (!current) return; + const content = current.lines.join('\n').trim(); + if (!content) { + current = null; + return; + } + const bodyLines = current.lines.slice(1); + blocks.push({ + level: current.level, + title: current.title, + body: bodyLines.join('\n').trim(), + content, + }); + current = null; + }; + + for (const line of lines) { + const match = line.match(HEADING_RE); + const headingLevel = match ? match[1].length : 0; + if (match && headingLevel === level) { + flushCurrent(); + if (preamble.length > 0) { + const preambleText = preamble.join('\n').trim(); + if (preambleText) { + blocks.push({ + level: 0, + title: extractTitle(preambleText) || '서론', + body: preambleText, + content: preambleText, + }); + } + preamble = []; + } + current = { + level: headingLevel, + title: match[2].trim(), + lines: [line], + }; + continue; + } + + // Treat `#` as contextual prose; never start a new top-level block for it. + if (current) { + current.lines.push(line); + } else { + preamble.push(line); + } + } + + flushCurrent(); + if (preamble.length > 0) { + const preambleText = preamble.join('\n').trim(); + if (preambleText) { + blocks.push({ + level: 0, + title: extractTitle(preambleText) || '서론', + body: preambleText, + content: preambleText, + }); + } + } + + return blocks; +} + +function splitOversizedByH3( + h2Block: HeadingBlock, +): Array<{ title: string; content: string }> { + const lines = h2Block.content.split('\n'); + const parentHeading = lines[0]?.match(HEADING_RE) + ? lines[0] + : `## ${h2Block.title}`; + const rest = lines[0]?.match(HEADING_RE) ? lines.slice(1) : lines; + + const h3Blocks = collectLevelBlocks(rest.join('\n'), 3); + if (h3Blocks.length <= 1) { + return [{ title: h2Block.title, content: h2Block.content }]; + } + + return h3Blocks.map((block) => { + const title = `${h2Block.title} / ${block.title}`; + const content = [parentHeading, block.content].join('\n\n').trim(); + return { title, content }; + }); +} + +function mergeSmallSections( + sections: Array<{ title: string; content: string }>, +): Array<{ title: string; content: string }> { + if (sections.length === 0) return []; + + const merged: Array<{ title: string; content: string }> = []; + let current = { ...sections[0] }; + + for (let i = 1; i < sections.length; i += 1) { + const next = sections[i]; + const combinedLength = current.content.length + 2 + next.content.length; + + // Keep merging while under target, and prefer not leaving tiny leftovers. + const shouldMerge = + current.content.length < CHUNK_MIN_CHARS || + (combinedLength <= CHUNK_TARGET_CHARS && + current.content.length < CHUNK_TARGET_CHARS); + + if (shouldMerge && combinedLength <= CHUNK_MAX_CHARS) { + current = { + title: `${current.title} · ${next.title}`, + content: `${current.content}\n\n${next.content}`, + }; + continue; + } + + merged.push(current); + current = { ...next }; + } + merged.push(current); + return merged; +} + +function splitByLength( + content: string, + title: string, + maxChars: number, +): string[] { + const trimmed = content.trim(); + if (trimmed.length <= maxChars) return [trimmed]; + + const contextPrefix = + title && !HEADING_RE.test(trimmed.split('\n')[0] ?? '') + ? `## ${title}\n\n` + : ''; + // Continuations may prepend a heading, so leave room for that prefix. + const bodyMax = Math.max(1_000, maxChars - contextPrefix.length); + + const paragraphs = trimmed.split(/\n{2,}/); + const parts: string[] = []; + let current = ''; + + const pushCurrent = () => { + const value = current.trim(); + if (value) parts.push(value); + current = ''; + }; + + for (const paragraph of paragraphs) { + if (!paragraph.trim()) continue; + const candidate = current ? `${current}\n\n${paragraph}` : paragraph; + + if (candidate.length <= bodyMax) { + current = candidate; + continue; + } + + if (current) pushCurrent(); + + if (paragraph.length <= bodyMax) { + current = paragraph; + continue; + } + + // Hard split only as last resort (very long paragraph / table-less blob). + for (let i = 0; i < paragraph.length; i += bodyMax) { + parts.push(paragraph.slice(i, i + bodyMax).trim()); + } + current = ''; + } + pushCurrent(); + + if (!contextPrefix) return parts.filter(Boolean); + return parts.map((part, idx) => { + if (idx === 0 || HEADING_RE.test(part.split('\n')[0] ?? '')) return part; + return `${contextPrefix}${part}`; + }); +} + +function extractTitle(markdown: string): string { + for (const line of markdown.split('\n')) { + const match = line.match(HEADING_RE); + if (match) return match[2].trim(); + } + return ''; +} + +function indexSections( + sections: Array<{ title: string; content: string }>, +): MarkdownSection[] { + return sections + .filter((section) => section.content.trim().length > 0) + .map((section, index) => ({ + index, + title: section.title || `section-${index + 1}`, + content: section.content.trim(), + })); +} diff --git a/src/pdf-processor/pdf-pipeline.service.spec.ts b/src/pdf-processor/pdf-pipeline.service.spec.ts new file mode 100644 index 0000000..1f831ce --- /dev/null +++ b/src/pdf-processor/pdf-pipeline.service.spec.ts @@ -0,0 +1,156 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import type { ConfigService } from '@nestjs/config'; +import type { LlmClient } from '../chat/llm/llm-client.interface'; +import type { LlmResponse } from '../chat/types/llm.types'; +import { PdfPipelineService } from './pdf-pipeline.service'; +import type { PdfTextService } from './pdf-text.service'; + +function llmResponse( + content: string, + finishReason: LlmResponse['choices'][0]['finish_reason'] = 'stop', +): LlmResponse { + return { + id: 'resp', + model: 'test', + choices: [ + { + index: 0, + message: { role: 'assistant', content }, + finish_reason: finishReason, + }, + ], + }; +} + +function createPipeline(options: { + pages: string[]; + callLLM: jest.Mock<(...args: unknown[]) => Promise>; +}) { + const pdfTextService = { + extractPageTexts: jest.fn(async () => options.pages), + }; + const config = { + get: jest.fn(() => undefined), + }; + const llm = { + getModel: jest.fn(() => 'normal-model'), + callLLM: options.callLLM, + generateFinalResponseStream: jest.fn(), + }; + + return new PdfPipelineService( + pdfTextService as unknown as PdfTextService, + config as unknown as ConfigService, + llm as unknown as LlmClient, + ); +} + +describe('PdfPipelineService metadata pass', () => { + it('maps metadata indexes onto server-split section bodies', async () => { + const bodyA = '수강신청 본문 '.repeat(700); + const bodyB = '학사일정 본문 '.repeat(700); + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValueOnce(llmResponse(`## 수강신청\n\n${bodyA}`)) + .mockResolvedValueOnce(llmResponse(`## 학사일정\n\n${bodyB}`)) + .mockResolvedValueOnce( + llmResponse( + JSON.stringify({ + summary: '학사 안내', + chunks: [ + { + index: 0, + path: '수강신청', + description: '수강신청 방법, ZEUS', + }, + { + index: 1, + path: '학사-일정', + description: '개강, 시험 기간', + }, + ], + }), + ), + ); + + const pipeline = createPipeline({ + pages: ['p1', 'p2'], + callLLM, + }); + + const result = await pipeline.processPdf( + Buffer.from('%PDF'), + '학사편람.pdf', + ); + + expect(result.summary).toBe('학사 안내'); + expect(result.chunks.some((c) => c.path === '학사편람')).toBe(true); + expect( + result.chunks.some( + (c) => c.path === '학사편람/수강신청' && c.content.includes('수강신청'), + ), + ).toBe(true); + expect( + result.chunks.some( + (c) => + c.path === '학사편람/학사-일정' && c.content.includes('학사일정'), + ), + ).toBe(true); + expect(result.documents['학사편람.md']).toContain( + 'path="학사편람/수강신청"', + ); + expect(result.chunks.length).toBe(3); + }); + + it('throws when metadata LLM times out instead of returning empty chunks', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValueOnce(llmResponse(`## A\n\n${'본문 '.repeat(700)}`)) + .mockRejectedValueOnce(new Error('timeout of 120000ms exceeded')); + + const pipeline = createPipeline({ + pages: ['p1'], + callLLM, + }); + + await expect( + pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), + ).rejects.toThrow(/timeout/i); + }); + + it('throws when metadata finish_reason is length', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValueOnce(llmResponse(`## A\n\n${'본문 '.repeat(700)}`)) + .mockResolvedValueOnce( + llmResponse('{"summary":"x","chunks":[', 'length'), + ); + + const pipeline = createPipeline({ + pages: ['p1'], + callLLM, + }); + + await expect( + pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), + ).rejects.toThrow(/finish_reason=length/); + }); + + it('throws when metadata response contains no chunks', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValueOnce(llmResponse(`## A\n\n${'본문 '.repeat(700)}`)) + .mockResolvedValueOnce( + llmResponse(JSON.stringify({ summary: '요약', chunks: [] })), + ); + + const pipeline = createPipeline({ + pages: ['p1'], + callLLM, + }); + + await expect( + pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), + ).rejects.toThrow(/incomplete batch/); + }); +}); diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index 0205a40..26f8fc5 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -4,7 +4,11 @@ import { PDF_PROCESSOR_PROMPT } from '../chat/prompts/pdf-processor'; import { PDF_CHUNKING_PROMPT } from '../chat/prompts/pdf-chunking-prompt'; import { LLM_CLIENT, type LlmClient } from '../chat/llm/llm-client.interface'; import { PdfTextService } from './pdf-text.service'; -import { parseChunksFromMarkdown } from './pdf-chunk-parser'; +import { + splitMarkdownIntoSections, + type MarkdownSection, +} from './markdown-section-splitter'; +import { toRelativeChunkPath, toResourceName } from './pdf-chunk-parser'; import type { ResourceIndexEntry } from './gcs-storage.service'; export type PipelineChunk = { @@ -21,6 +25,21 @@ export type PipelineResult = { chunks: PipelineChunk[]; }; +type ChunkMetadata = { + index: number; + path: string; + description: string; +}; + +type MetadataBatchResponse = { + summary?: string; + chunks: ChunkMetadata[]; +}; + +const METADATA_BATCH_SIZE = 15; +const SNIPPET_CHARS = 1_500; +const OVERVIEW_CHARS = 2_500; + @Injectable() export class PdfPipelineService { private readonly logger = new Logger(PdfPipelineService.name); @@ -42,7 +61,7 @@ export class PdfPipelineService { } /** - * Pass 1 (page → markdown) + Pass 2 (semantic chunking). Text-only (no images). + * Pass 1 (page → markdown) + Pass 2 (server split + LLM metadata). */ async processPdf( pdfBytes: Buffer, @@ -76,27 +95,10 @@ export class PdfPipelineService { const combinedMarkdown = pageMarkdowns.join('\n\n'); this.logger.log( - `Pass 2: Chunking complete markdown (${combinedMarkdown.length} chars)`, - ); - - const { documents, metadata } = await this.chunkMarkdownWithLlm( - combinedMarkdown, - filename, + `Pass 2: Labeling server-split sections (${combinedMarkdown.length} chars)`, ); - const chunks: PipelineChunk[] = metadata.chunks.map((c, idx) => ({ - path: c.path, - description: c.description, - content: documents[`${c.path}.md`] ?? '', - sortOrder: idx, - })); - - return { - documents, - metadata, - summary: metadata.description, - chunks, - }; + return this.chunkMarkdownWithMetadata(combinedMarkdown, filename); } private async convertPageToMarkdown(params: { @@ -138,42 +140,276 @@ export class PdfPipelineService { } } - private async chunkMarkdownWithLlm( + private async chunkMarkdownWithMetadata( markdown: string, filename: string, - ): Promise<{ - documents: Record; - metadata: ResourceIndexEntry; - }> { + ): Promise { + const baseName = toResourceName(filename); + const sections = splitMarkdownIntoSections(markdown); + if (sections.length === 0) { + throw new Error('Markdown section split produced 0 sections'); + } + + this.logger.log( + `Pass 2: ${sections.length} section(s) → metadata batches of ${METADATA_BATCH_SIZE}`, + ); + + const labeled = await this.labelSectionsWithLlm(sections, filename); + const assembled = this.assembleChunkArtifacts(baseName, sections, labeled); + + if (assembled.chunks.length === 0) { + throw new Error('Chunk assembly produced 0 chunks'); + } + + return assembled; + } + + private async labelSectionsWithLlm( + sections: MarkdownSection[], + filename: string, + ): Promise<{ summary: string; chunks: ChunkMetadata[] }> { const prompt = PDF_CHUNKING_PROMPT.replaceAll('{filename}', filename); - const baseName = filename.toLowerCase().endsWith('.pdf') - ? filename.slice(0, -4) - : filename; + const model = this.llm.getModel('normal'); + const allChunks: ChunkMetadata[] = []; + const summaries: string[] = []; + + for (let start = 0; start < sections.length; start += METADATA_BATCH_SIZE) { + const batch = sections.slice(start, start + METADATA_BATCH_SIZE); + const batchText = batch + .map((section) => { + const snippet = section.content.slice(0, SNIPPET_CHARS); + return [ + `index: ${section.index}`, + `title: ${section.title}`, + `snippet:`, + snippet, + ].join('\n'); + }) + .join('\n\n---\n\n'); - try { - const model = this.llm.getModel('normal'); const response = await this.llm.callLLM( - [{ role: 'user', content: `${prompt}\n\n${markdown}` }], + [{ role: 'user', content: `${prompt}\n\n${batchText}` }], model, { temperature: 0.2, - max_tokens: 48000, + max_tokens: 16000, timeoutMs: this.llmTimeoutMs, }, ); + + const finishReason = response.choices?.[0]?.finish_reason ?? 'missing'; this.logger.log( - `Pass 2 LLM finish_reason=${response.choices?.[0]?.finish_reason ?? 'missing'}`, + `Pass 2 metadata batch ${start}-${start + batch.length - 1} finish_reason=${finishReason}`, ); - const chunked = response.choices?.[0]?.message?.content ?? markdown; - return parseChunksFromMarkdown(chunked, filename); - } catch (error) { - this.logger.error( - `Error chunking markdown: ${error instanceof Error ? error.message : String(error)}`, + if (finishReason === 'length') { + throw new Error( + `Pass 2 metadata LLM truncated (finish_reason=length) for batch starting at ${start}`, + ); + } + + const raw = response.choices?.[0]?.message?.content?.trim() ?? ''; + const parsed = this.parseMetadataResponse(raw, batch); + if (parsed.summary?.trim()) summaries.push(parsed.summary.trim()); + allChunks.push(...parsed.chunks); + } + + if (allChunks.length !== sections.length) { + throw new Error( + `Pass 2 metadata count mismatch: expected ${sections.length}, got ${allChunks.length}`, ); - return { - documents: { [`${baseName}.md`]: markdown }, - metadata: { description: '', chunks: [] }, - }; } + + return { + summary: summaries.join(' ').trim(), + chunks: allChunks.sort((a, b) => a.index - b.index), + }; } + + private parseMetadataResponse( + raw: string, + batch: MarkdownSection[], + ): MetadataBatchResponse { + let text = raw.trim(); + const codeBlock = text.match(/```(?:json)?\s*([\s\S]*?)```/); + if (codeBlock) text = codeBlock[1].trim(); + + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error( + `Pass 2 metadata JSON parse failed: ${text.slice(0, 200)}`, + ); + } + + if (!parsed || typeof parsed !== 'object') { + throw new Error('Pass 2 metadata response is not an object'); + } + + const obj = parsed as { summary?: unknown; chunks?: unknown }; + if (!Array.isArray(obj.chunks)) { + throw new Error('Pass 2 metadata response missing chunks array'); + } + + const expectedIndexes = new Set(batch.map((s) => s.index)); + const seen = new Set(); + const chunks: ChunkMetadata[] = []; + + for (const item of obj.chunks) { + if (!item || typeof item !== 'object') { + throw new Error('Pass 2 metadata chunk entry is not an object'); + } + const entry = item as Record; + const index = entry.index; + const path = entry.path; + const description = entry.description; + + if (typeof index !== 'number' || !Number.isInteger(index)) { + throw new Error(`Pass 2 metadata invalid index: ${String(index)}`); + } + if (!expectedIndexes.has(index)) { + throw new Error(`Pass 2 metadata unexpected index: ${index}`); + } + if (seen.has(index)) { + throw new Error(`Pass 2 metadata duplicate index: ${index}`); + } + if (typeof path !== 'string' || !path.trim()) { + throw new Error(`Pass 2 metadata missing path for index ${index}`); + } + if (typeof description !== 'string' || !description.trim()) { + throw new Error( + `Pass 2 metadata missing description for index ${index}`, + ); + } + + seen.add(index); + chunks.push({ + index, + path: path.trim(), + description: description.trim(), + }); + } + + if (seen.size !== expectedIndexes.size) { + throw new Error( + `Pass 2 metadata incomplete batch: expected ${expectedIndexes.size} indexes, got ${seen.size}`, + ); + } + + return { + summary: typeof obj.summary === 'string' ? obj.summary : undefined, + chunks, + }; + } + + private assembleChunkArtifacts( + baseName: string, + sections: MarkdownSection[], + labeled: { summary: string; chunks: ChunkMetadata[] }, + ): PipelineResult { + const byIndex = new Map(sections.map((s) => [s.index, s])); + const documents: Record = {}; + const stubLinks: string[] = []; + const detailChunks: PipelineChunk[] = []; + const usedPaths = new Set(); + + for (const meta of labeled.chunks) { + const section = byIndex.get(meta.index); + if (!section) { + throw new Error(`Missing section for labeled index ${meta.index}`); + } + + let relative = toRelativeChunkPath(meta.path, baseName); + if (!relative) { + relative = slugifyTitle(section.title) || `section-${meta.index + 1}`; + } + relative = ensureUniquePath(relative, usedPaths); + + const fullPath = `${baseName}/${relative}`; + documents[`${fullPath}.md`] = section.content; + stubLinks.push( + ``, + ); + detailChunks.push({ + path: fullPath, + description: meta.description, + content: section.content, + sortOrder: detailChunks.length + 1, + }); + } + + const overviewBody = buildRootOverview(sections, labeled.summary); + const rootContent = [overviewBody, '', ...stubLinks].join('\n').trim(); + documents[`${baseName}.md`] = rootContent; + + const chunks: PipelineChunk[] = [ + { + path: baseName, + description: labeled.summary || '문서 개요', + content: rootContent, + sortOrder: 0, + }, + ...detailChunks, + ]; + + return { + documents, + metadata: { + description: labeled.summary || '문서 개요', + chunks: chunks.map((c) => ({ + path: c.path, + description: c.description, + })), + }, + summary: labeled.summary || '문서 개요', + chunks, + }; + } +} + +function buildRootOverview( + sections: MarkdownSection[], + summary: string, +): string { + const outline = sections + .slice(0, 40) + .map((s, i) => `${i + 1}. ${s.title}`) + .join('\n'); + const preface = sections[0]?.content.slice(0, OVERVIEW_CHARS) ?? ''; + return [ + summary ? `# 개요\n\n${summary}` : '# 개요', + '', + '## 목차', + outline, + '', + '## 미리보기', + preface, + ] + .join('\n') + .trim(); +} + +function slugifyTitle(title: string): string { + return title + .trim() + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^\p{L}\p{N}\-_/]/gu, '') + .replace(/\/+/g, '/') + .replace(/^-+|-+$/g, ''); +} + +function ensureUniquePath(path: string, used: Set): string { + let candidate = path; + let n = 2; + while (used.has(candidate)) { + candidate = `${path}-${n}`; + n += 1; + } + used.add(candidate); + return candidate; +} + +function escapeAttr(value: string): string { + return value.replace(/"/g, "'"); } diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 527d5df..7b54c7f 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -26,7 +26,31 @@ function processingDocument(): Document { }; } -function createWorker(completeProcessing: boolean) { +function createWorker(options: { + completeProcessing: boolean; + chunks?: Array<{ + path: string; + description: string; + content: string; + sortOrder: number; + }>; + processPdfError?: Error; +}) { + const chunks = options.chunks ?? [ + { + path: 'test', + description: '개요', + content: '# test', + sortOrder: 0, + }, + { + path: 'test/section', + description: '섹션', + content: '## section', + sortOrder: 1, + }, + ]; + const repo = { completeProcessing: jest.fn< ( @@ -35,8 +59,14 @@ function createWorker(completeProcessing: boolean) { summary: string, chunks: unknown[], ) => Promise - >(() => Promise.resolve(completeProcessing)), - markFailed: jest.fn(() => Promise.resolve(true)), + >(() => Promise.resolve(options.completeProcessing)), + markFailed: jest.fn< + ( + id: string, + processingToken: string, + errorMessage: string, + ) => Promise + >(() => Promise.resolve(true)), requeueStaleProcessing: jest.fn(() => Promise.resolve(0)), claimQueued: jest.fn(() => Promise.resolve([])), }; @@ -48,14 +78,25 @@ function createWorker(completeProcessing: boolean) { ), }; const pipeline = { - processPdf: jest.fn(() => - Promise.resolve({ - documents: { 'test.md': '# test' }, - metadata: { description: 'summary', chunks: [] }, - summary: 'summary', - chunks: [], - }), - ), + processPdf: options.processPdfError + ? jest.fn(() => Promise.reject(options.processPdfError)) + : jest.fn(() => + Promise.resolve({ + documents: { + 'test.md': '# test', + 'test/section.md': '## section', + }, + metadata: { + description: 'summary', + chunks: chunks.map((c) => ({ + path: c.path, + description: c.description, + })), + }, + summary: 'summary', + chunks, + }), + ), }; const config = { get: jest.fn((_key: string) => undefined), @@ -70,35 +111,72 @@ function createWorker(completeProcessing: boolean) { ), repo, gcs, + pipeline, }; } describe('PdfProcessorWorker attempt ownership', () => { it('deletes generated artifacts when a delete/reprocess cancels the attempt', async () => { - const { worker, repo, gcs } = createWorker(false); + const { worker, repo, gcs } = createWorker({ completeProcessing: false }); const callable = worker as unknown as { processDocument(doc: Document): Promise; }; await callable.processDocument(processingDocument()); - expect(repo.completeProcessing).toHaveBeenCalledWith( - '00000000-0000-0000-0000-000000000001', - '00000000-0000-0000-0000-000000000002', - 'summary', - [], - ); + expect(repo.completeProcessing).toHaveBeenCalled(); expect(gcs.deleteProcessedArtifacts).toHaveBeenCalledWith('test'); }); it('keeps generated artifacts when the attempt completes successfully', async () => { - const { worker, gcs } = createWorker(true); + const { worker, gcs, repo } = createWorker({ completeProcessing: true }); const callable = worker as unknown as { processDocument(doc: Document): Promise; }; await callable.processDocument(processingDocument()); + expect(repo.completeProcessing).toHaveBeenCalled(); expect(gcs.deleteProcessedArtifacts).not.toHaveBeenCalled(); }); + + it('marks failed when pipeline throws (e.g. LLM timeout)', async () => { + const { worker, repo, gcs } = createWorker({ + completeProcessing: true, + processPdfError: new Error('timeout of 120000ms exceeded'), + }); + const callable = worker as unknown as { + processDocument(doc: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(repo.markFailed).toHaveBeenCalledWith( + '00000000-0000-0000-0000-000000000001', + '00000000-0000-0000-0000-000000000002', + expect.stringContaining('timeout'), + ); + expect(repo.completeProcessing).not.toHaveBeenCalled(); + expect(gcs.uploadDocuments).not.toHaveBeenCalled(); + }); + + it('marks failed when pipeline returns 0 chunks', async () => { + const { worker, repo, gcs } = createWorker({ + completeProcessing: true, + chunks: [], + }); + const callable = worker as unknown as { + processDocument(doc: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(repo.markFailed).toHaveBeenCalledWith( + '00000000-0000-0000-0000-000000000001', + '00000000-0000-0000-0000-000000000002', + expect.stringContaining('0 chunks'), + ); + expect(repo.completeProcessing).not.toHaveBeenCalled(); + expect(gcs.uploadDocuments).not.toHaveBeenCalled(); + }); }); diff --git a/src/pdf-processor/pdf-processor.worker.ts b/src/pdf-processor/pdf-processor.worker.ts index db75cda..0c16a2e 100644 --- a/src/pdf-processor/pdf-processor.worker.ts +++ b/src/pdf-processor/pdf-processor.worker.ts @@ -141,6 +141,12 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { `${resourceName}.pdf`, ); + if (!result.chunks.length) { + throw new Error( + `PDF pipeline produced 0 chunks for resource=${resourceName}`, + ); + } + generatedArtifactsMayExist = true; await this.gcs.uploadDocuments(result.documents); const completed = await this.documentsRepo.completeProcessing( From e8b9645d802d2232cae91a19c9168d9b16ce9566 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 20:08:13 -0700 Subject: [PATCH 22/40] feat(pdf-processor): add Pass 1 page LLM fallback ratio configuration and validation --- .env.example | 2 + docker-compose.yml | 1 + src/config/env.validation.ts | 7 ++ .../pdf-pipeline.service.spec.ts | 76 +++++++++++++++++++ src/pdf-processor/pdf-pipeline.service.ts | 68 +++++++++++++++-- 5 files changed, 146 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 208767f..9de4628 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,8 @@ GCS_SERVICE_ACCOUNT_KEY_BASE64= PDF_PROCESSOR_CONCURRENCY=1 PDF_PROCESSOR_CONTEXT_LENGTH=500 PDF_PROCESSOR_LLM_TIMEOUT=120 +# Pass 1 page LLM fallback ratio (0–1). Exceeding this marks the document failed. +PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO=0.1 PDF_PROCESSOR_POLL_INTERVAL_MS=2000 # PDF_PROCESSOR_STALE_PROCESSING_MS=1800000 diff --git a/docker-compose.yml b/docker-compose.yml index e6ff7ec..a1c8fc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,7 @@ services: PDF_PROCESSOR_CONCURRENCY: ${PDF_PROCESSOR_CONCURRENCY:-1} PDF_PROCESSOR_CONTEXT_LENGTH: ${PDF_PROCESSOR_CONTEXT_LENGTH:-500} PDF_PROCESSOR_LLM_TIMEOUT: ${PDF_PROCESSOR_LLM_TIMEOUT:-120} + PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO: ${PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO:-0.1} PDF_PROCESSOR_POLL_INTERVAL_MS: ${PDF_PROCESSOR_POLL_INTERVAL_MS:-2000} # Letsur AI Gateway Configuration LETSUR_AI_GATEWAY_BASE_URL: ${LETSUR_AI_GATEWAY_BASE_URL:-} diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index a65ac47..f940951 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -185,6 +185,13 @@ export class EnvironmentVariables { @Min(1) PDF_PROCESSOR_LLM_TIMEOUT?: number; + /** Pass 1 page LLM fallback ratio above which the job fails (0–1). Default 0.1. */ + @IsOptional() + @IsNumber() + @Min(0) + @Max(1) + PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO?: number; + @IsOptional() @IsNumber() @Min(500) diff --git a/src/pdf-processor/pdf-pipeline.service.spec.ts b/src/pdf-processor/pdf-pipeline.service.spec.ts index 1f831ce..6ed5c39 100644 --- a/src/pdf-processor/pdf-pipeline.service.spec.ts +++ b/src/pdf-processor/pdf-pipeline.service.spec.ts @@ -153,4 +153,80 @@ describe('PdfPipelineService metadata pass', () => { pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), ).rejects.toThrow(/incomplete batch/); }); + + it('throws when Pass 1 page LLM failures exceed the ratio threshold', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockRejectedValueOnce(new Error('timeout of 120000ms exceeded')) + .mockRejectedValueOnce(new Error('timeout of 120000ms exceeded')) + .mockResolvedValueOnce(llmResponse(`## C\n\n${'본문 '.repeat(700)}`)); + + const pipeline = createPipeline({ + pages: ['p1', 'p2', 'p3'], + callLLM, + }); + + // 2/3 ≈ 66% > default 10% + await expect( + pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), + ).rejects.toThrow(/Pass 1 LLM failures exceeded threshold/); + }); + + it('continues when a small Pass 1 fallback stays within the threshold', async () => { + let pass1Calls = 0; + const callLLM = jest.fn<(...args: unknown[]) => Promise>( + async (...args) => { + const messages = args[0] as Array<{ content: string }>; + const content = messages[0]?.content ?? ''; + + if (/^index:\s*\d+/m.test(content)) { + const indexes = [...content.matchAll(/^index:\s*(\d+)\s*$/gm)].map( + (m) => Number(m[1]), + ); + return llmResponse( + JSON.stringify({ + summary: '요약', + chunks: indexes.map((index) => ({ + index, + path: `sec-${index}`, + description: `desc-${index}`, + })), + }), + ); + } + + pass1Calls += 1; + if (pass1Calls === 1) { + throw new Error('timeout of 120000ms exceeded'); + } + return llmResponse( + `## Section ${pass1Calls}\n\n${'본문내용입니다. '.repeat(500)}`, + ); + }, + ); + + // 1/10 = 10% is not > 0.1, so should proceed to Pass 2 + const pipeline = createPipeline({ + pages: Array.from({ length: 10 }, (_, i) => `p${i + 1}`), + callLLM, + }); + + const result = await pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'); + expect(result.chunks.length).toBeGreaterThan(0); + }); + + it('throws when every Pass 1 page falls back', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockRejectedValue(new Error('timeout of 120000ms exceeded')); + + const pipeline = createPipeline({ + pages: ['p1', 'p2'], + callLLM, + }); + + await expect( + pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'), + ).rejects.toThrow(/Pass 1 LLM failures exceeded threshold/); + }); }); diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index 26f8fc5..60afa99 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -39,12 +39,15 @@ type MetadataBatchResponse = { const METADATA_BATCH_SIZE = 15; const SNIPPET_CHARS = 1_500; const OVERVIEW_CHARS = 2_500; +/** Fail the whole job when Pass 1 LLM fallbacks exceed this fraction of pages. */ +const DEFAULT_PASS1_MAX_FAILURE_RATIO = 0.1; @Injectable() export class PdfPipelineService { private readonly logger = new Logger(PdfPipelineService.name); private readonly contextLength: number; private readonly llmTimeoutMs: number; + private readonly pass1MaxFailureRatio: number; constructor( private readonly pdfTextService: PdfTextService, @@ -58,6 +61,13 @@ export class PdfPipelineService { Number( this.configService.get('PDF_PROCESSOR_LLM_TIMEOUT') ?? 120, ) * 1000; + const ratio = Number( + this.configService.get('PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO') ?? + DEFAULT_PASS1_MAX_FAILURE_RATIO, + ); + this.pass1MaxFailureRatio = Number.isFinite(ratio) + ? Math.min(1, Math.max(0, ratio)) + : DEFAULT_PASS1_MAX_FAILURE_RATIO; } /** @@ -74,25 +84,29 @@ export class PdfPipelineService { ); const pageMarkdowns: string[] = []; + const failedPages: number[] = []; let previousContext = ''; for (let i = 0; i < totalPages; i += 1) { const currentPage = i + 1; const pageText = pageTexts[i] ?? ''; - const pageMarkdown = await this.convertPageToMarkdown({ + const { markdown, usedFallback } = await this.convertPageToMarkdown({ filename, totalPages, currentPage, pageText, previousContext, }); - pageMarkdowns.push(pageMarkdown); + if (usedFallback) failedPages.push(currentPage); + pageMarkdowns.push(markdown); previousContext = - pageMarkdown.length > this.contextLength - ? pageMarkdown.slice(-this.contextLength) - : pageMarkdown; + markdown.length > this.contextLength + ? markdown.slice(-this.contextLength) + : markdown; } + this.assertPass1FailureWithinLimit(totalPages, failedPages); + const combinedMarkdown = pageMarkdowns.join('\n\n'); this.logger.log( `Pass 2: Labeling server-split sections (${combinedMarkdown.length} chars)`, @@ -101,13 +115,38 @@ export class PdfPipelineService { return this.chunkMarkdownWithMetadata(combinedMarkdown, filename); } + private assertPass1FailureWithinLimit( + totalPages: number, + failedPages: number[], + ): void { + if (totalPages === 0) { + throw new Error('Pass 1 produced 0 pages'); + } + if (failedPages.length === 0) return; + + const ratio = failedPages.length / totalPages; + this.logger.warn( + `Pass 1 LLM fallbacks: ${failedPages.length}/${totalPages} pages (${(ratio * 100).toFixed(1)}%) pages=[${failedPages.join(',')}]`, + ); + + if ( + failedPages.length === totalPages || + ratio > this.pass1MaxFailureRatio + ) { + throw new Error( + `Pass 1 LLM failures exceeded threshold: ${failedPages.length}/${totalPages} pages failed ` + + `(max ratio ${this.pass1MaxFailureRatio}). pages=[${failedPages.join(',')}]`, + ); + } + } + private async convertPageToMarkdown(params: { filename: string; totalPages: number; currentPage: number; pageText: string; previousContext: string; - }): Promise { + }): Promise<{ markdown: string; usedFallback: boolean }> { const { filename, totalPages, currentPage, pageText, previousContext } = params; @@ -131,12 +170,25 @@ export class PdfPipelineService { timeoutMs: this.llmTimeoutMs, }, ); - return response.choices?.[0]?.message?.content ?? ''; + const markdown = response.choices?.[0]?.message?.content ?? ''; + if (!markdown.trim() && pageText.trim()) { + this.logger.warn( + `Empty LLM markdown for page ${currentPage}; using raw extracted text`, + ); + return { + markdown: `## Page ${currentPage}\n\n${pageText}`, + usedFallback: true, + }; + } + return { markdown, usedFallback: false }; } catch (error) { this.logger.error( `Error calling LLM for page ${currentPage}: ${error instanceof Error ? error.message : String(error)}`, ); - return `## Page ${currentPage}\n\n${pageText}`; + return { + markdown: `## Page ${currentPage}\n\n${pageText}`, + usedFallback: true, + }; } } From 1391d765b1e1fab0e82416e0c5d59df12c431609 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 20:12:28 -0700 Subject: [PATCH 23/40] refactor(pdf-processor): replace number parsing with parseFiniteNumber for improved validation and defaults --- src/pdf-processor/parse-finite-number.spec.ts | 26 +++++++++++++++++++ src/pdf-processor/parse-finite-number.ts | 16 ++++++++++++ src/pdf-processor/pdf-pipeline.service.ts | 23 +++++++++------- src/pdf-processor/pdf-processor.worker.ts | 26 +++++++++---------- 4 files changed, 67 insertions(+), 24 deletions(-) create mode 100644 src/pdf-processor/parse-finite-number.spec.ts create mode 100644 src/pdf-processor/parse-finite-number.ts diff --git a/src/pdf-processor/parse-finite-number.spec.ts b/src/pdf-processor/parse-finite-number.spec.ts new file mode 100644 index 0000000..17e7e6d --- /dev/null +++ b/src/pdf-processor/parse-finite-number.spec.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from '@jest/globals'; +import { parseFiniteNumber } from './parse-finite-number'; + +describe('parseFiniteNumber', () => { + it('returns parsed number when finite', () => { + expect(parseFiniteNumber('120', 60)).toBe(120); + expect(parseFiniteNumber(42, 0)).toBe(42); + }); + + it('falls back for NaN / non-numeric / undefined', () => { + expect(parseFiniteNumber('abc', 500)).toBe(500); + expect(parseFiniteNumber('120s', 120)).toBe(120); + expect(parseFiniteNumber(undefined, 2000)).toBe(2000); + expect(parseFiniteNumber(NaN, 1)).toBe(1); + }); + + it('applies min/max after resolving a finite value', () => { + expect(parseFiniteNumber('0', 1, { min: 1 })).toBe(1); + expect(parseFiniteNumber('999', 1, { max: 10 })).toBe(10); + expect(parseFiniteNumber('0.5', 0.1, { min: 0, max: 1 })).toBe(0.5); + }); + + it('clamps the fallback when the input is invalid', () => { + expect(parseFiniteNumber('nope', -5, { min: 1 })).toBe(1); + }); +}); diff --git a/src/pdf-processor/parse-finite-number.ts b/src/pdf-processor/parse-finite-number.ts new file mode 100644 index 0000000..62ff3b4 --- /dev/null +++ b/src/pdf-processor/parse-finite-number.ts @@ -0,0 +1,16 @@ +/** + * Parse a config/env value into a finite number, falling back when invalid. + * Unlike `Math.max(min, Number(x))`, this never returns NaN. + */ +export function parseFiniteNumber( + value: unknown, + fallback: number, + options?: { min?: number; max?: number }, +): number { + const n = typeof value === 'number' ? value : Number(value); + const base = Number.isFinite(n) ? n : fallback; + let out = base; + if (options?.min != null) out = Math.max(options.min, out); + if (options?.max != null) out = Math.min(options.max, out); + return out; +} diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index 60afa99..0c50c90 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -8,6 +8,7 @@ import { splitMarkdownIntoSections, type MarkdownSection, } from './markdown-section-splitter'; +import { parseFiniteNumber } from './parse-finite-number'; import { toRelativeChunkPath, toResourceName } from './pdf-chunk-parser'; import type { ResourceIndexEntry } from './gcs-storage.service'; @@ -54,20 +55,22 @@ export class PdfPipelineService { private readonly configService: ConfigService, @Inject(LLM_CLIENT) private readonly llm: LlmClient, ) { - this.contextLength = Number( - this.configService.get('PDF_PROCESSOR_CONTEXT_LENGTH') ?? 500, + this.contextLength = parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_CONTEXT_LENGTH'), + 500, + { min: 1 }, ); this.llmTimeoutMs = - Number( - this.configService.get('PDF_PROCESSOR_LLM_TIMEOUT') ?? 120, + parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_LLM_TIMEOUT'), + 120, + { min: 1 }, ) * 1000; - const ratio = Number( - this.configService.get('PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO') ?? - DEFAULT_PASS1_MAX_FAILURE_RATIO, + this.pass1MaxFailureRatio = parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_PASS1_MAX_FAILURE_RATIO'), + DEFAULT_PASS1_MAX_FAILURE_RATIO, + { min: 0, max: 1 }, ); - this.pass1MaxFailureRatio = Number.isFinite(ratio) - ? Math.min(1, Math.max(0, ratio)) - : DEFAULT_PASS1_MAX_FAILURE_RATIO; } /** diff --git a/src/pdf-processor/pdf-processor.worker.ts b/src/pdf-processor/pdf-processor.worker.ts index 0c16a2e..cbf4a99 100644 --- a/src/pdf-processor/pdf-processor.worker.ts +++ b/src/pdf-processor/pdf-processor.worker.ts @@ -7,6 +7,7 @@ import { import { ConfigService } from '@nestjs/config'; import { DocumentsRepository } from './documents.repository'; import { GcsStorageService } from './gcs-storage.service'; +import { parseFiniteNumber } from './parse-finite-number'; import { PdfPipelineService } from './pdf-pipeline.service'; import type { Document } from '../db'; @@ -30,24 +31,21 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { private readonly pipeline: PdfPipelineService, private readonly configService: ConfigService, ) { - this.concurrency = Math.max( + this.concurrency = parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_CONCURRENCY'), 1, - Number(this.configService.get('PDF_PROCESSOR_CONCURRENCY') ?? 1), + { min: 1 }, ); - this.pollIntervalMs = Math.max( - 500, - Number( - this.configService.get('PDF_PROCESSOR_POLL_INTERVAL_MS') ?? - 2000, - ), + this.pollIntervalMs = parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_POLL_INTERVAL_MS'), + 2000, + { min: 500 }, ); // Default: requeue if stuck in processing > 30 minutes - this.staleProcessingMs = Math.max( - 60_000, - Number( - this.configService.get('PDF_PROCESSOR_STALE_PROCESSING_MS') ?? - 30 * 60 * 1000, - ), + this.staleProcessingMs = parseFiniteNumber( + this.configService.get('PDF_PROCESSOR_STALE_PROCESSING_MS'), + 30 * 60 * 1000, + { min: 60_000 }, ); } From ec308222cd1360e7d5ff3ea454107bfd28d0dc45 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 20:14:23 -0700 Subject: [PATCH 24/40] refactor(pdf-processor): improve error handling and cleanup in PDF text extraction process --- src/pdf-processor/pdf-text.service.ts | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/pdf-processor/pdf-text.service.ts b/src/pdf-processor/pdf-text.service.ts index c0568bd..b04bcaa 100644 --- a/src/pdf-processor/pdf-text.service.ts +++ b/src/pdf-processor/pdf-text.service.ts @@ -30,28 +30,39 @@ export class PdfTextService { const pdf = await loadingTask.promise; const pageTexts: string[] = []; - for (let pageNum = 1; pageNum <= pdf.numPages; pageNum += 1) { - const page = await pdf.getPage(pageNum); - const textContent = await page.getTextContent(); - const raw = textContent.items - .map((item) => ('str' in item ? String(item.str) : '')) - .join(' '); + try { + for (let pageNum = 1; pageNum <= pdf.numPages; pageNum += 1) { + const page = await pdf.getPage(pageNum); + try { + const textContent = await page.getTextContent(); + const raw = textContent.items + .map((item) => ('str' in item ? String(item.str) : '')) + .join(' '); - if (isLikelyMojibake(raw)) { - const normalized = normalizeExtractedText(raw); - if (!normalized) { - this.logger.warn( - `Page ${pageNum}: Mojibake detected but recovery failed, skipping extracted text`, - ); - } else { - this.logger.log(`Page ${pageNum}: Fixed mojibake in extracted text`); + if (isLikelyMojibake(raw)) { + const normalized = normalizeExtractedText(raw); + if (!normalized) { + this.logger.warn( + `Page ${pageNum}: Mojibake detected but recovery failed, skipping extracted text`, + ); + } else { + this.logger.log( + `Page ${pageNum}: Fixed mojibake in extracted text`, + ); + } + pageTexts.push(normalized); + } else { + pageTexts.push(raw); + } + } finally { + page.cleanup(); } - pageTexts.push(normalized); - } else { - pageTexts.push(raw); } + return pageTexts; + } finally { + // pdfjs-dist v6: document uses cleanup(); loadingTask.destroy() tears down the worker. + await pdf.cleanup(); + await loadingTask.destroy(); } - - return pageTexts; } } From 9029926c7d62612791055b114f0d0225c61e24e8 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 20:19:48 -0700 Subject: [PATCH 25/40] feat(upload): add 503 Service Unavailable response for GCS errors in upload and delete operations --- src/upload/upload.controller.ts | 9 +++++- src/upload/upload.service.spec.ts | 47 +++++++++++++++++++++++++++---- src/upload/upload.service.ts | 13 +++++---- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index 65248a5..5fc7f26 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -154,6 +154,10 @@ export class UploadController { status: 409, description: '동일 resource_name 문서가 이미 존재', }) + @ApiResponse({ + status: 503, + description: '문서 저장소(GCS) 일시 장애', + }) async upload( @CurrentAdmin() admin: AdminContext, @Req() req: FastifyRequest, @@ -280,13 +284,16 @@ export class UploadController { }) @ApiParam({ name: 'id', description: '문서 UUID', type: String }) @ApiResponse({ status: 204, description: '삭제 성공' }) - @ApiResponse({ status: 400, description: 'GCS 산출물 삭제 실패' }) @ApiResponse({ status: 401, description: '인증 실패' }) @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) @ApiResponse({ status: 404, description: '문서 없음 또는 이미 삭제됨', }) + @ApiResponse({ + status: 503, + description: '문서 저장소(GCS) 일시 장애', + }) async delete(@Param('id') id: string): Promise { await this.uploadService.delete(id); } diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts index 3c6bcd2..c74fcb3 100644 --- a/src/upload/upload.service.spec.ts +++ b/src/upload/upload.service.spec.ts @@ -29,13 +29,14 @@ function document(overrides: Partial = {}): Document { function createService() { const repo = { createUploading: jest.fn<(...args: unknown[]) => Promise>(), - markQueuedAfterUpload: - jest.fn<(id: string) => Promise>(), - hardDelete: jest.fn(), - cancelAndSoftDelete: jest.fn(), + markQueuedAfterUpload: jest.fn<(id: string) => Promise>(), + hardDelete: jest.fn<(id: string) => Promise>(), + cancelAndSoftDelete: jest.fn<(id: string) => Promise>(), findById: jest.fn<(id: string) => Promise>(), updateExpiresAt: - jest.fn<(id: string, expiresAt: Date | null) => Promise>(), + jest.fn< + (id: string, expiresAt: Date | null) => Promise + >(), enqueueReprocess: jest.fn< ( @@ -49,7 +50,7 @@ function createService() { toGsPath: jest.fn((path: string) => `gs://bucket/${path}`), uploadPdf: jest.fn<(resourceName: string, pdfBytes: Buffer) => Promise>(), - deleteResourceArtifacts: jest.fn(), + deleteResourceArtifacts: jest.fn<(resourceName: string) => Promise>(), }; return { service: new UploadService( @@ -101,6 +102,23 @@ describe('UploadService atomic transitions', () => { expect(gcs.uploadPdf).not.toHaveBeenCalled(); }); + it('maps GCS upload failures to 503 without exposing the raw error', async () => { + const { service, repo, gcs } = createService(); + repo.createUploading.mockResolvedValue(document()); + gcs.uploadPdf.mockRejectedValue(new Error('bucket ACL denied xyz')); + repo.hardDelete.mockResolvedValue(undefined); + + await expect( + service.upload(Buffer.from('%PDF-test'), 'test.pdf', '테스트', 'admin-1'), + ).rejects.toMatchObject({ + response: { + statusCode: 503, + message: 'Document storage is temporarily unavailable', + }, + }); + expect(repo.hardDelete).toHaveBeenCalled(); + }); + it('cancels the DB processing attempt before deleting GCS artifacts', async () => { const { service, repo, gcs } = createService(); const calls: string[] = []; @@ -118,6 +136,23 @@ describe('UploadService atomic transitions', () => { expect(calls).toEqual(['cancel', 'delete-artifacts']); }); + it('maps GCS delete failures to 503 without exposing the raw error', async () => { + const { service, repo, gcs } = createService(); + repo.cancelAndSoftDelete.mockResolvedValue(document({ isActive: false })); + gcs.deleteResourceArtifacts.mockRejectedValue( + new Error('Permission denied on objects/test/'), + ); + + await expect( + service.delete('00000000-0000-0000-0000-000000000001'), + ).rejects.toMatchObject({ + response: { + statusCode: 503, + message: 'Document storage is temporarily unavailable', + }, + }); + }); + it.each(['uploading', 'queued', 'processing'] as const)( 'rejects reprocess while status is %s', async (status) => { diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index 6ad5cc1..c7d7cd8 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -4,6 +4,7 @@ import { NotFoundException, BadRequestException, ConflictException, + ServiceUnavailableException, HttpException, HttpStatus, } from '@nestjs/common'; @@ -30,7 +31,9 @@ export function parseExpiresAt(raw?: string | null): Date | null { const parsed = new Date(trimmed); if (Number.isNaN(parsed.getTime())) { - throw new BadRequestException('expiresAt must be a valid ISO-8601 datetime'); + throw new BadRequestException( + 'expiresAt must be a valid ISO-8601 datetime', + ); } if (parsed.getTime() <= Date.now()) { throw new BadRequestException('expiresAt must be in the future'); @@ -120,8 +123,8 @@ export class UploadService { `GCS upload failed: ${error instanceof Error ? error.message : String(error)}`, ); await this.rollbackUpload(reservation.id, resourceName); - throw new BadRequestException( - `GCS upload failed: ${error instanceof Error ? error.message : String(error)}`, + throw new ServiceUnavailableException( + 'Document storage is temporarily unavailable', ); } @@ -157,8 +160,8 @@ export class UploadService { this.logger.error( `GCS delete failed id=${id}: ${error instanceof Error ? error.message : String(error)}`, ); - throw new BadRequestException( - `GCS delete failed: ${error instanceof Error ? error.message : String(error)}`, + throw new ServiceUnavailableException( + 'Document storage is temporarily unavailable', ); } From 66e2b357d002cbb04e702032c221e3bd9a7cddfd Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 20:21:05 -0700 Subject: [PATCH 26/40] refactor(retrieval): remove content field from ReadyDocumentWithChunks and related logic --- src/retrieval/retrieval.repository.ts | 3 --- src/retrieval/retrieval.service.spec.ts | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/retrieval/retrieval.repository.ts b/src/retrieval/retrieval.repository.ts index b63f493..d303c9f 100644 --- a/src/retrieval/retrieval.repository.ts +++ b/src/retrieval/retrieval.repository.ts @@ -11,7 +11,6 @@ export type ReadyDocumentWithChunks = { chunks: Array<{ path: string; description: string; - content: string; sortOrder: number; }>; }; @@ -47,7 +46,6 @@ export class RetrievalRepository { chunkId: documentChunks.id, chunkPath: documentChunks.path, chunkDescription: documentChunks.description, - chunkContent: documentChunks.content, chunkSortOrder: documentChunks.sortOrder, }) .from(documents) @@ -77,7 +75,6 @@ export class RetrievalRepository { doc.chunks.push({ path: row.chunkPath, description: row.chunkDescription, - content: row.chunkContent, sortOrder: row.chunkSortOrder, }); } diff --git a/src/retrieval/retrieval.service.spec.ts b/src/retrieval/retrieval.service.spec.ts index 87e169d..d34756f 100644 --- a/src/retrieval/retrieval.service.spec.ts +++ b/src/retrieval/retrieval.service.spec.ts @@ -15,7 +15,6 @@ describe('RetrievalService', () => { { path: '학사편람/졸업요건', description: '졸업', - content: '본문', sortOrder: 0, }, ], @@ -71,7 +70,6 @@ describe('RetrievalService', () => { { path: 'doc/a', description: 'a', - content: 'c', sortOrder: 0, }, ], From 1cba493b3f392e0045b914a0588be40efeb029e9 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:21:07 -0700 Subject: [PATCH 27/40] feat(docker): add OpenRouter configuration and Swagger API credentials to docker-compose --- docker-compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a1c8fc8..41bfda7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,16 @@ services: LETSUR_AI_GATEWAY_MODEL_HEAVY: ${LETSUR_AI_GATEWAY_MODEL_HEAVY:-} LETSUR_AI_GATEWAY_X_TITLE: ${LETSUR_AI_GATEWAY_X_TITLE:-} LLM_PROVIDER: ${LLM_PROVIDER:-letsur} + # OpenRouter Configuration (LLM_PROVIDER=openrouter) + OPEN_ROUTER_API_KEY: ${OPEN_ROUTER_API_KEY:-} + OPEN_ROUTER_BASE_URL: ${OPEN_ROUTER_BASE_URL:-} + OPEN_ROUTER_MODEL_LIGHT: ${OPEN_ROUTER_MODEL_LIGHT:-} + OPEN_ROUTER_MODEL_NORMAL: ${OPEN_ROUTER_MODEL_NORMAL:-} + OPEN_ROUTER_MODEL_HEAVY: ${OPEN_ROUTER_MODEL_HEAVY:-} + OPEN_ROUTER_X_TITLE: ${OPEN_ROUTER_X_TITLE:-} + # Swagger API 문서 잠금 + SWAGGER_USER: ${SWAGGER_USER:-} + SWAGGER_PASSWORD: ${SWAGGER_PASSWORD:-} depends_on: postgres: condition: service_healthy From 0533bc814f641ad401edf969e76133303c5874ab Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:21:47 -0700 Subject: [PATCH 28/40] feat(chat): add Access-Control-Allow-Credentials header for CORS support --- src/chat/services/chat-stream.transport.spec.ts | 1 + src/chat/services/chat-stream.transport.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chat/services/chat-stream.transport.spec.ts b/src/chat/services/chat-stream.transport.spec.ts index 51cc37b..500d0be 100644 --- a/src/chat/services/chat-stream.transport.spec.ts +++ b/src/chat/services/chat-stream.transport.spec.ts @@ -27,6 +27,7 @@ describe('ChatStreamTransport', () => { expect.objectContaining({ 'Content-Type': 'text/event-stream', 'Access-Control-Allow-Origin': 'http://localhost:5173', + 'Access-Control-Allow-Credentials': 'true', }), ); }); diff --git a/src/chat/services/chat-stream.transport.ts b/src/chat/services/chat-stream.transport.ts index aac1ba5..0932a36 100644 --- a/src/chat/services/chat-stream.transport.ts +++ b/src/chat/services/chat-stream.transport.ts @@ -35,13 +35,14 @@ export class ChatStreamTransport { const corsOrigin = requestOrigin && allowedOrigins.includes(requestOrigin) ? requestOrigin - : (allowedOrigins[1] ?? '*'); + : allowedOrigins[1]; reply.raw.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive', 'Access-Control-Allow-Origin': corsOrigin, + 'Access-Control-Allow-Credentials': 'true', 'Access-Control-Allow-Methods': 'POST, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type, Authorization', }); From 0bf3908faf55b3f2431d2d4aa5ce9add9d0a738d Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:22:59 -0700 Subject: [PATCH 29/40] test(chat): add tests for SSE error handling and JSON parse failures in ChatStreamTransport --- .../services/chat-stream.transport.spec.ts | 41 +++++++++++ src/chat/services/chat-stream.transport.ts | 73 +++++++++++++++---- 2 files changed, 98 insertions(+), 16 deletions(-) diff --git a/src/chat/services/chat-stream.transport.spec.ts b/src/chat/services/chat-stream.transport.spec.ts index 500d0be..ee7df5e 100644 --- a/src/chat/services/chat-stream.transport.spec.ts +++ b/src/chat/services/chat-stream.transport.spec.ts @@ -72,6 +72,47 @@ describe('ChatStreamTransport', () => { ); }); + it('ends the SSE response and rejects when the upstream stream errors', async () => { + const transport = createTransport(); + const reply = { + raw: { write: jest.fn(), end: jest.fn(), writableEnded: false }, + }; + const stream = new PassThrough(); + const consumePromise = transport.consumeAndForward(stream, reply as never); + + stream.destroy(new Error('upstream timeout')); + + await expect(consumePromise).rejects.toThrow('upstream timeout'); + expect(reply.raw.write).toHaveBeenCalledWith( + `data: ${JSON.stringify({ error: 'upstream timeout' })}\n\n`, + ); + expect(reply.raw.end).toHaveBeenCalledTimes(1); + }); + + it('does not swallow SSE write failures as JSON parse failures', async () => { + const transport = createTransport(); + const reply = { + raw: { + write: jest.fn(() => { + throw new Error('socket closed'); + }), + end: jest.fn(), + writableEnded: false, + }, + }; + const stream = new PassThrough(); + const consumePromise = transport.consumeAndForward(stream, reply as never); + + stream.write( + `data: ${JSON.stringify({ + choices: [{ delta: { content: '안녕' } }], + })}\n\n`, + ); + + await expect(consumePromise).rejects.toThrow('socket closed'); + expect(reply.raw.end).toHaveBeenCalledTimes(1); + }); + it('writes resources and done events', () => { const transport = createTransport(); const reply = { diff --git a/src/chat/services/chat-stream.transport.ts b/src/chat/services/chat-stream.transport.ts index 0932a36..a9f335e 100644 --- a/src/chat/services/chat-stream.transport.ts +++ b/src/chat/services/chat-stream.transport.ts @@ -61,8 +61,11 @@ export class ChatStreamTransport { let model = ''; let usage: LlmUsage | null = null; let buffer = ''; + let settled = false; stream.on('data', (chunk: Buffer) => { + if (settled) return; + buffer += chunk.toString(); const lines = buffer.split('\n'); buffer = lines.pop() || ''; @@ -73,35 +76,73 @@ export class ChatStreamTransport { const data = line.slice(6).trim(); if (!data || data === '[DONE]') continue; + let parsed: { + choices?: Array<{ delta?: { content?: string } }>; + model?: string; + usage?: LlmUsage; + }; try { - const parsed = JSON.parse(data); - if (parsed.choices?.[0]?.delta?.content) { - const content = parsed.choices[0].delta.content; - accumulatedContent += content; + parsed = JSON.parse(data) as typeof parsed; + } catch { + // JSON 파싱 실패 시 해당 이벤트만 무시 + continue; + } + + const content = parsed.choices?.[0]?.delta?.content; + if (content) { + accumulatedContent += content; + try { reply.raw.write(`data: ${JSON.stringify({ content })}\n\n`); + } catch (error) { + settled = true; + const streamError = + error instanceof Error ? error : new Error(String(error)); + this.logger.error('SSE write error:', streamError); + if (!reply.raw.writableEnded) { + try { + reply.raw.end(); + } catch { + // Socket may already be unavailable. + } + } + if (!stream.destroyed) stream.destroy(); + reject(streamError); + return; } - if (parsed.model) { - model = parsed.model; - } - if (parsed.usage) { - usage = parsed.usage; - } - } catch { - // JSON 파싱 실패 시 무시 + } + if (parsed.model) { + model = parsed.model; + } + if (parsed.usage) { + usage = parsed.usage; } } }); stream.on('error', (error: Error) => { + if (settled) return; + settled = true; this.logger.error('Stream error:', error); - reply.raw.write( - `data: ${JSON.stringify({ error: error.message || 'Stream error' })}\n\n`, - ); - reply.raw.end(); + try { + reply.raw.write( + `data: ${JSON.stringify({ error: error.message || 'Stream error' })}\n\n`, + ); + } catch { + // Socket may already be unavailable. + } + if (!reply.raw.writableEnded) { + try { + reply.raw.end(); + } catch { + // Socket may already be unavailable. + } + } reject(error); }); stream.on('end', () => { + if (settled) return; + settled = true; resolve({ accumulatedContent, model, usage }); }); }); From c8413c560a1d8e4123ffa8d0ab3df01e7e305bef Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:24:07 -0700 Subject: [PATCH 30/40] feat(pdf-processor): implement normalizeRelativeChunkPath function and add tests for path normalization --- src/pdf-processor/pdf-chunk-parser.spec.ts | 13 +++++++++ src/pdf-processor/pdf-chunk-parser.ts | 23 ++++++++++++++- .../pdf-pipeline.service.spec.ts | 28 +++++++++++++++++++ src/pdf-processor/pdf-pipeline.service.ts | 10 +++++-- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/pdf-processor/pdf-chunk-parser.spec.ts b/src/pdf-processor/pdf-chunk-parser.spec.ts index 1edba82..89b6808 100644 --- a/src/pdf-processor/pdf-chunk-parser.spec.ts +++ b/src/pdf-processor/pdf-chunk-parser.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from '@jest/globals'; import { + normalizeRelativeChunkPath, parseChunksFromMarkdown, toRelativeChunkPath, toResourceName, @@ -26,6 +27,18 @@ describe('toRelativeChunkPath', () => { }); }); +describe('normalizeRelativeChunkPath', () => { + it('removes current-directory and empty segments', () => { + expect(normalizeRelativeChunkPath('./학사//졸업')).toBe('학사/졸업'); + }); + + it('rejects parent traversal with slash or backslash separators', () => { + expect(normalizeRelativeChunkPath('../외부/문서')).toBe(''); + expect(normalizeRelativeChunkPath('학사/../외부')).toBe(''); + expect(normalizeRelativeChunkPath('학사\\..\\외부')).toBe(''); + }); +}); + describe('parseChunksFromMarkdown', () => { it('parses summary, preserves overview, and normalizes relative paths', () => { const input = ` diff --git a/src/pdf-processor/pdf-chunk-parser.ts b/src/pdf-processor/pdf-chunk-parser.ts index 7043f64..b72321a 100644 --- a/src/pdf-processor/pdf-chunk-parser.ts +++ b/src/pdf-processor/pdf-chunk-parser.ts @@ -26,6 +26,25 @@ export function toRelativeChunkPath(raw: string, baseName: string): string { return p; } +/** + * Normalize an untrusted relative chunk path. + * Dot segments are removed, while parent traversal is rejected rather than + * resolved so an LLM path can never reference outside the document prefix. + */ +export function normalizeRelativeChunkPath(raw: string): string { + const segments = raw.replace(/\\/g, '/').split('/'); + const normalized: string[] = []; + + for (const segment of segments) { + const trimmed = segment.trim(); + if (!trimmed || trimmed === '.') continue; + if (trimmed === '..') return ''; + normalized.push(trimmed); + } + + return normalized.join('/'); +} + function resourceStem(resourceName: string): string { if ( resourceName.includes('.') && @@ -67,7 +86,9 @@ export function parseChunksFromMarkdown( /(.+?)<\/document>/gs; const chunks: ParsedChunk[] = []; for (const match of markdown.matchAll(chunkPattern)) { - const relative = toRelativeChunkPath(match[1], baseName); + const relative = normalizeRelativeChunkPath( + toRelativeChunkPath(match[1], baseName), + ); if (!relative) continue; chunks.push({ path: relative, diff --git a/src/pdf-processor/pdf-pipeline.service.spec.ts b/src/pdf-processor/pdf-pipeline.service.spec.ts index 6ed5c39..adc7031 100644 --- a/src/pdf-processor/pdf-pipeline.service.spec.ts +++ b/src/pdf-processor/pdf-pipeline.service.spec.ts @@ -154,6 +154,34 @@ describe('PdfPipelineService metadata pass', () => { ).rejects.toThrow(/incomplete batch/); }); + it('falls back to the section title when metadata path traverses upward', async () => { + const callLLM = jest + .fn<(...args: unknown[]) => Promise>() + .mockResolvedValueOnce(llmResponse(`## 안전한 제목\n\n${'본문 '.repeat(700)}`)) + .mockResolvedValueOnce( + llmResponse( + JSON.stringify({ + summary: '요약', + chunks: [ + { + index: 0, + path: '../외부/문서', + description: '설명', + }, + ], + }), + ), + ); + + const pipeline = createPipeline({ pages: ['p1'], callLLM }); + const result = await pipeline.processPdf(Buffer.from('%PDF'), 'doc.pdf'); + + expect(result.documents['doc/안전한-제목.md']).toBeDefined(); + expect(Object.keys(result.documents).some((path) => path.includes('..'))).toBe( + false, + ); + }); + it('throws when Pass 1 page LLM failures exceed the ratio threshold', async () => { const callLLM = jest .fn<(...args: unknown[]) => Promise>() diff --git a/src/pdf-processor/pdf-pipeline.service.ts b/src/pdf-processor/pdf-pipeline.service.ts index 0c50c90..6dc25f7 100644 --- a/src/pdf-processor/pdf-pipeline.service.ts +++ b/src/pdf-processor/pdf-pipeline.service.ts @@ -9,7 +9,11 @@ import { type MarkdownSection, } from './markdown-section-splitter'; import { parseFiniteNumber } from './parse-finite-number'; -import { toRelativeChunkPath, toResourceName } from './pdf-chunk-parser'; +import { + normalizeRelativeChunkPath, + toRelativeChunkPath, + toResourceName, +} from './pdf-chunk-parser'; import type { ResourceIndexEntry } from './gcs-storage.service'; export type PipelineChunk = { @@ -374,7 +378,9 @@ export class PdfPipelineService { throw new Error(`Missing section for labeled index ${meta.index}`); } - let relative = toRelativeChunkPath(meta.path, baseName); + let relative = normalizeRelativeChunkPath( + toRelativeChunkPath(meta.path, baseName), + ); if (!relative) { relative = slugifyTitle(section.title) || `section-${meta.index + 1}`; } From f019df9e5836672d5f959b76a91adcec4e7b4dbc Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:25:06 -0700 Subject: [PATCH 31/40] feat(pdf-processor): enhance GcsStorageService to aggregate deletion failures and improve error handling --- src/pdf-processor/gcs-storage.service.spec.ts | 44 ++++++++++++++++++- src/pdf-processor/gcs-storage.service.ts | 25 ++++++----- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/pdf-processor/gcs-storage.service.spec.ts b/src/pdf-processor/gcs-storage.service.spec.ts index f5bfdbf..c02f50b 100644 --- a/src/pdf-processor/gcs-storage.service.spec.ts +++ b/src/pdf-processor/gcs-storage.service.spec.ts @@ -1,5 +1,8 @@ -import { describe, expect, it } from '@jest/globals'; -import { decodeServiceAccountCredentials } from './gcs-storage.service'; +import { describe, expect, it, jest } from '@jest/globals'; +import { + decodeServiceAccountCredentials, + GcsStorageService, +} from './gcs-storage.service'; function encode(value: unknown): string { return Buffer.from(JSON.stringify(value), 'utf-8').toString('base64'); @@ -37,3 +40,40 @@ describe('decodeServiceAccountCredentials', () => { ).toThrow(/client_email and private_key/); }); }); + +describe('GcsStorageService deletion', () => { + it('attempts every deletion and propagates aggregated failures', async () => { + const warn = jest.fn(); + const service = Object.create( + GcsStorageService.prototype, + ) as GcsStorageService; + Object.defineProperty(service, 'logger', { value: { warn } }); + + type DeleteOptions = { ignoreNotFound: boolean }; + const firstDelete = jest.fn(async (_options: DeleteOptions) => undefined); + const failedDelete = jest.fn(async (_options: DeleteOptions) => { + throw new Error('permission denied'); + }); + const lastDelete = jest.fn(async (_options: DeleteOptions) => undefined); + const files = [ + { name: 'doc.pdf', delete: firstDelete }, + { name: 'doc.md', delete: failedDelete }, + { name: 'doc/chunk.md', delete: lastDelete }, + ]; + const deleteFiles = ( + service as unknown as { + deleteFiles(items: typeof files): Promise; + } + ).deleteFiles.bind(service); + + await expect(deleteFiles(files)).rejects.toThrow( + 'Failed to delete 1 GCS object(s)', + ); + expect(firstDelete).toHaveBeenCalledWith({ ignoreNotFound: true }); + expect(failedDelete).toHaveBeenCalledWith({ ignoreNotFound: true }); + expect(lastDelete).toHaveBeenCalledWith({ ignoreNotFound: true }); + expect(warn).toHaveBeenCalledWith( + 'Failed to delete doc.md: permission denied', + ); + }); +}); diff --git a/src/pdf-processor/gcs-storage.service.ts b/src/pdf-processor/gcs-storage.service.ts index 1c06dd1..b6bd980 100644 --- a/src/pdf-processor/gcs-storage.service.ts +++ b/src/pdf-processor/gcs-storage.service.ts @@ -139,16 +139,21 @@ export class GcsStorageService { } private async deleteFiles(files: File[]): Promise { - await Promise.all( - files.map(async (file) => { - try { - await file.delete({ ignoreNotFound: true }); - } catch (error) { - this.logger.warn( - `Failed to delete ${file.name}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - }), + const results = await Promise.allSettled( + files.map((file) => file.delete({ ignoreNotFound: true })), ); + + let failureCount = 0; + results.forEach((result, index) => { + if (result.status === 'fulfilled') return; + failureCount += 1; + this.logger.warn( + `Failed to delete ${files[index].name}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`, + ); + }); + + if (failureCount > 0) { + throw new Error(`Failed to delete ${failureCount} GCS object(s)`); + } } } From 038d523b2c6a6c53e0970b1dbea903be64b62a6c Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:27:36 -0700 Subject: [PATCH 32/40] feat(pdf-processor): implement heartbeat processing for document ownership management --- src/pdf-processor/documents.repository.ts | 22 +++++ .../pdf-processor.worker.spec.ts | 63 ++++++++++++++- src/pdf-processor/pdf-processor.worker.ts | 80 ++++++++++++++++++- 3 files changed, 158 insertions(+), 7 deletions(-) diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index b98f375..06e442f 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -192,6 +192,28 @@ export class DocumentsRepository { return result.length; } + /** + * Refresh the stale-processing lease only while this exact attempt owns it. + */ + async heartbeatProcessing( + documentId: string, + processingToken: string, + ): Promise { + const result = await this.db + .update(documents) + .set({ updatedAt: new Date() }) + .where( + and( + eq(documents.id, documentId), + eq(documents.status, 'processing'), + eq(documents.processingToken, processingToken), + eq(documents.isActive, true), + ), + ) + .returning({ id: documents.id }); + return result.length > 0; + } + /** * Persist chunks and mark ready only if this exact processing attempt still * owns the document. A delete/reprocess/stale recovery clears the token. diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 7b54c7f..7b1fd03 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -35,6 +35,9 @@ function createWorker(options: { sortOrder: number; }>; processPdfError?: Error; + uploadDocumentsError?: Error; + heartbeatOwned?: boolean; + markFailed?: boolean; }) { const chunks = options.chunks ?? [ { @@ -66,13 +69,18 @@ function createWorker(options: { processingToken: string, errorMessage: string, ) => Promise - >(() => Promise.resolve(true)), + >(() => Promise.resolve(options.markFailed ?? true)), + heartbeatProcessing: jest.fn< + (id: string, processingToken: string) => Promise + >(() => Promise.resolve(options.heartbeatOwned ?? true)), requeueStaleProcessing: jest.fn(() => Promise.resolve(0)), claimQueued: jest.fn(() => Promise.resolve([])), }; const gcs = { downloadPdf: jest.fn(() => Promise.resolve(Buffer.from('%PDF-test'))), - uploadDocuments: jest.fn(() => Promise.resolve()), + uploadDocuments: options.uploadDocumentsError + ? jest.fn(() => Promise.reject(options.uploadDocumentsError)) + : jest.fn(() => Promise.resolve()), deleteProcessedArtifacts: jest.fn<(resourceName: string) => Promise>( () => Promise.resolve(), ), @@ -116,7 +124,7 @@ function createWorker(options: { } describe('PdfProcessorWorker attempt ownership', () => { - it('deletes generated artifacts when a delete/reprocess cancels the attempt', async () => { + it('does not delete shared artifacts when completion loses ownership', async () => { const { worker, repo, gcs } = createWorker({ completeProcessing: false }); const callable = worker as unknown as { processDocument(doc: Document): Promise; @@ -125,7 +133,7 @@ describe('PdfProcessorWorker attempt ownership', () => { await callable.processDocument(processingDocument()); expect(repo.completeProcessing).toHaveBeenCalled(); - expect(gcs.deleteProcessedArtifacts).toHaveBeenCalledWith('test'); + expect(gcs.deleteProcessedArtifacts).not.toHaveBeenCalled(); }); it('keeps generated artifacts when the attempt completes successfully', async () => { @@ -140,6 +148,53 @@ describe('PdfProcessorWorker attempt ownership', () => { expect(gcs.deleteProcessedArtifacts).not.toHaveBeenCalled(); }); + it('heartbeats the processing token while an attempt is active', async () => { + const { worker, repo } = createWorker({ completeProcessing: true }); + const doc = processingDocument(); + const callable = worker as unknown as { + processDocument(document: Document): Promise; + }; + + await callable.processDocument(doc); + + expect(repo.heartbeatProcessing).toHaveBeenCalledWith( + doc.id, + '00000000-0000-0000-0000-000000000002', + ); + }); + + it('does not upload when the heartbeat reports ownership loss', async () => { + const { worker, repo, gcs } = createWorker({ + completeProcessing: false, + heartbeatOwned: false, + }); + const callable = worker as unknown as { + processDocument(document: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(repo.heartbeatProcessing).toHaveBeenCalled(); + expect(gcs.uploadDocuments).not.toHaveBeenCalled(); + expect(repo.completeProcessing).not.toHaveBeenCalled(); + }); + + it('skips cleanup when markFailed reports ownership loss', async () => { + const { worker, repo, gcs } = createWorker({ + completeProcessing: true, + uploadDocumentsError: new Error('upload interrupted'), + markFailed: false, + }); + const callable = worker as unknown as { + processDocument(document: Document): Promise; + }; + + await callable.processDocument(processingDocument()); + + expect(repo.markFailed).toHaveBeenCalled(); + expect(gcs.deleteProcessedArtifacts).not.toHaveBeenCalled(); + }); + it('marks failed when pipeline throws (e.g. LLM timeout)', async () => { const { worker, repo, gcs } = createWorker({ completeProcessing: true, diff --git a/src/pdf-processor/pdf-processor.worker.ts b/src/pdf-processor/pdf-processor.worker.ts index cbf4a99..16f8089 100644 --- a/src/pdf-processor/pdf-processor.worker.ts +++ b/src/pdf-processor/pdf-processor.worker.ts @@ -17,6 +17,7 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { private readonly concurrency: number; private readonly pollIntervalMs: number; private readonly staleProcessingMs: number; + private readonly heartbeatIntervalMs: number; private readonly staleCheckIntervalMs = 60_000; private activeCount = 0; private readonly activeDocumentIds = new Set(); @@ -47,6 +48,10 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { 30 * 60 * 1000, { min: 60_000 }, ); + this.heartbeatIntervalMs = Math.max( + 10_000, + Math.min(60_000, Math.floor(this.staleProcessingMs / 3)), + ); } async onModuleInit(): Promise { @@ -131,6 +136,7 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { return; } + const heartbeat = this.startHeartbeat(doc.id, processingToken); let generatedArtifactsMayExist = false; try { const pdfBytes = await this.gcs.downloadPdf(resourceName); @@ -145,6 +151,13 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { ); } + if (heartbeat.hasLostOwnership()) { + this.logger.warn( + `Discarding processing result after ownership loss: id=${doc.id} token=${processingToken}`, + ); + return; + } + generatedArtifactsMayExist = true; await this.gcs.uploadDocuments(result.documents); const completed = await this.documentsRepo.completeProcessing( @@ -157,7 +170,9 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { this.logger.warn( `Discarding cancelled processing result: id=${doc.id} token=${processingToken}`, ); - await this.cleanupGeneratedArtifacts(resourceName); + // Paths are shared by attempts. A newer owner may already have + // uploaded its artifacts, so an attempt that lost ownership must not + // delete them. return; } @@ -170,19 +185,78 @@ export class PdfProcessorWorker implements OnModuleInit, OnModuleDestroy { `Processing failed id=${doc.id} name=${resourceName}: ${message}`, error instanceof Error ? error.stack : undefined, ); + let markedFailed = false; try { - await this.documentsRepo.markFailed(doc.id, processingToken, message); + markedFailed = await this.documentsRepo.markFailed( + doc.id, + processingToken, + message, + ); } catch (markError) { this.logger.error( `Failed to persist processing error id=${doc.id}: ${markError instanceof Error ? markError.message : String(markError)}`, ); } - if (generatedArtifactsMayExist) { + if (generatedArtifactsMayExist && markedFailed) { await this.cleanupGeneratedArtifacts(resourceName); + } else if (generatedArtifactsMayExist) { + this.logger.warn( + `Skipping artifact cleanup after ownership loss: id=${doc.id} token=${processingToken}`, + ); } + } finally { + heartbeat.stop(); } } + private startHeartbeat( + documentId: string, + processingToken: string, + ): { + stop: () => void; + hasLostOwnership: () => boolean; + } { + let stopped = false; + let ownershipLost = false; + let inFlight = false; + + const beat = async (): Promise => { + if (stopped || ownershipLost || inFlight) return; + inFlight = true; + try { + const owned = await this.documentsRepo.heartbeatProcessing( + documentId, + processingToken, + ); + if (stopped) return; + if (!owned) { + ownershipLost = true; + this.logger.warn( + `Processing heartbeat lost ownership: id=${documentId} token=${processingToken}`, + ); + } + } catch (error) { + this.logger.error( + `Processing heartbeat failed id=${documentId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + inFlight = false; + } + }; + + const timer = setInterval(() => void beat(), this.heartbeatIntervalMs); + if (typeof timer.unref === 'function') timer.unref(); + void beat(); + + return { + stop: () => { + stopped = true; + clearInterval(timer); + }, + hasLostOwnership: () => ownershipLost, + }; + } + private async cleanupGeneratedArtifacts(resourceName: string): Promise { try { await this.gcs.deleteProcessedArtifacts(resourceName); From 9ded3df7f4497508bb08e536271600723382a5a6 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:28:34 -0700 Subject: [PATCH 33/40] feat(journal): add new journal entry for aberrant_alex_wilder with breakpoints enabled --- drizzle/0013_aberrant_alex_wilder.sql | 2 + drizzle/meta/0013_snapshot.json | 1402 +++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + src/db/schema.ts | 4 + 4 files changed, 1415 insertions(+) create mode 100644 drizzle/0013_aberrant_alex_wilder.sql create mode 100644 drizzle/meta/0013_snapshot.json diff --git a/drizzle/0013_aberrant_alex_wilder.sql b/drizzle/0013_aberrant_alex_wilder.sql new file mode 100644 index 0000000..bf3aa80 --- /dev/null +++ b/drizzle/0013_aberrant_alex_wilder.sql @@ -0,0 +1,2 @@ +CREATE INDEX "document_chunks_path_idx" ON "document_chunks" USING btree ("path");--> statement-breakpoint +CREATE UNIQUE INDEX "document_chunks_document_id_path_unique" ON "document_chunks" USING btree ("document_id","path"); \ No newline at end of file diff --git a/drizzle/meta/0013_snapshot.json b/drizzle/meta/0013_snapshot.json new file mode 100644 index 0000000..0c459e9 --- /dev/null +++ b/drizzle/meta/0013_snapshot.json @@ -0,0 +1,1402 @@ +{ + "id": "5f7ce177-093e-4298-bc1e-6c0e5b5b9332", + "prevId": "3c7f053c-f6f3-405d-a29d-82f19edb33aa", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_path_idx": { + "name": "document_chunks_path_idx", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_id_path_unique": { + "name": "document_chunks_document_id_path_unique", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_reprocessed_at": { + "name": "last_reprocessed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_expires_at_idx": { + "name": "documents_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index fc167f7..25dcb9f 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -92,6 +92,13 @@ "when": 1785445409151, "tag": "0012_brainy_dagger", "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1785475702240, + "tag": "0013_aberrant_alex_wilder", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index 7e70fc6..6615946 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -260,6 +260,10 @@ export const documentChunks = pgTable( table.documentId, table.sortOrder, ), + pathIdx: index('document_chunks_path_idx').on(table.path), + documentPathUnique: uniqueIndex( + 'document_chunks_document_id_path_unique', + ).on(table.documentId, table.path), }), ); From 7e24bec85f3e39666efb0e85a6b163b6851f86b0 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:29:12 -0700 Subject: [PATCH 34/40] feat(retrieval): add support for preserving unknown dotted path segments and enhance extension handling --- src/retrieval/retrieval.service.spec.ts | 17 +++++++++++++++++ src/retrieval/retrieval.service.ts | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/retrieval/retrieval.service.spec.ts b/src/retrieval/retrieval.service.spec.ts index d34756f..dba8e71 100644 --- a/src/retrieval/retrieval.service.spec.ts +++ b/src/retrieval/retrieval.service.spec.ts @@ -58,6 +58,23 @@ describe('RetrievalService', () => { expect(hits).toEqual([{ path: '학사편람/졸업요건', content: '졸업 본문' }]); }); + it('preserves dotted path segments that are not known extensions', async () => { + const repo = { + listReadyWithChunks: jest.fn(), + findChunkContentsByPaths: jest.fn(async (paths: string[]) => { + expect(paths).toEqual(['규정/3.2', '가이드/v1.2']); + return paths.map((path) => ({ path, content: `${path} 본문` })); + }), + }; + const service = new RetrievalService( + repo as unknown as RetrievalRepository, + ); + + const hits = await service.getContentsByPaths(['규정/3.2', '가이드/v1.2']); + + expect(hits).toHaveLength(2); + }); + it('uses title when summary is empty', async () => { const repo = { listReadyWithChunks: jest.fn(async () => [ diff --git a/src/retrieval/retrieval.service.ts b/src/retrieval/retrieval.service.ts index 0c61372..f234e92 100644 --- a/src/retrieval/retrieval.service.ts +++ b/src/retrieval/retrieval.service.ts @@ -8,6 +8,16 @@ import type { @Injectable() export class RetrievalService { + private static readonly KNOWN_EXTENSIONS = new Set([ + 'md', + 'pdf', + 'png', + 'jpg', + 'jpeg', + 'gif', + 'webp', + ]); + private readonly logger = new Logger(RetrievalService.name); constructor(private readonly retrievalRepo: RetrievalRepository) {} @@ -77,8 +87,8 @@ export class RetrievalService { private stripKnownExtension(path: string): string { if (!path.includes('.')) return path; const lastDot = path.lastIndexOf('.'); - const extension = path.substring(lastDot + 1); - if (extension.length <= 5 && /^[a-z0-9]+$/i.test(extension)) { + const extension = path.substring(lastDot + 1).toLowerCase(); + if (RetrievalService.KNOWN_EXTENSIONS.has(extension)) { return path.substring(0, lastDot); } return path; From af9ed51006672ddc05182abf3b55a36db6362a42 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Thu, 30 Jul 2026 22:30:21 -0700 Subject: [PATCH 35/40] feat(upload): enhance document management by adding ownership checks to reprocess and delete operations --- src/pdf-processor/documents.repository.ts | 32 ++++++++-- src/upload/upload.controller.ts | 14 +++-- src/upload/upload.service.spec.ts | 75 ++++++++++++++++++++--- src/upload/upload.service.ts | 25 ++++++-- 4 files changed, 122 insertions(+), 24 deletions(-) diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index 06e442f..ea70a8d 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -55,14 +55,24 @@ export class DocumentsRepository { return row; } - async updateExpiresAt(id: string, expiresAt: Date | null) { + async updateExpiresAt( + id: string, + uploadedByIdpUuid: string, + expiresAt: Date | null, + ) { const [row] = await this.db .update(documents) .set({ expiresAt, updatedAt: new Date(), }) - .where(and(eq(documents.id, id), eq(documents.isActive, true))) + .where( + and( + eq(documents.id, id), + eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), + eq(documents.isActive, true), + ), + ) .returning(); return row ?? null; } @@ -292,7 +302,7 @@ export class DocumentsRepository { /** * Cancel the current attempt before deleting external artifacts. */ - async cancelAndSoftDelete(id: string) { + async cancelAndSoftDelete(id: string, uploadedByIdpUuid: string) { const [row] = await this.db .update(documents) .set({ @@ -300,12 +310,23 @@ export class DocumentsRepository { processingToken: null, updatedAt: new Date(), }) - .where(and(eq(documents.id, id), eq(documents.isActive, true))) + .where( + and( + eq(documents.id, id), + eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), + eq(documents.isActive, true), + ), + ) .returning(); return row ?? null; } - async enqueueReprocess(id: string, cooldownBefore: Date, now: Date) { + async enqueueReprocess( + id: string, + uploadedByIdpUuid: string, + cooldownBefore: Date, + now: Date, + ) { return this.db.transaction(async (tx) => { const [row] = await tx .update(documents) @@ -320,6 +341,7 @@ export class DocumentsRepository { .where( and( eq(documents.id, id), + eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), eq(documents.isActive, true), inArray(documents.status, ['ready', 'failed']), or( diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index 5fc7f26..c37b297 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -271,8 +271,11 @@ export class UploadController { }, }, }) - async reprocess(@Param('id') id: string) { - return this.uploadService.reprocess(id); + async reprocess( + @CurrentAdmin() admin: AdminContext, + @Param('id') id: string, + ) { + return this.uploadService.reprocess(id, admin.uuid); } @Delete(':id') @@ -294,8 +297,11 @@ export class UploadController { status: 503, description: '문서 저장소(GCS) 일시 장애', }) - async delete(@Param('id') id: string): Promise { - await this.uploadService.delete(id); + async delete( + @CurrentAdmin() admin: AdminContext, + @Param('id') id: string, + ): Promise { + await this.uploadService.delete(id, admin.uuid); } } diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts index c74fcb3..b6fce99 100644 --- a/src/upload/upload.service.spec.ts +++ b/src/upload/upload.service.spec.ts @@ -31,16 +31,22 @@ function createService() { createUploading: jest.fn<(...args: unknown[]) => Promise>(), markQueuedAfterUpload: jest.fn<(id: string) => Promise>(), hardDelete: jest.fn<(id: string) => Promise>(), - cancelAndSoftDelete: jest.fn<(id: string) => Promise>(), + cancelAndSoftDelete: + jest.fn<(id: string, idpUuid: string) => Promise>(), findById: jest.fn<(id: string) => Promise>(), updateExpiresAt: jest.fn< - (id: string, expiresAt: Date | null) => Promise + ( + id: string, + idpUuid: string, + expiresAt: Date | null, + ) => Promise >(), enqueueReprocess: jest.fn< ( id: string, + idpUuid: string, cooldownBefore: Date, now: Date, ) => Promise @@ -131,9 +137,16 @@ describe('UploadService atomic transitions', () => { return Promise.resolve(); }); - await service.delete('00000000-0000-0000-0000-000000000001'); + await service.delete( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ); expect(calls).toEqual(['cancel', 'delete-artifacts']); + expect(repo.cancelAndSoftDelete).toHaveBeenCalledWith( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ); }); it('maps GCS delete failures to 503 without exposing the raw error', async () => { @@ -144,7 +157,10 @@ describe('UploadService atomic transitions', () => { ); await expect( - service.delete('00000000-0000-0000-0000-000000000001'), + service.delete( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ), ).rejects.toMatchObject({ response: { statusCode: 503, @@ -153,6 +169,36 @@ describe('UploadService atomic transitions', () => { }); }); + it('returns 404 without touching GCS when delete ownership does not match', async () => { + const { service, repo, gcs } = createService(); + repo.cancelAndSoftDelete.mockResolvedValue(null); + + await expect( + service.delete( + '00000000-0000-0000-0000-000000000001', + 'other-admin', + ), + ).rejects.toMatchObject({ status: 404 }); + expect(repo.cancelAndSoftDelete).toHaveBeenCalledWith( + '00000000-0000-0000-0000-000000000001', + 'other-admin', + ); + expect(gcs.deleteResourceArtifacts).not.toHaveBeenCalled(); + }); + + it('returns 404 when reprocess ownership does not match', async () => { + const { service, repo } = createService(); + repo.findById.mockResolvedValue(document({ uploadedByIdpUuid: 'admin-1' })); + + await expect( + service.reprocess( + '00000000-0000-0000-0000-000000000001', + 'other-admin', + ), + ).rejects.toMatchObject({ status: 404 }); + expect(repo.enqueueReprocess).not.toHaveBeenCalled(); + }); + it.each(['uploading', 'queued', 'processing'] as const)( 'rejects reprocess while status is %s', async (status) => { @@ -160,7 +206,10 @@ describe('UploadService atomic transitions', () => { repo.findById.mockResolvedValue(document({ status })); await expect( - service.reprocess('00000000-0000-0000-0000-000000000001'), + service.reprocess( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ), ).rejects.toBeInstanceOf(ConflictException); expect(repo.enqueueReprocess).not.toHaveBeenCalled(); }, @@ -176,7 +225,10 @@ describe('UploadService atomic transitions', () => { ); try { - await service.reprocess('00000000-0000-0000-0000-000000000001'); + await service.reprocess( + '00000000-0000-0000-0000-000000000001', + 'admin-1', + ); throw new Error('Expected reprocess to be rejected'); } catch (error) { expect(error).toEqual( @@ -202,7 +254,7 @@ describe('UploadService atomic transitions', () => { document({ status: 'queued', lastReprocessedAt: new Date() }), ); - await expect(service.reprocess(current.id)).resolves.toEqual( + await expect(service.reprocess(current.id, 'admin-1')).resolves.toEqual( expect.objectContaining({ status: 'queued', canReprocess: false }), ); }); @@ -219,10 +271,11 @@ describe('UploadService atomic transitions', () => { repo.findById.mockResolvedValue(current); repo.enqueueReprocess.mockResolvedValue(queued); - const result = await service.reprocess(current.id); + const result = await service.reprocess(current.id, 'admin-1'); expect(repo.enqueueReprocess).toHaveBeenCalledWith( current.id, + 'admin-1', expect.any(Date), expect.any(Date), ); @@ -264,7 +317,11 @@ describe('UploadService atomic transitions', () => { const result = await service.updateExpiresAt(current.id, 'admin-1', null); - expect(repo.updateExpiresAt).toHaveBeenCalledWith(current.id, null); + expect(repo.updateExpiresAt).toHaveBeenCalledWith( + current.id, + 'admin-1', + null, + ); expect(result.expiresAt).toBeNull(); expect(result.isExpired).toBe(false); }); diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index c7d7cd8..ceb68d3 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -148,8 +148,8 @@ export class UploadService { /** * Soft-delete DB row and remove GCS artifacts. */ - async delete(id: string): Promise { - const row = await this.documentsRepo.cancelAndSoftDelete(id); + async delete(id: string, idpUuid: string): Promise { + const row = await this.documentsRepo.cancelAndSoftDelete(id, idpUuid); if (!row) { throw new NotFoundException(`Document not found: ${id}`); } @@ -171,9 +171,13 @@ export class UploadService { /** * Clear chunks and re-enqueue for processing. */ - async reprocess(id: string): Promise { + async reprocess(id: string, idpUuid: string): Promise { const row = await this.documentsRepo.findById(id); - if (!row || !row.isActive) { + if ( + !row || + !row.isActive || + row.uploadedByIdpUuid !== idpUuid + ) { throw new NotFoundException(`Document not found: ${id}`); } @@ -183,13 +187,18 @@ export class UploadService { const cooldownBefore = new Date(now.getTime() - REPROCESS_COOLDOWN_MS); const updated = await this.documentsRepo.enqueueReprocess( id, + idpUuid, cooldownBefore, now, ); if (!updated) { // Re-read to classify a concurrent state transition accurately. const latest = await this.documentsRepo.findById(id); - if (!latest || !latest.isActive) { + if ( + !latest || + !latest.isActive || + latest.uploadedByIdpUuid !== idpUuid + ) { throw new NotFoundException(`Document not found: ${id}`); } this.assertReprocessEligible(latest, new Date()); @@ -215,7 +224,11 @@ export class UploadService { const expiresAt = expiresAtRaw === null ? null : parseExpiresAt(expiresAtRaw); - const updated = await this.documentsRepo.updateExpiresAt(id, expiresAt); + const updated = await this.documentsRepo.updateExpiresAt( + id, + idpUuid, + expiresAt, + ); if (!updated) { throw new NotFoundException(`Document not found: ${id}`); } From 251166fc99c49909dacf70ba49e2c5c7fb5c7642 Mon Sep 17 00:00:00 2001 From: yejuneric Date: Sat, 1 Aug 2026 23:47:06 +0900 Subject: [PATCH 36/40] feat: add organization-based document management --- drizzle/0014_unique_dracula.sql | 112 + drizzle/meta/0014_snapshot.json | 2009 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/app.module.ts | 2 + src/db/index.ts | 25 +- src/db/migration-lock.spec.ts | 40 + src/db/organizations.schema.spec.ts | 56 + src/db/schema.ts | 228 +- src/main.ts | 3 +- src/organizations/dto/membership.dto.ts | 62 + src/organizations/dto/organization.dto.ts | 41 + .../organization-access.policy.ts | 37 + .../organization-access.service.spec.ts | 255 +++ .../organization-access.service.ts | 143 ++ src/organizations/organization.types.ts | 32 + src/organizations/organizations.controller.ts | 163 ++ src/organizations/organizations.module.ts | 19 + .../organizations.repository.spec.ts | 48 + src/organizations/organizations.repository.ts | 1244 ++++++++++ .../organizations.service.spec.ts | 334 +++ src/organizations/organizations.service.ts | 325 +++ src/pdf-processor/documents.repository.ts | 163 +- .../pdf-processor.worker.spec.ts | 5 +- src/retrieval/retrieval.repository.spec.ts | 16 + src/upload/dto/document-list-item.dto.spec.ts | 54 + src/upload/dto/document-list-item.dto.ts | 66 + src/upload/dto/transfer-document.dto.ts | 8 + src/upload/upload.controller.ts | 206 +- src/upload/upload.module.ts | 10 +- src/upload/upload.service.spec.ts | 730 ++++-- src/upload/upload.service.ts | 475 +++- test/organization-database.e2e-spec.ts | 1096 +++++++++ test/organizations.e2e-spec.ts | 249 ++ 33 files changed, 7748 insertions(+), 515 deletions(-) create mode 100644 drizzle/0014_unique_dracula.sql create mode 100644 drizzle/meta/0014_snapshot.json create mode 100644 src/db/migration-lock.spec.ts create mode 100644 src/db/organizations.schema.spec.ts create mode 100644 src/organizations/dto/membership.dto.ts create mode 100644 src/organizations/dto/organization.dto.ts create mode 100644 src/organizations/organization-access.policy.ts create mode 100644 src/organizations/organization-access.service.spec.ts create mode 100644 src/organizations/organization-access.service.ts create mode 100644 src/organizations/organization.types.ts create mode 100644 src/organizations/organizations.controller.ts create mode 100644 src/organizations/organizations.module.ts create mode 100644 src/organizations/organizations.repository.spec.ts create mode 100644 src/organizations/organizations.repository.ts create mode 100644 src/organizations/organizations.service.spec.ts create mode 100644 src/organizations/organizations.service.ts create mode 100644 src/upload/dto/document-list-item.dto.spec.ts create mode 100644 src/upload/dto/transfer-document.dto.ts create mode 100644 test/organization-database.e2e-spec.ts create mode 100644 test/organizations.e2e-spec.ts diff --git a/drizzle/0014_unique_dracula.sql b/drizzle/0014_unique_dracula.sql new file mode 100644 index 0000000..435db28 --- /dev/null +++ b/drizzle/0014_unique_dracula.sql @@ -0,0 +1,112 @@ +CREATE TYPE "public"."organization_membership_status" AS ENUM('PENDING', 'ACCEPTED');--> statement-breakpoint +CREATE TYPE "public"."organization_role" AS ENUM('MANAGER', 'MEMBER');--> statement-breakpoint +CREATE TABLE "document_organization_shares" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "document_id" uuid NOT NULL, + "organization_id" uuid NOT NULL, + "shared_by_idp_uuid" varchar(255) NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "document_ownership_transfers" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "document_id" uuid NOT NULL, + "source_organization_id" uuid NOT NULL, + "target_organization_id" uuid NOT NULL, + "actor_idp_uuid" varchar(255) NOT NULL, + "transferred_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "organization_memberships" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "organization_id" uuid NOT NULL, + "invitee_email" varchar(255) NOT NULL, + "member_idp_uuid" varchar(255), + "role" "organization_role" DEFAULT 'MEMBER' NOT NULL, + "status" "organization_membership_status" DEFAULT 'PENDING' NOT NULL, + "invited_by_idp_uuid" varchar(255) NOT NULL, + "accepted_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "organizations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "name" varchar(255) NOT NULL, + "slug" varchar(255) NOT NULL, + "is_default" boolean DEFAULT false NOT NULL, + "created_by_idp_uuid" varchar(255), + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "organizations_slug_unique" UNIQUE("slug") +); +--> statement-breakpoint +INSERT INTO "organizations" ("name", "slug", "is_default", "created_by_idp_uuid") +VALUES ('인포팀', 'infoteam', true, NULL) +ON CONFLICT ("slug") DO UPDATE +SET "name" = EXCLUDED."name", + "is_default" = true, + "updated_at" = now();--> statement-breakpoint +INSERT INTO "organization_memberships" ( + "organization_id", + "invitee_email", + "member_idp_uuid", + "role", + "status", + "invited_by_idp_uuid", + "accepted_at" +) +SELECT + o."id", + lower(trim(a."email")), + a."idp_uuid", + 'MANAGER'::"organization_role", + 'ACCEPTED'::"organization_membership_status", + a."idp_uuid", + now() +FROM "admins" a +JOIN "organizations" o ON o."slug" = 'infoteam' +WHERE a."role" = 'SUPER_ADMIN' + AND NOT EXISTS ( + SELECT 1 + FROM "organization_memberships" existing + WHERE existing."organization_id" = o."id" + AND ( + existing."invitee_email" = lower(trim(a."email")) + OR existing."member_idp_uuid" = a."idp_uuid" + ) + );--> statement-breakpoint +ALTER TABLE "documents" ADD COLUMN "owner_organization_id" uuid;--> statement-breakpoint +UPDATE "documents" +SET "owner_organization_id" = ( + SELECT "id" FROM "organizations" WHERE "slug" = 'infoteam' +) +WHERE "owner_organization_id" IS NULL;--> statement-breakpoint +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM "documents" WHERE "owner_organization_id" IS NULL) THEN + RAISE EXCEPTION 'documents.owner_organization_id backfill failed'; + END IF; +END $$;--> statement-breakpoint +ALTER TABLE "documents" ALTER COLUMN "owner_organization_id" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "document_organization_shares" ADD CONSTRAINT "document_organization_shares_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "document_organization_shares" ADD CONSTRAINT "document_organization_shares_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organizations"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "document_ownership_transfers" ADD CONSTRAINT "document_ownership_transfers_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "document_ownership_transfers" ADD CONSTRAINT "document_ownership_transfers_source_organization_id_organizations_id_fk" FOREIGN KEY ("source_organization_id") REFERENCES "public"."organizations"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "document_ownership_transfers" ADD CONSTRAINT "document_ownership_transfers_target_organization_id_organizations_id_fk" FOREIGN KEY ("target_organization_id") REFERENCES "public"."organizations"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organizations"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "document_organization_shares_document_id_idx" ON "document_organization_shares" USING btree ("document_id");--> statement-breakpoint +CREATE INDEX "document_organization_shares_organization_id_idx" ON "document_organization_shares" USING btree ("organization_id");--> statement-breakpoint +CREATE UNIQUE INDEX "document_organization_shares_document_id_organization_id_unique" ON "document_organization_shares" USING btree ("document_id","organization_id");--> statement-breakpoint +CREATE INDEX "document_ownership_transfers_document_id_idx" ON "document_ownership_transfers" USING btree ("document_id");--> statement-breakpoint +CREATE INDEX "document_ownership_transfers_source_organization_id_idx" ON "document_ownership_transfers" USING btree ("source_organization_id");--> statement-breakpoint +CREATE INDEX "document_ownership_transfers_target_organization_id_idx" ON "document_ownership_transfers" USING btree ("target_organization_id");--> statement-breakpoint +CREATE INDEX "organization_memberships_organization_id_idx" ON "organization_memberships" USING btree ("organization_id");--> statement-breakpoint +CREATE INDEX "organization_memberships_member_idp_uuid_status_idx" ON "organization_memberships" USING btree ("member_idp_uuid","status");--> statement-breakpoint +CREATE INDEX "organization_memberships_invitee_email_status_idx" ON "organization_memberships" USING btree ("invitee_email","status");--> statement-breakpoint +CREATE UNIQUE INDEX "organization_memberships_organization_id_invitee_email_unique" ON "organization_memberships" USING btree ("organization_id","invitee_email") WHERE "organization_memberships"."status" = 'PENDING';--> statement-breakpoint +CREATE UNIQUE INDEX "organization_memberships_organization_id_member_idp_uuid_unique" ON "organization_memberships" USING btree ("organization_id","member_idp_uuid") WHERE "organization_memberships"."member_idp_uuid" IS NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX "organizations_single_default_unique" ON "organizations" USING btree ("is_default") WHERE "organizations"."is_default" = true;--> statement-breakpoint +CREATE INDEX "organizations_created_by_idp_uuid_idx" ON "organizations" USING btree ("created_by_idp_uuid");--> statement-breakpoint +ALTER TABLE "documents" ADD CONSTRAINT "documents_owner_organization_id_organizations_id_fk" FOREIGN KEY ("owner_organization_id") REFERENCES "public"."organizations"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "documents_owner_organization_id_idx" ON "documents" USING btree ("owner_organization_id"); diff --git a/drizzle/meta/0014_snapshot.json b/drizzle/meta/0014_snapshot.json new file mode 100644 index 0000000..0447c28 --- /dev/null +++ b/drizzle/meta/0014_snapshot.json @@ -0,0 +1,2009 @@ +{ + "id": "84c647e8-5db0-45bb-9c87-b53110fdb284", + "prevId": "5f7ce177-093e-4298-bc1e-6c0e5b5b9332", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_path_idx": { + "name": "document_chunks_path_idx", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_id_path_unique": { + "name": "document_chunks_document_id_path_unique", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_organization_shares": { + "name": "document_organization_shares", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "shared_by_idp_uuid": { + "name": "shared_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_organization_shares_document_id_idx": { + "name": "document_organization_shares_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_organization_shares_organization_id_idx": { + "name": "document_organization_shares_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_organization_shares_document_id_organization_id_unique": { + "name": "document_organization_shares_document_id_organization_id_unique", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_organization_shares_document_id_documents_id_fk": { + "name": "document_organization_shares_document_id_documents_id_fk", + "tableFrom": "document_organization_shares", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_organization_shares_organization_id_organizations_id_fk": { + "name": "document_organization_shares_organization_id_organizations_id_fk", + "tableFrom": "document_organization_shares", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_ownership_transfers": { + "name": "document_ownership_transfers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_organization_id": { + "name": "source_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "target_organization_id": { + "name": "target_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "actor_idp_uuid": { + "name": "actor_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "transferred_at": { + "name": "transferred_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_ownership_transfers_document_id_idx": { + "name": "document_ownership_transfers_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_ownership_transfers_source_organization_id_idx": { + "name": "document_ownership_transfers_source_organization_id_idx", + "columns": [ + { + "expression": "source_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_ownership_transfers_target_organization_id_idx": { + "name": "document_ownership_transfers_target_organization_id_idx", + "columns": [ + { + "expression": "target_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_ownership_transfers_document_id_documents_id_fk": { + "name": "document_ownership_transfers_document_id_documents_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "document_ownership_transfers_source_organization_id_organizations_id_fk": { + "name": "document_ownership_transfers_source_organization_id_organizations_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "organizations", + "columnsFrom": [ + "source_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "document_ownership_transfers_target_organization_id_organizations_id_fk": { + "name": "document_ownership_transfers_target_organization_id_organizations_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "organizations", + "columnsFrom": [ + "target_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner_organization_id": { + "name": "owner_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_reprocessed_at": { + "name": "last_reprocessed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_owner_organization_id_idx": { + "name": "documents_owner_organization_id_idx", + "columns": [ + { + "expression": "owner_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_expires_at_idx": { + "name": "documents_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_owner_organization_id_organizations_id_fk": { + "name": "documents_owner_organization_id_organizations_id_fk", + "tableFrom": "documents", + "tableTo": "organizations", + "columnsFrom": [ + "owner_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_memberships": { + "name": "organization_memberships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "member_idp_uuid": { + "name": "member_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "organization_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'MEMBER'" + }, + "status": { + "name": "status", + "type": "organization_membership_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "accepted_at": { + "name": "accepted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_memberships_organization_id_idx": { + "name": "organization_memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_member_idp_uuid_status_idx": { + "name": "organization_memberships_member_idp_uuid_status_idx", + "columns": [ + { + "expression": "member_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_invitee_email_status_idx": { + "name": "organization_memberships_invitee_email_status_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_organization_id_invitee_email_unique": { + "name": "organization_memberships_organization_id_invitee_email_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organization_memberships\".\"status\" = 'PENDING'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_organization_id_member_idp_uuid_unique": { + "name": "organization_memberships_organization_id_member_idp_uuid_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "member_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organization_memberships\".\"member_idp_uuid\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_memberships_organization_id_organizations_id_fk": { + "name": "organization_memberships_organization_id_organizations_id_fk", + "tableFrom": "organization_memberships", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_single_default_unique": { + "name": "organizations_single_default_unique", + "columns": [ + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organizations\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "organizations_created_by_idp_uuid_idx": { + "name": "organizations_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.organization_membership_status": { + "name": "organization_membership_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.organization_role": { + "name": "organization_role", + "schema": "public", + "values": [ + "MANAGER", + "MEMBER" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 25dcb9f..6cd5ec7 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1785475702240, "tag": "0013_aberrant_alex_wilder", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1785573225458, + "tag": "0014_unique_dracula", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app.module.ts b/src/app.module.ts index daf1fc2..38bd1d6 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -11,6 +11,7 @@ import { McpModule } from './mcp/mcp.module'; import { ChatModule } from './chat/chat.module'; import { UploadModule } from './upload/upload.module'; import { PdfProcessorModule } from './pdf-processor/pdf-processor.module'; +import { OrganizationsModule } from './organizations/organizations.module'; @Module({ imports: [ @@ -26,6 +27,7 @@ import { PdfProcessorModule } from './pdf-processor/pdf-processor.module'; McpModule, ChatModule, PdfProcessorModule, + OrganizationsModule, UploadModule, ], controllers: [AppController], diff --git a/src/db/index.ts b/src/db/index.ts index e4d8aa0..ae8d17d 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -16,6 +16,27 @@ export interface DatabaseConnectionParams { sslEnabled: boolean; } +const MIGRATION_LOCK_SQL = 'SELECT pg_advisory_lock(1128352846, 1667785076)'; +const MIGRATION_UNLOCK_SQL = + 'SELECT pg_advisory_unlock(1128352846, 1667785076)'; + +export interface MigrationAdvisoryLockClient { + unsafe(query: string): PromiseLike; +} + +/** Serialize startup migrators across every application instance. */ +export async function withMigrationAdvisoryLock( + client: MigrationAdvisoryLockClient, + operation: () => Promise, +): Promise { + await client.unsafe(MIGRATION_LOCK_SQL); + try { + return await operation(); + } finally { + await client.unsafe(MIGRATION_UNLOCK_SQL); + } +} + // Database connection factory with SSL options export const createDatabaseConnection = (params: DatabaseConnectionParams) => { const options = { @@ -59,7 +80,9 @@ export const runMigrations = async (params: DatabaseConnectionParams) => { : './drizzle'; // Local development path try { - await migrate(db, { migrationsFolder }); + await withMigrationAdvisoryLock(migrationClient, () => + migrate(db, { migrationsFolder }), + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); console.error('Migration failed:', errorMessage); diff --git a/src/db/migration-lock.spec.ts b/src/db/migration-lock.spec.ts new file mode 100644 index 0000000..2bda840 --- /dev/null +++ b/src/db/migration-lock.spec.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + type MigrationAdvisoryLockClient, + withMigrationAdvisoryLock, +} from './index'; + +describe('withMigrationAdvisoryLock', () => { + it('holds the session lock for the complete migration operation', async () => { + const events: string[] = []; + const client: MigrationAdvisoryLockClient = { + unsafe: jest.fn(async (query: string) => { + events.push(query.includes('unlock') ? 'unlock' : 'lock'); + }), + }; + + await withMigrationAdvisoryLock(client, async () => { + events.push('migrate'); + }); + + expect(events).toEqual(['lock', 'migrate', 'unlock']); + }); + + it('releases the session lock when migration fails', async () => { + const queries: string[] = []; + const client: MigrationAdvisoryLockClient = { + unsafe: jest.fn(async (query: string) => { + queries.push(query); + }), + }; + + await expect( + withMigrationAdvisoryLock(client, async () => { + throw new Error('migration failed'); + }), + ).rejects.toThrow('migration failed'); + expect(queries).toHaveLength(2); + expect(queries[0]).toContain('pg_advisory_lock'); + expect(queries[1]).toContain('pg_advisory_unlock'); + }); +}); diff --git a/src/db/organizations.schema.spec.ts b/src/db/organizations.schema.spec.ts new file mode 100644 index 0000000..49141e6 --- /dev/null +++ b/src/db/organizations.schema.spec.ts @@ -0,0 +1,56 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +describe('organization ownership migration', () => { + const sql = readFileSync( + join(process.cwd(), 'drizzle', '0014_unique_dracula.sql'), + 'utf8', + ); + + it('creates independent organization roles and all ownership tables', () => { + expect(sql).toContain( + `CREATE TYPE "public"."organization_role" AS ENUM('MANAGER', 'MEMBER')`, + ); + expect(sql).toContain('CREATE TABLE "organizations"'); + expect(sql).toContain('CREATE TABLE "organization_memberships"'); + expect(sql).toContain('CREATE TABLE "document_organization_shares"'); + expect(sql).toContain('CREATE TABLE "document_ownership_transfers"'); + }); + + it('creates infoteam and backfills every document before NOT NULL', () => { + const insert = sql.indexOf("VALUES ('인포팀', 'infoteam', true, NULL)"); + const addNullable = sql.indexOf('ADD COLUMN "owner_organization_id" uuid;'); + const backfill = sql.indexOf('UPDATE "documents"'); + const verify = sql.indexOf( + "RAISE EXCEPTION 'documents.owner_organization_id backfill failed'", + ); + const notNull = sql.indexOf( + 'ALTER COLUMN "owner_organization_id" SET NOT NULL', + ); + expect(sql).toContain('ON CONFLICT ("slug") DO UPDATE'); + expect(insert).toBeGreaterThan(-1); + expect(addNullable).toBeGreaterThan(insert); + expect(backfill).toBeGreaterThan(addNullable); + expect(verify).toBeGreaterThan(backfill); + expect(notNull).toBeGreaterThan(verify); + }); + + it('backfills existing SUPER_ADMINs as accepted default MANAGERs', () => { + expect(sql).toContain(`WHERE a."role" = 'SUPER_ADMIN'`); + expect(sql).toContain(`'MANAGER'::"organization_role"`); + expect(sql).toContain(`'ACCEPTED'::"organization_membership_status"`); + }); + + it('enforces single default and membership/share uniqueness', () => { + expect(sql).toContain('organizations_single_default_unique'); + expect(sql).toContain( + 'organization_memberships_organization_id_member_idp_uuid_unique', + ); + expect(sql).toContain( + 'organization_memberships_organization_id_invitee_email_unique" ON "organization_memberships" USING btree ("organization_id","invitee_email") WHERE "organization_memberships"."status" = \'PENDING\'', + ); + expect(sql).toContain( + 'document_organization_shares_document_id_organization_id_unique', + ); + }); +}); diff --git a/src/db/schema.ts b/src/db/schema.ts index 6615946..68355ea 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -37,6 +37,16 @@ export const collaboratorStatusEnum = pgEnum('collaborator_status', [ 'ACCEPTED', ]); +export const organizationRoleEnum = pgEnum('organization_role', [ + 'MANAGER', + 'MEMBER', +]); + +export const organizationMembershipStatusEnum = pgEnum( + 'organization_membership_status', + ['PENDING', 'ACCEPTED'], +); + export const documentStatusEnum = pgEnum('document_status', [ 'uploading', 'queued', @@ -68,6 +78,76 @@ export const admins = pgTable( emailIdx: index('admins_email_idx').on(table.email), }), ); + +/** + * 문서 관리 조직. 문서는 정확히 한 조직이 소유하며 다른 조직에 공유될 수 있다. + */ +export const organizations = pgTable( + 'organizations', + { + id: uuid('id').defaultRandom().primaryKey(), + name: varchar('name', { length: 255 }).notNull(), + slug: varchar('slug', { length: 255 }).notNull().unique(), + isDefault: boolean('is_default').notNull().default(false), + createdByIdpUuid: varchar('created_by_idp_uuid', { length: 255 }), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), + }, + (table) => ({ + defaultOrganizationUnique: uniqueIndex( + 'organizations_single_default_unique', + ) + .on(table.isDefault) + .where(sql`${table.isDefault} = true`), + createdByIdx: index('organizations_created_by_idp_uuid_idx').on( + table.createdByIdpUuid, + ), + }), +); + +/** 조직 초대와 수락된 멤버십을 함께 저장한다. */ +export const organizationMemberships = pgTable( + 'organization_memberships', + { + id: uuid('id').defaultRandom().primaryKey(), + organizationId: uuid('organization_id') + .notNull() + .references(() => organizations.id, { onDelete: 'restrict' }), + inviteeEmail: varchar('invitee_email', { length: 255 }).notNull(), + memberIdpUuid: varchar('member_idp_uuid', { length: 255 }), + role: organizationRoleEnum('role').notNull().default('MEMBER'), + status: organizationMembershipStatusEnum('status') + .notNull() + .default('PENDING'), + invitedByIdpUuid: varchar('invited_by_idp_uuid', { + length: 255, + }).notNull(), + acceptedAt: timestamp('accepted_at'), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), + }, + (table) => ({ + organizationIdx: index('organization_memberships_organization_id_idx').on( + table.organizationId, + ), + memberLookupIdx: index( + 'organization_memberships_member_idp_uuid_status_idx', + ).on(table.memberIdpUuid, table.status), + pendingInviteIdx: index( + 'organization_memberships_invitee_email_status_idx', + ).on(table.inviteeEmail, table.status), + uniqueOrganizationEmail: uniqueIndex( + 'organization_memberships_organization_id_invitee_email_unique', + ) + .on(table.organizationId, table.inviteeEmail) + .where(sql`${table.status} = 'PENDING'`), + uniqueOrganizationMember: uniqueIndex( + 'organization_memberships_organization_id_member_idp_uuid_unique', + ) + .on(table.organizationId, table.memberIdpUuid) + .where(sql`${table.memberIdpUuid} IS NOT NULL`), + }), +); /** * 위젯 키 테이블 * - 위젯 인증에 사용되는 키 정보를 저장 @@ -212,6 +292,9 @@ export const documents = pgTable( uploadedByIdpUuid: varchar('uploaded_by_idp_uuid', { length: 255, }).notNull(), + ownerOrganizationId: uuid('owner_organization_id') + .notNull() + .references(() => organizations.id, { onDelete: 'restrict' }), isActive: boolean('is_active').notNull().default(true), createdAt: timestamp('created_at').notNull().defaultNow(), updatedAt: timestamp('updated_at').notNull().defaultNow(), @@ -229,6 +312,9 @@ export const documents = pgTable( uploadedByIdpUuidIdx: index('documents_uploaded_by_idp_uuid_idx').on( table.uploadedByIdpUuid, ), + ownerOrganizationIdIdx: index('documents_owner_organization_id_idx').on( + table.ownerOrganizationId, + ), isActiveIdx: index('documents_is_active_idx').on(table.isActive), createdAtIdx: index('documents_created_at_idx').on(table.createdAt), expiresAtIdx: index('documents_expires_at_idx').on(table.expiresAt), @@ -267,6 +353,63 @@ export const documentChunks = pgTable( }), ); +/** 조직에 문서를 공유한다. 소유 조직에 대한 공유 행은 서비스에서 금지한다. */ +export const documentOrganizationShares = pgTable( + 'document_organization_shares', + { + id: uuid('id').defaultRandom().primaryKey(), + documentId: uuid('document_id') + .notNull() + .references(() => documents.id, { onDelete: 'cascade' }), + organizationId: uuid('organization_id') + .notNull() + .references(() => organizations.id, { onDelete: 'restrict' }), + sharedByIdpUuid: varchar('shared_by_idp_uuid', { length: 255 }).notNull(), + createdAt: timestamp('created_at').notNull().defaultNow(), + }, + (table) => ({ + documentIdx: index('document_organization_shares_document_id_idx').on( + table.documentId, + ), + organizationIdx: index( + 'document_organization_shares_organization_id_idx', + ).on(table.organizationId), + uniqueDocumentOrganization: uniqueIndex( + 'document_organization_shares_document_id_organization_id_unique', + ).on(table.documentId, table.organizationId), + }), +); + +/** 문서 소유권 이전 감사 로그. 정상 애플리케이션 흐름에서는 append-only이다. */ +export const documentOwnershipTransfers = pgTable( + 'document_ownership_transfers', + { + id: uuid('id').defaultRandom().primaryKey(), + documentId: uuid('document_id') + .notNull() + .references(() => documents.id, { onDelete: 'restrict' }), + sourceOrganizationId: uuid('source_organization_id') + .notNull() + .references(() => organizations.id, { onDelete: 'restrict' }), + targetOrganizationId: uuid('target_organization_id') + .notNull() + .references(() => organizations.id, { onDelete: 'restrict' }), + actorIdpUuid: varchar('actor_idp_uuid', { length: 255 }).notNull(), + transferredAt: timestamp('transferred_at').notNull().defaultNow(), + }, + (table) => ({ + documentIdx: index('document_ownership_transfers_document_id_idx').on( + table.documentId, + ), + sourceOrganizationIdx: index( + 'document_ownership_transfers_source_organization_id_idx', + ).on(table.sourceOrganizationId), + targetOrganizationIdx: index( + 'document_ownership_transfers_target_organization_id_idx', + ).on(table.targetOrganizationId), + }), +); + /** * 메시지 테이블 * - 채팅 메시지를 저장 @@ -364,8 +507,36 @@ export const usageDaily = pgTable( ); // Relations -export const documentsRelations = relations(documents, ({ many }) => ({ +export const organizationsRelations = relations(organizations, ({ many }) => ({ + memberships: many(organizationMemberships), + ownedDocuments: many(documents), + documentShares: many(documentOrganizationShares), + outgoingTransfers: many(documentOwnershipTransfers, { + relationName: 'transferSourceOrganization', + }), + incomingTransfers: many(documentOwnershipTransfers, { + relationName: 'transferTargetOrganization', + }), +})); + +export const organizationMembershipsRelations = relations( + organizationMemberships, + ({ one }) => ({ + organization: one(organizations, { + fields: [organizationMemberships.organizationId], + references: [organizations.id], + }), + }), +); + +export const documentsRelations = relations(documents, ({ one, many }) => ({ + ownerOrganization: one(organizations, { + fields: [documents.ownerOrganizationId], + references: [organizations.id], + }), chunks: many(documentChunks), + organizationShares: many(documentOrganizationShares), + ownershipTransfers: many(documentOwnershipTransfers), })); export const documentChunksRelations = relations(documentChunks, ({ one }) => ({ @@ -375,6 +546,40 @@ export const documentChunksRelations = relations(documentChunks, ({ one }) => ({ }), })); +export const documentOrganizationSharesRelations = relations( + documentOrganizationShares, + ({ one }) => ({ + document: one(documents, { + fields: [documentOrganizationShares.documentId], + references: [documents.id], + }), + organization: one(organizations, { + fields: [documentOrganizationShares.organizationId], + references: [organizations.id], + }), + }), +); + +export const documentOwnershipTransfersRelations = relations( + documentOwnershipTransfers, + ({ one }) => ({ + document: one(documents, { + fields: [documentOwnershipTransfers.documentId], + references: [documents.id], + }), + sourceOrganization: one(organizations, { + fields: [documentOwnershipTransfers.sourceOrganizationId], + references: [organizations.id], + relationName: 'transferSourceOrganization', + }), + targetOrganization: one(organizations, { + fields: [documentOwnershipTransfers.targetOrganizationId], + references: [organizations.id], + relationName: 'transferTargetOrganization', + }), + }), +); + export const widgetKeysRelations = relations(widgetKeys, ({ many }) => ({ sessions: many(sessions), usageDaily: many(usageDaily), @@ -431,6 +636,17 @@ export const usageDailyRelations = relations(usageDaily, ({ one }) => ({ export type Admin = typeof admins.$inferSelect; export type NewAdmin = typeof admins.$inferInsert; +export type Organization = typeof organizations.$inferSelect; +export type NewOrganization = typeof organizations.$inferInsert; + +export type OrganizationMembership = + typeof organizationMemberships.$inferSelect; +export type NewOrganizationMembership = + typeof organizationMemberships.$inferInsert; +export type OrganizationRole = (typeof organizationRoleEnum.enumValues)[number]; +export type OrganizationMembershipStatus = + (typeof organizationMembershipStatusEnum.enumValues)[number]; + export type UploadedResource = typeof uploadedResources.$inferSelect; export type NewUploadedResource = typeof uploadedResources.$inferInsert; @@ -441,6 +657,16 @@ export type DocumentStatus = (typeof documentStatusEnum.enumValues)[number]; export type DocumentChunk = typeof documentChunks.$inferSelect; export type NewDocumentChunk = typeof documentChunks.$inferInsert; +export type DocumentOrganizationShare = + typeof documentOrganizationShares.$inferSelect; +export type NewDocumentOrganizationShare = + typeof documentOrganizationShares.$inferInsert; + +export type DocumentOwnershipTransfer = + typeof documentOwnershipTransfers.$inferSelect; +export type NewDocumentOwnershipTransfer = + typeof documentOwnershipTransfers.$inferInsert; + export type WidgetKey = typeof widgetKeys.$inferSelect; export type NewWidgetKey = typeof widgetKeys.$inferInsert; diff --git a/src/main.ts b/src/main.ts index 34af67a..ad1cb6a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -72,7 +72,8 @@ async function bootstrap() { .addTag('Widget Messages', '(Public) 대화 내역 저장 및 조회') .addTag('Admin Management', '(Private) 위젯 키 관리') .addTag('Authentication', '(Private) Admin 인증 및 토큰 관리') - .addTag('Upload', '(Private) Super Admin 전용 PDF 업로드/삭제') + .addTag('Upload', '(Private) 조직 권한 기반 PDF 문서 관리') + .addTag('Organizations', '(Private) 조직 멤버십 및 문서 권한 관리') .addTag('Health', '(Public) 서버 상태 확인') .build(); diff --git a/src/organizations/dto/membership.dto.ts b/src/organizations/dto/membership.dto.ts new file mode 100644 index 0000000..38cc9c1 --- /dev/null +++ b/src/organizations/dto/membership.dto.ts @@ -0,0 +1,62 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsEmail, IsEnum } from 'class-validator'; +import { Transform } from 'class-transformer'; +import type { OrganizationMembershipStatus, OrganizationRole } from '../../db'; + +export const ORGANIZATION_ROLES: OrganizationRole[] = ['MANAGER', 'MEMBER']; + +export class InviteOrganizationMemberDto { + @ApiProperty({ example: 'member@example.com' }) + @Transform(({ value }) => + typeof value === 'string' ? value.trim().toLowerCase() : value, + ) + @IsEmail() + inviteeEmail: string; + + @ApiProperty({ enum: ORGANIZATION_ROLES, default: 'MEMBER' }) + @IsEnum(ORGANIZATION_ROLES) + role: OrganizationRole = 'MEMBER'; +} + +export class UpdateOrganizationMemberDto { + @ApiProperty({ enum: ORGANIZATION_ROLES }) + @IsEnum(ORGANIZATION_ROLES) + role: OrganizationRole; +} + +export class OrganizationMembershipDto { + @ApiProperty({ format: 'uuid' }) + id: string; + + @ApiProperty({ format: 'uuid' }) + organizationId: string; + + @ApiProperty() + inviteeEmail: string; + + @ApiProperty({ nullable: true }) + memberIdpUuid: string | null; + + @ApiProperty({ enum: ORGANIZATION_ROLES }) + role: OrganizationRole; + + @ApiProperty({ enum: ['PENDING', 'ACCEPTED'] }) + status: OrganizationMembershipStatus; + + @ApiProperty({ nullable: true }) + memberName: string | null; + + @ApiProperty({ type: String, format: 'date-time', nullable: true }) + acceptedAt: Date | null; + + @ApiProperty({ type: String, format: 'date-time' }) + createdAt: Date; +} + +export class OrganizationInvitationDto extends OrganizationMembershipDto { + @ApiProperty() + organizationName: string; + + @ApiProperty() + organizationSlug: string; +} diff --git a/src/organizations/dto/organization.dto.ts b/src/organizations/dto/organization.dto.ts new file mode 100644 index 0000000..15cdde6 --- /dev/null +++ b/src/organizations/dto/organization.dto.ts @@ -0,0 +1,41 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Transform } from 'class-transformer'; +import { IsNotEmpty, IsString, Matches, MaxLength } from 'class-validator'; + +export class CreateOrganizationDto { + @ApiProperty({ example: '학생지원팀', maxLength: 255 }) + @Transform(({ value }) => (typeof value === 'string' ? value.trim() : value)) + @IsString() + @IsNotEmpty() + @MaxLength(255) + name: string; + + @ApiProperty({ + example: 'student-support', + description: '소문자 영문/숫자와 단일 하이픈으로 구성된 고유 slug', + }) + @IsString() + @Matches(/^[a-z0-9]+(?:-[a-z0-9]+)*$/) + @MaxLength(255) + slug: string; +} + +export class OrganizationDto { + @ApiProperty({ format: 'uuid' }) + id: string; + + @ApiProperty() + name: string; + + @ApiProperty() + slug: string; + + @ApiProperty() + isDefault: boolean; + + @ApiProperty({ enum: ['SUPER_ADMIN', 'MANAGER', 'MEMBER'] }) + effectiveRole: 'SUPER_ADMIN' | 'MANAGER' | 'MEMBER'; + + @ApiProperty({ type: String, format: 'date-time' }) + createdAt: Date; +} diff --git a/src/organizations/organization-access.policy.ts b/src/organizations/organization-access.policy.ts new file mode 100644 index 0000000..353f866 --- /dev/null +++ b/src/organizations/organization-access.policy.ts @@ -0,0 +1,37 @@ +import type { OrganizationRole } from '../db'; +import type { DocumentAccessDecision } from './organization.types'; + +/** + * Pure document authorization policy shared by request-time checks and + * transaction-time reauthorization. + */ +export function evaluateDocumentAccess(input: { + document: DocumentAccessDecision['document']; + actorIdpUuid: string; + ownerRole: OrganizationRole | null; + shared: boolean; +}): DocumentAccessDecision { + if (input.ownerRole) { + const isManager = input.ownerRole === 'MANAGER'; + const isOwnUpload = input.document.uploadedByIdpUuid === input.actorIdpUuid; + return { + document: input.document, + relation: 'OWNER', + ownerRole: input.ownerRole, + canView: true, + canManage: isManager || isOwnUpload, + canShare: isManager, + canTransfer: isManager, + }; + } + + return { + document: input.document, + relation: 'SHARED', + ownerRole: null, + canView: input.shared, + canManage: false, + canShare: false, + canTransfer: false, + }; +} diff --git a/src/organizations/organization-access.service.spec.ts b/src/organizations/organization-access.service.spec.ts new file mode 100644 index 0000000..8df9275 --- /dev/null +++ b/src/organizations/organization-access.service.spec.ts @@ -0,0 +1,255 @@ +import { + BadRequestException, + ForbiddenException, + NotFoundException, +} from '@nestjs/common'; +import { describe, expect, it, jest } from '@jest/globals'; +import type { Document } from '../db'; +import { + evaluateDocumentAccess, + OrganizationAccessService, +} from './organization-access.service'; +import type { OrganizationsRepository } from './organizations.repository'; +import type { AdminPrincipal } from './organization.types'; + +const ORG_ID = '550e8400-e29b-41d4-a716-446655440010'; + +function document(overrides: Partial = {}): Document { + return { + id: '00000000-0000-0000-0000-000000000001', + title: 'Document', + resourceName: 'document', + summary: null, + gcsPdfPath: 'gs://bucket/document.pdf', + status: 'ready', + errorMessage: null, + processingToken: null, + uploadedByIdpUuid: 'uploader', + ownerOrganizationId: ORG_ID, + isActive: true, + createdAt: new Date(), + updatedAt: new Date(), + processedAt: new Date(), + lastReprocessedAt: null, + expiresAt: null, + ...overrides, + }; +} + +function principal(overrides: Partial = {}): AdminPrincipal { + return { + uuid: 'uploader', + email: 'user@example.com', + role: 'ADMIN', + ...overrides, + }; +} + +describe('evaluateDocumentAccess', () => { + it('allows a MEMBER to manage only their own upload', () => { + expect( + evaluateDocumentAccess({ + document: document(), + actorIdpUuid: 'uploader', + ownerRole: 'MEMBER', + shared: false, + }), + ).toMatchObject({ canView: true, canManage: true, canShare: false }); + + expect( + evaluateDocumentAccess({ + document: document(), + actorIdpUuid: 'other-member', + ownerRole: 'MEMBER', + shared: false, + }), + ).toMatchObject({ canView: true, canManage: false, canTransfer: false }); + }); + + it('allows an owner MANAGER to manage/share/transfer every owned document', () => { + expect( + evaluateDocumentAccess({ + document: document(), + actorIdpUuid: 'manager', + ownerRole: 'MANAGER', + shared: false, + }), + ).toMatchObject({ + relation: 'OWNER', + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }); + }); + + it('grants shared organizations view-only access', () => { + expect( + evaluateDocumentAccess({ + document: document(), + actorIdpUuid: 'shared-member', + ownerRole: null, + shared: true, + }), + ).toMatchObject({ + relation: 'SHARED', + canView: true, + canManage: false, + canShare: false, + canTransfer: false, + }); + }); + + it('removes all derived rights when no accepted membership remains', () => { + expect( + evaluateDocumentAccess({ + document: document(), + actorIdpUuid: 'uploader', + ownerRole: null, + shared: false, + }), + ).toMatchObject({ canView: false, canManage: false }); + }); +}); + +describe('OrganizationAccessService', () => { + function setup() { + const repo = { + findOrganization: jest.fn(async (id: string) => + id === 'invalid' + ? undefined + : { + id, + name: 'Org', + slug: 'org', + isDefault: false, + createdByIdpUuid: null, + createdAt: new Date(), + updatedAt: new Date(), + }, + ), + findDefaultOrganization: jest.fn(async () => ({ + id: ORG_ID, + name: '인포팀', + slug: 'infoteam', + isDefault: true, + createdByIdpUuid: null, + createdAt: new Date(), + updatedAt: new Date(), + })), + isCurrentSuperAdmin: jest.fn(async (actor: AdminPrincipal) => + Promise.resolve(actor.role === 'SUPER_ADMIN'), + ), + findAcceptedMembership: jest.fn< + OrganizationsRepository['findAcceptedMembership'] + >(async () => ({ + id: 'membership', + organizationId: ORG_ID, + inviteeEmail: 'user@example.com', + memberIdpUuid: 'uploader', + role: 'MEMBER' as const, + status: 'ACCEPTED' as const, + invitedByIdpUuid: 'manager', + acceptedAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + })), + findDocumentAccessState: jest.fn( + async (_id: string, actor: AdminPrincipal) => ({ + document: document(), + ownerRole: actor.uuid === 'uploader' ? ('MEMBER' as const) : null, + shared: false, + isSuperAdmin: actor.role === 'SUPER_ADMIN', + }), + ), + }; + return { + repo, + service: new OrganizationAccessService( + repo as unknown as OrganizationsRepository, + ), + }; + } + + it('uses the default organization only when organizationId is omitted', async () => { + const { service, repo } = setup(); + await service.resolveUploadOrganization(undefined, principal()); + expect(repo.findDefaultOrganization).toHaveBeenCalledTimes(1); + }); + + it('rejects an explicitly supplied blank organizationId without fallback', async () => { + const { service, repo } = setup(); + await expect( + service.resolveUploadOrganization(' ', principal()), + ).rejects.toBeInstanceOf(BadRequestException); + expect(repo.findDefaultOrganization).not.toHaveBeenCalled(); + }); + + it('does not fall back for a nonempty invalid organization id', async () => { + const { service, repo } = setup(); + await expect( + service.resolveUploadOrganization('invalid', principal()), + ).rejects.toBeInstanceOf(BadRequestException); + expect(repo.findDefaultOrganization).not.toHaveBeenCalled(); + }); + + it('rejects upload for a non-member before storage work', async () => { + const { service, repo } = setup(); + repo.findAcceptedMembership.mockResolvedValue(null); + await expect( + service.resolveUploadOrganization(ORG_ID, principal()), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('conceals active documents from unrelated organizations', async () => { + const { service, repo } = setup(); + repo.findDocumentAccessState.mockResolvedValue({ + document: document(), + ownerRole: null, + shared: false, + isSuperAdmin: false, + }); + await expect( + service.requireDocumentView(document().id, principal()), + ).rejects.toBeInstanceOf(NotFoundException); + await expect( + service.requireDocumentManage(document().id, principal()), + ).rejects.toBeInstanceOf(NotFoundException); + }); + + it('gives SUPER_ADMIN global organization and document access', async () => { + const { service, repo } = setup(); + const actor = principal({ role: 'SUPER_ADMIN', uuid: 'root' }); + await expect( + service.requireOrganizationManager(ORG_ID, actor), + ).resolves.toBeNull(); + await expect( + service.requireDocumentShare(document().id, actor), + ).resolves.toMatchObject({ + canManage: true, + canShare: true, + canTransfer: true, + }); + expect(repo.findAcceptedMembership).not.toHaveBeenCalled(); + }); + + it('does not trust a stale SUPER_ADMIN claim after database demotion', async () => { + const { service, repo } = setup(); + const actor = principal({ role: 'SUPER_ADMIN', uuid: 'demoted-root' }); + repo.isCurrentSuperAdmin.mockResolvedValue(false); + repo.findAcceptedMembership.mockResolvedValue(null); + repo.findDocumentAccessState.mockResolvedValue({ + document: document(), + ownerRole: null, + shared: false, + isSuperAdmin: false, + }); + + await expect( + service.requireOrganizationManager(ORG_ID, actor), + ).rejects.toBeInstanceOf(ForbiddenException); + await expect( + service.requireDocumentShare(document().id, actor), + ).rejects.toBeInstanceOf(NotFoundException); + }); +}); diff --git a/src/organizations/organization-access.service.ts b/src/organizations/organization-access.service.ts new file mode 100644 index 0000000..27f48c4 --- /dev/null +++ b/src/organizations/organization-access.service.ts @@ -0,0 +1,143 @@ +import { + BadRequestException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import type { OrganizationMembership } from '../db'; +import { isUUID } from 'class-validator'; +import { evaluateDocumentAccess } from './organization-access.policy'; +import { OrganizationsRepository } from './organizations.repository'; +import type { + AdminPrincipal, + DocumentAccessDecision, +} from './organization.types'; + +@Injectable() +export class OrganizationAccessService { + constructor(private readonly organizationsRepo: OrganizationsRepository) {} + + isSuperAdmin(principal: AdminPrincipal): boolean { + return principal.role === 'SUPER_ADMIN'; + } + + async requireOrganizationMember( + organizationId: string, + principal: AdminPrincipal, + ): Promise { + const organization = + await this.organizationsRepo.findOrganization(organizationId); + if (!organization) throw new NotFoundException('Organization not found'); + if (await this.organizationsRepo.isCurrentSuperAdmin(principal)) + return null; + + const membership = await this.organizationsRepo.findAcceptedMembership( + organizationId, + principal.uuid, + ); + if (!membership) { + throw new ForbiddenException('Accepted organization membership required'); + } + return membership; + } + + async requireOrganizationManager( + organizationId: string, + principal: AdminPrincipal, + ): Promise { + const membership = await this.requireOrganizationMember( + organizationId, + principal, + ); + if (membership && membership.role !== 'MANAGER') { + throw new ForbiddenException('Organization manager role required'); + } + return membership; + } + + async resolveUploadOrganization( + suppliedOrganizationId: string | undefined, + principal: AdminPrincipal, + ) { + const omitted = suppliedOrganizationId === undefined; + const normalized = suppliedOrganizationId?.trim(); + if (!omitted && (!normalized || !isUUID(normalized))) { + throw new BadRequestException('organizationId must be a UUID'); + } + const organization = omitted + ? await this.organizationsRepo.findDefaultOrganization() + : await this.organizationsRepo.findOrganization(normalized!); + if (!organization) { + throw new NotFoundException( + !omitted ? 'Organization not found' : 'Default organization not found', + ); + } + await this.requireOrganizationMember(organization.id, principal); + return organization; + } + + async getDocumentAccess( + documentId: string, + principal: AdminPrincipal, + ): Promise { + const state = await this.organizationsRepo.findDocumentAccessState( + documentId, + principal, + ); + if (!state?.document.isActive) { + throw new NotFoundException(`Document not found: ${documentId}`); + } + const document = state.document; + + if (state.isSuperAdmin) { + return { + document, + relation: 'OWNER', + ownerRole: null, + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }; + } + + return evaluateDocumentAccess({ + document, + actorIdpUuid: principal.uuid, + ownerRole: state.ownerRole, + shared: !state.ownerRole && state.shared, + }); + } + + async requireDocumentView(documentId: string, principal: AdminPrincipal) { + const access = await this.getDocumentAccess(documentId, principal); + if (!access.canView) { + throw new NotFoundException(`Document not found: ${documentId}`); + } + return access; + } + + async requireDocumentManage(documentId: string, principal: AdminPrincipal) { + const access = await this.getDocumentAccess(documentId, principal); + if (!access.canView) { + throw new NotFoundException(`Document not found: ${documentId}`); + } + if (!access.canManage) { + throw new ForbiddenException('Document management permission required'); + } + return access; + } + + async requireDocumentShare(documentId: string, principal: AdminPrincipal) { + const access = await this.getDocumentAccess(documentId, principal); + if (!access.canView) { + throw new NotFoundException(`Document not found: ${documentId}`); + } + if (!access.canShare) { + throw new ForbiddenException('Document sharing permission required'); + } + return access; + } +} + +export { evaluateDocumentAccess } from './organization-access.policy'; diff --git a/src/organizations/organization.types.ts b/src/organizations/organization.types.ts new file mode 100644 index 0000000..8c98fd1 --- /dev/null +++ b/src/organizations/organization.types.ts @@ -0,0 +1,32 @@ +import type { Document, OrganizationRole } from '../db'; + +export interface AdminPrincipal { + uuid: string; + email: string; + role: string; +} + +export interface OrganizationSummary { + id: string; + name: string; + slug: string; +} + +export type DocumentAccessRelation = 'OWNER' | 'SHARED'; + +export interface DocumentAccessDecision { + document: Document; + relation: DocumentAccessRelation; + ownerRole: OrganizationRole | null; + canView: boolean; + canManage: boolean; + canShare: boolean; + canTransfer: boolean; +} + +export interface DocumentAdministrationRecord { + document: Document; + ownerOrganization: OrganizationSummary; + uploader: { idpUuid: string; email: string; name: string } | null; + sharedOrganizations: OrganizationSummary[]; +} diff --git a/src/organizations/organizations.controller.ts b/src/organizations/organizations.controller.ts new file mode 100644 index 0000000..45e670f --- /dev/null +++ b/src/organizations/organizations.controller.ts @@ -0,0 +1,163 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Param, + ParseUUIDPipe, + Patch, + Post, + UseGuards, +} from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiResponse, + ApiTags, +} from '@nestjs/swagger'; +import { CurrentAdmin } from '../auth/decorators/current-admin.decorator'; +import { AdminContext } from '../auth/context/admin-context.entity'; +import { AdminJwtGuard } from '../auth/guards/admin-jwt.guard'; +import { SuperAdminGuard } from '../auth/guards/super-admin.guard'; +import { CreateOrganizationDto, OrganizationDto } from './dto/organization.dto'; +import { + InviteOrganizationMemberDto, + OrganizationInvitationDto, + OrganizationMembershipDto, + UpdateOrganizationMemberDto, +} from './dto/membership.dto'; +import { OrganizationsService } from './organizations.service'; + +@ApiTags('Organizations') +@ApiBearerAuth('bearerAuth') +@UseGuards(AdminJwtGuard) +@Controller('api/v1/admin') +export class OrganizationsController { + constructor(private readonly organizationsService: OrganizationsService) {} + + @Post('organizations') + @UseGuards(SuperAdminGuard) + @ApiOperation({ + summary: '조직 생성', + description: + 'SUPER_ADMIN만 호출할 수 있으며 생성자 MANAGER 멤버십도 같은 트랜잭션에서 생성합니다.', + }) + @ApiResponse({ status: 201, type: OrganizationDto }) + @ApiResponse({ status: 403, description: 'SUPER_ADMIN 권한 필요' }) + @ApiResponse({ status: 409, description: 'slug 중복' }) + createOrganization( + @Body() dto: CreateOrganizationDto, + @CurrentAdmin() admin: AdminContext, + ) { + return this.organizationsService.createOrganization(dto, admin); + } + + @Get('organizations') + @ApiOperation({ summary: '현재 사용자가 접근 가능한 조직 목록' }) + @ApiResponse({ status: 200, type: OrganizationDto, isArray: true }) + listOrganizations(@CurrentAdmin() admin: AdminContext) { + return this.organizationsService.listOrganizations(admin); + } + + @Get('organizations/:organizationId/members') + @ApiOperation({ summary: '조직 멤버 및 대기 중 초대 목록' }) + @ApiParam({ name: 'organizationId', format: 'uuid' }) + @ApiResponse({ status: 200, type: OrganizationMembershipDto, isArray: true }) + @ApiResponse({ status: 403, description: '조직 MANAGER 권한 필요' }) + listMembers( + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + @CurrentAdmin() admin: AdminContext, + ) { + return this.organizationsService.listMembers(organizationId, admin); + } + + @Post('organizations/:organizationId/members') + @ApiOperation({ + summary: '조직 멤버 초대', + description: + '알려진 관리자도 자동 수락되지 않으며 PENDING 상태로 명시적 수락을 기다립니다.', + }) + @ApiParam({ name: 'organizationId', format: 'uuid' }) + @ApiResponse({ status: 201, type: OrganizationMembershipDto }) + @ApiResponse({ status: 400, description: '자기 자신 초대' }) + @ApiResponse({ status: 403, description: '조직 MANAGER 권한 필요' }) + @ApiResponse({ status: 409, description: '중복 초대/멤버십' }) + inviteMember( + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + @Body() dto: InviteOrganizationMemberDto, + @CurrentAdmin() admin: AdminContext, + ) { + return this.organizationsService.inviteMember(organizationId, dto, admin); + } + + @Patch('organizations/:organizationId/members/:membershipId') + @ApiOperation({ summary: '조직 멤버 역할 변경' }) + @ApiResponse({ status: 200, type: OrganizationMembershipDto }) + @ApiResponse({ status: 400, description: '최종 MANAGER 강등 불가' }) + @ApiResponse({ status: 403, description: '조직 MANAGER 권한 필요' }) + updateMember( + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + @Param('membershipId', new ParseUUIDPipe()) membershipId: string, + @Body() dto: UpdateOrganizationMemberDto, + @CurrentAdmin() admin: AdminContext, + ) { + return this.organizationsService.updateMember( + organizationId, + membershipId, + dto, + admin, + ); + } + + @Delete('organizations/:organizationId/members/:membershipId') + @HttpCode(HttpStatus.NO_CONTENT) + @ApiOperation({ summary: '조직 멤버 또는 대기 중 초대 제거' }) + @ApiResponse({ status: 204, description: '제거 성공' }) + @ApiResponse({ status: 400, description: '최종 MANAGER 제거 불가' }) + @ApiResponse({ status: 403, description: '조직 MANAGER 권한 필요' }) + async removeMember( + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + @Param('membershipId', new ParseUUIDPipe()) membershipId: string, + @CurrentAdmin() admin: AdminContext, + ): Promise { + await this.organizationsService.removeMember( + organizationId, + membershipId, + admin, + ); + } + + @Get('organization-invitations') + @ApiOperation({ summary: '현재 이메일의 대기 중 조직 초대 목록' }) + @ApiResponse({ status: 200, type: OrganizationInvitationDto, isArray: true }) + listInvitations(@CurrentAdmin() admin: AdminContext) { + return this.organizationsService.listInvitations(admin); + } + + @Post('organization-invitations/:membershipId/accept') + @ApiOperation({ summary: '조직 초대 명시적 수락' }) + @ApiResponse({ status: 201, type: OrganizationMembershipDto }) + @ApiResponse({ status: 403, description: '초대 이메일 불일치' }) + @ApiResponse({ status: 409, description: '이미 처리됨 또는 멤버십 충돌' }) + acceptInvitation( + @Param('membershipId', new ParseUUIDPipe()) membershipId: string, + @CurrentAdmin() admin: AdminContext, + ) { + return this.organizationsService.acceptInvitation(membershipId, admin); + } + + @Delete('organization-invitations/:membershipId') + @HttpCode(HttpStatus.NO_CONTENT) + @ApiOperation({ summary: '조직 초대 거절' }) + @ApiResponse({ status: 204, description: '거절 성공' }) + @ApiResponse({ status: 403, description: '초대 이메일 불일치' }) + async rejectInvitation( + @Param('membershipId', new ParseUUIDPipe()) membershipId: string, + @CurrentAdmin() admin: AdminContext, + ): Promise { + await this.organizationsService.rejectInvitation(membershipId, admin); + } +} diff --git a/src/organizations/organizations.module.ts b/src/organizations/organizations.module.ts new file mode 100644 index 0000000..dcd76da --- /dev/null +++ b/src/organizations/organizations.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { AuthModule } from '../auth/auth.module'; +import { DbModule } from '../db/db.module'; +import { OrganizationAccessService } from './organization-access.service'; +import { OrganizationsController } from './organizations.controller'; +import { OrganizationsRepository } from './organizations.repository'; +import { OrganizationsService } from './organizations.service'; + +@Module({ + imports: [DbModule, AuthModule], + controllers: [OrganizationsController], + providers: [ + OrganizationsRepository, + OrganizationAccessService, + OrganizationsService, + ], + exports: [OrganizationsRepository, OrganizationAccessService], +}) +export class OrganizationsModule {} diff --git a/src/organizations/organizations.repository.spec.ts b/src/organizations/organizations.repository.spec.ts new file mode 100644 index 0000000..6d69fe8 --- /dev/null +++ b/src/organizations/organizations.repository.spec.ts @@ -0,0 +1,48 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +describe('OrganizationsRepository concurrency invariants', () => { + const source = readFileSync( + join(process.cwd(), 'src', 'organizations', 'organizations.repository.ts'), + 'utf8', + ); + + it('serializes final-manager changes on the organization row', () => { + expect(source).toContain( + 'await this.lockOrganizations(tx, [organizationId])', + ); + expect(source).toContain('this.countAcceptedManagers(tx, organizationId)'); + expect(source).toContain("return { kind: 'last_manager' }"); + expect(source).toContain( + 'this.isCurrentManager(tx, organizationId, actor)', + ); + }); + + it('locks and conditionally updates the current owner during transfer', () => { + expect(source).toContain( + 'SELECT id FROM documents WHERE id = ${documentId} FOR UPDATE', + ); + expect(source).toContain( + 'await this.lockOrganizations(tx, [\n input.expectedOwnerOrganizationId,\n input.targetOrganizationId,', + ); + expect(source).toContain('documents.ownerOrganizationId,'); + expect(source).toContain('input.expectedOwnerOrganizationId,'); + expect(source).toContain('tx.insert(documentOwnershipTransfers)'); + expect(source).toContain('.delete(documentOrganizationShares)'); + }); + + it('reauthorizes document management after locking the organization', () => { + expect(source).toContain('lockAndAuthorizeDocumentManage'); + expect(source).toContain('evaluateDocumentAccess({'); + expect(source).toContain( + "decision.canManage ? state : { kind: 'forbidden' }", + ); + }); + + it('does not touch chunks or processing state in share/transfer code', () => { + const transferSection = source.slice(source.indexOf('async setShare')); + expect(transferSection).not.toContain('documentChunks'); + expect(transferSection).not.toContain("status: 'queued'"); + expect(transferSection).not.toContain('processingToken'); + }); +}); diff --git a/src/organizations/organizations.repository.ts b/src/organizations/organizations.repository.ts new file mode 100644 index 0000000..396d587 --- /dev/null +++ b/src/organizations/organizations.repository.ts @@ -0,0 +1,1244 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { + and, + desc, + eq, + inArray, + isNotNull, + isNull, + lte, + or, + sql, +} from 'drizzle-orm'; +import { + admins, + DB_CONNECTION, + documentChunks, + documentOrganizationShares, + documentOwnershipTransfers, + documents, + organizationMemberships, + organizations, + type Database, + type Document, + type OrganizationMembership, + type OrganizationRole, +} from '../db'; +import { evaluateDocumentAccess } from './organization-access.policy'; +import type { + AdminPrincipal, + DocumentAdministrationRecord, +} from './organization.types'; + +export type MembershipMutationResult = + | { kind: 'updated'; membership: OrganizationMembership } + | { kind: 'not_found' } + | { kind: 'last_manager' } + | { kind: 'forbidden' }; + +export type DocumentMutationResult = + | { kind: 'ok'; document: Document } + | { kind: 'not_found' } + | { kind: 'stale_owner' } + | { kind: 'forbidden' } + | { kind: 'state_changed'; document: Document }; + +export class RepositoryAuthorizationError extends Error { + constructor() { + super('Organization permission changed'); + this.name = 'RepositoryAuthorizationError'; + } +} + +export class AmbiguousAdminEmailError extends Error { + constructor() { + super('Multiple administrator identities use the same normalized email'); + this.name = 'AmbiguousAdminEmailError'; + } +} + +@Injectable() +export class OrganizationsRepository { + constructor(@Inject(DB_CONNECTION) private readonly db: Database) {} + + findOrganization(id: string) { + return this.db.query.organizations.findFirst({ + where: eq(organizations.id, id), + }); + } + + findDefaultOrganization() { + return this.db.query.organizations.findFirst({ + where: eq(organizations.isDefault, true), + }); + } + + async createOrganization( + name: string, + slug: string, + creator: AdminPrincipal, + ) { + return this.db.transaction(async (tx) => { + if (!(await this.isCurrentSuperAdminInTransaction(tx, creator))) { + throw new RepositoryAuthorizationError(); + } + const [organization] = await tx + .insert(organizations) + .values({ name, slug, createdByIdpUuid: creator.uuid }) + .returning(); + if (!organization) throw new Error('Failed to create organization'); + + await tx.insert(organizationMemberships).values({ + organizationId: organization.id, + inviteeEmail: normalizeEmail(creator.email), + memberIdpUuid: creator.uuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: creator.uuid, + acceptedAt: new Date(), + }); + return organization; + }); + } + + async listAccessibleOrganizations(principal: AdminPrincipal) { + const superAdminCondition = this.currentSuperAdminCondition(principal); + return this.db + .select({ + organization: organizations, + membershipRole: sql`CASE + WHEN ${superAdminCondition} THEN NULL + ELSE ${organizationMemberships.role} + END`, + }) + .from(organizations) + .leftJoin( + organizationMemberships, + and( + eq(organizationMemberships.organizationId, organizations.id), + eq(organizationMemberships.memberIdpUuid, principal.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .where(or(superAdminCondition, isNotNull(organizationMemberships.id))) + .orderBy(organizations.name); + } + + async findAcceptedMembership( + organizationId: string, + memberIdpUuid: string, + ): Promise { + const [membership] = await this.db + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, organizationId), + eq(organizationMemberships.memberIdpUuid, memberIdpUuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .limit(1); + return membership ?? null; + } + + async isCurrentSuperAdmin(principal: AdminPrincipal): Promise { + if (principal.role !== 'SUPER_ADMIN') return false; + const [admin] = await this.db + .select({ id: admins.id }) + .from(admins) + .where( + and(eq(admins.idpUuid, principal.uuid), eq(admins.role, 'SUPER_ADMIN')), + ) + .limit(1); + return Boolean(admin); + } + + async findAcceptedMemberships( + organizationIds: string[], + memberIdpUuid: string, + ) { + if (organizationIds.length === 0) return []; + return this.db + .select() + .from(organizationMemberships) + .where( + and( + inArray(organizationMemberships.organizationId, organizationIds), + eq(organizationMemberships.memberIdpUuid, memberIdpUuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ); + } + + async listMembers(organizationId: string, principal: AdminPrincipal) { + return this.db + .select({ + membership: organizationMemberships, + memberName: admins.name, + }) + .from(organizationMemberships) + .leftJoin( + admins, + eq(admins.idpUuid, organizationMemberships.memberIdpUuid), + ) + .where( + and( + eq(organizationMemberships.organizationId, organizationId), + this.organizationAccessCondition(organizationId, principal, true), + ), + ) + .orderBy(organizationMemberships.createdAt); + } + + async findAdminByEmail(normalizedEmail: string) { + const matches = await this.db + .select({ idpUuid: admins.idpUuid }) + .from(admins) + .where(sql`lower(trim(${admins.email})) = ${normalizedEmail}`) + .limit(2); + if (matches.length > 1) throw new AmbiguousAdminEmailError(); + return matches[0] ?? null; + } + + async createInvitation(input: { + organizationId: string; + inviteeEmail: string; + inviteeIdpUuid: string | null; + role: OrganizationRole; + invitedByIdpUuid: string; + actor: AdminPrincipal; + }) { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.organizationId]); + if ( + !(await this.isCurrentManager(tx, input.organizationId, input.actor)) + ) { + throw new RepositoryAuthorizationError(); + } + const [membership] = await tx + .insert(organizationMemberships) + .values({ + organizationId: input.organizationId, + inviteeEmail: input.inviteeEmail, + memberIdpUuid: input.inviteeIdpUuid, + role: input.role, + status: 'PENDING', + invitedByIdpUuid: input.invitedByIdpUuid, + }) + .returning(); + if (!membership) throw new Error('Failed to create invitation'); + return membership; + }); + } + + async updateMembershipRole( + organizationId: string, + membershipId: string, + role: OrganizationRole, + actor: AdminPrincipal, + ): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [organizationId]); + if (!(await this.isCurrentManager(tx, organizationId, actor))) { + return { kind: 'forbidden' }; + } + const [current] = await tx + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.id, membershipId), + eq(organizationMemberships.organizationId, organizationId), + ), + ) + .limit(1); + if (!current) return { kind: 'not_found' }; + + if ( + current.status === 'ACCEPTED' && + current.role === 'MANAGER' && + role !== 'MANAGER' && + (await this.countAcceptedManagers(tx, organizationId)) <= 1 + ) { + return { kind: 'last_manager' }; + } + + const [membership] = await tx + .update(organizationMemberships) + .set({ role, updatedAt: new Date() }) + .where( + and( + eq(organizationMemberships.id, membershipId), + eq(organizationMemberships.organizationId, organizationId), + ), + ) + .returning(); + return membership + ? { kind: 'updated', membership } + : { kind: 'not_found' }; + }); + } + + async removeMembership( + organizationId: string, + membershipId: string, + actor: AdminPrincipal, + ): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [organizationId]); + if (!(await this.isCurrentManager(tx, organizationId, actor))) { + return { kind: 'forbidden' }; + } + const [current] = await tx + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.id, membershipId), + eq(organizationMemberships.organizationId, organizationId), + ), + ) + .limit(1); + if (!current) return { kind: 'not_found' }; + + if ( + current.status === 'ACCEPTED' && + current.role === 'MANAGER' && + (await this.countAcceptedManagers(tx, organizationId)) <= 1 + ) { + return { kind: 'last_manager' }; + } + + const [membership] = await tx + .delete(organizationMemberships) + .where( + and( + eq(organizationMemberships.id, membershipId), + eq(organizationMemberships.organizationId, organizationId), + ), + ) + .returning(); + return membership + ? { kind: 'updated', membership } + : { kind: 'not_found' }; + }); + } + + async listPendingInvitations(normalizedEmail: string, memberIdpUuid: string) { + return this.db + .select({ + membership: organizationMemberships, + organizationName: organizations.name, + organizationSlug: organizations.slug, + }) + .from(organizationMemberships) + .innerJoin( + organizations, + eq(organizationMemberships.organizationId, organizations.id), + ) + .where( + and( + eq(organizationMemberships.inviteeEmail, normalizedEmail), + eq(organizationMemberships.status, 'PENDING'), + or( + eq(organizationMemberships.memberIdpUuid, memberIdpUuid), + and( + isNull(organizationMemberships.memberIdpUuid), + sql`( + SELECT count(*) + FROM "admins" AS "invitation_identity" + WHERE lower(trim("invitation_identity"."email")) = ${normalizedEmail} + ) = 1`, + sql`EXISTS ( + SELECT 1 + FROM "admins" AS "current_invitation_identity" + WHERE lower(trim("current_invitation_identity"."email")) = ${normalizedEmail} + AND "current_invitation_identity"."idp_uuid" = ${memberIdpUuid} + )`, + ), + ), + ), + ) + .orderBy(desc(organizationMemberships.createdAt)); + } + + async findMembershipById(id: string) { + const [membership] = await this.db + .select() + .from(organizationMemberships) + .where(eq(organizationMemberships.id, id)) + .limit(1); + return membership ?? null; + } + + async acceptInvitation(id: string, normalizedEmail: string, idpUuid: string) { + return this.db.transaction(async (tx) => { + await tx.execute( + sql`SELECT id FROM organization_memberships WHERE id = ${id} FOR UPDATE`, + ); + const [current] = await tx + .select() + .from(organizationMemberships) + .where(eq(organizationMemberships.id, id)) + .limit(1); + if ( + !current || + current.inviteeEmail !== normalizedEmail || + current.status !== 'PENDING' + ) { + return null; + } + if (current.memberIdpUuid && current.memberIdpUuid !== idpUuid) { + throw new RepositoryAuthorizationError(); + } + if ( + !current.memberIdpUuid && + !(await this.isSoleNormalizedAdminIdentity( + tx, + normalizedEmail, + idpUuid, + )) + ) { + throw new RepositoryAuthorizationError(); + } + + const now = new Date(); + const [membership] = await tx + .update(organizationMemberships) + .set({ + memberIdpUuid: current.memberIdpUuid ?? idpUuid, + status: 'ACCEPTED', + acceptedAt: now, + updatedAt: now, + }) + .where( + and( + eq(organizationMemberships.id, id), + eq(organizationMemberships.status, 'PENDING'), + ), + ) + .returning(); + return membership ?? null; + }); + } + + async rejectInvitation( + id: string, + normalizedEmail: string, + memberIdpUuid: string, + ) { + return this.db.transaction(async (tx) => { + await tx.execute( + sql`SELECT id FROM organization_memberships WHERE id = ${id} FOR UPDATE`, + ); + const [current] = await tx + .select() + .from(organizationMemberships) + .where(eq(organizationMemberships.id, id)) + .limit(1); + if ( + !current || + current.inviteeEmail !== normalizedEmail || + current.status !== 'PENDING' + ) { + return null; + } + if (current.memberIdpUuid && current.memberIdpUuid !== memberIdpUuid) { + throw new RepositoryAuthorizationError(); + } + if ( + !current.memberIdpUuid && + !(await this.isSoleNormalizedAdminIdentity( + tx, + normalizedEmail, + memberIdpUuid, + )) + ) { + throw new RepositoryAuthorizationError(); + } + + const [membership] = await tx + .delete(organizationMemberships) + .where( + and( + eq(organizationMemberships.id, id), + eq(organizationMemberships.status, 'PENDING'), + ), + ) + .returning(); + return membership ?? null; + }); + } + + async findDocument(id: string) { + const [document] = await this.db + .select() + .from(documents) + .where(eq(documents.id, id)) + .limit(1); + return document ?? null; + } + + async findDocumentAccessState( + id: string, + principal: AdminPrincipal, + ): Promise<{ + document: Document; + ownerRole: OrganizationRole | null; + shared: boolean; + isSuperAdmin: boolean; + } | null> { + const superAdminExpression = this.currentSuperAdminCondition(principal); + const [row] = await this.db + .select({ + document: documents, + ownerRole: sql`( + SELECT "owner_membership"."role" + FROM "organization_memberships" AS "owner_membership" + WHERE "owner_membership"."organization_id" = "documents"."owner_organization_id" + AND "owner_membership"."member_idp_uuid" = ${principal.uuid} + AND "owner_membership"."status" = 'ACCEPTED' + LIMIT 1 + )`, + shared: sql`EXISTS ( + SELECT 1 + FROM "document_organization_shares" AS "access_share" + INNER JOIN "organization_memberships" AS "shared_membership" + ON "shared_membership"."organization_id" = "access_share"."organization_id" + WHERE "access_share"."document_id" = "documents"."id" + AND "shared_membership"."member_idp_uuid" = ${principal.uuid} + AND "shared_membership"."status" = 'ACCEPTED' + )`, + isSuperAdmin: superAdminExpression, + }) + .from(documents) + .where(eq(documents.id, id)) + .limit(1); + return row ?? null; + } + + async createUploadingDocument(input: { + title: string; + resourceName: string; + gcsPdfPath: string; + ownerOrganizationId: string; + expiresAt: Date | null; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.ownerOrganizationId]); + if ( + !(await this.isCurrentMember( + tx, + input.ownerOrganizationId, + input.actor, + )) + ) { + throw new RepositoryAuthorizationError(); + } + const [document] = await tx + .insert(documents) + .values({ + title: input.title, + resourceName: input.resourceName, + gcsPdfPath: input.gcsPdfPath, + uploadedByIdpUuid: input.actor.uuid, + ownerOrganizationId: input.ownerOrganizationId, + expiresAt: input.expiresAt, + status: 'uploading', + isActive: true, + }) + .returning(); + if (!document) throw new Error('Failed to insert document'); + return document; + }); + } + + async finalizeUploadingDocument(input: { + documentId: string; + expectedOwnerOrganizationId: string; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeDocumentManage( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + if (state.document.status !== 'uploading') { + return { kind: 'state_changed', document: state.document }; + } + + const [document] = await tx + .update(documents) + .set({ + status: 'queued', + errorMessage: null, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, input.documentId), + eq( + documents.ownerOrganizationId, + input.expectedOwnerOrganizationId, + ), + eq(documents.status, 'uploading'), + eq(documents.isActive, true), + ), + ) + .returning(); + return document + ? { kind: 'ok', document } + : { kind: 'state_changed', document: state.document }; + }); + } + + async hasAcceptedShare(documentId: string, memberIdpUuid: string) { + const [row] = await this.db + .select({ id: documentOrganizationShares.id }) + .from(documentOrganizationShares) + .innerJoin( + organizationMemberships, + eq( + organizationMemberships.organizationId, + documentOrganizationShares.organizationId, + ), + ) + .where( + and( + eq(documentOrganizationShares.documentId, documentId), + eq(organizationMemberships.memberIdpUuid, memberIdpUuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .limit(1); + return Boolean(row); + } + + async listOrganizationDocuments( + organizationId: string, + principal: AdminPrincipal, + options: { limit: number; offset: number }, + ) { + const ids = await this.db + .selectDistinct({ id: documents.id, createdAt: documents.createdAt }) + .from(documents) + .leftJoin( + documentOrganizationShares, + eq(documentOrganizationShares.documentId, documents.id), + ) + .where( + and( + eq(documents.isActive, true), + this.organizationAccessCondition(organizationId, principal, false), + or( + eq(documents.ownerOrganizationId, organizationId), + eq(documentOrganizationShares.organizationId, organizationId), + ), + ), + ) + .orderBy(desc(documents.createdAt)) + .limit(options.limit) + .offset(options.offset); + if (ids.length === 0) return []; + return this.db + .select() + .from(documents) + .where( + inArray( + documents.id, + ids.map((row) => row.id), + ), + ) + .orderBy(desc(documents.createdAt)); + } + + async listManageableDocuments( + principal: AdminPrincipal, + options: { limit: number; offset: number }, + ) { + return this.db + .selectDistinct({ document: documents }) + .from(documents) + .leftJoin( + organizationMemberships, + and( + eq( + organizationMemberships.organizationId, + documents.ownerOrganizationId, + ), + eq(organizationMemberships.memberIdpUuid, principal.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .where( + and( + eq(documents.isActive, true), + or( + this.currentSuperAdminCondition(principal), + and( + isNotNull(organizationMemberships.id), + or( + eq(organizationMemberships.role, 'MANAGER'), + eq(documents.uploadedByIdpUuid, principal.uuid), + ), + ), + ), + ), + ) + .orderBy(desc(documents.createdAt)) + .limit(options.limit) + .offset(options.offset) + .then((rows) => rows.map((row) => row.document)); + } + + async hydrateDocuments( + rows: Document[], + ): Promise { + if (rows.length === 0) return []; + const ownerIds = [...new Set(rows.map((row) => row.ownerOrganizationId))]; + const uploaderIds = [...new Set(rows.map((row) => row.uploadedByIdpUuid))]; + const documentIds = rows.map((row) => row.id); + + const [ownerRows, uploaderRows, shareRows] = await Promise.all([ + this.db + .select({ + id: organizations.id, + name: organizations.name, + slug: organizations.slug, + }) + .from(organizations) + .where(inArray(organizations.id, ownerIds)), + this.db + .select({ + idpUuid: admins.idpUuid, + email: admins.email, + name: admins.name, + }) + .from(admins) + .where(inArray(admins.idpUuid, uploaderIds)), + this.db + .select({ + documentId: documentOrganizationShares.documentId, + id: organizations.id, + name: organizations.name, + slug: organizations.slug, + }) + .from(documentOrganizationShares) + .innerJoin( + organizations, + eq(documentOrganizationShares.organizationId, organizations.id), + ) + .where(inArray(documentOrganizationShares.documentId, documentIds)), + ]); + + const owners = new Map(ownerRows.map((row) => [row.id, row])); + const uploaders = new Map(uploaderRows.map((row) => [row.idpUuid, row])); + const shares = new Map(); + for (const share of shareRows) { + const list = shares.get(share.documentId) ?? []; + list.push({ id: share.id, name: share.name, slug: share.slug }); + shares.set(share.documentId, list); + } + + return rows.map((document) => { + const ownerOrganization = owners.get(document.ownerOrganizationId); + if (!ownerOrganization) { + throw new Error( + `Owner organization missing for document ${document.id}`, + ); + } + return { + document, + ownerOrganization, + uploader: uploaders.get(document.uploadedByIdpUuid) ?? null, + sharedOrganizations: shares.get(document.id) ?? [], + }; + }); + } + + async updateDocumentExpiresAt(input: { + documentId: string; + expectedOwnerOrganizationId: string; + expiresAt: Date | null; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeDocumentManage( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + + const [document] = await tx + .update(documents) + .set({ expiresAt: input.expiresAt, updatedAt: new Date() }) + .where( + and( + eq(documents.id, input.documentId), + eq( + documents.ownerOrganizationId, + input.expectedOwnerOrganizationId, + ), + eq(documents.isActive, true), + ), + ) + .returning(); + return document ? { kind: 'ok', document } : { kind: 'stale_owner' }; + }); + } + + async cancelAndSoftDeleteDocument(input: { + documentId: string; + expectedOwnerOrganizationId: string; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeDocumentManage( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + + const [document] = await tx + .update(documents) + .set({ + isActive: false, + processingToken: null, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, input.documentId), + eq( + documents.ownerOrganizationId, + input.expectedOwnerOrganizationId, + ), + eq(documents.isActive, true), + ), + ) + .returning(); + return document ? { kind: 'ok', document } : { kind: 'stale_owner' }; + }); + } + + async enqueueDocumentReprocess(input: { + documentId: string; + expectedOwnerOrganizationId: string; + cooldownBefore: Date; + now: Date; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeDocumentManage( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + + const [document] = await tx + .update(documents) + .set({ + status: 'queued', + errorMessage: null, + processingToken: null, + processedAt: null, + lastReprocessedAt: input.now, + updatedAt: input.now, + }) + .where( + and( + eq(documents.id, input.documentId), + eq( + documents.ownerOrganizationId, + input.expectedOwnerOrganizationId, + ), + eq(documents.isActive, true), + inArray(documents.status, ['ready', 'failed']), + or( + isNull(documents.lastReprocessedAt), + lte(documents.lastReprocessedAt, input.cooldownBefore), + ), + ), + ) + .returning(); + if (!document) return { kind: 'state_changed', document: state.document }; + + await tx + .delete(documentChunks) + .where(eq(documentChunks.documentId, input.documentId)); + return { kind: 'ok', document }; + }); + } + + async setShare(input: { + documentId: string; + expectedOwnerOrganizationId: string; + targetOrganizationId: string; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeOwnerManager( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + await tx + .insert(documentOrganizationShares) + .values({ + documentId: input.documentId, + organizationId: input.targetOrganizationId, + sharedByIdpUuid: input.actor.uuid, + }) + .onConflictDoNothing({ + target: [ + documentOrganizationShares.documentId, + documentOrganizationShares.organizationId, + ], + }); + return state; + }); + } + + async removeShare(input: { + documentId: string; + expectedOwnerOrganizationId: string; + targetOrganizationId: string; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [input.expectedOwnerOrganizationId]); + const state = await this.lockAndAuthorizeOwnerManager( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + await tx + .delete(documentOrganizationShares) + .where( + and( + eq(documentOrganizationShares.documentId, input.documentId), + eq( + documentOrganizationShares.organizationId, + input.targetOrganizationId, + ), + ), + ); + return state; + }); + } + + async transferDocument(input: { + documentId: string; + expectedOwnerOrganizationId: string; + targetOrganizationId: string; + actor: AdminPrincipal; + }): Promise { + return this.db.transaction(async (tx) => { + await this.lockOrganizations(tx, [ + input.expectedOwnerOrganizationId, + input.targetOrganizationId, + ]); + const state = await this.lockAndAuthorizeOwnerManager( + tx, + input.documentId, + input.expectedOwnerOrganizationId, + input.actor, + ); + if (state.kind !== 'ok') return state; + + if (state.document.status === 'uploading') { + return { kind: 'state_changed', document: state.document }; + } + + if (!(await this.isCurrentSuperAdminInTransaction(tx, input.actor))) { + const [targetMembership] = await tx + .select({ role: organizationMemberships.role }) + .from(organizationMemberships) + .where( + and( + eq( + organizationMemberships.organizationId, + input.targetOrganizationId, + ), + eq(organizationMemberships.memberIdpUuid, input.actor.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + eq(organizationMemberships.role, 'MANAGER'), + ), + ) + .limit(1); + if (!targetMembership) return { kind: 'forbidden' }; + } + + const [updated] = await tx + .update(documents) + .set({ + ownerOrganizationId: input.targetOrganizationId, + updatedAt: new Date(), + }) + .where( + and( + eq(documents.id, input.documentId), + eq( + documents.ownerOrganizationId, + input.expectedOwnerOrganizationId, + ), + eq(documents.isActive, true), + ), + ) + .returning(); + if (!updated) return { kind: 'stale_owner' }; + + await tx + .delete(documentOrganizationShares) + .where( + and( + eq(documentOrganizationShares.documentId, input.documentId), + eq( + documentOrganizationShares.organizationId, + input.targetOrganizationId, + ), + ), + ); + await tx.insert(documentOwnershipTransfers).values({ + documentId: input.documentId, + sourceOrganizationId: input.expectedOwnerOrganizationId, + targetOrganizationId: input.targetOrganizationId, + actorIdpUuid: input.actor.uuid, + }); + return { kind: 'ok', document: updated }; + }); + } + + private async countAcceptedManagers( + tx: Parameters[0]>[0], + organizationId: string, + ): Promise { + const [row] = await tx + .select({ count: sql`count(*)::int` }) + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, organizationId), + eq(organizationMemberships.status, 'ACCEPTED'), + eq(organizationMemberships.role, 'MANAGER'), + ), + ); + return row?.count ?? 0; + } + + private async isSoleNormalizedAdminIdentity( + tx: Parameters[0]>[0], + normalizedEmail: string, + actorIdpUuid: string, + ): Promise { + // Prevent a case/space-variant admin identity from being inserted between + // the ambiguity check and binding an invitation to the caller. + await tx.execute(sql`LOCK TABLE "admins" IN SHARE MODE`); + const matches = await tx + .select({ idpUuid: admins.idpUuid }) + .from(admins) + .where(sql`lower(trim(${admins.email})) = ${normalizedEmail}`) + .limit(2); + return matches.length === 1 && matches[0]?.idpUuid === actorIdpUuid; + } + + private organizationAccessCondition( + organizationId: string, + principal: AdminPrincipal, + managerOnly: boolean, + ) { + const superAdminCondition = this.currentSuperAdminCondition(principal); + const roleCondition = managerOnly + ? sql`AND "access_membership"."role" = 'MANAGER'` + : sql``; + return sql`( + ${superAdminCondition} + OR EXISTS ( + SELECT 1 + FROM "organization_memberships" AS "access_membership" + WHERE "access_membership"."organization_id" = ${organizationId} + AND "access_membership"."member_idp_uuid" = ${principal.uuid} + AND "access_membership"."status" = 'ACCEPTED' + ${roleCondition} + ) + )`; + } + + private currentSuperAdminCondition(principal: AdminPrincipal) { + return principal.role === 'SUPER_ADMIN' + ? sql`EXISTS ( + SELECT 1 + FROM "admins" AS "current_super_admin" + WHERE "current_super_admin"."idp_uuid" = ${principal.uuid} + AND "current_super_admin"."role" = 'SUPER_ADMIN' + )` + : sql`false`; + } + + private async isCurrentSuperAdminInTransaction( + tx: Parameters[0]>[0], + actor: AdminPrincipal, + ): Promise { + if (actor.role !== 'SUPER_ADMIN') return false; + const [admin] = await tx + .select({ id: admins.id }) + .from(admins) + .where( + and(eq(admins.idpUuid, actor.uuid), eq(admins.role, 'SUPER_ADMIN')), + ) + .limit(1) + .for('share'); + return Boolean(admin); + } + + private async lockOrganizations( + tx: Parameters[0]>[0], + organizationIds: string[], + ): Promise { + const sortedIds = [...new Set(organizationIds)].sort(); + for (const organizationId of sortedIds) { + await tx.execute( + sql`SELECT id FROM organizations WHERE id = ${organizationId} FOR UPDATE`, + ); + } + } + + private async isCurrentManager( + tx: Parameters[0]>[0], + organizationId: string, + actor: AdminPrincipal, + ): Promise { + if (await this.isCurrentSuperAdminInTransaction(tx, actor)) return true; + const [membership] = await tx + .select({ id: organizationMemberships.id }) + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, organizationId), + eq(organizationMemberships.memberIdpUuid, actor.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + eq(organizationMemberships.role, 'MANAGER'), + ), + ) + .limit(1); + return Boolean(membership); + } + + private async isCurrentMember( + tx: Parameters[0]>[0], + organizationId: string, + actor: AdminPrincipal, + ): Promise { + if (await this.isCurrentSuperAdminInTransaction(tx, actor)) return true; + const [membership] = await tx + .select({ id: organizationMemberships.id }) + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, organizationId), + eq(organizationMemberships.memberIdpUuid, actor.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .limit(1); + return Boolean(membership); + } + + private async lockAndAuthorizeDocumentManage( + tx: Parameters[0]>[0], + documentId: string, + expectedOwnerOrganizationId: string, + actor: AdminPrincipal, + ): Promise { + const state = await this.lockDocument( + tx, + documentId, + expectedOwnerOrganizationId, + ); + if (state.kind !== 'ok') return state; + if (await this.isCurrentSuperAdminInTransaction(tx, actor)) return state; + + const [membership] = await tx + .select({ role: organizationMemberships.role }) + .from(organizationMemberships) + .where( + and( + eq( + organizationMemberships.organizationId, + expectedOwnerOrganizationId, + ), + eq(organizationMemberships.memberIdpUuid, actor.uuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ) + .limit(1); + const decision = evaluateDocumentAccess({ + document: state.document, + actorIdpUuid: actor.uuid, + ownerRole: membership?.role ?? null, + shared: false, + }); + return decision.canManage ? state : { kind: 'forbidden' }; + } + + private async lockAndAuthorizeOwnerManager( + tx: Parameters[0]>[0], + documentId: string, + expectedOwnerOrganizationId: string, + actor: AdminPrincipal, + ): Promise { + const state = await this.lockDocument( + tx, + documentId, + expectedOwnerOrganizationId, + ); + if (state.kind !== 'ok') return state; + return (await this.isCurrentManager(tx, expectedOwnerOrganizationId, actor)) + ? state + : { kind: 'forbidden' }; + } + + private async lockDocument( + tx: Parameters[0]>[0], + documentId: string, + expectedOwnerOrganizationId: string, + ): Promise { + await tx.execute( + sql`SELECT id FROM documents WHERE id = ${documentId} FOR UPDATE`, + ); + const [document] = await tx + .select() + .from(documents) + .where(eq(documents.id, documentId)) + .limit(1); + if (!document || !document.isActive) return { kind: 'not_found' }; + if (document.ownerOrganizationId !== expectedOwnerOrganizationId) { + return { kind: 'stale_owner' }; + } + return { kind: 'ok', document }; + } +} + +export function normalizeEmail(email: string): string { + return email.trim().toLowerCase(); +} diff --git a/src/organizations/organizations.service.spec.ts b/src/organizations/organizations.service.spec.ts new file mode 100644 index 0000000..5bfa3da --- /dev/null +++ b/src/organizations/organizations.service.spec.ts @@ -0,0 +1,334 @@ +import { + BadRequestException, + ConflictException, + ForbiddenException, +} from '@nestjs/common'; +import { describe, expect, it, jest } from '@jest/globals'; +import type { OrganizationAccessService } from './organization-access.service'; +import { + AmbiguousAdminEmailError, + RepositoryAuthorizationError, + type OrganizationsRepository, +} from './organizations.repository'; +import { OrganizationsService } from './organizations.service'; +import type { AdminPrincipal } from './organization.types'; + +const ORG_ID = '00000000-0000-0000-0000-000000000010'; + +function actor(overrides: Partial = {}): AdminPrincipal { + return { + uuid: 'manager', + email: 'Manager@Example.com ', + role: 'ADMIN', + ...overrides, + }; +} + +function pendingMembership(overrides: Record = {}) { + return { + id: '00000000-0000-0000-0000-000000000011', + organizationId: ORG_ID, + inviteeEmail: 'invitee@example.com', + memberIdpUuid: null, + role: 'MEMBER' as const, + status: 'PENDING' as const, + invitedByIdpUuid: 'manager', + acceptedAt: null, + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + }; +} + +function setup() { + const repo = { + createOrganization: jest.fn( + async () => ({ + id: ORG_ID, + name: 'Org', + slug: 'org', + isDefault: false, + createdByIdpUuid: 'root', + createdAt: new Date(), + updatedAt: new Date(), + }), + ), + listAccessibleOrganizations: jest.fn< + OrganizationsRepository['listAccessibleOrganizations'] + >(async () => []), + listMembers: jest.fn(async () => []), + findAdminByEmail: jest.fn(async () => null as { idpUuid: string } | null), + createInvitation: jest.fn( + async () => pendingMembership(), + ), + updateMembershipRole: jest.fn< + OrganizationsRepository['updateMembershipRole'] + >(async () => ({ + kind: 'updated' as const, + membership: pendingMembership(), + })), + removeMembership: jest.fn( + async () => ({ + kind: 'updated' as const, + membership: pendingMembership(), + }), + ), + listPendingInvitations: jest.fn(async () => []), + findMembershipById: jest.fn(async () => pendingMembership()), + acceptInvitation: jest.fn( + async () => + pendingMembership({ + memberIdpUuid: 'invitee', + status: 'ACCEPTED', + acceptedAt: new Date(), + }), + ), + rejectInvitation: jest.fn(async () => pendingMembership()), + }; + const access = { + isSuperAdmin: jest.fn( + (principal: AdminPrincipal) => principal.role === 'SUPER_ADMIN', + ), + requireOrganizationManager: jest.fn(async () => null), + }; + return { + repo, + access, + service: new OrganizationsService( + repo as unknown as OrganizationsRepository, + access as unknown as OrganizationAccessService, + ), + }; +} + +describe('OrganizationsService', () => { + it('returns every accepted organization for a multi-organization user', async () => { + const { service, repo } = setup(); + repo.listAccessibleOrganizations.mockResolvedValue([ + { + organization: { + id: ORG_ID, + name: 'Org A', + slug: 'org-a', + isDefault: false, + createdByIdpUuid: 'root', + createdAt: new Date(), + updatedAt: new Date(), + }, + membershipRole: 'MANAGER', + }, + { + organization: { + id: '00000000-0000-0000-0000-000000000020', + name: 'Org B', + slug: 'org-b', + isDefault: false, + createdByIdpUuid: 'root', + createdAt: new Date(), + updatedAt: new Date(), + }, + membershipRole: 'MEMBER', + }, + ]); + await expect(service.listOrganizations(actor())).resolves.toEqual([ + expect.objectContaining({ slug: 'org-a', effectiveRole: 'MANAGER' }), + expect.objectContaining({ slug: 'org-b', effectiveRole: 'MEMBER' }), + ]); + }); + + it('rejects organization creation by non-SUPER_ADMIN', async () => { + const { service, repo } = setup(); + await expect( + service.createOrganization({ name: 'Org', slug: 'org' }, actor()), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(repo.createOrganization).not.toHaveBeenCalled(); + }); + + it('uses the repository atomic create-with-manager operation', async () => { + const { service, repo } = setup(); + const root = actor({ role: 'SUPER_ADMIN', uuid: 'root' }); + await expect( + service.createOrganization({ name: ' Org ', slug: 'org' }, root), + ).resolves.toMatchObject({ effectiveRole: 'SUPER_ADMIN' }); + expect(repo.createOrganization).toHaveBeenCalledWith('Org', 'org', root); + }); + + it('returns only fields declared by the organization DTO', async () => { + const { service } = setup(); + const result = await service.createOrganization( + { name: 'Org', slug: 'org' }, + actor({ role: 'SUPER_ADMIN', uuid: 'root' }), + ); + expect(result).toEqual({ + id: ORG_ID, + name: 'Org', + slug: 'org', + isDefault: false, + effectiveRole: 'SUPER_ADMIN', + createdAt: expect.any(Date), + }); + }); + + it('rejects a stale SUPER_ADMIN claim at transaction time', async () => { + const { service, repo } = setup(); + repo.createOrganization.mockRejectedValue( + new RepositoryAuthorizationError(), + ); + await expect( + service.createOrganization( + { name: 'Org', slug: 'org' }, + actor({ role: 'SUPER_ADMIN', uuid: 'demoted-root' }), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('invites an unknown email as normalized PENDING', async () => { + const { service, repo } = setup(); + const result = await service.inviteMember( + ORG_ID, + { inviteeEmail: ' Invitee@Example.COM ', role: 'MEMBER' }, + actor(), + ); + expect(repo.createInvitation).toHaveBeenCalledWith( + expect.objectContaining({ + inviteeEmail: 'invitee@example.com', + inviteeIdpUuid: null, + }), + ); + expect(result.status).toBe('PENDING'); + expect(result).not.toHaveProperty('invitedByIdpUuid'); + expect(result).not.toHaveProperty('updatedAt'); + }); + + it('stores a known admin UUID but keeps the invite PENDING', async () => { + const { service, repo } = setup(); + repo.findAdminByEmail.mockResolvedValue({ idpUuid: 'known-user' }); + repo.createInvitation.mockResolvedValue( + pendingMembership({ memberIdpUuid: 'known-user' }), + ); + const result = await service.inviteMember( + ORG_ID, + { inviteeEmail: 'invitee@example.com', role: 'MEMBER' }, + actor(), + ); + expect(result).toMatchObject({ + memberIdpUuid: 'known-user', + status: 'PENDING', + }); + }); + + it('rejects invitations when a normalized email maps to multiple identities', async () => { + const { service, repo } = setup(); + repo.findAdminByEmail.mockRejectedValue(new AmbiguousAdminEmailError()); + await expect( + service.inviteMember( + ORG_ID, + { inviteeEmail: 'invitee@example.com', role: 'MEMBER' }, + actor(), + ), + ).rejects.toBeInstanceOf(ConflictException); + expect(repo.createInvitation).not.toHaveBeenCalled(); + }); + + it('does not allow a MEMBER through member management access', async () => { + const { service, repo, access } = setup(); + access.requireOrganizationManager.mockRejectedValue( + new ForbiddenException('Organization manager role required'), + ); + await expect( + service.inviteMember( + ORG_ID, + { inviteeEmail: 'invitee@example.com', role: 'MEMBER' }, + actor(), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(repo.createInvitation).not.toHaveBeenCalled(); + }); + + it('rejects an invitation if manager access is revoked before insertion', async () => { + const { service, repo } = setup(); + repo.createInvitation.mockRejectedValue(new RepositoryAuthorizationError()); + await expect( + service.inviteMember( + ORG_ID, + { inviteeEmail: 'invitee@example.com', role: 'MEMBER' }, + actor(), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('maps final-manager demotion/removal protection', async () => { + const { service, repo } = setup(); + repo.updateMembershipRole.mockResolvedValue({ kind: 'last_manager' }); + await expect( + service.updateMember(ORG_ID, 'membership', { role: 'MEMBER' }, actor()), + ).rejects.toBeInstanceOf(BadRequestException); + + repo.removeMembership.mockResolvedValue({ kind: 'last_manager' }); + await expect( + service.removeMember(ORG_ID, 'membership', actor()), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('forbids another email from accepting an invitation', async () => { + const { service, repo } = setup(); + repo.findMembershipById.mockResolvedValue(pendingMembership()); + await expect( + service.acceptInvitation( + pendingMembership().id, + actor({ email: 'other@example.com' }), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(repo.acceptInvitation).not.toHaveBeenCalled(); + }); + + it('forbids a different UUID with the same normalized email', async () => { + const { service, repo } = setup(); + repo.findMembershipById.mockResolvedValue( + pendingMembership({ memberIdpUuid: 'known-user' }), + ); + await expect( + service.acceptInvitation( + pendingMembership().id, + actor({ uuid: 'case-variant-user', email: 'INVITEE@example.com' }), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(repo.acceptInvitation).not.toHaveBeenCalled(); + }); + + it('maps a transaction-time invitation identity change to forbidden', async () => { + const { service, repo } = setup(); + repo.acceptInvitation.mockRejectedValue(new RepositoryAuthorizationError()); + await expect( + service.acceptInvitation( + pendingMembership().id, + actor({ uuid: 'invitee', email: 'invitee@example.com' }), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('maps a transaction-time invitation rejection identity change to forbidden', async () => { + const { service, repo } = setup(); + repo.rejectInvitation.mockRejectedValue(new RepositoryAuthorizationError()); + await expect( + service.rejectInvitation( + pendingMembership().id, + actor({ uuid: 'invitee', email: 'invitee@example.com' }), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('accepts only explicitly and sets the current IDP UUID through repository', async () => { + const { service, repo } = setup(); + const invitee = actor({ + uuid: 'invitee', + email: 'INVITEE@example.com', + }); + await service.acceptInvitation(pendingMembership().id, invitee); + expect(repo.acceptInvitation).toHaveBeenCalledWith( + pendingMembership().id, + 'invitee@example.com', + 'invitee', + ); + }); +}); diff --git a/src/organizations/organizations.service.ts b/src/organizations/organizations.service.ts new file mode 100644 index 0000000..1fef820 --- /dev/null +++ b/src/organizations/organizations.service.ts @@ -0,0 +1,325 @@ +import { + BadRequestException, + ConflictException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { OrganizationAccessService } from './organization-access.service'; +import { + AmbiguousAdminEmailError, + normalizeEmail, + OrganizationsRepository, + RepositoryAuthorizationError, +} from './organizations.repository'; +import type { AdminPrincipal } from './organization.types'; +import type { + InviteOrganizationMemberDto, + OrganizationInvitationDto, + OrganizationMembershipDto, + UpdateOrganizationMemberDto, +} from './dto/membership.dto'; +import type { + CreateOrganizationDto, + OrganizationDto, +} from './dto/organization.dto'; + +@Injectable() +export class OrganizationsService { + constructor( + private readonly organizationsRepo: OrganizationsRepository, + private readonly access: OrganizationAccessService, + ) {} + + async createOrganization( + dto: CreateOrganizationDto, + principal: AdminPrincipal, + ): Promise { + if (!this.access.isSuperAdmin(principal)) { + throw new ForbiddenException('Super admin role required'); + } + try { + const organization = await this.organizationsRepo.createOrganization( + dto.name.trim(), + dto.slug, + principal, + ); + return this.toOrganizationDto(organization, 'SUPER_ADMIN'); + } catch (error) { + if (error instanceof RepositoryAuthorizationError) { + throw new ForbiddenException('Super admin role required'); + } + if (isUniqueViolation(error)) { + throw new ConflictException('Organization slug already exists'); + } + throw error; + } + } + + async listOrganizations( + principal: AdminPrincipal, + ): Promise { + const rows = + await this.organizationsRepo.listAccessibleOrganizations(principal); + return rows.map((row) => + this.toOrganizationDto( + row.organization, + row.membershipRole ?? 'SUPER_ADMIN', + ), + ); + } + + async listMembers( + organizationId: string, + principal: AdminPrincipal, + ): Promise { + await this.access.requireOrganizationManager(organizationId, principal); + const rows = await this.organizationsRepo.listMembers( + organizationId, + principal, + ); + return rows.map((row) => + this.toMembershipDto(row.membership, row.memberName), + ); + } + + async inviteMember( + organizationId: string, + dto: InviteOrganizationMemberDto, + principal: AdminPrincipal, + ): Promise { + await this.access.requireOrganizationManager(organizationId, principal); + const inviteeEmail = normalizeEmail(dto.inviteeEmail); + if (inviteeEmail === normalizeEmail(principal.email)) { + throw new BadRequestException('Cannot invite yourself'); + } + try { + const knownAdmin = + await this.organizationsRepo.findAdminByEmail(inviteeEmail); + const membership = await this.organizationsRepo.createInvitation({ + organizationId, + inviteeEmail, + inviteeIdpUuid: knownAdmin?.idpUuid ?? null, + role: dto.role, + invitedByIdpUuid: principal.uuid, + actor: principal, + }); + return this.toMembershipDto(membership, null); + } catch (error) { + if (isUniqueViolation(error)) { + throw new ConflictException( + 'This email already has an invitation or membership', + ); + } + if (error instanceof AmbiguousAdminEmailError) { + throw new ConflictException( + 'Multiple administrator identities use this normalized email', + ); + } + if (error instanceof RepositoryAuthorizationError) { + throw new ForbiddenException('Organization manager role required'); + } + throw error; + } + } + + async updateMember( + organizationId: string, + membershipId: string, + dto: UpdateOrganizationMemberDto, + principal: AdminPrincipal, + ): Promise { + await this.access.requireOrganizationManager(organizationId, principal); + const result = await this.organizationsRepo.updateMembershipRole( + organizationId, + membershipId, + dto.role, + principal, + ); + if (result.kind === 'not_found') { + throw new NotFoundException('Organization membership not found'); + } + if (result.kind === 'forbidden') { + throw new ForbiddenException('Organization manager role required'); + } + if (result.kind === 'last_manager') { + throw new BadRequestException( + 'Cannot demote the final accepted organization manager', + ); + } + return this.toMembershipDto(result.membership, null); + } + + async removeMember( + organizationId: string, + membershipId: string, + principal: AdminPrincipal, + ): Promise { + await this.access.requireOrganizationManager(organizationId, principal); + const result = await this.organizationsRepo.removeMembership( + organizationId, + membershipId, + principal, + ); + if (result.kind === 'not_found') { + throw new NotFoundException('Organization membership not found'); + } + if (result.kind === 'forbidden') { + throw new ForbiddenException('Organization manager role required'); + } + if (result.kind === 'last_manager') { + throw new BadRequestException( + 'Cannot remove the final accepted organization manager', + ); + } + } + + async listInvitations( + principal: AdminPrincipal, + ): Promise { + const rows = await this.organizationsRepo.listPendingInvitations( + normalizeEmail(principal.email), + principal.uuid, + ); + return rows.map((row) => ({ + ...this.toMembershipDto(row.membership, null), + organizationName: row.organizationName, + organizationSlug: row.organizationSlug, + })); + } + + async acceptInvitation( + membershipId: string, + principal: AdminPrincipal, + ): Promise { + await this.assertInvitationOwner(membershipId, principal); + try { + const membership = await this.organizationsRepo.acceptInvitation( + membershipId, + normalizeEmail(principal.email), + principal.uuid, + ); + if (!membership) { + throw new ConflictException('Invitation is no longer pending'); + } + return this.toMembershipDto(membership, null); + } catch (error) { + if (error instanceof RepositoryAuthorizationError) { + throw new ForbiddenException('Invitation belongs to another identity'); + } + if (isUniqueViolation(error)) { + throw new ConflictException( + 'An accepted membership already exists for this organization', + ); + } + throw error; + } + } + + async rejectInvitation( + membershipId: string, + principal: AdminPrincipal, + ): Promise { + await this.assertInvitationOwner(membershipId, principal); + try { + const removed = await this.organizationsRepo.rejectInvitation( + membershipId, + normalizeEmail(principal.email), + principal.uuid, + ); + if (!removed) + throw new ConflictException('Invitation is no longer pending'); + } catch (error) { + if (error instanceof RepositoryAuthorizationError) { + throw new ForbiddenException('Invitation belongs to another identity'); + } + throw error; + } + } + + private async assertInvitationOwner( + membershipId: string, + principal: AdminPrincipal, + ): Promise { + const membership = + await this.organizationsRepo.findMembershipById(membershipId); + if (!membership) throw new NotFoundException('Invitation not found'); + if (membership.inviteeEmail !== normalizeEmail(principal.email)) { + throw new ForbiddenException('Only the invited email may respond'); + } + if ( + membership.memberIdpUuid && + membership.memberIdpUuid !== principal.uuid + ) { + throw new ForbiddenException('Invitation belongs to another identity'); + } + if (membership.status !== 'PENDING') { + throw new ConflictException('Invitation is no longer pending'); + } + } + + private toOrganizationDto( + organization: { + id: string; + name: string; + slug: string; + isDefault: boolean; + createdAt: Date; + }, + effectiveRole: OrganizationDto['effectiveRole'], + ): OrganizationDto { + return { + id: organization.id, + name: organization.name, + slug: organization.slug, + isDefault: organization.isDefault, + effectiveRole, + createdAt: organization.createdAt, + }; + } + + private toMembershipDto( + membership: { + id: string; + organizationId: string; + inviteeEmail: string; + memberIdpUuid: string | null; + role: 'MANAGER' | 'MEMBER'; + status: 'PENDING' | 'ACCEPTED'; + acceptedAt: Date | null; + createdAt: Date; + }, + memberName: string | null, + ): OrganizationMembershipDto { + return { + id: membership.id, + organizationId: membership.organizationId, + inviteeEmail: membership.inviteeEmail, + memberIdpUuid: membership.memberIdpUuid, + role: membership.role, + status: membership.status, + memberName, + acceptedAt: membership.acceptedAt, + createdAt: membership.createdAt, + }; + } +} + +function isUniqueViolation(error: unknown): boolean { + let current: unknown = error; + for (let depth = 0; depth < 4 && current; depth += 1) { + if ( + typeof current === 'object' && + current !== null && + 'code' in current && + current.code === '23505' + ) { + return true; + } + current = + typeof current === 'object' && current !== null && 'cause' in current + ? current.cause + : null; + } + return false; +} diff --git a/src/pdf-processor/documents.repository.ts b/src/pdf-processor/documents.repository.ts index ea70a8d..f84ea11 100644 --- a/src/pdf-processor/documents.repository.ts +++ b/src/pdf-processor/documents.repository.ts @@ -6,22 +6,19 @@ import { eq, and, or, - isNull, - lte, desc, asc, lt, } from 'drizzle-orm'; -import { DB_CONNECTION, documents, documentChunks } from '../db'; +import { + admins, + DB_CONNECTION, + documents, + documentChunks, + organizationMemberships, +} from '../db'; import type { Database, Document, DocumentChunk } from '../db'; - -export type CreateDocumentInput = { - title: string; - resourceName: string; - gcsPdfPath: string; - uploadedByIdpUuid: string; - expiresAt?: Date | null; -}; +import type { AdminPrincipal } from '../organizations/organization.types'; export type ReplaceChunksInput = { path: string; @@ -34,68 +31,6 @@ export type ReplaceChunksInput = { export class DocumentsRepository { constructor(@Inject(DB_CONNECTION) private readonly db: Database) {} - /** - * Atomically reserve an active resource name before uploading to GCS. - * The worker only claims `queued`, so it cannot observe a partial upload. - */ - async createUploading(input: CreateDocumentInput): Promise { - const [row] = await this.db - .insert(documents) - .values({ - title: input.title, - resourceName: input.resourceName, - gcsPdfPath: input.gcsPdfPath, - uploadedByIdpUuid: input.uploadedByIdpUuid, - expiresAt: input.expiresAt ?? null, - status: 'uploading', - isActive: true, - }) - .returning(); - if (!row) throw new Error('Failed to insert document'); - return row; - } - - async updateExpiresAt( - id: string, - uploadedByIdpUuid: string, - expiresAt: Date | null, - ) { - const [row] = await this.db - .update(documents) - .set({ - expiresAt, - updatedAt: new Date(), - }) - .where( - and( - eq(documents.id, id), - eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), - eq(documents.isActive, true), - ), - ) - .returning(); - return row ?? null; - } - - async markQueuedAfterUpload(id: string) { - const [row] = await this.db - .update(documents) - .set({ - status: 'queued', - errorMessage: null, - updatedAt: new Date(), - }) - .where( - and( - eq(documents.id, id), - eq(documents.status, 'uploading'), - eq(documents.isActive, true), - ), - ) - .returning(); - return row ?? null; - } - async hardDelete(id: string): Promise { await this.db.delete(documents).where(eq(documents.id, id)); } @@ -124,16 +59,35 @@ export class DocumentsRepository { } async listByUploader( - idpUuid: string, + principal: AdminPrincipal, options: { limit: number; offset: number }, ): Promise { + const currentSuperAdmin = + principal.role === 'SUPER_ADMIN' + ? sql`EXISTS ( + SELECT 1 + FROM ${admins} + WHERE ${admins.idpUuid} = ${principal.uuid} + AND ${admins.role} = 'SUPER_ADMIN' + )` + : sql`false`; return this.db .select() .from(documents) .where( and( - eq(documents.uploadedByIdpUuid, idpUuid), + eq(documents.uploadedByIdpUuid, principal.uuid), eq(documents.isActive, true), + or( + currentSuperAdmin, + sql`EXISTS ( + SELECT 1 + FROM ${organizationMemberships} + WHERE ${organizationMemberships.organizationId} = ${documents.ownerOrganizationId} + AND ${organizationMemberships.memberIdpUuid} = ${principal.uuid} + AND ${organizationMemberships.status} = 'ACCEPTED' + )`, + ), ), ) .orderBy(desc(documents.createdAt)) @@ -299,65 +253,6 @@ export class DocumentsRepository { return result.length > 0; } - /** - * Cancel the current attempt before deleting external artifacts. - */ - async cancelAndSoftDelete(id: string, uploadedByIdpUuid: string) { - const [row] = await this.db - .update(documents) - .set({ - isActive: false, - processingToken: null, - updatedAt: new Date(), - }) - .where( - and( - eq(documents.id, id), - eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), - eq(documents.isActive, true), - ), - ) - .returning(); - return row ?? null; - } - - async enqueueReprocess( - id: string, - uploadedByIdpUuid: string, - cooldownBefore: Date, - now: Date, - ) { - return this.db.transaction(async (tx) => { - const [row] = await tx - .update(documents) - .set({ - status: 'queued', - errorMessage: null, - processingToken: null, - processedAt: null, - lastReprocessedAt: now, - updatedAt: now, - }) - .where( - and( - eq(documents.id, id), - eq(documents.uploadedByIdpUuid, uploadedByIdpUuid), - eq(documents.isActive, true), - inArray(documents.status, ['ready', 'failed']), - or( - isNull(documents.lastReprocessedAt), - lte(documents.lastReprocessedAt, cooldownBefore), - ), - ), - ) - .returning(); - if (!row) return null; - - await tx.delete(documentChunks).where(eq(documentChunks.documentId, id)); - return row; - }); - } - async listChunks(documentId: string): Promise { return this.db .select() diff --git a/src/pdf-processor/pdf-processor.worker.spec.ts b/src/pdf-processor/pdf-processor.worker.spec.ts index 7b1fd03..8b85dde 100644 --- a/src/pdf-processor/pdf-processor.worker.spec.ts +++ b/src/pdf-processor/pdf-processor.worker.spec.ts @@ -17,6 +17,7 @@ function processingDocument(): Document { errorMessage: null, processingToken: '00000000-0000-0000-0000-000000000002', uploadedByIdpUuid: 'admin-1', + ownerOrganizationId: '00000000-0000-0000-0000-000000000010', isActive: true, createdAt: new Date(), updatedAt: new Date(), @@ -79,7 +80,7 @@ function createWorker(options: { const gcs = { downloadPdf: jest.fn(() => Promise.resolve(Buffer.from('%PDF-test'))), uploadDocuments: options.uploadDocumentsError - ? jest.fn(() => Promise.reject(options.uploadDocumentsError)) + ? jest.fn(() => Promise.reject(options.uploadDocumentsError!)) : jest.fn(() => Promise.resolve()), deleteProcessedArtifacts: jest.fn<(resourceName: string) => Promise>( () => Promise.resolve(), @@ -87,7 +88,7 @@ function createWorker(options: { }; const pipeline = { processPdf: options.processPdfError - ? jest.fn(() => Promise.reject(options.processPdfError)) + ? jest.fn(() => Promise.reject(options.processPdfError!)) : jest.fn(() => Promise.resolve({ documents: { diff --git a/src/retrieval/retrieval.repository.spec.ts b/src/retrieval/retrieval.repository.spec.ts index 0689609..d43862e 100644 --- a/src/retrieval/retrieval.repository.spec.ts +++ b/src/retrieval/retrieval.repository.spec.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from '@jest/globals'; +import { readFileSync } from 'fs'; +import { join } from 'path'; import { isExpiredAt } from './retrieval.repository'; describe('isExpiredAt', () => { @@ -18,3 +20,17 @@ describe('isExpiredAt', () => { expect(isExpiredAt(new Date('2026-07-30T11:59:59.000Z'), now)).toBe(true); }); }); + +describe('retrieval organization scope invariant', () => { + it('keeps chatbot retrieval global and free of organization predicates', () => { + const source = readFileSync( + join(process.cwd(), 'src', 'retrieval', 'retrieval.repository.ts'), + 'utf8', + ); + expect(source).not.toContain('ownerOrganizationId'); + expect(source).not.toContain('documentOrganizationShares'); + expect(source).toContain("eq(documents.status, 'ready')"); + expect(source).toContain('eq(documents.isActive, true)'); + expect(source).toContain('notExpiredCondition()'); + }); +}); diff --git a/src/upload/dto/document-list-item.dto.spec.ts b/src/upload/dto/document-list-item.dto.spec.ts new file mode 100644 index 0000000..09ee17c --- /dev/null +++ b/src/upload/dto/document-list-item.dto.spec.ts @@ -0,0 +1,54 @@ +import { Controller, Get } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import { + FastifyAdapter, + NestFastifyApplication, +} from '@nestjs/platform-fastify'; +import { ApiOkResponse, DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; +import { DocumentListItemDto } from './document-list-item.dto'; + +@Controller('document-list-item-schema-probe') +class DocumentListItemSchemaProbeController { + @Get() + @ApiOkResponse({ type: DocumentListItemDto }) + get(): void {} +} + +describe('DocumentListItemDto OpenAPI schema', () => { + it('describes nested organization and uploader response fields', async () => { + const moduleRef = await Test.createTestingModule({ + controllers: [DocumentListItemSchemaProbeController], + }).compile(); + const app = moduleRef.createNestApplication( + new FastifyAdapter(), + ); + try { + const document = SwaggerModule.createDocument( + app, + new DocumentBuilder().build(), + ); + const schemas = document.components?.schemas ?? {}; + const item = schemas.DocumentListItemDto as { + properties?: Record; + }; + + expect(JSON.stringify(item.properties?.ownerOrganization)).toContain( + '#/components/schemas/DocumentOrganizationSummaryDto', + ); + expect(JSON.stringify(item.properties?.uploader)).toContain( + '#/components/schemas/DocumentUploaderSummaryDto', + ); + expect(JSON.stringify(item.properties?.sharedOrganizations)).toContain( + '#/components/schemas/DocumentOrganizationSummaryDto', + ); + expect(schemas.DocumentOrganizationSummaryDto).toMatchObject({ + properties: { id: {}, name: {}, slug: {} }, + }); + expect(schemas.DocumentUploaderSummaryDto).toMatchObject({ + properties: { idpUuid: {}, email: {}, name: {} }, + }); + } finally { + await app.close(); + } + }); +}); diff --git a/src/upload/dto/document-list-item.dto.ts b/src/upload/dto/document-list-item.dto.ts index d7eaec1..a0ddbb0 100644 --- a/src/upload/dto/document-list-item.dto.ts +++ b/src/upload/dto/document-list-item.dto.ts @@ -9,6 +9,28 @@ const DOCUMENT_STATUSES: DocumentStatus[] = [ 'failed', ]; +export class DocumentOrganizationSummaryDto { + @ApiProperty({ format: 'uuid' }) + id: string; + + @ApiProperty() + name: string; + + @ApiProperty() + slug: string; +} + +export class DocumentUploaderSummaryDto { + @ApiProperty() + idpUuid: string; + + @ApiProperty() + email: string; + + @ApiProperty() + name: string; +} + export class DocumentListItemDto { @ApiProperty({ description: '문서 UUID', @@ -110,4 +132,48 @@ export class DocumentListItemDto { example: false, }) isExpired: boolean; + + @ApiProperty({ + description: '문서 소유 조직', + type: () => DocumentOrganizationSummaryDto, + example: { + id: '550e8400-e29b-41d4-a716-446655440000', + name: '인포팀', + slug: 'infoteam', + }, + }) + ownerOrganization: DocumentOrganizationSummaryDto; + + @ApiProperty({ + description: + '업로더 IDP 식별자와 공개 관리자 정보 (관리자 레코드가 없으면 null)', + type: () => DocumentUploaderSummaryDto, + nullable: true, + example: { + idpUuid: 'idp-user-uuid', + email: 'admin@example.com', + name: '관리자', + }, + }) + uploader: DocumentUploaderSummaryDto | null; + + @ApiProperty({ + description: '문서가 공유된 조직 목록', + type: () => DocumentOrganizationSummaryDto, + isArray: true, + example: [], + }) + sharedOrganizations: DocumentOrganizationSummaryDto[]; + + @ApiProperty({ enum: ['OWNER', 'SHARED'] }) + accessRelation: 'OWNER' | 'SHARED'; + + @ApiProperty({ description: '수정/삭제/재처리/유효기간 변경 가능 여부' }) + canManage: boolean; + + @ApiProperty({ description: '다른 조직으로 공유/공유 해제 가능 여부' }) + canShare: boolean; + + @ApiProperty({ description: '소유권 이전을 시작할 수 있는 권한 여부' }) + canTransfer: boolean; } diff --git a/src/upload/dto/transfer-document.dto.ts b/src/upload/dto/transfer-document.dto.ts new file mode 100644 index 0000000..5d1d645 --- /dev/null +++ b/src/upload/dto/transfer-document.dto.ts @@ -0,0 +1,8 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsUUID } from 'class-validator'; + +export class TransferDocumentDto { + @ApiProperty({ format: 'uuid', description: '새 소유 조직 UUID' }) + @IsUUID() + targetOrganizationId: string; +} diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index c37b297..728cdce 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -3,9 +3,11 @@ import { Controller, Get, Post, + Put, Patch, Delete, Param, + ParseUUIDPipe, Query, Req, UseGuards, @@ -26,12 +28,12 @@ import { import type { FastifyRequest } from 'fastify'; import { UploadService, PDF_MIME } from './upload.service'; import { AdminJwtGuard } from '../auth/guards/admin-jwt.guard'; -import { SuperAdminGuard } from '../auth/guards/super-admin.guard'; import { CurrentAdmin } from '../auth/decorators/current-admin.decorator'; import { AdminContext } from '../auth/context/admin-context.entity'; import { Readable } from 'stream'; import { DocumentListItemDto } from './dto/document-list-item.dto'; import { UpdateExpiresAtDto } from './dto/update-expires-at.dto'; +import { TransferDocumentDto } from './dto/transfer-document.dto'; async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { const chunks: Buffer[] = []; @@ -43,16 +45,16 @@ async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { @ApiTags('Upload') @Controller('api/v1/admin/upload') -@UseGuards(AdminJwtGuard, SuperAdminGuard) +@UseGuards(AdminJwtGuard) @ApiBearerAuth('bearerAuth') export class UploadController { constructor(private readonly uploadService: UploadService) {} @Get() @ApiOperation({ - summary: '내가 업로드한 문서 목록 조회 (Super Admin 전용)', + summary: '내가 업로드한 문서 목록', description: - '현재 로그인한 Super Admin이 업로드한 문서 목록을 최신순으로 반환합니다. 삭제되지 않은 문서만 포함되며, 처리 상태(status)를 포함합니다.', + '현재 로그인한 사용자가 직접 업로드한 활성 문서만 최신순으로 반환합니다.', }) @ApiQuery({ name: 'limit', @@ -74,7 +76,6 @@ export class UploadController { }) @ApiResponse({ status: 400, description: '잘못된 limit 또는 offset' }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) async listMyUploads( @CurrentAdmin() admin: AdminContext, @Query('limit') limit?: string, @@ -91,7 +92,55 @@ export class UploadController { ) { throw new BadRequestException('offset must be a non-negative number'); } - return this.uploadService.listMyUploads(admin.uuid, { + return this.uploadService.listMyUploads(admin, { + limit: limitNum, + offset: offsetNum, + }); + } + + @Get('manageable') + @ApiOperation({ + summary: '현재 사용자가 관리 가능한 문서 목록', + description: + 'SUPER_ADMIN의 전체 문서, MANAGER의 소유 조직 문서, MEMBER가 직접 업로드한 소유 조직 문서를 중복 없이 반환합니다.', + }) + @ApiQuery({ + name: 'limit', + required: false, + type: Number, + description: '최대 개수 (기본 50, 최대 100)', + }) + @ApiQuery({ + name: 'offset', + required: false, + type: Number, + description: '건너뛸 개수 (페이지네이션)', + }) + @ApiResponse({ + status: 200, + description: '성공', + type: DocumentListItemDto, + isArray: true, + }) + @ApiResponse({ status: 400, description: '잘못된 limit 또는 offset' }) + @ApiResponse({ status: 401, description: '인증 실패' }) + async listManageableDocuments( + @CurrentAdmin() admin: AdminContext, + @Query('limit') limit?: string, + @Query('offset') offset?: string, + ) { + const limitNum = limit != null ? parseInt(limit, 10) : undefined; + const offsetNum = offset != null ? parseInt(offset, 10) : undefined; + if (limit != null && (Number.isNaN(limitNum) || (limitNum as number) < 1)) { + throw new BadRequestException('limit must be a positive number'); + } + if ( + offset != null && + (Number.isNaN(offsetNum as number) || (offsetNum as number) < 0) + ) { + throw new BadRequestException('offset must be a non-negative number'); + } + return this.uploadService.listManageableDocuments(admin, { limit: limitNum, offset: offsetNum, }); @@ -109,15 +158,18 @@ export class UploadController { type: DocumentListItemDto, }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 403, description: '문서 조회 권한 없음' }) @ApiResponse({ status: 404, description: '문서 없음' }) - async getOne(@CurrentAdmin() admin: AdminContext, @Param('id') id: string) { - return this.uploadService.getById(id, admin.uuid); + async getOne( + @CurrentAdmin() admin: AdminContext, + @Param('id', new ParseUUIDPipe()) id: string, + ) { + return this.uploadService.getById(id, admin); } @Post() @ApiOperation({ - summary: 'PDF 파일 업로드 (Super Admin 전용)', + summary: '조직 소유 PDF 파일 업로드', description: 'PDF를 GCS에 저장하고 비동기 처리 큐에 등록합니다. 처리 완료를 기다리지 않으며 status=queued로 즉시 응답합니다.', }) @@ -136,6 +188,12 @@ export class UploadController { '문서 유효기간 (ISO-8601, optional). 미전송/빈 값이면 무기한. 과거 시각은 400.', nullable: true, }, + organizationId: { + type: 'string', + format: 'uuid', + description: + '소유 조직 UUID. 생략한 경우에만 출시 호환성을 위해 기본 조직을 사용하며, 빈 값은 잘못된 입력입니다.', + }, }, }, }) @@ -149,7 +207,8 @@ export class UploadController { description: '잘못된 요청 (PDF 아님, 필드 누락, 과거 expiresAt 등)', }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 403, description: '조직 멤버십 필요' }) + @ApiResponse({ status: 404, description: '지정 조직 없음' }) @ApiResponse({ status: 409, description: '동일 resource_name 문서가 이미 존재', @@ -173,6 +232,7 @@ export class UploadController { const parts = fastifyReq.parts(); let title = ''; let expiresAt: string | undefined; + let organizationId: string | undefined; let fileBuffer: Buffer | null = null; let filename = 'document.pdf'; let mimetype = ''; @@ -185,6 +245,9 @@ export class UploadController { } else if (part.fieldname === 'expiresAt') { const v = part.value; expiresAt = typeof v === 'string' ? v : undefined; + } else if (part.fieldname === 'organizationId') { + const v = part.value; + organizationId = typeof v === 'string' ? v : undefined; } } else if (part.type === 'file' && part.fieldname === 'file') { const filePart = part; @@ -210,7 +273,8 @@ export class UploadController { fileBuffer, filename, title.trim(), - admin.uuid, + admin, + organizationId, expiresAt, ); } @@ -230,14 +294,14 @@ export class UploadController { }) @ApiResponse({ status: 400, description: '잘못된 expiresAt' }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 403, description: '문서 관리 권한 없음' }) @ApiResponse({ status: 404, description: '문서 없음' }) async updateExpiresAt( @CurrentAdmin() admin: AdminContext, - @Param('id') id: string, + @Param('id', new ParseUUIDPipe()) id: string, @Body() body: UpdateExpiresAtDto, ) { - return this.uploadService.updateExpiresAt(id, admin.uuid, body.expiresAt); + return this.uploadService.updateExpiresAt(id, admin, body.expiresAt); } @Post(':id/reprocess') @@ -248,12 +312,12 @@ export class UploadController { }) @ApiParam({ name: 'id', description: '문서 UUID' }) @ApiResponse({ - status: 200, + status: 201, description: '재처리 큐 등록', type: DocumentListItemDto, }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 403, description: '문서 관리 권한 없음' }) @ApiResponse({ status: 404, description: '문서 없음' }) @ApiResponse({ status: 409, @@ -273,22 +337,22 @@ export class UploadController { }) async reprocess( @CurrentAdmin() admin: AdminContext, - @Param('id') id: string, + @Param('id', new ParseUUIDPipe()) id: string, ) { - return this.uploadService.reprocess(id, admin.uuid); + return this.uploadService.reprocess(id, admin); } @Delete(':id') @HttpCode(HttpStatus.NO_CONTENT) @ApiOperation({ - summary: '업로드 파일 삭제 (Super Admin 전용)', + summary: '업로드 파일 삭제', description: - 'GCS 산출물을 삭제하고 DB에서 soft-delete 합니다. Super Admin 역할만 호출 가능합니다.', + '문서 관리 권한을 확인한 뒤 GCS 산출물을 삭제하고 DB에서 soft-delete 합니다.', }) @ApiParam({ name: 'id', description: '문서 UUID', type: String }) @ApiResponse({ status: 204, description: '삭제 성공' }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: 'Super Admin 권한 필요' }) + @ApiResponse({ status: 403, description: '문서 관리 권한 없음' }) @ApiResponse({ status: 404, description: '문서 없음 또는 이미 삭제됨', @@ -299,9 +363,103 @@ export class UploadController { }) async delete( @CurrentAdmin() admin: AdminContext, - @Param('id') id: string, + @Param('id', new ParseUUIDPipe()) id: string, + ): Promise { + await this.uploadService.delete(id, admin); + } + + @Put(':id/shares/:organizationId') + @ApiOperation({ + summary: '문서를 다른 조직에 공유', + description: + '소유 조직 MANAGER 또는 SUPER_ADMIN만 가능하며 PDF/청크를 복제하지 않습니다.', + }) + @ApiResponse({ status: 200, type: DocumentListItemDto }) + @ApiResponse({ status: 400, description: '소유 조직으로 공유 시도' }) + @ApiResponse({ status: 403, description: '공유 권한 없음' }) + share( + @CurrentAdmin() admin: AdminContext, + @Param('id', new ParseUUIDPipe()) id: string, + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + ) { + return this.uploadService.shareDocument(id, organizationId, admin); + } + + @Delete(':id/shares/:organizationId') + @HttpCode(HttpStatus.NO_CONTENT) + @ApiOperation({ summary: '조직 문서 공유 해제' }) + @ApiResponse({ status: 204, description: '공유 해제 성공' }) + @ApiResponse({ status: 403, description: '공유 권한 없음' }) + async unshare( + @CurrentAdmin() admin: AdminContext, + @Param('id', new ParseUUIDPipe()) id: string, + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, ): Promise { - await this.uploadService.delete(id, admin.uuid); + await this.uploadService.unshareDocument(id, organizationId, admin); + } + + @Post(':id/transfer') + @ApiOperation({ + summary: '문서 소유권 이전', + description: + 'SUPER_ADMIN 또는 출발/대상 양쪽 조직의 MANAGER만 가능하며 공유 정리와 감사 로그를 원자적으로 기록합니다.', + }) + @ApiResponse({ status: 201, type: DocumentListItemDto }) + @ApiResponse({ + status: 403, + description: '출발 또는 대상 조직 MANAGER 권한 없음', + }) + @ApiResponse({ status: 409, description: '동시 소유권 변경' }) + transfer( + @CurrentAdmin() admin: AdminContext, + @Param('id', new ParseUUIDPipe()) id: string, + @Body() body: TransferDocumentDto, + ) { + return this.uploadService.transferDocument( + id, + body.targetOrganizationId, + admin, + ); + } +} + +@ApiTags('Organizations', 'Upload') +@Controller('api/v1/admin/organizations') +@UseGuards(AdminJwtGuard) +@ApiBearerAuth('bearerAuth') +export class OrganizationDocumentsController { + constructor(private readonly uploadService: UploadService) {} + + @Get(':organizationId/documents') + @ApiOperation({ + summary: '조직 문서 목록', + description: '조직이 소유한 문서와 조직에 공유된 문서를 함께 반환합니다.', + }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + @ApiQuery({ name: 'offset', required: false, type: Number }) + @ApiResponse({ status: 200, type: DocumentListItemDto, isArray: true }) + @ApiResponse({ status: 403, description: '조직 접근 권한 없음' }) + async list( + @CurrentAdmin() admin: AdminContext, + @Param('organizationId', new ParseUUIDPipe()) organizationId: string, + @Query('limit') limit?: string, + @Query('offset') offset?: string, + ) { + const limitNum = limit != null ? parseInt(limit, 10) : undefined; + const offsetNum = offset != null ? parseInt(offset, 10) : undefined; + if (limit != null && (Number.isNaN(limitNum) || (limitNum as number) < 1)) { + throw new BadRequestException('limit must be a positive number'); + } + if ( + offset != null && + (Number.isNaN(offsetNum as number) || (offsetNum as number) < 0) + ) { + throw new BadRequestException('offset must be a non-negative number'); + } + return this.uploadService.listOrganizationDocuments(organizationId, admin, { + limit: limitNum, + offset: offsetNum, + }); } } diff --git a/src/upload/upload.module.ts b/src/upload/upload.module.ts index 21c1aec..cc24ed6 100644 --- a/src/upload/upload.module.ts +++ b/src/upload/upload.module.ts @@ -1,13 +1,17 @@ import { Module } from '@nestjs/common'; -import { UploadController } from './upload.controller'; +import { + OrganizationDocumentsController, + UploadController, +} from './upload.controller'; import { UploadService } from './upload.service'; import { AuthModule } from '../auth/auth.module'; import { DbModule } from '../db/db.module'; import { PdfProcessorModule } from '../pdf-processor/pdf-processor.module'; +import { OrganizationsModule } from '../organizations/organizations.module'; @Module({ - imports: [AuthModule, DbModule, PdfProcessorModule], - controllers: [UploadController], + imports: [AuthModule, DbModule, PdfProcessorModule, OrganizationsModule], + controllers: [UploadController, OrganizationDocumentsController], providers: [UploadService], }) export class UploadModule {} diff --git a/src/upload/upload.service.spec.ts b/src/upload/upload.service.spec.ts index b6fce99..c41bd93 100644 --- a/src/upload/upload.service.spec.ts +++ b/src/upload/upload.service.spec.ts @@ -1,10 +1,32 @@ -import { BadRequestException, ConflictException } from '@nestjs/common'; +import { + BadRequestException, + ConflictException, + ForbiddenException, + NotFoundException, +} from '@nestjs/common'; import { describe, expect, it, jest } from '@jest/globals'; import type { Document } from '../db'; +import type { OrganizationAccessService } from '../organizations/organization-access.service'; +import { + RepositoryAuthorizationError, + type OrganizationsRepository, +} from '../organizations/organizations.repository'; +import type { AdminPrincipal } from '../organizations/organization.types'; import type { DocumentsRepository } from '../pdf-processor/documents.repository'; import type { GcsStorageService } from '../pdf-processor/gcs-storage.service'; import { parseExpiresAt, UploadService } from './upload.service'; +const ORGANIZATION_ID = '00000000-0000-0000-0000-000000000010'; + +function principal(overrides: Partial = {}): AdminPrincipal { + return { + uuid: 'admin-1', + email: 'admin@example.com', + role: 'ADMIN', + ...overrides, + }; +} + function document(overrides: Partial = {}): Document { return { id: '00000000-0000-0000-0000-000000000001', @@ -16,6 +38,7 @@ function document(overrides: Partial = {}): Document { errorMessage: null, processingToken: null, uploadedByIdpUuid: 'admin-1', + ownerOrganizationId: ORGANIZATION_ID, isActive: true, createdAt: new Date(), updatedAt: new Date(), @@ -28,317 +51,563 @@ function document(overrides: Partial = {}): Document { function createService() { const repo = { - createUploading: jest.fn<(...args: unknown[]) => Promise>(), - markQueuedAfterUpload: jest.fn<(id: string) => Promise>(), - hardDelete: jest.fn<(id: string) => Promise>(), - cancelAndSoftDelete: - jest.fn<(id: string, idpUuid: string) => Promise>(), - findById: jest.fn<(id: string) => Promise>(), - updateExpiresAt: - jest.fn< - ( - id: string, - idpUuid: string, - expiresAt: Date | null, - ) => Promise - >(), - enqueueReprocess: - jest.fn< - ( - id: string, - idpUuid: string, - cooldownBefore: Date, - now: Date, - ) => Promise - >(), + hardDelete: jest.fn(), + listByUploader: jest.fn(async () => [ + document(), + ]), }; const gcs = { toGsPath: jest.fn((path: string) => `gs://bucket/${path}`), - uploadPdf: - jest.fn<(resourceName: string, pdfBytes: Buffer) => Promise>(), - deleteResourceArtifacts: jest.fn<(resourceName: string) => Promise>(), + uploadPdf: jest.fn(), + deleteResourceArtifacts: + jest.fn(), + }; + const organizationsRepo = { + findDocument: jest.fn(async () => document()), + findOrganization: jest.fn(async (id: string) => ({ + id, + name: '조직', + slug: 'organization', + isDefault: false, + createdByIdpUuid: 'admin-1', + createdAt: new Date(), + updatedAt: new Date(), + })), + hydrateDocuments: jest.fn( + async (rows: Document[]) => + rows.map((row) => ({ + document: row, + ownerOrganization: { + id: row.ownerOrganizationId, + name: '인포팀', + slug: 'infoteam', + }, + uploader: { + idpUuid: row.uploadedByIdpUuid, + email: 'admin@example.com', + name: 'Admin', + }, + sharedOrganizations: [], + })), + ), + findAcceptedMemberships: jest.fn( + async (_organizationIds: string[], _memberIdpUuid: string) => [ + { + organizationId: ORGANIZATION_ID, + role: 'MANAGER' as 'MANAGER' | 'MEMBER', + }, + ], + ), + isCurrentSuperAdmin: jest.fn(async (actor: AdminPrincipal) => + Promise.resolve(actor.role === 'SUPER_ADMIN'), + ), + listManageableDocuments: jest.fn< + OrganizationsRepository['listManageableDocuments'] + >(async () => [document()]), + listOrganizationDocuments: jest.fn(async () => [document()]), + createUploadingDocument: jest.fn< + OrganizationsRepository['createUploadingDocument'] + >(async () => document()), + finalizeUploadingDocument: jest.fn< + OrganizationsRepository['finalizeUploadingDocument'] + >(async () => ({ kind: 'ok', document: document({ status: 'queued' }) })), + cancelAndSoftDeleteDocument: jest.fn< + OrganizationsRepository['cancelAndSoftDeleteDocument'] + >(async () => ({ kind: 'ok', document: document({ isActive: false }) })), + updateDocumentExpiresAt: jest.fn< + OrganizationsRepository['updateDocumentExpiresAt'] + >(async () => ({ kind: 'ok', document: document() })), + enqueueDocumentReprocess: jest.fn< + OrganizationsRepository['enqueueDocumentReprocess'] + >(async () => ({ kind: 'ok', document: document({ status: 'queued' }) })), + setShare: jest.fn(async () => ({ + kind: 'ok', + document: document(), + })), + removeShare: jest.fn(async () => ({ + kind: 'ok', + document: document(), + })), + transferDocument: jest.fn( + async () => ({ kind: 'ok', document: document() }), + ), + }; + const accessDecision = (row = document()) => ({ + document: row, + relation: 'OWNER' as const, + ownerRole: 'MANAGER' as const, + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }); + const access = { + isSuperAdmin: jest.fn( + (actor: AdminPrincipal) => actor.role === 'SUPER_ADMIN', + ), + resolveUploadOrganization: jest.fn< + OrganizationAccessService['resolveUploadOrganization'] + >(async () => ({ + id: ORGANIZATION_ID, + name: '인포팀', + slug: 'infoteam', + isDefault: true, + createdByIdpUuid: null, + createdAt: new Date(), + updatedAt: new Date(), + })), + requireDocumentView: jest.fn(async () => accessDecision()), + requireDocumentManage: jest.fn(async () => accessDecision()), + requireDocumentShare: jest.fn(async () => accessDecision()), + requireOrganizationMember: jest.fn(async () => null), + requireOrganizationManager: jest.fn(async () => null), }; return { service: new UploadService( repo as unknown as DocumentsRepository, gcs as unknown as GcsStorageService, + organizationsRepo as unknown as OrganizationsRepository, + access as unknown as OrganizationAccessService, ), repo, gcs, + organizationsRepo, + access, }; } -describe('UploadService atomic transitions', () => { - it('reserves the DB resource name before uploading to GCS', async () => { - const { service, repo, gcs } = createService(); - const calls: string[] = []; - const reserved = document(); - const queued = document({ status: 'queued' }); +describe('UploadService organization-aware transitions', () => { + it('validates organization access before reserving or uploading', async () => { + const { service, organizationsRepo, gcs, access } = createService(); + access.resolveUploadOrganization.mockRejectedValue( + new ForbiddenException('membership required'), + ); - repo.createUploading.mockImplementation(() => { - calls.push('reserve'); - return Promise.resolve(reserved); - }); - gcs.uploadPdf.mockImplementation(() => { - calls.push('upload'); - return Promise.resolve('gs://bucket/test.pdf'); - }); - repo.markQueuedAfterUpload.mockImplementation(() => { - calls.push('queue'); - return Promise.resolve(queued); - }); + await expect( + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), + ORGANIZATION_ID, + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(organizationsRepo.createUploadingDocument).not.toHaveBeenCalled(); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); + }); + + it('stops before GCS when membership is revoked before reservation', async () => { + const { service, organizationsRepo, gcs } = createService(); + organizationsRepo.createUploadingDocument.mockRejectedValue( + new RepositoryAuthorizationError(), + ); + + await expect( + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), + ORGANIZATION_ID, + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); + }); + + it('uses the default resolver only when organizationId is omitted', async () => { + const { service, organizationsRepo, gcs, access } = createService(); + gcs.uploadPdf.mockResolvedValue('gs://bucket/test.pdf'); await service.upload( Buffer.from('%PDF-test'), 'test.pdf', '테스트', - 'admin-1', + principal(), ); - expect(calls).toEqual(['reserve', 'upload', 'queue']); + expect(access.resolveUploadOrganization).toHaveBeenCalledWith( + undefined, + expect.objectContaining({ uuid: 'admin-1' }), + ); + expect(organizationsRepo.createUploadingDocument).toHaveBeenCalledWith( + expect.objectContaining({ + ownerOrganizationId: ORGANIZATION_ID, + actor: principal(), + }), + ); }); - it('returns conflict without touching GCS when the name is already reserved', async () => { - const { service, repo, gcs } = createService(); - repo.createUploading.mockRejectedValue({ code: '23505' }); + it('reauthorizes a direct read after hydration before returning data', async () => { + const { service, access, organizationsRepo } = createService(); + access.requireDocumentView + .mockResolvedValueOnce({ + document: document(), + relation: 'OWNER', + ownerRole: 'MANAGER', + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }) + .mockRejectedValueOnce(new NotFoundException('Document not found')); await expect( - service.upload(Buffer.from('%PDF-test'), 'test.pdf', '테스트', 'admin-1'), - ).rejects.toBeInstanceOf(ConflictException); - expect(gcs.uploadPdf).not.toHaveBeenCalled(); + service.getById(document().id, principal()), + ).rejects.toBeInstanceOf(NotFoundException); + expect(organizationsRepo.hydrateDocuments).toHaveBeenCalled(); + expect(access.requireDocumentView).toHaveBeenCalledTimes(2); }); - it('maps GCS upload failures to 503 without exposing the raw error', async () => { - const { service, repo, gcs } = createService(); - repo.createUploading.mockResolvedValue(document()); - gcs.uploadPdf.mockRejectedValue(new Error('bucket ACL denied xyz')); - repo.hardDelete.mockResolvedValue(undefined); - + it('does not fall back after an invalid supplied organizationId', async () => { + const { service, organizationsRepo, access } = createService(); + access.resolveUploadOrganization.mockRejectedValue( + new BadRequestException('invalid organization'), + ); await expect( - service.upload(Buffer.from('%PDF-test'), 'test.pdf', '테스트', 'admin-1'), - ).rejects.toMatchObject({ - response: { - statusCode: 503, - message: 'Document storage is temporarily unavailable', - }, - }); - expect(repo.hardDelete).toHaveBeenCalled(); + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), + 'invalid-id', + ), + ).rejects.toBeInstanceOf(BadRequestException); + expect(organizationsRepo.createUploadingDocument).not.toHaveBeenCalled(); }); - it('cancels the DB processing attempt before deleting GCS artifacts', async () => { - const { service, repo, gcs } = createService(); + it('reserves before GCS upload and queues afterward', async () => { + const { service, organizationsRepo, gcs } = createService(); const calls: string[] = []; - repo.cancelAndSoftDelete.mockImplementation(() => { - calls.push('cancel'); - return Promise.resolve(document({ isActive: false })); + organizationsRepo.createUploadingDocument.mockImplementation(async () => { + calls.push('reserve'); + return document(); }); - gcs.deleteResourceArtifacts.mockImplementation(() => { - calls.push('delete-artifacts'); - return Promise.resolve(); + gcs.uploadPdf.mockImplementation(async () => { + calls.push('upload'); + return 'gs://bucket/test.pdf'; + }); + organizationsRepo.finalizeUploadingDocument.mockImplementation(async () => { + calls.push('queue'); + return { kind: 'ok', document: document({ status: 'queued' }) }; }); - await service.delete( - '00000000-0000-0000-0000-000000000001', - 'admin-1', - ); - - expect(calls).toEqual(['cancel', 'delete-artifacts']); - expect(repo.cancelAndSoftDelete).toHaveBeenCalledWith( - '00000000-0000-0000-0000-000000000001', - 'admin-1', + await service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), ); + expect(calls).toEqual(['reserve', 'upload', 'queue']); }); - it('maps GCS delete failures to 503 without exposing the raw error', async () => { - const { service, repo, gcs } = createService(); - repo.cancelAndSoftDelete.mockResolvedValue(document({ isActive: false })); - gcs.deleteResourceArtifacts.mockRejectedValue( - new Error('Permission denied on objects/test/'), - ); - + it('returns conflict without GCS when resource name is reserved', async () => { + const { service, organizationsRepo, gcs } = createService(); + organizationsRepo.createUploadingDocument.mockRejectedValue({ + code: '23505', + }); await expect( - service.delete( - '00000000-0000-0000-0000-000000000001', - 'admin-1', + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), ), - ).rejects.toMatchObject({ - response: { - statusCode: 503, - message: 'Document storage is temporarily unavailable', - }, - }); + ).rejects.toBeInstanceOf(ConflictException); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); }); - it('returns 404 without touching GCS when delete ownership does not match', async () => { + it('rolls back DB reservation and maps GCS upload failure to 503', async () => { const { service, repo, gcs } = createService(); - repo.cancelAndSoftDelete.mockResolvedValue(null); - + gcs.uploadPdf.mockRejectedValue(new Error('secret storage error')); await expect( - service.delete( - '00000000-0000-0000-0000-000000000001', - 'other-admin', + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), ), - ).rejects.toMatchObject({ status: 404 }); - expect(repo.cancelAndSoftDelete).toHaveBeenCalledWith( - '00000000-0000-0000-0000-000000000001', - 'other-admin', - ); - expect(gcs.deleteResourceArtifacts).not.toHaveBeenCalled(); + ).rejects.toMatchObject({ status: 503 }); + expect(gcs.deleteResourceArtifacts).toHaveBeenCalledWith('test'); + expect(repo.hardDelete).toHaveBeenCalled(); }); - it('returns 404 when reprocess ownership does not match', async () => { - const { service, repo } = createService(); - repo.findById.mockResolvedValue(document({ uploadedByIdpUuid: 'admin-1' })); + it('reauthorizes after GCS upload before publishing the document', async () => { + const { service, repo, organizationsRepo, gcs } = createService(); + gcs.uploadPdf.mockResolvedValue('gs://bucket/test.pdf'); + organizationsRepo.finalizeUploadingDocument.mockResolvedValue({ + kind: 'forbidden', + }); await expect( - service.reprocess( - '00000000-0000-0000-0000-000000000001', - 'other-admin', + service.upload( + Buffer.from('%PDF-test'), + 'test.pdf', + '테스트', + principal(), + ORGANIZATION_ID, ), - ).rejects.toMatchObject({ status: 404 }); - expect(repo.enqueueReprocess).not.toHaveBeenCalled(); + ).rejects.toBeInstanceOf(ForbiddenException); + expect(organizationsRepo.finalizeUploadingDocument).toHaveBeenCalledWith({ + documentId: document().id, + expectedOwnerOrganizationId: ORGANIZATION_ID, + actor: principal(), + }); + expect(gcs.deleteResourceArtifacts).toHaveBeenCalledWith('test'); + expect(repo.hardDelete).toHaveBeenCalledWith(document().id); + }); + + it('uses expected owner organization in the soft-delete predicate', async () => { + const { service, organizationsRepo, gcs } = createService(); + gcs.deleteResourceArtifacts.mockResolvedValue(undefined); + await service.delete(document().id, principal()); + expect(organizationsRepo.cancelAndSoftDeleteDocument).toHaveBeenCalledWith({ + documentId: document().id, + expectedOwnerOrganizationId: ORGANIZATION_ID, + actor: principal(), + }); }); it.each(['uploading', 'queued', 'processing'] as const)( 'rejects reprocess while status is %s', async (status) => { - const { service, repo } = createService(); - repo.findById.mockResolvedValue(document({ status })); - + const { service, organizationsRepo, access } = createService(); + access.requireDocumentManage.mockResolvedValue({ + document: document({ status }), + relation: 'OWNER', + ownerRole: 'MANAGER', + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }); await expect( - service.reprocess( - '00000000-0000-0000-0000-000000000001', - 'admin-1', - ), + service.reprocess(document().id, principal()), ).rejects.toBeInstanceOf(ConflictException); - expect(repo.enqueueReprocess).not.toHaveBeenCalled(); + expect(organizationsRepo.enqueueDocumentReprocess).not.toHaveBeenCalled(); }, ); - it('rejects reprocess during the 24-hour cooldown', async () => { - const { service, repo } = createService(); - repo.findById.mockResolvedValue( - document({ - status: 'ready', - lastReprocessedAt: new Date(Date.now() - 23 * 60 * 60 * 1000), - }), - ); - - try { - await service.reprocess( - '00000000-0000-0000-0000-000000000001', - 'admin-1', - ); - throw new Error('Expected reprocess to be rejected'); - } catch (error) { - expect(error).toEqual( - expect.objectContaining({ - status: 429, - response: expect.objectContaining({ - retryAt: expect.any(String), - }), - }), - ); - } - expect(repo.enqueueReprocess).not.toHaveBeenCalled(); - }); - - it('allows reprocess after the 24-hour cooldown', async () => { - const { service, repo } = createService(); + it('preserves the reprocess cooldown and owner-state predicate', async () => { + const { service, organizationsRepo, access } = createService(); const current = document({ status: 'ready', lastReprocessedAt: new Date(Date.now() - 25 * 60 * 60 * 1000), }); - repo.findById.mockResolvedValue(current); - repo.enqueueReprocess.mockResolvedValue( - document({ status: 'queued', lastReprocessedAt: new Date() }), + access.requireDocumentManage.mockResolvedValue({ + document: current, + relation: 'OWNER', + ownerRole: 'MANAGER', + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }); + organizationsRepo.enqueueDocumentReprocess.mockResolvedValue({ + kind: 'ok', + document: document({ status: 'queued', lastReprocessedAt: new Date() }), + }); + await service.reprocess(current.id, principal()); + expect(organizationsRepo.enqueueDocumentReprocess).toHaveBeenCalledWith({ + documentId: current.id, + expectedOwnerOrganizationId: ORGANIZATION_ID, + cooldownBefore: expect.any(Date), + now: expect.any(Date), + actor: principal(), + }); + }); + + it('shared viewers cannot mutate because access is checked first', async () => { + const { service, organizationsRepo, access } = createService(); + access.requireDocumentManage.mockRejectedValue( + new ForbiddenException('Document management permission required'), ); + await expect( + service.delete(document().id, principal()), + ).rejects.toBeInstanceOf(ForbiddenException); + expect( + organizationsRepo.cancelAndSoftDeleteDocument, + ).not.toHaveBeenCalled(); + }); - await expect(service.reprocess(current.id, 'admin-1')).resolves.toEqual( - expect.objectContaining({ status: 'queued', canReprocess: false }), + it('sharing returns the committed result without a second authorization read', async () => { + const { service, gcs, organizationsRepo, access } = createService(); + await service.shareDocument( + document().id, + '00000000-0000-0000-0000-000000000020', + principal(), ); + expect(organizationsRepo.setShare).toHaveBeenCalled(); + expect(access.requireDocumentView).not.toHaveBeenCalled(); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); + expect(organizationsRepo.enqueueDocumentReprocess).not.toHaveBeenCalled(); }); - it.each(['ready', 'failed'] as const)( - 'atomically requeues a %s document', - async (status) => { - const { service, repo } = createService(); - const current = document({ status }); - const queued = document({ - status: 'queued', - lastReprocessedAt: new Date(), - }); - repo.findById.mockResolvedValue(current); - repo.enqueueReprocess.mockResolvedValue(queued); - - const result = await service.reprocess(current.id, 'admin-1'); - - expect(repo.enqueueReprocess).toHaveBeenCalledWith( - current.id, - 'admin-1', - expect.any(Date), - expect.any(Date), - ); - expect(result.status).toBe('queued'); - expect(result.canReprocess).toBe(false); - }, - ); + it('does not let an unauthorized actor share a document', async () => { + const { service, organizationsRepo, access } = createService(); + access.requireDocumentShare.mockRejectedValue( + new ForbiddenException('Document sharing permission required'), + ); + await expect( + service.shareDocument( + document().id, + '00000000-0000-0000-0000-000000000020', + principal(), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(organizationsRepo.setShare).not.toHaveBeenCalled(); + }); - it('passes expiresAt into createUploading and returns isExpired=false', async () => { - const { service, repo, gcs } = createService(); - const future = new Date(Date.now() + 60_000); - const reserved = document({ expiresAt: future }); - const queued = document({ status: 'queued', expiresAt: future }); - repo.createUploading.mockResolvedValue(reserved); - gcs.uploadPdf.mockResolvedValue('gs://bucket/test.pdf'); - repo.markQueuedAfterUpload.mockResolvedValue(queued); + it('transfer returns the committed result without a second authorization read', async () => { + const { service, gcs, organizationsRepo, access } = createService(); + const targetId = '00000000-0000-0000-0000-000000000020'; + organizationsRepo.transferDocument.mockResolvedValue({ + kind: 'ok', + document: document({ ownerOrganizationId: targetId }), + }); + await service.transferDocument(document().id, targetId, principal()); + expect(organizationsRepo.transferDocument).toHaveBeenCalledWith( + expect.objectContaining({ + expectedOwnerOrganizationId: ORGANIZATION_ID, + targetOrganizationId: targetId, + }), + ); + expect(access.requireDocumentView).not.toHaveBeenCalled(); + expect(gcs.uploadPdf).not.toHaveBeenCalled(); + expect(organizationsRepo.enqueueDocumentReprocess).not.toHaveBeenCalled(); + }); - const result = await service.upload( - Buffer.from('%PDF-test'), - 'test.pdf', - '테스트', - 'admin-1', - future.toISOString(), + it('reports an uploading transfer as a conflict without mutating processing', async () => { + const { service, organizationsRepo } = createService(); + organizationsRepo.transferDocument.mockResolvedValue({ + kind: 'state_changed', + document: document({ status: 'uploading' }), + }); + await expect( + service.transferDocument( + document().id, + '00000000-0000-0000-0000-000000000020', + principal(), + ), + ).rejects.toBeInstanceOf(ConflictException); + }); + + it('redacts uploader PII and other share recipients from shared viewers', async () => { + const { service, organizationsRepo } = createService(); + const sharedOrganizationId = '00000000-0000-0000-0000-000000000020'; + organizationsRepo.findAcceptedMemberships.mockResolvedValue([ + { + organizationId: sharedOrganizationId, + role: 'MEMBER', + }, + ]); + organizationsRepo.hydrateDocuments.mockImplementation(async (rows) => + rows.map((row) => ({ + document: row, + ownerOrganization: { + id: row.ownerOrganizationId, + name: 'Owner', + slug: 'owner', + }, + uploader: { + idpUuid: 'owner-user', + email: 'owner@example.com', + name: 'Owner User', + }, + sharedOrganizations: [ + { id: sharedOrganizationId, name: 'Viewer Org', slug: 'viewer' }, + { + id: '00000000-0000-0000-0000-000000000030', + name: 'Unrelated Recipient', + slug: 'unrelated', + }, + ], + })), ); - expect(repo.createUploading).toHaveBeenCalledWith( - expect.objectContaining({ expiresAt: expect.any(Date) }), + const [item] = await service.listOrganizationDocuments( + sharedOrganizationId, + principal({ uuid: 'shared-user' }), ); - expect(result.expiresAt).toEqual(future); - expect(result.isExpired).toBe(false); + expect(item).toMatchObject({ + accessRelation: 'SHARED', + canManage: false, + uploader: null, + sharedOrganizations: [], + }); }); - it('updates expiresAt for the owner and clears with null', async () => { - const { service, repo } = createService(); - const current = document({ status: 'ready' }); - const cleared = document({ status: 'ready', expiresAt: null }); - repo.findById.mockResolvedValue(current); - repo.updateExpiresAt.mockResolvedValue(cleared); - - const result = await service.updateExpiresAt(current.id, 'admin-1', null); + it('omits shared documents when the viewing membership has been removed', async () => { + const { service, organizationsRepo } = createService(); + const sharedOrganizationId = '00000000-0000-0000-0000-000000000020'; + organizationsRepo.findAcceptedMemberships.mockResolvedValue([]); + organizationsRepo.hydrateDocuments.mockImplementation(async (rows) => + rows.map((row) => ({ + document: row, + ownerOrganization: { + id: row.ownerOrganizationId, + name: 'Owner', + slug: 'owner', + }, + uploader: null, + sharedOrganizations: [ + { id: sharedOrganizationId, name: 'Viewer Org', slug: 'viewer' }, + ], + })), + ); - expect(repo.updateExpiresAt).toHaveBeenCalledWith( - current.id, - 'admin-1', - null, + await expect( + service.listOrganizationDocuments( + sharedOrganizationId, + principal({ uuid: 'removed-user' }), + ), + ).resolves.toEqual([]); + expect(organizationsRepo.findAcceptedMemberships).toHaveBeenCalledWith( + expect.arrayContaining([ORGANIZATION_ID, sharedOrganizationId]), + 'removed-user', ); - expect(result.expiresAt).toBeNull(); - expect(result.isExpired).toBe(false); }); - it('marks isExpired when expiresAt is in the past', async () => { - const { service, repo } = createService(); - const past = new Date(Date.now() - 60_000); - repo.findById.mockResolvedValue( - document({ status: 'ready', expiresAt: past }), + it('requires target-organization MANAGER permission for transfer', async () => { + const { service, organizationsRepo, access } = createService(); + access.requireOrganizationManager.mockRejectedValue( + new ForbiddenException('Organization manager role required'), ); + await expect( + service.transferDocument( + document().id, + '00000000-0000-0000-0000-000000000020', + principal(), + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(organizationsRepo.transferDocument).not.toHaveBeenCalled(); + }); - const result = await service.getById( - '00000000-0000-0000-0000-000000000001', - 'admin-1', + it('keeps the legacy list scoped to documents uploaded by the caller', async () => { + const { service, repo, organizationsRepo } = createService(); + repo.listByUploader.mockResolvedValue([document()]); + const result = await service.listMyUploads(principal()); + expect(repo.listByUploader).toHaveBeenCalledWith(principal(), { + limit: 50, + offset: 0, + }); + expect(organizationsRepo.listManageableDocuments).not.toHaveBeenCalled(); + expect(result).toHaveLength(1); + }); + + it('delegates manageable listing to the organization repository', async () => { + const { service, repo, organizationsRepo } = createService(); + organizationsRepo.listManageableDocuments.mockResolvedValue([document()]); + const result = await service.listManageableDocuments(principal()); + expect(repo.listByUploader).not.toHaveBeenCalled(); + expect(organizationsRepo.listManageableDocuments).toHaveBeenCalledWith( + principal(), + { limit: 50, offset: 0 }, ); - expect(result.isExpired).toBe(true); - expect(result.expiresAt).toEqual(past); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + canManage: true, + accessRelation: 'OWNER', + }); }); }); @@ -347,7 +616,6 @@ describe('parseExpiresAt', () => { expect(parseExpiresAt(undefined)).toBeNull(); expect(parseExpiresAt(null)).toBeNull(); expect(parseExpiresAt('')).toBeNull(); - expect(parseExpiresAt(' ')).toBeNull(); }); it('rejects invalid and past values', () => { @@ -357,7 +625,7 @@ describe('parseExpiresAt', () => { ).toThrow(BadRequestException); }); - it('accepts future ISO-8601', () => { + it('accepts a future ISO-8601 value', () => { const future = new Date(Date.now() + 60_000).toISOString(); expect(parseExpiresAt(future)?.toISOString()).toBe(future); }); diff --git a/src/upload/upload.service.ts b/src/upload/upload.service.ts index ceb68d3..8f62902 100644 --- a/src/upload/upload.service.ts +++ b/src/upload/upload.service.ts @@ -1,18 +1,29 @@ import { - Injectable, - Logger, - NotFoundException, BadRequestException, ConflictException, - ServiceUnavailableException, + ForbiddenException, HttpException, HttpStatus, + Injectable, + Logger, + NotFoundException, + ServiceUnavailableException, } from '@nestjs/common'; +import type { Document } from '../db'; import { DocumentsRepository } from '../pdf-processor/documents.repository'; import { GcsStorageService } from '../pdf-processor/gcs-storage.service'; import { toResourceName } from '../pdf-processor/pdf-chunk-parser'; import { isExpiredAt } from '../retrieval/retrieval.repository'; -import type { Document } from '../db'; +import { OrganizationAccessService } from '../organizations/organization-access.service'; +import { evaluateDocumentAccess } from '../organizations/organization-access.policy'; +import { + OrganizationsRepository, + RepositoryAuthorizationError, +} from '../organizations/organizations.repository'; +import type { + AdminPrincipal, + DocumentAccessDecision, +} from '../organizations/organization.types'; import type { DocumentListItemDto } from './dto/document-list-item.dto'; const PDF_MIME = 'application/pdf'; @@ -20,10 +31,6 @@ const DEFAULT_LIMIT = 50; const MAX_LIMIT = 100; export const REPROCESS_COOLDOWN_MS = 24 * 60 * 60 * 1000; -/** - * Parse optional ISO-8601 expiresAt. Empty/undefined → null (never expires). - * Invalid or past timestamps → 400. - */ export function parseExpiresAt(raw?: string | null): Date | null { if (raw == null) return null; const trimmed = raw.trim(); @@ -48,48 +55,79 @@ export class UploadService { constructor( private readonly documentsRepo: DocumentsRepository, private readonly gcs: GcsStorageService, + private readonly organizationsRepo: OrganizationsRepository, + private readonly access: OrganizationAccessService, ) {} async listMyUploads( - idpUuid: string, + principal: AdminPrincipal, options: { limit?: number; offset?: number } = {}, ): Promise { - const limit = Math.min(options.limit ?? DEFAULT_LIMIT, MAX_LIMIT); - const offset = Math.max(0, options.offset ?? 0); + const paging = this.normalizePaging(options); + const rows = await this.documentsRepo.listByUploader(principal, paging); + return this.toListItems(rows, principal, undefined, [], true); + } - const rows = await this.documentsRepo.listByUploader(idpUuid, { - limit, - offset, - }); + async listManageableDocuments( + principal: AdminPrincipal, + options: { limit?: number; offset?: number } = {}, + ): Promise { + const paging = this.normalizePaging(options); + const rows = await this.organizationsRepo.listManageableDocuments( + principal, + paging, + ); + return this.toListItems(rows, principal, undefined, [], true); + } - return rows.map((row) => this.toListItem(row)); + async listOrganizationDocuments( + organizationId: string, + principal: AdminPrincipal, + options: { limit?: number; offset?: number } = {}, + ): Promise { + await this.access.requireOrganizationMember(organizationId, principal); + const rows = await this.organizationsRepo.listOrganizationDocuments( + organizationId, + principal, + this.normalizePaging(options), + ); + return this.toListItems(rows, principal, organizationId, [], true); } - async getById(id: string, idpUuid: string): Promise { - const row = await this.documentsRepo.findById(id); - if (!row || !row.isActive) { - throw new NotFoundException(`Document not found: ${id}`); - } - if (row.uploadedByIdpUuid !== idpUuid) { - throw new NotFoundException(`Document not found: ${id}`); - } - return this.toListItem(row); + async getById( + id: string, + principal: AdminPrincipal, + ): Promise { + const decision = await this.access.requireDocumentView(id, principal); + return ( + await this.toListItems( + [decision.document], + principal, + undefined, + [decision], + false, + true, + ) + )[0]; } - /** - * Upload PDF to GCS and enqueue processing (status=queued). - */ async upload( fileBuffer: Buffer, filename: string, title: string, - idpUuid: string, + principal: AdminPrincipal, + organizationId?: string, expiresAtRaw?: string | null, ): Promise { if (!fileBuffer?.length) { throw new BadRequestException('file is required'); } + // Permission is checked before reserving a DB row or touching GCS. + const organization = await this.access.resolveUploadOrganization( + organizationId, + principal, + ); const expiresAt = parseExpiresAt(expiresAtRaw); const resourceName = toResourceName(filename || 'document.pdf'); if (!resourceName.trim()) { @@ -99,12 +137,13 @@ export class UploadService { const gcsPdfPath = this.gcs.toGsPath(`${resourceName}.pdf`); let reservation: Document; try { - reservation = await this.documentsRepo.createUploading({ + reservation = await this.organizationsRepo.createUploadingDocument({ title, resourceName, gcsPdfPath, - uploadedByIdpUuid: idpUuid, + ownerOrganizationId: organization.id, expiresAt, + actor: principal, }); } catch (error) { if (isUniqueViolation(error)) { @@ -112,6 +151,9 @@ export class UploadService { `An active document with resource name "${resourceName}" already exists`, ); } + if (error instanceof RepositoryAuthorizationError) { + throw new ForbiddenException('Organization membership changed'); + } throw error; } @@ -130,29 +172,38 @@ export class UploadService { let record: Document | null; try { - record = await this.documentsRepo.markQueuedAfterUpload(reservation.id); - if (!record) { - throw new Error('Upload reservation is no longer active'); + const result = await this.organizationsRepo.finalizeUploadingDocument({ + documentId: reservation.id, + expectedOwnerOrganizationId: organization.id, + actor: principal, + }); + this.assertDocumentMutation(result); + if (result.kind !== 'ok') { + throw new ConflictException('Upload reservation state changed'); } + record = result.document; } catch (error) { await this.rollbackUpload(reservation.id, resourceName); + if (error instanceof HttpException) throw error; throw new Error( `Failed to enqueue the uploaded document: ${error instanceof Error ? error.message : String(error)}`, ); } this.logger.log(`Upload queued: id=${record.id} resource=${resourceName}`); - return this.toListItem(record); + return (await this.toListItems([record], principal))[0]; } - /** - * Soft-delete DB row and remove GCS artifacts. - */ - async delete(id: string, idpUuid: string): Promise { - const row = await this.documentsRepo.cancelAndSoftDelete(id, idpUuid); - if (!row) { - throw new NotFoundException(`Document not found: ${id}`); - } + async delete(id: string, principal: AdminPrincipal): Promise { + const decision = await this.access.requireDocumentManage(id, principal); + const result = await this.organizationsRepo.cancelAndSoftDeleteDocument({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, + actor: principal, + }); + this.assertDocumentMutation(result); + if (result.kind !== 'ok') return; + const row = result.document; try { await this.gcs.deleteResourceArtifacts(row.resourceName); @@ -164,118 +215,306 @@ export class UploadService { 'Document storage is temporarily unavailable', ); } - this.logger.log(`Document deleted: id=${id} resource=${row.resourceName}`); } - /** - * Clear chunks and re-enqueue for processing. - */ - async reprocess(id: string, idpUuid: string): Promise { - const row = await this.documentsRepo.findById(id); - if ( - !row || - !row.isActive || - row.uploadedByIdpUuid !== idpUuid - ) { - throw new NotFoundException(`Document not found: ${id}`); - } - + async reprocess( + id: string, + principal: AdminPrincipal, + ): Promise { + const decision = await this.access.requireDocumentManage(id, principal); const now = new Date(); - this.assertReprocessEligible(row, now); + this.assertReprocessEligible(decision.document, now); - const cooldownBefore = new Date(now.getTime() - REPROCESS_COOLDOWN_MS); - const updated = await this.documentsRepo.enqueueReprocess( - id, - idpUuid, - cooldownBefore, + const result = await this.organizationsRepo.enqueueDocumentReprocess({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, + cooldownBefore: new Date(now.getTime() - REPROCESS_COOLDOWN_MS), now, - ); - if (!updated) { - // Re-read to classify a concurrent state transition accurately. - const latest = await this.documentsRepo.findById(id); - if ( - !latest || - !latest.isActive || - latest.uploadedByIdpUuid !== idpUuid - ) { - throw new NotFoundException(`Document not found: ${id}`); - } - this.assertReprocessEligible(latest, new Date()); + actor: principal, + }); + if (result.kind === 'state_changed') { + this.assertReprocessEligible(result.document, new Date()); throw new ConflictException('Document reprocess state changed'); } - + this.assertDocumentMutation(result); + if (result.kind !== 'ok') { + throw new ConflictException('Document reprocess state changed'); + } + const updated = result.document; this.logger.log(`Document requeued: id=${id}`); - return this.toListItem(updated); + return (await this.toListItems([updated], principal))[0]; } async updateExpiresAt( id: string, - idpUuid: string, + principal: AdminPrincipal, expiresAtRaw: string | null, ): Promise { - const row = await this.documentsRepo.findById(id); - if (!row || !row.isActive) { - throw new NotFoundException(`Document not found: ${id}`); - } - if (row.uploadedByIdpUuid !== idpUuid) { - throw new NotFoundException(`Document not found: ${id}`); - } - + const decision = await this.access.requireDocumentManage(id, principal); const expiresAt = expiresAtRaw === null ? null : parseExpiresAt(expiresAtRaw); - const updated = await this.documentsRepo.updateExpiresAt( - id, - idpUuid, + const result = await this.organizationsRepo.updateDocumentExpiresAt({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, expiresAt, + actor: principal, + }); + this.assertDocumentMutation(result); + if (result.kind !== 'ok') { + throw new ConflictException('Document ownership or state changed'); + } + const updated = result.document; + return (await this.toListItems([updated], principal))[0]; + } + + async shareDocument( + id: string, + targetOrganizationId: string, + principal: AdminPrincipal, + ): Promise { + const decision = await this.access.requireDocumentShare(id, principal); + if (decision.document.ownerOrganizationId === targetOrganizationId) { + throw new BadRequestException( + 'Owning organization cannot be a share target', + ); + } + if ( + !(await this.organizationsRepo.findOrganization(targetOrganizationId)) + ) { + throw new NotFoundException('Target organization not found'); + } + const result = await this.organizationsRepo.setShare({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, + targetOrganizationId, + actor: principal, + }); + this.assertDocumentMutation(result); + if (result.kind !== 'ok') { + throw new ConflictException('Document sharing state changed'); + } + return ( + await this.toListItems([result.document], principal, undefined, [ + { ...decision, document: result.document }, + ]) + )[0]; + } + + async unshareDocument( + id: string, + targetOrganizationId: string, + principal: AdminPrincipal, + ): Promise { + const decision = await this.access.requireDocumentShare(id, principal); + const result = await this.organizationsRepo.removeShare({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, + targetOrganizationId, + actor: principal, + }); + this.assertDocumentMutation(result); + } + + async transferDocument( + id: string, + targetOrganizationId: string, + principal: AdminPrincipal, + ): Promise { + const decision = await this.access.requireDocumentShare(id, principal); + if (decision.document.ownerOrganizationId === targetOrganizationId) { + throw new BadRequestException( + 'Document is already owned by target organization', + ); + } + if ( + !(await this.organizationsRepo.findOrganization(targetOrganizationId)) + ) { + throw new NotFoundException('Target organization not found'); + } + await this.access.requireOrganizationManager( + targetOrganizationId, + principal, ); - if (!updated) { - throw new NotFoundException(`Document not found: ${id}`); + const result = await this.organizationsRepo.transferDocument({ + documentId: id, + expectedOwnerOrganizationId: decision.document.ownerOrganizationId, + targetOrganizationId, + actor: principal, + }); + this.assertDocumentMutation(result); + if (result.kind !== 'ok') { + throw new ConflictException('Document transfer state changed'); } - return this.toListItem(updated); + return ( + await this.toListItems([result.document], principal, undefined, [ + { + document: result.document, + relation: 'OWNER', + ownerRole: this.access.isSuperAdmin(principal) ? null : 'MANAGER', + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }, + ]) + )[0]; } - private toListItem(row: Document): DocumentListItemDto { - const reprocessAvailableAt = row.lastReprocessedAt - ? new Date(row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS) - : null; - const statusAllowsReprocess = - row.status === 'ready' || row.status === 'failed'; - const canReprocess = - statusAllowsReprocess && - (!reprocessAvailableAt || reprocessAvailableAt.getTime() <= Date.now()); + private async toListItems( + rows: Document[], + principal: AdminPrincipal, + organizationContextId?: string, + knownDecisions: DocumentAccessDecision[] = [], + omitUnauthorized = false, + reauthorizeKnownDecisions = false, + ): Promise { + const records = await this.organizationsRepo.hydrateDocuments(rows); + const currentSuperAdmin = + await this.organizationsRepo.isCurrentSuperAdmin(principal); + const authorizationOrganizationIds = [ + ...new Set([ + ...rows.map((row) => row.ownerOrganizationId), + ...(organizationContextId ? [organizationContextId] : []), + ]), + ]; + const memberships = currentSuperAdmin + ? [] + : await this.organizationsRepo.findAcceptedMemberships( + authorizationOrganizationIds, + principal.uuid, + ); + const roleByOrganization = new Map( + memberships.map((membership) => [ + membership.organizationId, + membership.role, + ]), + ); + const currentKnownDecisions = reauthorizeKnownDecisions + ? await Promise.all( + knownDecisions.map((decision) => + this.access.requireDocumentView(decision.document.id, principal), + ), + ) + : knownDecisions; + const decisions = new Map( + currentKnownDecisions.map((decision) => [decision.document.id, decision]), + ); + + const items = records.map((record): DocumentListItemDto | null => { + const row = record.document; + let decision = decisions.get(row.id); + if (!decision) { + if (currentSuperAdmin) { + decision = { + document: row, + relation: + organizationContextId && + organizationContextId !== row.ownerOrganizationId + ? 'SHARED' + : 'OWNER', + ownerRole: null, + canView: true, + canManage: true, + canShare: true, + canTransfer: true, + }; + } else { + const ownerRole = + roleByOrganization.get(row.ownerOrganizationId) ?? null; + decision = evaluateDocumentAccess({ + document: row, + actorIdpUuid: principal.uuid, + ownerRole, + shared: + !ownerRole && + Boolean( + organizationContextId && + roleByOrganization.has(organizationContextId) && + record.sharedOrganizations.some( + (organization) => organization.id === organizationContextId, + ), + ), + }); + } + } + + if (omitUnauthorized && !decision.canView) return null; + + const reprocessAvailableAt = row.lastReprocessedAt + ? new Date(row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS) + : null; + const statusAllowsReprocess = + row.status === 'ready' || row.status === 'failed'; + return { + id: row.id, + title: row.title, + resourceName: row.resourceName, + status: row.status, + summary: row.summary, + gcsPdfPath: row.gcsPdfPath, + errorMessage: row.errorMessage, + uploadedAt: row.createdAt, + processedAt: row.processedAt, + lastReprocessedAt: row.lastReprocessedAt, + reprocessAvailableAt, + canReprocess: + decision.canManage && + statusAllowsReprocess && + (!reprocessAvailableAt || + reprocessAvailableAt.getTime() <= Date.now()), + expiresAt: row.expiresAt, + isExpired: isExpiredAt(row.expiresAt), + ownerOrganization: record.ownerOrganization, + uploader: + !currentSuperAdmin && decision.relation === 'SHARED' + ? null + : record.uploader, + sharedOrganizations: + !currentSuperAdmin && decision.relation === 'SHARED' + ? [] + : record.sharedOrganizations, + accessRelation: decision.relation, + canManage: decision.canManage, + canShare: decision.canShare, + canTransfer: decision.canTransfer, + }; + }); + return items.filter((item): item is DocumentListItemDto => item !== null); + } + private normalizePaging(options: { limit?: number; offset?: number }) { return { - id: row.id, - title: row.title, - resourceName: row.resourceName, - status: row.status, - summary: row.summary, - gcsPdfPath: row.gcsPdfPath, - errorMessage: row.errorMessage, - uploadedAt: row.createdAt, - processedAt: row.processedAt, - lastReprocessedAt: row.lastReprocessedAt, - reprocessAvailableAt, - canReprocess, - expiresAt: row.expiresAt, - isExpired: isExpiredAt(row.expiresAt), + limit: Math.min(options.limit ?? DEFAULT_LIMIT, MAX_LIMIT), + offset: Math.max(0, options.offset ?? 0), }; } + private assertDocumentMutation(result: { + kind: 'ok' | 'not_found' | 'stale_owner' | 'forbidden' | 'state_changed'; + }): void { + if (result.kind === 'ok') return; + if (result.kind === 'not_found') + throw new NotFoundException('Document not found'); + if (result.kind === 'forbidden') { + throw new ForbiddenException('Organization permission changed'); + } + if (result.kind === 'state_changed') + throw new ConflictException('Document state changed'); + throw new ConflictException('Document ownership changed'); + } + private assertReprocessEligible(row: Document, now: Date): void { if (row.status !== 'ready' && row.status !== 'failed') { throw new ConflictException( `Document cannot be reprocessed while status is "${row.status}"`, ); } - if (!row.lastReprocessedAt) return; const retryAt = new Date( row.lastReprocessedAt.getTime() + REPROCESS_COOLDOWN_MS, ); if (retryAt.getTime() <= now.getTime()) return; - throw new HttpException( { statusCode: HttpStatus.TOO_MANY_REQUESTS, diff --git a/test/organization-database.e2e-spec.ts b/test/organization-database.e2e-spec.ts new file mode 100644 index 0000000..2fecca3 --- /dev/null +++ b/test/organization-database.e2e-spec.ts @@ -0,0 +1,1096 @@ +import { ExecutionContext, NotFoundException } from '@nestjs/common'; +import { + FastifyAdapter, + NestFastifyApplication, +} from '@nestjs/platform-fastify'; +import { Test } from '@nestjs/testing'; +import { and, eq, inArray } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/postgres-js'; +import type { FastifyRequest } from 'fastify'; +import postgres from 'postgres'; +import { AdminContext } from '../src/auth/context/admin-context.entity'; +import { AdminJwtGuard } from '../src/auth/guards/admin-jwt.guard'; +import { + admins, + documentChunks, + documentOrganizationShares, + documentOwnershipTransfers, + documents, + organizationMemberships, + organizations, + type Database, +} from '../src/db'; +import * as schema from '../src/db/schema'; +import { OrganizationAccessService } from '../src/organizations/organization-access.service'; +import { + OrganizationsRepository, + RepositoryAuthorizationError, +} from '../src/organizations/organizations.repository'; +import { DocumentsRepository } from '../src/pdf-processor/documents.repository'; +import { GcsStorageService } from '../src/pdf-processor/gcs-storage.service'; +import { RetrievalRepository } from '../src/retrieval/retrieval.repository'; +import { UploadController } from '../src/upload/upload.controller'; +import { UploadService } from '../src/upload/upload.service'; + +type AdminRequest = FastifyRequest & { user?: AdminContext }; + +class HeaderAdminGuard { + canActivate(context: ExecutionContext): boolean { + const request = context.switchToHttp().getRequest(); + request.user = new AdminContext( + String(request.headers['x-admin-email'] ?? 'admin@example.com'), + String(request.headers['x-admin-uuid'] ?? 'admin'), + 'Database E2E Admin', + String(request.headers['x-admin-role'] ?? 'ADMIN'), + ); + return true; + } +} + +const describeDatabase = + process.env.ORGANIZATION_TEST_DB === 'true' ? describe : describe.skip; + +describeDatabase('Organization database invariants (e2e)', () => { + const testPrefix = `org-e2e-${Date.now()}`; + const managerUuid = `${testPrefix}-manager`; + const memberUuid = `${testPrefix}-member`; + const inviteeUuid = `${testPrefix}-invitee`; + const sourceOnlyManagerUuid = `${testPrefix}-source-only-manager`; + const concurrentManagerAUuid = `${testPrefix}-concurrent-manager-a`; + const concurrentManagerBUuid = `${testPrefix}-concurrent-manager-b`; + const collisionAdminAUuid = `${testPrefix}-collision-a`; + const collisionAdminBUuid = `${testPrefix}-collision-b`; + const listingSourceManagerUuid = `${testPrefix}-listing-source-manager`; + const listingMultiManagerUuid = `${testPrefix}-listing-multi-manager`; + let client: ReturnType; + let db: ReturnType>; + let repo: OrganizationsRepository; + let access: OrganizationAccessService; + let retrievalRepo: RetrievalRepository; + let app: NestFastifyApplication; + let sourceOrganizationId: string; + let targetOrganizationId: string; + let rootDocumentId: string; + let memberDocumentId: string; + let targetDocumentId: string; + + beforeAll(async () => { + const database = process.env.DB_NAME ?? ''; + if (!database.endsWith('_test')) { + throw new Error( + 'Organization database E2E requires DB_NAME ending in _test', + ); + } + client = postgres({ + host: process.env.DB_HOST ?? '127.0.0.1', + port: Number(process.env.DB_PORT ?? 5432), + database, + username: process.env.DB_USER ?? 'postgres', + password: process.env.DB_PASSWORD ?? 'postgres', + max: 5, + }); + db = drizzle(client, { schema }); + repo = new OrganizationsRepository(db as unknown as Database); + access = new OrganizationAccessService(repo); + retrievalRepo = new RetrievalRepository(db as unknown as Database); + + await db.insert(admins).values([ + { + idpUuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + name: 'Manager', + role: 'SUPER_ADMIN', + }, + { + idpUuid: memberUuid, + email: `${testPrefix}-member@example.com`, + name: 'Member', + }, + { + idpUuid: inviteeUuid, + email: `${testPrefix}-invitee@example.com`, + name: 'Invitee', + }, + ]); + + const [source, target] = await db + .insert(organizations) + .values([ + { + name: `${testPrefix} source`, + slug: `${testPrefix}-source`, + createdByIdpUuid: managerUuid, + }, + { + name: `${testPrefix} target`, + slug: `${testPrefix}-target`, + createdByIdpUuid: managerUuid, + }, + ]) + .returning(); + sourceOrganizationId = source.id; + targetOrganizationId = target.id; + + await db + .insert(organizationMemberships) + .values([ + { + organizationId: sourceOrganizationId, + inviteeEmail: `${testPrefix}-manager@example.com`, + memberIdpUuid: managerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: targetOrganizationId, + inviteeEmail: `${testPrefix}-manager@example.com`, + memberIdpUuid: managerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: sourceOrganizationId, + inviteeEmail: `${testPrefix}-member@example.com`, + memberIdpUuid: memberUuid, + role: 'MEMBER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + ]) + .returning(); + + await db.insert(admins).values([ + { + idpUuid: listingSourceManagerUuid, + email: `${testPrefix}-listing-source-manager@example.com`, + name: 'Listing Source Manager', + }, + { + idpUuid: listingMultiManagerUuid, + email: `${testPrefix}-listing-multi-manager@example.com`, + name: 'Listing Multi Manager', + }, + ]); + await db.insert(organizationMemberships).values([ + { + organizationId: sourceOrganizationId, + inviteeEmail: `${testPrefix}-listing-source-manager@example.com`, + memberIdpUuid: listingSourceManagerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: sourceOrganizationId, + inviteeEmail: `${testPrefix}-listing-multi-manager@example.com`, + memberIdpUuid: listingMultiManagerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: targetOrganizationId, + inviteeEmail: `${testPrefix}-listing-multi-manager@example.com`, + memberIdpUuid: listingMultiManagerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + ]); + + const listingDocuments = await db + .insert(documents) + .values([ + { + title: `${testPrefix} root upload`, + resourceName: `${testPrefix}-root-upload`, + gcsPdfPath: `gs://test/${testPrefix}-root-upload.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + }, + { + title: `${testPrefix} member upload`, + resourceName: `${testPrefix}-member-upload`, + gcsPdfPath: `gs://test/${testPrefix}-member-upload.pdf`, + status: 'ready', + uploadedByIdpUuid: memberUuid, + ownerOrganizationId: sourceOrganizationId, + }, + { + title: `${testPrefix} target upload`, + resourceName: `${testPrefix}-target-upload`, + gcsPdfPath: `gs://test/${testPrefix}-target-upload.pdf`, + status: 'ready', + uploadedByIdpUuid: listingMultiManagerUuid, + ownerOrganizationId: targetOrganizationId, + }, + ]) + .returning({ id: documents.id }); + [rootDocumentId, memberDocumentId, targetDocumentId] = listingDocuments.map( + (row) => row.id, + ); + await db.insert(documentOrganizationShares).values({ + documentId: rootDocumentId, + organizationId: targetOrganizationId, + sharedByIdpUuid: managerUuid, + }); + const documentsRepo = new DocumentsRepository(db as unknown as Database); + const uploadService = new UploadService( + documentsRepo, + {} as GcsStorageService, + repo, + access, + ); + const moduleRef = await Test.createTestingModule({ + controllers: [UploadController], + providers: [{ provide: UploadService, useValue: uploadService }], + }) + .overrideGuard(AdminJwtGuard) + .useClass(HeaderAdminGuard) + .compile(); + app = moduleRef.createNestApplication( + new FastifyAdapter(), + ); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + }); + + afterAll(async () => { + await app?.close(); + if (!db || !client) return; + const ownedOrganizations = await db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.createdByIdpUuid, managerUuid)); + const organizationIds = ownedOrganizations.map((row) => row.id); + if (organizationIds.length > 0) { + const ownedDocuments = await db + .select({ id: documents.id }) + .from(documents) + .where(inArray(documents.ownerOrganizationId, organizationIds)); + const documentIds = ownedDocuments.map((row) => row.id); + if (documentIds.length > 0) { + await db + .delete(documentOrganizationShares) + .where(inArray(documentOrganizationShares.documentId, documentIds)); + await db + .delete(documentOwnershipTransfers) + .where(inArray(documentOwnershipTransfers.documentId, documentIds)); + await db + .delete(documentChunks) + .where(inArray(documentChunks.documentId, documentIds)); + await db.delete(documents).where(inArray(documents.id, documentIds)); + } + await db + .delete(organizationMemberships) + .where( + inArray(organizationMemberships.organizationId, organizationIds), + ); + await db + .delete(organizations) + .where(inArray(organizations.id, organizationIds)); + } + await db + .delete(admins) + .where( + inArray(admins.idpUuid, [ + managerUuid, + memberUuid, + inviteeUuid, + sourceOnlyManagerUuid, + concurrentManagerAUuid, + concurrentManagerBUuid, + collisionAdminAUuid, + collisionAdminBUuid, + listingSourceManagerUuid, + listingMultiManagerUuid, + ]), + ); + await client.end(); + }); + + async function getUploadList(url: string, uuid: string, role = 'ADMIN') { + return app + .getHttpAdapter() + .getInstance() + .inject({ + method: 'GET', + url, + headers: { + 'x-admin-email': `${uuid}@example.com`, + 'x-admin-uuid': uuid, + 'x-admin-role': role, + }, + }); + } + + function responseIds(payload: string): string[] { + return JSON.parse(payload).map((row: { id: string }) => row.id); + } + + it('preserves the legacy upload list and exposes the separate manageable HTTP contract', async () => { + const memberOwn = await getUploadList('/api/v1/admin/upload', memberUuid); + expect(memberOwn.statusCode).toBe(200); + expect(responseIds(memberOwn.payload)).toEqual([memberDocumentId]); + + const superOwn = await getUploadList( + '/api/v1/admin/upload', + managerUuid, + 'SUPER_ADMIN', + ); + expect(superOwn.statusCode).toBe(200); + expect(responseIds(superOwn.payload)).toEqual([rootDocumentId]); + + const superManageable = await getUploadList( + '/api/v1/admin/upload/manageable?limit=100', + managerUuid, + 'SUPER_ADMIN', + ); + expect(superManageable.statusCode).toBe(200); + const superManageableIds = responseIds(superManageable.payload); + expect(superManageableIds).toEqual( + expect.arrayContaining([ + rootDocumentId, + memberDocumentId, + targetDocumentId, + ]), + ); + expect(new Set(superManageableIds).size).toBe(superManageableIds.length); + + const sourceManagerManageable = await getUploadList( + '/api/v1/admin/upload/manageable', + listingSourceManagerUuid, + ); + expect(sourceManagerManageable.statusCode).toBe(200); + expect(new Set(responseIds(sourceManagerManageable.payload))).toEqual( + new Set([rootDocumentId, memberDocumentId]), + ); + + const memberManageable = await getUploadList( + '/api/v1/admin/upload/manageable', + memberUuid, + ); + expect(memberManageable.statusCode).toBe(200); + expect(responseIds(memberManageable.payload)).toEqual([memberDocumentId]); + + const multiManagerManageable = await getUploadList( + '/api/v1/admin/upload/manageable', + listingMultiManagerUuid, + ); + expect(multiManagerManageable.statusCode).toBe(200); + const multiManagerIds = responseIds(multiManagerManageable.payload); + expect(new Set(multiManagerIds)).toEqual( + new Set([rootDocumentId, memberDocumentId, targetDocumentId]), + ); + expect(new Set(multiManagerIds).size).toBe(multiManagerIds.length); + + const staleSuperManageable = await getUploadList( + '/api/v1/admin/upload/manageable', + inviteeUuid, + 'SUPER_ADMIN', + ); + expect(staleSuperManageable.statusCode).toBe(200); + expect(responseIds(staleSuperManageable.payload)).toEqual([]); + }); + + it('supports one user as accepted MANAGER in multiple organizations', async () => { + const memberships = await db + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.memberIdpUuid, managerUuid), + eq(organizationMemberships.status, 'ACCEPTED'), + ), + ); + expect(memberships).toHaveLength(2); + }); + + it('creates an organization and creator MANAGER membership atomically', async () => { + const created = await repo.createOrganization( + `${testPrefix} atomic`, + `${testPrefix}-atomic`, + { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'SUPER_ADMIN', + }, + ); + await expect( + repo.findAcceptedMembership(created.id, managerUuid), + ).resolves.toMatchObject({ role: 'MANAGER', status: 'ACCEPTED' }); + }); + + it('keeps known-admin invitations PENDING until explicit acceptance', async () => { + const invitation = await repo.createInvitation({ + organizationId: targetOrganizationId, + inviteeEmail: `${testPrefix}-invitee@example.com`, + inviteeIdpUuid: inviteeUuid, + role: 'MEMBER', + invitedByIdpUuid: managerUuid, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }); + expect(invitation).toMatchObject({ + memberIdpUuid: inviteeUuid, + status: 'PENDING', + }); + expect( + await repo.findAcceptedMembership(targetOrganizationId, inviteeUuid), + ).toBeNull(); + await expect( + repo.listPendingInvitations(invitation.inviteeEmail, 'other-uuid'), + ).resolves.toHaveLength(0); + await expect( + repo.rejectInvitation( + invitation.id, + invitation.inviteeEmail, + 'other-uuid', + ), + ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + + await expect( + repo.acceptInvitation( + invitation.id, + invitation.inviteeEmail, + 'same-email-different-uuid', + ), + ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + + const accepted = await repo.acceptInvitation( + invitation.id, + invitation.inviteeEmail, + inviteeUuid, + ); + expect(accepted).toMatchObject({ + memberIdpUuid: inviteeUuid, + status: 'ACCEPTED', + acceptedAt: expect.any(Date), + }); + }); + + it('serializes concurrent mutations of different MANAGER rows', async () => { + await db.insert(admins).values([ + { + idpUuid: concurrentManagerAUuid, + email: `${testPrefix}-concurrent-a@example.com`, + name: 'Concurrent Manager A', + }, + { + idpUuid: concurrentManagerBUuid, + email: `${testPrefix}-concurrent-b@example.com`, + name: 'Concurrent Manager B', + }, + ]); + const [concurrentOrganization] = await db + .insert(organizations) + .values({ + name: `${testPrefix} concurrent managers`, + slug: `${testPrefix}-concurrent-managers`, + createdByIdpUuid: managerUuid, + }) + .returning(); + const managerRows = await db + .insert(organizationMemberships) + .values([ + { + organizationId: concurrentOrganization.id, + inviteeEmail: `${testPrefix}-concurrent-a@example.com`, + memberIdpUuid: concurrentManagerAUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: concurrentOrganization.id, + inviteeEmail: `${testPrefix}-concurrent-b@example.com`, + memberIdpUuid: concurrentManagerBUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + ]) + .returning(); + const root = { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'SUPER_ADMIN', + }; + const results = await Promise.all([ + repo.removeMembership(concurrentOrganization.id, managerRows[0].id, root), + repo.updateMembershipRole( + concurrentOrganization.id, + managerRows[1].id, + 'MEMBER', + root, + ), + ]); + expect( + results.filter((result) => result.kind === 'last_manager'), + ).toHaveLength(1); + const remainingManagers = await db + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, concurrentOrganization.id), + eq(organizationMemberships.status, 'ACCEPTED'), + eq(organizationMemberships.role, 'MANAGER'), + ), + ); + expect(remainingManagers).toHaveLength(1); + }); + + it('allows identified accepted users with colliding normalized emails but deduplicates pending invites', async () => { + const normalizedEmail = `${testPrefix}-collision@example.com`; + const [unboundInvitation] = await db + .insert(organizationMemberships) + .values({ + organizationId: targetOrganizationId, + inviteeEmail: normalizedEmail, + memberIdpUuid: null, + role: 'MEMBER', + status: 'PENDING', + invitedByIdpUuid: managerUuid, + }) + .returning(); + await db.insert(admins).values([ + { + idpUuid: collisionAdminAUuid, + email: `${testPrefix}-Collision@Example.com`, + name: 'Collision A', + role: 'SUPER_ADMIN', + }, + { + idpUuid: collisionAdminBUuid, + email: `${testPrefix}-collision@example.com`, + name: 'Collision B', + role: 'SUPER_ADMIN', + }, + ]); + const [collisionOrganization] = await db + .insert(organizations) + .values({ + name: `${testPrefix} collision`, + slug: `${testPrefix}-collision`, + createdByIdpUuid: managerUuid, + }) + .returning(); + await expect( + repo.listPendingInvitations(normalizedEmail, collisionAdminAUuid), + ).resolves.toHaveLength(0); + await expect( + repo.acceptInvitation( + unboundInvitation.id, + normalizedEmail, + collisionAdminAUuid, + ), + ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + await expect( + repo.rejectInvitation( + unboundInvitation.id, + normalizedEmail, + collisionAdminAUuid, + ), + ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + await expect( + db.insert(organizationMemberships).values([ + { + organizationId: collisionOrganization.id, + inviteeEmail: normalizedEmail, + memberIdpUuid: collisionAdminAUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + { + organizationId: collisionOrganization.id, + inviteeEmail: normalizedEmail, + memberIdpUuid: collisionAdminBUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }, + ]), + ).resolves.toBeDefined(); + await db.insert(organizationMemberships).values({ + organizationId: collisionOrganization.id, + inviteeEmail: `${testPrefix}-pending@example.com`, + role: 'MEMBER', + status: 'PENDING', + invitedByIdpUuid: managerUuid, + }); + await expect( + db.insert(organizationMemberships).values({ + organizationId: collisionOrganization.id, + inviteeEmail: `${testPrefix}-pending@example.com`, + role: 'MEMBER', + status: 'PENDING', + invitedByIdpUuid: managerUuid, + }), + ).rejects.toBeDefined(); + }); + + it("removes an uploader's rights immediately with their membership", async () => { + const [created] = await db + .insert(documents) + .values({ + title: `${testPrefix} member document`, + resourceName: `${testPrefix}-member-document`, + gcsPdfPath: `gs://test/${testPrefix}-member-document.pdf`, + status: 'ready', + uploadedByIdpUuid: memberUuid, + ownerOrganizationId: sourceOrganizationId, + }) + .returning(); + await expect( + access.requireDocumentManage(created.id, { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }), + ).resolves.toMatchObject({ canManage: true }); + await expect( + repo.updateDocumentExpiresAt({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toMatchObject({ kind: 'ok' }); + const reservation = await repo.createUploadingDocument({ + title: `${testPrefix} pending member upload`, + resourceName: `${testPrefix}-pending-member-upload`, + gcsPdfPath: `gs://test/${testPrefix}-pending-member-upload.pdf`, + ownerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }, + }); + await db.insert(organizationMemberships).values({ + organizationId: targetOrganizationId, + inviteeEmail: `${testPrefix}-member@example.com`, + memberIdpUuid: memberUuid, + role: 'MEMBER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }); + const [stillAuthorizedDocument] = await db + .insert(documents) + .values({ + title: `${testPrefix} older authorized member document`, + resourceName: `${testPrefix}-older-authorized-member-document`, + gcsPdfPath: `gs://test/${testPrefix}-older-authorized-member-document.pdf`, + status: 'ready', + uploadedByIdpUuid: memberUuid, + ownerOrganizationId: targetOrganizationId, + createdAt: new Date('2000-01-01T00:00:00.000Z'), + }) + .returning(); + + const [member] = await db + .select() + .from(organizationMemberships) + .where( + and( + eq(organizationMemberships.organizationId, sourceOrganizationId), + eq(organizationMemberships.memberIdpUuid, memberUuid), + ), + ); + await repo.removeMembership(sourceOrganizationId, member.id, { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }); + const ownAfterRemoval = await getUploadList( + '/api/v1/admin/upload?limit=1', + memberUuid, + ); + const manageableAfterRemoval = await getUploadList( + '/api/v1/admin/upload/manageable?limit=100', + memberUuid, + ); + expect(ownAfterRemoval.statusCode).toBe(200); + expect(manageableAfterRemoval.statusCode).toBe(200); + expect(responseIds(ownAfterRemoval.payload)).toEqual([ + stillAuthorizedDocument.id, + ]); + expect(responseIds(manageableAfterRemoval.payload)).toEqual([ + stillAuthorizedDocument.id, + ]); + await expect( + access.requireDocumentManage(created.id, { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }), + ).rejects.toBeInstanceOf(NotFoundException); + await expect( + repo.updateDocumentExpiresAt({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toEqual({ kind: 'forbidden' }); + await expect( + repo.finalizeUploadingDocument({ + documentId: reservation.id, + expectedOwnerOrganizationId: sourceOrganizationId, + actor: { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toEqual({ kind: 'forbidden' }); + await expect( + repo.createUploadingDocument({ + title: `${testPrefix} revoked upload`, + resourceName: `${testPrefix}-revoked-upload`, + gcsPdfPath: `gs://test/${testPrefix}-revoked-upload.pdf`, + ownerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: memberUuid, + email: `${testPrefix}-member@example.com`, + role: 'ADMIN', + }, + }), + ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + await expect( + repo.updateDocumentExpiresAt({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toMatchObject({ kind: 'ok' }); + }); + + it('gives shared-organization members view-only access', async () => { + const [created] = await db + .insert(documents) + .values({ + title: `${testPrefix} shared document`, + resourceName: `${testPrefix}-shared-document`, + gcsPdfPath: `gs://test/${testPrefix}-shared-document.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + }) + .returning(); + await repo.setShare({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + targetOrganizationId, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }); + + await expect( + access.requireDocumentView(created.id, { + uuid: inviteeUuid, + email: `${testPrefix}-invitee@example.com`, + role: 'ADMIN', + }), + ).resolves.toMatchObject({ relation: 'SHARED', canManage: false }); + await expect( + repo.updateDocumentExpiresAt({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: inviteeUuid, + email: `${testPrefix}-invitee@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toEqual({ kind: 'forbidden' }); + await expect( + repo.listOrganizationDocuments( + targetOrganizationId, + { + uuid: inviteeUuid, + email: `${testPrefix}-invitee@example.com`, + role: 'ADMIN', + }, + { + limit: 100, + offset: 0, + }, + ), + ).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ id: created.id })]), + ); + }); + + it('requires target MANAGER permission and returns manageable documents once', async () => { + await db.insert(admins).values({ + idpUuid: sourceOnlyManagerUuid, + email: `${testPrefix}-source-only@example.com`, + name: 'Source-only Manager', + }); + await db.insert(organizationMemberships).values({ + organizationId: sourceOrganizationId, + inviteeEmail: `${testPrefix}-source-only@example.com`, + memberIdpUuid: sourceOnlyManagerUuid, + role: 'MANAGER', + status: 'ACCEPTED', + invitedByIdpUuid: managerUuid, + acceptedAt: new Date(), + }); + const [created] = await db + .insert(documents) + .values({ + title: `${testPrefix} target permission`, + resourceName: `${testPrefix}-target-permission`, + gcsPdfPath: `gs://test/${testPrefix}-target-permission.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + }) + .returning(); + const sourceOnlyActor = { + uuid: sourceOnlyManagerUuid, + email: `${testPrefix}-source-only@example.com`, + role: 'ADMIN', + }; + await expect( + repo.transferDocument({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + targetOrganizationId, + actor: sourceOnlyActor, + }), + ).resolves.toEqual({ kind: 'forbidden' }); + + const manageable = await repo.listManageableDocuments( + { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + { limit: 100, offset: 0 }, + ); + expect(new Set(manageable.map((row) => row.id)).size).toBe( + manageable.length, + ); + }); + + it('transfers ownership, removes the target share, audits, and preserves chunks/state', async () => { + const [created] = await db + .insert(documents) + .values({ + title: `${testPrefix} transfer document`, + resourceName: `${testPrefix}-transfer-document`, + gcsPdfPath: `gs://test/${testPrefix}-transfer-document.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + }) + .returning(); + const [chunk] = await db + .insert(documentChunks) + .values({ + documentId: created.id, + path: `${testPrefix}/chunk`, + content: 'unchanged', + sortOrder: 0, + }) + .returning(); + await db.insert(documentOrganizationShares).values({ + documentId: created.id, + organizationId: targetOrganizationId, + sharedByIdpUuid: managerUuid, + }); + + const result = await repo.transferDocument({ + documentId: created.id, + expectedOwnerOrganizationId: sourceOrganizationId, + targetOrganizationId, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }); + expect(result).toMatchObject({ + kind: 'ok', + document: { + ownerOrganizationId: targetOrganizationId, + status: 'ready', + }, + }); + expect( + await db + .select() + .from(documentOrganizationShares) + .where(eq(documentOrganizationShares.documentId, created.id)), + ).toHaveLength(0); + expect( + await db + .select() + .from(documentOwnershipTransfers) + .where(eq(documentOwnershipTransfers.documentId, created.id)), + ).toHaveLength(1); + expect( + await db + .select() + .from(documentChunks) + .where(eq(documentChunks.documentId, created.id)), + ).toEqual([ + expect.objectContaining({ id: chunk.id, content: 'unchanged' }), + ]); + }); + + it('does not transfer an incomplete uploading reservation', async () => { + const reservation = await repo.createUploadingDocument({ + title: `${testPrefix} incomplete transfer`, + resourceName: `${testPrefix}-incomplete-transfer`, + gcsPdfPath: `gs://test/${testPrefix}-incomplete-transfer.pdf`, + ownerOrganizationId: sourceOrganizationId, + expiresAt: null, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }); + await expect( + repo.transferDocument({ + documentId: reservation.id, + expectedOwnerOrganizationId: sourceOrganizationId, + targetOrganizationId, + actor: { + uuid: managerUuid, + email: `${testPrefix}-manager@example.com`, + role: 'ADMIN', + }, + }), + ).resolves.toMatchObject({ kind: 'state_changed' }); + await expect( + db + .select() + .from(documentOwnershipTransfers) + .where(eq(documentOwnershipTransfers.documentId, reservation.id)), + ).resolves.toHaveLength(0); + }); + + it('keeps ready, active, non-expired chatbot retrieval global across owner organizations', async () => { + const rows = await db + .insert(documents) + .values([ + { + title: `${testPrefix} retrieval source`, + resourceName: `${testPrefix}-retrieval-source`, + gcsPdfPath: `gs://test/${testPrefix}-retrieval-source.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + expiresAt: new Date(Date.now() + 60_000), + }, + { + title: `${testPrefix} retrieval target`, + resourceName: `${testPrefix}-retrieval-target`, + gcsPdfPath: `gs://test/${testPrefix}-retrieval-target.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: targetOrganizationId, + expiresAt: null, + }, + { + title: `${testPrefix} retrieval inactive`, + resourceName: `${testPrefix}-retrieval-inactive`, + gcsPdfPath: `gs://test/${testPrefix}-retrieval-inactive.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + isActive: false, + }, + { + title: `${testPrefix} retrieval expired`, + resourceName: `${testPrefix}-retrieval-expired`, + gcsPdfPath: `gs://test/${testPrefix}-retrieval-expired.pdf`, + status: 'ready', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: targetOrganizationId, + expiresAt: new Date(Date.now() - 60_000), + }, + { + title: `${testPrefix} retrieval queued`, + resourceName: `${testPrefix}-retrieval-queued`, + gcsPdfPath: `gs://test/${testPrefix}-retrieval-queued.pdf`, + status: 'queued', + uploadedByIdpUuid: managerUuid, + ownerOrganizationId: sourceOrganizationId, + }, + ]) + .returning(); + await db.insert(documentChunks).values( + rows.map((row, index) => ({ + documentId: row.id, + path: `${testPrefix}/retrieval-${index}`, + description: `retrieval ${index}`, + content: `content ${index}`, + sortOrder: 0, + })), + ); + + const catalog = await retrievalRepo.listReadyWithChunks(); + const catalogIds = new Set(catalog.map((row) => row.id)); + expect(catalogIds.has(rows[0].id)).toBe(true); + expect(catalogIds.has(rows[1].id)).toBe(true); + expect(catalogIds.has(rows[2].id)).toBe(false); + expect(catalogIds.has(rows[3].id)).toBe(false); + expect(catalogIds.has(rows[4].id)).toBe(false); + + const paths = rows.map((_, index) => `${testPrefix}/retrieval-${index}`); + const contents = await retrievalRepo.findChunkContentsByPaths(paths); + expect(contents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: paths[0], content: 'content 0' }), + expect.objectContaining({ path: paths[1], content: 'content 1' }), + ]), + ); + const retrievedPaths = new Set(contents.map((row) => row.path)); + expect(retrievedPaths.has(paths[2])).toBe(false); + expect(retrievedPaths.has(paths[3])).toBe(false); + expect(retrievedPaths.has(paths[4])).toBe(false); + }); +}); diff --git a/test/organizations.e2e-spec.ts b/test/organizations.e2e-spec.ts new file mode 100644 index 0000000..18c2752 --- /dev/null +++ b/test/organizations.e2e-spec.ts @@ -0,0 +1,249 @@ +import { + BadRequestException, + ExecutionContext, + ValidationPipe, +} from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import { + FastifyAdapter, + NestFastifyApplication, +} from '@nestjs/platform-fastify'; +import type { FastifyRequest } from 'fastify'; +import { AdminContext } from '../src/auth/context/admin-context.entity'; +import { AdminJwtGuard } from '../src/auth/guards/admin-jwt.guard'; +import { SuperAdminGuard } from '../src/auth/guards/super-admin.guard'; +import { OrganizationAccessService } from '../src/organizations/organization-access.service'; +import { OrganizationsController } from '../src/organizations/organizations.controller'; +import { OrganizationsRepository } from '../src/organizations/organizations.repository'; +import { OrganizationsService } from '../src/organizations/organizations.service'; + +type AdminRequest = FastifyRequest & { user?: AdminContext }; + +class TestAdminGuard { + canActivate(context: ExecutionContext): boolean { + const request = context.switchToHttp().getRequest(); + request.user = new AdminContext( + String(request.headers['x-admin-email'] ?? 'admin@example.com'), + String(request.headers['x-admin-uuid'] ?? 'admin'), + 'Test Admin', + String(request.headers['x-admin-role'] ?? 'ADMIN'), + ); + return true; + } +} + +const ORG_ID = '550e8400-e29b-41d4-a716-446655440010'; +const MEMBERSHIP_ID = '550e8400-e29b-41d4-a716-446655440011'; + +function pendingMembership() { + return { + id: MEMBERSHIP_ID, + organizationId: ORG_ID, + inviteeEmail: 'invitee@example.com', + memberIdpUuid: 'invitee', + role: 'MEMBER' as const, + status: 'PENDING' as const, + invitedByIdpUuid: 'manager', + acceptedAt: null, + createdAt: new Date('2026-08-01T00:00:00.000Z'), + updatedAt: new Date('2026-08-01T00:00:00.000Z'), + }; +} + +describe('Organization administration API (e2e)', () => { + let app: NestFastifyApplication | undefined; + const repo = { + createOrganization: jest.fn(async () => ({ + id: ORG_ID, + name: 'Student Support', + slug: 'student-support', + isDefault: false, + createdByIdpUuid: 'root', + createdAt: new Date('2026-08-01T00:00:00.000Z'), + updatedAt: new Date('2026-08-01T00:00:00.000Z'), + })), + findAdminByEmail: jest.fn(async () => ({ idpUuid: 'invitee' })), + createInvitation: jest.fn(async () => pendingMembership()), + findMembershipById: jest.fn(async () => pendingMembership()), + acceptInvitation: jest.fn(async () => ({ + ...pendingMembership(), + status: 'ACCEPTED' as const, + acceptedAt: new Date('2026-08-01T00:01:00.000Z'), + })), + }; + const access = { + isSuperAdmin: jest.fn( + (admin: AdminContext) => admin.role === 'SUPER_ADMIN', + ), + requireOrganizationManager: jest.fn(async () => null), + }; + + beforeAll(async () => { + const moduleRef = await Test.createTestingModule({ + controllers: [OrganizationsController], + providers: [ + OrganizationsService, + SuperAdminGuard, + { provide: OrganizationsRepository, useValue: repo }, + { provide: OrganizationAccessService, useValue: access }, + ], + }) + .overrideGuard(AdminJwtGuard) + .useClass(TestAdminGuard) + .compile(); + + app = moduleRef.createNestApplication( + new FastifyAdapter(), + ); + app.useGlobalPipes( + new ValidationPipe({ + transform: true, + whitelist: true, + forbidNonWhitelisted: true, + exceptionFactory: (errors) => + new BadRequestException({ + statusCode: 400, + message: errors.flatMap((error) => + Object.values(error.constraints ?? {}), + ), + error: 'Bad Request', + }), + }), + ); + await app.init(); + await app.getHttpAdapter().getInstance().ready(); + }); + + afterEach(() => jest.clearAllMocks()); + afterAll(async () => app?.close()); + + it('rejects organization creation by a non-SUPER_ADMIN', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: '/api/v1/admin/organizations', + headers: { 'x-admin-role': 'ADMIN' }, + payload: { name: 'Student Support', slug: 'student-support' }, + }); + expect(response.statusCode).toBe(403); + expect(repo.createOrganization).not.toHaveBeenCalled(); + }); + + it('creates an organization through the atomic service operation for SUPER_ADMIN', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: '/api/v1/admin/organizations', + headers: { + 'x-admin-role': 'SUPER_ADMIN', + 'x-admin-uuid': 'root', + }, + payload: { name: 'Student Support', slug: 'student-support' }, + }); + expect(response.statusCode).toBe(201); + expect(JSON.parse(response.payload)).toEqual({ + id: ORG_ID, + name: 'Student Support', + slug: 'student-support', + isDefault: false, + effectiveRole: 'SUPER_ADMIN', + createdAt: '2026-08-01T00:00:00.000Z', + }); + expect(repo.createOrganization).toHaveBeenCalledTimes(1); + }); + + it('keeps a known admin invitation PENDING', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: `/api/v1/admin/organizations/${ORG_ID}/members`, + headers: { + 'x-admin-email': 'manager@example.com', + 'x-admin-uuid': 'manager', + }, + payload: { inviteeEmail: ' Invitee@Example.com ', role: 'MEMBER' }, + }); + expect(response.statusCode).toBe(201); + expect(JSON.parse(response.payload)).toEqual({ + id: MEMBERSHIP_ID, + organizationId: ORG_ID, + inviteeEmail: 'invitee@example.com', + memberIdpUuid: 'invitee', + role: 'MEMBER', + status: 'PENDING', + memberName: null, + acceptedAt: null, + createdAt: '2026-08-01T00:00:00.000Z', + }); + }); + + it('forbids a different email from accepting the invitation', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: `/api/v1/admin/organization-invitations/${MEMBERSHIP_ID}/accept`, + headers: { + 'x-admin-email': 'other@example.com', + 'x-admin-uuid': 'other', + }, + }); + expect(response.statusCode).toBe(403); + expect(repo.acceptInvitation).not.toHaveBeenCalled(); + }); + + it('accepts explicitly for the invited normalized email', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: `/api/v1/admin/organization-invitations/${MEMBERSHIP_ID}/accept`, + headers: { + 'x-admin-email': 'INVITEE@example.com', + 'x-admin-uuid': 'invitee', + }, + }); + expect(response.statusCode).toBe(201); + expect(JSON.parse(response.payload).status).toBe('ACCEPTED'); + expect(repo.acceptInvitation).toHaveBeenCalledWith( + MEMBERSHIP_ID, + 'invitee@example.com', + 'invitee', + ); + }); + + it('rejects invalid organization slugs through the global validation contract', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: '/api/v1/admin/organizations', + headers: { 'x-admin-role': 'SUPER_ADMIN' }, + payload: { name: 'Invalid', slug: 'Not Valid' }, + }); + expect(response.statusCode).toBe(400); + }); + + it('rejects an organization name containing only whitespace', async () => { + const response = await app! + .getHttpAdapter() + .getInstance() + .inject({ + method: 'POST', + url: '/api/v1/admin/organizations', + headers: { 'x-admin-role': 'SUPER_ADMIN' }, + payload: { name: ' ', slug: 'blank-name' }, + }); + expect(response.statusCode).toBe(400); + expect(repo.createOrganization).not.toHaveBeenCalled(); + }); +}); From 70b6e7f5236289b1be0ddb80b9f7726adc69b898 Mon Sep 17 00:00:00 2001 From: yejuneric Date: Sun, 2 Aug 2026 01:11:45 +0900 Subject: [PATCH 37/40] docs: align upload detail Swagger response --- src/upload/upload.controller.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/upload/upload.controller.ts b/src/upload/upload.controller.ts index 728cdce..a5a09dc 100644 --- a/src/upload/upload.controller.ts +++ b/src/upload/upload.controller.ts @@ -158,7 +158,6 @@ export class UploadController { type: DocumentListItemDto, }) @ApiResponse({ status: 401, description: '인증 실패' }) - @ApiResponse({ status: 403, description: '문서 조회 권한 없음' }) @ApiResponse({ status: 404, description: '문서 없음' }) async getOne( @CurrentAdmin() admin: AdminContext, From 79024c11b4405bdc3821d27577cba4576a17f49c Mon Sep 17 00:00:00 2001 From: yejuneric Date: Sun, 2 Aug 2026 02:21:54 +0900 Subject: [PATCH 38/40] fix: stabilize migration locking and document ordering --- src/db/index.ts | 61 +++++++++++-- src/db/migration-lock.spec.ts | 87 +++++++++++++++++-- src/organizations/organizations.repository.ts | 6 +- test/organization-database.e2e-spec.ts | 1 + 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/src/db/index.ts b/src/db/index.ts index ae8d17d..351eb4c 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -19,22 +19,69 @@ export interface DatabaseConnectionParams { const MIGRATION_LOCK_SQL = 'SELECT pg_advisory_lock(1128352846, 1667785076)'; const MIGRATION_UNLOCK_SQL = 'SELECT pg_advisory_unlock(1128352846, 1667785076)'; +const MIGRATION_LOCK_TIMEOUT_MS = 30_000; +const MIGRATION_LOCK_TIMEOUT_SQL = `SET lock_timeout = '${MIGRATION_LOCK_TIMEOUT_MS}ms'`; +const MIGRATION_LOCK_TIMEOUT_RESET_SQL = 'SET lock_timeout = DEFAULT'; export interface MigrationAdvisoryLockClient { unsafe(query: string): PromiseLike; } +export interface ReservedMigrationConnection extends MigrationAdvisoryLockClient { + release(): void; +} + +export interface ReservableMigrationClient< + TConnection extends ReservedMigrationConnection = ReservedMigrationConnection, +> { + reserve(): Promise; + end(): Promise; +} + +export async function withReservedMigrationConnection< + TConnection extends ReservedMigrationConnection, + T, +>( + client: ReservableMigrationClient, + operation: (connection: TConnection) => Promise, +): Promise { + let connection: TConnection | undefined; + try { + connection = await client.reserve(); + return await operation(connection); + } finally { + try { + connection?.release(); + } finally { + await client.end(); + } + } +} + /** Serialize startup migrators across every application instance. */ export async function withMigrationAdvisoryLock( client: MigrationAdvisoryLockClient, operation: () => Promise, ): Promise { + await client.unsafe(MIGRATION_LOCK_TIMEOUT_SQL); await client.unsafe(MIGRATION_LOCK_SQL); + let operationFailed = false; + let operationError: unknown; + let result: T | undefined; + try { + await client.unsafe(MIGRATION_LOCK_TIMEOUT_RESET_SQL); + result = await operation(); + } catch (error) { + operationFailed = true; + operationError = error; + } try { - return await operation(); - } finally { await client.unsafe(MIGRATION_UNLOCK_SQL); + } catch (unlockError) { + if (!operationFailed) throw unlockError; } + if (operationFailed) throw operationError; + return result as T; } // Database connection factory with SSL options @@ -71,8 +118,6 @@ export const runMigrations = async (params: DatabaseConnectionParams) => { max: 1, ssl: params.sslEnabled ? { rejectUnauthorized: false } : false, }); - const db = drizzle(migrationClient); - // Determine migrations folder path based on environment const migrationsFolder = process.env.NODE_ENV === 'production' @@ -80,15 +125,15 @@ export const runMigrations = async (params: DatabaseConnectionParams) => { : './drizzle'; // Local development path try { - await withMigrationAdvisoryLock(migrationClient, () => - migrate(db, { migrationsFolder }), + await withReservedMigrationConnection(migrationClient, (connection) => + withMigrationAdvisoryLock(connection, () => + migrate(drizzle(connection), { migrationsFolder }), + ), ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); console.error('Migration failed:', errorMessage); throw error; - } finally { - await migrationClient.end(); } }; diff --git a/src/db/migration-lock.spec.ts b/src/db/migration-lock.spec.ts index 2bda840..0f120d6 100644 --- a/src/db/migration-lock.spec.ts +++ b/src/db/migration-lock.spec.ts @@ -1,7 +1,10 @@ import { describe, expect, it, jest } from '@jest/globals'; import { type MigrationAdvisoryLockClient, + type ReservableMigrationClient, + type ReservedMigrationConnection, withMigrationAdvisoryLock, + withReservedMigrationConnection, } from './index'; describe('withMigrationAdvisoryLock', () => { @@ -9,7 +12,9 @@ describe('withMigrationAdvisoryLock', () => { const events: string[] = []; const client: MigrationAdvisoryLockClient = { unsafe: jest.fn(async (query: string) => { - events.push(query.includes('unlock') ? 'unlock' : 'lock'); + if (query.includes('lock_timeout = DEFAULT')) events.push('reset'); + else if (query.includes('lock_timeout')) events.push('timeout'); + else events.push(query.includes('unlock') ? 'unlock' : 'lock'); }), }; @@ -17,7 +22,7 @@ describe('withMigrationAdvisoryLock', () => { events.push('migrate'); }); - expect(events).toEqual(['lock', 'migrate', 'unlock']); + expect(events).toEqual(['timeout', 'lock', 'reset', 'migrate', 'unlock']); }); it('releases the session lock when migration fails', async () => { @@ -33,8 +38,80 @@ describe('withMigrationAdvisoryLock', () => { throw new Error('migration failed'); }), ).rejects.toThrow('migration failed'); - expect(queries).toHaveLength(2); - expect(queries[0]).toContain('pg_advisory_lock'); - expect(queries[1]).toContain('pg_advisory_unlock'); + expect(queries.some((query) => query.includes('pg_advisory_lock'))).toBe( + true, + ); + expect(queries.at(-1)).toContain('pg_advisory_unlock'); + }); + + it('does not replace a migration failure with an unlock failure', async () => { + const migrationError = new Error('migration failed'); + const client: MigrationAdvisoryLockClient = { + unsafe: jest.fn(async (query: string) => { + if (query.includes('pg_advisory_unlock')) { + throw new Error('unlock failed'); + } + }), + }; + + await expect( + withMigrationAdvisoryLock(client, async () => { + throw migrationError; + }), + ).rejects.toBe(migrationError); + }); + + it.each([false, true])( + 'always releases the reserved connection when operation failure is %s', + async (shouldFail) => { + const release = jest.fn(); + const connection: ReservedMigrationConnection = { + unsafe: jest.fn(async () => undefined), + release, + }; + const client: ReservableMigrationClient = { + reserve: jest.fn(async () => connection), + end: jest.fn(async () => undefined), + }; + const operation = withReservedMigrationConnection(client, async () => { + if (shouldFail) throw new Error('migration failed'); + return 'migrated'; + }); + + if (shouldFail) + await expect(operation).rejects.toThrow('migration failed'); + else await expect(operation).resolves.toBe('migrated'); + expect(release).toHaveBeenCalledTimes(1); + expect(client.end).toHaveBeenCalledTimes(1); + }, + ); + + it('closes the migration client when reserving a connection fails', async () => { + const client: ReservableMigrationClient = { + reserve: jest.fn(async () => { + throw new Error('reserve failed'); + }), + end: jest.fn(async () => undefined), + }; + + await expect( + withReservedMigrationConnection(client, async () => undefined), + ).rejects.toThrow('reserve failed'); + expect(client.end).toHaveBeenCalledTimes(1); + }); + + it('sets a finite wait timeout before acquiring the advisory lock', async () => { + const queries: string[] = []; + const client: MigrationAdvisoryLockClient = { + unsafe: jest.fn(async (query: string) => { + queries.push(query); + }), + }; + + await withMigrationAdvisoryLock(client, async () => undefined); + + expect(queries[0]).toContain('lock_timeout'); + expect(queries[0]).toContain('30000ms'); + expect(queries[1]).toContain('pg_advisory_lock'); }); }); diff --git a/src/organizations/organizations.repository.ts b/src/organizations/organizations.repository.ts index 396d587..ff5fc87 100644 --- a/src/organizations/organizations.repository.ts +++ b/src/organizations/organizations.repository.ts @@ -642,7 +642,7 @@ export class OrganizationsRepository { ), ), ) - .orderBy(desc(documents.createdAt)) + .orderBy(desc(documents.createdAt), desc(documents.id)) .limit(options.limit) .offset(options.offset); if (ids.length === 0) return []; @@ -655,7 +655,7 @@ export class OrganizationsRepository { ids.map((row) => row.id), ), ) - .orderBy(desc(documents.createdAt)); + .orderBy(desc(documents.createdAt), desc(documents.id)); } async listManageableDocuments( @@ -691,7 +691,7 @@ export class OrganizationsRepository { ), ), ) - .orderBy(desc(documents.createdAt)) + .orderBy(desc(documents.createdAt), desc(documents.id)) .limit(options.limit) .offset(options.offset) .then((rows) => rows.map((row) => row.document)); diff --git a/test/organization-database.e2e-spec.ts b/test/organization-database.e2e-spec.ts index 2fecca3..036fcb8 100644 --- a/test/organization-database.e2e-spec.ts +++ b/test/organization-database.e2e-spec.ts @@ -930,6 +930,7 @@ describeDatabase('Organization database invariants (e2e)', () => { .values({ documentId: created.id, path: `${testPrefix}/chunk`, + description: '', content: 'unchanged', sortOrder: 0, }) From 03a0b403567a68a94caf9ac3518ac11f24a2b6aa Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Tue, 4 Aug 2026 16:38:20 -0700 Subject: [PATCH 39/40] fix: improve migration locking and ensure client connection closure --- src/db/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/db/index.ts b/src/db/index.ts index 351eb4c..e8a73a7 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -125,15 +125,17 @@ export const runMigrations = async (params: DatabaseConnectionParams) => { : './drizzle'; // Local development path try { - await withReservedMigrationConnection(migrationClient, (connection) => - withMigrationAdvisoryLock(connection, () => - migrate(drizzle(connection), { migrationsFolder }), - ), + // max:1 keeps advisory lock + migrate on the same session. Do not pass + // reserve() to drizzle — that wrapper lacks options.parsers. + await withMigrationAdvisoryLock(migrationClient, () => + migrate(drizzle(migrationClient), { migrationsFolder }), ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); console.error('Migration failed:', errorMessage); throw error; + } finally { + await migrationClient.end(); } }; From 10a373fdc5301871184480aee8be1f627845e5e2 Mon Sep 17 00:00:00 2001 From: ikjunchoi Date: Fri, 7 Aug 2026 00:26:42 -0700 Subject: [PATCH 40/40] feat: add unique index for normalized email and improve admin identity handling --- drizzle/0015_marvelous_lady_mastermind.sql | 15 + drizzle/meta/0015_snapshot.json | 2024 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/db/schema.ts | 4 + src/organizations/organizations.repository.ts | 8 +- test/organization-database.e2e-spec.ts | 61 +- 6 files changed, 2087 insertions(+), 32 deletions(-) create mode 100644 drizzle/0015_marvelous_lady_mastermind.sql create mode 100644 drizzle/meta/0015_snapshot.json diff --git a/drizzle/0015_marvelous_lady_mastermind.sql b/drizzle/0015_marvelous_lady_mastermind.sql new file mode 100644 index 0000000..10f6471 --- /dev/null +++ b/drizzle/0015_marvelous_lady_mastermind.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM ( + SELECT lower(trim("email")) AS normalized_email + FROM "admins" + GROUP BY 1 + HAVING count(*) > 1 + ) duplicates + ) THEN + RAISE EXCEPTION 'Cannot create admins_normalized_email_unique: duplicate lower(trim(email)) values exist in admins'; + END IF; +END $$;--> statement-breakpoint +CREATE UNIQUE INDEX "admins_normalized_email_unique" ON "admins" USING btree (lower(trim("email"))); diff --git a/drizzle/meta/0015_snapshot.json b/drizzle/meta/0015_snapshot.json new file mode 100644 index 0000000..f704cca --- /dev/null +++ b/drizzle/meta/0015_snapshot.json @@ -0,0 +1,2024 @@ +{ + "id": "4b301ad5-44be-495a-8612-96a601a9456c", + "prevId": "84c647e8-5db0-45bb-9c87-b53110fdb284", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admins": { + "name": "admins", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "idp_uuid": { + "name": "idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "admin_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ADMIN'" + }, + "last_login_at": { + "name": "last_login_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admins_idp_uuid_idx": { + "name": "admins_idp_uuid_idx", + "columns": [ + { + "expression": "idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_email_idx": { + "name": "admins_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admins_normalized_email_unique": { + "name": "admins_normalized_email_unique", + "columns": [ + { + "expression": "lower(trim(\"email\"))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admins_idp_uuid_unique": { + "name": "admins_idp_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "idp_uuid" + ] + }, + "admins_email_unique": { + "name": "admins_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_chunks": { + "name": "document_chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_chunks_document_id_idx": { + "name": "document_chunks_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_sort_idx": { + "name": "document_chunks_document_sort_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_path_idx": { + "name": "document_chunks_path_idx", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_chunks_document_id_path_unique": { + "name": "document_chunks_document_id_path_unique", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_chunks_document_id_documents_id_fk": { + "name": "document_chunks_document_id_documents_id_fk", + "tableFrom": "document_chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_organization_shares": { + "name": "document_organization_shares", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "shared_by_idp_uuid": { + "name": "shared_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_organization_shares_document_id_idx": { + "name": "document_organization_shares_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_organization_shares_organization_id_idx": { + "name": "document_organization_shares_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_organization_shares_document_id_organization_id_unique": { + "name": "document_organization_shares_document_id_organization_id_unique", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_organization_shares_document_id_documents_id_fk": { + "name": "document_organization_shares_document_id_documents_id_fk", + "tableFrom": "document_organization_shares", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_organization_shares_organization_id_organizations_id_fk": { + "name": "document_organization_shares_organization_id_organizations_id_fk", + "tableFrom": "document_organization_shares", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_ownership_transfers": { + "name": "document_ownership_transfers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "document_id": { + "name": "document_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_organization_id": { + "name": "source_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "target_organization_id": { + "name": "target_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "actor_idp_uuid": { + "name": "actor_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "transferred_at": { + "name": "transferred_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_ownership_transfers_document_id_idx": { + "name": "document_ownership_transfers_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_ownership_transfers_source_organization_id_idx": { + "name": "document_ownership_transfers_source_organization_id_idx", + "columns": [ + { + "expression": "source_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_ownership_transfers_target_organization_id_idx": { + "name": "document_ownership_transfers_target_organization_id_idx", + "columns": [ + { + "expression": "target_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_ownership_transfers_document_id_documents_id_fk": { + "name": "document_ownership_transfers_document_id_documents_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "document_ownership_transfers_source_organization_id_organizations_id_fk": { + "name": "document_ownership_transfers_source_organization_id_organizations_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "organizations", + "columnsFrom": [ + "source_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "document_ownership_transfers_target_organization_id_organizations_id_fk": { + "name": "document_ownership_transfers_target_organization_id_organizations_id_fk", + "tableFrom": "document_ownership_transfers", + "tableTo": "organizations", + "columnsFrom": [ + "target_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "resource_name": { + "name": "resource_name", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gcs_pdf_path": { + "name": "gcs_pdf_path", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "document_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_token": { + "name": "processing_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner_organization_id": { + "name": "owner_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_reprocessed_at": { + "name": "last_reprocessed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "documents_resource_name_active_unique": { + "name": "documents_resource_name_active_unique", + "columns": [ + { + "expression": "resource_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"documents\".\"is_active\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_status_idx": { + "name": "documents_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_uploaded_by_idp_uuid_idx": { + "name": "documents_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_owner_organization_id_idx": { + "name": "documents_owner_organization_id_idx", + "columns": [ + { + "expression": "owner_organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_is_active_idx": { + "name": "documents_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_created_at_idx": { + "name": "documents_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "documents_expires_at_idx": { + "name": "documents_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_owner_organization_id_organizations_id_fk": { + "name": "documents_owner_organization_id_organizations_id_fk", + "tableFrom": "documents", + "tableTo": "organizations", + "columnsFrom": [ + "owner_organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_feedbacks": { + "name": "message_feedbacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "message_id": { + "name": "message_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "message_feedback_rating", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "message_feedbacks_message_id_unique": { + "name": "message_feedbacks_message_id_unique", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "message_feedbacks_rating_created_at_idx": { + "name": "message_feedbacks_rating_created_at_idx", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_feedbacks_message_id_messages_id_fk": { + "name": "message_feedbacks_message_id_messages_id_fk", + "tableFrom": "message_feedbacks", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "message_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "messages_session_id_idx": { + "name": "messages_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_created_at_idx": { + "name": "messages_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "messages_session_created_idx": { + "name": "messages_session_created_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_session_id_sessions_id_fk": { + "name": "messages_session_id_sessions_id_fk", + "tableFrom": "messages", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_memberships": { + "name": "organization_memberships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "member_idp_uuid": { + "name": "member_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "organization_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'MEMBER'" + }, + "status": { + "name": "status", + "type": "organization_membership_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "accepted_at": { + "name": "accepted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_memberships_organization_id_idx": { + "name": "organization_memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_member_idp_uuid_status_idx": { + "name": "organization_memberships_member_idp_uuid_status_idx", + "columns": [ + { + "expression": "member_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_invitee_email_status_idx": { + "name": "organization_memberships_invitee_email_status_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_organization_id_invitee_email_unique": { + "name": "organization_memberships_organization_id_invitee_email_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organization_memberships\".\"status\" = 'PENDING'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_memberships_organization_id_member_idp_uuid_unique": { + "name": "organization_memberships_organization_id_member_idp_uuid_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "member_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organization_memberships\".\"member_idp_uuid\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_memberships_organization_id_organizations_id_fk": { + "name": "organization_memberships_organization_id_organizations_id_fk", + "tableFrom": "organization_memberships", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_single_default_unique": { + "name": "organizations_single_default_unique", + "columns": [ + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"organizations\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "organizations_created_by_idp_uuid_idx": { + "name": "organizations_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page_url": { + "name": "page_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sessions_session_token_idx": { + "name": "sessions_session_token_idx", + "columns": [ + { + "expression": "session_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_widget_key_id_idx": { + "name": "sessions_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sessions_widget_key_id_widget_keys_id_fk": { + "name": "sessions_widget_key_id_widget_keys_id_fk", + "tableFrom": "sessions", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_session_token_unique": { + "name": "sessions_session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "session_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.uploaded_resources": { + "name": "uploaded_resources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "uploaded_by_idp_uuid": { + "name": "uploaded_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uploaded_resources_uploaded_by_idp_uuid_idx": { + "name": "uploaded_resources_uploaded_by_idp_uuid_idx", + "columns": [ + { + "expression": "uploaded_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_is_active_idx": { + "name": "uploaded_resources_is_active_idx", + "columns": [ + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uploaded_resources_created_at_idx": { + "name": "uploaded_resources_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_daily": { + "name": "usage_daily", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_answers": { + "name": "total_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bad_answers": { + "name": "bad_answers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "usage_daily_widget_key_date_idx": { + "name": "usage_daily_widget_key_date_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_daily_widget_key_id_date_domain_unique": { + "name": "usage_daily_widget_key_id_date_domain_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_daily_widget_key_id_widget_keys_id_fk": { + "name": "usage_daily_widget_key_id_widget_keys_id_fk", + "tableFrom": "usage_daily", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_daily_total_answers_non_negative": { + "name": "usage_daily_total_answers_non_negative", + "value": "\"usage_daily\".\"total_answers\" >= 0" + }, + "usage_daily_bad_answers_non_negative": { + "name": "usage_daily_bad_answers_non_negative", + "value": "\"usage_daily\".\"bad_answers\" >= 0" + }, + "usage_daily_bad_answers_lte_total": { + "name": "usage_daily_bad_answers_lte_total", + "value": "\"usage_daily\".\"bad_answers\" <= \"usage_daily\".\"total_answers\"" + } + }, + "isRLSEnabled": false + }, + "public.widget_key_collaborators": { + "name": "widget_key_collaborators", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "widget_key_id": { + "name": "widget_key_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invitee_email": { + "name": "invitee_email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "invitee_idp_uuid": { + "name": "invitee_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "collaborator_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'VIEWER'" + }, + "status": { + "name": "status", + "type": "collaborator_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'PENDING'" + }, + "invited_by_idp_uuid": { + "name": "invited_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_key_collaborators_widget_key_id_idx": { + "name": "widget_key_collaborators_widget_key_id_idx", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_email_idx": { + "name": "widget_key_collaborators_invitee_email_idx", + "columns": [ + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_invitee_idp_uuid_idx": { + "name": "widget_key_collaborators_invitee_idp_uuid_idx", + "columns": [ + { + "expression": "invitee_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_key_collaborators_widget_key_id_invitee_email_unique": { + "name": "widget_key_collaborators_widget_key_id_invitee_email_unique", + "columns": [ + { + "expression": "widget_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invitee_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "widget_key_collaborators_widget_key_id_widget_keys_id_fk": { + "name": "widget_key_collaborators_widget_key_id_widget_keys_id_fk", + "tableFrom": "widget_key_collaborators", + "tableTo": "widget_keys", + "columnsFrom": [ + "widget_key_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.widget_keys": { + "name": "widget_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "secret_key": { + "name": "secret_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "widget_key_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'ACTIVE'" + }, + "allowed_domains": { + "name": "allowed_domains", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "allowed_app_ids": { + "name": "allowed_app_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by_idp_uuid": { + "name": "created_by_idp_uuid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "widget_keys_secret_key_idx": { + "name": "widget_keys_secret_key_idx", + "columns": [ + { + "expression": "secret_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_status_idx": { + "name": "widget_keys_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "widget_keys_created_by_idp_uuid_idx": { + "name": "widget_keys_created_by_idp_uuid_idx", + "columns": [ + { + "expression": "created_by_idp_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "widget_keys_secret_key_unique": { + "name": "widget_keys_secret_key_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.admin_role": { + "name": "admin_role", + "schema": "public", + "values": [ + "SUPER_ADMIN", + "ADMIN" + ] + }, + "public.collaborator_role": { + "name": "collaborator_role", + "schema": "public", + "values": [ + "VIEWER" + ] + }, + "public.collaborator_status": { + "name": "collaborator_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.document_status": { + "name": "document_status", + "schema": "public", + "values": [ + "uploading", + "queued", + "processing", + "ready", + "failed" + ] + }, + "public.message_feedback_rating": { + "name": "message_feedback_rating", + "schema": "public", + "values": [ + "GOOD", + "BAD" + ] + }, + "public.message_role": { + "name": "message_role", + "schema": "public", + "values": [ + "user", + "assistant" + ] + }, + "public.organization_membership_status": { + "name": "organization_membership_status", + "schema": "public", + "values": [ + "PENDING", + "ACCEPTED" + ] + }, + "public.organization_role": { + "name": "organization_role", + "schema": "public", + "values": [ + "MANAGER", + "MEMBER" + ] + }, + "public.widget_key_status": { + "name": "widget_key_status", + "schema": "public", + "values": [ + "ACTIVE", + "REVOKED" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 6cd5ec7..04740d4 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1785573225458, "tag": "0014_unique_dracula", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1786086938468, + "tag": "0015_marvelous_lady_mastermind", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index 68355ea..76b2f42 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -76,6 +76,10 @@ export const admins = pgTable( (table) => ({ idpUuidIdx: index('admins_idp_uuid_idx').on(table.idpUuid), emailIdx: index('admins_email_idx').on(table.email), + normalizedEmailUnique: uniqueIndex('admins_normalized_email_unique').using( + 'btree', + sql`lower(trim(${table.email}))`, + ), }), ); diff --git a/src/organizations/organizations.repository.ts b/src/organizations/organizations.repository.ts index ff5fc87..382f802 100644 --- a/src/organizations/organizations.repository.ts +++ b/src/organizations/organizations.repository.ts @@ -1051,14 +1051,14 @@ export class OrganizationsRepository { normalizedEmail: string, actorIdpUuid: string, ): Promise { - // Prevent a case/space-variant admin identity from being inserted between - // the ambiguity check and binding an invitation to the caller. - await tx.execute(sql`LOCK TABLE "admins" IN SHARE MODE`); + // Unique index on lower(trim(email)) already prevents case/space-variant + // duplicates; only lock matching rows instead of the whole admins table. const matches = await tx .select({ idpUuid: admins.idpUuid }) .from(admins) .where(sql`lower(trim(${admins.email})) = ${normalizedEmail}`) - .limit(2); + .limit(2) + .for('share'); return matches.length === 1 && matches[0]?.idpUuid === actorIdpUuid; } diff --git a/test/organization-database.e2e-spec.ts b/test/organization-database.e2e-spec.ts index 036fcb8..4b62193 100644 --- a/test/organization-database.e2e-spec.ts +++ b/test/organization-database.e2e-spec.ts @@ -555,7 +555,7 @@ describeDatabase('Organization database invariants (e2e)', () => { expect(remainingManagers).toHaveLength(1); }); - it('allows identified accepted users with colliding normalized emails but deduplicates pending invites', async () => { + it('enforces sole normalized admin identity and still deduplicates pending invites', async () => { const normalizedEmail = `${testPrefix}-collision@example.com`; const [unboundInvitation] = await db .insert(organizationMemberships) @@ -568,45 +568,50 @@ describeDatabase('Organization database invariants (e2e)', () => { invitedByIdpUuid: managerUuid, }) .returning(); - await db.insert(admins).values([ - { - idpUuid: collisionAdminAUuid, - email: `${testPrefix}-Collision@Example.com`, - name: 'Collision A', - role: 'SUPER_ADMIN', - }, - { + await db.insert(admins).values({ + idpUuid: collisionAdminAUuid, + email: `${testPrefix}-Collision@Example.com`, + name: 'Collision A', + role: 'SUPER_ADMIN', + }); + await expect( + db.insert(admins).values({ idpUuid: collisionAdminBUuid, email: `${testPrefix}-collision@example.com`, name: 'Collision B', role: 'SUPER_ADMIN', - }, - ]); - const [collisionOrganization] = await db - .insert(organizations) - .values({ - name: `${testPrefix} collision`, - slug: `${testPrefix}-collision`, - createdByIdpUuid: managerUuid, - }) - .returning(); + }), + ).rejects.toBeDefined(); + await db.insert(admins).values({ + idpUuid: collisionAdminBUuid, + email: `${testPrefix}-collision-b@example.com`, + name: 'Collision B', + role: 'SUPER_ADMIN', + }); + await expect( repo.listPendingInvitations(normalizedEmail, collisionAdminAUuid), - ).resolves.toHaveLength(0); + ).resolves.toHaveLength(1); await expect( repo.acceptInvitation( unboundInvitation.id, normalizedEmail, collisionAdminAUuid, ), - ).rejects.toBeInstanceOf(RepositoryAuthorizationError); - await expect( - repo.rejectInvitation( - unboundInvitation.id, - normalizedEmail, - collisionAdminAUuid, - ), - ).rejects.toBeInstanceOf(RepositoryAuthorizationError); + ).resolves.toMatchObject({ + id: unboundInvitation.id, + status: 'ACCEPTED', + memberIdpUuid: collisionAdminAUuid, + }); + + const [collisionOrganization] = await db + .insert(organizations) + .values({ + name: `${testPrefix} collision`, + slug: `${testPrefix}-collision`, + createdByIdpUuid: managerUuid, + }) + .returning(); await expect( db.insert(organizationMemberships).values([ {