From c4a264d42a47d5eac2579a1d3085f7f73a5ea698 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Tue, 12 May 2026 11:51:15 +0530 Subject: [PATCH] Kakfa Related Changes in Payload --- KAFKA_CONSUMER_MAPPING.md | 129 ++++++ KAFKA_EVENTS.md | 377 ++++++++++++++++++ .../response/services/response.service.ts | 7 + survey_template.csv | 7 + 4 files changed, 520 insertions(+) create mode 100644 KAFKA_CONSUMER_MAPPING.md create mode 100644 KAFKA_EVENTS.md create mode 100644 survey_template.csv diff --git a/KAFKA_CONSUMER_MAPPING.md b/KAFKA_CONSUMER_MAPPING.md new file mode 100644 index 00000000..0799b5dd --- /dev/null +++ b/KAFKA_CONSUMER_MAPPING.md @@ -0,0 +1,129 @@ +# Kafka Consumer — Central DB Mapping + +This document maps every column of both Central DB tables to the exact Kafka event +fields that populate them. Columns marked as `null for now` will be auto-populated +once the corresponding field is added to the Survey Service. + +--- + +## Table 1 — SurveyList (Survey Master) + +Populate from: **`SURVEY_CREATED`**, **`SURVEY_UPDATED`**, **`SURVEY_PUBLISHED`**, **`SURVEY_CLOSED`**, **`SURVEY_DELETED`** + +| Central DB Column | Kafka Field | Value / Notes | +|---|---|---| +| `SurveyID` | `data.surveyId` | UUID — use as primary key | +| `SurveyName` | `data.surveyTitle` | string | +| `TenantID` | `data.tenantId` | UUID | +| `TargetRole` | `data.targetRoles` | Array e.g. `["teacher", "admin"]` — store as JSON array | +| `TargetGeo` | — | **Save `null`** — field not in Survey Service yet, will be added later | +| `Context` | `data.contextType` | Enum: `none \| learner \| center \| teacher \| self` | +| `ContextId` | — | **Save `null`** — survey-level contextId not in Survey Service yet, will be added later | +| `Type` | `data.surveyType` | Free-form string, can be null | +| `CreatedAt` | `data.createdAt` | ISO timestamp | +| `CreatedBy` | `data.createdBy` | UUID of the creating user | +| `SurveyRolloutStartDate` | — | **Save `null`** — not in Survey Service yet; will be added later | +| `SurveyRolloutEndDate` | — | **Save `null`** — not in Survey Service yet; will be added later | +| `IsActive` | `data.status` | Derive: `isActive = (status === 'published')` | +| `SurveyForm` | `data.sections[]` | Full nested JSON — sections + fields. See structure below. Store as JSONB. | + +### SurveyForm JSON structure + +Stored as a **transformed, report-friendly JSON** — technical rendering fields are stripped out. +Use `fieldId` to join against `SurveySummary` keys in `SurveyTracker`. + +```json +[ + { + "sectionId": "uuid", + "sectionTitle": "string", + "sectionDescription": "string | null", + "order": 0, + "fields": [ + { + "fieldId": "uuid", + "fieldName": "string", + "label": "Human readable question label", + "type": "text | textarea | number | email | phone | date | time | datetime | select | multi_select | radio | checkbox | rating | scale | image_upload | video_upload | file_upload | signature | location | matrix", + "required": false, + "helpText": "string | null", + "placeholder": "string | null", + "order": 0 + } + ] + } +] +``` + +> Stripped from raw source: `uiConfig`, `uploadConfig`, `dataSource`, `validations`, `conditionalLogic`, `defaultValue`, `isVisible` + +### Event → Action mapping for SurveyList + +| Kafka Event | Action | +|---|---| +| `SURVEY_CREATED` | **INSERT** — all available columns, `null` for deferred ones | +| `SURVEY_UPDATED` | **UPDATE** — SurveyName, SurveyForm, TargetRole, Context, Type | +| `SURVEY_PUBLISHED` | **UPDATE** — `IsActive = true` | +| `SURVEY_CLOSED` | **UPDATE** — `IsActive = false` | +| `SURVEY_DELETED` | **DELETE** (or soft-delete) by SurveyID | + +--- + +## Table 2 — SurveyTracker + +Populate from: **`RESPONSE_STARTED`**, **`RESPONSE_UPDATED`**, **`RESPONSE_SUBMITTED`** + +| Central DB Column | Kafka Field | Value / Notes | +|---|---|---| +| `SurveyTrackingID` | `data.responseId` | Use `responseId` directly as the PK — avoids needing a separate generated ID | +| `SurveyID` | `data.surveyId` | UUID | +| `TenantID` | `data.tenantId` | UUID | +| `TargetRoleUserId` | `data.respondentId` | UUID — present in `RESPONSE_STARTED` and `RESPONSE_SUBMITTED`. Not in `RESPONSE_UPDATED` — keep from the existing row | +| `Context` | `data.contextType` | Enum: `none \| learner \| center \| teacher \| self \| null` — now included in all response events | +| `ContextId` | `data.contextId` | UUID or null — the specific learner/center this response is for — now included in all response events | +| `SurveySummary` | `data.responseData` | Present in both `RESPONSE_UPDATED` (partial) and `RESPONSE_SUBMITTED` (complete). Save `null` only if `responseData` is empty. Always check `status` before using for reports. | +| `SurveyResponseStatusIndividual` | `data.status` | `in_progress` → `submitted` | +| `CreatedAt` | `timestamp` (envelope) | Top-level `timestamp` from the `RESPONSE_STARTED` event | +| `UpdatedAt` | `timestamp` (envelope) | Top-level `timestamp` from `RESPONSE_UPDATED` / `RESPONSE_SUBMITTED` | + +### SurveySummary (responseData) structure + +```json +{ + "": "text answer", + "": 42, + "": "selected_option_value", + "": ["option1", "option2"], + "": "2026-05-11" +} +``` + +File upload fields are **not** in `responseData`. They are in a separate `fileUploadIds` map: +```json +{ + "": ["fileId_1", "fileId_2"] +} +``` + +### Event → Action mapping for SurveyTracker + +| Kafka Event | Action | +|---|---| +| `RESPONSE_STARTED` | **INSERT** — SurveyTrackingID, SurveyID, TenantID, TargetRoleUserId, Context, ContextId, Status = `in_progress`, CreatedAt, SurveySummary = `null` | +| `RESPONSE_UPDATED` | **UPDATE** — Status, Context, ContextId, SurveySummary = partial responseData, UpdatedAt | +| `RESPONSE_SUBMITTED` | **UPDATE** — Status = `submitted`, SurveySummary = full responseData, UpdatedAt | + +--- + +## Deferred Columns — Will Auto-Populate Later + +These columns are `null` today. Once the Survey Service adds the corresponding fields, +the Kafka events will start carrying them and the consumer will populate them automatically +without any consumer-side code changes — as long as the INSERT/UPDATE logic does not skip null fields. + +| Column | Table | Will come from | +|---|---|---| +| `SurveyRolloutStartDate` | SurveyList | New field in Survey entity — will come in `SURVEY_CREATED` / `SURVEY_UPDATED` events | +| `SurveyRolloutEndDate` | SurveyList | New field in Survey entity — will come in `SURVEY_CREATED` / `SURVEY_UPDATED` events | +| `TargetGeo` | SurveyList | New field in Survey entity — will come in `SURVEY_CREATED` / `SURVEY_UPDATED` events | +| `ContextId` | SurveyList | New field in Survey entity — will come in `SURVEY_CREATED` / `SURVEY_UPDATED` events | diff --git a/KAFKA_EVENTS.md b/KAFKA_EVENTS.md new file mode 100644 index 00000000..4ca19403 --- /dev/null +++ b/KAFKA_EVENTS.md @@ -0,0 +1,377 @@ +# Survey Service — Kafka Events Reference + +This document describes all Kafka events produced by the Survey Service. +Consumer services can use this as the contract for what to expect on the topic. + +--- + +## Configuration + +| Property | Env Variable | Default | +|---|---|---| +| Topic | `KAFKA_TOPIC` | `survey-topic` | +| Brokers | `KAFKA_HOST` | `localhost:9092` | +| Enabled | `KAFKA_ENABLED` | `false` | +| Client ID | — | `survey-service` | + +> **Note:** All events are silently skipped (no error thrown) when `KAFKA_ENABLED=false`. + +--- + +## Message Envelope + +Every message published to Kafka follows this top-level wrapper: + +```json +{ + "eventType": "", + "timestamp": "2026-05-11T10:00:00.000Z", + "": "", + "data": { ... } +} +``` + +- `eventType` — one of the event types listed below +- `timestamp` — ISO 8601 string, UTC, set at time of publish +- `` — also used as the **Kafka message key** for partition ordering + +--- + +## Survey Events + +Produced by: `src/modules/survey/services/survey.service.ts` + +### `SURVEY_CREATED` + +Fired when a new survey is successfully created (including all sections and fields). + +**Kafka message key:** `surveyId` + +```json +{ + "eventType": "SURVEY_CREATED", + "timestamp": "2026-05-11T10:00:00.000Z", + "surveyId": "uuid", + "data": { + "surveyId": "uuid", + "tenantId": "uuid", + "surveyTitle": "string", + "surveyDescription": "string | null", + "status": "draft", + "surveyType": "string | null", + "settings": {}, + "theme": {}, + "targetRoles": ["string"] , + "contextType": "none | learner | center | teacher | self", + "createdBy": "uuid", + "updatedBy": "uuid", + "version": 0, + "publishedAt": null, + "closedAt": null, + "createdAt": "ISO timestamp", + "updatedAt": "ISO timestamp", + "sections": [ + { + "sectionId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "sectionTitle": "string", + "sectionDescription": "string | null", + "displayOrder": 0, + "isVisible": true, + "conditionalLogic": null, + "fields": [ + { + "fieldId": "uuid", + "sectionId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "fieldName": "string", + "fieldLabel": "string", + "fieldType": "string", + "isRequired": false, + "displayOrder": 0, + "placeholder": "string | null", + "helpText": "string | null", + "defaultValue": "any | null", + "validations": {}, + "dataSource": null, + "uploadConfig": null, + "uiConfig": {}, + "conditionalLogic": null, + "options": [] + } + ] + } + ] + } +} +``` + +--- + +### `SURVEY_UPDATED` + +Fired when an existing survey's metadata is updated (title, description, settings, theme, etc.). +Not fired for section/field changes. + +**Kafka message key:** `surveyId` + +```json +{ + "eventType": "SURVEY_UPDATED", + "timestamp": "2026-05-11T10:00:00.000Z", + "surveyId": "uuid", + "data": { + // Same full survey object as SURVEY_CREATED, with updated values + } +} +``` + +--- + +### `SURVEY_PUBLISHED` + +Fired when a survey's status transitions from `draft` → `published`. + +**Kafka message key:** `surveyId` + +```json +{ + "eventType": "SURVEY_PUBLISHED", + "timestamp": "2026-05-11T10:00:00.000Z", + "surveyId": "uuid", + "data": { + // Same full survey object with status = "published" and publishedAt set + } +} +``` + +--- + +### `SURVEY_CLOSED` + +Fired when a survey's status transitions from `published` → `closed`. + +**Kafka message key:** `surveyId` + +```json +{ + "eventType": "SURVEY_CLOSED", + "timestamp": "2026-05-11T10:00:00.000Z", + "surveyId": "uuid", + "data": { + // Same full survey object with status = "closed" and closedAt set + } +} +``` + +--- + +### `SURVEY_DELETED` + +Fired when a survey is deleted. Note: only contains IDs, not the full survey object. + +**Kafka message key:** `surveyId` + +```json +{ + "eventType": "SURVEY_DELETED", + "timestamp": "2026-05-11T10:00:00.000Z", + "surveyId": "uuid", + "data": { + "surveyId": "uuid", + "tenantId": "uuid" + } +} +``` + +--- + +## Response Events + +Produced by: `src/modules/response/services/response.service.ts` + +### `RESPONSE_STARTED` + +Fired when a user begins filling a survey (a new response record is created with status `in_progress`). + +**Kafka message key:** `responseId` + +```json +{ + "eventType": "RESPONSE_STARTED", + "timestamp": "2026-05-11T10:00:00.000Z", + "responseId": "uuid", + "data": { + "responseId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "respondentId": "uuid", + "contextType": "none | learner | center | teacher | self | null", + "contextId": "uuid | null", + "status": "in_progress" + } +} +``` + +--- + +### `RESPONSE_UPDATED` + +Fired when a user saves a draft / partially updates their in-progress response. + +**Kafka message key:** `responseId` + +```json +{ + "eventType": "RESPONSE_UPDATED", + "timestamp": "2026-05-11T10:00:00.000Z", + "responseId": "uuid", + "data": { + "responseId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "contextType": "none | learner | center | teacher | self | null", + "contextId": "uuid | null", + "status": "in_progress", + "responseData": { + "": "", + "": ["option1", "option2"] + } + } +} +``` + +> `respondentId` is **not included** in this event — look up the existing row by `responseId` to get it. +> `responseData` here is **partial** — only fields answered so far. Always check `status` before using it for reporting. + +--- + +### `RESPONSE_SUBMITTED` + +Fired when a user submits their completed response. This is the richest response event. + +**Kafka message key:** `responseId` + +```json +{ + "eventType": "RESPONSE_SUBMITTED", + "timestamp": "2026-05-11T10:00:00.000Z", + "responseId": "uuid", + "data": { + "responseId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "respondentId": "uuid", + "contextType": "none | learner | center | teacher | self | null", + "contextId": "uuid | null", + "status": "submitted", + "responseData": { + "": "", + "": ["option1", "option2"] + }, + "submittedAt": "ISO timestamp" + } +} +``` + +#### `responseData` structure + +`responseData` is a flat key-value map where keys are `fieldId` UUIDs: + +| Field Type | Value Format | +|---|---| +| Text / Number / Date | `"string"` or `number` | +| Single-select / Radio | `"selectedOptionValue"` | +| Multi-select / Checkbox | `["option1", "option2"]` | +| File upload fields | **Not in `responseData`** — see `fileUploadIds` below | + +File upload field IDs are stored separately in `fileUploadIds`: +```json +{ + "": ["fileId_1", "fileId_2"] +} +``` +These `fileId` values correspond to records produced by `FILE_UPLOADED` events. + +--- + +## File Events + +Produced by: `src/modules/file-upload/services/file-upload.service.ts` + +### `FILE_UPLOADED` + +Fired when a file (image/video) is successfully uploaded and the DB record is created. + +**Kafka message key:** `fileId` + +```json +{ + "eventType": "FILE_UPLOADED", + "timestamp": "2026-05-11T10:00:00.000Z", + "fileId": "uuid", + "data": { + "fileId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid", + "fieldId": "uuid", + "fileType": "image | video" + } +} +``` + +--- + +### `FILE_DELETED` + +Fired when a file is soft-deleted (the record is kept in DB with `status = deleted`). + +**Kafka message key:** `fileId` + +```json +{ + "eventType": "FILE_DELETED", + "timestamp": "2026-05-11T10:00:00.000Z", + "fileId": "uuid", + "data": { + "fileId": "uuid", + "surveyId": "uuid", + "tenantId": "uuid" + } +} +``` + +--- + +## Event Summary Table + +| Event Type | Trigger | Message Key | Source File | +|---|---|---|---| +| `SURVEY_CREATED` | New survey created | `surveyId` | `survey.service.ts` | +| `SURVEY_UPDATED` | Survey metadata updated | `surveyId` | `survey.service.ts` | +| `SURVEY_PUBLISHED` | Survey goes live | `surveyId` | `survey.service.ts` | +| `SURVEY_CLOSED` | Survey accepting closed | `surveyId` | `survey.service.ts` | +| `SURVEY_DELETED` | Survey deleted | `surveyId` | `survey.service.ts` | +| `RESPONSE_STARTED` | User begins a response | `responseId` | `response.service.ts` | +| `RESPONSE_UPDATED` | Draft response saved | `responseId` | `response.service.ts` | +| `RESPONSE_SUBMITTED` | Response submitted | `responseId` | `response.service.ts` | +| `FILE_UPLOADED` | File uploaded | `fileId` | `file-upload.service.ts` | +| `FILE_DELETED` | File soft-deleted | `fileId` | `file-upload.service.ts` | + +--- + +## Consumer Notes + +1. **Filter by `eventType`** — All 10 event types land on the same topic (`survey-topic`). Your consumer must switch on `eventType` to route to the correct handler. + +2. **Multi-tenancy** — Every event carries a `tenantId`. Always scope any writes in your consumer by `tenantId`. + +3. **`responseData` is sent in both `RESPONSE_UPDATED` and `RESPONSE_SUBMITTED`** — `RESPONSE_UPDATED` carries partial data (fields answered so far), `RESPONSE_SUBMITTED` carries the final complete data. Always check `status` before using `responseData` for reporting — only `submitted` responses are complete. + +4. **File uploads are linked via `fileUploadIds`** — The `RESPONSE_SUBMITTED` event does not embed file content. Use the `fileId` values from `fileUploadIds` to cross-reference against `FILE_UPLOADED` events or call the Survey Service API to get presigned URLs. + +5. **Idempotency** — Kafka delivery may result in duplicate messages. Use `responseId` / `surveyId` / `fileId` as idempotency keys in your consumer (upsert by ID rather than blindly inserting). + +6. **Kafka is fire-and-forget from this service** — Kafka publish failures are logged but do not fail the HTTP request. Your consumer should not assume every action on the Survey Service will produce a Kafka event. diff --git a/src/modules/response/services/response.service.ts b/src/modules/response/services/response.service.ts index 6e03ab9f..86f5c2d8 100644 --- a/src/modules/response/services/response.service.ts +++ b/src/modules/response/services/response.service.ts @@ -119,6 +119,8 @@ export class ResponseService { surveyId: saved.surveyId, tenantId: saved.tenantId, respondentId: saved.respondentId, + contextType: saved.contextType, + contextId: saved.contextId, status: saved.status, }, saved.responseId) .catch((err) => @@ -307,7 +309,10 @@ export class ResponseService { responseId: saved.responseId, surveyId: saved.surveyId, tenantId: saved.tenantId, + contextType: saved.contextType, + contextId: saved.contextId, status: saved.status, + responseData: saved.responseData, }, saved.responseId) .catch((err) => this.loggerService.error('Kafka publish failed', err.message, apiId, userId) @@ -417,6 +422,8 @@ export class ResponseService { surveyId: saved.surveyId, tenantId: saved.tenantId, respondentId: saved.respondentId, + contextType: saved.contextType, + contextId: saved.contextId, status: saved.status, responseData: saved.responseData, submittedAt: saved.submittedAt, diff --git a/survey_template.csv b/survey_template.csv new file mode 100644 index 00000000..8ade2605 --- /dev/null +++ b/survey_template.csv @@ -0,0 +1,7 @@ +Question ID (optional),Section Name *,Display Order,Field Name *,Question *,Question Type *,Required?,Placeholder (optional),Option R1,Option R2,Option R3,Min Value (optional),Max Value (optional),File Upload Required,File Format,Helper Text (optional),⬇ Show only when Question ID →,⬇ Condition Operator →,⬇ Answer equals → +,Personal Details,1,full_name,What is your full name?,Short Text,Yes,Enter your full name,,,,,,,,,,, +,Personal Details,2,age,What is your age?,Number,Yes,Enter your age,,,, 1,120,,,,,, +,Personal Details,3,gender,What is your gender?,Radio (Single Choice),Yes,,Male,Female,Other,,,,,,,,, +,Personal Details,4,phone_number,What is your mobile number?,Mobile Number,No,Enter 10-digit number,,,,,,,,,,, +,Health Info,5,has_chronic_illness,Do you have any chronic illness?,Radio (Single Choice),Yes,,Yes,No,,,,,,,,,, +,Health Info,6,illness_details,Please describe your illness,Long Text,Yes,Describe here,,,,,,,,,has_chronic_illness,equals,Yes