From a4a6191f669d8f759d784d737216191d18eda3af Mon Sep 17 00:00:00 2001 From: Ricardo Zanini Date: Fri, 22 May 2026 09:55:58 -0400 Subject: [PATCH 1/3] docs: add MODE 3 Kafka presentation slides design spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design for 3 new slides (16-18) to be added to Data Index POC presentation: - Slide 16: MODE 3 Architecture (Kafka + Direct Write) - Slide 17: Three-way comparison (MODE 1 vs MODE 2 vs MODE 3) - Slide 18: Updated decision framework Sequential evolution narrative: FluentBit → Elasticsearch → Kafka Focus on architecture and decision criteria, no implementation timeline Related to #23 --- ...6-05-22-mode3-kafka-presentation-design.md | 424 ++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-22-mode3-kafka-presentation-design.md diff --git a/docs/superpowers/specs/2026-05-22-mode3-kafka-presentation-design.md b/docs/superpowers/specs/2026-05-22-mode3-kafka-presentation-design.md new file mode 100644 index 000000000..1e961b19d --- /dev/null +++ b/docs/superpowers/specs/2026-05-22-mode3-kafka-presentation-design.md @@ -0,0 +1,424 @@ +# MODE 3 Kafka Presentation Slides Design + +**Date:** 2026-05-22 +**Purpose:** Add MODE 3 (Kafka-based ingestion) slides to Data Index POC presentation +**Scope:** 3 new slides (16-18) for architecture comparison and decision framework +**Target Audience:** Backend engineers + technical leadership +**Presentation Date:** 2026-05-22 + +--- + +## Overview + +This design documents the addition of 3 slides to the existing Data Index v1.0.0 POC presentation to introduce MODE 3 (Kafka-based event ingestion for PostgreSQL). The slides present MODE 3 as a sequential evolution from MODE 1 and MODE 2, providing architectural comparison and decision criteria without committing to an implementation timeline. + +### Context + +- **Existing presentation:** 15 slides covering MODE 1 (PostgreSQL + Triggers) and MODE 2 (Elasticsearch + Transforms) +- **GitHub Issue:** #23 - Implement Kafka-based event ingestion service for PostgreSQL backend (MODE 3) +- **Status:** MODE 3 is designed but not yet implemented +- **Goal:** Present MODE 3 as a viable architectural option alongside MODE 1 and MODE 2 + +--- + +## Architecture + +### Presentation Structure + +**Current (15 slides):** +1. Introduction & Context (Slides 1-3) +2. MODE 1: PostgreSQL + Triggers (Slides 4-5) +3. MODE 2: Elasticsearch + Transforms (Slides 6-7) +4. Comparison & Demo (Slides 8-10) +5. Capacity & Tuning (Slides 11-13) +6. Decision Framework & Productization (Slides 14-15) + +**New (18 slides total):** +- Add 3 new slides after Slide 8 (Comparison) +- Insert between current comparison and demo sections +- New slides: 16 (MODE 3 Architecture), 17 (Three-way comparison), 18 (Updated decision framework) +- Renumber existing slides 9-15 to 19-25 + +### Narrative Flow + +The presentation now tells a complete architectural evolution story: + +1. **Introduction:** Context and overall architecture +2. **Deep Dive:** MODE 1 (triggers), MODE 2 (transforms), MODE 3 (Kafka) +3. **Comparison:** Three-way comparison across all dimensions +4. **Decision:** How to choose the right mode +5. **Demo:** Live demonstration (MODE 1/MODE 2) +6. **Operations:** Tuning, reliability, productization + +--- + +## Components + +### Slide 16: MODE 3 Architecture + +**Purpose:** Introduce MODE 3 architecture and key characteristics + +**Visual Elements:** + +1. **Title:** "MODE 3: Kafka + Direct Write (PostgreSQL)" with purple/violet badge (#7b1fa2) + +2. **Architecture Diagram:** + ``` + Quarkus Flow → Kafka Topics → Kafka Listener Service → PostgreSQL → GraphQL API + (CloudEvents) (workflow-events, (Java/Quarkus) (normalized (SmallRye) + task-events) tables only) + ``` + +3. **Key Characteristics Box:** + - Normalization: Immediate (< 10ms) + - Consistency: At-least-once delivery + - Security: Kafka encryption + auth + - Infrastructure: Leverages existing Kafka + +4. **When to Choose MODE 3 Box:** + - Existing Kafka infrastructure + - Sensitive data concerns (PII, credit cards) + - Need replay capabilities from Kafka offsets + - Direct write control preferred over triggers + +**Technical Details:** + +- **No raw tables:** Unlike MODE 1, MODE 3 writes directly to normalized tables +- **No triggers:** Normalization logic implemented in Java (ported from SQL triggers) +- **Separate deployment:** Kafka Listener Service is a standalone pod (not part of data-index-service) +- **Same GraphQL API:** Identical interface to MODE 1 and MODE 2 + +**Color Scheme:** +- MODE 3 primary: Purple/Violet (#7b1fa2) +- Kafka: Purple (#7b1fa2) +- Listener Service: Purple (#7b1fa2) +- PostgreSQL: Blue (#1976d2) - reuse MODE 1 color +- GraphQL API: Green (#2e7d32) - consistent across all modes + +--- + +### Slide 17: MODE 1 vs MODE 2 vs MODE 3 Comparison + +**Purpose:** Comprehensive comparison across all three storage backends + +**Visual Elements:** + +1. **Title:** "Storage Backend Comparison" + +2. **Three-column comparison table:** + +| Feature | MODE 1 (PostgreSQL) | MODE 2 (Elasticsearch) | MODE 3 (Kafka) | +|---------|---------------------|------------------------|----------------| +| **Ingestion** | FluentBit → Raw tables | FluentBit → Raw indices | Kafka → Listener Service | +| **Normalization** | PostgreSQL Triggers (< 1ms) | ES Transforms (~1s) | Java Service (< 10ms) | +| **Consistency** | ACID transactions | Eventual consistency | At-least-once delivery | +| **Scale Target** | < 50K workflows/day | > 50K workflows/day | 50K+ workflows/day | +| **Infrastructure** | PostgreSQL only | Elasticsearch cluster | Kafka + PostgreSQL | +| **Security** | Log files (plaintext) | Log files (plaintext) | Kafka encryption + auth | +| **Replay** | Log file replay (idempotent) | Log file replay (idempotent) | Kafka offset replay | +| **GraphQL API** | ✅ Identical | ✅ Identical | ✅ Identical | +| **Operational Complexity** | Low (triggers automatic) | Medium (transform monitoring) | Medium (Kafka + service) | +| **Best For** | Simple deployments, ACID required | Large scale, full-text search | Existing Kafka, sensitive data | + +**Visual Enhancements:** +- Color-coded column headers (Blue for MODE 1, Orange for MODE 2, Purple for MODE 3) +- Green checkmarks for "GraphQL API" row (all identical) +- "Best For" row highlighted with border or background color +- Bold text for category headers + +**Key Messages:** +- All three modes share the same GraphQL API +- Infrastructure and operational complexity are the main differentiators +- MODE 3 offers unique security benefits via Kafka encryption + +--- + +### Slide 18: Architectural Decision Framework (Updated) + +**Purpose:** Guide selection between MODE 1, MODE 2, and MODE 3 based on requirements + +**Visual Elements:** + +1. **Title:** "Choosing Your Storage Backend" + +2. **Decision Tree:** + ``` + Start Here + ↓ + Do you have existing Kafka infrastructure? + ↓ YES → MODE 3 (Kafka + PostgreSQL) + ↓ NO + ↓ + Do you need full-text search or > 50K workflows/day? + ↓ YES → MODE 2 (Elasticsearch + Transforms) + ↓ NO + ↓ + MODE 1 (PostgreSQL + Triggers) + ``` + +3. **Decision Factors Table:** + +| Factor | MODE 1 | MODE 2 | MODE 3 | +|--------|--------|--------|--------| +| **Scale Threshold** | < 50K workflows/day | > 50K workflows/day | 50K+ workflows/day | +| **Primary Driver** | Simplicity, ACID | Search, aggregations | Security, existing Kafka | +| **Infrastructure** | PostgreSQL | Elasticsearch | Kafka + PostgreSQL | +| **Operational Effort** | Low | Medium | Medium | +| **Data Security** | Log-based (standard) | Log-based (standard) | Kafka encryption | +| **Migration Path** | → MODE 2 (scale)
→ MODE 3 (security) | — | — | + +4. **Key Recommendation Box:** + - "All three modes share identical GraphQL API" + - "Choose based on infrastructure and requirements, not features" + - "MODE 1 is default; MODE 2 for scale; MODE 3 for enterprise security" + +**Design Rationale:** +- Decision tree leads with infrastructure (Kafka) as primary decision point +- Positions MODE 1 as the default/simple choice +- MODE 2 for scale/search needs +- MODE 3 for enterprise security and existing Kafka investments + +--- + +## Data Flow + +### How MODE 3 Differs from MODE 1 and MODE 2 + +**MODE 1 (PostgreSQL + Triggers):** +``` +Quarkus Flow → /tmp/quarkus-flow-events.log (JSON) + ↓ (FluentBit tail) + PostgreSQL raw tables (JSONB) + ↓ (BEFORE INSERT triggers) + PostgreSQL normalized tables + ↓ (JPA/Hibernate) + GraphQL API (SmallRye GraphQL) +``` + +**MODE 2 (Elasticsearch + Transforms):** +``` +Quarkus Flow → /tmp/quarkus-flow-events.log (JSON) + ↓ (FluentBit tail) + Elasticsearch raw indices (workflow-events, task-events) + ↓ (ES Transform, continuous, 1s) + Elasticsearch normalized indices (workflow-instances, task-executions) + ↓ (Elasticsearch Java Client) + GraphQL API (SmallRye GraphQL) +``` + +**MODE 3 (Kafka + Direct Write):** +``` +Quarkus Flow → Kafka Topics (CloudEvents) + ↓ (workflow-events, task-events) + Kafka Listener Service (Java/Quarkus) + ↓ (Parse CloudEvents, normalize in Java) + PostgreSQL normalized tables ONLY + ↓ (JDBC UPSERT) + GraphQL API (SmallRye GraphQL) +``` + +**Key Differences:** +- **No intermediate storage:** MODE 3 skips raw tables/indices entirely +- **No triggers/transforms:** Normalization happens in Java code before database write +- **Direct UPSERT:** Uses JDBC `INSERT ... ON CONFLICT DO UPDATE` directly +- **CloudEvents format:** Kafka messages use CloudEvents spec (structured event envelope) + +--- + +## Implementation Details + +### Slide Generation + +**File Structure:** +``` +presentations/2026-05-20-data-index-poc/ +├── diagrams/ +│ └── mode3-architecture.svg # New MODE 3 architecture diagram +├── templates/ +│ ├── slide-16-mode3-arch.html # New MODE 3 architecture slide +│ ├── slide-17-three-way-comparison.html # New 3-way comparison slide +│ └── slide-18-decision-updated.html # Updated decision framework slide +├── output/ +│ ├── slide-16-mode3-arch.png # Generated PNG +│ ├── slide-17-three-way-comparison.png +│ └── slide-18-decision-updated.png +├── SLIDES.md # Updated with 3 new slides +└── generate-slides.js # Updated to generate slides 16-18 +``` + +**Generation Process:** +1. Create MODE 3 architecture diagram SVG (diagrams/mode3-architecture.svg) +2. Create HTML templates for 3 new slides (templates/slide-16-*.html, slide-17-*.html, slide-18-*.html) +3. Update generate-slides.js to include new slides +4. Update SLIDES.md documentation with new slide descriptions +5. Run `npm run generate` to create PNG outputs + +**Renumbering:** +- Existing slides 9-15 become 19-25 +- Requires updating template filenames and generate-slides.js references + +--- + +## Testing + +### Validation Checklist + +**Visual Consistency:** +- [ ] MODE 3 purple color (#7b1fa2) is distinct from MODE 1 blue and MODE 2 orange +- [ ] All three slides use consistent fonts, sizing, and layout +- [ ] Diagrams match the visual style of existing slides (clean, professional) +- [ ] Tables are readable at 1920x1080 resolution + +**Content Accuracy:** +- [ ] Architecture diagram matches GitHub issue #23 description +- [ ] Comparison table accurately represents all three modes +- [ ] Decision framework logic is sound (Kafka first, then search/scale) +- [ ] No contradictions with existing slides 1-15 + +**Technical Correctness:** +- [ ] CloudEvents format mentioned (correct Kafka message format) +- [ ] Normalization logic described as "ported from triggers" (accurate) +- [ ] Scale targets are consistent (< 50K for MODE 1, > 50K for MODE 2, 50K+ for MODE 3) +- [ ] GraphQL API described as identical across all modes (correct) + +**Presentation Flow:** +- [ ] Slides 16-18 fit naturally after Slide 8 (current comparison) +- [ ] Narrative arc: MODE 1 → MODE 2 → MODE 3 → Three-way comparison → Decision +- [ ] No redundancy with existing slides +- [ ] Smooth transition to demo section (now Slide 19) + +--- + +## Error Handling + +### Potential Issues and Mitigations + +**Issue 1: MODE 3 appears more complex than MODE 1/2** +- **Mitigation:** Emphasize "existing Kafka infrastructure" benefit in "When to Choose" box +- **Mitigation:** Position as "enterprise option" for security and control + +**Issue 2: Audience may think MODE 3 is required** +- **Mitigation:** Decision framework clearly shows MODE 1 as default +- **Mitigation:** "Choose based on infrastructure" messaging + +**Issue 3: Implementation timeline questions** +- **Mitigation:** No roadmap slide (by design) +- **Mitigation:** If asked, refer to GitHub issue #23 (week of 2026-05-26 target) + +**Issue 4: Security comparison may seem unfair to MODE 1/2** +- **Mitigation:** Use neutral language: "Log-based (standard)" vs "Kafka encryption" +- **Mitigation:** Emphasize all modes are production-ready and secure + +--- + +## Key Principles + +### Design Decisions + +1. **Sequential Evolution Narrative:** + - MODE 1 (simple, triggers) → MODE 2 (scale, search) → MODE 3 (security, enterprise) + - Tells story of architectural maturity + - Positions MODE 3 as additive, not replacement + +2. **Identical GraphQL API:** + - Emphasized on every slide + - Reinforces that choice is infrastructure-driven, not feature-driven + +3. **No Implementation Timeline:** + - Avoids commitment pressure + - Keeps focus on architecture and decision criteria + - Leaves flexibility for post-presentation prioritization + +4. **Security as Primary MODE 3 Differentiator:** + - Kafka encryption + auth + - Addresses Walter's concern from GitHub issue + - Appeals to enterprise customers with compliance needs + +5. **Infrastructure-First Decision Framework:** + - "Do you have Kafka?" is first question + - Avoids customers building Kafka just for MODE 3 + - Leverages existing investments + +--- + +## Non-Requirements + +### What This Design Does NOT Include + +- ❌ **Implementation timeline slide:** No roadmap or delivery dates +- ❌ **Demo section for MODE 3:** Not yet implemented, can't demo +- ❌ **CODE examples:** No CloudEvents schemas or Java code snippets +- ❌ **Kafka configuration details:** No broker settings, topic naming, partitioning +- ❌ **Operator integration:** How operator deploys MODE 3 (future work) +- ❌ **Performance benchmarks:** No comparison data (not yet built) +- ❌ **Migration guides:** How to switch between modes (future documentation) + +### Rationale + +MODE 3 is designed but not implemented. These slides present the architecture and decision criteria without overcommitting or creating expectations for features that don't exist yet. The focus is on education and positioning, not delivery promises. + +--- + +## Success Criteria + +**Presentation Goals Met:** +- [ ] Audience understands MODE 3 architecture at high level +- [ ] Audience can explain when MODE 3 is appropriate vs MODE 1/2 +- [ ] Audience sees MODE 3 as complementary (not replacement) to other modes +- [ ] No confusion about implementation status (not yet built) +- [ ] Smooth integration into existing 15-slide presentation + +**Technical Accuracy:** +- [ ] Architecture diagram matches GitHub issue #23 design +- [ ] Comparison table is factually correct +- [ ] Decision framework logic is sound +- [ ] No contradictions with existing slides or CLAUDE.md + +**Visual Quality:** +- [ ] Slides match existing design language +- [ ] MODE 3 purple color is professional and distinct +- [ ] Diagrams are clear and readable +- [ ] Tables fit on slide without requiring small fonts + +--- + +## Future Work + +### Post-Presentation Enhancements + +**After MODE 3 Implementation (Week of 2026-05-26):** +1. Add demo section (slides showing MODE 3 deployment and testing) +2. Add performance benchmarks (MODE 1 vs MODE 3 comparison) +3. Add CloudEvents schema examples +4. Update decision framework with real-world data + +**Documentation Updates:** +1. Create `data-index/docs/deployment/MODE3_KAFKA_INGESTION.md` +2. Update architecture diagrams in `data-index-docs/` +3. Add Kafka listener service configuration guide +4. Document migration paths between modes + +**Presentation Maintenance:** +1. Create versioned presentation copies (pre-MODE3, post-MODE3) +2. Archive old slides for reference +3. Update SLIDES.md with MODE 3 demo section (future) + +--- + +## References + +- **GitHub Issue:** [#23 - Implement Kafka-based event ingestion service for PostgreSQL backend (MODE 3)](https://github.com/kubesmarts/logic-apps/issues/23) +- **Existing Presentation:** `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/` +- **CLAUDE.md:** `/Users/ricferna/dev/github/kubesmarts/logic-apps/CLAUDE.md` (current architecture documentation) +- **MODE 1 Documentation:** `data-index/docs/deployment/MODE1_HANDOFF.md` +- **MODE 2 Documentation:** `data-index/docs/deployment/MODE2_HANDOFF.md` + +--- + +## Notes + +- Target presentation date: 2026-05-22 (today) +- Presentation will be delivered before MODE 3 implementation begins +- Slides serve dual purpose: education (for team) and positioning (for stakeholders) +- MODE 3 is designed for customers with existing Kafka infrastructure and security requirements +- No commitment to delivery timeline during presentation (refer to GitHub issue if pressed) From d60178ee6399dcc1ff1af4b2c472dd7227d45d4d Mon Sep 17 00:00:00 2001 From: Ricardo Zanini Date: Fri, 22 May 2026 10:01:57 -0400 Subject: [PATCH 2/3] docs: add MODE 3 presentation and e2e tests implementation plan Plan includes: - Part 1: Add 3 MODE 3 slides to presentation (16-18) - Part 2: Deploy MODE 1 to KIND and run e2e tests - 15 tasks with bite-sized steps - Quick reference guide for presentation demo Related to #23 --- ...2026-05-22-mode3-presentation-e2e-tests.md | 1197 +++++++++++++++++ 1 file changed, 1197 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-22-mode3-presentation-e2e-tests.md diff --git a/docs/superpowers/plans/2026-05-22-mode3-presentation-e2e-tests.md b/docs/superpowers/plans/2026-05-22-mode3-presentation-e2e-tests.md new file mode 100644 index 000000000..887ea8296 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-mode3-presentation-e2e-tests.md @@ -0,0 +1,1197 @@ +# MODE 3 Presentation Slides and E2E Tests Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add 3 MODE 3 slides to the Data Index presentation and run e2e tests on KIND cluster for today's presentation + +**Architecture:** Add slides 16-18 (MODE 3 architecture, three-way comparison, updated decision framework) to existing 15-slide presentation, then deploy MODE 1 to KIND and verify with GraphQL queries + +**Tech Stack:** HTML5, SVG, Puppeteer (slide generation), KIND, Kubernetes, PostgreSQL, FluentBit, GraphQL + +--- + +## Part 1: Presentation Updates + +### Task 1: Create MODE 3 Architecture Diagram SVG + +**Files:** +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/diagrams/mode3-architecture.svg` + +- [ ] **Step 1: Create MODE 3 architecture SVG diagram** + +```svg + + + + Kafka Topics + CloudEvents Format + + + + + + + + + + workflow-events, task-events + + + + Kafka Listener Service + Java/Quarkus Consumer + Parse CloudEvents + Normalize + + + + JDBC UPSERT + + + + PostgreSQL Normalized Tables (Direct Write) + + + workflow_instances + id, name, version, status + start, end, input, output + error (type, title, detail) + + + task_instances + id, workflow_id, name, status + start, end, input, output + error (type, title, detail) + + + + ⚡ No triggers, No raw tables - Direct UPSERT with idempotency logic in Java + + + + JPA/Hibernate + + + + GraphQL API + SmallRye GraphQL + + + + 🔒 Security + Kafka Encryption + SASL/SSL Auth + No log files + +``` + +- [ ] **Step 2: Verify SVG renders correctly** + +Open in browser: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/diagrams/mode3-architecture.svg` + +Expected: Purple/violet color scheme, Kafka → Listener → PostgreSQL → GraphQL flow, security badge visible + +- [ ] **Step 3: Commit diagram** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps +git add presentations/2026-05-20-data-index-poc/diagrams/mode3-architecture.svg +git commit -m "feat(presentation): add MODE 3 architecture diagram + +- Kafka topics → Listener service → PostgreSQL (direct write) +- Purple/violet color scheme (#7b1fa2) +- Security badge highlighting Kafka encryption +- No raw tables or triggers (direct UPSERT) + +Related to #23" +``` + +--- + +### Task 2: Update CSS for MODE 3 Badge Styling + +**Files:** +- Modify: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/styles/slides.css:189-216` + +- [ ] **Step 1: Add MODE 3 badge and box styles to CSS** + +Add after line 196 (after `.badge-mode2`): + +```css +.badge-mode3 { + border: 3px solid #7b1fa2; +} + +.box-purple { + background: #f3e5f5; + border: 3px solid #7b1fa2; +} + +.badge-mode3 .badge-title { + color: #7b1fa2; +} +``` + +- [ ] **Step 2: Verify CSS is valid** + +Check syntax with: `cat /Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/styles/slides.css | grep -A 3 "badge-mode3"` + +Expected: 3 new CSS rules for MODE 3 purple styling + +- [ ] **Step 3: Commit CSS changes** + +```bash +git add presentations/2026-05-20-data-index-poc/styles/slides.css +git commit -m "feat(presentation): add MODE 3 purple styling to CSS + +- .badge-mode3 with purple border (#7b1fa2) +- .box-purple for MODE 3 content boxes +- Consistent with MODE 1 (blue) and MODE 2 (orange) + +Related to #23" +``` + +--- + +### Task 3: Create Slide 16 - MODE 3 Architecture + +**Files:** +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-16-mode3-arch.html` + +- [ ] **Step 1: Create HTML template for Slide 16** + +```html + + + + + + MODE 3 Architecture + + + +
+

MODE 3: Kafka + Direct Write (PostgreSQL)

+

Enterprise event ingestion with Kafka security

+ +
+ +
+ +
+
+

Key Characteristics:

+
    +
  • < 10ms normalization
    • +
  • At-least-once delivery
    • +
  • Kafka encryption + auth
    • +
  • Horizontal scaling
    • +
+
+
+

When to Choose MODE 3:

+
    +
  • Existing Kafka infrastructure
    • +
  • Sensitive data (PII, credit cards)
    • +
  • Kafka offset replay needed
    • +
  • Direct write control preferred
    • +
+
+
+
+ + +``` + +- [ ] **Step 2: Verify HTML renders correctly** + +Open in browser: `file:///Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-16-mode3-arch.html` + +Expected: MODE 3 title, architecture diagram, two-column layout with characteristics and use cases + +- [ ] **Step 3: Commit slide template** + +```bash +git add presentations/2026-05-20-data-index-poc/templates/slide-16-mode3-arch.html +git commit -m "feat(presentation): add Slide 16 - MODE 3 Architecture + +- Kafka → Listener Service → PostgreSQL flow +- Key characteristics: < 10ms, at-least-once, Kafka security +- When to choose: Existing Kafka, sensitive data, replay needs + +Related to #23" +``` + +--- + +### Task 4: Create Slide 17 - Three-Way Comparison + +**Files:** +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-17-three-way-comparison.html` + +- [ ] **Step 1: Create HTML template for Slide 17** + +```html + + + + + + MODE 1 vs MODE 2 vs MODE 3 Comparison + + + +
+

Storage Backend Comparison

+

Choose based on infrastructure and requirements

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureMODE 1
PostgreSQL		MODE 2
Elasticsearch		MODE 3
Kafka
IngestionFluentBit → Raw tablesFluentBit → Raw indicesKafka → Listener Service
NormalizationPG Triggers (< 1ms)ES Transforms (~1s)Java Service (< 10ms)
ConsistencyACID transactionsEventual consistencyAt-least-once delivery
Scale Target< 50K workflows/day> 50K workflows/day50K+ workflows/day
InfrastructurePostgreSQL onlyElasticsearch clusterKafka + PostgreSQL
SecurityLog-based (standard)Log-based (standard)Kafka encryption + auth
ReplayLog file (idempotent)Log file (idempotent)Kafka offset replay
GraphQL API✅ Identical
Ops ComplexityLow (triggers auto)Medium (transform monitoring)Medium (Kafka + service)
Best ForSimple deployments
ACID required		Large scale
Full-text search		Existing Kafka
Sensitive data
+
+
+ + +``` + +- [ ] **Step 2: Verify HTML renders correctly** + +Open in browser: `file:///Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-17-three-way-comparison.html` + +Expected: Three-column table with MODE 1 (blue), MODE 2 (orange), MODE 3 (purple) headers, GraphQL row highlighted green + +- [ ] **Step 3: Commit slide template** + +```bash +git add presentations/2026-05-20-data-index-poc/templates/slide-17-three-way-comparison.html +git commit -m "feat(presentation): add Slide 17 - Three-way comparison + +- MODE 1 vs MODE 2 vs MODE 3 comparison table +- Color-coded headers (blue, orange, purple) +- GraphQL API row highlighted (identical across modes) +- Best For row shows key differentiators + +Related to #23" +``` + +--- + +### Task 5: Create Slide 18 - Updated Decision Framework + +**Files:** +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-18-decision-updated.html` + +- [ ] **Step 1: Create HTML template for Slide 18** + +```html + + + + + + Architectural Decision Framework + + + +
+

Choosing Your Storage Backend

+

Infrastructure-driven decision framework

+ +
+ +
+

Decision Tree

+
+
Start Here ↓
+
+
❓ Do you have existing Kafka infrastructure?
+
+ ✅ YES → MODE 3 (Kafka + PostgreSQL) +
+
❌ NO ↓
+
+ ❓ Do you need full-text search or > 50K workflows/day? +
+
+ ✅ YES → MODE 2 (Elasticsearch + Transforms) +
+
❌ NO ↓
+
+ → MODE 1 (PostgreSQL + Triggers) +
+
+
+
+ + +
+

Decision Factors

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FactorMODE 1MODE 2MODE 3
Primary DriverSimplicity, ACIDSearch, aggregationsSecurity, existing Kafka
InfrastructurePostgreSQLElasticsearchKafka + PostgreSQL
Operational EffortLowMediumMedium
Data SecurityLog-based (standard)Log-based (standard)Kafka encryption
Migration Path→ MODE 2 (scale)
→ MODE 3 (security)		——
+
+ + +
+

Key Recommendations

+
    +
  • All three modes share identical GraphQL API
    • +
  • Choose based on infrastructure and requirements, not features
    • +
  • MODE 1 is default; MODE 2 for scale; MODE 3 for enterprise security
    • +
+
+
+
+ + +``` + +- [ ] **Step 2: Verify HTML renders correctly** + +Open in browser: `file:///Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/templates/slide-18-decision-updated.html` + +Expected: Decision tree flowchart, decision factors table, green recommendation box at bottom + +- [ ] **Step 3: Commit slide template** + +```bash +git add presentations/2026-05-20-data-index-poc/templates/slide-18-decision-updated.html +git commit -m "feat(presentation): add Slide 18 - Updated decision framework + +- Decision tree: Kafka first, then search/scale, default MODE 1 +- Decision factors table comparing all three modes +- Key recommendations box (identical GraphQL API) + +Related to #23" +``` + +--- + +### Task 6: Update generate-slides.js + +**Files:** +- Modify: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/generate-slides.js:5-21` + +- [ ] **Step 1: Insert new slides and renumber existing slides 9-15 to 19-25** + +Replace lines 5-21 with: + +```javascript +const slides = [ + { name: 'slide-01-title', file: 'templates/slide-01-title.html' }, + { name: 'slide-02-migration', file: 'templates/slide-02-migration.html' }, + { name: 'slide-03-event-flow', file: 'templates/slide-03-event-flow.html' }, + { name: 'slide-04-mode1-arch', file: 'templates/slide-04-mode1-arch.html' }, + { name: 'slide-05-mode1-trigger', file: 'templates/slide-05-mode1-trigger.html' }, + { name: 'slide-06-mode2-arch', file: 'templates/slide-06-mode2-arch.html' }, + { name: 'slide-07-mode2-transform', file: 'templates/slide-07-mode2-transform.html' }, + { name: 'slide-08-comparison', file: 'templates/slide-08-comparison.html' }, + { name: 'slide-16-mode3-arch', file: 'templates/slide-16-mode3-arch.html' }, + { name: 'slide-17-three-way-comparison', file: 'templates/slide-17-three-way-comparison.html' }, + { name: 'slide-18-decision-updated', file: 'templates/slide-18-decision-updated.html' }, + { name: 'slide-19-demo-setup', file: 'templates/slide-09-demo-setup.html' }, + { name: 'slide-20-demo-commands', file: 'templates/slide-10-demo-commands.html' }, + { name: 'slide-21-fluentbit', file: 'templates/slide-11-fluentbit.html' }, + { name: 'slide-22-log-replay', file: 'templates/slide-12-log-replay.html' }, + { name: 'slide-23-reliability', file: 'templates/slide-13-reliability.html' }, + { name: 'slide-24-decision', file: 'templates/slide-14-decision.html' }, + { name: 'slide-25-redhat', file: 'templates/slide-15-redhat.html' } +]; +``` + +- [ ] **Step 2: Verify JavaScript syntax** + +Run: `node -c /Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/generate-slides.js` + +Expected: No output (syntax valid) + +- [ ] **Step 3: Commit generator script changes** + +```bash +git add presentations/2026-05-20-data-index-poc/generate-slides.js +git commit -m "feat(presentation): update slide generator for MODE 3 slides + +- Insert slides 16-18 after slide 8 +- Renumber existing slides 9-15 to 19-25 +- Total slides: 18 (was 15) + +Related to #23" +``` + +--- + +### Task 7: Update SLIDES.md Documentation + +**Files:** +- Modify: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/SLIDES.md:1-7` +- Modify: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/SLIDES.md:62-82` + +- [ ] **Step 1: Update header metadata** + +Replace lines 1-7 with: + +```markdown +# Data Index v1.0.0 POC Presentation Slides + +**Generated:** 2026-05-22 +**Total Slides:** 18 +**Format:** PNG (1920x1080) +**Target Audience:** Backend engineers + technical leadership +**Duration:** 35-50 minutes +``` + +- [ ] **Step 2: Add MODE 3 slides section and renumber demo section** + +Replace lines 62-82 with: + +```markdown +--- + +### MODE 3: Kafka + Direct Write (Slides 16-18) + +**Slide 16: MODE 3 Architecture** +- Kafka-based event ingestion architecture +- Kafka Topics → Listener Service → PostgreSQL (direct write, no triggers/raw tables) +- Key characteristics: < 10ms normalization, at-least-once delivery, Kafka encryption +- When to choose MODE 3: Existing Kafka, sensitive data, offset replay + +**Slide 17: MODE 1 vs MODE 2 vs MODE 3 Comparison** +- Three-way comparison table +- All dimensions: ingestion, normalization, consistency, scale, security, replay +- GraphQL API identical across all modes (highlighted row) +- Best For row: Simple/ACID (MODE 1), Scale/Search (MODE 2), Kafka/Security (MODE 3) + +**Slide 18: Architectural Decision Framework (Updated)** +- Decision tree: Kafka first, then search/scale, default MODE 1 +- Decision factors table: primary driver, infrastructure, operational effort, security +- Key recommendations: identical GraphQL API, choose by infrastructure, MODE 1 default + +--- + +### Demo (Slides 19-20) + +**Slide 19: Demo Environment Setup** (was Slide 9) +- KIND cluster deployment scripts +- MODE 1 components: PostgreSQL + FluentBit + Data Index + Workflow App +- MODE 2 components: Elasticsearch + FluentBit + Data Index + Workflow App + +**Slide 20: Demo Commands** (was Slide 10) +- Verify components running +- Trigger workflow execution +- Watch events flow (FluentBit logs) +- Query via GraphQL +- Verify data normalization +``` + +- [ ] **Step 3: Update capacity section slide numbers** + +Replace lines 84-105 (Capacity & Tuning section header) with: + +```markdown +--- + +### Capacity & Tuning (Slides 21-23) + +**Slide 21: FluentBit Scalability** (was Slide 11) +``` + +Continue pattern for slides 22-23 (old 12-13). + +- [ ] **Step 4: Update decision framework section slide numbers** + +Replace lines 107-120 (Decision Framework & Productization section) with: + +```markdown +--- + +### Decision Framework & Productization (Slides 24-25) + +**Slide 24: Architectural Decision Framework** (was Slide 14) +- Choose MODE 1 if: < 50K workflows/day, ACID required, simpler operations +- Choose MODE 2 if: > 50K workflows/day, full-text search, complex aggregations +- Decision factors table: Scale threshold, consistency, operational complexity, migration path +- NOTE: This is the old two-way decision framework; Slide 18 is the updated three-way version + +**Slide 25: Red Hat Productization** (was Slide 15) +``` + +- [ ] **Step 5: Update file structure section** + +Replace slide filenames in lines 122-142 to include slides 16-25. + +- [ ] **Step 6: Update presentation flow section** + +Replace lines 179-200 (Presentation Flow) with: + +```markdown +## Presentation Flow (35-50 min) + +### Introduction (5 min) +- Slides 1-3: Context, migration rationale, overall architecture + +### Technical Deep Dive (20-25 min) +- Slides 4-5: MODE 1 architecture & logic +- Slides 6-7: MODE 2 architecture & logic +- Slide 8: MODE 1 vs MODE 2 comparison (two-way) +- Slides 16-17: MODE 3 architecture & three-way comparison +- Slide 18: Updated decision framework (three-way) + +### Live Demo (10-15 min) +- Slides 19-20: Demo setup and commands +- Live demonstration on KIND cluster (MODE 1) + +### Capacity & Tuning Discussion (5-10 min) +- Slides 21-23: FluentBit scalability, log replay, reliability + +### Decision Making (5-10 min) +- Slide 24: Two-way decision framework (legacy) +- Slide 25: Red Hat productization strategy +- Team discussion and Q&A +``` + +- [ ] **Step 7: Update key messages** + +Add to line 214 (after existing key messages): + +```markdown +8. **Three Storage Modes:** PostgreSQL (default), Elasticsearch (scale), Kafka (enterprise security) +9. **Infrastructure-Driven Choice:** Existing Kafka → MODE 3, else MODE 2 for scale, else MODE 1 +``` + +- [ ] **Step 8: Commit documentation changes** + +```bash +git add presentations/2026-05-20-data-index-poc/SLIDES.md +git commit -m "docs(presentation): update SLIDES.md for MODE 3 content + +- Total slides: 18 (was 15) +- Add MODE 3 section (slides 16-18) +- Renumber demo and capacity sections (19-25) +- Update presentation flow (35-50 min duration) + +Related to #23" +``` + +--- + +### Task 8: Generate Slide PNGs + +**Files:** +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-16-mode3-arch.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-17-three-way-comparison.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-18-decision-updated.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-19-demo-setup.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-20-demo-commands.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-21-fluentbit.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-22-log-replay.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-23-reliability.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-24-decision.png` +- Create: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-25-redhat.png` + +- [ ] **Step 1: Clean old output directory** + +```bash +rm -f /Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-*.png +``` + +Expected: All old PNG files removed + +- [ ] **Step 2: Generate all slides** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc +npm run generate +``` + +Expected output: +``` +Starting slide generation... + +Generating slide-01-title... +✓ slide-01-title.png +... +Generating slide-16-mode3-arch... +✓ slide-16-mode3-arch.png +Generating slide-17-three-way-comparison... +✓ slide-17-three-way-comparison.png +Generating slide-18-decision-updated... +✓ slide-18-decision-updated.png +... +✓ All slides generated successfully! +Output: presentation/output/ +``` + +- [ ] **Step 3: Verify new slides exist** + +```bash +ls -lh /Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/slide-{16,17,18}*.png +``` + +Expected: 3 PNG files (slide-16-mode3-arch.png, slide-17-three-way-comparison.png, slide-18-decision-updated.png), each 200-400 KB + +- [ ] **Step 4: Commit generated slides** + +```bash +git add presentations/2026-05-20-data-index-poc/output/*.png +git commit -m "feat(presentation): generate MODE 3 slides and renumbered slides + +- New slides: 16 (MODE 3 arch), 17 (three-way comparison), 18 (decision) +- Renumbered slides: 19-25 (previously 9-15) +- Total presentation: 18 slides (1920x1080 PNG) + +Related to #23" +``` + +--- + +## Part 2: E2E Tests on KIND + +### Task 9: Create Git Branch for Work + +**Files:** +- None (git operation) + +- [ ] **Step 1: Create and checkout new branch** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps +git checkout -b presentation/mode3-slides-e2e-tests +``` + +Expected: `Switched to a new branch 'presentation/mode3-slides-e2e-tests'` + +- [ ] **Step 2: Verify branch created** + +```bash +git branch --show-current +``` + +Expected: `presentation/mode3-slides-e2e-tests` + +- [ ] **Step 3: Push branch to remote** + +```bash +git push -u origin presentation/mode3-slides-e2e-tests +``` + +Expected: Branch pushed to remote with tracking set + +--- + +### Task 10: Setup KIND Cluster + +**Files:** +- None (infrastructure setup) + +- [ ] **Step 1: Check if KIND cluster exists** + +```bash +kind get clusters +``` + +Expected: List of existing clusters (may be empty) or `No kind clusters found.` + +- [ ] **Step 2: Delete existing cluster if present** + +```bash +kind delete cluster --name data-index-demo 2>/dev/null || true +``` + +Expected: Cluster deleted or no-op if not present + +- [ ] **Step 3: Create new KIND cluster** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/kind +./setup-cluster.sh +``` + +Expected output: +``` +Creating kind cluster... +✓ Control plane node ready +✓ Worker nodes ready +✓ Cluster created successfully +``` + +- [ ] **Step 4: Verify cluster is running** + +```bash +kubectl cluster-info --context kind-data-index-demo +``` + +Expected: +``` +Kubernetes control plane is running at https://127.0.0.1:... +CoreDNS is running at https://127.0.0.1:.../api/v1/namespaces/kube-system/services/kube-dns:dns/proxy +``` + +--- + +### Task 11: Deploy PostgreSQL and Data Index Service (MODE 1) + +**Files:** +- None (deployment operations) + +- [ ] **Step 1: Install PostgreSQL with Helm** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/kind +MODE=postgresql ./install-dependencies.sh +``` + +Expected output: +``` +Installing PostgreSQL... +✓ PostgreSQL Helm chart installed +✓ PostgreSQL pod running +✓ Database initialized +``` + +- [ ] **Step 2: Wait for PostgreSQL to be ready** + +```bash +kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=postgresql -n data-index --timeout=300s +``` + +Expected: `pod/postgresql-0 condition met` + +- [ ] **Step 3: Deploy Data Index service** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/kind +./deploy-data-index.sh postgresql +``` + +Expected output: +``` +Building data-index container image... +✓ Image built: kubesmarts/data-index-service:999-SNAPSHOT +Loading image to KIND cluster... +✓ Image loaded +Applying Kubernetes manifests... +✓ Namespace created +✓ ConfigMap created +✓ Deployment created +✓ Service created +``` + +- [ ] **Step 4: Wait for Data Index to be ready** + +```bash +kubectl wait --for=condition=ready pod -l app=data-index -n data-index --timeout=300s +``` + +Expected: `pod/data-index-... condition met` + +- [ ] **Step 5: Verify Data Index is healthy** + +```bash +kubectl port-forward -n data-index svc/data-index 8080:8080 & +sleep 3 +curl -f http://localhost:8080/q/health/ready +``` + +Expected: `{"status":"UP",...}` + +--- + +### Task 12: Deploy FluentBit DaemonSet (MODE 1) + +**Files:** +- None (deployment operations) + +- [ ] **Step 1: Generate FluentBit ConfigMap** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/fluentbit/mode1-postgresql-triggers +./generate-configmap.sh +``` + +Expected output: +``` +Generating FluentBit ConfigMap... +✓ ConfigMap generated: kubernetes/configmap.yaml +``` + +- [ ] **Step 2: Apply FluentBit ConfigMap** + +```bash +kubectl apply -f /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/fluentbit/mode1-postgresql-triggers/kubernetes/configmap.yaml +``` + +Expected: `configmap/workflows-fluent-bit-config created` + +- [ ] **Step 3: Deploy FluentBit DaemonSet** + +```bash +kubectl apply -f /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/fluentbit/mode1-postgresql-triggers/kubernetes/daemonset.yaml +``` + +Expected: `daemonset.apps/workflows-fluent-bit-mode1 created` + +- [ ] **Step 4: Verify FluentBit pods are running** + +```bash +kubectl wait --for=condition=ready pod -l app=workflows-fluent-bit-mode1 -n logging --timeout=120s +``` + +Expected: `pod/workflows-fluent-bit-mode1-... condition met` (one per node) + +--- + +### Task 13: Deploy Test Workflow Application + +**Files:** +- None (deployment operations) + +- [ ] **Step 1: Deploy test workflow app** + +```bash +cd /Users/ricferna/dev/github/kubesmarts/logic-apps/data-index/scripts/kind +./deploy-workflow-app.sh +``` + +Expected output: +``` +Building workflow-app container image... +✓ Image built +Loading image to KIND cluster... +✓ Image loaded +Deploying workflow app... +✓ Deployment created +✓ Service created +``` + +- [ ] **Step 2: Wait for workflow app to be ready** + +```bash +kubectl wait --for=condition=ready pod -l app=workflow-app -n workflows --timeout=120s +``` + +Expected: `pod/workflow-app-... condition met` + +- [ ] **Step 3: Trigger test workflow execution** + +```bash +kubectl port-forward -n workflows svc/workflow-app 8081:8080 & +sleep 3 +curl -X POST http://localhost:8081/workflows/test-workflow \ + -H "Content-Type: application/json" \ + -d '{"orderId": "12345", "customerId": "customer-1"}' +``` + +Expected: `{"id":"...", "status":"RUNNING"}` + +- [ ] **Step 4: Wait for events to be processed** + +```bash +sleep 5 +``` + +Expected: FluentBit ingests logs, triggers normalize, data appears in PostgreSQL + +--- + +### Task 14: Verify E2E Flow with GraphQL Queries + +**Files:** +- None (verification) + +- [ ] **Step 1: Query workflow instances via GraphQL** + +```bash +curl -X POST http://localhost:8080/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ getWorkflowInstances { id name status start } }" + }' +``` + +Expected: +```json +{ + "data": { + "getWorkflowInstances": [ + { + "id": "...", + "name": "test-workflow", + "status": "COMPLETED", + "start": "2026-05-22T..." + } + ] + } +} +``` + +- [ ] **Step 2: Query task executions for workflow** + +```bash +curl -X POST http://localhost:8080/graphql \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ getWorkflowInstances { id taskExecutions { id name status } } }" + }' +``` + +Expected: Workflow with nested task executions (at least 1 task) + +- [ ] **Step 3: Verify data normalization (check PostgreSQL directly)** + +```bash +kubectl exec -it -n data-index postgresql-0 -- psql -U postgres -d dataindex -c "SELECT id, name, status, start FROM workflow_instances;" +``` + +Expected: Table with workflow instance data matching GraphQL query + +- [ ] **Step 4: Check FluentBit logs for ingestion** + +```bash +kubectl logs -n logging -l app=workflows-fluent-bit-mode1 --tail=50 | grep -i "workflow\|task" +``` + +Expected: Log lines showing FluentBit processing workflow/task events + +--- + +### Task 15: Leave Cluster Ready for Presentation + +**Files:** +- None (verification) + +- [ ] **Step 1: Stop port-forward processes** + +```bash +pkill -f "kubectl port-forward" || true +``` + +Expected: All port-forward processes killed + +- [ ] **Step 2: Verify all components are healthy** + +```bash +kubectl get pods -A | grep -E "data-index|postgresql|fluent-bit|workflow-app" +``` + +Expected: All pods in `Running` state, `READY` column shows all containers ready (e.g., `1/1`) + +- [ ] **Step 3: Create quick reference for presentation** + +```bash +cat > /Users/ricferna/dev/github/kubesmarts/logic-apps/PRESENTATION_QUICK_START.md <<'EOF' +# Data Index Presentation - Quick Start + +**Cluster:** KIND `data-index-demo` +**Date:** 2026-05-22 + +## Components Running + +- PostgreSQL (data-index namespace) +- Data Index Service (data-index namespace, MODE 1) +- FluentBit DaemonSet (logging namespace, MODE 1) +- Workflow App (workflows namespace) + +## Demo Commands + +### 1. Port-forward Data Index +```bash +kubectl port-forward -n data-index svc/data-index 8080:8080 & +``` + +### 2. Trigger workflow +```bash +kubectl port-forward -n workflows svc/workflow-app 8081:8080 & +curl -X POST http://localhost:8081/workflows/test-workflow \ + -H "Content-Type: application/json" \ + -d '{"orderId": "99999", "customerId": "demo-customer"}' +``` + +### 3. Query via GraphQL +```bash +curl -X POST http://localhost:8080/graphql \ + -H "Content-Type: application/json" \ + -d '{"query": "{ getWorkflowInstances { id name status start taskExecutions { id name status } } }"}' +``` + +### 4. Watch FluentBit logs +```bash +kubectl logs -n logging -l app=workflows-fluent-bit-mode1 -f +``` + +### 5. Check PostgreSQL +```bash +kubectl exec -it -n data-index postgresql-0 -- psql -U postgres -d dataindex +\dt +SELECT * FROM workflow_instances; +``` + +## Slides + +Presentation slides: `/Users/ricferna/dev/github/kubesmarts/logic-apps/presentations/2026-05-20-data-index-poc/output/` + +Total slides: 18 (MODE 1, MODE 2, MODE 3 architecture + comparison) + +## Cleanup (after presentation) + +```bash +kind delete cluster --name data-index-demo +``` +EOF +``` + +Expected: Quick reference file created + +- [ ] **Step 4: Verify quick reference file** + +```bash +cat /Users/ricferna/dev/github/kubesmarts/logic-apps/PRESENTATION_QUICK_START.md | head -20 +``` + +Expected: Quick reference content visible + +- [ ] **Step 5: Final commit** + +```bash +git add PRESENTATION_QUICK_START.md +git commit -m "docs: add presentation quick start guide + +- KIND cluster setup and running +- MODE 1 e2e tests passing +- GraphQL queries verified +- FluentBit ingestion working +- Demo commands ready for presentation + +Related to #23" +``` + +--- + +## Execution Summary + +**Part 1: Presentation Updates (Tasks 1-8)** +- 3 new slides (MODE 3 architecture, three-way comparison, decision framework) +- Updated slide generator and documentation +- Total slides: 18 (was 15) +- Duration: 35-50 minutes (was 30-45) + +**Part 2: E2E Tests on KIND (Tasks 9-15)** +- KIND cluster created and configured +- MODE 1 deployed: PostgreSQL + FluentBit + Data Index + Workflow App +- E2E flow verified: Workflow execution → FluentBit → PostgreSQL triggers → GraphQL API +- Quick reference guide created for presentation demo + +**Deliverables:** +- ✅ MODE 3 slides added to presentation +- ✅ E2E tests running on KIND +- ✅ Cluster ready for live demo +- ✅ Quick start guide for presentation + +**Branch:** `presentation/mode3-slides-e2e-tests` + +**Related Issue:** #23 - Implement Kafka-based event ingestion service for PostgreSQL backend (MODE 3) From 7da96a666c6ffb7f7677a40747ff367d880bc5cc Mon Sep 17 00:00:00 2001 From: Ricardo Zanini Date: Wed, 26 Aug 2026 14:26:22 -0400 Subject: [PATCH 3/3] feat: remove redundant task_execution_id column Remove task_execution_id column from task_instances table across all modes. TaskExecution.id is now derived from instanceId + taskPosition composite key. Database (MODE 1 - PostgreSQL): - Update V1__initial_schema.sql to remove task_execution_id column - Update trigger function to remove task_execution_id from INSERT/UPDATE - Primary key remains (instance_id, task_position) Domain Model: - TaskExecution.getId() now derives ID from instanceId + taskPosition - Format: "instanceId:taskPosition" JPA (MODE 1): - TaskInstanceEntity uses composite @Id on instanceId and taskPosition - Update equals() and hashCode() to use composite key - TaskInstanceEntityMapper updated for derived ID - TaskExecutionJPAStorage.get() parses derived ID and queries by composite key Kafka Ingestion (MODE 3): - Remove generateTaskExecutionId() from Mapper and TaskExecutionProcessor - Update task-instance-upsert.sql to remove task_execution_id column - Update TaskPersistence.setTaskParameters() for new column count Elasticsearch (MODE 2): - Remove taskExecutionId from task-events.json index template GraphQL API: - TaskExecutionJPAStorage.get() parses "instanceId:taskPosition" format - Queries by composite key (instance_id, task_position) Tests: - Remove setTaskExecutionId() calls from WorkflowInstanceGraphQLApiTest - ID now derived automatically from instanceId + taskPosition Documentation: - Update CLAUDE.md field mapping table - Update composite key section to reflect column removal - Add guidance on derived ID format Fixes #31 --- CLAUDE.md | 33 ++++++++------- .../processor/TaskExecutionProcessor.java | 11 ----- .../persistence/TaskPersistence.java | 36 ++++++++--------- .../resources/sql/task-instance-upsert.sql | 4 +- .../ingestion/kafka/service/Mapper.java | 9 ----- .../WorkflowInstanceGraphQLApiTest.java | 3 -- .../logic/dataindex/model/TaskExecution.java | 17 +++++++- .../index-templates/task-events.json | 8 ---- .../db/migration/V1__initial_schema.sql | 15 +------ .../storage/jpa/TaskExecutionJPAStorage.java | 38 ++++++++++++++++++ .../jpa/entity/TaskInstanceEntity.java | 40 +++++++------------ .../jpa/mapper/TaskInstanceEntityMapper.java | 10 ++--- 12 files changed, 112 insertions(+), 112 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 469edc9ba..9157d7f43 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -397,32 +397,32 @@ task.completed event: taskExecutionId = 97469968-64b5-3cdd-b809-e58e4b6e3be7 - ON CONFLICT: `(task_execution_id)` → never matches - Result: Always INSERTs new row (4 rows for 2 tasks) -**Solution (V2 Migration):** +**Current Solution:** - Primary key: `(instance_id, task_position)` - composite key - ON CONFLICT: `(instance_id, task_position)` → matches on same task - Result: UPDATEs existing row on subsequent events (2 rows for 2 tasks) +- **No task_execution_id column** - ID is derived from composite key **Rationale:** - `(instance_id, task_position)` uniquely identifies a task execution within a workflow - Same workflow + same position = same task (regardless of changing taskExecutionId) - Enables proper UPSERT behavior despite Quarkus Flow's ID generation +- TaskExecution.id is derived as `instanceId + ":" + taskPosition` -**Migration:** `V2__fix_task_instances_composite_key.sql` -- Drops old PK on `task_execution_id` -- Adds composite PK on `(instance_id, task_position)` -- Updates trigger function to use composite key for conflict resolution -- Cleans up existing duplicates (keeps latest by `last_event_time`) -- Adds index on `task_execution_id` for GraphQL queries by ID +**Schema (V1):** +- Composite PK on `(instance_id, task_position)` +- Trigger function uses composite key for conflict resolution +- No task_execution_id column (derived in application layer) **DO:** - ✅ Use composite key `(instance_id, task_position)` for task identity -- ✅ Keep `task_execution_id` as a regular column (still exposed in GraphQL) +- ✅ Derive TaskExecution.id from `instanceId + ":" + taskPosition` - ✅ Trust `task_position` to be stable across task lifecycle **DON'T:** -- ❌ Don't rely on `taskExecutionId` for UPSERT logic (it changes per event) +- ❌ Don't add task_execution_id column to database +- ❌ Don't rely on Quarkus Flow's taskExecutionId for identity (it changes per event) - ❌ Don't assume Quarkus Flow will fix this (working as designed for their use case) -- ❌ Don't remove `task_execution_id` column (GraphQL API still needs it) ### Metrics & Observability @@ -474,16 +474,19 @@ data-index.metrics.transform.poll-interval=30s **Database columns → JPA Entity → GraphQL:** ``` -task_instances.task_execution_id → TaskInstanceEntity.taskExecutionId → TaskExecution.id -task_instances.start → TaskInstanceEntity.start → TaskExecution.start (@JsonProperty("startDate")) -task_instances.end → TaskInstanceEntity.end → TaskExecution.end (@JsonProperty("endDate")) -task_instances.input → TaskInstanceEntity.input → TaskExecution.input (@Ignore, exposed via getInputData()) +task_instances.instance_id → TaskInstanceEntity.instanceId → TaskExecution.instanceId +task_instances.task_position → TaskInstanceEntity.taskPosition → TaskExecution.taskPosition +(derived ID) → TaskInstanceEntity.getId() → TaskExecution.getId() (returns "instanceId:taskPosition") +task_instances.start → TaskInstanceEntity.start → TaskExecution.start (@JsonProperty("startDate")) +task_instances.end → TaskInstanceEntity.end → TaskExecution.end (@JsonProperty("endDate")) +task_instances.input → TaskInstanceEntity.input → TaskExecution.input (@Ignore, exposed via getInputData()) ``` **IMPORTANT:** +- TaskExecution.id is derived from instanceId + taskPosition (no database column) - Database uses `start`/`end` (reserved SQL keywords, quoted) - GraphQL uses `startDate`/`endDate` (via `@JsonProperty`) -- Never use old names: `enter`/`exit`, `triggerTime`/`leaveTime` +- Never use old names: `enter`/`exit`, `triggerTime`/`leaveTime`, `taskExecutionId` ### Error Handling diff --git a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/TaskExecutionProcessor.java b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/TaskExecutionProcessor.java index 9d3290980..79d72598c 100644 --- a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/TaskExecutionProcessor.java +++ b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/TaskExecutionProcessor.java @@ -55,11 +55,6 @@ public void processBatch(List events) { log.debug("Processing task batch size: {}", events.size()); try { - for (TaskExecution event : events) { - Objects.requireNonNull(event, "event cannot be null"); - event.setId(generateTaskExecutionId(event)); - } - this.taskPersistence.persistBatch(events); log.debug("Successfully processed task batch size: {}", events.size()); @@ -68,10 +63,4 @@ public void processBatch(List events) { throw new ProcessEventFailedException("Failed to process task event batch", e); } } - - private String generateTaskExecutionId(TaskExecution taskExecutionEvent) { - return taskExecutionEvent.getInstanceId() - + ":" - + taskExecutionEvent.getTaskPosition(); - } } diff --git a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/persistence/TaskPersistence.java b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/persistence/TaskPersistence.java index d3d81a769..2fba4574d 100644 --- a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/persistence/TaskPersistence.java +++ b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/processor/persistence/TaskPersistence.java @@ -140,33 +140,31 @@ private void tryCreatePlaceholder(TaskExecution event, Connection conn) throws S private void setTaskParameters(PreparedStatement stmt, TaskExecution event) throws SQLException { - stmt.setString(1, event.getId()); - stmt.setString(2, event.getInstanceId()); - stmt.setString(3, event.getTaskName()); - stmt.setString(4, event.getTaskPosition()); - stmt.setString(5, event.getStatus()); - stmt.setObject(6, Optional.ofNullable(event.getStart()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); - stmt.setObject(7, Optional.ofNullable(event.getEnd()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); - stmt.setString(8, toJsonString(event.getInput())); - stmt.setString(9, toJsonString(event.getOutput())); - + stmt.setString(1, event.getInstanceId()); + stmt.setString(2, event.getTaskName()); + stmt.setString(3, event.getTaskPosition()); + stmt.setString(4, event.getStatus()); + stmt.setObject(5, Optional.ofNullable(event.getStart()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); + stmt.setObject(6, Optional.ofNullable(event.getEnd()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); + stmt.setString(7, toJsonString(event.getInput())); + stmt.setString(8, toJsonString(event.getOutput())); // Error fields if (event.getError() != null) { - stmt.setString(10, event.getError().getType()); - stmt.setString(11, event.getError().getTitle()); - stmt.setString(12, event.getError().getDetail()); - stmt.setObject(13, event.getError().getStatus()); - stmt.setString(14, event.getError().getInstance()); + stmt.setString(9, event.getError().getType()); + stmt.setString(10, event.getError().getTitle()); + stmt.setString(11, event.getError().getDetail()); + stmt.setObject(12, event.getError().getStatus()); + stmt.setString(13, event.getError().getInstance()); } else { + stmt.setNull(9, java.sql.Types.VARCHAR); stmt.setNull(10, java.sql.Types.VARCHAR); stmt.setNull(11, java.sql.Types.VARCHAR); - stmt.setNull(12, java.sql.Types.VARCHAR); - stmt.setNull(13, java.sql.Types.INTEGER); - stmt.setNull(14, java.sql.Types.VARCHAR); + stmt.setNull(12, java.sql.Types.INTEGER); + stmt.setNull(13, java.sql.Types.VARCHAR); } - stmt.setObject(15, Optional.ofNullable(event.getEventTimestamp()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); + stmt.setObject(14, Optional.ofNullable(event.getEventTimestamp()).map(ZonedDateTime::toOffsetDateTime).orElse(null)); } private String toJsonString(JsonNode node) { diff --git a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/resources/sql/task-instance-upsert.sql b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/resources/sql/task-instance-upsert.sql index 59828f062..4f16a2513 100644 --- a/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/resources/sql/task-instance-upsert.sql +++ b/data-index/data-index-ingestion/data-index-ingestion-kafka-processor/src/main/resources/sql/task-instance-upsert.sql @@ -1,9 +1,9 @@ INSERT INTO task_instances ( - task_execution_id, instance_id, task_name, task_position, status, + instance_id, task_name, task_position, status, start, "end", input, output, error_type, error_title, error_detail, error_status, error_instance, last_event_time, created_at, updated_at -) VALUES (?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?::jsonb, ?, ?, ?, ?, ?, ?, NOW(), NOW()) +) VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?::jsonb, ?, ?, ?, ?, ?, ?, NOW(), NOW()) ON CONFLICT (instance_id, task_position) DO UPDATE SET instance_id = COALESCE(EXCLUDED.instance_id, task_instances.instance_id), task_name = COALESCE(EXCLUDED.task_name, task_instances.task_name), diff --git a/data-index/data-index-ingestion/data-index-ingestion-kafka-service/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/service/Mapper.java b/data-index/data-index-ingestion/data-index-ingestion-kafka-service/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/service/Mapper.java index dca727aaa..ea73f1962 100644 --- a/data-index/data-index-ingestion/data-index-ingestion-kafka-service/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/service/Mapper.java +++ b/data-index/data-index-ingestion/data-index-ingestion-kafka-service/src/main/java/org/kubesmarts/logic/dataindex/ingestion/kafka/service/Mapper.java @@ -24,9 +24,6 @@ import org.kubesmarts.logic.dataindex.model.TaskExecution; import org.kubesmarts.logic.dataindex.model.WorkflowInstance; import org.kubesmarts.logic.dataindex.model.WorkflowInstanceStatus; -import java.time.OffsetDateTime; -import java.util.UUID; - public final class Mapper { private Mapper() {} @@ -91,7 +88,6 @@ public static TaskExecution mapTaskExecutionEvent(CloudEvent cloudEvent, TaskCED taskExecution.setInstanceId(instanceId); taskExecution.setEventTimestamp(cloudEvent.getTime().toZonedDateTime()); taskExecution.setStatus(status); - taskExecution.setId(generateTaskExecutionId(instanceId, taskPosition, cloudEvent.getTime())); taskExecution.setTaskPosition(taskPosition); taskExecution.setTaskName(taskPosition.substring(taskPosition.lastIndexOf("/") + 1)); @@ -141,9 +137,4 @@ private static Error mapError(WorkflowError workflowError) { error.setInstance(workflowError.instance()); return error; } - - private static String generateTaskExecutionId(String instanceId, String taskPosition, OffsetDateTime time) { - String composite = instanceId + ":" + taskPosition + ":" + time.toInstant().toEpochMilli(); - return UUID.nameUUIDFromBytes(composite.getBytes()).toString(); - } } diff --git a/data-index/data-index-integration-tests/data-index-integration-tests-postgresql/src/test/java/org/kubesmarts/logic/dataindex/graphql/WorkflowInstanceGraphQLApiTest.java b/data-index/data-index-integration-tests/data-index-integration-tests-postgresql/src/test/java/org/kubesmarts/logic/dataindex/graphql/WorkflowInstanceGraphQLApiTest.java index 2aed4f65a..836133e90 100644 --- a/data-index/data-index-integration-tests/data-index-integration-tests-postgresql/src/test/java/org/kubesmarts/logic/dataindex/graphql/WorkflowInstanceGraphQLApiTest.java +++ b/data-index/data-index-integration-tests/data-index-integration-tests-postgresql/src/test/java/org/kubesmarts/logic/dataindex/graphql/WorkflowInstanceGraphQLApiTest.java @@ -94,7 +94,6 @@ public void setupTestData() throws Exception { List tasks1 = new ArrayList<>(); TaskInstanceEntity task1 = new TaskInstanceEntity(); - task1.setTaskExecutionId("task-1-1"); task1.setInstanceId(TEST_WORKFLOW_ID_1); task1.setTaskName("validateInput"); task1.setTaskPosition("/do/0"); @@ -107,7 +106,6 @@ public void setupTestData() throws Exception { tasks1.add(task1); TaskInstanceEntity task2 = new TaskInstanceEntity(); - task2.setTaskExecutionId("task-1-2"); task2.setInstanceId(TEST_WORKFLOW_ID_1); task2.setTaskName("processData"); task2.setTaskPosition("/do/1"); @@ -144,7 +142,6 @@ public void setupTestData() throws Exception { List tasks2 = new ArrayList<>(); TaskInstanceEntity task3 = new TaskInstanceEntity(); - task3.setTaskExecutionId("task-2-1"); task3.setInstanceId(TEST_WORKFLOW_ID_2); task3.setTaskName("failingTask"); task3.setTaskPosition("/do/0"); diff --git a/data-index/data-index-model/src/main/java/org/kubesmarts/logic/dataindex/model/TaskExecution.java b/data-index/data-index-model/src/main/java/org/kubesmarts/logic/dataindex/model/TaskExecution.java index 3d6c61cbe..e560e474e 100644 --- a/data-index/data-index-model/src/main/java/org/kubesmarts/logic/dataindex/model/TaskExecution.java +++ b/data-index/data-index-model/src/main/java/org/kubesmarts/logic/dataindex/model/TaskExecution.java @@ -98,10 +98,25 @@ public class TaskExecution { */ private ZonedDateTime eventTimestamp; + /** + * Get derived task execution ID. + *

ID is derived from instanceId + taskPosition (format: "instanceId:taskPosition") + *

No database column - computed from composite key + */ public String getId() { - return id; + if (id != null) { + return id; + } + if (instanceId != null && taskPosition != null) { + return instanceId + ":" + taskPosition; + } + return null; } + /** + * Set task execution ID. + *

For deserialization only - ID is normally derived from instanceId + taskPosition + */ public void setId(String id) { this.id = id; } diff --git a/data-index/data-index-storage/data-index-storage-elasticsearch-schema/src/main/resources/elasticsearch/index-templates/task-events.json b/data-index/data-index-storage/data-index-storage-elasticsearch-schema/src/main/resources/elasticsearch/index-templates/task-events.json index b4cdea8f7..8dc6ba400 100644 --- a/data-index/data-index-storage/data-index-storage-elasticsearch-schema/src/main/resources/elasticsearch/index-templates/task-events.json +++ b/data-index/data-index-storage/data-index-storage-elasticsearch-schema/src/main/resources/elasticsearch/index-templates/task-events.json @@ -31,14 +31,6 @@ } } }, - "taskExecutionId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, "taskPosition": { "type": "text", "fields": { diff --git a/data-index/data-index-storage/data-index-storage-migrations/src/main/resources/db/migration/V1__initial_schema.sql b/data-index/data-index-storage/data-index-storage-migrations/src/main/resources/db/migration/V1__initial_schema.sql index 4cc9d2a65..14c9f5b51 100644 --- a/data-index/data-index-storage/data-index-storage-migrations/src/main/resources/db/migration/V1__initial_schema.sql +++ b/data-index/data-index-storage/data-index-storage-migrations/src/main/resources/db/migration/V1__initial_schema.sql @@ -64,7 +64,6 @@ CREATE INDEX IF NOT EXISTS idx_workflow_instances_start ON workflow_instances (s CREATE INDEX IF NOT EXISTS idx_workflow_instances_last_event_time ON workflow_instances (last_event_time DESC); CREATE TABLE IF NOT EXISTS task_instances ( - task_execution_id VARCHAR(255), instance_id VARCHAR(255) NOT NULL, task_name VARCHAR(255), task_position VARCHAR(255) NOT NULL, @@ -82,8 +81,7 @@ CREATE TABLE IF NOT EXISTS task_instances ( created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), -- Composite primary key: (instance_id, task_position) uniquely identifies a task - -- Quarkus Flow generates different taskExecutionId per event (started, completed, etc.) - -- Same workflow + same position = same task execution + -- TaskExecution.id is derived from instanceId + taskPosition (no column needed) PRIMARY KEY (instance_id, task_position), CONSTRAINT fk_task_instance_workflow FOREIGN KEY (instance_id) REFERENCES workflow_instances(id) ON DELETE CASCADE ); @@ -91,7 +89,6 @@ CREATE TABLE IF NOT EXISTS task_instances ( CREATE INDEX IF NOT EXISTS idx_task_instances_instance_id ON task_instances (instance_id); CREATE INDEX IF NOT EXISTS idx_task_instances_status ON task_instances (status); CREATE INDEX IF NOT EXISTS idx_task_instances_last_event_time ON task_instances (last_event_time DESC); -CREATE INDEX IF NOT EXISTS idx_task_instances_task_execution_id ON task_instances (task_execution_id); -- ============================================================================ -- TRIGGER FUNCTIONS (Extract from JSONB and normalize with idempotency) @@ -218,7 +215,6 @@ BEGIN -- instance already exists because workflow events have already created it. BEGIN INSERT INTO task_instances ( - task_execution_id, instance_id, task_name, task_position, @@ -236,7 +232,6 @@ BEGIN created_at, updated_at ) VALUES ( - NEW.data->>'taskExecutionId', NEW.data->>'instanceId', NEW.data->>'taskName', NEW.data->>'taskPosition', @@ -267,9 +262,6 @@ BEGIN NEW.time ) ON CONFLICT (instance_id, task_position) DO UPDATE SET - -- Update task_execution_id to the latest (even though it changes per event) - task_execution_id = EXCLUDED.task_execution_id, - -- Keep first non-null values (immutable fields) task_name = COALESCE(task_instances.task_name, EXCLUDED.task_name), @@ -318,7 +310,6 @@ BEGIN ON CONFLICT (id) DO NOTHING; INSERT INTO task_instances ( - task_execution_id, instance_id, task_name, task_position, @@ -336,7 +327,6 @@ BEGIN created_at, updated_at ) VALUES ( - NEW.data->>'taskExecutionId', NEW.data->>'instanceId', NEW.data->>'taskName', NEW.data->>'taskPosition', @@ -367,9 +357,6 @@ BEGIN NEW.time ) ON CONFLICT (instance_id, task_position) DO UPDATE SET - -- Update task_execution_id to the latest (even though it changes per event) - task_execution_id = EXCLUDED.task_execution_id, - -- Keep first non-null values (immutable fields) task_name = COALESCE(task_instances.task_name, EXCLUDED.task_name), diff --git a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/TaskExecutionJPAStorage.java b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/TaskExecutionJPAStorage.java index 150093878..4897c5729 100644 --- a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/TaskExecutionJPAStorage.java +++ b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/TaskExecutionJPAStorage.java @@ -62,6 +62,44 @@ protected TaskExecutionJPAStorage() { super(); } + /** + * Get task execution by derived ID. + *

ID format: "instanceId:taskPosition" + *

Parses the composite ID and queries by both instanceId and taskPosition. + * + * @param id Derived ID in format "instanceId:taskPosition" + * @return TaskExecution or null if not found + */ + @Override + public TaskExecution get(String id) { + if (id == null || id.isEmpty()) { + return null; + } + + // Parse composite ID: "instanceId:taskPosition" + int separatorIndex = id.indexOf(':'); + if (separatorIndex == -1 || separatorIndex == 0 || separatorIndex == id.length() - 1) { + throw new IllegalArgumentException("Invalid task execution ID format. Expected 'instanceId:taskPosition', got: " + id); + } + + String instanceId = id.substring(0, separatorIndex); + String taskPosition = id.substring(separatorIndex + 1); + + // Query by composite key + List entities = em + .createQuery("SELECT t FROM TaskInstanceEntity t WHERE t.instanceId = :instanceId AND t.taskPosition = :taskPosition", TaskInstanceEntity.class) + .setParameter("instanceId", instanceId) + .setParameter("taskPosition", taskPosition) + .setMaxResults(1) + .getResultList(); + + if (entities.isEmpty()) { + return null; + } + + return mapToModel.apply(entities.get(0)); + } + @Override public List findByWorkflowInstanceId(String workflowInstanceId) { List entities = em diff --git a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/entity/TaskInstanceEntity.java b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/entity/TaskInstanceEntity.java index 9c4e69473..6a5d5fd5b 100644 --- a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/entity/TaskInstanceEntity.java +++ b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/entity/TaskInstanceEntity.java @@ -57,19 +57,11 @@ public class TaskInstanceEntity extends AbstractEntity { /** - * Task execution ID (primary key). - *

Source: taskExecutionId from Quarkus Flow events - *

Extracted by trigger from: data->>'taskExecutionId' - */ - @Id - @Column(name = "task_execution_id") - private String taskExecutionId; - - /** - * Workflow instance ID (foreign key). + * Workflow instance ID (part of composite primary key). *

Source: instanceId from Quarkus Flow events *

Extracted by trigger from: data->>'instanceId' */ + @Id @Column(name = "instance_id", nullable = false) private String instanceId; @@ -81,11 +73,13 @@ public class TaskInstanceEntity extends AbstractEntity { private String taskName; /** - * Task position in workflow document (JSONPointer). + * Task position in workflow document (JSONPointer, part of composite primary key). *

Source: taskPosition from Quarkus Flow task events *

Extracted by trigger from: data->>'taskPosition' *

Examples: "do/0/set-0", "fork/branches/0/do/1" */ + @Id + @Column(name = "task_position", nullable = false) private String taskPosition; /** @@ -164,15 +158,11 @@ public class TaskInstanceEntity extends AbstractEntity { @Override public String getId() { - return taskExecutionId; - } - - public String getTaskExecutionId() { - return taskExecutionId; - } - - public void setTaskExecutionId(String taskExecutionId) { - this.taskExecutionId = taskExecutionId; + // Return derived ID from composite key + if (instanceId != null && taskPosition != null) { + return instanceId + ":" + taskPosition; + } + return null; } public String getInstanceId() { @@ -280,21 +270,21 @@ public boolean equals(Object o) { return false; } TaskInstanceEntity that = (TaskInstanceEntity) o; - return Objects.equals(taskExecutionId, that.taskExecutionId); + return Objects.equals(instanceId, that.instanceId) && + Objects.equals(taskPosition, that.taskPosition); } @Override public int hashCode() { - return Objects.hash(taskExecutionId); + return Objects.hash(instanceId, taskPosition); } @Override public String toString() { return "TaskInstanceEntity{" + - "taskExecutionId='" + taskExecutionId + '\'' + - ", instanceId='" + instanceId + '\'' + - ", taskName='" + taskName + '\'' + + "instanceId='" + instanceId + '\'' + ", taskPosition='" + taskPosition + '\'' + + ", taskName='" + taskName + '\'' + ", status='" + status + '\'' + ", start=" + start + ", end=" + end + diff --git a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/mapper/TaskInstanceEntityMapper.java b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/mapper/TaskInstanceEntityMapper.java index 729efb8d8..6f7959c1a 100644 --- a/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/mapper/TaskInstanceEntityMapper.java +++ b/data-index/data-index-storage/data-index-storage-postgresql/src/main/java/org/kubesmarts/logic/dataindex/storage/jpa/mapper/TaskInstanceEntityMapper.java @@ -37,8 +37,8 @@ public interface TaskInstanceEntityMapper { /** * Convert JPA entity to domain model. * Used when reading from database to return via GraphQL API. + * ID is derived from instanceId + taskPosition in both entity and model. */ - @Mapping(target = "id", source = "taskExecutionId") @Mapping(target = "start", source = "start") @Mapping(target = "end", source = "end") @Mapping(target = "input", source = "input") @@ -49,8 +49,8 @@ public interface TaskInstanceEntityMapper { * Convert domain model to JPA entity. * Used when writing to database (though Data Index v1.0.0 is read-only, this may be used for tests). */ - @Mapping(target = "taskExecutionId", source = "id") - @Mapping(target = "instanceId", ignore = true) // Will be set from relationship + @Mapping(target = "instanceId", source = "instanceId") + @Mapping(target = "taskPosition", source = "taskPosition") @Mapping(target = "workflowInstance", ignore = true) // Will be set by relationship @Mapping(target = "createdAt", ignore = true) @Mapping(target = "updatedAt", ignore = true) @@ -60,8 +60,8 @@ public interface TaskInstanceEntityMapper { * Update existing entity from model. * Useful for merge operations. */ - @Mapping(target = "taskExecutionId", ignore = true) // Primary key, don't update - @Mapping(target = "instanceId", ignore = true) + @Mapping(target = "instanceId", ignore = true) // Primary key, don't update + @Mapping(target = "taskPosition", ignore = true) // Primary key, don't update @Mapping(target = "workflowInstance", ignore = true) @Mapping(target = "createdAt", ignore = true) @Mapping(target = "updatedAt", ignore = true)