From c709a6b5643fe9b33a5e899ed8a5178b36e48d85 Mon Sep 17 00:00:00 2001 From: redshift <213178690+sh1ftred@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:47:08 +0000 Subject: [PATCH 01/47] Chat sync built with applesauce and pns storage. (#121) * updated Topup container UI and now working on new chat sync. * beofre swtiching to pns * event creation and publishng works. now syncing * eventSync should be working now. * Chat sync works manually. * added copy button for msgs * chat sync pubkey not found issue * fixed the state bug * could fetch URLs!@ * real time syncing now works * modularized and events loading works now. but doesn't show up on sidebar and also title selection is broken. * live sync works. needs more robust testing and ephemeral pns keys. * option to disable chat sync. * option for local only storage. * local messages are persistent and fixed sorting bugs. * a version that isn't working. local to cloud sync * fixed the chat sync toggle bug * fully applesuace integrated with negentropy * sync is perfect. fixing live sync * now it syncs after every message. * live sync works great * Update * remove local storage altogether * fixing new conv createion * fully using conv ref rn * made single account login at all times. * checkpoint * Fix visible
tags in tables * Filter out staging providers in production across ApiKeysTab, ModelsTab, and useApiState hook * sync with derivedPnsKeys * finding EOSE * publishing 1081 event * pnskeys sync works now. * checkout: syncing with new pnskeys * temp * sync works end toend. only the order isn't perfect yet. * fixed sorting of events. itworks great now. * fixing event syncing with manual trigger. * debugging. * handling messgae edits * chekc * hopefully fixed stale sync * fixed editing * fixed build --------- Co-authored-by: redshift <213178690+1ftredsh@users.noreply.github.com> Co-authored-by: Evan Yang --- CHAT_SYNC.md | 105 ++++ INCREMENTAL_SYNC_ARCHITECTURE.md | 520 +++++++++++++++++ REFACTORING.md | 632 --------------------- ZUSTAND_EVENT_DATABASE_ARCHITECTURE.md | 584 +++++++++++++++++++ app/page.tsx | 12 +- components/ClientProviders.tsx | 11 +- components/LoginModal.tsx | 12 +- components/NostrProvider.tsx | 6 +- components/SettingsModal.tsx | 1 - components/TopUpPromptModal.tsx | 36 +- components/chat/ChatContainer.tsx | 8 +- components/chat/ChatMessages.tsx | 176 +++++- components/chat/MainChatArea.tsx | 4 - components/chat/Sidebar.tsx | 24 +- components/settings/GeneralTab.tsx | 35 +- components/settings/NostrRelayManager.tsx | 2 +- context/ChatProvider.tsx | 39 +- hooks/useApiState.ts | 1 - hooks/useCashuWithXYZ.ts | 5 +- hooks/useChatActions.ts | 47 +- hooks/useChatHistorySync.ts | 1 + hooks/useChatSync.ts | 190 +++++++ hooks/useChatSync1081.ts | 657 ++++++++++++++++++++++ hooks/useConversationState.ts | 187 ++++-- lib/applesauce-core.ts | 14 +- lib/eventDatabase/README.md | 229 ++++++++ lib/eventDatabase/eventStore.ts | 305 ++++++++++ lib/eventDatabase/filters.ts | 102 ++++ lib/eventDatabase/index.ts | 48 ++ lib/eventDatabase/replaceables.ts | 71 +++ lib/eventDatabase/types.ts | 69 +++ lib/nostr.ts | 4 + lib/pns.ts | 86 +++ mocks/handlers.ts | 11 + next.config.ts | 5 +- tsconfig.tsbuildinfo | 2 +- types/chat.ts | 4 + utils/apiUtils.ts | 1 - utils/cashuUtils.ts | 1 - utils/conversationUtils.ts | 175 +++++- utils/eventProcessing.ts | 333 +++++++++++ utils/messageUtils.ts | 4 +- utils/storageManager.ts | 201 +++++++ utils/storageUtils.ts | 18 +- 44 files changed, 4229 insertions(+), 749 deletions(-) create mode 100644 CHAT_SYNC.md create mode 100644 INCREMENTAL_SYNC_ARCHITECTURE.md delete mode 100644 REFACTORING.md create mode 100644 ZUSTAND_EVENT_DATABASE_ARCHITECTURE.md create mode 100644 hooks/useChatSync.ts create mode 100644 hooks/useChatSync1081.ts create mode 100644 lib/eventDatabase/README.md create mode 100644 lib/eventDatabase/eventStore.ts create mode 100644 lib/eventDatabase/filters.ts create mode 100644 lib/eventDatabase/index.ts create mode 100644 lib/eventDatabase/replaceables.ts create mode 100644 lib/eventDatabase/types.ts create mode 100644 lib/pns.ts create mode 100644 utils/eventProcessing.ts create mode 100644 utils/storageManager.ts diff --git a/CHAT_SYNC.md b/CHAT_SYNC.md new file mode 100644 index 00000000..37f94f38 --- /dev/null +++ b/CHAT_SYNC.md @@ -0,0 +1,105 @@ +# Chat Sync Feature Implementation Plan + +## Overview +This document outlines the implementation of a robust chat sync feature for the Routstr chat application using Nostr and PNS (Private Nostor Storage). + +## Requirements + +### 2. Negentropy-based Event Filtering +- Create a list of known events +- Use negentropy protocol to only sync events not already in the local list +- Avoid duplicate syncing and reduce bandwidth usage + +### 3. Message Sync for Missing EventIds +- Create function to identify messages in storage that don't have eventIds +- Sync these messages to Nostr one by one with appropriate delays +- Handle rate limiting and network issues gracefully + +### 6. PNS KEYS CANNOT BE CREATED WITH OTHER LOGINS> HELP!! + +## Implementation Details + +### Core Components + +#### 1. Enhanced useChatSync Hook +- Add pagination support for initial sync +- Implement negentropy filtering +- Add real-time subscription handling +- Improve error handling and retry logic + +#### 2. Event Management Utilities +- Functions to track known events +- Negentropy implementation for efficient syncing +- Message-to-event mapping utilities + +#### 3. Sync Status Management +- Track sync progress and status +- Provide user feedback for sync operations +- Handle offline/online scenarios + +### Technical Considerations + +#### Performance +- Implement efficient pagination for large histories +- Use negentropy to minimize data transfer +- Cache frequently accessed data + +#### Reliability +- Add retry mechanisms for failed operations +- Handle network interruptions gracefully +- Implement proper error recovery + +#### User Experience +- Show sync status indicators +- Provide feedback for sync operations +- Handle background syncing seamlessly + +## Implementation Steps + +1. **Update CHAT_SYNC.md** with detailed plan ✓ +2. **Implement pagination** in useChatSync hook +3. **Add negentropy filtering** for efficient syncing +4. **Create message sync utility** for missing eventIds +5. **Implement real-time subscriptions** +6. **Add active conversation handling** +7. **Implement error handling and retry logic** +8. **Add sync status indicators** +9. **Write comprehensive tests** +10. **Performance optimization** + +## File Structure + +``` +hooks/ +├── useChatSync.ts (enhanced) +├── useChatHistorySync.ts (existing) +└── useChatRealtimeSync.ts (new) + +utils/ +├── conversationUtils.ts (enhanced) +├── messageUtils.ts (enhanced) +└── syncUtils.ts (new) + +types/ +└── chat.ts (enhanced with sync types) +``` + +## Dependencies + +- nostr-tools (for Nostr operations) +- applesauce-core (for relay pool) +- negentropy (for efficient syncing) +- Existing PNS utilities + +## Testing Strategy + +- Unit tests for sync utilities +- Integration tests for real-time syncing +- Performance tests for large datasets +- Error scenario testing + +## Deployment Considerations + +- Gradual rollout with feature flags +- Monitoring and logging for sync operations +- Fallback mechanisms for compatibility \ No newline at end of file diff --git a/INCREMENTAL_SYNC_ARCHITECTURE.md b/INCREMENTAL_SYNC_ARCHITECTURE.md new file mode 100644 index 00000000..4192e2d5 --- /dev/null +++ b/INCREMENTAL_SYNC_ARCHITECTURE.md @@ -0,0 +1,520 @@ +# Incremental Chat Sync Architecture + +## Overview + +This document describes the refactored chat synchronization system that processes Nostr events incrementally, providing a better user experience by displaying messages as they arrive rather than waiting for all events to be fetched. + +The system now supports **two modes of operation**: +1. **Batch Sync Mode** ([`useChatSync`](hooks/useChatSync.ts)) - Fetch all events at once or incrementally on demand +2. **Real-time Streaming Mode** ([`useChatSyncPro`](hooks/useChatSyncPro.ts)) - Live subscription with automatic conversation updates + +## Key Components + +### 1. Event Processing Utilities (`utils/eventProcessing.ts`) + +Modular, reusable functions for processing Nostr events: + +#### Core Functions + +- **`decryptPnsEventToInner(pnsEvent, pnsKeys)`** + - Decrypts a PNS event (Kind 1080) to an inner event (Kind 20001) + - Returns `InnerEvent | null` + +- **`extractConversationMetadata(innerEvent)`** + - Extracts conversation ID, role, timestamps, and links from inner event + - Returns `ConversationMetadata | null` + +- **`innerEventToMessage(innerEvent)`** + - Converts an inner event to a Message object + - Handles JSON parsing and metadata attachment + +- **`addMessageToConversation(conversation, message, options)`** + - Adds a message to a conversation with deduplication + - Sorts messages by timestamp if requested + - Returns updated conversation + +- **`shouldUpdateTitle(message, conversation)`** + - Determines if a message should trigger title update + - Returns true if message is earlier than current earliest + +- **`generateTitleFromMessage(message, maxLength)`** + - Generates a conversation title from message content + - Handles both string and multimodal content + +- **`updateConversationTitle(conversation, message)`** + - Updates conversation title if message is the earliest + - Only updates for the earliest message in the conversation + +- **`processInnerEvent(conversationsMap, innerEvent)`** + - High-level function that processes an inner event + - Creates or updates conversations in the map + - Handles title updates automatically + +### 2. Storage Manager (`utils/storageManager.ts`) + +Manages batched, debounced updates to localStorage to prevent thrashing: + +#### StorageBatchManager Class + +```typescript +class StorageBatchManager { + // Queue a single conversation update + queueUpdate(conversation: Conversation): void + + // Queue multiple conversations + queueBatchUpdate(conversations: Conversation[]): void + + // Flush all pending updates immediately + flush(): void + + // Clear pending updates without flushing + clear(): void + + // Initialize with existing conversations + initialize(conversations: Conversation[]): void + + // Get conversation from in-memory snapshot + getConversation(conversationId: string): Conversation | undefined + + // Get all conversations + getAllConversations(): Conversation[] +} +``` + +**Features:** +- Default 500ms debounce delay (configurable) +- Maintains in-memory snapshot of all conversations +- Automatically merges pending updates with existing data +- Global singleton pattern via `getStorageManager()` + +### 3. Refactored Chat Sync Hook (`hooks/useChatSync.ts`) + +#### New Functions + +**`processPnsEventIncremental(pnsEvent, pnsKeys, conversationsMap, onConversationUpdate)`** +- Processes a single PNS event incrementally +- Decrypts, processes, and optionally triggers callback +- Used internally for incremental sync + +**`syncConversationsIncremental(onConversationUpdate?, onComplete?)`** +- Main incremental sync function +- Processes events one by one +- Calls `onConversationUpdate` for each conversation as it's updated +- Calls `onComplete` with final array when done +- Queues batch storage update +- Returns: `Promise` + +**`syncConversations()` (Backward Compatible)** +- Wrapper around `syncConversationsIncremental()` +- Maintains existing API for compatibility +- Returns: `Promise` + +#### Return Interface + +```typescript +interface ChatSyncHook { + isSyncing: boolean; + lastSyncTime: number | null; + error: string | null; + publishMessage: (...) => Promise; + syncConversations: () => Promise; + syncConversationsIncremental: ( + onConversationUpdate?: (conv: Conversation) => void, + onComplete?: (conversations: Conversation[]) => void + ) => Promise; +} +``` + +### 4. Real-time Sync Hook (`hooks/useChatSyncPro.ts`) + +Enhanced to process events and maintain conversations in real-time: + +#### Key Features + +- **Live Event Subscription**: Uses RxJS streams to subscribe to Nostr relays +- **Automatic Decryption**: Decrypts PNS events as they arrive +- **Real-time Conversation Updates**: Maintains conversations map and updates state +- **Batched Storage**: Uses storage manager for efficient persistence +- **Deduplication**: Prevents duplicate events from being processed + +#### New State + +```typescript +const [conversations, setConversations] = useState([]) +const conversationsMapRef = useRef>(new Map()) +const pnsKeysRef = useRef(null) +``` + +#### Core Functions + +**`getPnsKeys()`** +- Derives and caches PNS keys from login credentials +- Returns cached keys for efficiency +- Returns null if no login available + +**`processEvent(event: NostrEvent)`** +- Decrypts PNS event to inner event +- Updates conversations map using [`processInnerEvent()`](utils/eventProcessing.ts:244) +- Updates state with sorted conversations +- Queues storage update + +**Event Subscription** +- Subscribes to `kind1080Events$` RxJS stream +- Processes each event as it arrives +- Updates both raw events array and conversations +- Flushes storage on unmount + +#### Return Interface + +```typescript +{ + events: NostrEvent[]; // Raw PNS events + conversations: Conversation[]; // Processed conversations + loading: boolean; + error: string | null; +} +``` + +### 5. Updated Conversation State Hook (`hooks/useConversationState.ts`) + +#### New Functions + +**`handleConversationUpdate(updatedConversation)`** +- Callback for incremental conversation updates +- Merges or adds conversations to state +- Called as each event is processed + +**`syncWithNostr()` (Updated)** +- Now uses `syncConversationsIncremental()` +- Provides `handleConversationUpdate` callback +- Provides completion callback for logging +- Storage is handled automatically by storage manager + +## Data Flow + +### Real-time Streaming Flow (useChatSyncPro) + +``` +┌─────────────────────────────────────────┐ +│ Nostr Relay Event Stream (RxJS) │ +└───────────────┬─────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────┐ +│ Event arrives in subscription │ +└───────────────┬─────────────────────────┘ + │ + ├──────────────────────────┐ + ▼ ▼ +┌─────────────────────────┐ ┌────────────────────────┐ +│ Update raw events array │ │ processEvent() │ +│ (with deduplication) │ │ - Decrypt PNS event │ +└─────────────────────────┘ │ - Process inner event │ + │ - Update conversations │ + └──────────┬───────────────┘ + │ + ┌──────────┴───────────┐ + ▼ ▼ + ┌──────────────────┐ ┌──────────────────┐ + │ Update UI State │ │ Queue Storage │ + │ (immediate) │ │ (debounced 500ms)│ + └──────────────────┘ └──────────────────┘ +``` + +### Incremental Sync Flow (useChatSync) + +``` +┌─────────────────────────────────────────┐ +│ User triggers sync │ +└───────────────┬─────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────┐ +│ syncConversationsIncremental() │ +│ - Fetch all PNS events │ +└───────────────┬─────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────┐ +│ For each PNS event: │ +│ ┌─────────────────────────────────────┐ │ +│ │ 1. Decrypt to inner event │ │ +│ │ 2. Process inner event │ │ +│ │ 3. Update conversations map │ │ +│ │ 4. Check title update │ │ +│ │ 5. Call onConversationUpdate() │ │ +│ └─────────────────────────────────────┘ │ +└───────────────┬─────────────────────────┘ + │ + ├──────────────┬─────────────────────┐ + ▼ ▼ ▼ +┌─────────────────────┐ ┌──────────────┐ ┌────────────────────┐ +│ UI Update │ │ Queue │ │ Title Update │ +│ (immediate) │ │ Storage │ │ (if earliest msg) │ +│ │ │ (debounced) │ │ │ +└─────────────────────┘ └──────────────┘ └────────────────────┘ + │ + ▼ + ┌──────────────┐ + │ Batch Flush │ + │ (after 500ms)│ + └──────────────┘ +``` + +## Title Update Logic + +### When Titles Are Updated + +Titles are updated when a new message arrives that is **earlier** than the current earliest message in the conversation. + +### Implementation + +1. **Track earliest message**: Each conversation's messages are sorted by `_createdAt` +2. **Compare timestamps**: When a new message arrives, compare its timestamp with the earliest +3. **Update if earlier**: If the new message is earlier, generate a new title from it +4. **Active conversation only**: Currently optimized for active conversation updates + +### Example + +```typescript +// Initial state: Conversation has messages from 1000s onwards +conversation.messages[0]._createdAt = 1000 + +// New message arrives with earlier timestamp +newMessage._createdAt = 500 + +// shouldUpdateTitle() returns true +// Title is updated from newMessage content +conversation.title = generateTitleFromMessage(newMessage) +``` + +## Storage Strategy + +### Immediate State Updates + +- Conversations state is updated immediately as events are processed +- UI reflects changes in real-time + +### Batched Storage Writes + +- localStorage writes are debounced (500ms default) +- Multiple rapid updates are batched into single write +- Prevents localStorage thrashing +- Reduces performance impact + +### Storage Manager Lifecycle + +```typescript +// Initialize with existing conversations +const storageManager = getStorageManager(); +storageManager.initialize(existingConversations); + +// Queue updates as they arrive +storageManager.queueUpdate(updatedConversation); + +// Or queue batch +storageManager.queueBatchUpdate(conversations); + +// Automatic flush after debounce delay +// Or manual flush +storageManager.flush(); +``` + +## Benefits + +### User Experience +✅ Messages appear as they arrive (not all at once) +✅ Faster perceived load time +✅ Real-time sync feel +✅ Progress feedback during sync + +### Performance +✅ Lower memory usage (process one at a time) +✅ Batched storage writes prevent thrashing +✅ Incremental rendering reduces UI lag +✅ Debouncing reduces I/O operations + +### Maintainability +✅ Modular, testable functions +✅ Clear separation of concerns +✅ Reusable components +✅ Type-safe implementation + +### Compatibility +✅ Backward compatible wrapper +✅ Gradual migration path +✅ Works with existing code +✅ Optional incremental mode + +## Usage Examples + +### Basic Sync (Backward Compatible) + +```typescript +const { syncConversations, isSyncing } = useChatSync(relays); + +// Old way still works +const conversations = await syncConversations(); +``` + +### Incremental Sync with Updates + +```typescript +const { syncConversationsIncremental, isSyncing } = useChatSync(relays); + +// With callbacks for real-time updates +await syncConversationsIncremental( + (conversation) => { + console.log('Conversation updated:', conversation.id); + // Update UI immediately + }, + (allConversations) => { + console.log('Sync complete:', allConversations.length); + } +); +``` + +### In useConversationState + +```typescript +const syncWithNostr = useCallback(async () => { + await syncConversationsIncremental( + handleConversationUpdate, // Updates state as events arrive + (finalConversations) => { + console.log(`Loaded ${finalConversations.length} conversations`); + } + ); +}, [syncConversationsIncremental, handleConversationUpdate]); +``` + +## Migration Guide + +### For Existing Code + +No changes required! The existing `syncConversations()` function still works: + +```typescript +// This continues to work +const conversations = await syncConversations(); +``` + +### To Enable Incremental Updates + +Replace `syncConversations()` with `syncConversationsIncremental()` and provide callbacks: + +```typescript +// Before +const conversations = await syncConversations(); +setConversations(conversations); + +// After +await syncConversationsIncremental( + (conv) => setConversations(prev => [...prev, conv]), + (allConvs) => console.log('Done!') +); +``` + +## Testing Recommendations + +### Unit Tests + +1. **Event Processing Utilities** + - Test decryption with valid/invalid events + - Test message conversion + - Test title generation + - Test conversation updates + +2. **Storage Manager** + - Test debouncing behavior + - Test batch updates + - Test flush operations + - Test initialization + +3. **Sync Functions** + - Test incremental processing + - Test callbacks + - Test error handling + - Test backward compatibility + +### Integration Tests + +1. Test complete sync flow +2. Test with multiple conversations +3. Test title updates with early messages +4. Test storage persistence + +## Performance Considerations + +### Memory Usage +- **Before**: All events loaded and processed in memory at once +- **After**: Events processed one at a time, lower peak memory + +### Storage I/O +- **Before**: Potentially multiple localStorage writes +- **After**: Batched writes with 500ms debounce + +### UI Responsiveness +- **Before**: UI blocks until all events processed +- **After**: UI updates incrementally, stays responsive + +## Future Enhancements + +1. **Real-time Subscriptions**: Integrate with `useChatSyncPro` for live event streaming +2. **Optimistic Updates**: Show local changes immediately before sync confirmation +3. **Conflict Resolution**: Handle concurrent edits from multiple devices +4. **Partial Sync**: Only fetch events since last sync timestamp +5. **Background Sync**: Sync in background without blocking UI +6. **Progress Indicators**: Show sync progress (X of Y events processed) + +## Troubleshooting + +### Events Not Appearing + +Check that: +1. PNS keys are correctly derived +2. Events are properly encrypted/decrypted +3. Callbacks are being called +4. Storage manager is initialized + +### Storage Not Persisting + +Check that: +1. Storage manager flush is being called +2. Debounce delay hasn't prevented flush +3. No localStorage quota exceeded +4. Browser allows localStorage + +### Title Not Updating + +Check that: +1. Message has `_createdAt` timestamp +2. Message is actually earlier than existing messages +3. Conversation exists in state +4. Title generation is working + +## Choosing Between Sync Modes + +### Use `useChatSyncPro` when: +✅ You want real-time updates as events arrive +✅ You need live subscription to relay feeds +✅ You want automatic conversation management +✅ You're building a chat interface with live updates + +### Use `useChatSync` when: +✅ You need on-demand sync (manual trigger) +✅ You want more control over the sync process +✅ You need to sync with specific callbacks +✅ You're implementing import/export features + +### Use both when: +✅ You want real-time updates + manual refresh capability +✅ You need different sync strategies for different features + +## API Reference + +See individual file documentation: +- [`utils/eventProcessing.ts`](utils/eventProcessing.ts) - Event processing utilities +- [`utils/storageManager.ts`](utils/storageManager.ts) - Batched storage manager +- [`hooks/useChatSync.ts`](hooks/useChatSync.ts) - On-demand sync with incremental support +- [`hooks/useChatSyncPro.ts`](hooks/useChatSyncPro.ts) - Real-time streaming sync +- [`hooks/useConversationState.ts`](hooks/useConversationState.ts) - Conversation state management \ No newline at end of file diff --git a/REFACTORING.md b/REFACTORING.md deleted file mode 100644 index 3ee8b847..00000000 --- a/REFACTORING.md +++ /dev/null @@ -1,632 +0,0 @@ -# Refactoring Plan: Modular Architecture for Routstr Chat - -## Current Issues - -### 1. **Scattered Wallet Logic** -Currently, wallet functionality is fragmented across: -- **hooks/**: `useCashuWallet`, `useCreateCashuWallet`, `useCashuToken`, `useWalletOperations`, `useNutzaps`, `useCashuHistory` -- **lib/**: `cashu.ts`, `cashuLightning.ts` -- **utils/**: `cashuUtils.ts`, `walletUtils.ts` -- **stores/**: `cashuStore.ts`, `nutzapStore.ts`, `transactionHistoryStore.ts` - -### 2. **Scattered Chat Logic** -Chat functionality is spread across: -- **hooks/**: `useChatActions`, `useConversationState` -- **utils/**: `messageUtils`, `apiUtils`, `conversationUtils` -- **context/**: `ChatProvider` - -### 3. **Scattered Nostr Logic** -Nostr infrastructure is distributed between: -- **hooks/**: `useNostr`, `useAuthor`, `useCurrentUser`, `useLoginActions` -- **lib/**: `nostr.ts`, `nostr-kinds.ts`, `nostrTimestamps.ts` -- **context/**: `NostrContext`, `NostrProvider` -- **utils/**: `nip60Utils` - -### 4. **General Problems** -- ❌ No clear separation between domain logic and UI/infrastructure -- ❌ Hooks contain too much business logic (hard to test) -- ❌ Utils folder is a catch-all with no clear organization -- ❌ Difficult to understand dependencies between modules -- ❌ **Cannot easily extract wallet functionality for reuse in other projects** -- ❌ High cognitive load for new contributors - ---- - -## Proposed Architecture - -### Feature-Based Organization with Domain-Driven Design - -``` -src/ -├── features/ # Feature modules (self-contained domains) -│ ├── wallet/ # 🎯 Wallet feature (easily extractable!) -│ │ ├── core/ # Pure business logic (framework-agnostic) -│ │ │ ├── domain/ # Domain models and types -│ │ │ │ ├── Proof.ts -│ │ │ │ ├── Token.ts -│ │ │ │ ├── Mint.ts -│ │ │ │ └── Transaction.ts -│ │ │ ├── services/ # Business logic services -│ │ │ │ ├── CashuWalletService.ts -│ │ │ │ ├── TokenService.ts -│ │ │ │ ├── MintService.ts -│ │ │ │ ├── LightningService.ts -│ │ │ │ └── NutzapService.ts -│ │ │ └── utils/ # Wallet-specific utilities -│ │ │ ├── balance.ts -│ │ │ ├── fees.ts -│ │ │ └── formatting.ts -│ │ ├── infrastructure/ # External dependencies & adapters -│ │ │ ├── api/ # API clients -│ │ │ │ └── cashu-mint-client.ts -│ │ │ ├── storage/ # Storage adapters -│ │ │ │ ├── ProofStorage.ts -│ │ │ │ └── WalletStorage.ts -│ │ │ └── nostr/ # Nostr integration for wallet -│ │ │ ├── nip60-adapter.ts -│ │ │ └── nutzap-adapter.ts -│ │ ├── state/ # State management -│ │ │ ├── cashuStore.ts -│ │ │ ├── nutzapStore.ts -│ │ │ └── transactionHistoryStore.ts -│ │ ├── hooks/ # React hooks for wallet -│ │ │ ├── useCashuWallet.ts -│ │ │ ├── useCashuToken.ts -│ │ │ ├── useLightning.ts -│ │ │ └── useWalletBalance.ts -│ │ ├── components/ # Wallet UI components -│ │ │ ├── WalletDisplay.tsx -│ │ │ ├── DepositModal.tsx -│ │ │ ├── SendTokenForm.tsx -│ │ │ └── MintSelector.tsx -│ │ ├── index.ts # Public API (what to expose) -│ │ └── README.md # Feature documentation -│ │ -│ ├── chat/ # Chat feature -│ │ ├── core/ -│ │ │ ├── domain/ -│ │ │ │ ├── Message.ts -│ │ │ │ ├── Conversation.ts -│ │ │ │ └── Model.ts -│ │ │ ├── services/ -│ │ │ │ ├── ChatService.ts -│ │ │ │ ├── ConversationService.ts -│ │ │ │ ├── StreamingService.ts -│ │ │ │ └── MessageService.ts -│ │ │ └── utils/ -│ │ │ ├── message-formatting.ts -│ │ │ └── thinking-parser.ts -│ │ ├── infrastructure/ -│ │ │ ├── api/ -│ │ │ │ └── ai-api-client.ts -│ │ │ └── storage/ -│ │ │ └── ConversationStorage.ts -│ │ ├── state/ -│ │ │ └── chatStore.ts -│ │ ├── hooks/ -│ │ │ ├── useChatActions.ts -│ │ │ ├── useConversationState.ts -│ │ │ └── useStreamingResponse.ts -│ │ ├── components/ -│ │ │ ├── ChatContainer.tsx -│ │ │ ├── ChatMessages.tsx -│ │ │ ├── ChatInput.tsx -│ │ │ └── MessageContent.tsx -│ │ ├── index.ts -│ │ └── README.md -│ │ -│ └── nostr/ # Nostr feature (identity & relay management) -│ ├── core/ -│ │ ├── domain/ -│ │ │ ├── Event.ts -│ │ │ ├── User.ts -│ │ │ └── Relay.ts -│ │ ├── services/ -│ │ │ ├── NostrService.ts -│ │ │ ├── RelayService.ts -│ │ │ ├── AuthService.ts -│ │ │ └── EventPublisher.ts -│ │ └── utils/ -│ │ ├── nip-04.ts -│ │ ├── nip-44.ts -│ │ └── key-management.ts -│ ├── infrastructure/ -│ │ ├── relay/ -│ │ │ └── relay-pool.ts -│ │ └── storage/ -│ │ └── NostrStorage.ts -│ ├── state/ -│ │ └── nostrStore.ts -│ ├── hooks/ -│ │ ├── useNostr.ts -│ │ ├── useCurrentUser.ts -│ │ └── useRelays.ts -│ ├── providers/ -│ │ └── NostrProvider.tsx -│ ├── index.ts -│ └── README.md -│ -├── shared/ # Shared utilities and infrastructure -│ ├── types/ # Global TypeScript types -│ │ └── index.ts -│ ├── config/ # App-wide configuration -│ │ ├── constants.ts -│ │ └── env.ts -│ ├── lib/ # Third-party library wrappers -│ │ └── query-client.ts -│ ├── hooks/ # Generic React hooks -│ │ ├── useLocalStorage.ts -│ │ ├── useMediaQuery.ts -│ │ └── useDebounce.ts -│ └── utils/ # Generic utilities -│ ├── storage.ts -│ ├── formatting.ts -│ └── validation.ts -│ -├── components/ # Shared/Layout UI components only -│ ├── ui/ # Shadcn/Radix primitives -│ │ ├── button.tsx -│ │ ├── dialog.tsx -│ │ └── ... -│ └── layout/ -│ ├── AppLayout.tsx -│ └── Header.tsx -│ -└── app/ # Next.js app directory (routes only) - ├── layout.tsx - ├── page.tsx - └── ... -``` - ---- - -## Migration Strategy - -### Phase 1: Extract Wallet Feature (Week 1-2) - -**Goal**: Make wallet functionality a self-contained, reusable module - -#### Step 1.1: Create Domain Models -```typescript -// features/wallet/core/domain/Proof.ts -export interface Proof { - id: string; - amount: number; - secret: string; - C: string; -} - -// features/wallet/core/domain/Token.ts -export interface CashuToken { - mint: string; - proofs: Proof[]; - del?: string[]; -} - -// features/wallet/core/domain/Mint.ts -export interface Mint { - url: string; - info?: MintInfo; - keysets?: MintKeyset[]; - active: boolean; -} -``` - -#### Step 1.2: Create Service Layer -```typescript -// features/wallet/core/services/CashuWalletService.ts -export class CashuWalletService { - constructor( - private storage: IWalletStorage, - private mintClient: ICashuMintClient, - private nostrAdapter?: INip60Adapter - ) {} - - async createWallet(privkey: string, mints: string[]): Promise { - // Pure business logic - no React, no hooks - } - - async getBalance(): Promise { - // ... - } - - async sendToken(amount: number, mintUrl: string): Promise { - // ... - } -} -``` - -#### Step 1.3: Create Infrastructure Adapters -```typescript -// features/wallet/infrastructure/storage/ProofStorage.ts -export interface IProofStorage { - saveProofs(proofs: Proof[]): Promise; - getProofs(mintUrl: string): Promise; - removeProofs(proofs: Proof[]): Promise; -} - -export class LocalStorageProofAdapter implements IProofStorage { - // Implementation using localStorage -} - -export class Nip60ProofAdapter implements IProofStorage { - // Implementation using Nostr NIP-60 -} -``` - -#### Step 1.4: Create React Hooks (Thin wrappers) -```typescript -// features/wallet/hooks/useCashuWallet.ts -export function useCashuWallet() { - const service = useWalletService(); // Dependency injection - const queryClient = useQueryClient(); - - return useQuery({ - queryKey: ['wallet'], - queryFn: () => service.getWallet() - }); -} -``` - -#### Step 1.5: Public API Export -```typescript -// features/wallet/index.ts -// Services (for framework-agnostic usage) -export { CashuWalletService } from './core/services/CashuWalletService'; -export { TokenService } from './core/services/TokenService'; - -// Hooks (for React usage) -export { useCashuWallet } from './hooks/useCashuWallet'; -export { useCashuToken } from './hooks/useCashuToken'; - -// Types -export type * from './core/domain'; - -// Components (optional) -export { WalletDisplay } from './components/WalletDisplay'; -``` - -**Migration Checklist for Wallet:** -- [ ] Move `lib/cashu.ts` → `features/wallet/core/services/` -- [ ] Move `lib/cashuLightning.ts` → `features/wallet/core/services/LightningService.ts` -- [ ] Move `utils/cashuUtils.ts` → `features/wallet/core/utils/` -- [ ] Move `utils/walletUtils.ts` → `features/wallet/core/utils/` -- [ ] Move `stores/cashuStore.ts` → `features/wallet/state/` -- [ ] Move `stores/nutzapStore.ts` → `features/wallet/state/` -- [ ] Move hooks → `features/wallet/hooks/` -- [ ] Move wallet components → `features/wallet/components/` -- [ ] Create adapters for external dependencies -- [ ] Write `features/wallet/README.md` with usage examples - ---- - -### Phase 2: Extract Chat Feature (Week 3) - -**Migration Checklist:** -- [ ] Create domain models (Message, Conversation, Model) -- [ ] Extract `utils/apiUtils.ts` → `features/chat/infrastructure/api/` -- [ ] Extract `utils/messageUtils.ts` → `features/chat/core/utils/` -- [ ] Extract `utils/conversationUtils.ts` → `features/chat/core/utils/` -- [ ] Move `hooks/useChatActions.ts` → `features/chat/hooks/` -- [ ] Move chat components → `features/chat/components/` -- [ ] Create ChatService for business logic -- [ ] Create `features/chat/index.ts` public API - ---- - -### Phase 3: Extract Nostr Feature (Week 4) - -**Migration Checklist:** -- [ ] Create domain models (Event, User, Relay) -- [ ] Extract `lib/nostr.ts` → `features/nostr/core/services/` -- [ ] Extract NIP utilities → `features/nostr/core/utils/` -- [ ] Move `context/NostrContext.tsx` → `features/nostr/providers/` -- [ ] Move hooks → `features/nostr/hooks/` -- [ ] Create NostrService for relay management -- [ ] Create `features/nostr/index.ts` public API - ---- - -### Phase 4: Cleanup & Reorganize Shared Code (Week 5) - -**Migration Checklist:** -- [ ] Move generic hooks → `shared/hooks/` -- [ ] Move generic utils → `shared/utils/` -- [ ] Create `shared/types/` for global types -- [ ] Keep UI components in root `components/` (they're truly shared) -- [ ] Update all import paths -- [ ] Remove old `utils/`, `lib/`, `hooks/` folders - ---- - -## Benefits of New Structure - -### ✅ For External Contributors - -1. **Clear Entry Points**: - - "Want to work on wallet? Look in `features/wallet/`" - - "Want to fix chat? Look in `features/chat/`" - -2. **Self-Documenting**: - - Each feature has its own README - - Clear separation: domain → services → infrastructure → UI - -3. **Easy Testing**: - - Core business logic has no React dependencies - - Can test services in isolation - - Mock infrastructure easily - -### ✅ For Code Reuse - -**Example: Using wallet in another project** - -```typescript -// In another project (non-React) -import { CashuWalletService } from '@routstr/wallet'; - -const walletService = new CashuWalletService( - new MyCustomStorage(), - new CashuMintClient() -); - -const balance = await walletService.getBalance(); -``` - -```typescript -// In another React project -import { useCashuWallet, WalletDisplay } from '@routstr/wallet'; - -function App() { - const { balance } = useCashuWallet(); - return ; -} -``` - -### ✅ For Maintainability - -- **Dependency Injection**: Services don't hard-code dependencies -- **Testability**: Business logic separated from React -- **Boundaries**: Clear contracts between features -- **Scalability**: Add new features without touching existing ones - ---- - -## Example: Before & After - -### Before (Current) -```typescript -// hooks/useCashuWallet.ts - 482 lines of mixed concerns -- Nostr queries -- State management -- Business logic -- Error handling -- React hooks -- Cashu SDK calls -``` - -### After (Proposed) -```typescript -// features/wallet/core/services/CashuWalletService.ts -class CashuWalletService { - async getWallet(): Promise { - // Pure business logic - } -} - -// features/wallet/infrastructure/nostr/nip60-adapter.ts -class Nip60Adapter implements IWalletStorage { - // Nostr-specific implementation -} - -// features/wallet/hooks/useCashuWallet.ts (30 lines) -export function useCashuWallet() { - const service = useWalletService(); - return useQuery(['wallet'], () => service.getWallet()); -} -``` - ---- - -## Publishing Wallet as Standalone Package - -Once refactored, you can publish the wallet feature: - -```json -// package.json for @routstr/wallet -{ - "name": "@routstr/wallet", - "version": "1.0.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": "./dist/index.js", - "./react": "./dist/hooks/index.js", - "./components": "./dist/components/index.js" - }, - "peerDependencies": { - "react": ">=18", - "@tanstack/react-query": ">=5" - } -} -``` - -Usage in other projects: -```bash -npm install @routstr/wallet -``` - ---- - -## Step-by-Step Migration Commands - -### 1. Create Feature Directories -```bash -mkdir -p features/wallet/{core/{domain,services,utils},infrastructure/{api,storage,nostr},state,hooks,components} -mkdir -p features/chat/{core/{domain,services,utils},infrastructure/{api,storage},state,hooks,components} -mkdir -p features/nostr/{core/{domain,services,utils},infrastructure/{relay,storage},state,hooks,providers} -mkdir -p shared/{types,config,lib,hooks,utils} -``` - -### 2. Start with Wallet Migration -```bash -# Move domain logic -mv lib/cashu.ts features/wallet/core/services/CashuWalletService.ts -mv lib/cashuLightning.ts features/wallet/core/services/LightningService.ts - -# Move utilities -mv utils/cashuUtils.ts features/wallet/core/utils/ -mv utils/walletUtils.ts features/wallet/core/utils/ - -# Move state -mv stores/cashuStore.ts features/wallet/state/ -mv stores/nutzapStore.ts features/wallet/state/ - -# Move hooks -mv hooks/useCashuWallet.ts features/wallet/hooks/ -mv hooks/useCashuToken.ts features/wallet/hooks/ -mv hooks/useNutzaps.ts features/wallet/hooks/ -``` - -### 3. Fix Imports -After moving files, update all imports: -```typescript -// Old -import { useCashuWallet } from '@/hooks/useCashuWallet'; - -// New -import { useCashuWallet } from '@/features/wallet'; -``` - ---- - -## Testing Strategy - -### Core Services (Pure Functions) -```typescript -// features/wallet/core/services/__tests__/CashuWalletService.test.ts -describe('CashuWalletService', () => { - it('should calculate balance correctly', () => { - const service = new CashuWalletService(mockStorage, mockClient); - const balance = service.calculateBalance(mockProofs); - expect(balance).toBe(1000); - }); -}); -``` - -### Infrastructure Adapters -```typescript -// features/wallet/infrastructure/storage/__tests__/Nip60Adapter.test.ts -describe('Nip60Adapter', () => { - it('should save proofs to Nostr', async () => { - const adapter = new Nip60Adapter(mockNostr); - await adapter.saveProofs(mockProofs); - expect(mockNostr.event).toHaveBeenCalled(); - }); -}); -``` - -### React Hooks -```typescript -// features/wallet/hooks/__tests__/useCashuWallet.test.tsx -import { renderHook } from '@testing-library/react'; - -it('should fetch wallet data', async () => { - const { result } = renderHook(() => useCashuWallet()); - await waitFor(() => expect(result.current.isSuccess).toBe(true)); -}); -``` - ---- - -## Documentation for Each Feature - -Each feature should have a README: - -### `features/wallet/README.md` -```markdown -# Wallet Feature - -## Overview -Cashu ecash wallet with NIP-60 integration. - -## Usage - -### In React -\`\`\`tsx -import { useCashuWallet } from '@/features/wallet'; - -function MyComponent() { - const { balance } = useCashuWallet(); - return
Balance: {balance}
; -} -\`\`\` - -### Without React -\`\`\`ts -import { CashuWalletService } from '@/features/wallet'; - -const service = new CashuWalletService(storage, client); -const balance = await service.getBalance(); -\`\`\` - -## Architecture -- Core: Pure business logic -- Infrastructure: External dependencies -- Hooks: React integration -``` - ---- - -## Estimated Timeline - -| Phase | Duration | Tasks | -|-------|----------|-------| -| Phase 1 | 2 weeks | Extract Wallet Feature | -| Phase 2 | 1 week | Extract Chat Feature | -| Phase 3 | 1 week | Extract Nostr Feature | -| Phase 4 | 1 week | Cleanup & Documentation | -| **Total** | **5 weeks** | **Complete Refactor** | - ---- - -## Success Metrics - -✅ **Wallet is 100% extractable** - Can be used in any JS project -✅ **All features have <100 LOC hooks** - Business logic moved to services -✅ **90%+ test coverage** - Core services fully tested -✅ **Documentation complete** - Each feature has README -✅ **New contributor onboarding < 1 hour** - Clear structure - ---- - -## Questions & Considerations - -1. **Do we want to keep using Zustand?** - - Consider moving to services + React Query for server state - - Keep Zustand only for UI state - -2. **Monorepo or Separate Packages?** - - Consider Turborepo/Nx if you want to publish `@routstr/wallet` separately - -3. **TypeScript Strict Mode?** - - Now is a good time to enable `strict: true` - -4. **Testing Framework?** - - Add Vitest for unit tests - - Add Playwright for E2E - ---- - -## Next Steps - -1. **Review this plan** with the team -2. **Start with Phase 1** (Wallet) - it's the highest priority -3. **Create feature branch** `refactor/modular-architecture` -4. **Migrate incrementally** - don't break main -5. **Write tests** as you migrate -6. **Update documentation** continuously - ---- - -**Questions? Start with `features/wallet/` and make it work. Then replicate the pattern for other features.** - diff --git a/ZUSTAND_EVENT_DATABASE_ARCHITECTURE.md b/ZUSTAND_EVENT_DATABASE_ARCHITECTURE.md new file mode 100644 index 00000000..b25e390b --- /dev/null +++ b/ZUSTAND_EVENT_DATABASE_ARCHITECTURE.md @@ -0,0 +1,584 @@ +# 🏗️ Zustand-based Event Database Architecture Plan + +## Overview + +This document outlines the architecture for implementing an `IEventDatabase` interface using Zustand for browser-based Nostr event storage. This will serve as a complementary layer to the applesauce-core `EventStore`. + +## Requirements Summary + +- **Storage Size**: ~1000-5000 events +- **Performance**: Not critical (basic queries acceptable) +- **Persistence**: Required (using Zustand persist middleware) +- **Integration**: Complementary to EventStore (EventStore accepts IEventDatabase in constructor) +- **Search**: Skip full-text search (NIP-50) +- **Primary Use Case**: Caching chat messages (PNS events) + +## Feasibility Analysis + +### ✅ Feasible Features + +| Method | Implementation | Notes | +|--------|---------------|-------| +| `add(event)` | O(1) map insertion | Straightforward | +| `remove(event)` | O(1) map deletion | Direct lookup | +| `hasEvent(id)` | O(1) existence check | Simple map lookup | +| `getEvent(id)` | O(1) retrieval | Direct access | +| `hasReplaceable(...)` | O(1) index lookup | Composite key index | +| `getReplaceable(...)` | O(1) retrieval | Index-based | +| `getReplaceableHistory(...)` | O(h) where h=history size | Array iteration | +| `getByFilters(filters)` | O(n) linear scan | No SQL indexing | +| `getTimeline(filters)` | O(n log n) with sort | Filter + sort | +| `removeByFilters(filters)` | O(n) scan + delete | Batch removal | + +### ⚠️ Limitations + +1. **No FTS**: Full-text search not supported (skipped per requirements) +2. **Filter Performance**: O(n) instead of indexed lookups +3. **Storage Limits**: LocalStorage ~5-10MB = max ~5000-10000 events +4. **No Transactions**: Zustand updates are atomic but no multi-operation transactions +5. **Cross-tab Sync**: Requires storage event listeners (not built-in) + +## Data Structure Design + +### Core State Shape + +```typescript +interface EventStoreState { + // Primary storage: event ID -> NostrEvent + events: Record; + + // Replaceable event index: "kind:pubkey:identifier" -> latest event ID + replaceableIndex: Record; + + // Replaceable history: "kind:pubkey:identifier" -> event ID array (newest first) + replaceableHistory: Record; + + // Optional: Tag index for common queries + // Format: "tagName:tagValue" -> event ID array + tagIndex?: Record; + + // Methods (IEventDatabase interface) + add: (event: NostrEvent) => NostrEvent; + remove: (event: string | NostrEvent) => boolean; + removeByFilters: (filters: Filter | Filter[]) => number; + hasEvent: (id: string) => boolean; + getEvent: (id: string) => NostrEvent | undefined; + hasReplaceable: (kind: number, pubkey: string, identifier?: string) => boolean; + getReplaceable: (kind: number, pubkey: string, identifier?: string) => NostrEvent | undefined; + getReplaceableHistory: (kind: number, pubkey: string, identifier?: string) => NostrEvent[] | undefined; + getByFilters: (filters: Filter | Filter[]) => NostrEvent[]; + getTimeline: (filters: Filter | Filter[]) => NostrEvent[]; + + // Utility methods + clearStore: () => void; + getStats: () => { totalEvents: number; replaceableCount: number }; +} +``` + +### Key Design Decisions + +1. **Record vs Array**: Use `Record` for O(1) lookups by ID +2. **Composite Keys**: Format `"kind:pubkey:identifier"` for replaceable event indexing +3. **History as Arrays**: Newest events first for efficient access +4. **Optional Tag Index**: Build on-demand for performance-critical queries + +## Replaceable Event Handling + +### Replaceable Event Detection + +```typescript +function isReplaceable(kind: number): boolean { + // Kinds 10000-19999 and 30000-39999 are replaceable + return (kind >= 10000 && kind < 20000) || (kind >= 30000 && kind < 40000); +} + +function isParameterized(kind: number): boolean { + // Kinds 30000-39999 use 'd' tag as identifier + return kind >= 30000 && kind < 40000; +} + +function getReplaceableKey(kind: number, pubkey: string, identifier: string = ''): string { + return `${kind}:${pubkey}:${identifier}`; +} +``` + +### Add Logic for Replaceable Events + +```typescript +// When adding a replaceable event: +1. Generate composite key +2. Check if newer than existing version (compare created_at) +3. If newer: + - Update replaceableIndex to point to new event + - Prepend to replaceableHistory array +4. Always add to events map (keep all versions) +``` + +### Identifier Extraction + +```typescript +function getIdentifier(event: NostrEvent, kind: number): string { + if (!isParameterized(kind)) return ''; + + const dTag = event.tags.find(t => t[0] === 'd'); + return dTag && dTag[1] ? dTag[1] : ''; +} +``` + +## Filter Implementation + +### Filter Matching Algorithm + +```typescript +function matchesFilter(event: NostrEvent, filter: Filter): boolean { + // 1. Match IDs (prefix matching) + if (filter.ids) { + if (!filter.ids.some(id => event.id.startsWith(id))) return false; + } + + // 2. Match authors (prefix matching) + if (filter.authors) { + if (!filter.authors.some(author => event.pubkey.startsWith(author))) return false; + } + + // 3. Match kinds (exact match) + if (filter.kinds) { + if (!filter.kinds.includes(event.kind)) return false; + } + + // 4. Match time range + if (filter.since && event.created_at < filter.since) return false; + if (filter.until && event.created_at > filter.until) return false; + + // 5. Match tags (#e, #p, etc.) + for (const [key, values] of Object.entries(filter)) { + if (key.startsWith('#')) { + const tagName = key.slice(1); + const eventTagValues = event.tags + .filter(t => t[0] === tagName) + .map(t => t[1]); + + if (!values.some(v => eventTagValues.includes(v))) return false; + } + } + + return true; +} +``` + +### Multi-Filter Support + +```typescript +function getByFilters(filters: Filter | Filter[]): NostrEvent[] { + const filterArray = Array.isArray(filters) ? filters : [filters]; + const allEvents = Object.values(this.events); + + // Union: event matches if it matches ANY filter + const matchingEvents = allEvents.filter(event => + filterArray.some(filter => matchesFilter(event, filter)) + ); + + // Apply limit if specified (use smallest limit across all filters) + const limit = Math.min( + ...filterArray.map(f => f.limit || Infinity) + ); + + if (limit !== Infinity) { + return matchingEvents.slice(0, limit); + } + + return matchingEvents; +} +``` + +## Persistence Strategy + +### Zustand Persist Middleware + +```typescript +import { persist, createJSONStorage } from 'zustand/middleware'; + +const useEventStore = create()( + persist( + (set, get) => ({ + events: {}, + replaceableIndex: {}, + replaceableHistory: {}, + + // ... methods + }), + { + name: 'nostr-event-store', + storage: createJSONStorage(() => localStorage), + + // Only persist data, not methods + partialize: (state) => ({ + events: state.events, + replaceableIndex: state.replaceableIndex, + replaceableHistory: state.replaceableHistory, + }), + + // Optional: Merge strategy for hydration + merge: (persistedState, currentState) => ({ + ...currentState, + ...persistedState, + }), + } + ) +); +``` + +### Storage Size Management + +```typescript +// Monitor storage usage +function getStorageSize(): number { + const str = JSON.stringify(localStorage); + return new Blob([str]).size; +} + +// Implement LRU eviction (optional) +function pruneOldEvents(beforeTimestamp: number): number { + // Remove events older than timestamp + // Update indexes accordingly +} + +// Utility to check if near storage limit +function isNearStorageLimit(): boolean { + const size = getStorageSize(); + const limit = 5 * 1024 * 1024; // 5MB + return size > limit * 0.8; // 80% of limit +} +``` + +## File Structure + +``` +lib/ + eventDatabase/ + index.ts # Public exports + eventStore.ts # Main Zustand store implementation + types.ts # TypeScript type definitions + filters.ts # Filter matching logic + replaceables.ts # Replaceable event utilities + utils.ts # Helper functions + README.md # Usage documentation +``` + +## Integration with EventStore + +### Current Usage (useChatSync.ts:102) + +```typescript +const store = new EventStore(); +``` + +### After Implementation + +```typescript +import { useEventStore } from '@/lib/eventDatabase'; +import { EventStore } from 'applesauce-core'; + +// Get the event database instance +const eventDatabase = useEventStore.getState(); + +// Pass to EventStore constructor +const store = new EventStore(eventDatabase); + +// EventStore will now use our Zustand-based database for storage +``` + +### Singleton Pattern + +```typescript +// Create a singleton EventStore instance +let eventStoreInstance: EventStore | null = null; + +export function getEventStore(): EventStore { + if (!eventStoreInstance) { + const eventDatabase = useEventStore.getState(); + eventStoreInstance = new EventStore(eventDatabase); + } + return eventStoreInstance; +} +``` + +## Performance Optimizations + +### For 1000-5000 Events + +1. **Batch Operations** + ```typescript + function addBatch(events: NostrEvent[]): void { + set((state) => { + const newEvents = { ...state.events }; + const newIndex = { ...state.replaceableIndex }; + const newHistory = { ...state.replaceableHistory }; + + for (const event of events) { + // Batch process all events + // Update structures + } + + return { events: newEvents, replaceableIndex: newIndex, replaceableHistory: newHistory }; + }); + } + ``` + +2. **Lazy Tag Indexing** + ```typescript + // Build tag index only when needed + function buildTagIndex(tagName: string): void { + const index: Record = {}; + + for (const event of Object.values(this.events)) { + event.tags + .filter(t => t[0] === tagName) + .forEach(t => { + const value = t[1]; + if (!index[value]) index[value] = []; + index[value].push(event.id); + }); + } + + set({ tagIndex: { ...this.tagIndex, [`${tagName}:*`]: index } }); + } + ``` + +3. **Memoized Filter Results** + ```typescript + const filterCache = new Map(); + + function getByFiltersCached(filters: Filter | Filter[]): NostrEvent[] { + const key = JSON.stringify(filters); + if (filterCache.has(key)) { + return filterCache.get(key)!; + } + + const results = getByFilters(filters); + filterCache.set(key, results); + return results; + } + ``` + +4. **Debounced Persistence** + ```typescript + // Prevent excessive localStorage writes + const debouncedPersist = debounce(() => { + // Trigger manual persistence if needed + }, 1000); + ``` + +## Implementation Phases + +### Phase 1: Core Storage (Priority: High) + +- [ ] Create type definitions matching IEventDatabase +- [ ] Implement basic Zustand store structure +- [ ] Add `add()` method with event storage +- [ ] Add `remove()` method +- [ ] Implement `hasEvent()` and `getEvent()` +- [ ] Add persistence with Zustand persist middleware +- [ ] Write unit tests for core operations + +### Phase 2: Replaceable Events (Priority: High) + +- [ ] Implement replaceable detection logic +- [ ] Add replaceable index management +- [ ] Implement `getReplaceable()` with latest version logic +- [ ] Add `hasReplaceable()` check +- [ ] Implement `getReplaceableHistory()` with all versions +- [ ] Handle replaceable event updates correctly +- [ ] Write tests for replaceable event scenarios + +### Phase 3: Filtering (Priority: Medium) + +- [ ] Implement filter matching algorithm +- [ ] Add multi-filter support (union logic) +- [ ] Implement `getByFilters()` method +- [ ] Add `getTimeline()` with sorting +- [ ] Implement `removeByFilters()` for batch deletion +- [ ] Handle edge cases (empty filters, limit, etc.) +- [ ] Write comprehensive filter tests + +### Phase 4: Integration & Documentation (Priority: Medium) + +- [ ] Integrate with EventStore in useChatSync.ts +- [ ] Create singleton EventStore instance +- [ ] Add usage examples and documentation +- [ ] Test with real chat data/PNS events +- [ ] Add storage size monitoring +- [ ] Document known limitations +- [ ] Create migration guide + +### Phase 5: Optimizations (Priority: Low) + +- [ ] Implement batch operations +- [ ] Add optional tag indexing +- [ ] Implement storage pruning/LRU eviction +- [ ] Add performance monitoring +- [ ] Optimize filter queries +- [ ] Add cross-tab synchronization (optional) + +## Known Limitations & Workarounds + +| Limitation | Impact | Severity | Workaround | +|------------|--------|----------|------------| +| No SQL indexing | O(n) filter queries | Medium | Pre-build tag indexes for common queries | +| LocalStorage size (~10MB) | Max ~5000-10000 events | Low | Implement event pruning/archival | +| No transactions | Potential race conditions | Low | Zustand updates are atomic per operation | +| O(n) filter matching | Scales linearly with events | Low | Acceptable for <5000 events (~5-10ms) | +| No real-time cross-tab | State not synced across tabs | Low | Use storage event listeners | +| No FTS | Can't search event content | N/A | Skipped per requirements | + +## Usage Examples + +### Basic Usage + +```typescript +import { useEventStore } from '@/lib/eventDatabase'; + +// In a component or hook +const { add, getEvent, getByFilters } = useEventStore(); + +// Add an event +const event: NostrEvent = { /* ... */ }; +add(event); + +// Get an event by ID +const retrieved = getEvent(event.id); + +// Query events +const chatEvents = getByFilters({ + kinds: [1080], // PNS events + authors: [pubkey], +}); +``` + +### With EventStore + +```typescript +import { getEventStore } from '@/lib/eventDatabase'; + +// Get the singleton EventStore instance +const eventStore = getEventStore(); + +// Now EventStore uses our Zustand database +eventStore.add(event); + +// Subscribe to events +eventStore.event(eventId).subscribe(event => { + console.log('Event updated:', event); +}); +``` + +### Advanced Filtering + +```typescript +// Get timeline of last 50 messages +const timeline = useEventStore.getState().getTimeline({ + kinds: [1080], + authors: [pubkey], + limit: 50, +}); + +// Get events by multiple filters (union) +const events = useEventStore.getState().getByFilters([ + { kinds: [1], authors: [user1] }, + { kinds: [1], authors: [user2] }, +]); + +// Get replaceable event +const profile = useEventStore.getState().getReplaceable( + 0, // kind: metadata + pubkey, // author + '' // no identifier for kind 0 +); +``` + +### Storage Management + +```typescript +// Check storage usage +const stats = useEventStore.getState().getStats(); +console.log(`Stored ${stats.totalEvents} events`); + +// Prune old events +const removed = useEventStore.getState().removeByFilters({ + until: Date.now() / 1000 - 30 * 24 * 60 * 60, // Older than 30 days +}); +console.log(`Removed ${removed} old events`); + +// Clear entire store +useEventStore.getState().clearStore(); +``` + +## Testing Strategy + +### Unit Tests + +1. **Core Operations** + - Add/remove events + - Event existence checks + - Event retrieval + +2. **Replaceable Events** + - Latest version retrieval + - History tracking + - Update scenarios + +3. **Filter Matching** + - Single filter matching + - Multi-filter union + - Edge cases (empty, limits) + +### Integration Tests + +1. **EventStore Integration** + - Constructor injection + - Event synchronization + - Observable streams + +2. **Persistence** + - LocalStorage hydration + - Cross-session persistence + - Storage limits + +### Performance Tests + +1. **Scalability** + - 1000 events: <5ms queries + - 5000 events: <20ms queries + - 10000 events: <50ms queries + +2. **Memory Usage** + - Monitor heap size + - Check for memory leaks + - Storage size tracking + +## Success Criteria + +- ✅ Implements complete IEventDatabase interface +- ✅ Persists events across browser sessions +- ✅ Handles 1000-5000 events efficiently +- ✅ Integrates seamlessly with EventStore +- ✅ Properly manages replaceable events +- ✅ Filter queries complete in <50ms for 5000 events +- ✅ Comprehensive documentation and examples +- ✅ Full test coverage (>80%) + +## Next Steps + +1. Review and approve this architecture plan +2. Switch to Code mode to begin implementation +3. Start with Phase 1 (Core Storage) +4. Iteratively implement remaining phases +5. Integrate with existing chat sync functionality + +## References + +- [IEventDatabase Interface](https://github.com/hzrd149/applesauce/blob/master/packages/core/src/event-store/interfaces.ts) +- [Zustand Documentation](https://docs.pmnd.rs/zustand/) +- [Zustand Persist Middleware](https://docs.pmnd.rs/zustand/integrations/persisting-store-data) +- [Nostr NIPs](https://github.com/nostr-protocol/nips) +- [Replaceable Events (NIP-01)](https://github.com/nostr-protocol/nips/blob/master/01.md#kinds) \ No newline at end of file diff --git a/app/page.tsx b/app/page.tsx index bdd1d2fd..c2543b4f 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -71,23 +71,20 @@ function ChatPageContent() { useEffect(() => { let topUpTimer: NodeJS.Timeout | null = null; - if (!isBalanceLoading && balance === 0 && isAuthenticated && !isSettingsOpen) { + if (!isBalanceLoading && balance === 0 && !isSettingsOpen) { if (!hasSeenTopUpPrompt() && !topUpPromptDismissed) { setIsTopUpPromptOpen(false); topUpTimer = setTimeout(() => { + markTopUpPromptSeen(); setIsTopUpPromptOpen(true); }, 500); - } else { - setIsTopUpPromptOpen(false); } - } else { - setIsTopUpPromptOpen(false); - } + } return () => { if (topUpTimer) clearTimeout(topUpTimer); }; - }, [balance, isBalanceLoading, isAuthenticated, isSettingsOpen, topUpPromptDismissed]); + }, [balance, isBalanceLoading, isSettingsOpen, topUpPromptDismissed]); const handleTopUp = (_amount?: number) => {}; @@ -194,6 +191,7 @@ function ChatPageContent() { isOpen={isLoginModalOpen} onClose={() => setIsLoginModalOpen(false)} onLogin={() => setIsLoginModalOpen(false)} + logout={logout} /> {/* Top-up Prompt */} diff --git a/components/ClientProviders.tsx b/components/ClientProviders.tsx index afaf3bb9..4f1b7e55 100644 --- a/components/ClientProviders.tsx +++ b/components/ClientProviders.tsx @@ -22,10 +22,10 @@ import { AppConfig } from '@/context/AppContext'; const presetRelays = [ { url: 'wss://relay.routstr.com', name: 'Routstr Relay' }, - { url: 'wss://relay.damus.io', name: 'Damus' }, { url: 'wss://nos.lol', name: 'nos.lol' }, - { url: 'wss://relay.nostr.band', name: 'Nostr.Band' }, { url: 'wss://relay.primal.net', name: 'Primal' }, + { url: 'wss://relay.damus.io', name: 'Damus' }, + { url: 'wss://relay.nostr.band', name: 'Nostr.Band' }, { url: 'wss://relay.chorus.community', name: 'Chorus Relay' }, ]; @@ -76,9 +76,10 @@ export default function ClientProviders({ children }: { children: ReactNode }) { useReactEffect(() => { if (logins.length === 0) { try { - const sk = generateSecretKey(); - const nsec = nip19.nsecEncode(sk); - loginActions.nsec(nsec); + // const sk = generateSecretKey(); + // const nsec = nip19.nsecEncode(sk); + // console.log("RTRUE nsse", nsec) + // loginActions.nsec(nsec); } catch (err) { // no-op } diff --git a/components/LoginModal.tsx b/components/LoginModal.tsx index 8ba30e6a..5439bdf4 100644 --- a/components/LoginModal.tsx +++ b/components/LoginModal.tsx @@ -5,16 +5,18 @@ import { generateSecretKey, getPublicKey, nip19 } from 'nostr-tools'; import { useLoginActions } from '@/hooks/useLoginActions'; import { useLoggedInAccounts } from '@/hooks/useLoggedInAccounts'; import { Shield, Eye, EyeOff, Copy, Check } from 'lucide-react'; +import { hasCreatedEphemeralNsec, markEphemeralNsecDeleted } from '@/utils/storageUtils'; interface LoginModalProps { isOpen: boolean; onClose: () => void; onLogin: () => void; + logout: () => void; } type SignupStep = 'initial' | 'save-keys'; -export default function LoginModal({ isOpen, onClose, onLogin }: LoginModalProps) { +export default function LoginModal({ isOpen, onClose, onLogin, logout }: LoginModalProps) { const [nsec, setNsec] = useState(''); const [error, setError] = useState(null); const [isLoading, setIsLoading] = useState(false); @@ -55,6 +57,10 @@ export default function LoginModal({ isOpen, onClose, onLogin }: LoginModalProps if (!('nostr' in window)) { throw new Error('Nostr extension not found. Please install a NIP-07 extension.'); } + if (hasCreatedEphemeralNsec()) { + logout() + markEphemeralNsecDeleted() + } const login = await loginActions.extension(); setLogin(login.id); onLogin(); @@ -73,6 +79,10 @@ export default function LoginModal({ isOpen, onClose, onLogin }: LoginModalProps setError(null); try { + if (hasCreatedEphemeralNsec()) { + logout() + markEphemeralNsecDeleted() + } const login = loginActions.nsec(nsec); setLogin(login.id); onLogin(); diff --git a/components/NostrProvider.tsx b/components/NostrProvider.tsx index ab508c74..e3ff734f 100644 --- a/components/NostrProvider.tsx +++ b/components/NostrProvider.tsx @@ -1,7 +1,6 @@ import { NostrEvent, NPool, NRelay1 } from '@nostrify/nostrify'; import { NostrContext } from '@nostrify/react'; import React, { useRef, useEffect } from 'react'; // Import useEffect -import { storeEventTimestamp } from '@/lib/nostrTimestamps'; import { useAppContext } from '@/hooks/useAppContext'; interface NostrProviderProps { @@ -19,16 +18,13 @@ class TimestampTrackingNPool extends NPool { ): Promise { // Call the original event method await super.event(event, opts); - - // Store the timestamp after successful publishing - storeEventTimestamp(event.pubkey, event.kind); } } const NostrProvider: React.FC = (props) => { const { children } = props; - const { config, presetRelays } = useAppContext(); // Keep presetRelays even if not used directly here + const { config } = useAppContext(); // Keep presetRelays even if not used directly here // NPool instance created once const pool = useRef(undefined); diff --git a/components/SettingsModal.tsx b/components/SettingsModal.tsx index 4a88ac29..99a933dd 100644 --- a/components/SettingsModal.tsx +++ b/components/SettingsModal.tsx @@ -131,7 +131,6 @@ const SettingsModal = ({ {activeTab === 'settings' ? ( = ({ isOpen, onClose, on const [nwcCustomAmount, setNwcCustomAmount] = useState(''); const [isPayingWithNWC, setIsPayingWithNWC] = useState(false); + + const { logins } = useNostrLogin(); + const loginActions = useLoginActions(); + const { isAuthenticated } = useAuth(); + useEffect(() => { let unsubConnect: undefined | (() => void); let unsubDisconnect: undefined | (() => void); @@ -113,6 +123,14 @@ const TopUpPromptModal: React.FC = ({ isOpen, onClose, on const quickAmounts = [500, 1000, 5000]; + const createNsecForLogin = () => { + const sk = generateSecretKey(); + const nsec = nip19.nsecEncode(sk); + console.log("RTRUE nsse", nsec) + loginActions.nsec(nsec); + markEphemeralNsecCreated() + } + const copyInvoiceToClipboard = async () => { if (!invoice) return; try { @@ -141,6 +159,8 @@ const TopUpPromptModal: React.FC = ({ isOpen, onClose, on setTimeout(() => setError(null), 2000); return; } + + createNsecForLogin() try { setIsReceivingToken(true); @@ -183,6 +203,8 @@ const TopUpPromptModal: React.FC = ({ isOpen, onClose, on return; } + createNsecForLogin() + const amt = amount !== undefined ? amount : parseInt(customAmount); if (isNaN(amt) || amt <= 0) { setError('Enter a valid amount'); @@ -412,7 +434,7 @@ const TopUpPromptModal: React.FC = ({ isOpen, onClose, on {/* Tab Content Container */} -
+
{/* Lightning Tab */} {activeTab === 'lightning' && (
@@ -558,6 +580,7 @@ const TopUpPromptModal: React.FC = ({ isOpen, onClose, on
-
+
+ {totalVersions > 1 && ( +
+ + {currentVersion} / {totalVersions} + +
+ )} +
+ {totalVersions > 1 && ( +
+ + {currentVersion} / {totalVersions} + +
+ )} +
{conversations.length === 0 ? (

No saved conversations

) : ( - [...conversations].reverse().map(conversation => ( + [...conversations].map(conversation => (
{ @@ -157,4 +171,4 @@ export default function Sidebar({ )}
); -} \ No newline at end of file +} diff --git a/components/settings/GeneralTab.tsx b/components/settings/GeneralTab.tsx index 95ecb1ee..fd9e3358 100644 --- a/components/settings/GeneralTab.tsx +++ b/components/settings/GeneralTab.tsx @@ -6,10 +6,10 @@ import NWCWalletManager from './NWCWalletManager'; // Import the NWC wallet mana import { useLoggedInAccounts } from '@/hooks/useLoggedInAccounts'; import { useLoginActions } from '@/hooks/useLoginActions'; import { useNostrLogin } from '@nostrify/react/login'; +import { useChatSync } from '@/hooks/useChatSync'; interface GeneralTabProps { publicKey: string | undefined; - nsecData?: { nsec: `nsec1${string}`; } | { bunkerPubkey: string; clientNsec: `nsec1${string}`; relays: string[]; } | { [key: string]: unknown; } | null; loginType: 'nsec' | "bunker" | "extension" | `x-${string}` | undefined; logout?: () => void; router?: AppRouterInstance; @@ -19,7 +19,6 @@ interface GeneralTabProps { const GeneralTab: React.FC = ({ publicKey, - nsecData, loginType, logout, router, @@ -39,6 +38,7 @@ const GeneralTab: React.FC = ({ const { currentUser, otherUsers, setLogin, removeLogin } = useLoggedInAccounts(); const { logins } = useNostrLogin(); const loginActions = useLoginActions(); + const { chatSyncEnabled, setChatSyncEnabled } = useChatSync(); useEffect(() => { if (localStorage.getItem('nsec_storing_skipped') === 'true') { @@ -81,6 +81,34 @@ const GeneralTab: React.FC = ({ )} + {/* Chat Sync Settings */} +
+

Chat Sync

+
+
+
+
Enable Chat Sync
+
Sync chat messages with Nostr relays
+
+ +
+
+
+ {/* Nostr Relays */} @@ -151,10 +179,11 @@ const GeneralTab: React.FC = ({
- {loginType === 'nsec' && nsecData && ( + {loginType === 'nsec' && logins[0]?.data && ( )} {logout && router && ( @@ -210,24 +213,6 @@ const GeneralTab: React.FC = ({ )}
- {showNsec && ( -
- - {nsecValue ? `${nsecValue.slice(0, 4)}${'•'.repeat(Math.max(0, nsecValue.length - 4))}` : ''} - - -
- )} {/* Version Information */} diff --git a/lib/pns.ts b/lib/pns.ts index 65071544..c820f57c 100644 --- a/lib/pns.ts +++ b/lib/pns.ts @@ -5,7 +5,8 @@ import { nip44, getPublicKey, finalizeEvent, Event } from 'nostr-tools'; // Constants export const KIND_PNS = 1080; -export const SALT_PNS = process.env.NODE_ENV === 'development' ? 'routstr-chat-sync-test3' : 'routstr-chat-sync-v1'; +export const SALT_PNS = 'routstr-chat-sync-v1'; +// export const SALT_PNS = process.env.NODE_ENV === 'development' ? 'routstr-chat-sync-test3' : 'routstr-chat-sync-v1'; // Types export interface PnsKeys { diff --git a/utils/apiUtils.ts b/utils/apiUtils.ts index ac24cdac..a5ce1eb8 100644 --- a/utils/apiUtils.ts +++ b/utils/apiUtils.ts @@ -159,7 +159,10 @@ async function routstrRequest(params: { body: JSON.stringify({ model: modelIdForRequest, messages: apiMessages, - stream: true + stream: true, + tools: [ + { type: "web_search" }, + ] }) }); } catch (error: any) { From 96f5c1cdbdc702466efe040076fa0e4c1d4e38e5 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Mon, 8 Dec 2025 09:55:25 +0800 Subject: [PATCH 05/47] sync works --- features/wallet/hooks/useCashuWallet.ts | 10 +++- hooks/useChatSync1081.ts | 77 +++++++++++++++++++------ hooks/useConversationState.ts | 1 + 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/features/wallet/hooks/useCashuWallet.ts b/features/wallet/hooks/useCashuWallet.ts index a2d31d3c..4f45f54c 100644 --- a/features/wallet/hooks/useCashuWallet.ts +++ b/features/wallet/hooks/useCashuWallet.ts @@ -352,10 +352,14 @@ export function useCashuWallet() { const newDeletedEvents = Array.from(deletedEventsTemp); - let allDeletedEvents = newDeletedEvents; + let allDeletedEvents = [...existingDeletedEvents, ...newDeletedEvents]; + + // Remove deleted events older than 30 days + const thirtyDaysAgo = Math.floor(Date.now() / 1000) - (7 * 24 * 60 * 60); + allDeletedEvents = allDeletedEvents.filter(e => e.timestamp > thirtyDaysAgo); + // Update local storage with combined events (existing + new) - if (newDeletedEvents.length > 0) { - allDeletedEvents = [...existingDeletedEvents, ...newDeletedEvents]; + if (newDeletedEvents.length > 0 || allDeletedEvents.length !== existingDeletedEvents.length) { setDeletedEvents(allDeletedEvents); } diff --git a/hooks/useChatSync1081.ts b/hooks/useChatSync1081.ts index 7468599e..f9f15603 100644 --- a/hooks/useChatSync1081.ts +++ b/hooks/useChatSync1081.ts @@ -1,5 +1,5 @@ import { useEffect, useState, useRef } from 'react' -import { BehaviorSubject, Subject, filter, shareReplay, combineLatest, switchMap, tap, map, defaultIfEmpty, merge, catchError, EMPTY, scan, distinctUntilChanged, from, mergeMap, withLatestFrom, share, take, timeout, of } from 'rxjs' +import { BehaviorSubject, Subject, filter, shareReplay, combineLatest, switchMap, tap, map, defaultIfEmpty, merge, catchError, EMPTY, scan, distinctUntilChanged, from, mergeMap, withLatestFrom, share, take, timeout, of, retry } from 'rxjs' import { nip19, generateSecretKey } from 'nostr-tools' import type { NostrEvent } from 'nostr-tools' import { KIND_PNS, PnsKeys, derivePnsKeys, SALT_PNS } from '@/lib/pns' @@ -42,7 +42,24 @@ export const syncDerivedPnsTrigger$ = new Subject() // Function to trigger derived PNS sync manually export function triggerDerivedPnsSync() { - console.log('[useChatSync1081] Manual derived PNS sync triggered') + const observers = syncDerivedPnsTrigger$.observers.length + const relays = relayUrls$.getValue() + const signer = userSigner$.getValue() + + console.log('[useChatSync1081] Manual derived PNS sync triggered. Observers:', observers) + + if (observers === 0) { + console.warn('[useChatSync1081] No observers for syncDerivedPnsTrigger$. Sync stream may be dead or not initialized.') + } + + if (relays.length === 0) { + console.warn('[useChatSync1081] Triggered sync but relayUrls is empty! Sync will not proceed to network.') + } + + if (!signer) { + console.warn('[useChatSync1081] Triggered sync but userSigner is null! Decryption will fail.') + } + syncDerivedPnsTrigger$.next() } const relayUrlsDefined$ = relayUrls$.pipe( @@ -326,6 +343,9 @@ const processStored1081Events$ = combineLatest([ userSignerDefined$, userPubkeyDefined$ ]).pipe( + tap(([count, signer, pubkey]) => { + console.log('[useChatSync1081] processStored1081Events$ input update:', { count, hasSigner: !!signer, pubkey }) + }), filter(([count]) => count > 0), switchMap(([_, signerInfo, userPubkey]) => { // Read all 1081 events for this user from the store @@ -389,12 +409,18 @@ const syncStatsDerivedPns = { // Combined stream for derived PNS sync - emits when pubkeys/relays are ready OR when manually triggered const syncDerivedPnsInputs$ = merge( // Initial emission when pubkeys and relays are defined - combineLatest([derivedPnsPubkeys$, relayUrlsDefined$]), + combineLatest([derivedPnsPubkeys$, relayUrlsDefined$]).pipe( + tap(([pubkeys, relays]) => console.log('[useChatSync1081] Auto-sync input emitted', pubkeys.length, relays.length)) + ), // Re-emit current values when sync is manually triggered syncDerivedPnsTrigger$.pipe( - switchMap(() => { + tap(() => console.log('[useChatSync1081] syncDerivedPnsTrigger$ processing started')), + withLatestFrom(relayUrls$), + switchMap(([_, relayUrls]) => { + console.log('[useChatSync1081] Manual trigger processing with relays:', relayUrls.length) return derivedPnsPubkeys$.pipe( take(1), + tap(pubkeys => console.log('[useChatSync1081] Current derived PNS pubkeys:', pubkeys.length)), switchMap(pubkeys => { if (pubkeys.length === 0) { console.log('[useChatSync1081] No derived PNS keys found during manual trigger, attempting to process stored 1081 events') @@ -412,18 +438,20 @@ const syncDerivedPnsInputs$ = merge( ) } return of(pubkeys) - }) + }), + map(pubkeys => [pubkeys, relayUrls] as [string[], string[]]) ) }), - withLatestFrom(relayUrlsDefined$), tap(([pubkeys, relayUrls]) => { - console.log('[useChatSync1081] Manual derived PNS trigger payload:', { + console.log('[useChatSync1081] Manual derived PNS trigger payload ready:', { pubkeysCount: pubkeys.length, - relayUrls, + relayUrlsCount: relayUrls.length, }) }), - map(([pubkeys, relayUrls]) => [pubkeys, relayUrls] as [string[], string[]]) + filter(([_, relayUrls]) => relayUrls.length > 0) ) +).pipe( + share() // Share the merged stream to prevent multiple subscriptions to source observables ) // Sync kind 1080 events for all derived PNS pubkeys @@ -449,14 +477,16 @@ const syncDerivedPnsEvents$ = syncDerivedPnsInputs$.pipe( eventStore.add(event) }), catchError((err) => { - if (err.name === 'EmptyError') { - console.log('[useChatSync1081] Derived PNS sync complete - no events') - return EMPTY - } - throw err + console.error('[useChatSync1081] Derived PNS sync error:', err) + return EMPTY }), ) }), + tap({ + error: (err) => console.error('[useChatSync1081] syncDerivedPnsEvents$ stream error:', err), + complete: () => console.log('[useChatSync1081] syncDerivedPnsEvents$ stream completed'), + }), + retry(1), shareReplay(1), ) @@ -484,14 +514,16 @@ const liveDerivedPnsEvents$ = combineLatest([derivedPnsPubkeys$, relayUrlsDefine }), defaultIfEmpty(null), catchError((err) => { - if (err.name === 'EmptyError') { - console.log('[useChatSync1081] Derived PNS sync complete - no events') - return EMPTY - } - throw err + console.error('[useChatSync1081] Live derived PNS sync error:', err) + return EMPTY }), ) }), + tap({ + error: (err) => console.error('[useChatSync1081] liveDerivedPnsEvents$ stream error:', err), + complete: () => console.log('[useChatSync1081] liveDerivedPnsEvents$ stream completed'), + }), + retry(1), shareReplay(1), ) @@ -561,6 +593,7 @@ export function useChatSync1081() { // Subscribe to derived PNS events sync useEffect(() => { + console.log('[useChatSync1081] Subscribing to syncDerivedPnsEvents$') setLoadingDerivedPns(true) syncCountDerivedPnsRef.current = 0 @@ -593,8 +626,14 @@ export function useChatSync1081() { }, }) + // Also subscribe to the trigger to ensure the stream stays alive if it's hot + const triggerSub = syncDerivedPnsTrigger$.subscribe(() => { + console.log('[useChatSync1081] syncDerivedPnsTrigger$ fired in component') + }) + return () => { sub.unsubscribe() + triggerSub.unsubscribe() } }, []) diff --git a/hooks/useConversationState.ts b/hooks/useConversationState.ts index fcfe43c4..05f20e3c 100644 --- a/hooks/useConversationState.ts +++ b/hooks/useConversationState.ts @@ -72,6 +72,7 @@ export const useConversationState = (): UseConversationStateReturn => { const isSyncing = isPublishing || loading1081 || loadingDerivedPns; const syncWithNostr = useCallback(async () => { + console.log('[useConversationState] syncWithNostr triggered') triggerDerivedPnsSync(); triggerProcessStored1081Events(); }, [triggerDerivedPnsSync, triggerProcessStored1081Events]); From de249c7234f2075e76879aebb90e3ce714adcd49 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Mon, 8 Dec 2025 15:35:29 +0800 Subject: [PATCH 06/47] more robust publishing --- hooks/useChatSync1081.ts | 190 ++++++++++++++++++---------------- hooks/useConversationState.ts | 11 ++ lib/pns.ts | 26 +++++ utils/eventProcessing.ts | 17 ++- 4 files changed, 153 insertions(+), 91 deletions(-) diff --git a/hooks/useChatSync1081.ts b/hooks/useChatSync1081.ts index f9f15603..5909a0ac 100644 --- a/hooks/useChatSync1081.ts +++ b/hooks/useChatSync1081.ts @@ -41,16 +41,12 @@ export const relayUrls$ = new BehaviorSubject([]) export const syncDerivedPnsTrigger$ = new Subject() // Function to trigger derived PNS sync manually +// This now works reliably because it uses an imperative approach that creates fresh subscriptions export function triggerDerivedPnsSync() { - const observers = syncDerivedPnsTrigger$.observers.length const relays = relayUrls$.getValue() const signer = userSigner$.getValue() - console.log('[useChatSync1081] Manual derived PNS sync triggered. Observers:', observers) - - if (observers === 0) { - console.warn('[useChatSync1081] No observers for syncDerivedPnsTrigger$. Sync stream may be dead or not initialized.') - } + console.log('[useChatSync1081] Manual derived PNS sync triggered.') if (relays.length === 0) { console.warn('[useChatSync1081] Triggered sync but relayUrls is empty! Sync will not proceed to network.') @@ -60,6 +56,7 @@ export function triggerDerivedPnsSync() { console.warn('[useChatSync1081] Triggered sync but userSigner is null! Decryption will fail.') } + // Emit trigger - the subscription at module level handles this imperatively syncDerivedPnsTrigger$.next() } const relayUrlsDefined$ = relayUrls$.pipe( @@ -406,88 +403,103 @@ const syncStatsDerivedPns = { lastSyncTime: null as Date | null, } -// Combined stream for derived PNS sync - emits when pubkeys/relays are ready OR when manually triggered -const syncDerivedPnsInputs$ = merge( - // Initial emission when pubkeys and relays are defined - combineLatest([derivedPnsPubkeys$, relayUrlsDefined$]).pipe( - tap(([pubkeys, relays]) => console.log('[useChatSync1081] Auto-sync input emitted', pubkeys.length, relays.length)) - ), - // Re-emit current values when sync is manually triggered - syncDerivedPnsTrigger$.pipe( - tap(() => console.log('[useChatSync1081] syncDerivedPnsTrigger$ processing started')), - withLatestFrom(relayUrls$), - switchMap(([_, relayUrls]) => { - console.log('[useChatSync1081] Manual trigger processing with relays:', relayUrls.length) - return derivedPnsPubkeys$.pipe( - take(1), - tap(pubkeys => console.log('[useChatSync1081] Current derived PNS pubkeys:', pubkeys.length)), - switchMap(pubkeys => { - if (pubkeys.length === 0) { - console.log('[useChatSync1081] No derived PNS keys found during manual trigger, attempting to process stored 1081 events') - triggerProcessStored1081Events() - - // Wait for keys to be derived - return derivedPnsPubkeys$.pipe( - filter(keys => keys.length > 0), - take(1), - timeout(5000), // Wait up to 5 seconds - catchError(err => { - console.warn('[useChatSync1081] Timed out waiting for derived PNS keys:', err) - return of([]) - }) - ) - } - return of(pubkeys) - }), - map(pubkeys => [pubkeys, relayUrls] as [string[], string[]]) - ) +// Subject to emit sync results - allows imperative triggering of new sync subscriptions +const syncDerivedPnsResults$ = new Subject() + +// Imperative function to perform derived PNS sync - creates a fresh subscription each time +function performDerivedPnsSync(pubkeys: string[], relayUrls: string[]): void { + if (pubkeys.length === 0 || relayUrls.length === 0) { + console.log('[useChatSync1081] performDerivedPnsSync: skipping, no pubkeys or relays') + return + } + + // Reset sync stats for new sync + syncStatsDerivedPns.eventsReceived = 0 + syncStatsDerivedPns.lastSyncTime = new Date() + + // Create filter for all derived PNS pubkeys + const kind1080Filter = { + kinds: [KIND_PNS], + authors: pubkeys, + } + + console.log('[useChatSync1081] Syncing derived PNS events for pubkeys:', pubkeys) + + // Create fresh subscription for each sync - this won't be affected by previous completions + relayPool.sync(relayUrls, eventStore, kind1080Filter, SyncDirection.BOTH).pipe( + tap((event: NostrEvent) => { + syncStatsDerivedPns.eventsReceived++ + console.log('[useChatSync1081] Synced derived PNS event:', event.id, 'from:', event.pubkey.slice(0, 8)) + eventStore.add(event) }), - tap(([pubkeys, relayUrls]) => { - console.log('[useChatSync1081] Manual derived PNS trigger payload ready:', { - pubkeysCount: pubkeys.length, - relayUrlsCount: relayUrls.length, - }) + catchError((err: any) => { + // Handle EmptyError which can happen when sync completes with no events + if (err?.name === 'EmptyError' || err?.message === 'no elements in sequence') { + console.log('[useChatSync1081] Derived PNS sync complete - no events to sync') + return EMPTY + } + console.error('[useChatSync1081] Derived PNS sync error:', err) + return EMPTY }), - filter(([_, relayUrls]) => relayUrls.length > 0) - ) -).pipe( - share() // Share the merged stream to prevent multiple subscriptions to source observables -) + ).subscribe({ + next: (event) => syncDerivedPnsResults$.next(event), + error: (err) => console.error('[useChatSync1081] performDerivedPnsSync error:', err), + complete: () => console.log('[useChatSync1081] performDerivedPnsSync complete'), + }) +} -// Sync kind 1080 events for all derived PNS pubkeys -const syncDerivedPnsEvents$ = syncDerivedPnsInputs$.pipe( +// Auto-sync when pubkeys/relays become available (initial sync) +const autoSyncDerivedPns$ = combineLatest([derivedPnsPubkeys$, relayUrlsDefined$]).pipe( filter(([pubkeys, _]) => pubkeys.length > 0), - switchMap(([pubkeys, relayUrls]) => { - // Reset sync stats for new sync - syncStatsDerivedPns.eventsReceived = 0 - syncStatsDerivedPns.lastSyncTime = new Date() - - // Create filter for all derived PNS pubkeys - const kind1080Filter = { - kinds: [KIND_PNS], - authors: pubkeys, - } - - console.log('[useChatSync1081] Syncing derived PNS events for pubkeys:', pubkeys) + take(1), // Only auto-sync once on initial load + tap(([pubkeys, relayUrls]) => { + console.log('[useChatSync1081] Auto-triggering initial derived PNS sync') + performDerivedPnsSync(pubkeys, relayUrls) + }), +) - return relayPool.sync(relayUrls, eventStore, kind1080Filter, SyncDirection.BOTH).pipe( - tap((event: NostrEvent) => { - syncStatsDerivedPns.eventsReceived++ - console.log('[useChatSync1081] Synced derived PNS event:', event.id, 'from:', event.pubkey.slice(0, 8)) - eventStore.add(event) +// Handle manual sync triggers - creates fresh sync each time +syncDerivedPnsTrigger$.pipe( + tap(() => console.log('[useChatSync1081] Manual sync trigger received')), + withLatestFrom(relayUrls$), + switchMap(([_, relayUrls]) => { + return derivedPnsPubkeys$.pipe( + take(1), + tap(pubkeys => { + if (pubkeys.length === 0) { + console.log('[useChatSync1081] No derived PNS keys, triggering stored events processing') + triggerProcessStored1081Events() + } }), - catchError((err) => { - console.error('[useChatSync1081] Derived PNS sync error:', err) - return EMPTY + switchMap(pubkeys => { + if (pubkeys.length === 0) { + // Wait for keys to be derived + return derivedPnsPubkeys$.pipe( + filter(keys => keys.length > 0), + take(1), + timeout({ first: 5000 }), + catchError(err => { + console.warn('[useChatSync1081] Timed out waiting for derived PNS keys:', err) + return of([]) + }) + ) + } + return of(pubkeys) }), + tap(pubkeys => { + if (pubkeys.length > 0 && relayUrls.length > 0) { + performDerivedPnsSync(pubkeys, relayUrls) + } else { + console.warn('[useChatSync1081] Cannot sync: pubkeys or relays missing', { pubkeys: pubkeys.length, relays: relayUrls.length }) + } + }) ) - }), - tap({ - error: (err) => console.error('[useChatSync1081] syncDerivedPnsEvents$ stream error:', err), - complete: () => console.log('[useChatSync1081] syncDerivedPnsEvents$ stream completed'), - }), - retry(1), - shareReplay(1), + }) +).subscribe() // This subscription never completes since syncDerivedPnsTrigger$ is a Subject + +// Expose results stream for components to subscribe to +const syncDerivedPnsEvents$ = syncDerivedPnsResults$.pipe( + share() // Use share() instead of shareReplay(1) to avoid caching completed state ) // Sync kind 1080 events for all derived PNS pubkeys @@ -503,13 +515,13 @@ const liveDerivedPnsEvents$ = combineLatest([derivedPnsPubkeys$, relayUrlsDefine authors: pubkeys } - console.log('[useChatSync1081] Live derived PNS events for pubkeys:', pubkeys) + // console.log('[useChatSync1081] Live derived PNS events for pubkeys:', pubkeys) return relayPool.subscription(relayUrls, kind1080Filter).pipe( onlyEvents(), tap((event: NostrEvent) => { syncStatsDerivedPns.eventsReceived++ - console.log('[useChatSync1081] Live derived PNS event:', event.id, 'from:', event.pubkey.slice(0, 8)) + // console.log('[useChatSync1081] Live derived PNS event:', event.id, 'from:', event.pubkey.slice(0, 8)) eventStore.add(event) }), defaultIfEmpty(null), @@ -551,8 +563,14 @@ export function useChatSync1081() { return () => sub.unsubscribe() }, []) + // Subscribe to auto-sync on initial load useEffect(() => { - console.log("CURRNET ", currentDerivedPnsKeys); + const sub = autoSyncDerivedPns$.subscribe() + return () => sub.unsubscribe() + }, []) + + useEffect(() => { + console.log("CURRENT ", currentDerivedPnsKeys); // Find the first PNS keys with SALT_PNS from currentDerivedPnsKeys const firstPnsKeysWithSalt = Array.from(currentDerivedPnsKeys.values()).find( pnsKeys => pnsKeys.salt === SALT_PNS @@ -626,14 +644,8 @@ export function useChatSync1081() { }, }) - // Also subscribe to the trigger to ensure the stream stays alive if it's hot - const triggerSub = syncDerivedPnsTrigger$.subscribe(() => { - console.log('[useChatSync1081] syncDerivedPnsTrigger$ fired in component') - }) - return () => { sub.unsubscribe() - triggerSub.unsubscribe() } }, []) diff --git a/hooks/useConversationState.ts b/hooks/useConversationState.ts index 05f20e3c..1981f3a4 100644 --- a/hooks/useConversationState.ts +++ b/hooks/useConversationState.ts @@ -130,6 +130,8 @@ export const useConversationState = (): UseConversationStateReturn => { } let hasNewEvents = false; + let failedDecryptions = 0; + let successfulDecryptions = 0; const eventsToLoad = eventStore.getByFilters({ kinds: [1080] }); eventsToLoad.forEach((event) => { @@ -141,6 +143,9 @@ export const useConversationState = (): UseConversationStateReturn => { // Decrypt and process the event const innerEvent = decryptPnsEventToInner(event, currentPnsKeys); if (!innerEvent) { + failedDecryptions++; + // Still mark as processed to avoid repeated decryption attempts + processedEventIdsRef.current.add(event.id); return; } @@ -148,8 +153,14 @@ export const useConversationState = (): UseConversationStateReturn => { processInnerEvent(conversationsMapRef.current, innerEvent); processedEventIdsRef.current.add(event.id); hasNewEvents = true; + successfulDecryptions++; }); + // Log decryption statistics for debugging + if (failedDecryptions > 0 || successfulDecryptions > 0) { + console.log(`[useConversationState] PNS event decryption stats: ${successfulDecryptions} successful, ${failedDecryptions} failed`); + } + // Update state with new conversations array if we processed any new events if (hasNewEvents) { const updatedConversations = Array.from(conversationsMapRef.current.values()); diff --git a/lib/pns.ts b/lib/pns.ts index c820f57c..a629b9cc 100644 --- a/lib/pns.ts +++ b/lib/pns.ts @@ -81,6 +81,32 @@ export function decryptPnsEvent( // Parse the decrypted contents as JSON return JSON.parse(plaintext); } catch (error) { + if (error instanceof Error) { + // Handle specific "invalid MAC" error which indicates decryption failure + if (error.message === 'invalid MAC') { + console.warn('PNS event decryption failed: invalid MAC - likely encrypted with different keys', { + eventId: pnsEvent.id, + pubkey: pnsEvent.pubkey + }); + return null; + } + + // Handle JSON parse errors separately + if (error.message.includes('Unexpected token') || error.message.includes('JSON')) { + console.warn('Failed to parse decrypted PNS event content as JSON:', { + eventId: pnsEvent.id, + error: error.message + }); + return null; + } + + console.error('PNS event decryption error:', error.message, { + eventId: pnsEvent.id, + pubkey: pnsEvent.pubkey + }); + return null; + } + console.error('Failed to decrypt PNS event:', error); return null; } diff --git a/utils/eventProcessing.ts b/utils/eventProcessing.ts index db3594f7..c2a1d0e7 100644 --- a/utils/eventProcessing.ts +++ b/utils/eventProcessing.ts @@ -43,7 +43,17 @@ export function decryptPnsEventToInner( // Decrypt PNS Event -> Inner Event const inner = decryptPnsEvent(pnsEvent, pnsKeys); - if (!inner || inner.kind !== KIND_CHAT_INNER) { + if (!inner) { + // Log when decryption returns null (already logged in decryptPnsEvent) + return null; + } + + if (inner.kind !== KIND_CHAT_INNER) { + console.warn('PNS event decrypted but not a chat inner event:', { + eventId: pnsEvent.id, + actualKind: inner.kind, + expectedKind: KIND_CHAT_INNER + }); return null; } @@ -53,7 +63,10 @@ export function decryptPnsEventToInner( id: pnsEvent.id, }; } catch (error) { - console.warn('Failed to decrypt PNS event:', error); + console.warn('Unexpected error in decryptPnsEventToInner:', { + eventId: pnsEvent.id, + error: error instanceof Error ? error.message : error + }); return null; } } From e6e87e5c0c88ec521c992d4144d18ff7eb2b88f3 Mon Sep 17 00:00:00 2001 From: redshift <213178690+1ftredsh@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:32:31 +0800 Subject: [PATCH 07/47] deletion works perfectly. --- components/chat/Sidebar.tsx | 4 +-- hooks/useConversationState.ts | 54 +++++++++++++++++++++++++++++++---- hooks/useDeletionSync.ts | 37 ++++++++++++++++++++++++ lib/pns.ts | 35 +++++++++++++++++++++++ utils/apiUtils.ts | 3 -- 5 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 hooks/useDeletionSync.ts diff --git a/components/chat/Sidebar.tsx b/components/chat/Sidebar.tsx index e682df2e..8e8fcf1d 100644 --- a/components/chat/Sidebar.tsx +++ b/components/chat/Sidebar.tsx @@ -12,7 +12,7 @@ interface SidebarProps { activeConversationId: string | null; createNewConversation: () => void; loadConversation: (id: string) => void; - deleteConversation: (id: string, e: React.MouseEvent) => void; + deleteConversation: (id: string, e: React.MouseEvent) => Promise; setIsSettingsOpen: (isOpen: boolean) => void; setInitialSettingsTab: (tab: 'settings' | 'wallet' | 'history' | 'api-keys') => void; balance: number; @@ -118,7 +118,7 @@ export default function Sidebar({ {conversation.title} ) : (