From 227433f9dc28649e62ff53e0d4a2a8cd7b931bc7 Mon Sep 17 00:00:00 2001
From: StrathCole <7449529+StrathCole@users.noreply.github.com>
Date: Thu, 23 Oct 2025 07:56:16 +0200
Subject: [PATCH] Implement flush threshold for Pebble batch operations

Added a flush threshold to prevent Pebble batch from exceeding 4 GB limit during write operations.
---
 pebble.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/pebble.go b/pebble.go
index ca55d00..4ee2cf8 100644
--- a/pebble.go
+++ b/pebble.go
@@ -21,6 +21,7 @@ type PebbleDB struct {
 }
 
 var _ DB = (*PebbleDB)(nil)
+const flushThreshold = 3_500_000_000 // ~3.5 GB
 
 func NewPebbleDB(name string, dir string) (*PebbleDB, error) {
 	opts := &pebble.Options{}
@@ -250,6 +251,14 @@ func (b *pebbleDBBatch) Set(key, value []byte) error {
 		return errBatchClosed
 	}
 
+	// Prevent Pebble batch from exceeding 4 GB hard limit
+	if b.batch.Len() > flushThreshold {
+		if err := b.batch.Commit(pebble.Sync); err != nil {
+			return err
+		}
+		b.batch.Reset()
+	}
+
 	return b.batch.Set(key, value, nil)
 }
 
@@ -262,6 +271,14 @@ func (b *pebbleDBBatch) Delete(key []byte) error {
 		return errBatchClosed
 	}
 
+	// Prevent Pebble batch from exceeding 4 GB hard limit
+	if b.batch.Len() > flushThreshold {
+		if err := b.batch.Commit(pebble.Sync); err != nil {
+			return err
+		}
+		b.batch.Reset()
+	}
+
 	return b.batch.Delete(key, nil)
 }