wip-nov2

Signed-off-by: Eduardo Silva <[email protected]>

Author: edsiper

CHUNKS.md

@@ -91,6 +91,47 @@ Content Data <       |  |  records  |  |
 Fluent Bit API provides backward compatibility with the previous metadata and content
 format found on series v1.8.
 
+Starting with the Fluent Bit release that introduces direct route persistence, the
+fourth metadata byte now carries feature flags. A zero value preserves the legacy
+layout, while a non-zero value indicates that additional structures follow the tag.
+When the ``FLB_CHUNK_FLAG_DIRECT_ROUTES`` bit is set the tag is terminated with a
+single ``\0`` byte and a routing payload is appended.  Fluent Bit v3.0 and later
+also set the ``FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS`` bit to store each destination's
+alias (or generated name) alongside its numeric identifier so routes can survive
+configuration changes that renumber outputs:
+
+```
+Metadata
+                --   +---------+-------+
+               /     |  0xF1   | 0x77  |  <- Magic Bytes
+              /      +---------+-------+
+             <       |  Type   | Flags |  <- Chunk type and flag bits
+              \      +---------+-------+
+               \     |      Tag        |  <- Tag string (no size prefix)
+                     +---------+-------+
+                     |  0x00   |       |  <- Tag terminator (present when flags != 0)
+                     +---------+-------+
+                     | Routing Length  |  <- uint16_t big endian
+                     +---------+-------+
+                     | Route Count     |  <- uint16_t big endian
+                     +---------+-------+
+                     | Output IDs ...  |  <- Each stored as uint16_t (big endian)
+                     +---------+-------+
+                     | Label Lens ...  |  <- Present when FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS
+                     +---------+-------+
+                     | Label Bytes ... |  <- Concatenated label strings (no terminators)
+                --   +-----------------+
+```
+
+The routing payload captures the direct route mapping so that filesystem chunks
+loaded by the storage backlog re-use the same outputs after a restart. Chunks
+without direct routes keep the legacy layout (flags byte set to zero) and remain
+fully backwards compatible across Fluent Bit versions. When labels are stored the
+reader first reconstructs routes by matching aliases or numbered names and only
+falls back to numeric identifiers if the textual metadata cannot be matched. This
+ensures that chunks continue to flow to the intended destinations even when the
+output configuration is re-ordered.
+
 ### Fluent Bit <= v1.8
 
 Up to Fluent Bit <= 1.8.x, the metadata and content data is simple, where metadata

include/fluent-bit/flb_input_chunk.h

@@ -25,6 +25,8 @@
 #include <fluent-bit/flb_config.h>
 #include <fluent-bit/flb_routes_mask.h>
 
+struct cio_chunk;
+
 #include <monkey/mk_core.h>
 #include <msgpack.h>
 
@@ -42,6 +44,16 @@
 /* Number of bytes reserved for Metadata Header on Chunks */
 #define FLB_INPUT_CHUNK_META_HEADER   4
 
+/* Chunk metadata flags */
+#define FLB_CHUNK_FLAG_DIRECT_ROUTES        (1 << 0)
+#define FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS  (1 << 1)
+
+struct flb_chunk_direct_route {
+    uint16_t id;
+    uint16_t label_length;
+    const char *label;
+};
+
 /* Chunks magic bytes (starting from Fluent Bit v1.8.10) */
 #define FLB_INPUT_CHUNK_MAGIC_BYTE_0  (unsigned char) 0xF1
 #define FLB_INPUT_CHUNK_MAGIC_BYTE_1  (unsigned char) 0x77
@@ -110,6 +122,18 @@ int flb_input_chunk_get_event_type(struct flb_input_chunk *ic);
 int flb_input_chunk_get_tag(struct flb_input_chunk *ic,
                             const char **tag_buf, int *tag_len);
 
+int flb_input_chunk_write_header_v2(struct cio_chunk *chunk,
+                                    int event_type,
+                                    char *tag, int tag_len,
+                                    const struct flb_chunk_direct_route *routes,
+                                    int route_count);
+int flb_input_chunk_has_direct_routes(struct flb_input_chunk *ic);
+int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic,
+                                      struct flb_chunk_direct_route **routes,
+                                      int *route_count);
+void flb_input_chunk_destroy_direct_routes(struct flb_chunk_direct_route *routes,
+                                           int route_count);
+
 void flb_input_chunk_ring_buffer_cleanup(struct flb_input_instance *ins);
 void flb_input_chunk_ring_buffer_collector(struct flb_config *ctx, void *data);
 ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic);

lib/chunkio/src/cio_file.c

@@ -1106,7 +1106,7 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size)
     /* set new position for the content data */
     cur_content_data = cio_file_st_get_content(cf->map);
     new_content_data = meta + size;
-    memmove(new_content_data, cur_content_data, size);
+    memmove(new_content_data, cur_content_data, cf->data_size);
 
     /* copy new metadata */
     memcpy(meta, buf, size);