Implement scale-safe lookup calls

Add transparent Level 2 lookup split/stitch behavior across C, Go, and Rust.

Cover lookup scale, malformed responses, interop, coverage, and benchmark validation.

Author: ktsaou

.agents/sow/current/SOW-0021-20260613-netipc-at-scale.md

@@ -2,9 +2,9 @@
 
 ## Status
 
-Status: open
+Status: closed
 
-Sub-state: follow-up tracked from SOW-0006; not yet scheduled.
+Sub-state: superseded before implementation by `SOW-0021 - NetIPC At Scale`.
 
 ## Requirements
 
@@ -152,21 +152,28 @@ Pending.
 
 - Created as the tracked follow-up for SOW-0006's explicitly scoped-out transparent split/stitch work.
 
+### 2026-06-13
+
+- Closed before implementation because `SOW-0021 - NetIPC At Scale` supersedes this narrower request-splitting SOW.
+- The old all-or-error oversized single-item assumption is explicitly replaced by SOW-0021's item-level outcome contract.
+- The old staged C/Rust/Go parity option is explicitly replaced by SOW-0021's all-language completion requirement.
+
 ## Validation
 
-Pending.
+- Not executed in this SOW because the implementation and validation scope moved to SOW-0021 before this SOW started.
+- SOW-0021 owns the full C/Rust/Go, POSIX/Windows, interop, coverage, benchmark, and downstream vendoring validation.
 
 ## Outcome
 
-Pending.
+Closed as superseded by `SOW-0021 - NetIPC At Scale`.
 
 ## Lessons Extracted
 
 Pending.
 
 ## Followup
 
-None yet.
+The active implementation and validation follow-up is SOW-0021.
 
 ## Regression Log
 

…28-transparent-lookup-batch-splitting.md …28-transparent-lookup-batch-splitting.md.agents/sow/pending/SOW-0007-20260528-transparent-lookup-batch-splitting.md renamed to .agents/sow/done/SOW-0007-20260528-transparent-lookup-batch-splitting.md .agents/sow/pending/SOW-0007-20260528-transparent-lookup-batch-splitting.md renamed to .agents/sow/done/SOW-0007-20260528-transparent-lookup-batch-splitting.md

@@ -311,6 +311,9 @@ if(NOT NETIPC_WINDOWS_RUNTIME AND NOT APPLE)
     target_link_libraries(test_service_extra PRIVATE
         netipc_service netipc_shm netipc_uds netipc_protocol Threads::Threads
     )
+    target_include_directories(test_service_extra PRIVATE
+        src/libnetdata/netipc/src/service
+    )
     target_compile_definitions(test_service_extra PRIVATE NIPC_INTERNAL_TESTING=1)
     add_test(NAME test_service_extra COMMAND test_service_extra)
     set_tests_properties(test_service_extra PROPERTIES TIMEOUT 120)
@@ -568,6 +571,15 @@ if(NOT NETIPC_WINDOWS_RUNTIME AND NOT APPLE AND CARGO_EXECUTABLE)
         TIMEOUT 120
         ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_c>;INTEROP_SVC_RS=${INTEROP_SVC_RS};INTEROP_SVC_GO=${INTEROP_SVC_GO}"
     )
+
+    add_test(NAME test_lookup_scale_interop
+        COMMAND bash "${CMAKE_SOURCE_DIR}/tests/test_lookup_scale_interop.sh"
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    )
+    set_tests_properties(test_lookup_scale_interop PROPERTIES
+        TIMEOUT 180
+        ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_c>;INTEROP_SVC_RS=${INTEROP_SVC_RS};INTEROP_SVC_GO=${INTEROP_SVC_GO};NIPC_LOOKUP_SCALE_ITEMS=8192"
+    )
 endif()
 
 # --- L3 cache tests (Linux only) -------------------------------------------
@@ -663,6 +675,15 @@ if(NOT NETIPC_WINDOWS_RUNTIME AND NOT APPLE AND CARGO_EXECUTABLE)
         TIMEOUT 120
         ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_c>;INTEROP_SVC_RS=${INTEROP_SVC_RS};INTEROP_SVC_GO=${INTEROP_SVC_GO}"
     )
+
+    add_test(NAME test_lookup_scale_shm_interop
+        COMMAND bash "${CMAKE_SOURCE_DIR}/tests/test_lookup_scale_interop.sh"
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    )
+    set_tests_properties(test_lookup_scale_shm_interop PROPERTIES
+        TIMEOUT 180
+        ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_c>;INTEROP_SVC_RS=${INTEROP_SVC_RS};INTEROP_SVC_GO=${INTEROP_SVC_GO};NIPC_LOOKUP_SCALE_ITEMS=8192;NIPC_PROFILE=shm"
+    )
 endif()
 
 # --- Cross-language L3 cache interop tests (Linux only) ---------------------
@@ -964,6 +985,26 @@ if(NETIPC_WINDOWS_RUNTIME AND CARGO_EXECUTABLE)
         RESOURCE_LOCK win_service_interop_tests
         ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_win_c>;INTEROP_SVC_RS=${INTEROP_SVC_WIN_RS};INTEROP_SVC_GO=${INTEROP_SVC_WIN_GO}"
     )
+
+    add_test(NAME test_lookup_scale_win_interop
+        COMMAND bash "${CMAKE_SOURCE_DIR}/tests/test_lookup_scale_win_interop.sh"
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    )
+    set_tests_properties(test_lookup_scale_win_interop PROPERTIES
+        TIMEOUT 300
+        RESOURCE_LOCK win_service_interop_tests
+        ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_win_c>;INTEROP_SVC_RS=${INTEROP_SVC_WIN_RS};INTEROP_SVC_GO=${INTEROP_SVC_WIN_GO};NIPC_LOOKUP_SCALE_ITEMS=8192"
+    )
+
+    add_test(NAME test_lookup_scale_win_shm_interop
+        COMMAND bash "${CMAKE_SOURCE_DIR}/tests/test_lookup_scale_win_shm_interop.sh"
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    )
+    set_tests_properties(test_lookup_scale_win_shm_interop PROPERTIES
+        TIMEOUT 300
+        RESOURCE_LOCK win_service_interop_tests
+        ENVIRONMENT "INTEROP_SVC_C=$<TARGET_FILE:interop_service_win_c>;INTEROP_SVC_RS=${INTEROP_SVC_WIN_RS};INTEROP_SVC_GO=${INTEROP_SVC_WIN_GO};NIPC_LOOKUP_SCALE_ITEMS=8192"
+    )
 endif()
 
 # --- Cross-language Windows L3 cache interop tests --------------------------

CMakeLists.txt

@@ -14,6 +14,28 @@ same wire contracts and typed APIs. The goal is simple:
 This README is a summary of the current verified state of the repository.
 The authoritative specifications live under [docs/](docs/README.md).
 
+## Non-Negotiable Contract
+
+- These rules apply to every implementation language and every service unless
+  a future spec explicitly changes the global contract.
+- C, Rust, and Go must implement the same wire contract and typed behavior.
+- NetIPC does not do backward-compatible, forward-compatible, or best-effort
+  decoding across method, layout, status, echoed-key, or generation drift.
+- Netdata plugins and NetIPC providers/clients must match the documented
+  contract exactly. Any mismatch is rejected.
+- Mixed-generation stitched lookup responses are not supported.
+- Level 2 lookup callers pass typed semantic keys; they do not split requests
+  around transport payload budgets and do not stitch responses manually.
+- Lookup `PAYLOAD_EXCEEDED` is an internal Level 2 retry signal. The client
+  retries only the affected suffix and returns one logical response.
+- Lookup `OVERSIZED_ITEM` is a final per-item outcome. One oversized valid item
+  must not invalidate the rest of the logical lookup batch.
+- Payload budgets and logical lookup ceilings are initialization policy.
+  Zero-valued fields use documented defaults; consumers can override them for
+  small IoT systems or large-memory HPC deployments.
+- Named defaults are defaults, not protocol hard limits. Explicit client/server
+  initialization config is the deployment authority.
+
 ## Service Model
 
 The public contract is service-oriented, not plugin-oriented:

README.md

@@ -48,6 +48,9 @@
 #define RESPONSE_BUF_SIZE  65536
 #define MAX_LATENCY_SAMPLES (10 * 1000 * 1000) /* 10M samples max */
 #define DEFAULT_DURATION   30 /* seconds */
+#define LOOKUP_METHOD_MAX_ITEMS 32768u
+#define LOOKUP_METHOD_PATH_BYTES 64u
+#define LOOKUP_METHOD_BUF_BYTES_PER_ITEM 256u
 
 /* Profiles for SHM vs baseline */
 #define BENCH_PROFILE_UDS  NIPC_PROFILE_BASELINE
@@ -1216,19 +1219,31 @@ static int parse_lookup_method_scenario(const char *scenario,
     else
         return 0;
 
-    size_t scenario_len = strlen(scenario);
-    if (scenario_len >= 4 && strcmp(scenario + scenario_len - 4, "-256") == 0)
-        *item_count = 256;
-    else if (scenario_len >= 3 && strcmp(scenario + scenario_len - 3, "-16") == 0)
-        *item_count = 16;
-    else if (scenario_len >= 2 && strcmp(scenario + scenario_len - 2, "-1") == 0)
-        *item_count = 1;
-    else
+    const char *dash = strrchr(scenario, '-');
+    if (!dash || dash[1] == '\0')
+        return 0;
+
+    char *end = NULL;
+    errno = 0;
+    unsigned long parsed = strtoul(dash + 1, &end, 10);
+    if (errno != 0 || !end || *end != '\0' || parsed == 0 ||
+        parsed > LOOKUP_METHOD_MAX_ITEMS)
         return 0;
 
+    if (parsed != 1 && parsed != 16 && parsed != 256 &&
+        parsed != 8192 && parsed != 32768)
+        return 0;
+
+    *item_count = (uint32_t)parsed;
     return 1;
 }
 
+static size_t lookup_method_buffer_size(uint32_t item_count)
+{
+    size_t scaled = (size_t)item_count * LOOKUP_METHOD_BUF_BYTES_PER_ITEM + 4096u;
+    return scaled > RESPONSE_BUF_SIZE ? scaled : RESPONSE_BUF_SIZE;
+}
+
 static int run_lookup_method_bench(int duration_sec,
                                    const char *scenario,
                                    uint64_t target_rps)
@@ -1241,9 +1256,17 @@ static int run_lookup_method_bench(int duration_sec,
         return 1;
     }
 
-    char path_storage[256][64];
-    nipc_str_view_t paths[256];
-    uint32_t pids[256];
+    char (*path_storage)[LOOKUP_METHOD_PATH_BYTES] =
+        calloc(item_count, sizeof(*path_storage));
+    nipc_str_view_t *paths = calloc(item_count, sizeof(*paths));
+    uint32_t *pids = malloc((size_t)item_count * sizeof(*pids));
+    if (!path_storage || !paths || !pids) {
+        free(path_storage);
+        free(paths);
+        free(pids);
+        return 1;
+    }
+
     for (uint32_t i = 0; i < item_count; i++) {
         int n = snprintf(path_storage[i], sizeof(path_storage[i]),
                          "/sys/fs/cgroup/bench/cg-%03u", i);
@@ -1252,9 +1275,13 @@ static int run_lookup_method_bench(int duration_sec,
         pids[i] = 1000u + i;
     }
 
-    uint8_t *req_buf = malloc(RESPONSE_BUF_SIZE);
-    uint8_t *resp_buf = malloc(RESPONSE_BUF_SIZE);
+    size_t io_buf_size = lookup_method_buffer_size(item_count);
+    uint8_t *req_buf = malloc(io_buf_size);
+    uint8_t *resp_buf = malloc(io_buf_size);
     if (!req_buf || !resp_buf) {
+        free(path_storage);
+        free(paths);
+        free(pids);
         free(req_buf);
         free(resp_buf);
         return 1;
@@ -1284,12 +1311,12 @@ static int run_lookup_method_bench(int duration_sec,
         nipc_error_t err;
 
         if (is_apps) {
-            req_len = nipc_apps_lookup_req_encode(pids, item_count, req_buf, RESPONSE_BUF_SIZE);
+            req_len = nipc_apps_lookup_req_encode(pids, item_count, req_buf, io_buf_size);
             if (req_len == 0) {
                 errors++;
                 continue;
             }
-            err = nipc_dispatch_apps_lookup(req_buf, req_len, resp_buf, RESPONSE_BUF_SIZE,
+            err = nipc_dispatch_apps_lookup(req_buf, req_len, resp_buf, io_buf_size,
                                             &resp_len, apps_lookup_bench_handler, &ctx);
             if (err == NIPC_OK) {
                 nipc_apps_lookup_resp_view_t view;
@@ -1298,12 +1325,12 @@ static int run_lookup_method_bench(int duration_sec,
                     err = NIPC_ERR_BAD_ITEM_COUNT;
             }
         } else {
-            req_len = nipc_cgroups_lookup_req_encode(paths, item_count, req_buf, RESPONSE_BUF_SIZE);
+            req_len = nipc_cgroups_lookup_req_encode(paths, item_count, req_buf, io_buf_size);
             if (req_len == 0) {
                 errors++;
                 continue;
             }
-            err = nipc_dispatch_cgroups_lookup(req_buf, req_len, resp_buf, RESPONSE_BUF_SIZE,
+            err = nipc_dispatch_cgroups_lookup(req_buf, req_len, resp_buf, io_buf_size,
                                                &resp_len, cgroups_lookup_bench_handler, &ctx);
             if (err == NIPC_OK) {
                 nipc_cgroups_lookup_resp_view_t view;
@@ -1345,6 +1372,9 @@ static int run_lookup_method_bench(int duration_sec,
         fprintf(stderr, "lookup-method-bench: %lu errors\n", (unsigned long)errors);
 
     latency_free(&lr);
+    free(path_storage);
+    free(paths);
+    free(pids);
     free(req_buf);
     free(resp_buf);
     return errors > 0 ? 1 : 0;