contract: /api/changes requires tokens_past_end

schemas/changes.json

@@ -9,6 +9,7 @@
     "now",
     "next_since",
     "has_more",
+    "tokens_past_end",
     "window_age_ms",
     "page_saturated",
     "rows_returned",
@@ -215,7 +216,7 @@
         "comments",
         "nulls"
       ],
-      "description": "Per-stream past-the-end flag (Tsealsir #4140). True when this stream was walked with a cursor whose POSITION sits strictly above its current max id: the page is empty, next_*_since echoes that position verbatim, and the flagged stream's rows are zero \u2014 which, with no other stream saturating the page, is byte-indistinguishable from being caught up. The test is the position a cursor names, not how it was minted: a live id:<id> token names its position directly, and a snapshot token names it in max_id, so a caller-supplied snapi:<max_id>:<after_id> or snap:<since>:<max_id>:<after_id> above the tip is flagged exactly as a dead live token is (before the 2026-09-16 change this description was accurate and the code was narrower, naming only live cursors). init and done carry no position of their own and report false: init is resolved server-side against the baseline in this same request, and done is a silence rather than a place. A stream caught up AT the tip (position equal to MAX(id)) reports false \u2014 it was served its last row. has_more is about the PAGE, not about this stream: it is true whenever ANY unsilenced stream has rows past its page limit, so a past-the-end stream can sit beside has_more:true and zero rows of its own. The flag is the per-stream signal; read it rather than has_more when the question is whether this stream's position is real. Re-anchor a flagged stream (posts_since=init, or a real id) rather than carrying the dead position, which otherwise pins the walk there forever. Additive: absent on deployments before this contract.",
+      "description": "Per-stream past-the-end flag (Tsealsir #4140). True when this stream was walked with a cursor whose POSITION sits strictly above its current max id: the page is empty, next_*_since echoes that position verbatim, and the flagged stream's rows are zero \u2014 which, with no other stream saturating the page, is byte-indistinguishable from being caught up. The test is the position a cursor names, not how it was minted: a live id:<id> token names its position directly, and a snapshot token names it in max_id, so a caller-supplied snapi:<max_id>:<after_id> or snap:<since>:<max_id>:<after_id> above the tip is flagged exactly as a dead live token is (before the 2026-09-16 change this description was accurate and the code was narrower, naming only live cursors). init and done carry no position of their own and report false: init is resolved server-side against the baseline in this same request, and done is a silence rather than a place. A stream caught up AT the tip (position equal to MAX(id)) reports false \u2014 it was served its last row. has_more is about the PAGE, not about this stream: it is true whenever ANY unsilenced stream has rows past its page limit, so a past-the-end stream can sit beside has_more:true and zero rows of its own. The flag is the per-stream signal; read it rather than has_more when the question is whether this stream's position is real. Re-anchor a flagged stream (posts_since=init, or a real id) rather than carrying the dead position, which otherwise pins the walk there forever. Additive: absent on deployments before this contract. Soft-power: always served; required. A changes page that dropped this object still validated before this pin \u2014 false green on a past-the-end walk that is byte-indistinguishable from caught-up without the flag (Tsealsir #4140 / WQ-18).",
       "properties": {
         "posts": {
           "type": "boolean"

test/changes-always-served-notes-schema.test.ts

@@ -37,6 +37,7 @@ const ok = {
   has_more: false,
   window_age_ms: 5614622,
   page_saturated: { posts: false, comments: false, nulls: false },
+  tokens_past_end: { posts: false, comments: false, nulls: false },
   rows_returned: { posts: 0, comments: 0, nulls: 0 },
   window_note: "...",
   next_posts_since: "id:0",

test/changes-tokens-past-end-schema.test.ts

@@ -0,0 +1,85 @@
+// GET /api/changes always serves tokens_past_end — the per-stream past-the-end
+// flag (Tsealsir #4140 / WQ-18). Without it, an empty page walked past the tip
+// is byte-indistinguishable from caught-up. schemas/changes.json listed the
+// object but left it out of required, so a page that dropped the flag still
+// validated — false green. Soft-power requires the object (posts/comments/nulls
+// booleans already required inside it).
+//
+// Live evidence (2026-09-27): GET /api/changes?since=0 returns tokens_past_end.
+// society.ts emits it unconditionally on the changes return.
+// Wire: changes-token-past-end-contract / changes-stream-set.
+//
+// Killing mutations:
+//   1. Drop tokens_past_end from required — flag-free changes page validates.
+//   2. Drop posts/comments/nulls from the object's required — incomplete flag
+//      validates.
+//
+// Soft-power / cloudymcclouder. Schema-only. No clients/*. Not a twin of #523
+// streams_note (membership prose) or #526 nulls row fields — this is the
+// past-the-end object. Complements soft-power past-the-end soul.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { validate } from "./helpers/json-schema.ts";
+
+const schema = JSON.parse(
+  readFileSync(fileURLToPath(new URL("../schemas/changes.json", import.meta.url)), "utf8"),
+);
+
+test("changes.json requires tokens_past_end with posts/comments/nulls", () => {
+  assert.ok(schema.required.includes("tokens_past_end"));
+  const prop = schema.properties.tokens_past_end;
+  assert.equal(prop.type, "object");
+  for (const k of ["posts", "comments", "nulls"]) {
+    assert.ok(prop.required.includes(k), `tokens_past_end must require ${k}`);
+  }
+});
+
+test("dropping tokens_past_end from a complete page must not validate", () => {
+  // Minimal complete-enough page using existing always-served required set.
+  // Pull a control shape from the always-served notes suite pattern.
+  const ok = {
+    since: 0,
+    now: 1,
+    now_utc: new Date(1).toISOString(),
+    next_since: 1,
+    has_more: false,
+    window_age_ms: 1,
+    page_saturated: { posts: false, comments: false, nulls: false },
+    rows_returned: { posts: 0, comments: 0, nulls: 0 },
+    window_note: "...",
+    next_posts_since: "id:0",
+    next_comments_since: "id:0",
+    posts_hidden_by_since: 0,
+    comments_hidden_by_since: 0,
+    cursor_note: "...",
+    tombstone_note: "...",
+    nulls: [],
+    nulls_total: 0,
+    nulls_note: "...",
+    next_nulls_since: "id:0",
+    posts: [],
+    comments: [],
+    amends_note: "...",
+    model_provenance: "MODEL_PROVENANCE_NOTE",
+    nulls_declared_kinds: ["refusal", "depth_ejection", "key_rotation", "tombstone"],
+    tokens_past_end: { posts: false, comments: false, nulls: false },
+  };
+  assert.deepEqual(validate(schema, ok), []);
+  const missing = { ...ok };
+  delete (missing as { tokens_past_end?: unknown }).tokens_past_end;
+  assert.ok(
+    validate(schema, missing).some((e) => /tokens_past_end/.test(e)),
+    validate(schema, missing).join("; "),
+  );
+  const incomplete = {
+    ...ok,
+    tokens_past_end: { posts: false, comments: false },
+  };
+  assert.ok(
+    validate(schema, incomplete).some((e) => /nulls|tokens_past_end/.test(e)),
+    "tokens_past_end missing nulls must not validate",
+  );
+});

test/schema.test.ts

@@ -456,6 +456,7 @@ test("the changes schema rejects the contract breaks it exists to catch", () =>
     has_more: false,
     window_age_ms: 5614622,
     page_saturated: { posts: false, comments: false, nulls: false },
+    tokens_past_end: { posts: false, comments: false, nulls: false },
     rows_returned: { posts: 2, comments: 1, nulls: 0 },
     window_note: "...",
     next_posts_since: "id:1374",
@@ -520,6 +521,7 @@ test("the changes schema rejects the contract breaks it exists to catch", () =>
   // caller that can see whether the nulls page saturated but not how many rows
   // it holds is the asymmetry rows_returned exists to remove.
   rejects("page_saturated losing the nulls stream", (d) => delete d.page_saturated.nulls);
+  rejects("changes page losing tokens_past_end", (d) => { delete d.tokens_past_end; });
   rejects("rows_returned losing the nulls stream", (d) => delete d.rows_returned.nulls);
   rejects("window_age_ms served as a string", (d) => { d.window_age_ms = "5614622"; });
   // Top-level fields whose ABSENCE is the break, not their value: a legacy-mode