feat(backend): Signal support for handshake nonce (#5905)

Author: jacekradko

.changeset/six-ears-wash.md

@@ -0,0 +1,36 @@
+---
+'@clerk/backend': minor
+---
+
+## Optimize handshake payload delivery with nonce-based fetching
+
+This change introduces a significant optimization to the handshake flow by replacing direct payload delivery with a nonce-based approach to overcome browser cookie size limitations.
+
+## Problem Solved
+Previously, the handshake payload (an encoded JWT containing set-cookie headers) was sent directly in a cookie. Since browsers limit cookies to ~4KB, this severely restricted the practical size of session tokens, which are also JWTs stored in cookies but embedded within the handshake payload.
+
+## Solution 
+We now use a conditional approach based on payload size:
+- **Small payloads (≤2KB)**: Continue using the direct approach for optimal performance
+- **Large payloads (>2KB)**: Use nonce-based fetching to avoid cookie size limits
+
+For large payloads, we:
+1. Generate a short nonce (ID) for each handshake instance
+2. Send only the nonce in the `__clerk_handshake_nonce` cookie
+3. Use the nonce to fetch the actual handshake payload via a dedicated BAPI endpoint
+
+## New Handshake Flow (for payloads >2KB)
+1. User visits `example.com`
+2. Client app middleware triggers handshake → `307 FAPI/v1/client/handshake`
+3. FAPI handshake resolves → `307 example.com` with `__clerk_handshake_nonce` cookie containing the nonce
+4. Client app middleware makes `GET BAPI/v1/clients/handshake_payload?nonce=<nonce_value>` request (BAPI)
+5. BAPI returns array of set-cookie header values
+6. Client app middleware applies headers to the response
+
+## Traditional Flow (for payloads ≤2KB)
+No changes. Continues to work as before with direct payload delivery in cookies for optimal performance.
+
+## Trade-offs
+- **Added**: One additional BAPI call per handshake (only for payloads >2KB)
+- **Removed**: Cookie size restrictions that previously limited session token size
+

packages/backend/src/constants.ts

@@ -35,6 +35,7 @@ const QueryParameters = {
   LegacyDevBrowser: '__dev_session',
   HandshakeReason: '__clerk_hs_reason',
   HandshakeNonce: Cookies.HandshakeNonce,
+  HandshakeFormat: 'format',
 } as const;
 
 const Headers = {

packages/backend/src/tokens/__tests__/handshake.test.ts

@@ -155,6 +155,7 @@ describe('HandshakeService', () => {
       expect(url.searchParams.get('redirect_url')).toBe('https://example.com/');
       expect(url.searchParams.get(constants.QueryParameters.SuffixedCookies)).toBe('true');
       expect(url.searchParams.get(constants.QueryParameters.HandshakeReason)).toBe('test-reason');
+      expect(url.searchParams.get(constants.QueryParameters.HandshakeFormat)).toBe('nonce');
     });
 
     it('should include dev browser token in development mode', () => {
@@ -168,6 +169,7 @@ describe('HandshakeService', () => {
       const url = new URL(location);
 
       expect(url.searchParams.get(constants.QueryParameters.DevBrowser)).toBe('dev-token');
+      expect(url.searchParams.get(constants.QueryParameters.HandshakeFormat)).toBe('nonce');
     });
 
     it('should throw error if clerkUrl is missing', () => {

packages/backend/src/tokens/handshake.ts

@@ -149,6 +149,7 @@ export class HandshakeService {
       this.authenticateContext.usesSuffixedCookies().toString(),
     );
     url.searchParams.append(constants.QueryParameters.HandshakeReason, reason);
+    url.searchParams.append(constants.QueryParameters.HandshakeFormat, 'nonce');
 
     if (this.authenticateContext.instanceType === 'development' && this.authenticateContext.devBrowserToken) {
       url.searchParams.append(constants.QueryParameters.DevBrowser, this.authenticateContext.devBrowserToken);