StreamingTextResponse not compatible with Remix action function

[cephalization]

When using the package in an action, like so,

export const action = async ({ request }: ActionArgs) => {
    const { messages, transcriptId } = z
        .object({
            messages: z.array(
                z
                    .object({
                        content: z.string(),
                        role: z.union([z.literal('user'), z.literal('system'), z.literal('assistant')]),
                    })
                    .passthrough(),
            ),
            transcriptId: z.string(),
        })
        .parse(await request.json());

    // Load transcript set from Redis, turn it into a string
    const transcriptArray = await transcriptQueries.getTranscriptArray(redisClient, { transcriptKey: transcriptId });
    const transcript = transcriptArray;
    const prompt = `<my prompt>`;
    const content = prompt.replace('{transcript}', constrainLinesToTokenLimit(transcript, prompt, 6000).join('\n'));

    const newMessages = [
        {
            role: 'system',
            content,
        },
        ...messages,
    ] as const;

    const response = await openai.createChatCompletion({
        // model: 'gpt-3.5-turbo-16k',
        model: 'gpt-4',
        // @ts-expect-error
        messages: newMessages,
        stream: true,
    });
    const stream = OpenAIStream(response);

    return new StreamingTextResponse(stream);
};

The action returns an internal error

TypeError: First parameter has member 'readable' that is not a ReadableStream.
dashboard:dev:     at assertReadableStream (/Users/tony/repos/teno/node_modules/web-streams-polyfill/src/lib/validators/readable-stream.ts:5:11)
dashboard:dev:     at convertReadableWritablePair (/Users/tony/repos/teno/node_modules/web-streams-polyfill/src/lib/validators/readable-writable-pair.ts:15:3)
dashboard:dev:     at ReadableStream.pipeThrough (/Users/tony/repos/teno/node_modules/web-streams-polyfill/src/lib/readable-stream.ts:211:23)
dashboard:dev:     at AIStream (/Users/tony/repos/teno/node_modules/ai/dist/index.js:147:29)
dashboard:dev:     at OpenAIStream (/Users/tony/repos/teno/node_modules/ai/dist/index.js:170:10)
dashboard:dev:     at action7 (/Users/tony/repos/teno/apps/dashboard/app/routes/api.chat.ts:68:17)
dashboard:dev:     at processTicksAndRejections (node:internal/process/task_queues:95:5)
dashboard:dev:     at Object.callRouteActionRR (/Users/tony/repos/teno/node_modules/@remix-run/node/node_modules/@remix-run/server-runtime/dist/data.js:35:16)
dashboard:dev:     at callLoaderOrAction (/Users/tony/repos/teno/node_modules/@remix-run/router/router.ts:3568:16)
dashboard:dev:     at submit (/Users/tony/repos/teno/node_modules/@remix-run/router/router.ts:2835:16)
dashboard:dev:     at queryImpl (/Users/tony/repos/teno/node_modules/@remix-run/router/router.ts:2770:22)
dashboard:dev:     at Object.queryRoute (/Users/tony/repos/teno/node_modules/@remix-run/router/router.ts:2720:18)
dashboard:dev:     at handleResourceRequestRR (/Users/tony/repos/teno/node_modules/@remix-run/node/node_modules/@remix-run/server-runtime/dist/server.js:251:20)
dashboard:dev:     at requestHandler (/Users/tony/repos/teno/node_modules/@remix-run/node/node_modules/@remix-run/server-runtime/dist/server.js:59:18)
dashboard:dev:     at /Users/tony/repos/teno/node_modules/@remix-run/express/dist/server.js:39:22

This appears to be due to some web streams workaround that remix installs. It even occurs when hosted on vercel. This error does not occur if I move all of this code into a dedicated express api but that defeats the purpose of me using remix as it would be my only external api call.

Anyone had luck fixing this? I've tried implementing the solutions here MattiasBuelens/web-streams-polyfill#93 with no luck.

I will try to produce a reproduction repo soon, but using any remix template should immediately reproduce the issue.

[cephalization]

Here is the polyfill code that is throwing the error

https://github.com/MattiasBuelens/web-streams-polyfill/blob/master/src/lib/validators/readable-stream.ts

[cephalization]

I had to perform some unspeakable acts, but this package can be made compatible with remix actions now.

import { createEventStreamTransformer, trimStartOfStreamHelper } from 'ai';
import { ReadableStream as PolyfillReadableStream } from 'web-streams-polyfill';
import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';

// @ts-expect-error bad types
const toPolyfillReadable = createReadableStreamWrapper(PolyfillReadableStream);
const toNativeReadable = createReadableStreamWrapper(ReadableStream);

export class StreamingTextResponse extends Response {
	constructor(res: ReadableStream, init?: ResponseInit) {
		const headers: HeadersInit = {
			'Content-Type': 'text/plain; charset=utf-8',
			...init?.headers,
		};
		super(res, { ...init, status: 200, headers });
		this.getRequestHeaders();
	}

	getRequestHeaders() {
		return addRawHeaders(this.headers);
	}
}

function addRawHeaders(headers: Headers) {
	// @ts-expect-error shame on me
	headers.raw = function () {
		const rawHeaders = {};
		const headerEntries = headers.entries();
		for (const [key, value] of headerEntries) {
			const headerKey = key.toLowerCase();
			if (rawHeaders.hasOwnProperty(headerKey)) {
				// @ts-expect-error shame on me
				rawHeaders[headerKey].push(value);
			} else {
				// @ts-expect-error shame on me
				rawHeaders[headerKey] = [value];
			}
		}
		return rawHeaders;
	};
	return headers;
}

function parseOpenAIStream(): (data: string) => string | void {
	const trimStartOfStream = trimStartOfStreamHelper();
	return (data) => {
		// TODO: Needs a type
		const json = JSON.parse(data);

		// this can be used for either chat or completion models
		const text = trimStartOfStream(json.choices[0]?.delta?.content ?? json.choices[0]?.text ?? '');

		return text;
	};
}

export function OpenAIStream(response: Response): ReadableStream<any> {
	if (!response.ok || !response.body) {
		throw new Error(`Failed to convert the response to stream. Received status code: ${response.status}.`);
	}

	const responseBodyStream = toPolyfillReadable(response.body);

	// @ts-expect-error bad types
	return toNativeReadable(responseBodyStream.pipeThrough(createEventStreamTransformer(parseOpenAIStream())));
}

Just use this OpenAIStream instead of the one returned from ai and also return this StreamingTextResponse from your action and it will work.

There were two key issues:

• OpenAIStream uses pipeThrough, which does not work in remix without patching the polyfill that remix server uses
• StreamingTextResponse does not include request.headers.raw(), which @vercel/remix uses when deployed on vercel

Using these two patched functions in place of the ai ones used in the examples makes everything work in vercel!

[MaxLeiter]

Hi all,

After some research / asking around, I believe this is a remix issue rather than an SDK one (but please let me know if you disagree!)

Basically Remix uses node-fetch and we're using the spec compliant fetch implementation. If you use the edge runtime on Vercel it should work:

export const config = { runtime: "edge" };

You shouldn't need to use the edge runtime though (even though we recommend it) so I'll look into the appropriate way to handle this. It seems like we'll either need to add polyfills to vercel/remix or submit something upstream.

[cephalization]

It would be really funny if this just worked 😅

I tried this while the action was on the same route as the component and it did not work. Let me try with the action on a dedicated resource route

[cephalization]

This does not work for local dev, which is a non starter for me unfortunately

[AvidDabbler]

This is the same for HuggingFaceStream. Would love for this to work.

@cephalization if you have any workarounds or end up using something else would love to know and would I can do the same for you 😄

[cephalization]

My workaround reimplementation above works on local and hosted so I am happy with it for now. We will see if that changes when I try to upgrade the package in the future 😂

[MaxLeiter]

I've found these relevant upstream issues:

1. Remix should not rely on Response.headers.raw remix-run/remix#4354
2. Request.body does not seem to interface correctly with ReadableStream remix-run/remix#3862
   2a. which is really about Investigate using native streams MattiasBuelens/web-streams-polyfill#20

[cephalization]

@AvidDabbler copy paste the code from this comment into a new file, then import and use StreamingTextResponse from your new file.

To get huggingfacestream working, you can likely tweak the openaistream I have shown with whatever hugging face specific logic there is, if any. Checkout the source code from this repo and see what you can make happen

[AvidDabbler]

@AvidDabbler copy paste the code from this comment into a new file, then import and use StreamingTextResponse from your new file.

To get huggingfacestream working, you can likely tweak the openaistream I have shown with whatever hugging face specific logic there is, if any. Checkout the source code from this repo and see what you can make happen

IDK how i missed that 🤦

[yarapolana]

@cephalization Hey how is your client side?
I get Could not parse content as FormData.
And when I use the alternative await request.json()

I get a HTML stream as if you were doing a get request.

[cephalization]

@yarapolana

My action looks just like it does at the beginning of this post. It is on its own route called routes/api.chat.ts and in the frontend code, that uses the useChat hook, I give it the api location of /api/chat

I am using ai@^2.1.7 if it helps

[yarapolana]

@yarapolana

My action looks just like it does at the beginning of this post. It is on its own route called routes/api.chat.ts and in the frontend code, that uses the useChat hook, I give it the api location of /api/chat

I am using ai@^2.1.7 if it helps

Tried for two days to work around it, and downgraded ai lib but I still get the full html as response.
To clarify the messages from usechat give me
<!doctype html ...etc

and that html response does not include the actual streaming values.
just styling/meta/head etc

I am using ChakraUI

Im going to try this on a new project and see what happens.

Thanks for sharing though.

[cephalization]

@yarapolana I can give you my code snippets.

Here is my usage of the chat hook

        const { messages, append, reload, stop, isLoading, input, setInput } = useChat({
		api: '/api/chat',
		headers: {
			'Content-Type': 'application/json',
		},
		initialMessages,
		id,
		body: {
			id,
			transcriptId: meeting.transcript.redisKey,
		},
	});

Here is my /api/chat route

import type { ActionArgs } from '@vercel/remix';
import type { ChatCompletionRequestMessage } from 'openai-edge';
import { Configuration, OpenAIApi } from 'openai-edge';
import { z } from 'zod';
import { countMessageTokens, optimizeTranscriptModel } from 'llm';

import { transcriptQueries, redis } from '@/server/kv.server';
import { OpenAIStream, StreamingTextResponse } from '@/server/streamingTextResponse.server';
import { checkAuth } from '@/server/auth.utils.server';

const oConfig = new Configuration({
	apiKey: process.env.OPENAI_API_KEY ?? '',
});
const openai = new OpenAIApi(oConfig);

export const action = async ({ request }: ActionArgs) => {
	checkAuth(request);

	const { messages, transcriptId } = z
		.object({
			messages: z.array(
				z
					.object({
						content: z.string(),
						role: z.union([z.literal('user'), z.literal('system'), z.literal('assistant')]),
					})
					.passthrough(),
			),
			transcriptId: z.string(),
		})
		.parse(await request.json());

	// Load transcript set from Redis, turn it into a string
	const transcriptArray = await transcriptQueries.getTranscriptArray(redis, { transcriptKey: transcriptId });
	const transcript = transcriptArray;
	const prompt = `HERE IS MY PROMPT`;
	const { model, shortenedTranscript } = optimizeTranscriptModel(transcript, {
		promptTokenBuffer: countMessageTokens(prompt),
	});
	const content = prompt.replace('{transcript}', shortenedTranscript.join('\n'));

	const newMessages: ChatCompletionRequestMessage[] = [
		{
			role: 'system',
			content,
		},
		...messages,
	];

	const response = await openai.createChatCompletion({
		model,
		messages: newMessages,
		stream: true,
	});
	const stream = OpenAIStream(response);

	const sResponse = new StreamingTextResponse(stream);

	return sResponse;
};

[yarapolana]

@cephalization
Ah I am embarrassed right now, didn't think of using the action inside a specific api route, had it with all the rest of the page. Its working fine 😅