Fix broken HTML inlining of non UTF-8 decodable binary data from Flight payload

packages/next/src/client/app-index.tsx

@@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document
 
 const encoder = new TextEncoder()
 
-let initialServerDataBuffer: string[] | undefined = undefined
+let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined
 let initialServerDataWriter: ReadableStreamDefaultController | undefined =
   undefined
 let initialServerDataLoaded = false
@@ -56,6 +56,7 @@ function nextServerDataCallback(
     | [isBootStrap: 0]
     | [isNotBootstrap: 1, responsePartial: string]
     | [isFormState: 2, formState: any]
+    | [isBinary: 3, responseBase64Partial: string]
 ): void {
   if (seg[0] === 0) {
     initialServerDataBuffer = []
@@ -70,6 +71,22 @@ function nextServerDataCallback(
     }
   } else if (seg[0] === 2) {
     initialFormStateData = seg[1]
+  } else if (seg[0] === 3) {
+    if (!initialServerDataBuffer)
+      throw new Error('Unexpected server data: missing bootstrap script.')
+
+    // Decode the base64 string back to binary data.
+    const binaryString = atob(seg[1])
+    const decodedChunk = new Uint8Array(binaryString.length)
+    for (var i = 0; i < binaryString.length; i++) {
+      decodedChunk[i] = binaryString.charCodeAt(i)
+    }
+
+    if (initialServerDataWriter) {
+      initialServerDataWriter.enqueue(decodedChunk)
+    } else {
+      initialServerDataBuffer.push(decodedChunk)
+    }
   }
 }
 
@@ -84,7 +101,7 @@ function nextServerDataCallback(
 function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) {
   if (initialServerDataBuffer) {
     initialServerDataBuffer.forEach((val) => {
-      ctr.enqueue(encoder.encode(val))
+      ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val)
     })
     if (initialServerDataLoaded && !initialServerDataFlushed) {
       ctr.close()

packages/next/src/server/app-render/use-flight-response.tsx

@@ -9,6 +9,7 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge'
 const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0
 const INLINE_FLIGHT_PAYLOAD_DATA = 1
 const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2
+const INLINE_FLIGHT_PAYLOAD_BINARY = 3
 
 const flightResponses = new WeakMap<BinaryStreamOf<any>, Promise<any>>()
 const encoder = new TextEncoder()
@@ -96,10 +97,8 @@ export function createInlinedDataReadableStream(
     ? `<script nonce=${JSON.stringify(nonce)}>`
     : '<script>'
 
-  const decoder = new TextDecoder('utf-8', { fatal: true })
-  const decoderOptions = { stream: true }
-
   const flightReader = flightStream.getReader()
+  const decoder = new TextDecoder('utf-8', { fatal: true })
 
   const readable = new ReadableStream({
     type: 'bytes',
@@ -114,15 +113,26 @@ export function createInlinedDataReadableStream(
     async pull(controller) {
       try {
         const { done, value } = await flightReader.read()
-        if (done) {
-          const tail = decoder.decode(value, { stream: false })
-          if (tail.length) {
-            writeFlightDataInstruction(controller, startScriptTag, tail)
+
+        if (value) {
+          try {
+            const decodedString = decoder.decode(value, { stream: !done })
+
+            // The chunk cannot be decoded as valid UTF-8 string as it might
+            // have arbitrary binary data.
+            writeFlightDataInstruction(
+              controller,
+              startScriptTag,
+              decodedString
+            )
+          } catch {
+            // The chunk cannot be decoded as valid UTF-8 string.
+            writeFlightDataInstruction(controller, startScriptTag, value)
           }
+        }
+
+        if (done) {
           controller.close()
-        } else {
-          const chunkAsString = decoder.decode(value, decoderOptions)
-          writeFlightDataInstruction(controller, startScriptTag, chunkAsString)
         }
       } catch (error) {
         // There was a problem in the upstream reader or during decoding or enqueuing
@@ -154,13 +164,28 @@ function writeInitialInstructions(
 function writeFlightDataInstruction(
   controller: ReadableStreamDefaultController,
   scriptStart: string,
-  chunkAsString: string
+  chunk: string | Uint8Array
 ) {
+  let htmlInlinedData: string
+
+  if (typeof chunk === 'string') {
+    htmlInlinedData = htmlEscapeJsonString(
+      JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunk])
+    )
+  } else {
+    // The chunk cannot be embedded as a UTF-8 string in the script tag.
+    // Instead let's inline it in base64.
+    // Credits to Devon Govett (devongovett) for the technique.
+    // https://github.com/devongovett/rsc-html-stream
+    const base64 = btoa(String.fromCodePoint(...chunk))
+    htmlInlinedData = htmlEscapeJsonString(
+      JSON.stringify([INLINE_FLIGHT_PAYLOAD_BINARY, base64])
+    )
+  }
+
   controller.enqueue(
     encoder.encode(
-      `${scriptStart}self.__next_f.push(${htmlEscapeJsonString(
-        JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunkAsString])
-      )})</script>`
+      `${scriptStart}self.__next_f.push(${htmlInlinedData})</script>`
     )
   )
 }

packages/next/src/server/stream-utils/node-web-streams-helper.ts

@@ -81,7 +81,12 @@ export async function streamToString(
 
   // @ts-expect-error TypeScript gets this wrong (https://nodejs.org/api/webstreams.html#async-iteration)
   for await (const chunk of stream) {
-    string += decoder.decode(chunk, { stream: true })
+    try {
+      string += decoder.decode(chunk, { stream: true })
+    } catch (e) {
+      console.error('!!!!Error decoding chunk', e)
+      throw e
+    }
   }
 
   string += decoder.decode()

test/e2e/app-dir/binary/app/client.js

@@ -0,0 +1,19 @@
+'use client'
+
+import { useEffect, useState } from 'react'
+
+export function Client({ binary, arbitrary }) {
+  const [hydrated, setHydrated] = useState(false)
+
+  useEffect(() => {
+    setHydrated(true)
+  }, [])
+
+  return (
+    <>
+      <div>utf8 binary: {new TextDecoder().decode(binary)}</div>
+      <div>arbitrary binary: {String(arbitrary)}</div>
+      <div>hydrated: {String(hydrated)}</div>
+    </>
+  )
+}

test/e2e/app-dir/binary/app/layout.js

@@ -0,0 +1,12 @@
+export const metadata = {
+  title: 'Next.js',
+  description: 'Generated by Next.js',
+}
+
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/app-dir/binary/app/page.js

@@ -0,0 +1,8 @@
+import { Client } from './client'
+
+export default function Page() {
+  const binaryData = new Uint8Array([104, 101, 108, 108, 111])
+  const nonUtf8BinaryData = new Uint8Array([0xff, 0, 1, 2, 3])
+
+  return <Client binary={binaryData} arbitrary={nonUtf8BinaryData} />
+}

test/e2e/app-dir/binary/next.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  experimental: {
+    // This ensures that we're running the experimental React.
+    taint: true,
+  },
+}

test/e2e/app-dir/binary/rsc-binary.test.ts

@@ -0,0 +1,32 @@
+import { nextTestSetup } from 'e2e-utils'
+import { check } from 'next-test-utils'
+
+describe('RSC binary serialization', () => {
+  const { next, skipped } = nextTestSetup({
+    files: __dirname,
+    skipDeployment: true,
+    dependencies: {
+      react: '19.0.0-beta-4508873393-20240430',
+      'react-dom': '19.0.0-beta-4508873393-20240430',
+      'server-only': 'latest',
+    },
+  })
+  if (skipped) return
+
+  afterEach(async () => {
+    await next.stop()
+  })
+
+  it('should correctly encode/decode binaries and hydrate', async function () {
+    const browser = await next.browser('/')
+    await check(async () => {
+      const content = await browser.elementByCss('body').text()
+
+      return content.includes('utf8 binary: hello') &&
+        content.includes('arbitrary binary: 255,0,1,2,3') &&
+        content.includes('hydrated: true')
+        ? 'success'
+        : 'fail'
+    }, 'success')
+  })
+})