Revert "Fix broken HTML inlining of non UTF-8 decodable binary data f… (

#65906)

…rom Flight payload (#65664)"

This reverts commit a34d909.

This PR appears to be causing issues with PPR when deployed.
[x-ref](https://github.com/vercel/vercel/actions/runs/9133310975/job/25116552847?pr=11610)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->

packages/next/src/client/app-index.tsx

@@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document
 
 const encoder = new TextEncoder()
 
-let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined
+let initialServerDataBuffer: string[] | undefined = undefined
 let initialServerDataWriter: ReadableStreamDefaultController | undefined =
   undefined
 let initialServerDataLoaded = false
@@ -56,7 +56,6 @@ function nextServerDataCallback(
     | [isBootStrap: 0]
     | [isNotBootstrap: 1, responsePartial: string]
     | [isFormState: 2, formState: any]
-    | [isBinary: 3, responseBase64Partial: string]
 ): void {
   if (seg[0] === 0) {
     initialServerDataBuffer = []
@@ -71,22 +70,6 @@ function nextServerDataCallback(
     }
   } else if (seg[0] === 2) {
     initialFormStateData = seg[1]
-  } else if (seg[0] === 3) {
-    if (!initialServerDataBuffer)
-      throw new Error('Unexpected server data: missing bootstrap script.')
-
-    // Decode the base64 string back to binary data.
-    const binaryString = atob(seg[1])
-    const decodedChunk = new Uint8Array(binaryString.length)
-    for (var i = 0; i < binaryString.length; i++) {
-      decodedChunk[i] = binaryString.charCodeAt(i)
-    }
-
-    if (initialServerDataWriter) {
-      initialServerDataWriter.enqueue(decodedChunk)
-    } else {
-      initialServerDataBuffer.push(decodedChunk)
-    }
   }
 }
 
@@ -101,7 +84,7 @@ function nextServerDataCallback(
 function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) {
   if (initialServerDataBuffer) {
     initialServerDataBuffer.forEach((val) => {
-      ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val)
+      ctr.enqueue(encoder.encode(val))
     })
     if (initialServerDataLoaded && !initialServerDataFlushed) {
       ctr.close()

packages/next/src/server/app-render/app-render.tsx

@@ -398,7 +398,7 @@ function createFlightDataResolver(ctx: AppRenderContext) {
   // Generate the flight data and as soon as it can, convert it into a string.
   const promise = generateFlight(ctx)
     .then(async (result) => ({
-      flightData: await result.toUnchunkedBuffer(true),
+      flightData: await result.toUnchunkedString(true),
     }))
     // Otherwise if it errored, return the error.
     .catch((err) => ({ err }))

packages/next/src/server/app-render/use-flight-response.tsx

@@ -9,7 +9,6 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge'
 const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0
 const INLINE_FLIGHT_PAYLOAD_DATA = 1
 const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2
-const INLINE_FLIGHT_PAYLOAD_BINARY = 3
 
 const flightResponses = new WeakMap<BinaryStreamOf<any>, Promise<any>>()
 const encoder = new TextEncoder()
@@ -97,8 +96,10 @@ export function createInlinedDataReadableStream(
     ? `<script nonce=${JSON.stringify(nonce)}>`
     : '<script>'
 
-  const flightReader = flightStream.getReader()
   const decoder = new TextDecoder('utf-8', { fatal: true })
+  const decoderOptions = { stream: true }
+
+  const flightReader = flightStream.getReader()
 
   const readable = new ReadableStream({
     type: 'bytes',
@@ -113,26 +114,15 @@ export function createInlinedDataReadableStream(
     async pull(controller) {
       try {
         const { done, value } = await flightReader.read()
-
-        if (value) {
-          try {
-            const decodedString = decoder.decode(value, { stream: !done })
-
-            // The chunk cannot be decoded as valid UTF-8 string as it might
-            // have arbitrary binary data.
-            writeFlightDataInstruction(
-              controller,
-              startScriptTag,
-              decodedString
-            )
-          } catch {
-            // The chunk cannot be decoded as valid UTF-8 string.
-            writeFlightDataInstruction(controller, startScriptTag, value)
-          }
-        }
-
         if (done) {
+          const tail = decoder.decode(value, { stream: false })
+          if (tail.length) {
+            writeFlightDataInstruction(controller, startScriptTag, tail)
+          }
           controller.close()
+        } else {
+          const chunkAsString = decoder.decode(value, decoderOptions)
+          writeFlightDataInstruction(controller, startScriptTag, chunkAsString)
         }
       } catch (error) {
         // There was a problem in the upstream reader or during decoding or enqueuing
@@ -164,28 +154,13 @@ function writeInitialInstructions(
 function writeFlightDataInstruction(
   controller: ReadableStreamDefaultController,
   scriptStart: string,
-  chunk: string | Uint8Array
+  chunkAsString: string
 ) {
-  let htmlInlinedData: string
-
-  if (typeof chunk === 'string') {
-    htmlInlinedData = htmlEscapeJsonString(
-      JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunk])
-    )
-  } else {
-    // The chunk cannot be embedded as a UTF-8 string in the script tag.
-    // Instead let's inline it in base64.
-    // Credits to Devon Govett (devongovett) for the technique.
-    // https://github.com/devongovett/rsc-html-stream
-    const base64 = btoa(String.fromCodePoint(...chunk))
-    htmlInlinedData = htmlEscapeJsonString(
-      JSON.stringify([INLINE_FLIGHT_PAYLOAD_BINARY, base64])
-    )
-  }
-
   controller.enqueue(
     encoder.encode(
-      `${scriptStart}self.__next_f.push(${htmlInlinedData})</script>`
+      `${scriptStart}self.__next_f.push(${htmlEscapeJsonString(
+        JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunkAsString])
+      )})</script>`
     )
   )
 }

packages/next/src/server/base-server.ts

@@ -13,8 +13,6 @@ import type { ParsedUrlQuery } from 'querystring'
 import type { RenderOptsPartial as PagesRenderOptsPartial } from './render'
 import type { RenderOptsPartial as AppRenderOptsPartial } from './app-render/types'
 import type {
-  CachedAppPageValue,
-  CachedPageValue,
   ResponseCacheBase,
   ResponseCacheEntry,
   ResponseGenerator,
@@ -2545,29 +2543,16 @@ export default abstract class Server<
         return null
       }
 
-      if (isAppPath) {
-        return {
-          value: {
-            kind: 'APP_PAGE',
-            html: result,
-            headers,
-            rscData: metadata.flightData,
-            postponed: metadata.postponed,
-            status: res.statusCode,
-          } satisfies CachedAppPageValue,
-          revalidate: metadata.revalidate,
-        }
-      }
-
       // We now have a valid HTML result that we can return to the user.
       return {
         value: {
           kind: 'PAGE',
           html: result,
-          pageData: metadata.pageData,
+          pageData: metadata.pageData ?? metadata.flightData,
+          postponed: metadata.postponed,
           headers,
-          status: res.statusCode,
-        } satisfies CachedPageValue,
+          status: isAppPath ? res.statusCode : undefined,
+        },
         revalidate: metadata.revalidate,
       }
     }
@@ -2680,6 +2665,7 @@ export default abstract class Server<
               value: {
                 kind: 'PAGE',
                 html: RenderResult.fromStatic(html),
+                postponed: undefined,
                 status: undefined,
                 headers: undefined,
                 pageData: {},
@@ -2748,7 +2734,7 @@ export default abstract class Server<
     }
 
     const didPostpone =
-      cacheEntry.value?.kind === 'APP_PAGE' &&
+      cacheEntry.value?.kind === 'PAGE' &&
       typeof cacheEntry.value.postponed === 'string'
 
     if (
@@ -2915,11 +2901,7 @@ export default abstract class Server<
     } else if (isAppPath) {
       // If the request has a postponed state and it's a resume request we
       // should error.
-      if (
-        cachedData.kind === 'APP_PAGE' &&
-        cachedData.postponed &&
-        minimalPostponed
-      ) {
+      if (cachedData.postponed && minimalPostponed) {
         throw new Error(
           'Invariant: postponed state should not be present on a resume request'
         )
@@ -2967,11 +2949,7 @@ export default abstract class Server<
       }
 
       // Mark that the request did postpone if this is a data request.
-      if (
-        cachedData.kind === 'APP_PAGE' &&
-        cachedData.postponed &&
-        isRSCRequest
-      ) {
+      if (cachedData.postponed && isRSCRequest) {
         res.setHeader(NEXT_DID_POSTPONE_HEADER, '1')
       }
 
@@ -2982,15 +2960,8 @@ export default abstract class Server<
       if (isDataReq && !isPreviewMode) {
         // If this is a dynamic RSC request, then stream the response.
         if (isDynamicRSCRequest) {
-          if (cachedData.kind !== 'APP_PAGE') {
-            console.error({ url: req.url, pathname }, cachedData)
-            throw new Error(
-              `Invariant: expected cache data kind of APP_PAGE got ${cachedData.kind}`
-            )
-          }
-
-          if (cachedData.rscData) {
-            throw new Error('Invariant: Expected rscData to be undefined')
+          if (cachedData.pageData) {
+            throw new Error('Invariant: Expected pageData to be undefined')
           }
 
           if (cachedData.postponed) {
@@ -3009,23 +2980,17 @@ export default abstract class Server<
           }
         }
 
-        if (cachedData.kind !== 'APP_PAGE') {
-          throw new Error(
-            `Invariant: expected cached data to be APP_PAGE got ${cachedData.kind}`
-          )
-        }
-
-        if (!Buffer.isBuffer(cachedData.rscData)) {
+        if (typeof cachedData.pageData !== 'string') {
           throw new Error(
-            `Invariant: expected rscData to be a Buffer, got ${typeof cachedData.rscData}`
+            `Invariant: expected pageData to be a string, got ${typeof cachedData.pageData}`
           )
         }
 
         // As this isn't a prefetch request, we should serve the static flight
         // data.
         return {
           type: 'rsc',
-          body: RenderResult.fromStatic(cachedData.rscData),
+          body: RenderResult.fromStatic(cachedData.pageData),
           revalidate: cacheEntry.revalidate,
         }
       }
@@ -3036,10 +3001,7 @@ export default abstract class Server<
       // If there's no postponed state, we should just serve the HTML. This
       // should also be the case for a resume request because it's completed
       // as a server render (rather than a static render).
-      if (
-        !(cachedData.kind === 'APP_PAGE' && cachedData.postponed) ||
-        this.minimalMode
-      ) {
+      if (!cachedData.postponed || this.minimalMode) {
         return {
           type: 'html',
           body,
@@ -3068,7 +3030,7 @@ export default abstract class Server<
             throw new Error('Invariant: expected a result to be returned')
           }
 
-          if (result.value?.kind !== 'APP_PAGE') {
+          if (result.value?.kind !== 'PAGE') {
             throw new Error(
               `Invariant: expected a page response, got ${result.value?.kind}`
             )
@@ -3094,11 +3056,6 @@ export default abstract class Server<
         revalidate: 0,
       }
     } else if (isDataReq) {
-      if (cachedData.kind !== 'PAGE') {
-        throw new Error(
-          `Invariant: expected cached data to be PAGE got ${cachedData.kind}`
-        )
-      }
       return {
         type: 'json',
         body: RenderResult.fromStatic(JSON.stringify(cachedData.pageData)),

packages/next/src/server/lib/incremental-cache/fetch-cache.ts

@@ -109,10 +109,7 @@ export default class FetchCache implements CacheHandler {
             }
             // rough estimate of size of cache value
             return (
-              value.html.length +
-              (JSON.stringify(
-                value.kind === 'APP_PAGE' ? value.rscData : value.pageData
-              )?.length || 0)
+              value.html.length + (JSON.stringify(value.pageData)?.length || 0)
             )
           },
         })