diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts
index 1113116f49b8d..8223e506e8037 100644
--- a/packages/next/src/build/webpack-config.ts
+++ b/packages/next/src/build/webpack-config.ts
@@ -455,12 +455,20 @@ export default async function getBaseWebpackConfig(
     : []
 
   const swcLoaderForMiddlewareLayer = useSWCLoader
-    ? swcServerLayerLoader
+    ? getSwcLoader({
+        serverComponents: false,
+        isReactServerLayer: false,
+      })
     : // When using Babel, we will have to use SWC to do the optimization
       // for middleware to tree shake the unused default optimized imports like "next/server".
       // This will cause some performance overhead but
       // acceptable as Babel will not be recommended.
-      [swcServerLayerLoader, babelLoader]
+      [
+        getSwcLoader({
+          serverComponents: false,
+          isReactServerLayer: false,
+        }),
+      ]
 
   // client components layers: SSR + browser
   const swcLoaderForClientLayer = [
diff --git a/packages/next/src/lib/constants.ts b/packages/next/src/lib/constants.ts
index b943c19cecdf1..25d026a4a43c9 100644
--- a/packages/next/src/lib/constants.ts
+++ b/packages/next/src/lib/constants.ts
@@ -154,9 +154,12 @@ const WEBPACK_LAYERS = {
       WEBPACK_LAYERS_NAMES.actionBrowser,
       WEBPACK_LAYERS_NAMES.appMetadataRoute,
       WEBPACK_LAYERS_NAMES.appRouteHandler,
+    ],
+    nonClientServerTarget: [
+      // plus middleware and pages api
       WEBPACK_LAYERS_NAMES.middleware,
+      WEBPACK_LAYERS_NAMES.api,
     ],
-    nonClientServerTarget: [WEBPACK_LAYERS_NAMES.api],
     app: [
       WEBPACK_LAYERS_NAMES.reactServerComponents,
       WEBPACK_LAYERS_NAMES.actionBrowser,
diff --git a/test/e2e/module-layer/index.test.ts b/test/e2e/module-layer/index.test.ts
index 701050aaaa34f..4aef0e35246fe 100644
--- a/test/e2e/module-layer/index.test.ts
+++ b/test/e2e/module-layer/index.test.ts
@@ -1,12 +1,11 @@
 import { createNextDescribe } from 'e2e-utils'
-import { getRedboxSource, hasRedbox } from 'next-test-utils'
 
 createNextDescribe(
   'module layer',
   {
     files: __dirname,
   },
-  ({ next, isNextStart, isNextDev }) => {
+  ({ next, isNextStart }) => {
     function runTests() {
       it('should render routes marked with restriction marks without errors', async () => {
         const routes = [
@@ -61,60 +60,45 @@ createNextDescribe(
       }
     }
 
-    describe('with server-only in server targets', () => {
-      runTests()
-    })
+    describe('no server-only in server targets', () => {
+      const middlewareFile = 'middleware.js'
+      // const pagesApiFile = 'pages/api/hello.js'
+      let middlewareContent = ''
+      // let pagesApiContent = ''
 
-    // Should error for using mixed (with client-only) in server targets
-    if (isNextDev) {
-      describe('no server-only in server targets', () => {
-        const middlewareFile = 'middleware.js'
-        // const pagesApiFile = 'pages/api/hello.js'
-        let middlewareContent = ''
-        // let pagesApiContent = ''
+      beforeAll(async () => {
+        await next.stop()
 
-        beforeAll(async () => {
-          await next.stop()
+        middlewareContent = await next.readFile(middlewareFile)
+        // pagesApiContent = await next.readFile(pagesApiFile)
 
-          middlewareContent = await next.readFile(middlewareFile)
-          // pagesApiContent = await next.readFile(pagesApiFile)
-
-          await next.patchFile(
-            middlewareFile,
-            middlewareContent
-              // .replace("import 'server-only'", "// import 'server-only'")
-              .replace(
-                "// import './lib/mixed-lib'",
-                "import './lib/mixed-lib'"
-              )
-          )
-
-          // await next.patchFile(
-          //   pagesApiFile,
-          //   pagesApiContent
-          //     .replace("import 'server-only'", "// import 'server-only'")
-          //     .replace(
-          //       "// import '../../lib/mixed-lib'",
-          //       "import '../../lib/mixed-lib'"
-          //     )
-          // )
-
-          await next.start()
-        })
-        afterAll(async () => {
-          await next.patchFile(middlewareFile, middlewareContent)
-          // await next.patchFile(pagesApiFile, pagesApiContent)
-        })
+        await next.patchFile(
+          middlewareFile,
+          middlewareContent
+            .replace("import 'server-only'", "// import 'server-only'")
+            .replace("// import './lib/mixed-lib'", "import './lib/mixed-lib'")
+        )
 
-        it('should error when import client-only in middleware', async () => {
-          const browser = await next.browser('/')
+        // await next.patchFile(
+        //   pagesApiFile,
+        //   pagesApiContent
+        //     .replace("import 'server-only'", "// import 'server-only'")
+        //     .replace(
+        //       "// import '../../lib/mixed-lib'",
+        //       "import '../../lib/mixed-lib'"
+        //     )
+        // )
 
-          expect(await hasRedbox(browser, true)).toBe(true)
-          expect(await getRedboxSource(browser)).toContain(
-            `You're importing a component that imports client-only. It only works in a Client Component but none of its parents are marked with "use client", so they're Server Components by default.`
-          )
-        })
+        await next.start()
       })
-    }
+      afterAll(async () => {
+        await next.patchFile(middlewareFile, middlewareContent)
+        // await next.patchFile(pagesApiFile, pagesApiContent)
+      })
+      runTests()
+    })
+    describe('with server-only in server targets', () => {
+      runTests()
+    })
   }
 )