diff --git a/packages/next/src/build/handle-externals.ts b/packages/next/src/build/handle-externals.ts
index 853ab2e3c68ab..2e0eb816b1495 100644
--- a/packages/next/src/build/handle-externals.ts
+++ b/packages/next/src/build/handle-externals.ts
@@ -44,6 +44,7 @@ export async function resolveExternal(
   context: string,
   request: string,
   isEsmRequested: boolean,
+  hasAppDir: boolean,
   getResolve: (
     options: any
   ) => (
@@ -65,7 +66,11 @@ export async function resolveExternal(
 
   let preferEsmOptions =
     esmExternals && isEsmRequested ? [true, false] : [false]
-
+  // Disable esm resolving for app/ and pages/ so for esm package using under pages/
+  // won't load react through esm loader
+  if (hasAppDir) {
+    preferEsmOptions = [false]
+  }
   for (const preferEsm of preferEsmOptions) {
     const resolve = getResolve(
       preferEsm ? esmResolveOptions : nodeResolveOptions
@@ -130,10 +135,12 @@ export function makeExternalHandler({
   config,
   optOutBundlingPackageRegex,
   dir,
+  hasAppDir,
 }: {
   config: NextConfigComplete
   optOutBundlingPackageRegex: RegExp
   dir: string
+  hasAppDir: boolean
 }) {
   let resolvedExternalPackageDirs: Map<string, string>
   const looseEsmExternals = config.experimental?.esmExternals === 'loose'
@@ -286,6 +293,7 @@ export function makeExternalHandler({
       context,
       request,
       isEsmRequested,
+      hasAppDir,
       getResolve,
       isLocal ? resolveNextExternal : undefined
     )
@@ -345,6 +353,7 @@ export function makeExternalHandler({
           config.experimental.esmExternals,
           context,
           pkg + '/package.json',
+          hasAppDir,
           isEsmRequested,
           getResolve,
           isLocal ? resolveNextExternal : undefined
diff --git a/packages/next/src/build/webpack-config-rules/resolve.ts b/packages/next/src/build/webpack-config-rules/resolve.ts
deleted file mode 100644
index f50f6c92ee629..0000000000000
--- a/packages/next/src/build/webpack-config-rules/resolve.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import {
-  COMPILER_NAMES,
-  type CompilerNameValues,
-} from '../../shared/lib/constants'
-
-// exports.<conditionName>
-export const edgeConditionNames = [
-  'edge-light',
-  'worker',
-  // inherits the default conditions
-  '...',
-]
-
-const mainFieldsPerCompiler: Record<
-  CompilerNameValues | 'app-router-server',
-  string[]
-> = {
-  // For default case, prefer CJS over ESM on server side. e.g. pages dir SSR
-  [COMPILER_NAMES.server]: ['main', 'module'],
-  [COMPILER_NAMES.client]: ['browser', 'module', 'main'],
-  [COMPILER_NAMES.edgeServer]: edgeConditionNames,
-  // For app router since everything is bundled, prefer ESM over CJS
-  'app-router-server': ['module', 'main'],
-}
-
-export function getMainField(
-  pageType: 'app' | 'pages',
-  compilerType: CompilerNameValues
-) {
-  if (compilerType === COMPILER_NAMES.edgeServer) {
-    return edgeConditionNames
-  } else if (compilerType === COMPILER_NAMES.client) {
-    return mainFieldsPerCompiler[COMPILER_NAMES.client]
-  }
-
-  // Prefer module fields over main fields for isomorphic packages on server layer
-  return pageType === 'app'
-    ? mainFieldsPerCompiler['app-router-server']
-    : mainFieldsPerCompiler[COMPILER_NAMES.server]
-}
diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts
index a20cf97a1e97f..1664d4377787c 100644
--- a/packages/next/src/build/webpack-config.ts
+++ b/packages/next/src/build/webpack-config.ts
@@ -74,10 +74,6 @@ import { needsExperimentalReact } from '../lib/needs-experimental-react'
 import { getDefineEnvPlugin } from './webpack/plugins/define-env-plugin'
 import type { SWCLoaderOptions } from './webpack/loaders/next-swc-loader'
 import { isResourceInPackages, makeExternalHandler } from './handle-externals'
-import {
-  getMainField,
-  edgeConditionNames,
-} from './webpack-config-rules/resolve'
 
 type ExcludesFalse = <T>(x: T | false) => x is T
 type ClientEntries = {
@@ -108,6 +104,21 @@ const babelIncludeRegexes: RegExp[] = [
 const asyncStoragesRegex =
   /next[\\/]dist[\\/](esm[\\/])?client[\\/]components[\\/](static-generation-async-storage|action-async-storage|request-async-storage)/
 
+// exports.<conditionName>
+const edgeConditionNames = [
+  'edge-light',
+  'worker',
+  // inherits the default conditions
+  '...',
+]
+
+// packageJson.<mainField>
+const mainFieldsPerCompiler: Record<CompilerNameValues, string[]> = {
+  [COMPILER_NAMES.server]: ['main', 'module'],
+  [COMPILER_NAMES.client]: ['browser', 'module', 'main'],
+  [COMPILER_NAMES.edgeServer]: edgeConditionNames,
+}
+
 // Support for NODE_PATH
 const nodePathList = (process.env.NODE_PATH || '')
   .split(process.platform === 'win32' ? ';' : ':')
@@ -920,8 +931,7 @@ export default async function getBaseWebpackConfig(
           },
         }
       : undefined),
-    // default main fields use pages dir ones, and customize app router ones in loaders.
-    mainFields: getMainField('pages', compilerType),
+    mainFields: mainFieldsPerCompiler[compilerType],
     ...(isEdgeServer && {
       conditionNames: edgeConditionNames,
     }),
@@ -1029,6 +1039,7 @@ export default async function getBaseWebpackConfig(
     config,
     optOutBundlingPackageRegex,
     dir,
+    hasAppDir,
   })
 
   const shouldIncludeExternalDirs =
@@ -1599,7 +1610,6 @@ export default async function getBaseWebpackConfig(
                   ],
                 },
                 resolve: {
-                  mainFields: getMainField('app', compilerType),
                   conditionNames: reactServerCondition,
                   // If missing the alias override here, the default alias will be used which aliases
                   // react to the direct file path, not the package name. In that case the condition
@@ -1744,9 +1754,6 @@ export default async function getBaseWebpackConfig(
                     ],
                     exclude: [codeCondition.exclude],
                     use: swcLoaderForClientLayer,
-                    resolve: {
-                      mainFields: getMainField('app', compilerType),
-                    },
                   },
                 ]
               : []),
diff --git a/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts b/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
index 5ea193152a283..1f61d3520d29d 100644
--- a/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
+++ b/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
@@ -743,6 +743,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
             context,
             request,
             isEsmRequested,
+            !!this.appDirEnabled,
             (options) => (_: string, resRequest: string) => {
               return getResolve(options)(parent, resRequest, job)
             },
diff --git a/test/e2e/app-dir/app-external/app-external.test.ts b/test/e2e/app-dir/app-external/app-external.test.ts
index b4e66cb205966..38592f5ba1b47 100644
--- a/test/e2e/app-dir/app-external/app-external.test.ts
+++ b/test/e2e/app-dir/app-external/app-external.test.ts
@@ -39,7 +39,7 @@ createNextDescribe(
     buildCommand: 'yarn build',
     skipDeployment: true,
   },
-  ({ next }) => {
+  ({ next, isNextDev }) => {
     it('should be able to opt-out 3rd party packages being bundled in server components', async () => {
       await next.fetch('/react-server/optout').then(async (response) => {
         const result = await resolveStreamResponse(response)
@@ -47,7 +47,6 @@ createNextDescribe(
         expect(result).toContain('Server subpath: subpath.default')
         expect(result).toContain('Client: index.default')
         expect(result).toContain('Client subpath: subpath.default')
-        expect(result).toContain('opt-out-react-version: 18.2.0')
       })
     })
 
@@ -92,23 +91,24 @@ createNextDescribe(
     })
 
     it('should resolve 3rd party package exports based on the react-server condition', async () => {
-      const $ = await next.render$('/react-server/3rd-party-package')
-
-      const result = $('body').text()
-
-      // Package should be resolved based on the react-server condition,
-      // as well as package's internal & external dependencies.
-      expect(result).toContain(
-        'Server: index.react-server:react.subset:dep.server'
-      )
-      expect(result).toContain('Client: index.default:react.full:dep.default')
-
-      // Subpath exports should be resolved based on the condition too.
-      expect(result).toContain('Server subpath: subpath.react-server')
-      expect(result).toContain('Client subpath: subpath.default')
-
-      // Prefer `module` field for isomorphic packages.
-      expect($('#main-field').text()).toContain('server-module-field:module')
+      await next
+        .fetch('/react-server/3rd-party-package')
+        .then(async (response) => {
+          const result = await resolveStreamResponse(response)
+
+          // Package should be resolved based on the react-server condition,
+          // as well as package's internal & external dependencies.
+          expect(result).toContain(
+            'Server: index.react-server:react.subset:dep.server'
+          )
+          expect(result).toContain(
+            'Client: index.default:react.full:dep.default'
+          )
+
+          // Subpath exports should be resolved based on the condition too.
+          expect(result).toContain('Server subpath: subpath.react-server')
+          expect(result).toContain('Client subpath: subpath.default')
+        })
     })
 
     it('should correctly collect global css imports and mark them as side effects', async () => {
diff --git a/test/e2e/app-dir/app-external/app/react-server/3rd-party-package/page.js b/test/e2e/app-dir/app-external/app/react-server/3rd-party-package/page.js
index 92e9f01672faa..33141e12f7685 100644
--- a/test/e2e/app-dir/app-external/app/react-server/3rd-party-package/page.js
+++ b/test/e2e/app-dir/app-external/app/react-server/3rd-party-package/page.js
@@ -1,6 +1,5 @@
 import v from 'conditional-exports'
 import v1 from 'conditional-exports/subpath'
-import { name as serverFieldName } from 'server-module-field'
 
 import Client from './client'
 
@@ -12,8 +11,6 @@ export default function Page() {
       {`Server subpath: ${v1}`}
       <br />
       <Client />
-      <br />
-      <div id="main-field">{`Server module field: ${serverFieldName}`}</div>
     </div>
   )
 }
diff --git a/test/e2e/app-dir/app-external/app/react-server/optout/page.js b/test/e2e/app-dir/app-external/app/react-server/optout/page.js
index 45ba1ccff0358..fc7bd55ab86c3 100644
--- a/test/e2e/app-dir/app-external/app/react-server/optout/page.js
+++ b/test/e2e/app-dir/app-external/app/react-server/optout/page.js
@@ -1,6 +1,5 @@
 import v from 'conditional-exports-optout'
 import v1 from 'conditional-exports-optout/subpath'
-import { getReactVersion } from 'conditional-exports-optout/react'
 
 import Client from './client'
 
@@ -12,9 +11,6 @@ export default function Page() {
       {`Server subpath: ${v1}`}
       <br />
       <Client />
-      <p id="optout-react-version">
-        {`opt-out-react-version: ${getReactVersion()}`}
-      </p>
     </div>
   )
 }
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/package.json b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/package.json
index 3355c20aad28a..fe1b70d109b2a 100644
--- a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/package.json
+++ b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/package.json
@@ -10,9 +10,6 @@
       "react-server": "./subpath.server.js",
       "default": "./subpath.js"
     },
-    "./react": {
-      "import": "./react.js"
-    },
     "./package.json": "./package.json"
   }
 }
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/react.js b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/react.js
deleted file mode 100644
index 4f2c2283ed693..0000000000000
--- a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports-optout/react.js
+++ /dev/null
@@ -1,5 +0,0 @@
-import React from 'react'
-
-export function getReactVersion() {
-  return React.version
-}
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/package.json b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/package.json
index 06e09e177ae16..b51ade2e7acfe 100644
--- a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/package.json
+++ b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/package.json
@@ -16,9 +16,6 @@
       "react-server": "./subpath.server.js",
       "default": "./subpath.js"
     },
-    "./react": {
-      "import": "./react.js"
-    },
     "./package.json": "./package.json"
   }
 }
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/react.js b/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/react.js
deleted file mode 100644
index 4f2c2283ed693..0000000000000
--- a/test/e2e/app-dir/app-external/node_modules_bak/conditional-exports/react.js
+++ /dev/null
@@ -1,5 +0,0 @@
-import React from 'react'
-
-export function getReactVersion() {
-  return React.version
-}
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.cjs b/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.cjs
deleted file mode 100644
index bead07e159aa3..0000000000000
--- a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.cjs
+++ /dev/null
@@ -1 +0,0 @@
-exports.name = 'server-module-field:main'
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.esm.js b/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.esm.js
deleted file mode 100644
index 02218634f7d07..0000000000000
--- a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/index.esm.js
+++ /dev/null
@@ -1 +0,0 @@
-export const name = 'server-module-field:module'
diff --git a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/package.json b/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/package.json
deleted file mode 100644
index d6cb0ed97cb3d..0000000000000
--- a/test/e2e/app-dir/app-external/node_modules_bak/server-module-field/package.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "main": "./index.cjs",
-  "module": "./index.esm.js"
-}