diff --git a/packages/nextjs/src/config/index.ts b/packages/nextjs/src/config/index.ts
index 21beb61b482c..e8485a2f8be4 100644
--- a/packages/nextjs/src/config/index.ts
+++ b/packages/nextjs/src/config/index.ts
@@ -12,21 +12,17 @@ export function withSentryConfig(
   userNextConfig: ExportedNextConfig = {},
   userSentryWebpackPluginOptions: Partial<SentryWebpackPluginOptions> = {},
 ): NextConfigFunction | NextConfigObject {
-  const partialConfig = {
-    // TODO When we add a way to disable the webpack plugin, doing so should turn this off, too
-    productionBrowserSourceMaps: true,
-    webpack: constructWebpackConfigFunction(userNextConfig, userSentryWebpackPluginOptions),
-  };
+  const newWebpackConfig = constructWebpackConfigFunction(userNextConfig, userSentryWebpackPluginOptions);
 
   // If the user has passed us a function, we need to return a function, so that we have access to `phase` and
   // `defaults` in order to pass them along to the user's function
   if (typeof userNextConfig === 'function') {
     return (phase: string, defaults: { defaultConfig: { [key: string]: unknown } }): NextConfigObject => ({
       ...userNextConfig(phase, defaults),
-      ...partialConfig,
+      webpack: newWebpackConfig,
     });
   }
 
   // Otherwise, we can just merge their config with ours and return an object.
-  return { ...userNextConfig, ...partialConfig };
+  return { ...userNextConfig, webpack: newWebpackConfig };
 }
diff --git a/packages/nextjs/test/config.test.ts b/packages/nextjs/test/config.test.ts
index c1d7f1ab249c..7023dc9ba4d3 100644
--- a/packages/nextjs/test/config.test.ts
+++ b/packages/nextjs/test/config.test.ts
@@ -115,7 +115,6 @@ describe('withSentryConfig', () => {
 
     expect(finalConfig).toEqual(
       expect.objectContaining({
-        productionBrowserSourceMaps: true,
         webpack: expect.any(Function), // `webpack` is tested specifically elsewhere
       }),
     );
@@ -133,7 +132,6 @@ describe('withSentryConfig', () => {
     expect(finalConfig).toEqual(
       expect.objectContaining({
         ...userNextConfig,
-        productionBrowserSourceMaps: true,
         webpack: expect.any(Function), // `webpack` is tested specifically elsewhere
       }),
     );
@@ -147,7 +145,6 @@ describe('withSentryConfig', () => {
     expect(finalConfig).toEqual(
       expect.objectContaining({
         ...userNextConfigFunction(),
-        productionBrowserSourceMaps: true,
         webpack: expect.any(Function), // `webpack` is tested specifically elsewhere
       }),
     );