diff --git a/eng/testing/workloads-testing.targets b/eng/testing/workloads-testing.targets
index 40f134fc8bb3d..67a781d1058b6 100644
--- a/eng/testing/workloads-testing.targets
+++ b/eng/testing/workloads-testing.targets
@@ -89,9 +89,6 @@
           WorkingDirectory="$(InstallerProjectRoot)pkg/sfx/Microsoft.NETCore.App" />
 
     <ItemGroup>
-      <_NuGetSourceForWorkloads Include="dotnet6" Value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
-      <_NuGetSourceForWorkloads Include="dotnet7" Value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
-
       <_BuiltNuGets Include="$(LibrariesShippingPackagesDir)\*.nupkg" />
     </ItemGroup>
 
@@ -111,7 +108,7 @@
                      WorkloadId="@(WorkloadIdForTesting)"
                      VersionBand="$(SdkBandVersion)"
                      LocalNuGetsPath="$(LibrariesShippingPackagesDir)"
-                     ExtraNuGetSources="@(_NuGetSourceForWorkloads)"
+                     TemplateNuGetConfigPath="$(RepoRoot)NuGet.config"
                      SdkDir="$(SdkWithWorkloadForTestingPath)" />
     <WriteLinesToFile File="$(SdkWithWorkload_WorkloadStampPath)" Lines="" Overwrite="true" />
   </Target>
@@ -124,7 +121,7 @@
                      WorkloadId="@(WorkloadIdForTesting)"
                      VersionBand="$(SdkBandVersion)"
                      LocalNuGetsPath="$(LibrariesShippingPackagesDir)"
-                     ExtraNuGetSources="@(_NuGetSourceForWorkloads)"
+                     TemplateNuGetConfigPath="$(RepoRoot)NuGet.config"
                      SdkDir="$(SdkWithNoWorkloadForTestingPath)"
                      OnlyUpdateManifests="true"/>
 
diff --git a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs
index 6c5f4a71a0857..08f933b2b6e01 100644
--- a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs
+++ b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs
@@ -27,14 +27,30 @@ public class InstallWorkloadFromArtifacts : Task
         [Required, NotNull]
         public string?        LocalNuGetsPath    { get; set; }
 
+        [Required, NotNull]
+        public string?        TemplateNuGetConfigPath { get; set; }
+
         [Required, NotNull]
         public string?        SdkDir             { get; set; }
 
         public bool           OnlyUpdateManifests{ get; set; }
 
-        public ITaskItem[]    ExtraNuGetSources  { get; set; } = Array.Empty<ITaskItem>();
+        private const string s_nugetInsertionTag = "<!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->";
 
         public override bool Execute()
+        {
+            try
+            {
+                return ExecuteInternal();
+            }
+            catch (LogAsErrorException laee)
+            {
+                Log.LogError(laee.Message);
+                return false;
+            }
+        }
+
+        private bool ExecuteInternal()
         {
             if (!HasMetadata(WorkloadId, nameof(WorkloadId), "Version") ||
                 !HasMetadata(WorkloadId, nameof(WorkloadId), "ManifestName"))
@@ -48,6 +64,12 @@ public override bool Execute()
                 return false;
             }
 
+            if (!File.Exists(TemplateNuGetConfigPath))
+            {
+                Log.LogError($"Cannot find TemplateNuGetConfigPath={TemplateNuGetConfigPath}");
+                return false;
+            }
+
             Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Installing workload manifest {WorkloadId.ItemSpec} **{Environment.NewLine}");
 
             string nugetConfigContents = GetNuGetConfig();
@@ -86,25 +108,11 @@ public override bool Execute()
 
         private string GetNuGetConfig()
         {
-            StringBuilder nugetConfigBuilder = new();
-            nugetConfigBuilder.AppendLine($"<configuration>{Environment.NewLine}<packageSources>");
-
-            nugetConfigBuilder.AppendLine($@"<add key=""nuget-local"" value=""{LocalNuGetsPath}"" />");
-            foreach (ITaskItem source in ExtraNuGetSources)
-            {
-                string key = source.ItemSpec;
-                string value = source.GetMetadata("Value");
-                if (string.IsNullOrEmpty(value))
-                {
-                    Log.LogWarning($"ExtraNuGetSource {key} is missing Value metadata");
-                    continue;
-                }
-
-                nugetConfigBuilder.AppendLine($@"<add key=""{key}"" value=""{value}"" />");
-            }
+            string contents = File.ReadAllText(TemplateNuGetConfigPath);
+            if (contents.IndexOf(s_nugetInsertionTag) < 0)
+                throw new LogAsErrorException($"Could not find {s_nugetInsertionTag} in {TemplateNuGetConfigPath}");
 
-            nugetConfigBuilder.AppendLine($"</packageSources>{Environment.NewLine}</configuration>");
-            return nugetConfigBuilder.ToString();
+            return contents.Replace(s_nugetInsertionTag, $@"<add key=""nuget-local"" value=""{LocalNuGetsPath}"" />");
         }
 
         private bool InstallWorkloadManifest(string name, string version, string nugetConfigContents, bool stopOnMissing)
diff --git a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj
index bec88895f4c28..738fbedb31eaa 100644
--- a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj
+++ b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj
@@ -7,6 +7,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="..\Common\Utils.cs" />
+    <Compile Include="..\Common\LogAsErrorException.cs" />
 
     <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
   </ItemGroup>