update SnippetGenerator to add language hint (#22090)

* update SnippetGenerator to add language hint
* allow enforcing of unused snippets

eng/SnippetGenerator/CSharpProcessor.cs

@@ -11,10 +11,10 @@ namespace SnippetGenerator
 {
     public class CSharpProcessor
     {
-        private static readonly string _snippetFormat = "{3} <code snippet=\"{0}\">{1}{2} </code>";
-        private static readonly string _snippetExampleFormat = "{3} <example snippet=\"{0}\">{1}{3} <code>{1}{2} </code>{1}{3} </example>";
+        private static readonly string _snippetFormat = "{3} <code snippet=\"{0}\" language=\"csharp\">{1}{2} </code>";
+        private static readonly string _snippetExampleFormat = "{3} <example snippet=\"{0}\">{1}{3} <code language=\"csharp\">{1}{2} </code>{1}{3} </example>";
 
-        private static readonly Regex _snippetRegex = new Regex("^(?<indent>\\s*)\\/{3}\\s*<code snippet=\"(?<name>[\\w:]+)\">.*?\\s*<\\/code>",
+        private static readonly Regex _snippetRegex = new Regex("^(?<indent>\\s*)\\/{3}\\s*<code snippet=\"(?<name>[\\w:]+)\"[^>]*?>.*?\\s*<\\/code>",
             RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.IgnoreCase);
 
         private static readonly Regex _snippetExampleRegex = new Regex("^(?<indent>\\s*)\\/{3}\\s*<example snippet=\"(?<name>[\\w:]+)\">.*?\\s*<\\/example>",

eng/SnippetGenerator/DirectoryProcessor.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -37,9 +38,8 @@ public DirectoryProcessor(string directory)
             _snippets = new Lazy<Task<List<Snippet>>>(DiscoverSnippetsAsync);
         }
 
-        public async Task ProcessAsync()
+        public async Task<IEnumerable<string>> ProcessAsync()
         {
-
             List<string> files = new List<string>();
             files.AddRange(Directory.EnumerateFiles(_directory, "*.md", SearchOption.AllDirectories));
             files.AddRange(Directory.EnumerateFiles(_directory, "*.cs", SearchOption.AllDirectories));
@@ -61,8 +61,10 @@ async ValueTask<string> SnippetProvider(string s)
                     }
 
                     var selectedSnippet = selectedSnippets.Single();
+                    selectedSnippet.IsUsed = true;
                     Console.WriteLine($"Replaced {selectedSnippet.Name} in {file}");
-                    return FormatSnippet(selectedSnippet.Text);
+                    var result = FormatSnippet(selectedSnippet.Text);
+                    return result;
                 }
 
                 var originalText = await File.ReadAllTextAsync(file);
@@ -86,13 +88,30 @@ async ValueTask<string> SnippetProvider(string s)
                     await File.WriteAllTextAsync(file, text, _utf8EncodingWithoutBOM);
                 }
             }
+            var snippets = await _snippets.Value;
+            var unUsedSnippets = snippets
+                .Where(s => !s.IsUsed)
+                .Select(s => $"{GetServiceDirName(s.FilePath)}: {s.Name}");
+            return unUsedSnippets;
+
+        }
+
+        private string GetServiceDirName(string path)
+        {
+            string sdk = $"{Path.DirectorySeparatorChar}sdk{Path.DirectorySeparatorChar}";
+            int start = path.IndexOf(sdk) + sdk.Length;
+            int end = path.IndexOf(Path.DirectorySeparatorChar, start);
+            return path.Substring(start, end - start);
         }
 
         private async Task<List<Snippet>> DiscoverSnippetsAsync()
         {
             var snippets = await GetSnippetsInDirectoryAsync(_directory);
+            if (snippets.Count == 0)
+            {
+                return snippets;
+            }
             Console.WriteLine($"Discovered snippets:");
-
             foreach (var snippet in snippets)
             {
                 Console.WriteLine($" {snippet.Name} in {snippet.FilePath}");

eng/SnippetGenerator/Program.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -13,27 +14,49 @@ namespace SnippetGenerator
 {
     public class Program
     {
-
-        [Option(ShortName = "b")]
-        public string BasePath { get; set; }
+        [Option(ShortName = "b")] public string BasePath { get; set; }
+        [Option(ShortName = "sm")] public bool StrictMode { get; set; }
 
         public async Task OnExecuteAsync()
         {
+            List<string> unUsedSnippets = null;
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
             var baseDirectory = new DirectoryInfo(BasePath).Name;
             if (baseDirectory.Equals("sdk"))
             {
-                var tasks = new List<Task>();
+                var tasks = new List<Task<IEnumerable<string>>>();
                 foreach (var sdkDir in Directory.GetDirectories(BasePath))
                 {
                     tasks.Add(new DirectoryProcessor(sdkDir).ProcessAsync());
                 }
 
                 await Task.WhenAll(tasks);
+                unUsedSnippets = tasks
+                    .SelectMany(t => t.Result)
+                    .ToList();
+            }
+            else
+            {
+                unUsedSnippets = (await new DirectoryProcessor(BasePath).ProcessAsync()).ToList();
+            }
+            Console.WriteLine();
+            Console.ForegroundColor = ConsoleColor.Red;
+            string message = $"Not all snippets were used.\n{string.Join(Environment.NewLine, unUsedSnippets)}";
+            unUsedSnippets.Sort();
+            if (StrictMode)
+            {
+                if (unUsedSnippets.Any())
+                {
+                    throw new InvalidOperationException(message);
+                }
             }
             else
             {
-                await new DirectoryProcessor(BasePath).ProcessAsync();
+                Console.WriteLine(message);
             }
+            sw.Stop();
+            Console.WriteLine($"SnippetGenerator completed in {sw.Elapsed}");
         }
 
         public static int Main(string[] args)
@@ -57,4 +80,4 @@ public static int Main(string[] args)
             }
         }
     }
-}
+}

eng/SnippetGenerator/Snippet.cs

@@ -5,11 +5,12 @@
 
 namespace SnippetGenerator
 {
-    internal class Snippet
+    public class Snippet
     {
         public string Name { get; }
         public SourceText Text { get; }
         public string FilePath { get; }
+        public bool IsUsed { get; set; }
 
         public Snippet(string name, SourceText text, string filePath)
         {

eng/SnippetGenerator/SnippetGenerator.Tests/CSharpProcessorTests.cs

@@ -1,26 +1,28 @@
 using System;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using NUnit.Framework;
-using SnippetGenerator;
 
 namespace SnippetGenerator.Tests
 {
     public class Tests
     {
         private const string Processed = "processed";
+        private const string ProcessedAgain = "processed again";
 
         [Test]
         [TestCaseSource(nameof(CodeInputs))]
-        public void CSharpProcsesorFindsCodeXMLDocs(string code, string expected)
+        public async Task CSharpProcsesorFindsCodeXMLDocs(string code, string expected)
         {
-            var actual = CSharpProcessor.Process(code, SnippetProvider);
+            var actual = await CSharpProcessor.ProcessAsync(code, SnippetProvider);
             Assert.AreEqual(expected, actual);
 
-            var reProcessed = CSharpProcessor.Process(actual, SnippetProvider);
-            Assert.AreEqual(expected, reProcessed);
+            var reProcessed = await CSharpProcessor.ProcessAsync(actual, SnippetProvider2);
+            Assert.AreEqual(expected.Replace(Processed, ProcessedAgain), reProcessed);
         }
 
-        private string SnippetProvider(string s) => Processed;
+        private ValueTask<string> SnippetProvider(string s) => new(Processed);
+        private ValueTask<string> SnippetProvider2(string s) => new(ProcessedAgain);
 
         public static IEnumerable<object[]> CodeInputs()
         {
@@ -31,7 +33,7 @@ public static IEnumerable<object[]> CodeInputs()
                 "    foo" + Environment.NewLine +
                 "        {",
                 @"    /// </remarks>" + Environment.NewLine +
-                @"    /// <code snippet=""Snippet:A"">" + Environment.NewLine +
+                @"    /// <code snippet=""Snippet:A"" language=""csharp"">" + Environment.NewLine +
                 $"    /// {Processed} </code>" + Environment.NewLine +
                 "    foo" + Environment.NewLine +
                 "        {"
@@ -40,7 +42,7 @@ public static IEnumerable<object[]> CodeInputs()
             yield return new[]
             {
                 @"/// <code snippet=""Snippet:B""></code>",
-                @"/// <code snippet=""Snippet:B"">" + Environment.NewLine +
+                @"/// <code snippet=""Snippet:B"" language=""csharp"">" + Environment.NewLine +
                 $"/// {Processed} </code>"
             };
 
@@ -50,7 +52,7 @@ public static IEnumerable<object[]> CodeInputs()
                 @"    /// <code snippet=""Snippet:C""></code>" + Environment.NewLine +
                 "     foo",
                 @"    /// Example of enumerating an AsyncPageable using the <c> async foreach </c> loop:" + Environment.NewLine +
-                @"    /// <code snippet=""Snippet:C"">" + Environment.NewLine +
+                @"    /// <code snippet=""Snippet:C"" language=""csharp"">" + Environment.NewLine +
                 $"    /// {Processed} </code>" + Environment.NewLine +
                 "     foo"
             };
@@ -62,11 +64,19 @@ public static IEnumerable<object[]> CodeInputs()
                 "     foo",
                 @"    /// Example of enumerating an AsyncPageable using the <c> async foreach </c> loop:" + Environment.NewLine +
                 @"    /// <example snippet=""Snippet:Example"">" + Environment.NewLine +
-                @"    /// <code>" + Environment.NewLine +
+                @"    /// <code language=""csharp"">" + Environment.NewLine +
                 $"    /// {Processed} </code>" + Environment.NewLine +
                 @"    /// </example>" + Environment.NewLine +
                 "     foo"
             };
+
+            yield return new[]
+            {
+                @"/// <code snippet=""Snippet:AcceptsAnyTagSuffix"" any string here></code>",
+                @"/// <code snippet=""Snippet:AcceptsAnyTagSuffix"" language=""csharp"">" + Environment.NewLine +
+                $"/// {Processed} </code>"
+            };
+
         }
     }
 }

eng/SnippetGenerator/SnippetGenerator.Tests/SnippetGenerator.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>netcoreapp2.1</TargetFramework>
+        <TargetFramework>net5.0</TargetFramework>
 
         <IsPackable>false</IsPackable>
     </PropertyGroup>

eng/SnippetGenerator/SnippetGenerator.sln

@@ -20,6 +20,7 @@ Global
 		{49A0F579-8121-472C-A2A7-B812FEC8CA18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{49A0F579-8121-472C-A2A7-B812FEC8CA18}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{49A0F579-8121-472C-A2A7-B812FEC8CA18}.Release|Any CPU.Build.0 = Release|Any CPU
+		{49A0F579-8121-472C-A2A7-B812FEC8CA18}.Debug|Any CPU.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

eng/scripts/Update-Snippets.ps1

@@ -2,7 +2,10 @@
 param (
     [Parameter(Position=0)]
     [ValidateNotNullOrEmpty()]
-    [string] $ServiceDirectory
+    [string] $ServiceDirectory,
+
+    [Parameter()]
+    [switch] $StrictMode
 )
 
 $generatorProject = "$PSScriptRoot/../SnippetGenerator/SnippetGenerator.csproj";
@@ -14,4 +17,10 @@ if ($ServiceDirectory -and ($ServiceDirectory -ne "*")) {
     $root += '/' + $ServiceDirectory
 }
 
-Resolve-Path "$root" | %{ dotnet run -p $generatorProject -b "$_" -c Release }
+if (-not (Test-Path env:TF_BUILD)) { $StrictMode = $true }
+
+if($StrictMode) {
+    Resolve-Path "$root" | %{ dotnet run -p $generatorProject -b "$_" -sm -c Release }
+} else {
+    Resolve-Path "$root" | %{ dotnet run -p $generatorProject -b "$_" -c Release }
+}

sdk/attestation/Azure.Security.Attestation/src/AttestationAdministrationClient.cs

@@ -198,7 +198,7 @@ private async Task<AttestationResponse<string>> GetPolicyInternalAsync(Attestati
         /// </item>
         /// </list>
         /// To verify the hash, clients can generate an attestation token and verify the hash generated from that token:
-        /// <code snippet="Snippet:VerifySigningHash">
+        /// <code snippet="Snippet:VerifySigningHash" language="csharp">
         /// // The SetPolicyAsync API will create an AttestationToken signed with the TokenSigningKey to transmit the policy.
         /// // To verify that the policy specified by the caller was received by the service inside the enclave, we
         /// // verify that the hash of the policy document returned from the Attestation Service matches the hash
@@ -282,7 +282,7 @@ public virtual AttestationResponse<PolicyModificationResult> SetPolicy(
         /// </item>
         /// </list>
         /// To verify the hash, clients can generate an attestation token and verify the hash generated from that token:
-        /// <code snippet="Snippet:VerifySigningHash">
+        /// <code snippet="Snippet:VerifySigningHash" language="csharp">
         /// // The SetPolicyAsync API will create an AttestationToken signed with the TokenSigningKey to transmit the policy.
         /// // To verify that the policy specified by the caller was received by the service inside the enclave, we
         /// // verify that the hash of the policy document returned from the Attestation Service matches the hash

sdk/core/Azure.Core/src/AsyncPageable.cs

@@ -15,7 +15,7 @@ namespace Azure
     /// <typeparam name="T">The type of the values.</typeparam>
     /// <example>
     /// Example of enumerating an AsyncPageable using the <c> async foreach </c> loop:
-    /// <code snippet="Snippet:AsyncPageable">
+    /// <code snippet="Snippet:AsyncPageable" language="csharp">
     /// // call a service method, which returns AsyncPageable&lt;T&gt;
     /// AsyncPageable&lt;SecretProperties&gt; allSecretProperties = client.GetPropertiesOfSecretsAsync();
     ///
@@ -25,7 +25,7 @@ namespace Azure
     /// }
     /// </code>
     /// or using a while loop:
-    /// <code snippet="Snippet:AsyncPageableLoop">
+    /// <code snippet="Snippet:AsyncPageableLoop" language="csharp">
     /// // call a service method, which returns AsyncPageable&lt;T&gt;
     /// AsyncPageable&lt;SecretProperties&gt; allSecretProperties = client.GetPropertiesOfSecretsAsync();
     ///

sdk/core/Azure.Core/src/AzureNamedKeyCredential.cs

@@ -62,7 +62,7 @@ public void Update(string name, string key)
         /// <param name="name">The name of the <paramref name="key"/>.</param>
         /// <param name="key">The key to use for authenticating with the Azure service.</param>
         /// <example>
-        /// <code snippet="Snippet:AzureNamedKeyCredential_Deconstruct">
+        /// <code snippet="Snippet:AzureNamedKeyCredential_Deconstruct" language="csharp">
         /// var credential = new AzureNamedKeyCredential(&quot;SomeName&quot;, &quot;SomeKey&quot;);
         ///
         /// (string name, string key) = credential;

sdk/core/Azure.Core/src/GeoJson/GeoLineString.cs

@@ -12,7 +12,7 @@ namespace Azure.Core.GeoJson
     /// </summary>
     /// <example>
     /// Creating a line:
-    /// <code snippet="Snippet:CreateLineString">
+    /// <code snippet="Snippet:CreateLineString" language="csharp">
     /// var line = new GeoLineString(new[]
     /// {
     ///     new GeoPosition(-122.108727, 47.649383),

sdk/core/Azure.Core/src/GeoJson/GeoPoint.cs

@@ -11,7 +11,7 @@ namespace Azure.Core.GeoJson
     /// </summary>
     /// <example>
     /// Creating a point:
-    /// <code snippet="Snippet:CreatePoint">
+    /// <code snippet="Snippet:CreatePoint" language="csharp">
     /// var point = new GeoPoint(-122.091954, 47.607148);
     /// </code>
     /// </example>

sdk/core/Azure.Core/src/GeoJson/GeoPolygon.cs

@@ -12,7 +12,7 @@ namespace Azure.Core.GeoJson
     /// </summary>
     /// <example>
     /// Creating a polygon:
-    /// <code snippet="Snippet:CreatePolygon">
+    /// <code snippet="Snippet:CreatePolygon" language="csharp">
     /// var polygon = new GeoPolygon(new[]
     /// {
     ///     new GeoPosition(-122.108727, 47.649383),
@@ -23,7 +23,7 @@ namespace Azure.Core.GeoJson
     /// });
     /// </code>
     /// Creating a polygon with holes:
-    /// <code snippet="Snippet:CreatePolygonWithHoles">
+    /// <code snippet="Snippet:CreatePolygonWithHoles" language="csharp">
     /// var polygon = new GeoPolygon(new[]
     /// {
     ///     // Outer ring

sdk/core/Azure.Core/src/Pipeline/HttpPipeline.cs

@@ -117,7 +117,7 @@ public Response SendRequest(Request request, CancellationToken cancellationToken
         /// <returns>The <see cref="IDisposable"/> instance that needs to be disposed when client request id shouldn't be sent anymore.</returns>
         /// <example>
         /// Sample usage:
-        /// <code snippet="Snippet:ClientRequestId">
+        /// <code snippet="Snippet:ClientRequestId" language="csharp">
         /// var secretClient = new SecretClient(new Uri(&quot;http://example.com&quot;), new DefaultAzureCredential());
         ///
         /// using (HttpPipeline.CreateClientRequestIdScope(&quot;&lt;custom-client-request-id&gt;&quot;))

sdk/core/Azure.Core/src/SyncAsyncEventArgs.cs

@@ -35,7 +35,7 @@ public class SyncAsyncEventArgs : EventArgs
         /// how the event is being raised and implement your handler
         /// accordingly.  Here's an example handler that's safe to invoke from
         /// both sync and async code paths.
-        /// <code snippet="Snippet:Azure_Core_Samples_EventSamples_CombinedHandler">
+        /// <code snippet="Snippet:Azure_Core_Samples_EventSamples_CombinedHandler" language="csharp">
         /// var client = new AlarmClient();
         /// client.Ring += async (SyncAsyncEventArgs e) =&gt;
         /// {