Remove unused resource strings from System.Private.CoreLib

[stephentoub]

cc: @jkotas, @danmosemsft

[danmoseley]

I thought we did a pass not that long ago..

[stephentoub]

For reference, this was done with just a simple little text-based utility, so it's possible there are some false negatives, but hopefully not many.

using System.Collections.Generic;
using System.IO;
using System.Xml;
using System.Linq;
using System.Text;

class Program
{
    static void Main()
    {
        var doc = new XmlDocument();
        doc.Load(@"D:\repos\coreclr\src\System.Private.CoreLib\Resources\Strings.resx");

        var names = new HashSet<string>(doc.SelectNodes("/root/data").Cast<XmlNode>().Select(n => n.Attributes["name"].Value));

        var found = new HashSet<string>();
        foreach (string path in Directory.EnumerateFiles(@"D:\repos\coreclr\src\", "*", SearchOption.AllDirectories))
        {
            if (Path.GetExtension(path) == ".resx") continue;
            string text = File.ReadAllText(path);
            foreach (string name in names)
                if (text.Contains(name))
                    found.Add(name);
        }
        names.ExceptWith(found);

        var lines = new List<string>(File.ReadAllLines(@"D:\repos\coreclr\src\System.Private.CoreLib\Resources\Strings.resx"));
        for (int i = 0; i < lines.Count; i++)
        {
            if (!lines[i].Contains("data name")) continue;
            if (names.Any(n => !n.StartsWith("event_") && lines[i].Contains(n)))
            {
                lines.RemoveRange(i, 3);
                i--;
            }
        }
        File.WriteAllLines(@"D:\repos\coreclr\src\System.Private.CoreLib\Resources\Strings.resx", lines, new UTF8Encoding(true));
    }
}

[stephentoub]

Hmm... it looks like native code can actually throw with these managed resources, hadn't realized that... need to tweak the code and fix the PR accordingly....

[stephentoub]

Fixed. Basically everything is used after factoring in the native references. I just removed a few event_* resources no longer used.

[danmoseley]

Ah that explains why I left so many of those. I was using this: https://gist.github.com/danmosemsft/9da8ef840afac4edc6efd6db19638a76 which has heuristics to exclude the ones called from native.

[danmoseley]

I'll paste it in here in case it's useful in future and my gist is lost.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace FindDeadResources
{
    class Program
    {
            static void Main(string[] args)
            {
                if(args.Length < 1)
                {
                    Console.WriteLine("Usage: FindDeadResources paths\\to\\code ...");
                    return;
                }

                HashSet<string> stringsUsed = new HashSet<string>();
                foreach (string codeDir in args)
                {
                    Console.Write($"===Processing {codeDir}  ");

                    if (!Directory.Exists(codeDir))
                    {
                        throw new Exception($"Can't find directory {codeDir}");
                    }

                    foreach (string path in Directory.EnumerateFiles(codeDir, "*.cs", SearchOption.AllDirectories))
                    {
                        //Console.Write(".");
                        ProcessCs(path, stringsUsed);
                    }
                    
                    Console.Write(".");

                    foreach (string path in Directory.EnumerateFiles(codeDir, "*.cpp", SearchOption.AllDirectories))
                    {
                        //Console.Write(".");
                        ProcessCs(path, stringsUsed);
                    }
                    
                    Console.Write(".");
                    
                    foreach (string path in Directory.EnumerateFiles(codeDir, "*.h", SearchOption.AllDirectories))
                    {
                        //Console.Write(".");
                        ProcessCs(path, stringsUsed);
                    }
                    
                    Console.Write(".");
                    
                    foreach (string path in Directory.EnumerateFiles(codeDir, "*.hpp", SearchOption.AllDirectories))
                    {
                        //Console.Write(".");
                        ProcessCs(path, stringsUsed);
                    }
                    
                    Console.Write(".");

                    //Console.WriteLine();
                }

                bool foundResx = false;

                foreach (string codeDir in args)
                {
                    //foreach (string resxPath in Directory.EnumerateFiles(codeDir, "*.resx", SearchOption.AllDirectories))
                    string resxPath = @"c:\git\coreclr\src\mscorlib\Resources\strings.resx";
                    {
                        foundResx = true;
                        ProcessResx(resxPath, stringsUsed);
                    }
                }

                if (!foundResx) Console.WriteLine("Found no resx's");

                Console.WriteLine();
            }

            static Regex[] srRegexs = { 

            // SR.ArgumentException_ValueTupleIncorrectType            
            new Regex(@"SR\.([\w_]+)", RegexOptions.Compiled), 
            
            // ThrowHelper.ThrowSecurityException(ExceptionResource.Security_RegistryPermission);
            new Regex(@"ExceptionResource\.([\w_]+)", RegexOptions.Compiled),
           
            // Resources.GetResourceString("Argument_InvalidTypeName")
            new Regex(@"Resources.GetResourceString\(""([\w_]+)""", RegexOptions.Compiled),
            
            // ResId.NotSupported_Constructor
            new Regex(@"ResId\.([\w_]+)", RegexOptions.Compiled),
            
            // Result.SetFailure(ParseFailureKind.Format, "Format_BadDateTime", null);
            new Regex(@"\.SetFailure\([^,]+, *""([\w_]+)""", RegexOptions.Compiled),
            
            // ResMgrGetString(W("ReflectionTypeLoad_LoadFailed"), &gc.str
            new Regex(@"ResMgrGetString.*""([\w_]+)""", RegexOptions.Compiled),
            
	    // EEResourceException e(kAppDomainUnloadedException, W("Remoting_AppDomainUnloaded_ThreadUnwound"));
	    new Regex(@"EEResourceException.*""([\w_]+)""", RegexOptions.Compiled),
	    
 	    // EX_THROW(EEArgumentException, (kArgumentNullException, argName, W("ArgumentNull_Generic")));
 	    new Regex(@"EX_THROW.*""([\w_]+)""", RegexOptions.Compiled),
 	     	    
	    // TryDemand(SECURITY_SKIP_VER, kFieldAccessException, W("Acc_RvaStatic"));
	    new Regex(@"TryDemand.*""([\w_]+)""", RegexOptions.Compiled),
	    	    
	    // LPCWSTR argName = W("Arg_InvalidHandle");
	    new Regex(@"LPCWSTR argName =.*""([\w_]+)""", RegexOptions.Compiled),
	    
	    // COMPlusThrow(kOverflowException, W("Overflow_Int32"));
	    new Regex(@"COMPlusThrow.*""([\w_]+)""", RegexOptions.Compiled),
	    
	    // from CThrowArgumentOutOfRange(W("startIndex"), W("ArgumentOutOfRange_StartIndex")); etc
	    // new Regex(@"LPCWSTR argName =.*""([\w_]+)""", RegexOptions.Compiled),
	    
	    // FCThrowXXXX(kNullReferenceException, W("NullReference_This"));
	    new Regex(@"FCThrow.*""([\w_]+)""", RegexOptions.Compiled),
            
            };
            
            private static void ProcessCs(string csPath, HashSet<string> stringsUsed)
            {
                foreach (Regex regex in srRegexs) 
                {
			string fileContents = File.ReadAllText(csPath);
			MatchCollection matches = regex.Matches(fileContents);

			foreach(Match match in matches)
			{
			    string stringName = match.Groups[1].Value;
			    //Console.WriteLine($"Found {stringName}");
			    stringsUsed.Add(stringName);
			}
		}
            }

            private static void ProcessResx(string resxPath, HashSet<string> stringsUsed)
            {
                // IDs that are constructed
                var immune = new HashSet<string> { 
			"Globalization_cp_1200" ,
			"Globalization_cp_12000",
			"Globalization_cp_12001",
			"Globalization_cp_1201" ,
			"Globalization_cp_20127",
			"Globalization_cp_28591",
			"Globalization_cp_65000",
			"Globalization_cp_65001",
			"AssertionFailed",
			"AssumptionFailed",
			"InvariantFailed",
			"PostconditionFailed",
			"PostconditionOnExceptionFailed",
			"PreconditionFailed",
			"AssertionFailed_Cnd",
			"AssumptionFailed_Cnd",
			"InvariantFailed_Cnd",
			"PostconditionFailed_Cnd",
			"PostconditionOnExceptionFailed_Cnd",
			"PreconditionFailed_Cnd",
			"event_Barrier_PhaseFinished",
			"event_ConcurrentBag_TryPeekSteals",
			"event_ConcurrentBag_TryTakeSteals",
			"event_ConcurrentDictionary_AcquiringAllLocks",
			"event_ConcurrentStack_FastPopFailed",
			"event_ConcurrentStack_FastPushFailed",
			"event_ParallelFork",
			"event_ParallelInvokeBegin",
			"event_ParallelInvokeEnd",
			"event_ParallelJoin",
			"event_ParallelLoopBegin",
			"event_ParallelLoopEnd",
			"event_SpinLock_FastPathFailed",
			"event_SpinWait_NextSpinWillYield",
			"event_TaskCompleted",
			"event_TaskScheduled",
			"event_TaskStarted",
			"event_TaskWaitBegin",
			"event_TaskWaitEnd",
			// from COMPlusThrow(kOverflowException, W("Overflow_Int32")); etc
	
			"Acc_CreateAbst",
			"Acc_CreateGeneric",
			"Acc_CreateInterface",
			"Acc_ReadOnly",
			"Arg_CannotHaveNegativeValue",
			"Arg_DlgtNullInst",
			"Arg_DlgtTargMeth",
			"Arg_InvalidANSIString",
			"Arg_InvalidBase",
			"Arg_InvalidHandle",
			"Arg_InvalidOleVariantTypeException",
			"Arg_InvalidUTF8String",
			"Arg_MethodAccessException",
			"Arg_MissingFieldException",
			"Arg_MustBeEnum",
			"Arg_MustBeInterface",
			"Arg_MustBeString",
			"Arg_MustBeStringPtrNotAtom",
			"Arg_NoDefCTor",
			"Arg_NoITypeInfo",
			"Arg_NoITypeLib",
			"Arg_NotFoundIFace",
			"Arg_NullIndex",
			"Arg_ObjObj",
			"Arg_OleAutDateInvalid",
			"Arg_OleAutDateScale",
			"Arg_ParmCnt",
			"Arg_PrimWiden",
			"ArgumentNull_Array",
			"ArgumentNull_ArrayElement",
			"ArgumentNull_AssemblyName",
			"ArgumentNull_AssemblyNameName",
			"ArgumentNull_FileName",
			"ArgumentNull_GUID",
			"ArgumentNull_Generic",
			"ArgumentNull_Obj",
			"ArgumentNull_Path",
			"ArgumentNull_SafeHandle",
			"ArgumentNull_String",
			"ArgumentNull_Type",
			"ArgumentOutOfRange_ArrayLBAndLength",
			"ArgumentOutOfRange_Capacity",
			"ArgumentOutOfRange_Count",
			"ArgumentOutOfRange_Enum",
			"ArgumentOutOfRange_Index",
			"ArgumentOutOfRange_NeedNonNegNum",
			"ArgumentOutOfRange_NeedNonNegOrNegative1",
			"Argument_AlreadyACCW",
			"Argument_BadConstantValue",
			"Argument_BadFormatSpecifier",
			"Argument_BadObjRef",
			"Argument_BadSigFormat",
			"Argument_CORDBBadMethod",
			"Argument_CORDBBadVarArgCallConv",
			"Argument_CannotCreateString",
			"Argument_CannotCreateTypedReference",
			"Argument_CantCallSecObjFunc",
			"Argument_DuplicateTypeName",
			"Argument_EmptyFileName",
			"Argument_GenericsInvalid",
			"Argument_InterfaceMap",
			"Argument_InvalidAssemblyName",
			"Argument_InvalidFlag",
			"Argument_InvalidGenericArg",
			"Argument_InvalidValue",
			"Argument_MissingDefaultConstructor",
			"Argument_MustBeRuntimeType",
			"Argument_MustHaveLayoutOrBeBlittable",
			"Argument_NeedNonGenericObject",
			"Argument_NeedNonGenericType",
			"Argument_NoUnderlyingCCW",
			"Argument_NoUninitializedStrings",
			"Argument_NotATP",
			"Argument_ObjIsWinRTObject",
			"Argument_StringZeroLength",
			"Argument_TypeMustBeVisibleFromCom",
			"Argument_TypeNotValid",
			"Argument_VerStringTooLong",
			"ArrayTypeMismatch_CantAssignType",
			"Format_BadBase",
			"Format_EmptyInputString",
			"Format_ExtraJunkAtEnd",
			"Format_NoParsibleDigits",
			"Format_StringZeroLength",
			"InvalidCast_DownCastArrayElement",
			"InvalidCast_OATypeMismatch",
			"InvalidCast_StoreArrayElement",
			"InvalidOperation_CantInstantiateAbstractClass",
			"InvalidOperation_CantInstantiateFunctionPointer",
			"InvalidOperation_CriticalTransparentAreMutuallyExclusive",
			"InvalidOperation_EnumEnded",
			"InvalidOperation_HandleIsNotInitialized",
			"InvalidOperation_StrongNameKeyPairRequired",
			"InvalidOperation_TypeCannotBeBoxed",
			"Invoke",
			"NotSupported_ByRefLike",
			"NotSupported_ByRefLikeArray",
			"NotSupported_ByRefReturn",
			"NotSupported_VoidArray",		
			"NotSupported_ChangeType",
			"NotSupported_CollectibleAssemblyResolve",
			"NotSupported_CollectibleBoundNonCollectible",
			"NotSupported_CollectibleCOM",
			"NotSupported_CollectibleDelegateMarshal",
			"NotSupported_CollectibleNotYet",
			"NotSupported_CollectibleResolveFailure",
			"NotSupported_DelegateMarshalToWrongDomain",
			"NotSupported_GenericMethod",
			"NotSupported_IDispInvokeDefaultMemberWithNamedArgs",
			"NotSupported_ManagedActivation",
			"NotSupported_NativeCallableTarget",
			"NotSupported_NonBlittableTypes",
			"NotSupported_NonReflectedType",
			"NotSupported_NonStaticMethod",
			"NotSupported_OleAutBadVarType",
			"NotSupported_OpenType",
			"NotSupported_PIAInAppxProcess",
			"NotSupported_SignalAndWaitSTAThread",
			"NotSupported_TooManyArgs",
			"NotSupported_Type",
			"NotSupported_ValueClassCM",
			"NotSupported_WinRT_PartialTrust",
			"NullReference_This",
			"NumberFormatInfo",
			"Overflow_Currency",
			"Overflow_Int16",
			"Overflow_Int32",
			"Overflow_Int64",
			"Overflow_NegativeUnsigned",
			"Overflow_SByte",
			"Overflow_UInt32",
			"Overflow_UInt64",
			"PlatformNotSupported_NamedSyncObjectWaitAnyWaitAll",
			"PlatformNotSupported_WinRT",
			"Policy_CannotLoadSemiTrustAssembliesDuringInit",
			"Rank_MultiDimNotSupported",
			"Remoting_AppDomainUnloaded_ThreadUnwound",
			"UnauthorizedAccess_SystemDomain",
	


			// from CThrowArgumentOutOfRange(W("startIndex"), W("ArgumentOutOfRange_StartIndex")); etc
			"Arg_ArgumentOutOfRangeException",
			"Arg_InvalidHandle",
			"Arg_InvalidSwitchName",
			"Arg_LongerThanDestArray",
			"Arg_LongerThanSrcArray",
			"Arg_MustBePrimArray",
			"ArgumentNull_TypedRefType",
			"ArgumentOutOfRange_ArrayLB",
			"ArgumentOutOfRange_Count",
			"ArgumentOutOfRange_DecimalRound",
			"ArgumentOutOfRange_Index",
			"ArgumentOutOfRange_NeedNonNegNum",
			"ArgumentOutOfRange_NegativeLength",
			"ArgumentOutOfRange_PartialWCHAR",
			"ArgumentOutOfRange_StartIndex",
			"Argument_ArgumentZero",
			"Argument_HandleLeak",
			"Argument_InvalidOffLen",
			"Argument_MustHaveLayoutOrBeBlittable",
			"Argument_NeedStructWithNoRefs",
			"Argument_StructMustNotBeValueClass",
			"ArrayTypeMismatch_CantAssignType",
			"ArrayTypeMismatch_ConstrainedCopy",
			"IndexOutOfRange_ArrayRankIndex",
			"IndexOutOfRange_IORaceCondition",
			"InvalidOperation_HandleIsNotInitialized",
			"NotSupported_DynamicAssemblyNoRunAccess",
			"NotSupported_Type",
			"NullReference_This",
			"Overflow_Currency",
			"Overflow_Decimal",
			"Overflow_Int32",
			"Rank_MustMatch"
                };

                XDocument resxDoc = XDocument.Load(resxPath, LoadOptions.PreserveWhitespace);
                List<XElement> nodesToRemove = new List<XElement>();
                int stringsKept = 0;
                int immuneKept = 0;

                foreach(XElement dataNode in resxDoc.Descendants(XName.Get("data")))
                {
                    string stringName = dataNode.Attribute("name").Value;
                    if (stringsUsed.Contains(stringName))
                    {
                        stringsKept++;
                    }
                    else if (immune.Contains(stringName))
                    {
                        immuneKept++;
                    }
                    else
                    { 
                        nodesToRemove.Add(dataNode);
                    }
                }

                foreach(XElement nodeToRemove in nodesToRemove)
                {
                    RemoveWithNextWhitespace(nodeToRemove);
                }

                Console.WriteLine($"Removed {nodesToRemove.Count} strings, kept {stringsKept}, immune {immuneKept}");

                resxDoc.Save(resxPath);
        }

        public static void RemoveWithNextWhitespace(XElement element)
        {
            IEnumerable<XText> textNodes
                = element.NodesAfterSelf()
                         .TakeWhile(node => node is XText).Cast<XText>();
            if (element.ElementsAfterSelf().Any()) {
                // Easy case, remove following text nodes.
                textNodes.ToList().ForEach(node => node.Remove());
            } else {
                // Remove trailing whitespace.
                textNodes.TakeWhile(text => !text.Value.Contains("\n"))
                         .ToList().ForEach(text => text.Remove());
                // Fetch text node containing newline, if any.
                XText newLineTextNode
                    = element.NodesAfterSelf().OfType<XText>().FirstOrDefault();
                if (newLineTextNode != null) {
                    string value = newLineTextNode.Value;
                    if (value.Length > 1) {
                        // Composite text node, trim until newline (inclusive).
                        newLineTextNode.AddAfterSelf(
                            new XText(value.Substring(value.IndexOf('\n') + 1)));
                    }
                    // Remove original node.
                    newLineTextNode.Remove();
                }
            }
            element.Remove();
        }

    }
}

[danmoseley]

@stephentoub it looks like some or all of the ones you have left (eg event_ParallelJoin) I had excluded explicitly because they were constructed ID's.

[stephentoub]

@danmosemsft, yes, but constructed for things that no longer exist. These "event_" resources are used by EventSource, with "event_" prefixed to the name of an EventSource event... but Parallel, Barrier, ConcurrentStack, and ConcurrentBag aren't in CoreLib anymore, and neither are their events.

[stephentoub]

@dotnet-bot test this please

[stephentoub]

@dotnet-bot test this please

[stephentoub]

@dotnet-bot test Windows_NT x64 Checked CoreFX Tests please
@dotnet-bot test Windows_NT x64 Release CoreFX Tests please