From 86d6c687d4b6c4e892c958c9e85ccbddab4a7ffe Mon Sep 17 00:00:00 2001 From: sergey ignatov Date: Mon, 15 Aug 2016 16:41:50 +0300 Subject: [PATCH] Jitted Code Pitching Feature implemented --- src/inc/CMakeLists.txt | 4 + src/inc/clrconfigvalues.h | 8 + src/vm/CMakeLists.txt | 10 + src/vm/codeman.cpp | 32 +- src/vm/codeman.h | 4 +- src/vm/codepitchingmanager.cpp | 526 +++++++++++++++++++++++++++++++++ src/vm/dynamicmethod.h | 7 +- src/vm/gdbjit.cpp | 2 +- src/vm/gdbjit.h | 2 +- src/vm/hosting.cpp | 15 +- src/vm/method.hpp | 11 + src/vm/methodtablebuilder.cpp | 6 + src/vm/prestub.cpp | 30 +- src/vm/util.cpp | 12 +- src/vm/util.hpp | 6 +- 15 files changed, 639 insertions(+), 36 deletions(-) create mode 100644 src/vm/codepitchingmanager.cpp diff --git a/src/inc/CMakeLists.txt b/src/inc/CMakeLists.txt index 40499b44eab8..4c82f157bc5f 100644 --- a/src/inc/CMakeLists.txt +++ b/src/inc/CMakeLists.txt @@ -64,6 +64,10 @@ endforeach(IDL_SOURCE) add_compile_options(-fPIC) endif(WIN32) +if(FEATURE_JIT_PITCHING) + add_definitions(-DFEATURE_JIT_PITCHING) +endif(FEATURE_JIT_PITCHING) + # Compile *_i.cpp to lib _add_library(corguids ${CORGUIDS_SOURCES}) diff --git a/src/inc/clrconfigvalues.h b/src/inc/clrconfigvalues.h index 8a21a9d8fdf8..cfcf8e03affc 100644 --- a/src/inc/clrconfigvalues.h +++ b/src/inc/clrconfigvalues.h @@ -132,6 +132,14 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_FinalizeOnShutdown, W("FinalizeOnShutdown"), D // ARM // RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_ARMEnabled, W("ARMEnabled"), (DWORD)0, "Set it to 1 to enable ARM") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchEnabled, W("JitPitchEnabled"), (DWORD)0, "Set it to 1 to enable Jit Pitching") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMemThreshold, W("JitPitchMemThreshold"), (DWORD)0, "Pitching jits when code heap usage is larger than this (in bytes)") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMethodSizeThreshold, W("JitPitchMethodSizeThreshold"), (DWORD)0, "Pitching jit for methods whose native code size larger than this (in bytes)") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMaxLevel, W("JitPitchMaxLevel"), (DWORD)0, "Pitching jits for all methods as it possible") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchTimeInterval, W("JitPitchTimeInterval"), (DWORD)0, "Time interval between jit pitchings in ms") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchPrintStat, W("JitPitchPrintStat"), (DWORD)0, "Print statistics about Jit Pitching") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMinVal, W("JitPitchMinVal"), (DWORD)0, "Pitching jit if the value of the inner counter greater than this value (for debugging purpose only)") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_JitPitchMaxVal, W("JitPitchMaxVal"), (DWORD)0xffffffff, "Pitching jit the value of the inner counter less then this value (for debuggin purpose only)") // // Assembly Loader diff --git a/src/vm/CMakeLists.txt b/src/vm/CMakeLists.txt index adb84095587a..8bb2292816da 100644 --- a/src/vm/CMakeLists.txt +++ b/src/vm/CMakeLists.txt @@ -33,6 +33,10 @@ if(FEATURE_GDBJIT) add_definitions(-DFEATURE_GDBJIT) endif(FEATURE_GDBJIT) +if(FEATURE_JIT_PITCHING) + add_definitions(-DFEATURE_JIT_PITCHING) +endif(FEATURE_JIT_PITCHING) + set(VM_SOURCES_DAC_AND_WKS_COMMON appdomain.cpp array.cpp @@ -137,6 +141,12 @@ if(FEATURE_READYTORUN) ) endif(FEATURE_READYTORUN) +if(FEATURE_JIT_PITCHING) + list(APPEND VM_SOURCES_DAC_AND_WKS_COMMON + codepitchingmanager.cpp + ) +endif(FEATURE_JIT_PITCHING) + set(VM_SOURCES_DAC ${VM_SOURCES_DAC_AND_WKS_COMMON} contexts.cpp diff --git a/src/vm/codeman.cpp b/src/vm/codeman.cpp index d934b824f66e..4239022e6a6e 100644 --- a/src/vm/codeman.cpp +++ b/src/vm/codeman.cpp @@ -2186,7 +2186,7 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi } CONTRACT_END; size_t initialRequestSize = pInfo->getRequestSize(); - size_t minReserveSize = VIRTUAL_ALLOC_RESERVE_GRANULARITY; // ( 64 KB) + size_t minReserveSize = VIRTUAL_ALLOC_RESERVE_GRANULARITY; // ( 64 KB) #ifdef _WIN64 if (pInfo->m_hiAddr == 0) @@ -2390,12 +2390,12 @@ void* EEJitManager::allocCodeRaw(CodeHeapRequestInfo *pInfo, { // Let us create a new heap. - DomainCodeHeapList *pList = GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator); + DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator); if (pList == NULL) { // not found so need to create the first one pList = CreateCodeHeapList(pInfo); - _ASSERTE(pList == GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator)); + _ASSERTE(pList == GetCodeHeapList(pInfo, pInfo->m_pAllocator)); } _ASSERTE(pList); @@ -2478,23 +2478,29 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll SIZE_T totalSize = blockSize; + CodeHeader * pCodeHdr = NULL; + + CodeHeapRequestInfo requestInfo(pMD); +#if defined(FEATURE_JIT_PITCHING) + if (pMD->IsPitchable() && CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMethodSizeThreshold) < blockSize) + { + requestInfo.SetDynamicDomain(); + } +#endif + #if defined(USE_INDIRECT_CODEHEADER) SIZE_T realHeaderSize = offsetof(RealCodeHeader, unwindInfos[0]) + (sizeof(T_RUNTIME_FUNCTION) * nUnwindInfos); // if this is a LCG method then we will be allocating the RealCodeHeader // following the code so that the code block can be removed easily by // the LCG code heap. - if (pMD->IsLCGMethod()) + if (requestInfo.IsDynamicDomain()) { totalSize = ALIGN_UP(totalSize, sizeof(void*)) + realHeaderSize; static_assert_no_msg(CODE_SIZE_ALIGN >= sizeof(void*)); } #endif // USE_INDIRECT_CODEHEADER - CodeHeader * pCodeHdr = NULL; - - CodeHeapRequestInfo requestInfo(pMD); - // Scope the lock { CrstHolder ch(&m_CodeHeapCritSec); @@ -2521,7 +2527,7 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll pCodeHdr = ((CodeHeader *)pCode) - 1; #ifdef USE_INDIRECT_CODEHEADER - if (pMD->IsLCGMethod()) + if (requestInfo.IsDynamicDomain()) { pCodeHdr->SetRealCodeHeader((BYTE*)pCode + ALIGN_UP(blockSize, sizeof(void*))); } @@ -2550,7 +2556,7 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll RETURN(pCodeHdr); } -EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(MethodDesc *pMD, LoaderAllocator *pAllocator, BOOL fDynamicOnly) +EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(CodeHeapRequestInfo *pInfo, LoaderAllocator *pAllocator, BOOL fDynamicOnly) { CONTRACTL { NOTHROW; @@ -2564,7 +2570,7 @@ EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(MethodDesc *pMD, // get the appropriate list of heaps // pMD is NULL for NGen modules during Module::LoadTokenTables - if (fDynamicOnly || (pMD != NULL && pMD->IsLCGMethod())) + if (fDynamicOnly || (pInfo != NULL && pInfo->IsDynamicDomain())) { ppList = m_DynamicDomainCodeHeaps.Table(); count = m_DynamicDomainCodeHeaps.Count(); @@ -2605,7 +2611,7 @@ HeapList* EEJitManager::GetCodeHeap(CodeHeapRequestInfo *pInfo) // loop through the m_DomainCodeHeaps to find the AppDomain // if not found, then create it - DomainCodeHeapList *pList = GetCodeHeapList(pInfo->m_pMD, pInfo->m_pAllocator); + DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator); if (pList) { // Set pResult to the largest non-full HeapList @@ -2726,7 +2732,7 @@ bool EEJitManager::CanUseCodeHeap(CodeHeapRequestInfo *pInfo, HeapList *pCodeHea } } - return retVal; + return retVal; } EEJitManager::DomainCodeHeapList * EEJitManager::CreateCodeHeapList(CodeHeapRequestInfo *pInfo) diff --git a/src/vm/codeman.h b/src/vm/codeman.h index f85eeb59db05..afef682e2a18 100644 --- a/src/vm/codeman.h +++ b/src/vm/codeman.h @@ -369,6 +369,8 @@ struct CodeHeapRequestInfo bool m_isCollectible; bool IsDynamicDomain() { return m_isDynamicDomain; } + void SetDynamicDomain() { m_isDynamicDomain = true; } + bool IsCollectible() { return m_isCollectible; } size_t getRequestSize() { return m_requestSize; } @@ -1095,7 +1097,7 @@ private : size_t header, size_t blockSize, unsigned align, HeapList ** ppCodeHeap /* Writeback, Can be null */ ); - DomainCodeHeapList *GetCodeHeapList(MethodDesc *pMD, LoaderAllocator *pAllocator, BOOL fDynamicOnly = FALSE); + DomainCodeHeapList *GetCodeHeapList(CodeHeapRequestInfo *pInfo, LoaderAllocator *pAllocator, BOOL fDynamicOnly = FALSE); DomainCodeHeapList *CreateCodeHeapList(CodeHeapRequestInfo *pInfo); LoaderHeap* GetJitMetaHeap(MethodDesc *pMD); #endif // !CROSSGEN_COMPILE diff --git a/src/vm/codepitchingmanager.cpp b/src/vm/codepitchingmanager.cpp new file mode 100644 index 000000000000..5f69d88f2003 --- /dev/null +++ b/src/vm/codepitchingmanager.cpp @@ -0,0 +1,526 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// =========================================================================== +// File: CodePitchingManager.cpp +// + +// =========================================================================== +// This file contains the implementation for code pitching. +// Its distinctive features and algorithm are: +// +// 1. All its code is under #if defined(FEATURE_JIT_PITCHING) and doesn't mess up with other code +// 2. This feature is working only if the options INTERNAL_JitPitchEnabled != 0 and INTERNAL_JitPitchMemThreshold > 0 +// 3. Jitted code can be pitched only for methods that are not Dynamic, FCall or Virtual +// 4. If the size of the generated native code exceeds the value of INTERNAL_JitDPitchMethodSizeThreshold this code is +// placed in the special heap code list. Each heap block in this list stores the code for only one method and has the +// sufficient size for the code of a method aligned to 4K. The pointers to such methods are stored in the +// "PitchingCandidateMethods" hash map. +// 5. If the entrypoint of a method is backpatched this method is excluded from the "PitchingCandidateMethods" hash map +// and stored in "NotForPitchingMethods" hashmap. +// 6. When the total size of the generated native code exceeds the value of INTERNAL_JitPitchMemThreshold option, the +// execution of the program is stopped and stack frames for all the threads are inspected and pointers to methods +// being executed are stored in the "ExecutedMethods" hash map +// 7. The code for all the methods from the "PitchingCandidateMethods" that are not in the "ExecutedMethods" is pitched. +// (All heap blocks for these methods are set in the initial state and can be reused for newly compiled methods, pointers +// to the code for non-executed methods are set to NULL). +// 8. If the code for the given method is pitched once, this method is stored in the "NotForPitchingMethods" hashmap. Thus, +// if this method is compiled the second time, it is considered as called repeatedly, therefore, pitching for it is inexpedient, +// and the newly compiled code stored in the usual heap. +// 9. The coreclr code with this feature is built by the option +// ./build.sh cmakeargs -DFEATURE_JIT_PITCHING=true +// =========================================================================== + +#include "common.h" + +#ifndef DACCESS_COMPILE + +#if defined(FEATURE_JIT_PITCHING) + +#include "nibblemapmacros.h" +#include "threadsuspend.h" + +static PtrHashMap* s_pPitchingCandidateMethods = NULL; +static PtrHashMap* s_pPitchingCandidateSizes = NULL; +static SimpleRWLock* s_pPitchingCandidateMethodsLock = NULL; + +static PtrHashMap* s_pExecutedMethods = NULL; +static SimpleRWLock* s_pExecutedMethodsLock = NULL; + +static PtrHashMap* s_pNotForPitchingMethods = NULL; +static SimpleRWLock* s_pNotForPitchingMethodsLock = NULL; + +#ifdef _DEBUG +static PtrHashMap* s_pPitchedMethods = NULL; +static SimpleRWLock* s_pPitchedMethodsLock = NULL; +#endif + +static ULONG s_totalNCSize = 0; +static SimpleRWLock* s_totalNCSizeLock = NULL; + +static ULONG s_jitPitchedBytes = 0; + +static INT64 s_JitPitchLastTick = 0; + +static bool s_printStack = false; + +static void CreateRWLock(SimpleRWLock** lock) +{ + if (*lock == NULL) + { + void *pLockSpace = SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(SimpleRWLock))); + SimpleRWLock *pLock = new (pLockSpace) SimpleRWLock(COOPERATIVE_OR_PREEMPTIVE, LOCK_TYPE_DEFAULT); + + if (FastInterlockCompareExchangePointer(lock, pLock, NULL) != NULL) + SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()->BackoutMem(pLockSpace, sizeof(SimpleRWLock)); + } +} + +static COUNT_T GetFullHash(MethodDesc* pMD) +{ + const char *moduleName = pMD->GetModule()->GetSimpleName(); + + COUNT_T hash = HashStringA(moduleName); // Start the hash with the Module name + + SString className, methodName, methodSig; + + pMD->GetMethodInfo(className, methodName, methodSig); + + hash = HashCOUNT_T(hash, className.Hash()); // Hash in the name of the Class name + hash = HashCOUNT_T(hash, methodName.Hash()); // Hash in the name of the Method name + hash = HashCOUNT_T(hash, methodSig.Hash()); // Hash in the name of the Method signature + + return hash; +} + +bool MethodDesc::IsPitchable() +{ + if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) == 0) || + (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) == 0)) + return FALSE; + + if (this == NULL) + return FALSE; + + if (IsLCGMethod() || IsVtableMethod() || IsInterface() || IsVirtual()) + return FALSE; + + if (s_pNotForPitchingMethodsLock != NULL) + { + SimpleReadLockHolder srlh(s_pNotForPitchingMethodsLock); + if (s_pNotForPitchingMethods != NULL) + { + UPTR key = (UPTR)GetFullHash(this); + MethodDesc *pFound = (MethodDesc *)s_pNotForPitchingMethods->LookupValue(key, (LPVOID)this); + if (pFound != (MethodDesc *)INVALIDENTRY) + { + return FALSE; + } + } + } + return TRUE; +} + + +static BOOL IsOwnerOfRWLock(LPVOID lock) +{ + // @TODO - SimpleRWLock does not have knowledge of which thread gets the writer + // lock, so no way to verify + return TRUE; +} + +EXTERN_C bool LookupOrCreateInNotForPitching(MethodDesc* pMD) +{ + CONTRACTL + { + MODE_COOPERATIVE; + GC_TRIGGERS; + THROWS; + } + CONTRACTL_END; + + if (pMD != NULL && pMD->IsPitchable()) + { + CreateRWLock(&s_pNotForPitchingMethodsLock); + + UPTR key = (UPTR)GetFullHash(pMD); + + if (s_pNotForPitchingMethods == NULL) + { + SimpleWriteLockHolder swlh(s_pNotForPitchingMethodsLock); + if (s_pNotForPitchingMethods == NULL) + { + PtrHashMap *pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap(); + LockOwner lock = {s_pNotForPitchingMethodsLock, IsOwnerOfRWLock}; + pMap->Init(32, NULL, FALSE, &lock); + s_pNotForPitchingMethods = pMap; + } + } + else + { + SimpleReadLockHolder srlh(s_pNotForPitchingMethodsLock); + MethodDesc *pFound = (MethodDesc *)s_pNotForPitchingMethods->LookupValue(key, (LPVOID)pMD); + if (pFound != (MethodDesc *)INVALIDENTRY) + return TRUE; + } + + { + SimpleWriteLockHolder swlh(s_pNotForPitchingMethodsLock); + s_pNotForPitchingMethods->InsertValue(key, (LPVOID)pMD); + } + } + return FALSE; +} + +static void LookupOrCreateInPitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode) +{ + CONTRACTL + { + MODE_COOPERATIVE; + GC_TRIGGERS; + THROWS; + } + CONTRACTL_END; + + if (!pMD->IsPitchable()) + return; + + PCODE prCode = pMD->GetPreImplementedCode(); + if (prCode != NULL) + return; + + if (!pMD->HasPrecode()) + return; + + // We lazily allocate the reader/writer lock we synchronize access to the hash with. + CreateRWLock(&s_pPitchingCandidateMethodsLock); + + // Now we have a lock we can use to synchronize the remainder of the init. + + UPTR key = (UPTR)GetFullHash(pMD); + + if (s_pPitchingCandidateMethods == NULL) + { + SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock); + if (s_pPitchingCandidateMethods == NULL) + { + PtrHashMap *pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap(); + LockOwner lock = {s_pPitchingCandidateMethodsLock, IsOwnerOfRWLock}; + pMap->Init(32, NULL, FALSE, &lock); + s_pPitchingCandidateMethods = pMap; + + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap(); + pMap->Init(32, NULL, FALSE, &lock); + s_pPitchingCandidateSizes = pMap; + } + } + } + else + { + // Try getting an existing value first. + SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock); + MethodDesc *pFound = (MethodDesc *)s_pPitchingCandidateMethods->LookupValue(key, (LPVOID)pMD); + if (pFound != (MethodDesc *)INVALIDENTRY) + return; + } + + { + SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock); + s_pPitchingCandidateMethods->InsertValue(key, (LPVOID)pMD); + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + s_pPitchingCandidateSizes->InsertValue((UPTR)pMD, (LPVOID)((ULONGLONG)(sizeOfCode << 1))); + } +#ifdef _DEBUG + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + printf("Candidate %lld %s :: %s %s\n", + sizeOfCode, pMD->m_pszDebugClassName, pMD->GetName(), pMD->m_pszDebugMethodSignature); + } +#endif + } +} + +EXTERN_C void DeleteFromPitchingCandidate(MethodDesc* pMD) +{ + CONTRACTL + { + MODE_COOPERATIVE; + GC_TRIGGERS; + THROWS; + } + CONTRACTL_END; + + if (pMD != NULL && pMD->IsPitchable()) + { + PCODE pCode = pMD->GetPreImplementedCode(); + if (pCode != NULL) + return; + + _ASSERTE((s_pPitchingCandidateMethodsLock == NULL && s_pPitchingCandidateMethods == NULL) || + (s_pPitchingCandidateMethodsLock != NULL && s_pPitchingCandidateMethods != NULL)); + + if (s_pPitchingCandidateMethodsLock == NULL || s_pPitchingCandidateMethods == NULL) + return; + + UPTR key = (UPTR)GetFullHash(pMD); + + { + SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock); + MethodDesc *pFound = (MethodDesc *)s_pPitchingCandidateMethods->LookupValue(key, (LPVOID)pMD); + if (pFound == (MethodDesc *)INVALIDENTRY) + return; + } + + { + SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock); + s_pPitchingCandidateMethods->DeleteValue(key, (LPVOID)pMD); + } + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + _ASSERTE(s_pPitchingCandidateSizes == NULL); + LPVOID pitchedBytes; + { + SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock); + pitchedBytes = s_pPitchingCandidateSizes->LookupValue((UPTR)pMD, NULL); + _ASSERTE(pitchedBytes != (LPVOID)INVALIDENTRY); + } + { + SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock); + s_pPitchingCandidateSizes->DeleteValue((UPTR)pMD, pitchedBytes); + } + } + } +} + +EXTERN_C void MarkMethodNotPitchingCandidate(MethodDesc* pMD) +{ + + DeleteFromPitchingCandidate(pMD); + (void)LookupOrCreateInNotForPitching(pMD); +} + +StackWalkAction CrawlFrameVisitor(CrawlFrame* pCf, Thread* pMdThread) +{ + CONTRACTL + { + THROWS; + GC_NOTRIGGER; + SO_TOLERANT; + MODE_ANY; + } + CONTRACTL_END; + + MethodDesc* pMD = pCf->GetFunction(); + + // Filter out methods we don't care about + if (pMD == nullptr || !pMD->IsPitchable()) + { + return SWA_CONTINUE; + } + + if (s_pExecutedMethods == NULL) + { + PtrHashMap *pMap = new (SystemDomain::GetGlobalLoaderAllocator()->GetLowFrequencyHeap()) PtrHashMap(); + pMap->Init(TRUE, NULL); + s_pExecutedMethods = pMap; + } + + UPTR key = (UPTR)GetFullHash(pMD); + MethodDesc *pFound = (MethodDesc *)s_pExecutedMethods->LookupValue(key, (LPVOID)pMD); + if (pFound == (MethodDesc *)INVALIDENTRY) + { + s_pExecutedMethods->InsertValue(key, (LPVOID)pMD); + } + + return SWA_CONTINUE; +} + +// Visitor for stack walk callback. +StackWalkAction StackWalkCallback(CrawlFrame* pCf, VOID* data) +{ + WRAPPER_NO_CONTRACT; + + // WalkInfo* info = (WalkInfo*) data; + return CrawlFrameVisitor(pCf, (Thread *)data); +} + +static ULONGLONG s_PitchedMethodCounter = 0; +void MethodDesc::PitchNativeCode() +{ + WRAPPER_NO_CONTRACT; + SUPPORTS_DAC; + + g_IBCLogger.LogMethodDescAccess(this); + + if (!IsPitchable()) + return; + + PCODE pCode = GetNativeCode(); + + if (pCode == NULL) + return; + + _ASSERTE(HasPrecode()); + + _ASSERTE(HasNativeCode()); + + ++s_PitchedMethodCounter; + + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMinVal) > s_PitchedMethodCounter) + { + return; + } + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMaxVal) < s_PitchedMethodCounter) + { + return; + } + + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMaxLevel) == 0) + { + if (LookupOrCreateInNotForPitching(this)) + return; + } + + MethodTable * pMT = GetMethodTable(); + _ASSERTE(pMT != NULL); + + CodeHeader* pCH = ((CodeHeader*)(pCode & ~1)) - 1; + _ASSERTE(pCH->GetMethodDesc() == this); + + HostCodeHeap* pHeap = HostCodeHeap::GetCodeHeap((TADDR)pCode); + pHeap->GetJitManager()->FreeCodeMemory(pHeap, (void*)pCode); + + ClearFlagsOnUpdate(); + + _ASSERTE(HasPrecode()); + GetPrecode()->Reset(); + + if (HasNativeCodeSlot()) + { + RelativePointer *pRelPtr = (RelativePointer *)GetAddrOfNativeCodeSlot(); + pRelPtr->SetValueMaybeNull(NULL); + } + else + { +#ifdef FEATURE_INTERPRETER + SetNativeCodeInterlocked(NULL, NULL, FALSE); +#else + SetNativeCodeInterlocked(NULL, NULL); +#endif + } + + _ASSERTE(!HasNativeCode()); + + UPTR key = (UPTR)GetFullHash(this); + ULONGLONG pitchedBytes; + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + SimpleReadLockHolder srlh(s_pPitchingCandidateMethodsLock); + pitchedBytes = (ULONGLONG)s_pPitchingCandidateSizes->LookupValue((UPTR)this, NULL); + _ASSERTE(pitchedBytes != (ULONGLONG)INVALIDENTRY); + if (pitchedBytes == (ULONGLONG)INVALIDENTRY) + pitchedBytes = 0; + s_jitPitchedBytes += (pitchedBytes >> 1); + } + { + SimpleWriteLockHolder swlh(s_pPitchingCandidateMethodsLock); + s_pPitchingCandidateMethods->DeleteValue(key, (LPVOID)this); + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + s_pPitchingCandidateSizes->DeleteValue((UPTR)this, (LPVOID)pitchedBytes); + } + } + + if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + printf("Pitched %lld %lld %s :: %s %s\n", + s_PitchedMethodCounter, pitchedBytes, m_pszDebugClassName, GetName(), m_pszDebugMethodSignature); + } + + DACNotify::DoJITPitchingNotification(this); +} + +EXTERN_C void CheckStacksAndPitch() +{ + if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) != 0) && + (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) != 0) && + (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchTimeInterval) == 0 || + ((::GetTickCount() - s_JitPitchLastTick) > CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchTimeInterval)))) + { + CreateRWLock(&s_totalNCSizeLock); + + SimpleReadLockHolder srlh(s_totalNCSizeLock); + + if ((s_totalNCSize - s_jitPitchedBytes) > CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) && s_pPitchingCandidateMethods != NULL) + { + EX_TRY + { + // Suspend the runtime. + ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_OTHER); + + // Walk all other threads. + Thread* pThread = nullptr; + while ((pThread = ThreadStore::GetThreadList(pThread)) != nullptr) + { + pThread->StackWalkFrames(StackWalkCallback, (VOID *)pThread, ALLOW_ASYNC_STACK_WALK); + } + + if (s_pExecutedMethods) + { + PtrHashMap::PtrIterator i = s_pPitchingCandidateMethods->begin(); + while (!i.end()) + { + MethodDesc *pMD = (MethodDesc *) i.GetValue(); + UPTR key = (UPTR)GetFullHash(pMD); + MethodDesc *pFound = (MethodDesc *)s_pExecutedMethods->LookupValue(key, (LPVOID)pMD); + ++i; + if (pFound == (MethodDesc *)INVALIDENTRY) + { + pMD->PitchNativeCode(); + } + } + for (PtrHashMap::PtrIterator i = s_pExecutedMethods->begin(); !i.end();) + { + MethodDesc *pMD = (MethodDesc *) i.GetValue(); + UPTR key = (UPTR)GetFullHash(pMD); + ++i; + s_pExecutedMethods->DeleteValue(key, (LPVOID)pMD); + } + delete s_pExecutedMethods; + s_pExecutedMethods = NULL; + } + + s_JitPitchLastTick = ::GetTickCount(); + + ThreadSuspend::RestartEE(FALSE, TRUE); + } + EX_CATCH + { + } + EX_END_CATCH(SwallowAllExceptions); + } + } +} + +EXTERN_C void SavePitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode) +{ + if (pMD->IsPitchable()) + { + if (sizeOfCode > 0 && CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMethodSizeThreshold) < sizeOfCode) + { + LookupOrCreateInPitchingCandidate(pMD, sizeOfCode); + } + } + if (sizeOfCode > 0 && CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchPrintStat) != 0) + { + SimpleWriteLockHolder swlh(s_totalNCSizeLock); + s_totalNCSize += sizeOfCode; + printf("jitted %lld (bytes) pitched %lld (bytes)\n", s_totalNCSize, s_jitPitchedBytes); + } +} +#endif + +#endif diff --git a/src/vm/dynamicmethod.h b/src/vm/dynamicmethod.h index f9a92b0af0e8..7fd63e59b90e 100644 --- a/src/vm/dynamicmethod.h +++ b/src/vm/dynamicmethod.h @@ -287,7 +287,7 @@ class HostCodeHeap : CodeHeap public: // Space for header is reserved immediately before. It is not included in size. virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) DAC_EMPTY_RET(NULL); - + virtual ~HostCodeHeap() DAC_EMPTY(); LoaderAllocator* GetAllocator() { return m_pAllocator; } @@ -307,6 +307,11 @@ class HostCodeHeap : CodeHeap void FreeMemForCode(void * codeStart); +#if defined(FEATURE_JIT_PITCHING) +public: + PTR_EEJitManager GetJitManager() { return m_pJitManager; } +#endif + }; // class HostCodeHeap //--------------------------------------------------------------------------------------- diff --git a/src/vm/gdbjit.cpp b/src/vm/gdbjit.cpp index 4a1d00797942..6d4366aab76e 100644 --- a/src/vm/gdbjit.cpp +++ b/src/vm/gdbjit.cpp @@ -2153,7 +2153,7 @@ void NotifyGdb::OnMethodCompiled(MethodDesc* methodDescPtr) __jit_debug_register_code(); } -void NotifyGdb::MethodDropped(MethodDesc* methodDescPtr) +void NotifyGdb::MethodPitched(MethodDesc* methodDescPtr) { static const int textSectionIndex = GetSectionIndex(".text"); diff --git a/src/vm/gdbjit.h b/src/vm/gdbjit.h index 427c09d42296..65592b529bb1 100644 --- a/src/vm/gdbjit.h +++ b/src/vm/gdbjit.h @@ -331,7 +331,7 @@ class NotifyGdb { public: static void MethodCompiled(MethodDesc* methodDescPtr); - static void MethodDropped(MethodDesc* methodDescPtr); + static void MethodPitched(MethodDesc* methodDescPtr); template class DeleteValuesOnDestructSHashTraits : public PARENT_TRAITS { diff --git a/src/vm/hosting.cpp b/src/vm/hosting.cpp index d47bc282383b..035fff8812ff 100644 --- a/src/vm/hosting.cpp +++ b/src/vm/hosting.cpp @@ -480,12 +480,15 @@ BOOL EEHeapFree(HANDLE hHeap, DWORD dwFlags, LPVOID lpMem) #ifdef _DEBUG GlobalAllocStore::RemoveAlloc (lpMem); - // Check the heap handle to detect heap contamination - lpMem = (BYTE*)lpMem - OS_HEAP_ALIGN; - HANDLE storedHeapHandle = *((HANDLE*)lpMem); - if(storedHeapHandle != hHeap) - _ASSERTE(!"Heap contamination detected! HeapFree was called on a heap other than the one that memory was allocated from.\n" - "Possible cause: you used new (executable) to allocate the memory, but didn't use DeleteExecutable() to free it."); + if (lpMem != NULL) + { + // Check the heap handle to detect heap contamination + lpMem = (BYTE*)lpMem - OS_HEAP_ALIGN; + HANDLE storedHeapHandle = *((HANDLE*)lpMem); + if(storedHeapHandle != hHeap) + _ASSERTE(!"Heap contamination detected! HeapFree was called on a heap other than the one that memory was allocated from.\n" + "Possible cause: you used new (executable) to allocate the memory, but didn't use DeleteExecutable() to free it."); + } #endif // DON'T REMOVE THIS SEEMINGLY USELESS CAST // diff --git a/src/vm/method.hpp b/src/vm/method.hpp index 71e838a94acd..36a23716b4d0 100644 --- a/src/vm/method.hpp +++ b/src/vm/method.hpp @@ -1318,6 +1318,12 @@ class MethodDesc BOOL IsNativeCodeStableAfterInit() { LIMITED_METHOD_DAC_CONTRACT; + +#if defined(FEATURE_JIT_PITCHING) + if (IsPitchable()) + return false; +#endif + return #ifdef FEATURE_TIERED_COMPILATION !IsEligibleForTieredCompilation() && @@ -1434,6 +1440,11 @@ class MethodDesc // - ngened code if IsPreImplemented() PCODE GetNativeCode(); +#if defined(FEATURE_JIT_PITCHING) + bool IsPitchable(); + void PitchNativeCode(); +#endif + //================================================================ // FindOrCreateAssociatedMethodDesc // diff --git a/src/vm/methodtablebuilder.cpp b/src/vm/methodtablebuilder.cpp index f5a256a14fdb..9bc51b83b2e8 100644 --- a/src/vm/methodtablebuilder.cpp +++ b/src/vm/methodtablebuilder.cpp @@ -6834,6 +6834,12 @@ MethodTableBuilder::NeedsNativeCodeSlot(bmtMDMethod * pMDMethod) } #endif +#if defined(FEATURE_JIT_PITCHING) + if ((CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchEnabled) != 0) && + (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_JitPitchMemThreshold) != 0)) + return TRUE; +#endif + return GetModule()->IsEditAndContinueEnabled(); } diff --git a/src/vm/prestub.cpp b/src/vm/prestub.cpp index 1ebcdc4b4df5..cd6a30063df4 100644 --- a/src/vm/prestub.cpp +++ b/src/vm/prestub.cpp @@ -52,7 +52,14 @@ #include "callcounter.h" #endif -#ifndef DACCESS_COMPILE +#ifndef DACCESS_COMPILE + +#if defined(FEATURE_JIT_PITCHING) +EXTERN_C void CheckStacksAndPitch(); +EXTERN_C void SavePitchingCandidate(MethodDesc* pMD, ULONG sizeOfCode); +EXTERN_C void DeleteFromPitchingCandidate(MethodDesc* pMD); +EXTERN_C void MarkMethodNotPitchingCandidate(MethodDesc* pMD); +#endif EXTERN_C void STDCALL ThePreStub(); @@ -262,7 +269,6 @@ void DACNotifyCompilationFinished(MethodDesc *methodDesc) // This function creates a DeadlockAware list of methods being jitted // which prevents us from trying to JIT the same method more that once. - PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS flags) { STANDARD_VM_CONTRACT; @@ -276,6 +282,10 @@ PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS fla GetMethodTable()->GetDebugClassName(), m_pszDebugMethodName)); +#if defined(FEATURE_JIT_PITCHING) + CheckStacksAndPitch(); +#endif + PCODE pCode = NULL; ULONG sizeOfCode = 0; #if defined(FEATURE_INTERPRETER) || defined(FEATURE_TIERED_COMPILATION) @@ -548,7 +558,7 @@ PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS fla #endif // FEATURE_INTERPRETER #ifdef FEATURE_MULTICOREJIT - + // If called from multi-core JIT background thread, store code under lock, delay patching until code is queried from application threads if (fBackgroundThread) { @@ -596,6 +606,12 @@ PCODE MethodDesc::MakeJitWorker(COR_ILMETHOD_DECODER* ILHeader, CORJIT_FLAGS fla pCode = GetNativeCode(); goto Done; } +#if defined(FEATURE_JIT_PITCHING) + else + { + SavePitchingCandidate(this, sizeOfCode); + } +#endif } #ifdef FEATURE_INTERPRETER @@ -1343,7 +1359,9 @@ PCODE MethodDesc::DoPrestub(MethodTable *pDispatchingMT) #endif LOG((LF_CLASSLOADER, LL_INFO10000, " In PreStubWorker, method already jitted, backpatching call point\n")); - +#if defined(FEATURE_JIT_PITCHING) + MarkMethodNotPitchingCandidate(this); +#endif RETURN DoBackpatch(pMT, pDispatchingMT, TRUE); } @@ -2127,6 +2145,10 @@ EXTERN_C PCODE STDCALL ExternalMethodFixupWorker(TransitionBlock * pTransitionBl pCode = PatchNonVirtualExternalMethod(pMD, pCode, pImportSection, pIndirection); } } + +#if defined (FEATURE_JIT_PITCHING) + DeleteFromPitchingCandidate(pMD); +#endif } // Force a GC on every jit if the stress level is high enough diff --git a/src/vm/util.cpp b/src/vm/util.cpp index 260e0daa388d..724536a231c4 100644 --- a/src/vm/util.cpp +++ b/src/vm/util.cpp @@ -2871,7 +2871,7 @@ void DACNotify::DoJITNotification(MethodDesc *MethodDescPtr) DACNotifyExceptionHelper(Args, 2); } -void DACNotify::DoJITDiscardNotification(MethodDesc *MethodDescPtr) +void DACNotify::DoJITPitchingNotification(MethodDesc *MethodDescPtr) { CONTRACTL { @@ -2883,9 +2883,9 @@ void DACNotify::DoJITDiscardNotification(MethodDesc *MethodDescPtr) CONTRACTL_END; #if defined(FEATURE_GDBJIT) && defined(FEATURE_PAL) && !defined(CROSSGEN_COMPILE) - NotifyGdb::MethodDropped(MethodDescPtr); + NotifyGdb::MethodPitched(MethodDescPtr); #endif - TADDR Args[2] = { JIT_DISCARD_NOTIFICATION, (TADDR) MethodDescPtr }; + TADDR Args[2] = { JIT_PITCHING_NOTIFICATION, (TADDR) MethodDescPtr }; DACNotifyExceptionHelper(Args, 2); } @@ -3007,10 +3007,10 @@ BOOL DACNotify::ParseJITNotification(TADDR Args[], TADDR& MethodDescPtr) return TRUE; } -BOOL DACNotify::ParseJITDiscardNotification(TADDR Args[], TADDR& MethodDescPtr) +BOOL DACNotify::ParseJITPitchingNotification(TADDR Args[], TADDR& MethodDescPtr) { - _ASSERTE(Args[0] == JIT_DISCARD_NOTIFICATION); - if (Args[0] != JIT_DISCARD_NOTIFICATION) + _ASSERTE(Args[0] == JIT_PITCHING_NOTIFICATION); + if (Args[0] != JIT_PITCHING_NOTIFICATION) { return FALSE; } diff --git a/src/vm/util.hpp b/src/vm/util.hpp index 1f86d6c2d515..80cf97055b59 100644 --- a/src/vm/util.hpp +++ b/src/vm/util.hpp @@ -1064,7 +1064,7 @@ class DACNotify MODULE_LOAD_NOTIFICATION=1, MODULE_UNLOAD_NOTIFICATION=2, JIT_NOTIFICATION=3, - JIT_DISCARD_NOTIFICATION=4, + JIT_PITCHING_NOTIFICATION=4, EXCEPTION_NOTIFICATION=5, GC_NOTIFICATION= 6, CATCH_ENTER_NOTIFICATION = 7, @@ -1072,7 +1072,7 @@ class DACNotify // called from the runtime static void DoJITNotification(MethodDesc *MethodDescPtr); - static void DoJITDiscardNotification(MethodDesc *MethodDescPtr); + static void DoJITPitchingNotification(MethodDesc *MethodDescPtr); static void DoModuleLoadNotification(Module *Module); static void DoModuleUnloadNotification(Module *Module); static void DoExceptionNotification(class Thread* ThreadPtr); @@ -1082,7 +1082,7 @@ class DACNotify // called from the DAC static int GetType(TADDR Args[]); static BOOL ParseJITNotification(TADDR Args[], TADDR& MethodDescPtr); - static BOOL ParseJITDiscardNotification(TADDR Args[], TADDR& MethodDescPtr); + static BOOL ParseJITPitchingNotification(TADDR Args[], TADDR& MethodDescPtr); static BOOL ParseModuleLoadNotification(TADDR Args[], TADDR& ModulePtr); static BOOL ParseModuleUnloadNotification(TADDR Args[], TADDR& ModulePtr); static BOOL ParseExceptionNotification(TADDR Args[], TADDR& ThreadPtr);