Feature: dynamic expansion for generic dictionaries

src/debug/daccess/nidump.cpp

@@ -6984,15 +6984,15 @@ NativeImageDumper::DumpMethodTable( PTR_MethodTable mt, const char * name,
         if( currentDictionary != NULL )
         {
             PTR_DictionaryEntry entry(currentDictionary->EntryAddr(0));
-
+            
             PTR_DictionaryLayout layout( clazz->GetDictionaryLayout() );
 
             DisplayStartStructure( "Dictionary",
                                    DPtrToPreferredAddr(currentDictionary),
                                    //if there is a layout, use it to compute
                                    //the size, otherwise there is just the one
                                    //entry.
-                                   DictionaryLayout::GetFirstDictionaryBucketSize(mt->GetNumGenericArgs(), layout),
+                                   DictionaryLayout::GetDictionarySizeFromLayout(mt->GetNumGenericArgs(), layout),
                                    METHODTABLES );
 
             DisplayStartArrayWithOffset( m_pEntries, NULL, Dictionary,
@@ -8018,7 +8018,7 @@ void NativeImageDumper::DumpMethodDesc( PTR_MethodDesc md, PTR_Module module )
             {
                 PTR_DictionaryLayout layout(wrapped->IsSharedByGenericMethodInstantiations()
                                             ? dac_cast<TADDR>(wrapped->GetDictLayoutRaw()) : NULL );
-                dictSize = DictionaryLayout::GetFirstDictionaryBucketSize(imd->GetNumGenericMethodArgs(),
+                dictSize = DictionaryLayout::GetDictionarySizeFromLayout(imd->GetNumGenericMethodArgs(), 
                                                                           layout);
             }
         }

src/vm/appdomain.cpp

@@ -2068,6 +2068,10 @@ void SystemDomain::LoadBaseSystemClasses()
     g_pDelegateClass = MscorlibBinder::GetClass(CLASS__DELEGATE);
     g_pMulticastDelegateClass = MscorlibBinder::GetClass(CLASS__MULTICAST_DELEGATE);
 
+#ifndef CROSSGEN_COMPILE
+    CrossLoaderAllocatorHashSetup::EnsureTypesLoaded();
+#endif
+
     // used by IsImplicitInterfaceOfSZArray
     MscorlibBinder::GetClass(CLASS__IENUMERABLEGENERIC);
     MscorlibBinder::GetClass(CLASS__ICOLLECTIONGENERIC);
@@ -2083,10 +2087,6 @@ void SystemDomain::LoadBaseSystemClasses()
     g_pUtf8StringClass = MscorlibBinder::GetClass(CLASS__UTF8_STRING);
 #endif // FEATURE_UTF8STRING
 
-#ifndef CROSSGEN_COMPILE
-    CrossLoaderAllocatorHashSetup::EnsureTypesLoaded();
-#endif
-
 #ifndef CROSSGEN_COMPILE
     ECall::PopulateManagedStringConstructors();
 #endif // CROSSGEN_COMPILE

src/vm/ceeload.cpp

@@ -556,6 +556,7 @@ void Module::Initialize(AllocMemTracker *pamTracker, LPCWSTR szName)
     m_FixupCrst.Init(CrstModuleFixup, (CrstFlags)(CRST_HOST_BREAKABLE|CRST_REENTRANCY));
     m_InstMethodHashTableCrst.Init(CrstInstMethodHashTable, CRST_REENTRANCY);
     m_ISymUnmanagedReaderCrst.Init(CrstISymUnmanagedReader, CRST_DEBUGGER_THREAD);
+    m_DictionaryCrst.Init(CrstDomainLocalBlock);
 
     if (!m_file->HasNativeImage())
     {
@@ -687,8 +688,12 @@ void Module::Initialize(AllocMemTracker *pamTracker, LPCWSTR szName)
     }
 #endif // defined (PROFILING_SUPPORTED) &&!defined(DACCESS_COMPILE) && !defined(CROSSGEN_COMPILE)
 
-    LOG((LF_CLASSLOADER, LL_INFO10, "Loaded pModule: \"%ws\".\n", GetDebugName()));
+#ifndef CROSSGEN_COMPILE
+    m_dynamicSlotsHashForTypes.Init(GetLoaderAllocator());
+    m_dynamicSlotsHashForMethods.Init(GetLoaderAllocator());
+#endif
 
+    LOG((LF_CLASSLOADER, LL_INFO10, "Loaded pModule: \"%ws\".\n", GetDebugName()));
 }
 
 #endif // DACCESS_COMPILE
@@ -13233,6 +13238,327 @@ void ReflectionModule::ReleaseILData()
 
     Module::ReleaseILData();
 }
+
+TypeHandle* AllocateNewMethodDictionaryForExpansion(InstantiatedMethodDesc* pIMD, DWORD cbSize)
+{
+    TypeHandle* pInstOrPerInstInfo = (TypeHandle*)(void*)pIMD->GetLoaderAllocator()->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbSize + sizeof(void*)));
+    ZeroMemory(pInstOrPerInstInfo, cbSize + sizeof(void*));
+
+    // Slot[-1] points at previous dictionary to help with diagnostics when investigating crashes
+    *(byte**)pInstOrPerInstInfo = (byte*)pIMD->m_pPerInstInfo.GetValue() + 1;
+    pInstOrPerInstInfo++;
+
+    // Copy old dictionary entry contents
+    memcpy(pInstOrPerInstInfo, (const void*)pIMD->m_pPerInstInfo.GetValue(), pIMD->GetDictionarySlotsSize());
+
+    ULONG_PTR* pSizeSlot = ((ULONG_PTR*)pInstOrPerInstInfo) + pIMD->GetNumGenericMethodArgs();
+    *pSizeSlot = cbSize;
+
+    return pInstOrPerInstInfo;
+}
+
+TypeHandle* AllocateNewTypeDictionaryForExpansion(MethodTable* pMT, DWORD cbSize)
+{
+    TypeHandle* pInstOrPerInstInfo = (TypeHandle*)(void*)pMT->GetLoaderAllocator()->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbSize + sizeof(void*)));
+    ZeroMemory(pInstOrPerInstInfo, cbSize + sizeof(void*));
+
+    // Slot[-1] points at previous dictionary to help with diagnostics when investigating crashes
+    *(byte**)pInstOrPerInstInfo = (byte*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue() + 1;
+    pInstOrPerInstInfo++;
+
+    // Copy old dictionary entry contents
+    memcpy(pInstOrPerInstInfo, (const void*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue(), pMT->GetDictionarySlotsSize());
+
+    ULONG_PTR* pSizeSlot = ((ULONG_PTR*)pInstOrPerInstInfo) + pMT->GetNumGenericArgs();
+    *pSizeSlot = cbSize;
+
+    return pInstOrPerInstInfo;
+}
+
+#ifdef _DEBUG
+void Module::EnsureTypeRecorded(MethodTable* pMT)
+{
+    _ASSERTE(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+
+    BOOL typeExistsInHashtable = FALSE;
+    auto lamda = [&typeExistsInHashtable, pMT](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTValue)
+    {
+        typeExistsInHashtable = (pMT == pMTValue);
+        return pMT != pMTValue;
+    };
+
+    m_dynamicSlotsHashForTypes.VisitValuesOfKey(pMT->GetCanonicalMethodTable(), lamda);
+    _ASSERTE(typeExistsInHashtable);
+}
+
+void Module::EnsureMethodRecorded(MethodDesc* pMD)
+{
+    _ASSERTE(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+
+    BOOL methodExistsInHashtable = FALSE;
+    auto lamda = [&methodExistsInHashtable, pMD](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDValue)
+    {
+        methodExistsInHashtable = (pMD== pMDValue);
+        return pMD != pMDValue;
+    };
+
+    m_dynamicSlotsHashForMethods.VisitValuesOfKey(pMD->GetExistingWrappedMethodDesc(), lamda);
+    _ASSERTE(methodExistsInHashtable);
+}
+#endif
+
+void Module::RecordTypeForDictionaryExpansion_Locked(MethodTable* pGenericParentMT, MethodTable* pDependencyMT)
+{
+    CONTRACTL
+    {
+        GC_TRIGGERS;
+        PRECONDITION(CheckPointer(pGenericParentMT) && CheckPointer(pDependencyMT));
+        PRECONDITION(pGenericParentMT->HasInstantiation() && pGenericParentMT != pGenericParentMT->GetCanonicalMethodTable());
+        PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+    }
+    CONTRACTL_END
+
+    DictionaryLayout* pDictLayout = pDependencyMT->GetClass()->GetDictionaryLayout();
+    if (pDictLayout != NULL && pDictLayout->GetMaxSlots() > 0)
+    {
+        DWORD sizeFromDictLayout = DictionaryLayout::GetDictionarySizeFromLayout(pDependencyMT->GetNumGenericArgs(), pDictLayout);
+        if (pDependencyMT->GetDictionarySlotsSize() != sizeFromDictLayout)
+        {
+            _ASSERT(pDependencyMT->GetDictionarySlotsSize() < sizeFromDictLayout);
+
+            //
+            // Another thread got a chance to expand the dictionary layout and expand the dictionary slots of
+            // other types, but not for this one yet because we're still in the process of recording it for 
+            // expansions.
+            // Expand the dictionary slots here before finally adding the type to the hashtable.
+            //
+
+            TypeHandle* pInstOrPerInstInfo = AllocateNewTypeDictionaryForExpansion(pDependencyMT, sizeFromDictLayout);
+
+            // Publish the new dictionary slots to the type.
+            TypeHandle** pPerInstInfo = (TypeHandle**)pDependencyMT->GetPerInstInfo()->GetValuePtr();
+            FastInterlockExchangePointer(pPerInstInfo + (pDependencyMT->GetNumDicts() - 1), pInstOrPerInstInfo);
+
+            FlushProcessWriteBuffers();
+        }
+    }
+
+    GCX_COOP();
+    m_dynamicSlotsHashForTypes.Add(pGenericParentMT->GetCanonicalMethodTable(), pDependencyMT, GetLoaderAllocator());
+}
+
+void Module::RecordMethodForDictionaryExpansion_Locked(MethodDesc* pDependencyMD)
+{
+    CONTRACTL
+    {
+        GC_TRIGGERS;
+        PRECONDITION(CheckPointer(pDependencyMD) && pDependencyMD->HasMethodInstantiation() && pDependencyMD->IsInstantiatingStub());
+        PRECONDITION(pDependencyMD->GetDictionaryLayout() != NULL && pDependencyMD->GetDictionaryLayout()->GetMaxSlots() > 0);
+        PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+    }
+    CONTRACTL_END
+
+    DictionaryLayout* pDictLayout = pDependencyMD->GetDictionaryLayout();
+    InstantiatedMethodDesc* pIMDDependency = pDependencyMD->AsInstantiatedMethodDesc();
+
+    DWORD sizeFromDictLayout = DictionaryLayout::GetDictionarySizeFromLayout(pDependencyMD->GetNumGenericMethodArgs(), pDictLayout);
+    if (pIMDDependency->GetDictionarySlotsSize() != sizeFromDictLayout)
+    {
+        _ASSERT(pIMDDependency->GetDictionarySlotsSize() < sizeFromDictLayout);
+
+        //
+        // Another thread got a chance to expand the dictionary layout and expand the dictionary slots of
+        // other methods, but not for this one yet because we're still in the process of recording it for 
+        // expansions.
+        // Expand the dictionary slots here before finally adding the method to the hashtable.
+        //
+
+        TypeHandle* pInstOrPerInstInfo = AllocateNewMethodDictionaryForExpansion(pIMDDependency, sizeFromDictLayout);
+
+        FastInterlockExchangePointer((TypeHandle**)pIMDDependency->m_pPerInstInfo.GetValuePtr(), pInstOrPerInstInfo);
+
+        FlushProcessWriteBuffers();
+    }
+
+    GCX_COOP();
+
+    _ASSERTE(pDependencyMD->GetExistingWrappedMethodDesc() != NULL);
+    m_dynamicSlotsHashForMethods.Add(pDependencyMD->GetExistingWrappedMethodDesc(), pDependencyMD, GetLoaderAllocator());
+}
+
+void Module::ExpandTypeDictionaries_Locked(MethodTable* pMT, DictionaryLayout* pOldLayout, DictionaryLayout* pNewLayout)
+{
+    CONTRACTL
+    {
+        STANDARD_VM_CHECK;
+        INJECT_FAULT(ThrowOutOfMemory(););
+        PRECONDITION(CheckPointer(pOldLayout) && CheckPointer(pNewLayout));
+        PRECONDITION(CheckPointer(pMT) && pMT->HasInstantiation() && pMT->GetClass()->GetDictionaryLayout() == pOldLayout);
+        PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+    }
+    CONTRACTL_END
+
+    GCX_COOP();
+
+    MethodTable* pCanonMT = pMT->GetCanonicalMethodTable();
+    DWORD oldInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMT->GetNumGenericArgs(), pOldLayout);
+    DWORD newInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMT->GetNumGenericArgs(), pNewLayout);
+
+    //
+    // Dictionary expansion for types needs to be done in multiple steps, given how derived types do not directly embed dictionaries
+    // from parent types, but instead reference them from directly from the parent types. Also, this is necessary to ensure correctness
+    // for lock-free read operations for dictionary slots:
+    //      1) Allocate new dictionaries for all instantiated types of the same typedef as the one being expanded.
+    //      2) After all allocations and initializations are completed, publish the dictionaries to the types in #1 after 
+    //         flushing write buffers
+    //      3) For all types that derive from #1, update the embedded dictinoary pointer to the newly allocated one.
+    //
+
+    struct NewDictionary
+    {
+        MethodTable* pMT;
+        TypeHandle* pDictSlots;
+    };
+    StackSArray<NewDictionary> dictionaryUpdates;
+
+#ifdef _DEBUG
+    auto expandPerInstInfos = [oldInfoSize, newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+#else
+    auto expandPerInstInfos = [newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+#endif
+    {
+        if (!pMTToUpdate->HasSameTypeDefAs(pMTKey))
+            return true;
+
+        _ASSERTE(pMTToUpdate != pMTToUpdate->GetCanonicalMethodTable() && pMTToUpdate->GetCanonicalMethodTable() == pMTKey);
+        _ASSERTE(pMTToUpdate->GetDictionarySlotsSize() == oldInfoSize);
+
+        TypeHandle* pInstOrPerInstInfo = AllocateNewTypeDictionaryForExpansion(pMTToUpdate, newInfoSize);
+
+        NewDictionary entry;
+        entry.pMT = pMTToUpdate;
+        entry.pDictSlots = pInstOrPerInstInfo;
+        dictionaryUpdates.Append(entry);
+
+        return true; // Keep walking
+    };
+
+    m_dynamicSlotsHashForTypes.VisitValuesOfKey(pCanonMT, expandPerInstInfos);
+
+    // Flush write buffers to ensure new dictionaries are fully writted and initalized before publishing them
+    FlushProcessWriteBuffers();
+
+    for (SArray<NewDictionary>::Iterator i = dictionaryUpdates.Begin(); i != dictionaryUpdates.End(); i++)
+    {
+        MethodTable* pMT = i->pMT;
+        TypeHandle** pPerInstInfo = (TypeHandle**)pMT->GetPerInstInfo()->GetValuePtr();
+        FastInterlockExchangePointer(pPerInstInfo + (pMT->GetNumDicts() - 1), i->pDictSlots);
+        _ASSERTE(pMT->GetDictionarySlotsSize() == newInfoSize);
+        _ASSERTE((TypeHandle*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue() == i->pDictSlots);
+    }
+
+    auto updateDependentDicts = [](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+    {
+        if (pMTToUpdate->HasSameTypeDefAs(pMTKey))
+            return true;
+
+        MethodTable* pCurrentMT = pMTToUpdate->GetParentMethodTable();
+        while (pCurrentMT)
+        {
+            if (pCurrentMT->HasSameTypeDefAs(pMTKey))
+            {
+                DWORD dictToUpdate = pCurrentMT->GetNumDicts() - 1;
+                Dictionary* pUpdatedParentDict = pCurrentMT->GetPerInstInfo()[dictToUpdate].GetValue();
+                TypeHandle** pPerInstInfo = (TypeHandle**)pMTToUpdate->GetPerInstInfo()->GetValuePtr();
+                FastInterlockExchangePointer(pPerInstInfo + dictToUpdate, (TypeHandle*)pUpdatedParentDict);
+                _ASSERTE(pMTToUpdate->GetPerInstInfo()[dictToUpdate].GetValue() == pUpdatedParentDict);
+
+                return true; // Keep walking
+            }
+            pCurrentMT = pCurrentMT->GetParentMethodTable();
+        }
+
+        UNREACHABLE();
+    };
+
+    m_dynamicSlotsHashForTypes.VisitValuesOfKey(pCanonMT, updateDependentDicts);
+
+    // Ensure no other thread uses old dictionary pointers
+    FlushProcessWriteBuffers();
+}
+
+void Module::ExpandMethodDictionaries_Locked(MethodDesc* pMD, DictionaryLayout* pOldLayout, DictionaryLayout* pNewLayout)
+{
+    CONTRACTL
+    {
+        STANDARD_VM_CHECK;
+        INJECT_FAULT(ThrowOutOfMemory(););
+        PRECONDITION(CheckPointer(pOldLayout) && CheckPointer(pNewLayout));
+        PRECONDITION(CheckPointer(pMD));
+        PRECONDITION(pMD->HasMethodInstantiation() && pMD->GetDictionaryLayout() == pOldLayout);
+        PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+    }
+    CONTRACTL_END
+
+    GCX_COOP();
+
+    //
+    // Dictionary expansion for methods needs to be done in two steps to ensure correctness for lock-free read operations 
+    // for dictionary slots:
+    //      1) Allocate new dictionaries for all instantiated methods sharing the same canonical form as the input method
+    //      2) After all allocations and initializations are completed, publish the dictionaries to the methods after 
+    //         flushing write buffers
+    //
+
+    MethodDesc* pCanonMD = pMD->IsInstantiatingStub() ? pMD->GetExistingWrappedMethodDesc() : pMD;
+    _ASSERTE(pCanonMD != NULL);
+    DWORD oldInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMD->GetNumGenericMethodArgs(), pOldLayout);
+    DWORD newInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMD->GetNumGenericMethodArgs(), pNewLayout);
+
+    struct NewDictionary
+    {
+        InstantiatedMethodDesc* pIMD;
+        TypeHandle* pDictSlots;
+    };
+    StackSArray<NewDictionary> dictionaryUpdates;
+
+#ifdef _DEBUG
+    auto lambda = [oldInfoSize, newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDToUpdate)
+#else
+    auto lambda = [newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDToUpdate)
+#endif
+    {
+        // Update m_pPerInstInfo for the pMethodDesc being visited here
+        _ASSERTE(pMDToUpdate->IsInstantiatingStub() && pMDToUpdate->GetExistingWrappedMethodDesc() == pMDKey);
+
+        InstantiatedMethodDesc* pInstantiatedMD = pMDToUpdate->AsInstantiatedMethodDesc();
+        _ASSERTE(pInstantiatedMD->GetDictionarySlotsSize() == oldInfoSize);
+
+        TypeHandle* pInstOrPerInstInfo = AllocateNewMethodDictionaryForExpansion(pInstantiatedMD, newInfoSize);
+
+        NewDictionary entry;
+        entry.pIMD = pInstantiatedMD;
+        entry.pDictSlots = pInstOrPerInstInfo;
+        dictionaryUpdates.Append(entry);
+
+        return true; // Keep walking
+    };
+
+    m_dynamicSlotsHashForMethods.VisitValuesOfKey(pCanonMD, lambda);
+
+    // Flush write buffers to ensure new dictionaries are fully writted and initalized before publishing them
+    FlushProcessWriteBuffers();
+
+    for (SArray<NewDictionary>::Iterator i = dictionaryUpdates.Begin(); i != dictionaryUpdates.End(); i++)
+    {
+        FastInterlockExchangePointer((TypeHandle**)i->pIMD->m_pPerInstInfo.GetValuePtr(), i->pDictSlots);
+        _ASSERTE((TypeHandle*)i->pIMD->m_pPerInstInfo.GetValue() == i->pDictSlots);
+        _ASSERTE(i->pIMD->GetDictionarySlotsSize() == newInfoSize);
+    }
+
+    // Ensure no other thread uses old dictionary pointers
+    FlushProcessWriteBuffers();
+}
 #endif // !CROSSGEN_COMPILE
 
 #endif // !DACCESS_COMPILE