diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 034bd7264bdd54..195e28ee845edb 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -4764,17 +4764,14 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal { return optAssertionProp_Update(call, call, stmt); } - else if (!optLocalAssertionProp && (call->gtCallType == CT_HELPER)) - { - if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ISINSTANCEOFINTERFACE) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ISINSTANCEOFARRAY) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ISINSTANCEOFCLASS) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ISINSTANCEOFANY) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_CHKCASTINTERFACE) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_CHKCASTARRAY) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_CHKCASTCLASS) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_CHKCASTANY) || - call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_CHKCASTCLASS_SPECIAL)) + else if (!optLocalAssertionProp && call->IsHelperCall()) + { + const CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); + if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY) || + (helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFANY) || + (helper == CORINFO_HELP_CHKCASTINTERFACE) || (helper == CORINFO_HELP_CHKCASTARRAY) || + (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTANY) || + (helper == CORINFO_HELP_CHKCASTCLASS_SPECIAL)) { GenTree* arg1 = call->gtArgs.GetArgByIndex(1)->GetNode(); if (arg1->gtOper != GT_LCL_VAR) @@ -4804,6 +4801,11 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal return optAssertionProp_Update(arg1, call, stmt); } + + // TODO-InlineCast: check optAssertionIsNonNull for the object argument and replace + // the helper with its nonnull version, e.g.: + // CORINFO_HELP_ISINSTANCEOFANY -> CORINFO_HELP_ISINSTANCEOFANY_NONNULL + // so then fgLateCastExpansion can skip the null check. } } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index e09aac3ac123cf..3efa0e844b0141 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5046,6 +5046,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Expand thread local access DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess); + // Expand casts + DoPhase(this, PHASE_EXPAND_CASTS, &Compiler::fgLateCastExpansion); + // Insert GC Polls DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index cdff1b9fbfe0ae..0069686b70cfef 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5832,6 +5832,9 @@ class Compiler bool fgVNBasedIntrinsicExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call); bool fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call); + PhaseStatus fgLateCastExpansion(); + bool fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call); + PhaseStatus fgInsertGCPolls(); BasicBlock* fgCreateGCPoll(GCPollType pollType, BasicBlock* block); @@ -7519,6 +7522,7 @@ class Compiler #define OMF_HAS_TLS_FIELD 0x00010000 // Method contains TLS field access #define OMF_HAS_SPECIAL_INTRINSICS 0x00020000 // Method contains special intrinsics expanded in late phases #define OMF_HAS_RECURSIVE_TAILCALL 0x00040000 // Method contains recursive tail call +#define OMF_HAS_EXPANDABLE_CAST 0x00080000 // Method contains casts eligible for late expansion // clang-format on @@ -7559,6 +7563,16 @@ class Compiler optMethodFlags |= OMF_HAS_STATIC_INIT; } + bool doesMethodHaveExpandableCasts() + { + return (optMethodFlags & OMF_HAS_EXPANDABLE_CAST) != 0; + } + + void setMethodHasExpandableCasts() + { + optMethodFlags |= OMF_HAS_EXPANDABLE_CAST; + } + bool doesMethodHaveGuardedDevirtualization() const { return (optMethodFlags & OMF_HAS_GUARDEDDEVIRT) != 0; diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index a51ab4adc4f88e..23930985319769 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -98,6 +98,7 @@ CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, f CompPhaseNameMacro(PHASE_STRESS_SPLIT_TREE, "Stress gtSplitTree", false, -1, false) CompPhaseNameMacro(PHASE_EXPAND_RTLOOKUPS, "Expand runtime lookups", false, -1, true) CompPhaseNameMacro(PHASE_EXPAND_STATIC_INIT, "Expand static init", false, -1, true) +CompPhaseNameMacro(PHASE_EXPAND_CASTS, "Expand casts", false, -1, true) CompPhaseNameMacro(PHASE_EXPAND_TLS, "Expand TLS access", false, -1, true) CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", false, -1, true) CompPhaseNameMacro(PHASE_CREATE_THROW_HELPERS, "Create throw helper blocks", false, -1, true) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 0902c00c1650c1..8fea9d559d2ce5 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4080,6 +4080,7 @@ enum GenTreeCallFlags : unsigned int GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type + GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed. }; inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a) @@ -5588,8 +5589,9 @@ struct GenTreeCall final : public GenTree CORINFO_CLASS_HANDLE gtRetClsHnd; // The return type handle of the call if it is a struct; always available union { - void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined - CORINFO_CLASS_HANDLE gtInitClsHnd; // Used by static init helpers, represents a class they init + void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined + CORINFO_CLASS_HANDLE gtInitClsHnd; // Used by static init helpers, represents a class they init + IL_OFFSET gtCastHelperILOffset; // Used by cast helpers to save corresponding IL offset }; union { diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index 9b9fb009a6f50f..ac2a65b33ef95c 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -33,6 +33,55 @@ static GenTree* SpillExpression(Compiler* comp, GenTree* expr, BasicBlock* exprB return comp->gtNewLclVarNode(tmpNum); }; +//------------------------------------------------------------------------------ +// SplitAtTreeAndReplaceItWithLocal : Split block at the given tree and replace it with a local +// See comments in gtSplitTree and fgSplitBlockBeforeTree +// TODO: use this function in more places in this file. +// +// Arguments: +// comp - Compiler instance +// block - Block to split +// stmt - Statement containing the tree to split at +// tree - Tree to split at +// topBlock - [out] Top block after the split +// bottomBlock - [out] Bottom block after the split +// +// Return Value: +// Number of the local that replaces the tree +// +static unsigned SplitAtTreeAndReplaceItWithLocal( + Compiler* comp, BasicBlock* block, Statement* stmt, GenTree* tree, BasicBlock** topBlock, BasicBlock** bottomBlock) +{ + BasicBlock* prevBb = block; + GenTree** callUse = nullptr; + Statement* newFirstStmt = nullptr; + block = comp->fgSplitBlockBeforeTree(block, stmt, tree, &newFirstStmt, &callUse); + assert(prevBb != nullptr && block != nullptr); + + // Block ops inserted by the split need to be morphed here since we are after morph. + // We cannot morph stmt yet as we may modify it further below, and the morphing + // could invalidate callUse + while ((newFirstStmt != nullptr) && (newFirstStmt != stmt)) + { + comp->fgMorphStmtBlockOps(block, newFirstStmt); + newFirstStmt = newFirstStmt->GetNextStmt(); + } + + // Grab a temp to store the result. + const unsigned tmpNum = comp->lvaGrabTemp(true DEBUGARG("replacement local")); + comp->lvaTable[tmpNum].lvType = tree->TypeGet(); + + // Replace the original call with that temp + *callUse = comp->gtNewLclvNode(tmpNum, tree->TypeGet()); + + comp->fgMorphStmtBlockOps(block, stmt); + comp->gtUpdateStmtSideEffects(stmt); + + *topBlock = prevBb; + *bottomBlock = block; + return tmpNum; +} + //------------------------------------------------------------------------------ // gtNewRuntimeLookupHelperCallNode : Helper to create a runtime lookup call helper node. // @@ -1765,3 +1814,309 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, JITDUMP("ReadUtf8: succesfully expanded!\n") return true; } + +//------------------------------------------------------------------------------ +// fgLateCastExpansion: Partially inline various cast helpers, e.g.: +// +// tmp = CORINFO_HELP_ISINSTANCEOFINTERFACE(clsHandle, obj); +// +// into: +// +// tmp = obj; +// if ((obj != null) && (obj->pMT != likelyClassHandle)) +// { +// tmp = CORINFO_HELP_ISINSTANCEOFINTERFACE(clsHandle, obj); +// } +// +// The goal is to move cast expansion logic from the importer to this phase, for now, +// this phase only supports "isinst" and for profiled casts only. +// +// Returns: +// PhaseStatus indicating what, if anything, was changed. +// +PhaseStatus Compiler::fgLateCastExpansion() +{ + if (!doesMethodHaveExpandableCasts()) + { + // Nothing to expand in the current method + return PhaseStatus::MODIFIED_NOTHING; + } + + if (!opts.IsOptimizedWithProfile()) + { + // Currently, we're only interested in expanding cast helpers using profile data + return PhaseStatus::MODIFIED_NOTHING; + } + + if (JitConfig.JitConsumeProfileForCasts() == 0) + { + return PhaseStatus::MODIFIED_NOTHING; + } + + const bool preferSize = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_SIZE_OPT); + if (preferSize) + { + // The optimization comes with a codegen size increase + JITDUMP("Optimized for size - bail out.\n"); + return PhaseStatus::MODIFIED_NOTHING; + } + return fgExpandHelper<&Compiler::fgLateCastExpansionForCall>(true); +} + +//------------------------------------------------------------------------------ +// PickLikelyClass: picks a likely class handle corresponding to the given IL offset +// +// Arguments: +// comp - Compiler instance +// offset - IL offset +// likelihood - [out] likelihood of the returned class +// +// Returns: +// Likely class handle or NO_CLASS_HANDLE +// +static CORINFO_CLASS_HANDLE PickLikelyClass(Compiler* comp, IL_OFFSET offset, unsigned* likelihood) +{ + // TODO-InlineCast: consider merging this helper with pickGDV + + const int maxLikelyClasses = 8; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + unsigned likelyClassCount = getLikelyClasses(likelyClasses, maxLikelyClasses, comp->fgPgoSchema, + comp->fgPgoSchemaCount, comp->fgPgoData, (int)offset); + + if (likelyClassCount == 0) + { + return NO_CLASS_HANDLE; + } + +#ifdef DEBUG + // Print all the candidates and their likelihoods to the log + for (UINT32 i = 0; i < likelyClassCount; i++) + { + const char* className = comp->eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, + likelyClasses[i].likelihood); + } + + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + if (JitConfig.JitRandomGuardedDevirtualization() != 0) + { + // Reuse the random inliner's random state. + CLRRandom* const random = + comp->impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + unsigned index = static_cast(random->Next(static_cast(likelyClassCount))); + *likelihood = 100; + return (CORINFO_CLASS_HANDLE)likelyClasses[index].handle; + } +#endif + *likelihood = likelyClasses[0].likelihood; + return (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; +} + +//------------------------------------------------------------------------------ +// fgLateCastExpansionForCall : Expand specific cast helper, see +// fgLateCastExpansion's comments. +// +// Arguments: +// block - Block containing the cast helper to expand +// stmt - Statement containing the cast helper +// call - The cast helper +// +// Returns: +// True if expanded, false otherwise. +// +bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call) +{ + if (!call->IsHelperCall() || !impIsCastHelperMayHaveProfileData(call->GetHelperNum())) + { + // Not a cast helper we're interested in + return false; + } + + if ((call->gtCallMoreFlags & GTF_CALL_M_CAST_CAN_BE_EXPANDED) == 0) + { + // It's not eligible for expansion (already expanded in importer) + // To be removed once we move cast expansion here completely. + return false; + } + + // Helper calls are never tail calls + assert(!call->IsTailCall()); + + BasicBlock* block = *pBlock; + JITDUMP("Attempting to expand a cast helper call in " FMT_BB "...\n", block->bbNum); + DISPTREE(call); + JITDUMP("\n"); + + // Currently, we only expand "isinst" and only using profile data. The long-term plan is to + // move cast expansion logic here from the importer completely. + unsigned likelihood = 100; + CORINFO_CLASS_HANDLE likelyCls = PickLikelyClass(this, call->gtCastHelperILOffset, &likelihood); + if (likelyCls == NO_CLASS_HANDLE) + { + // TODO: make null significant, so it could mean our object is likely just null + JITDUMP("Likely class is null - bail out.\n"); + return false; + } + + // if there is a dominating candidate with >= 50% likelihood, use it + const unsigned likelihoodMinThreshold = 50; + if (likelihood < likelihoodMinThreshold) + { + JITDUMP("Likely class likelihood is below %u%% - bail out.\n", likelihoodMinThreshold); + return false; + } + + // E.g. "call CORINFO_HELP_ISINSTANCEOFCLASS(class, obj)" + GenTree* clsArg = call->gtArgs.GetUserArgByIndex(0)->GetNode(); + CORINFO_CLASS_HANDLE expectedCls = gtGetHelperArgClassHandle(clsArg); + if (expectedCls == NO_CLASS_HANDLE) + { + // clsArg doesn't represent a class handle - bail out + // TODO-InlineCast: use VN if it's available (depends on when this phase is executed) + JITDUMP("clsArg is not a constant handle - bail out.\n"); + return false; + } + + const TypeCompareState castResult = info.compCompHnd->compareTypesForCast(likelyCls, expectedCls); + if (castResult == TypeCompareState::May) + { + JITDUMP("compareTypesForCast returned May for this candidate\n"); + return false; + } + + if ((info.compCompHnd->getClassAttribs(likelyCls) & (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) != 0) + { + // Possible scenario: someone changed Foo to be an interface, + // but static profile data still report it as a normal likely class. + JITDUMP("Likely class is abstract/interface - bail out (stale PGO data?).\n"); + return false; + } + + DebugInfo debugInfo = stmt->GetDebugInfo(); + + BasicBlock* firstBb; + BasicBlock* lastBb; + const unsigned tmpNum = SplitAtTreeAndReplaceItWithLocal(this, block, stmt, call, &firstBb, &lastBb); + lvaSetClass(tmpNum, expectedCls); + GenTree* tmpNode = gtNewLclvNode(tmpNum, call->TypeGet()); + *pBlock = lastBb; + + // We're going to expand this "isinst" like this: + // + // prevBb: [weight: 1.0] + // ... + // + // nullcheckBb (BBJ_COND): [weight: 1.0] + // tmp = obj; + // if (tmp == null) + // goto lastBlock; + // + // typeCheckBb (BBJ_COND): [weight: 0.5] + // if (tmp->pMT == likelyCls) + // goto typeCheckSucceedBb; + // + // fallbackBb (BBJ_ALWAYS): [weight: ] + // tmp = helper_call(expectedCls, obj); + // goto lastBlock; + // + // typeCheckSucceedBb (BBJ_ALWAYS): [weight: ] + // no-op (or tmp = null; in case of 'MustNot') + // + // lastBlock (BBJ_any): [weight: 1.0] + // use(tmp); + // + + // Block 1: nullcheckBb + // TODO-InlineCast: assertionprop should leave us a mark that objArg is never null, so we can omit this check + // it's too late to rely on upstream phases to do this for us (unless we do optRepeat). + GenTree* nullcheckOp = gtNewOperNode(GT_EQ, TYP_INT, tmpNode, gtNewNull()); + nullcheckOp->gtFlags |= GTF_RELOP_JMP_USED; + BasicBlock* nullcheckBb = fgNewBBFromTreeAfter(BBJ_COND, firstBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), + debugInfo, lastBb, true); + + // The very first statement in the whole expansion is to assign obj to tmp. + // We assume it's the value we're going to return in most cases. + GenTree* originalObj = gtCloneExpr(call->gtArgs.GetUserArgByIndex(1)->GetNode()); + Statement* assignTmp = fgNewStmtAtBeg(nullcheckBb, gtNewTempStore(tmpNum, originalObj), debugInfo); + gtSetStmtInfo(assignTmp); + fgSetStmtSeq(assignTmp); + + // Block 2: typeCheckBb + // TODO-InlineCast: if likelyCls == expectedCls we can consider saving to a local to re-use. + GenTree* likelyClsNode = gtNewIconEmbClsHndNode(likelyCls); + GenTree* mtCheck = gtNewOperNode(GT_EQ, TYP_INT, gtNewMethodTableLookup(gtCloneExpr(tmpNode)), likelyClsNode); + mtCheck->gtFlags |= GTF_RELOP_JMP_USED; + GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, mtCheck); + BasicBlock* typeCheckBb = fgNewBBFromTreeAfter(BBJ_COND, nullcheckBb, jtrue, debugInfo, lastBb, true); + + // Block 3: fallbackBb + GenTree* fallbackTree = gtNewTempStore(tmpNum, call); + BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, typeCheckBb, fallbackTree, debugInfo, lastBb, true); + + // Block 4: typeCheckSucceedBb + GenTree* typeCheckSucceedTree; + if (castResult == TypeCompareState::MustNot) + { + // With TypeCompareState::MustNot it means our likely class never passes the type check. + // it means we just check obj's type for being likelyclass and return null if it's true. + typeCheckSucceedTree = gtNewTempStore(tmpNum, gtNewNull()); + } + else + { + // tmp is already assigned to obj, so we don't need to do anything here + // some downstream phase will collect this block. It's done for simplicity. + typeCheckSucceedTree = gtNewNothingNode(); + } + BasicBlock* typeCheckSucceedBb = + fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, typeCheckSucceedTree, debugInfo, lastBb); + + // + // Wire up the blocks + // + firstBb->SetTarget(nullcheckBb); + nullcheckBb->SetTrueTarget(lastBb); + nullcheckBb->SetFalseTarget(typeCheckBb); + typeCheckBb->SetTrueTarget(typeCheckSucceedBb); + typeCheckBb->SetFalseTarget(fallbackBb); + fallbackBb->SetTarget(lastBb); + fgRemoveRefPred(lastBb, firstBb); + fgAddRefPred(nullcheckBb, firstBb); + fgAddRefPred(typeCheckBb, nullcheckBb); + fgAddRefPred(lastBb, nullcheckBb); + fgAddRefPred(fallbackBb, typeCheckBb); + fgAddRefPred(lastBb, typeCheckSucceedBb); + fgAddRefPred(typeCheckSucceedBb, typeCheckBb); + fgAddRefPred(lastBb, fallbackBb); + + // + // Re-distribute weights + // We assume obj is 50%/50% null/not-null (TODO: use profile data) + // and rely on profile for the slow path. + // + nullcheckBb->inheritWeight(firstBb); + typeCheckBb->inheritWeightPercentage(nullcheckBb, 50); + fallbackBb->inheritWeightPercentage(typeCheckBb, 100 - likelihood); + typeCheckSucceedBb->inheritWeightPercentage(typeCheckBb, likelihood); + lastBb->inheritWeight(firstBb); + + // + // Update bbNatLoopNum for all new blocks and validate EH regions + // + nullcheckBb->bbNatLoopNum = firstBb->bbNatLoopNum; + fallbackBb->bbNatLoopNum = firstBb->bbNatLoopNum; + typeCheckBb->bbNatLoopNum = firstBb->bbNatLoopNum; + typeCheckSucceedBb->bbNatLoopNum = firstBb->bbNatLoopNum; + assert(BasicBlock::sameEHRegion(firstBb, lastBb)); + assert(BasicBlock::sameEHRegion(firstBb, nullcheckBb)); + assert(BasicBlock::sameEHRegion(firstBb, fallbackBb)); + assert(BasicBlock::sameEHRegion(firstBb, typeCheckBb)); + + // Bonus step: merge prevBb with nullcheckBb as they are likely to be mergeable + if (fgCanCompactBlocks(firstBb, nullcheckBb)) + { + fgCompactBlocks(firstBb, nullcheckBb); + } + + return true; +} diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index a0981b548ecdf4..84e402a85b0ce8 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -5520,7 +5520,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree( } // Check if this cast helper have some profile data - if (impIsCastHelperMayHaveProfileData(helper)) + // "isinst" with profile data is moved to a late phase. + // The long-term plan is to move all non-trivial expansions there. + if (impIsCastHelperMayHaveProfileData(helper) && isCastClass) { const int maxLikelyClasses = 32; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; @@ -5619,6 +5621,13 @@ GenTree* Compiler::impCastClassOrIsInstToTree( compCurBB->SetFlags(BBF_HAS_HISTOGRAM_PROFILE); } } + else if (!isCastClass && impIsCastHelperMayHaveProfileData(helper)) + { + // Maybe the late-cast-expand phase will have a better luck expanding this cast. + // TODO: enable for cast-class as well. + call->gtCallMoreFlags |= GTF_CALL_M_CAST_CAN_BE_EXPANDED; + call->gtCastHelperILOffset = ilOffset; + } return call; } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index d90e9f7870ec9e..ca7c46ef6654dd 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -7697,10 +7697,18 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call) optMethodFlags |= OMF_NEEDS_GCPOLLS; } - if (fgGlobalMorph && IsStaticHelperEligibleForExpansion(call)) + if (fgGlobalMorph) { - // Current method has potential candidates for fgExpandStaticInit phase - setMethodHasStaticInit(); + if (IsStaticHelperEligibleForExpansion(call)) + { + // Current method has potential candidates for fgExpandStaticInit phase + setMethodHasStaticInit(); + } + else if ((call->gtCallMoreFlags & GTF_CALL_M_CAST_CAN_BE_EXPANDED) != 0) + { + // Current method has potential candidates for fgLateCastExpansion phase + setMethodHasExpandableCasts(); + } } // Morph Type.op_Equality, Type.op_Inequality, and Enum.HasFlag