From dcb4b164b265bc260ca233fa205982deb150855e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 6 Feb 2024 16:11:58 -0800 Subject: [PATCH 001/201] Introduce regMaskGpr, regMaskFloat, regMaskPredicate --- src/coreclr/jit/target.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 1e32a1d88946cc..129a43518a6920 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -211,7 +211,27 @@ enum _regMask_enum : unsigned #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; +#define regMaskGpr regMaskTP +#define regMaskFloat regMaskTP +#define regMaskPredicate regMaskTP + +// We should replace `regMaskTP` with `regMaskTP_Any` and that +// tells that it represents group of registers of "any one" +// of "gpr" or "vector" or "predicate" category +// Since for all the platforms that we support, there are less than +// 32 registers for these categories, it is suffice to have it +// "unsigned". +// +// We will add a "//TODO: regMaskAnyOne" through out the code +// that we know that the existing `regMaskTP` contains data that +// can be represented by `regMaskAnyOne`. In other words, at +// those places, we never pass both gpr and vector registers +// together as part of `regMaskTP`. This will be eventually +// converted to "unsigned" +#define regMaskAnyOne regMaskTP +#define regMaskAll regMaskTP #else +// x86 and arm typedef unsigned regMaskTP; #endif From 87b0f9721a5c1a6badd5f0691429f05172ec875b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 6 Feb 2024 16:12:24 -0800 Subject: [PATCH 002/201] Add IsGpr() and IsFloat() --- src/coreclr/jit/compiler.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 48bd2523cd03eb..c112c1d607d051 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8443,6 +8443,18 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // Infrastructure functions: start/stop/reserve/emit. // +#ifdef TARGET_ARM64 + static bool IsGpr(regMaskTP regMask) + { + return (regMask & RBM_ALLFLOAT) == RBM_NONE; + } + + static bool IsFloat(regMaskTP regMask) + { + return (regMask & ~RBM_ALLFLOAT) == RBM_NONE; + } +#endif + void unwindBegProlog(); void unwindEndProlog(); void unwindBegEpilog(); From ba628553b99ee4dd45056f979d3214e1d1ae1abe Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 6 Feb 2024 16:17:07 -0800 Subject: [PATCH 003/201] Renamed to regMaskOnlyOne and regMaskAny --- src/coreclr/jit/target.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 129a43518a6920..e69a6d2a5cd202 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -222,14 +222,14 @@ typedef unsigned __int64 regMaskTP; // 32 registers for these categories, it is suffice to have it // "unsigned". // -// We will add a "//TODO: regMaskAnyOne" through out the code +// We will add a "//TODO: regMaskOnlyOne" through out the code // that we know that the existing `regMaskTP` contains data that -// can be represented by `regMaskAnyOne`. In other words, at +// can be represented by `regMaskOnlyOne`. In other words, at // those places, we never pass both gpr and vector registers // together as part of `regMaskTP`. This will be eventually // converted to "unsigned" -#define regMaskAnyOne regMaskTP -#define regMaskAll regMaskTP +#define regMaskOnlyOne regMaskTP +#define regMaskAny regMaskTP #else // x86 and arm typedef unsigned regMaskTP; From 375bf8e16b12b8489bcca1de5d75aad1f4ef8bad Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 13:35:02 -0800 Subject: [PATCH 004/201] convert from #define to typedef for better intellisense support --- src/coreclr/jit/target.h | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index e69a6d2a5cd202..865829dd96c8b4 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -211,16 +211,10 @@ enum _regMask_enum : unsigned #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; -#define regMaskGpr regMaskTP -#define regMaskFloat regMaskTP -#define regMaskPredicate regMaskTP +typedef unsigned __int64 regMaskGpr; +typedef unsigned __int64 regMaskFloat; +typedef unsigned __int64 regMaskPredicate; -// We should replace `regMaskTP` with `regMaskTP_Any` and that -// tells that it represents group of registers of "any one" -// of "gpr" or "vector" or "predicate" category -// Since for all the platforms that we support, there are less than -// 32 registers for these categories, it is suffice to have it -// "unsigned". // // We will add a "//TODO: regMaskOnlyOne" through out the code // that we know that the existing `regMaskTP` contains data that @@ -228,11 +222,23 @@ typedef unsigned __int64 regMaskTP; // those places, we never pass both gpr and vector registers // together as part of `regMaskTP`. This will be eventually // converted to "unsigned" -#define regMaskOnlyOne regMaskTP -#define regMaskAny regMaskTP +typedef unsigned __int64 regMaskOnlyOne; + +// `regMaskAny` tells that the mask can contain any of the gpr/vector +// registers. Once we identify all the places with `regMaskOnlyOne`, +// `regMaskFloat`, `regMaskGpr`, we will revisit `regMaskAny` and try +// to either: +// 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. +// 2. Have a data structure like struct to pass all these together +typedef unsigned __int64 regMaskAny; #else // x86 and arm typedef unsigned regMaskTP; +#define regMaskGpr regMaskTP +#define regMaskFloat regMaskTP +#define regMaskPredicate regMaskTP +#define regMaskOnlyOne regMaskTP +#define regMaskAny regMaskTP #endif #if REGMASK_BITS == 8 From da715848f3b3288ff64cced56c26bfdc5cd5dc8d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 13:35:33 -0800 Subject: [PATCH 005/201] Make IsGprRegMask() and IsFloatRegMask() --- src/coreclr/jit/compiler.h | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index c112c1d607d051..53e211ace0b827 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8443,17 +8443,27 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // Infrastructure functions: start/stop/reserve/emit. // -#ifdef TARGET_ARM64 - static bool IsGpr(regMaskTP regMask) +//#ifdef TARGET_ARM64 + static bool IsGprRegMask(regMaskTP regMask) { - return (regMask & RBM_ALLFLOAT) == RBM_NONE; +#ifdef TARGET_AMD64 + // TODO: We need compiler object for TARGET_AMD64 + return true; +#else + return (regMask & ~RBM_ALLFLOAT) == regMask; +#endif } - static bool IsFloat(regMaskTP regMask) + static bool IsFloatRegMask(regMaskTP regMask) { - return (regMask & ~RBM_ALLFLOAT) == RBM_NONE; - } +#ifdef TARGET_AMD64 + // TODO: We need compiler object for TARGET_AMD64 + return true; +#else + return (regMask & RBM_ALLFLOAT) == regMask; #endif + } +//#endif void unwindBegProlog(); void unwindEndProlog(); From 0ba172e9cae51231758334f997f87fbc1543be60 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 13:35:59 -0800 Subject: [PATCH 006/201] Update codegen.h and related code --- src/coreclr/jit/codegen.h | 43 ++++++++++++++-------------- src/coreclr/jit/codegenarm.cpp | 46 ++++++++++++++++-------------- src/coreclr/jit/codegenarm64.cpp | 31 +++++++++++--------- src/coreclr/jit/codegenarmarch.cpp | 4 +-- src/coreclr/jit/codegencommon.cpp | 15 +++++++--- src/coreclr/jit/codegeninterface.h | 2 +- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 24 +++++++++------- src/coreclr/jit/regset.h | 2 +- 9 files changed, 91 insertions(+), 78 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 0f91f798bb9468..8b5b6f61da2063 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -329,16 +329,16 @@ class CodeGen final : public CodeGenInterface } }; - static void genBuildRegPairsStack(regMaskTP regsMask, ArrayStack* regStack); + static void genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack); static void genSetUseSaveNextPairs(ArrayStack* regStack); - static int genGetSlotSizeForRegsInMask(regMaskTP regsMask); + static int genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask); - void genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset); - void genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset); + void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); + void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); - void genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); - void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); + void genSaveCalleeSavedRegistersHelp(regMaskAny regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); + void genRestoreCalleeSavedRegistersHelp(regMaskAny regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); @@ -355,9 +355,9 @@ class CodeGen final : public CodeGenInterface void genStackProbe(ssize_t frameSize, regNumber rOffset, regNumber rLimit, regNumber rPageSize); #endif - void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn); + void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn); - void genPoisonFrame(regMaskTP bbRegLiveIn); + void genPoisonFrame(regMaskAny bbRegLiveIn); #if defined(TARGET_ARM) @@ -366,11 +366,9 @@ class CodeGen final : public CodeGenInterface bool genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg); - void genPushFltRegs(regMaskTP regMask); - void genPopFltRegs(regMaskTP regMask); - regMaskTP genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskCalleeSavedFloat); - - regMaskTP genJmpCallArgMask(); + void genPushFltRegs(regMaskFloat regMask); + void genPopFltRegs(regMaskFloat regMask); + regMaskGpr genStackAllocRegisterMask(unsigned frameSize, regMaskFloat maskCalleeSavedFloat); void genFreeLclFrame(unsigned frameSize, /* IN OUT */ bool* pUnwindStarted); @@ -387,7 +385,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskTP fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) + regMaskAny fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer unsigned fiSpDelta; // Stack pointer delta unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP @@ -403,7 +401,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) + regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) @@ -438,7 +436,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) @@ -457,7 +455,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) @@ -481,7 +479,8 @@ class CodeGen final : public CodeGenInterface #endif // TARGET_XARCH - void genZeroInitFltRegs(const regMaskTP& initFltRegs, const regMaskTP& initDblRegs, const regNumber& initReg); + void genZeroInitFltRegs(const regMaskFloat& initFltRegs, const regMaskFloat& initDblRegs, + const regNumber& initReg); regNumber genGetZeroReg(regNumber initReg, bool* pInitRegZeroed); @@ -530,7 +529,7 @@ class CodeGen final : public CodeGenInterface CLANG_FORMAT_COMMENT_ANCHOR; #if defined(TARGET_ARM) - bool genCanUsePopToReturn(regMaskTP maskPopRegsInt, bool jmpEpilog); + bool genCanUsePopToReturn(bool jmpEpilog); #endif #if defined(TARGET_ARM64) @@ -542,7 +541,7 @@ class CodeGen final : public CodeGenInterface void genPopCalleeSavedRegisters(bool jmpEpilog = false); #if defined(TARGET_XARCH) - unsigned genPopCalleeSavedRegistersFromMask(regMaskTP rsPopRegs); + unsigned genPopCalleeSavedRegistersFromMask(regMaskGpr rsPopRegs); #endif // !defined(TARGET_XARCH) #endif // !defined(TARGET_ARM64) @@ -682,8 +681,8 @@ class CodeGen final : public CodeGenInterface void genSinglePush(); void genSinglePop(); - regMaskTP genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* noRefRegs); - void genPopRegs(regMaskTP regs, regMaskTP byrefRegs, regMaskTP noRefRegs); + regMaskGpr genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskGpr* noRefRegs); + void genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 4a8c08a89858e8..b73f8bab2196b4 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1715,7 +1715,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // For generating Enter callout we would need two registers and one of them has to be r0 to pass profiler handle. // The call target register could be any free register. regNumber argReg = REG_PROFILER_ENTER_ARG; - regMaskTP argRegMask = genRegMask(argReg); + regMaskGpr argRegMask = genRegMask(argReg); assert((regSet.rsMaskPreSpillRegArg & argRegMask) != 0); if (compiler->compProfilerMethHndIndirected) @@ -1891,7 +1891,7 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) { assert(compiler->compGeneratingProlog); @@ -1943,10 +1943,10 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni compiler->unwindAllocStack(frameSize); } -void CodeGen::genPushFltRegs(regMaskTP regMask) +void CodeGen::genPushFltRegs(regMaskFloat regMask) { - assert(regMask != 0); // Don't call uness we have some registers to push - assert((regMask & RBM_ALLFLOAT) == regMask); // Only floasting point registers should be in regMask + assert(regMask != 0); // Don't call unless we have some registers to push + assert(Compiler::IsFloatRegMask(regMask)); // Only floating point registers should be in regMask regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); @@ -1962,10 +1962,10 @@ void CodeGen::genPushFltRegs(regMaskTP regMask) GetEmitter()->emitIns_R_I(INS_vpush, EA_8BYTE, lowReg, slots / 2); } -void CodeGen::genPopFltRegs(regMaskTP regMask) +void CodeGen::genPopFltRegs(regMaskFloat regMask) { assert(regMask != 0); // Don't call uness we have some registers to pop - assert((regMask & RBM_ALLFLOAT) == regMask); // Only floasting point registers should be in regMask + assert(Compiler::IsFloatRegMask(regMask)); // Only floasting point registers should be in regMask regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); @@ -2100,7 +2100,7 @@ void CodeGen::genMov32RelocatableImmediate(emitAttr size, BYTE* addr, regNumber * instead of using "sub sp" / "add sp". Returns RBM_NONE if either frame size * is zero, or if we should use "sub sp" / "add sp" instead of push/pop. */ -regMaskTP CodeGen::genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskCalleeSavedFloat) +regMaskGpr CodeGen::genStackAllocRegisterMask(unsigned frameSize, regMaskFloat maskCalleeSavedFloat) { assert(compiler->compGeneratingProlog || compiler->compGeneratingEpilog); @@ -2109,6 +2109,8 @@ regMaskTP CodeGen::genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskC if (maskCalleeSavedFloat != RBM_NONE) return RBM_NONE; + assert(Compiler::IsFloatRegMask(maskCalleeSavedFloat)); + // Allocate space for small frames by pushing extra registers. It generates smaller and faster code // that extra sub sp,XXX/add sp,XXX. // R0 and R1 may be used by return value. Keep things simple and just skip the optimization @@ -2157,7 +2159,7 @@ void CodeGen::instGen_MemoryBarrier(BarrierKind barrierKind) } } -bool CodeGen::genCanUsePopToReturn(regMaskTP maskPopRegsInt, bool jmpEpilog) +bool CodeGen::genCanUsePopToReturn(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); @@ -2171,9 +2173,9 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskTP maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; - regMaskTP maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT; - regMaskTP maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; + regMaskAny maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskFloat maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT; + regMaskGpr maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; // First, pop float registers @@ -2187,7 +2189,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) if (!jmpEpilog) { - regMaskTP maskStackAlloc = genStackAllocRegisterMask(compiler->compLclFrameSize, maskPopRegsFloat); + regMaskGpr maskStackAlloc = genStackAllocRegisterMask(compiler->compLclFrameSize, maskPopRegsFloat); maskPopRegsInt |= maskStackAlloc; } @@ -2197,7 +2199,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) maskPopRegsInt |= RBM_FPBASE; } - if (genCanUsePopToReturn(maskPopRegsInt, jmpEpilog)) + if (genCanUsePopToReturn(jmpEpilog)) { maskPopRegsInt |= RBM_PC; // Record the fact that we use a pop to the PC to perform the return @@ -2332,10 +2334,10 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); - regMaskTP maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskTP maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat; + regMaskFloat maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; + regMaskGpr maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat; - regMaskTP maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPushRegsFloat); + regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPushRegsFloat); maskPushRegsInt |= maskStackAlloc; assert(FitsIn(maskPushRegsInt)); @@ -2350,7 +2352,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) bool isFilter = (block->bbCatchTyp == BBCT_FILTER); - regMaskTP maskArgRegsLiveIn; + regMaskGpr maskArgRegsLiveIn; if (isFilter) { maskArgRegsLiveIn = RBM_R0 | RBM_R1; @@ -2428,10 +2430,10 @@ void CodeGen::genFuncletEpilog() /* The saved regs info saves the LR register. We need to pop the PC register to return */ assert(genFuncletInfo.fiSaveRegs & RBM_LR); - regMaskTP maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskTP maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat; + regMaskFloat maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; + regMaskGpr maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat; - regMaskTP maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPopRegsFloat); + regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPopRegsFloat); maskPopRegsInt |= maskStackAlloc; if (maskStackAlloc == RBM_NONE) @@ -2488,7 +2490,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() unsigned preSpillRegArgSize = genCountBits(regSet.rsMaskPreSpillRegs(true)) * REGSIZE_BYTES; genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; - regMaskTP rsMaskSaveRegs = regSet.rsMaskCalleeSaved; + regMaskAny rsMaskSaveRegs = regSet.rsMaskCalleeSaved; unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 4efe4a235f6a7d..410099206b74a0 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -718,7 +718,7 @@ void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, reg // no return value; the regStack argument is modified. // // static -void CodeGen::genBuildRegPairsStack(regMaskTP regsMask, ArrayStack* regStack) +void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack) { assert(regStack != nullptr); assert(regStack->Height() == 0); @@ -820,7 +820,7 @@ void CodeGen::genSetUseSaveNextPairs(ArrayStack* regStack) // Note: Because int and float register type sizes match we can call this function with a mask that includes both. // // static -int CodeGen::genGetSlotSizeForRegsInMask(regMaskTP regsMask) +int CodeGen::genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask) { assert((regsMask & (RBM_CALLEE_SAVED | RBM_FP | RBM_LR)) == regsMask); // Do not expect anything else. @@ -836,8 +836,10 @@ int CodeGen::genGetSlotSizeForRegsInMask(regMaskTP regsMask) // spDelta - if non-zero, the amount to add to SP before the first register save (or together with it); // spOffset - the offset from SP that is the beginning of the callee-saved register area; // -void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset) +void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset) { + assert(Compiler::IsGprRegMask(regsMask) != Compiler::IsFloatRegMask(regsMask)); + const int slotSize = genGetSlotSizeForRegsInMask(regsMask); ArrayStack regStack(compiler->getAllocator(CMK_Codegen)); @@ -902,7 +904,7 @@ void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, i // The save set can contain LR in which case LR is saved along with the other callee-saved registers. // But currently Jit doesn't use frames without frame pointer on arm64. // -void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskAny regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); @@ -926,8 +928,8 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // Save integer registers at higher addresses than floating-point registers. - regMaskTP maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; - regMaskTP maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; + regMaskFloat maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; + regMaskGpr maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; if (maskSaveRegsFloat != RBM_NONE) { @@ -951,7 +953,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // spDelta - if non-zero, the amount to add to SP after the last register restore (or together with it); // spOffset - the offset from SP that is the beginning of the callee-saved register area; // -void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset) +void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset) { const int slotSize = genGetSlotSizeForRegsInMask(regsMask); @@ -1017,7 +1019,7 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskAny regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta >= 0); unsigned regsToRestoreCount = genCountBits(regsToRestoreMask); @@ -1043,8 +1045,8 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in // Save integer registers at higher addresses than floating-point registers. - regMaskTP maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; - regMaskTP maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; + regMaskFloat maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; + regMaskGpr maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; // Restore in the opposite order of saving. @@ -1368,8 +1370,8 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); - regMaskTP maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskTP maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat; + regMaskFloat maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; + regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat; // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskSaveRegsInt & RBM_LR) != 0); @@ -1377,7 +1379,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) bool isFilter = (block->bbCatchTyp == BBCT_FILTER); - regMaskTP maskArgRegsLiveIn; + regMaskGpr maskArgRegsLiveIn; if (isFilter) { maskArgRegsLiveIn = RBM_R0 | RBM_R1; @@ -5541,9 +5543,10 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) { assert(compiler->compGeneratingProlog); + assert(Compiler::IsGprRegMask(maskArgRegsLiveIn)); if (frameSize == 0) { diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index cb04cf702f2b94..053a0fa4ce01f0 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4946,8 +4946,8 @@ void CodeGen::genPushCalleeSavedRegisters() #endif // DEBUG #if defined(TARGET_ARM) - regMaskTP maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskTP maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat; + regMaskFloat maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT; + regMaskGpr maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat; maskPushRegsInt |= genStackAllocRegisterMask(compiler->compLclFrameSize, maskPushRegsFloat); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index c468473067b6e1..0cb3558b137661 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -4559,9 +4559,13 @@ void CodeGen::genCheckUseBlockInit() * initialized to 0. (Arm Only) Else copies from the integer register which * is slower. */ -void CodeGen::genZeroInitFltRegs(const regMaskTP& initFltRegs, const regMaskTP& initDblRegs, const regNumber& initReg) +void CodeGen::genZeroInitFltRegs(const regMaskFloat& initFltRegs, + const regMaskFloat& initDblRegs, + const regNumber& initReg) { assert(compiler->compGeneratingProlog); + assert(Compiler::IsFloatRegMask(initFltRegs)); + assert(Compiler::IsFloatRegMask(initDblRegs)); // The first float/double reg that is initialized to 0. So they can be used to // initialize the remaining registers. @@ -6790,7 +6794,7 @@ void CodeGen::genSinglePop() // Notes: // This function does not check if the register is marked as used, etc. // -regMaskTP CodeGen::genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* noRefRegs) +regMaskGpr CodeGen::genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskGpr* noRefRegs) { *byrefRegs = RBM_NONE; *noRefRegs = RBM_NONE; @@ -6800,6 +6804,8 @@ regMaskTP CodeGen::genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* return RBM_NONE; } + assert(Compiler::IsGprRegMask(regs)); + #if FEATURE_FIXED_OUT_ARGS NYI("Don't call genPushRegs with real regs!"); @@ -6863,13 +6869,14 @@ regMaskTP CodeGen::genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* // Return Value: // None // -void CodeGen::genPopRegs(regMaskTP regs, regMaskTP byrefRegs, regMaskTP noRefRegs) +void CodeGen::genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs) { if (regs == RBM_NONE) { return; } + assert(Compiler::IsGprRegMask(regs)); #if FEATURE_FIXED_OUT_ARGS NYI("Don't call genPopRegs with real regs!"); @@ -8300,7 +8307,7 @@ unsigned CodeGenInterface::getCurrentStackLevel() const // This function emits code to poison address exposed non-zero-inited local variables. We expect this function // to be called when emitting code for the scratch BB that comes right after the prolog. // The variables are poisoned using 0xcdcdcdcd. -void CodeGen::genPoisonFrame(regMaskTP regLiveIn) +void CodeGen::genPoisonFrame(regMaskAny regLiveIn) { assert(compiler->compShouldPoisonFrame()); #if defined(TARGET_XARCH) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index a28a60b50ca8cd..c21ac9ae9a5cfa 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -36,7 +36,7 @@ class emitter; struct RegState { - regMaskTP rsCalleeRegArgMaskLiveIn; // mask of register arguments (live on entry to method) + regMaskOnlyOne rsCalleeRegArgMaskLiveIn; // mask of register arguments (live on entry to method) unsigned rsCalleeRegArgCount; // total number of incoming register arguments of this kind (int or float) bool rsIsFloat; // true for float argument registers, false for integer argument registers }; diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 913f3a47002a51..0ff3c03c8821bb 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -201,7 +201,7 @@ void CodeGen::genCodeForBBlist() // change? We cleared them out above. Maybe we should just not clear them out, but update the ones that change // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). - regMaskTP newLiveRegSet = RBM_NONE; + regMaskAny newLiveRegSet = RBM_NONE; regMaskTP newRegGCrefSet = RBM_NONE; regMaskTP newRegByrefSet = RBM_NONE; #ifdef DEBUG diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 03a255d98078a9..6cd464686707ba 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -127,7 +127,7 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg) } regNumber regGSCheck; - regMaskTP regMaskGSCheck = RBM_NONE; + regMaskGpr regMaskGSCheck = RBM_NONE; if (!pushReg) { @@ -162,9 +162,9 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg) #endif // !TARGET_X86 } - regMaskTP byrefPushedRegs = RBM_NONE; - regMaskTP norefPushedRegs = RBM_NONE; - regMaskTP pushedRegs = RBM_NONE; + regMaskGpr byrefPushedRegs = RBM_NONE; + regMaskGpr norefPushedRegs = RBM_NONE; + regMaskGpr pushedRegs = RBM_NONE; if (compiler->gsGlobalSecurityCookieAddr == nullptr) { @@ -9994,10 +9994,10 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // if (doesSupersetOfNormalPops) { - regMaskTP rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; - regMaskTP tier0CalleeSaves = - ((regMaskTP)compiler->info.compPatchpointInfo->CalleeSaveRegisters()) & RBM_OSR_INT_CALLEE_SAVED; - regMaskTP additionalCalleeSaves = rsPopRegs & ~tier0CalleeSaves; + regMaskGpr rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr tier0CalleeSaves = + ((regMaskGpr)compiler->info.compPatchpointInfo->CalleeSaveRegisters()) & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr additionalCalleeSaves = rsPopRegs & ~tier0CalleeSaves; // Registers saved by the OSR prolog. // @@ -10014,7 +10014,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // Registers saved by a normal prolog // - regMaskTP rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; const unsigned popCount = genPopCalleeSavedRegistersFromMask(rsPopRegs); noway_assert(compiler->compCalleeRegsPushed == popCount); } @@ -10023,8 +10023,10 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // genPopCalleeSavedRegistersFromMask: pop specified set of callee saves // in the "standard" order // -unsigned CodeGen::genPopCalleeSavedRegistersFromMask(regMaskTP rsPopRegs) +unsigned CodeGen::genPopCalleeSavedRegistersFromMask(regMaskGpr rsPopRegs) { + assert(Compiler::IsGprRegMask(rsPopRegs)); + unsigned popCount = 0; if ((rsPopRegs & RBM_EBX) != 0) { @@ -10642,7 +10644,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) // Callee saved int registers are pushed to stack. genPushCalleeSavedRegisters(); - regMaskTP maskArgRegsLiveIn; + regMaskGpr maskArgRegsLiveIn; if ((block->bbCatchTyp == BBCT_FINALLY) || (block->bbCatchTyp == BBCT_FAULT)) { maskArgRegsLiveIn = RBM_ARG_0; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 73eb08aa943eb9..5e931f82bb47af 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -124,7 +124,7 @@ class RegSet regMaskTP _rsMaskVars; // backing store for rsMaskVars property #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - regMaskTP rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog + regMaskAny rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 public: // TODO-Cleanup: Should be private, but Compiler uses it From 8639bb7a5384081cd1c47d8a8b6539c2a354b1d3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 13:45:26 -0800 Subject: [PATCH 007/201] Update codegenarm.cpp and related code --- src/coreclr/jit/codegenarm.cpp | 8 ++++---- src/coreclr/jit/regset.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index b73f8bab2196b4..e16e44ef9a44a1 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1951,7 +1951,7 @@ void CodeGen::genPushFltRegs(regMaskFloat regMask) regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); // regMask should be contiguously set - regMaskTP tmpMask = ((regMask >> lowReg) + 1); // tmpMask should have a single bit set + regMaskFloat tmpMask = ((regMask >> lowReg) + 1); // tmpMask should have a single bit set assert((tmpMask & (tmpMask - 1)) == 0); assert(lowReg == REG_F16); // Currently we expect to start at F16 in the unwind codes @@ -1970,7 +1970,7 @@ void CodeGen::genPopFltRegs(regMaskFloat regMask) regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); // regMask should be contiguously set - regMaskTP tmpMask = ((regMask >> lowReg) + 1); // tmpMask should have a single bit set + regMaskFloat tmpMask = ((regMask >> lowReg) + 1); // tmpMask should have a single bit set assert((tmpMask & (tmpMask - 1)) == 0); // Our calling convention requires that we only use vpop for TYP_DOUBLE registers @@ -2646,9 +2646,9 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu regNumber rAddr; regNumber rCnt = REG_NA; // Invalid - regMaskTP regMask; + regMaskAny regMask; - regMaskTP availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers + regMaskAny availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers availMask &= ~intRegState.rsCalleeRegArgMaskLiveIn; // Remove all of the incoming argument registers as they are // currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg as we will zero it and maybe use it for diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 5e931f82bb47af..02c5460197bd49 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -68,14 +68,14 @@ class RegSet private: bool rsNeededSpillReg; // true if this method needed to spill any registers - regMaskTP rsModifiedRegsMask; // mask of the registers modified by the current function. + regMaskAny rsModifiedRegsMask; // mask of the registers modified by the current function. #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. #endif // DEBUG public: - regMaskTP rsGetModifiedRegsMask() const + regMaskAny rsGetModifiedRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; From 3c5bdaf18bb553818c4413266ae2e2865fdc3131 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 14:01:51 -0800 Subject: [PATCH 008/201] Update codegenarm64.cpp and related code --- src/coreclr/jit/codegenarm64.cpp | 26 +++++++++++++------------- src/coreclr/jit/codegencommon.cpp | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 410099206b74a0..ea0e102dcbd2a2 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,7 +36,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskTP rsRestoreRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskAny rsRestoreRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; if (isFramePointerUsed()) { @@ -45,7 +45,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) rsRestoreRegs |= RBM_LR; // We must save/restore the return address (in the LR register) - regMaskTP regsToRestoreMask = rsRestoreRegs; + regMaskAny regsToRestoreMask = rsRestoreRegs; const int totalFrameSize = genTotalFrameSize(); @@ -1574,8 +1574,8 @@ void CodeGen::genFuncletEpilog() unwindStarted = true; } - regMaskTP maskRestoreRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskTP maskRestoreRegsInt = genFuncletInfo.fiSaveRegs & ~maskRestoreRegsFloat; + regMaskFloat maskRestoreRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; + regMaskGpr maskRestoreRegsInt = genFuncletInfo.fiSaveRegs & ~maskRestoreRegsFloat; // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskRestoreRegsInt & RBM_LR) != 0); @@ -1598,7 +1598,7 @@ void CodeGen::genFuncletEpilog() } } - regMaskTP regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; + regMaskAny regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; if ((genFuncletInfo.fiFrameType == 1) || (genFuncletInfo.fiFrameType == 2) || (genFuncletInfo.fiFrameType == 3)) { regsToRestoreMask &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end @@ -1735,7 +1735,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiFunction_CallerSP_to_FP_delta = genCallerSPtoFPdelta() - osrPad; - regMaskTP rsMaskSaveRegs = regSet.rsMaskCalleeSaved; + regMaskAny rsMaskSaveRegs = regSet.rsMaskCalleeSaved; assert((rsMaskSaveRegs & RBM_LR) != 0); assert((rsMaskSaveRegs & RBM_FP) != 0); @@ -4334,9 +4334,9 @@ void CodeGen::genCodeForSwap(GenTreeOp* tree) assert(!varTypeIsFloating(type1)); regNumber oldOp1Reg = lcl1->GetRegNum(); - regMaskTP oldOp1RegMask = genRegMask(oldOp1Reg); + regMaskAny oldOp1RegMask = genRegMask(oldOp1Reg); regNumber oldOp2Reg = lcl2->GetRegNum(); - regMaskTP oldOp2RegMask = genRegMask(oldOp2Reg); + regMaskAny oldOp2RegMask = genRegMask(oldOp2Reg); // We don't call genUpdateVarReg because we don't have a tree node with the new register. varDsc1->SetRegNum(oldOp2Reg); @@ -5097,8 +5097,8 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; } - regMaskTP callTargetMask = genRegMask(callTargetReg); - regMaskTP callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskGpr callTargetMask = genRegMask(callTargetReg); + regMaskAny callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet noway_assert((callTargetMask & callKillSet) == callTargetMask); @@ -5120,7 +5120,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, false /* isJump */ ); - regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskAny killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); } @@ -5602,13 +5602,13 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni // until this is complete since the tickles could cause a stack overflow, and we need to be able to crawl // the stack afterward (which means the stack pointer needs to be known). - regMaskTP availMask = RBM_ALLINT & (regSet.rsGetModifiedRegsMask() | ~RBM_INT_CALLEE_SAVED); + regMaskGpr availMask = RBM_ALLINT & (regSet.rsGetModifiedRegsMask() | ~RBM_INT_CALLEE_SAVED); availMask &= ~maskArgRegsLiveIn; // Remove all of the incoming argument registers as they are currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg regNumber rOffset = initReg; regNumber rLimit; - regMaskTP tempMask; + regMaskGpr tempMask; // We pick the next lowest register number for rLimit noway_assert(availMask != RBM_NONE); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 0cb3558b137661..993c86f97e983d 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -625,7 +625,7 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // Return Value: // Mask of register kills -- registers whose values are no longer guaranteed to be the same. // -regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) +regMaskAny Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) { switch (helper) { From bad9e8b309a95da7187701f773da88f9f110c53d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Feb 2024 14:14:17 -0800 Subject: [PATCH 009/201] Update codegenarmarch.cpp and related code --- src/coreclr/jit/codegenarmarch.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 053a0fa4ce01f0..a18a6572957396 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3433,7 +3433,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskTP killMask = RBM_CALLEE_TRASH; + regMaskAny killMask = RBM_CALLEE_TRASH; if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); @@ -3576,7 +3576,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (call->IsFastTailCall()) { - regMaskTP trashedByEpilog = RBM_CALLEE_SAVED; + regMaskAny trashedByEpilog = RBM_CALLEE_SAVED; // The epilog may use and trash REG_GSCOOKIE_TMP_0/1. Make sure we have no // non-standard args that may be trash if this is a tailcall. @@ -3834,7 +3834,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - regMaskTP tempMask = genRegMask(varDsc->GetRegNum()); + regMaskAny tempMask = genRegMask(varDsc->GetRegNum()); regSet.RemoveMaskVars(tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) @@ -3850,7 +3850,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) #endif // Next move any un-enregistered register arguments back to their register. - regMaskTP fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. + regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { @@ -4082,14 +4082,14 @@ void CodeGen::genJmpMethod(GenTree* jmp) assert(compiler->info.compIsVarArgs); assert(firstArgVarNum != BAD_VAR_NUM); - regMaskTP remainingIntArgMask = RBM_ARG_REGS & ~fixedIntArgMask; + regMaskGpr remainingIntArgMask = RBM_ARG_REGS & ~fixedIntArgMask; if (remainingIntArgMask != RBM_NONE) { GetEmitter()->emitDisableGC(); for (int argNum = 0, argOffset = 0; argNum < MAX_REG_ARG; ++argNum) { regNumber argReg = intArgRegs[argNum]; - regMaskTP argRegMask = genRegMask(argReg); + regMaskGpr argRegMask = genRegMask(argReg); if ((remainingIntArgMask & argRegMask) != 0) { @@ -4898,7 +4898,7 @@ void CodeGen::genPushCalleeSavedRegisters() intRegState.rsCalleeRegArgMaskLiveIn); #endif - regMaskTP rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskAny rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -5053,8 +5053,8 @@ void CodeGen::genPushCalleeSavedRegisters() int offset; // This will be the starting place for saving the callee-saved registers, in increasing order. - regMaskTP maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskTP maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat; + regMaskFloat maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT; + regMaskGpr maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat; #ifdef DEBUG if (verbose) From 8384ff2978583f060da5c0f34183edc774dfab3a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 13:52:50 -0800 Subject: [PATCH 010/201] Partial update codegencommon.cpp --- src/coreclr/jit/codegencommon.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 993c86f97e983d..db4c2aaba1c2c9 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -128,7 +128,7 @@ CodeGen::CodeGen(Compiler* theCompiler) : CodeGenInterface(theCompiler) #if defined(TARGET_XARCH) // Shouldn't be used before it is set in genFnProlog() - compiler->compCalleeFPRegsSavedMask = (regMaskTP)-1; + compiler->compCalleeFPRegsSavedMask = (regMaskAny)-1; #endif // defined(TARGET_XARCH) #endif // DEBUG @@ -511,9 +511,9 @@ void CodeGenInterface::genUpdateLife(VARSET_VALARG_TP newLife) // Return the register mask for the given register variable // inline -regMaskTP CodeGenInterface::genGetRegMask(const LclVarDsc* varDsc) +regMaskOnlyOne CodeGenInterface::genGetRegMask(const LclVarDsc* varDsc) { - regMaskTP regMask = RBM_NONE; + regMaskOnlyOne regMask = RBM_NONE; assert(varDsc->lvIsInReg()); @@ -531,11 +531,11 @@ regMaskTP CodeGenInterface::genGetRegMask(const LclVarDsc* varDsc) // Return the register mask for the given lclVar or regVar tree node // inline -regMaskTP CodeGenInterface::genGetRegMask(GenTree* tree) +regMaskOnlyOne CodeGenInterface::genGetRegMask(GenTree* tree) { assert(tree->gtOper == GT_LCL_VAR); - regMaskTP regMask = RBM_NONE; + regMaskOnlyOne regMask = RBM_NONE; const LclVarDsc* varDsc = compiler->lvaGetDesc(tree->AsLclVarCommon()); if (varDsc->lvPromoted) { @@ -562,7 +562,7 @@ regMaskTP CodeGenInterface::genGetRegMask(GenTree* tree) // inline void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bool isDying DEBUGARG(GenTree* tree)) { - regMaskTP regMask = genGetRegMask(varDsc); + regMaskOnlyOne regMask = genGetRegMask(varDsc); #ifdef DEBUG if (compiler->verbose) From c78dc4bd9e5d96b7454afdb30807db2ffc3927b0 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 16:46:46 -0800 Subject: [PATCH 011/201] Update all gcRef and gcByRef to regMaskGpr --- src/coreclr/jit/codegenarm.cpp | 6 +-- src/coreclr/jit/codegenarm64.cpp | 16 ++++--- src/coreclr/jit/codegencommon.cpp | 14 ++++--- src/coreclr/jit/codegenlinear.cpp | 8 ++-- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/compiler.h | 21 ++++------ src/coreclr/jit/emit.cpp | 70 ++++++++++++++++++------------- src/coreclr/jit/emit.h | 48 ++++++++++----------- src/coreclr/jit/emitarm.cpp | 13 +++--- src/coreclr/jit/emitarm.h | 8 ++-- src/coreclr/jit/emitarm64.cpp | 13 +++--- src/coreclr/jit/emitarm64.h | 12 +++--- src/coreclr/jit/emitinl.h | 2 +- src/coreclr/jit/emitpub.h | 4 +- src/coreclr/jit/emitxarch.cpp | 14 ++++--- src/coreclr/jit/emitxarch.h | 12 +++--- src/coreclr/jit/gcinfo.cpp | 30 +++++++------ src/coreclr/jit/jitgcinfo.h | 16 +++---- src/coreclr/jit/target.h | 3 +- 19 files changed, 166 insertions(+), 146 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index e16e44ef9a44a1..27158487ba7ab7 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1946,7 +1946,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni void CodeGen::genPushFltRegs(regMaskFloat regMask) { assert(regMask != 0); // Don't call unless we have some registers to push - assert(Compiler::IsFloatRegMask(regMask)); // Only floating point registers should be in regMask + assert(compiler->IsFloatRegMask(regMask)); // Only floating point registers should be in regMask regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); @@ -1965,7 +1965,7 @@ void CodeGen::genPushFltRegs(regMaskFloat regMask) void CodeGen::genPopFltRegs(regMaskFloat regMask) { assert(regMask != 0); // Don't call uness we have some registers to pop - assert(Compiler::IsFloatRegMask(regMask)); // Only floasting point registers should be in regMask + assert(compiler->IsFloatRegMask(regMask)); // Only floasting point registers should be in regMask regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); int slots = genCountBits(regMask); @@ -2109,7 +2109,7 @@ regMaskGpr CodeGen::genStackAllocRegisterMask(unsigned frameSize, regMaskFloat m if (maskCalleeSavedFloat != RBM_NONE) return RBM_NONE; - assert(Compiler::IsFloatRegMask(maskCalleeSavedFloat)); + assert(compiler->IsFloatRegMask(maskCalleeSavedFloat)); // Allocate space for small frames by pushing extra registers. It generates smaller and faster code // that extra sub sp,XXX/add sp,XXX. diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index ea0e102dcbd2a2..7d08d089fde1e3 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -838,7 +838,7 @@ int CodeGen::genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask) // void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset) { - assert(Compiler::IsGprRegMask(regsMask) != Compiler::IsFloatRegMask(regsMask)); + assert(compiler->IsOnlyOneRegMask(regsMask)); const int slotSize = genGetSlotSizeForRegsInMask(regsMask); @@ -4333,10 +4333,14 @@ void CodeGen::genCodeForSwap(GenTreeOp* tree) // FP swap is not yet implemented (and should have NYI'd in LSRA) assert(!varTypeIsFloating(type1)); - regNumber oldOp1Reg = lcl1->GetRegNum(); - regMaskAny oldOp1RegMask = genRegMask(oldOp1Reg); - regNumber oldOp2Reg = lcl2->GetRegNum(); - regMaskAny oldOp2RegMask = genRegMask(oldOp2Reg); + regNumber oldOp1Reg = lcl1->GetRegNum(); + regNumber oldOp2Reg = lcl2->GetRegNum(); + + regMaskGpr oldOp1RegMask = genRegMask(oldOp1Reg); + regMaskGpr oldOp2RegMask = genRegMask(oldOp2Reg); + + assert(compiler->IsGprRegMask(oldOp1RegMask)); + assert(compiler->IsGprRegMask(oldOp2RegMask)); // We don't call genUpdateVarReg because we don't have a tree node with the new register. varDsc1->SetRegNum(oldOp2Reg); @@ -5546,7 +5550,7 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) { assert(compiler->compGeneratingProlog); - assert(Compiler::IsGprRegMask(maskArgRegsLiveIn)); + assert(compiler->IsGprRegMask(maskArgRegsLiveIn)); if (frameSize == 0) { diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index db4c2aaba1c2c9..2b838c5e743a11 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -4564,8 +4564,8 @@ void CodeGen::genZeroInitFltRegs(const regMaskFloat& initFltRegs, const regNumber& initReg) { assert(compiler->compGeneratingProlog); - assert(Compiler::IsFloatRegMask(initFltRegs)); - assert(Compiler::IsFloatRegMask(initDblRegs)); + assert(compiler->IsFloatRegMask(initFltRegs)); + assert(compiler->IsFloatRegMask(initDblRegs)); // The first float/double reg that is initialized to 0. So they can be used to // initialize the remaining registers. @@ -5206,8 +5206,10 @@ void CodeGen::genReserveProlog(BasicBlock* block) void CodeGen::genReserveEpilog(BasicBlock* block) { - regMaskTP gcrefRegsArg = gcInfo.gcRegGCrefSetCur; - regMaskTP byrefRegsArg = gcInfo.gcRegByrefSetCur; + regMaskGpr gcrefRegsArg = gcInfo.gcRegGCrefSetCur; + regMaskGpr byrefRegsArg = gcInfo.gcRegByrefSetCur; + assert(compiler->IsGprRegMask(gcrefRegsArg)); + assert(compiler->IsGprRegMask(byrefRegsArg)); /* The return value is special-cased: make sure it goes live for the epilog */ @@ -6804,7 +6806,7 @@ regMaskGpr CodeGen::genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskG return RBM_NONE; } - assert(Compiler::IsGprRegMask(regs)); + assert(compiler->IsGprRegMask(regs)); #if FEATURE_FIXED_OUT_ARGS @@ -6876,7 +6878,7 @@ void CodeGen::genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRef return; } - assert(Compiler::IsGprRegMask(regs)); + assert(compiler->IsGprRegMask(regs)); #if FEATURE_FIXED_OUT_ARGS NYI("Don't call genPopRegs with real regs!"); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 0ff3c03c8821bb..11e9eccff8e700 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -202,8 +202,8 @@ void CodeGen::genCodeForBBlist() // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). regMaskAny newLiveRegSet = RBM_NONE; - regMaskTP newRegGCrefSet = RBM_NONE; - regMaskTP newRegByrefSet = RBM_NONE; + regMaskGpr newRegGCrefSet = RBM_NONE; + regMaskGpr newRegByrefSet = RBM_NONE; #ifdef DEBUG VARSET_TP removedGCVars(VarSetOps::MakeEmpty(compiler)); VARSET_TP addedGCVars(VarSetOps::MakeEmpty(compiler)); @@ -480,8 +480,8 @@ void CodeGen::genCodeForBBlist() /* Make sure we didn't bungle pointer register tracking */ - regMaskTP ptrRegs = gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur; - regMaskTP nonVarPtrRegs = ptrRegs & ~regSet.GetMaskVars(); + regMaskGpr ptrRegs = gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur; + regMaskGpr nonVarPtrRegs = ptrRegs & ~regSet.GetMaskVars(); // If return is a GC-type, clear it. Note that if a common // epilog is generated (genReturnBB) it has a void return diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 6cd464686707ba..def2c6bacb177b 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -10025,7 +10025,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // unsigned CodeGen::genPopCalleeSavedRegistersFromMask(regMaskGpr rsPopRegs) { - assert(Compiler::IsGprRegMask(rsPopRegs)); + assert(compiler->IsGprRegMask(rsPopRegs)); unsigned popCount = 0; if ((rsPopRegs & RBM_EBX) != 0) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 53e211ace0b827..14fb7619c7bd88 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8443,27 +8443,20 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // Infrastructure functions: start/stop/reserve/emit. // -//#ifdef TARGET_ARM64 - static bool IsGprRegMask(regMaskTP regMask) + bool IsGprRegMask(regMaskTP regMask) { -#ifdef TARGET_AMD64 - // TODO: We need compiler object for TARGET_AMD64 - return true; -#else return (regMask & ~RBM_ALLFLOAT) == regMask; -#endif } - static bool IsFloatRegMask(regMaskTP regMask) + bool IsFloatRegMask(regMaskTP regMask) { -#ifdef TARGET_AMD64 - // TODO: We need compiler object for TARGET_AMD64 - return true; -#else return (regMask & RBM_ALLFLOAT) == regMask; -#endif } -//#endif + + bool IsOnlyOneRegMask(regMaskTP regMask) + { + return IsGprRegMask(regMask) != IsFloatRegMask(regMask); + } void unwindBegProlog(); void unwindEndProlog(); diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index d3ac84e7919a1d..fdc29c9a4a3af5 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -2062,8 +2062,8 @@ void emitter::emitEndProlog() void emitter::emitCreatePlaceholderIG(insGroupPlaceholderType igType, BasicBlock* igBB, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, bool last) { assert(igBB != nullptr); @@ -2891,8 +2891,8 @@ bool emitter::emitNoGChelper(CORINFO_METHOD_HANDLE methHnd) */ void* emitter::emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs DEBUG_ARG(BasicBlock* block)) + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block)) { /* Create a new IG if the current one is non-empty */ @@ -3603,11 +3603,13 @@ void emitter::emitSetSecondRetRegGCType(instrDescCGCA* id, emitAttr secondRetSiz emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSizeIn MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)) { + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); emitAttr retSize = (retSizeIn != EA_UNKNOWN) ? retSizeIn : EA_PTRSIZE; bool gcRefRegsInScratch = ((gcrefRegs & RBM_CALLEE_TRASH) != 0); @@ -3686,11 +3688,14 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt, emitter::instrDesc* emitter::emitNewInstrCallDir(int argCnt, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSizeIn MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)) { + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); + emitAttr retSize = (retSizeIn != EA_UNKNOWN) ? retSizeIn : EA_PTRSIZE; // Allocate a larger descriptor if new GC values need to be saved @@ -3848,14 +3853,17 @@ void emitter::emitDispGCDeltaTitle(const char* title) // prevRegs - The live GC registers before the recent instruction. // curRegs - The live GC registers after the recent instruction. // -void emitter::emitDispGCRegDelta(const char* title, regMaskTP prevRegs, regMaskTP curRegs) +void emitter::emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMaskGpr curRegs) { + assert(emitComp->IsGprRegMask(prevRegs)); + assert(emitComp->IsGprRegMask(curRegs)); + if (prevRegs != curRegs) { emitDispGCDeltaTitle(title); - regMaskTP sameRegs = prevRegs & curRegs; - regMaskTP removedRegs = prevRegs - sameRegs; - regMaskTP addedRegs = curRegs - sameRegs; + regMaskGpr sameRegs = prevRegs & curRegs; + regMaskGpr removedRegs = prevRegs - sameRegs; + regMaskGpr addedRegs = curRegs - sameRegs; if (removedRegs != RBM_NONE) { printf(" -"); @@ -7296,7 +7304,7 @@ unsigned emitter::emitEndCodeGen(Compiler* comp, /* Update the set of live GC ref registers */ { - regMaskTP GCregs = ig->igGCregs; + regMaskGpr GCregs = ig->igGCregs; if (GCregs != emitThisGCrefRegs) { @@ -8906,8 +8914,9 @@ void emitter::emitRecordGCcall(BYTE* codePos, unsigned char callInstrSize) * Record a new set of live GC ref registers. */ -void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr) +void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskGpr regs, BYTE* addr) { + assert(emitComp->IsGprRegMask(regs)); assert(emitIssuing); // Don't track GC changes in epilogs @@ -8916,14 +8925,14 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr) return; } - regMaskTP life; - regMaskTP dead; - regMaskTP chg; + regMaskGpr life; + regMaskGpr dead; + regMaskGpr chg; assert(needsGC(gcType)); - regMaskTP& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; - regMaskTP& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; + regMaskGpr& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; + regMaskGpr& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; assert(emitThisXXrefRegs != regs); if (emitFullGCinfo) @@ -8944,7 +8953,7 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr) do { - regMaskTP bit = genFindLowestBit(chg); + regMaskGpr bit = genFindLowestBit(chg); regNumber reg = genRegNumFromMask(bit); if (life & bit) @@ -8977,8 +8986,9 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr) * Record the fact that the given register now contains a live GC ref. */ -void emitter::emitGCregLiveSet(GCtype gcType, regMaskTP regMask, BYTE* addr, bool isThis) +void emitter::emitGCregLiveSet(GCtype gcType, regMaskGpr regMask, BYTE* addr, bool isThis) { + assert(emitComp->IsGprRegMask(regMask)); assert(emitIssuing); assert(needsGC(gcType)); @@ -9008,7 +9018,7 @@ void emitter::emitGCregLiveSet(GCtype gcType, regMaskTP regMask, BYTE* addr, boo * Record the fact that the given register no longer contains a live GC ref. */ -void emitter::emitGCregDeadSet(GCtype gcType, regMaskTP regMask, BYTE* addr) +void emitter::emitGCregDeadSet(GCtype gcType, regMaskGpr regMask, BYTE* addr) { assert(emitIssuing); assert(needsGC(gcType)); @@ -9247,8 +9257,8 @@ void emitter::emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr) regMaskTP regMask = genRegMask(reg); - regMaskTP& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; - regMaskTP& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; + regMaskGpr& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; + regMaskGpr& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; if ((emitThisXXrefRegs & regMask) == 0) { @@ -9285,7 +9295,7 @@ void emitter::emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr) * Record the fact that the given set of registers no longer contain live GC refs. */ -void emitter::emitGCregDeadUpdMask(regMaskTP regs, BYTE* addr) +void emitter::emitGCregDeadUpdMask(regMaskGpr regs, BYTE* addr) { assert(emitIssuing); @@ -9297,7 +9307,7 @@ void emitter::emitGCregDeadUpdMask(regMaskTP regs, BYTE* addr) // First, handle the gcref regs going dead - regMaskTP gcrefRegs = emitThisGCrefRegs & regs; + regMaskGpr gcrefRegs = emitThisGCrefRegs & regs; // "this" can never go dead in synchronized methods, except in the epilog // after the call to CORINFO_HELP_MON_EXIT. @@ -9317,7 +9327,7 @@ void emitter::emitGCregDeadUpdMask(regMaskTP regs, BYTE* addr) // Second, handle the byref regs going dead - regMaskTP byrefRegs = emitThisByrefRegs & regs; + regMaskGpr byrefRegs = emitThisByrefRegs & regs; if (byrefRegs) { @@ -10364,7 +10374,7 @@ const char* emitter::emitOffsetToLabel(unsigned offs) // Return value: // the saved set of registers. // -regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) +regMaskAny emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? bool isNoGCHelper = emitNoGChelper(methHnd); @@ -10373,7 +10383,7 @@ regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd); // Get the set of registers that this call kills and remove it from the saved set. - regMaskTP savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc); + regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc); #ifdef DEBUG if (emitComp->verbose) @@ -10410,7 +10420,7 @@ regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) // Return Value: // Mask of GC register kills // -regMaskTP emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) +regMaskAny emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); regMaskTP result; diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 45be55272f661f..e70b1a3783ac45 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -246,11 +246,11 @@ struct insPlaceholderGroupData insGroup* igPhNext; BasicBlock* igPhBB; VARSET_TP igPhInitGCrefVars; - regMaskTP igPhInitGCrefRegs; - regMaskTP igPhInitByrefRegs; + regMaskGpr igPhInitGCrefRegs; + regMaskGpr igPhInitByrefRegs; VARSET_TP igPhPrevGCrefVars; - regMaskTP igPhPrevGCrefRegs; - regMaskTP igPhPrevByrefRegs; + regMaskGpr igPhPrevGCrefRegs; + regMaskGpr igPhPrevByrefRegs; insGroupPlaceholderType igPhType; }; // end of struct insPlaceholderGroupData @@ -2137,8 +2137,8 @@ class emitter VARSET_TP idcGCvars; // ... updated GC vars or ssize_t idcDisp; // ... big addrmode disp - regMaskTP idcGcrefRegs; // ... gcref registers - regMaskTP idcByrefRegs; // ... byref registers + regMaskGpr idcGcrefRegs; // ... gcref registers + regMaskGpr idcByrefRegs; // ... byref registers unsigned idcArgCnt; // ... lots of args or (<0 ==> caller pops args) #if MULTIREG_HAS_SECOND_GC_RET @@ -2250,11 +2250,11 @@ class emitter VARSET_TP debugPrevGCrefVars; VARSET_TP debugThisGCrefVars; regPtrDsc* debugPrevRegPtrDsc; - regMaskTP debugPrevGCrefRegs; - regMaskTP debugPrevByrefRegs; + regMaskGpr debugPrevGCrefRegs; + regMaskGpr debugPrevByrefRegs; void emitDispInsIndent(); void emitDispGCDeltaTitle(const char* title); - void emitDispGCRegDelta(const char* title, regMaskTP prevRegs, regMaskTP curRegs); + void emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMaskGpr curRegs); void emitDispGCVarDelta(); void emitDispRegPtrListDelta(); void emitDispGCInfoDelta(); @@ -2646,12 +2646,12 @@ class emitter // out, and GCrefRegs is always saved. VARSET_TP emitPrevGCrefVars; - regMaskTP emitPrevGCrefRegs; - regMaskTP emitPrevByrefRegs; + regMaskGpr emitPrevGCrefRegs; + regMaskGpr emitPrevByrefRegs; VARSET_TP emitInitGCrefVars; - regMaskTP emitInitGCrefRegs; - regMaskTP emitInitByrefRegs; + regMaskGpr emitInitGCrefRegs; + regMaskGpr emitInitByrefRegs; // If this is set, we ignore comparing emitPrev* and emitInit* to determine // whether to save GC state (to save space in the IG), and always save it. @@ -2669,8 +2669,8 @@ class emitter // used due to bugs. VARSET_TP emitThisGCrefVars; - regMaskTP emitThisGCrefRegs; // Current set of registers holding GC references - regMaskTP emitThisByrefRegs; // Current set of registers holding BYREF references + regMaskGpr emitThisGCrefRegs; // Current set of registers holding GC references + regMaskGpr emitThisByrefRegs; // Current set of registers holding BYREF references bool emitThisGCrefVset; // Is "emitThisGCrefVars" up to date? @@ -2680,7 +2680,7 @@ class emitter void emitSetSecondRetRegGCType(instrDescCGCA* id, emitAttr secondRetSize); #endif // MULTIREG_HAS_SECOND_GC_RET - static void emitEncodeCallGCregs(regMaskTP regs, instrDesc* id); + static void emitEncodeCallGCregs(regMaskGpr regs, instrDesc* id); static unsigned emitDecodeCallGCregs(instrDesc* id); unsigned emitNxtIGnum; @@ -2853,8 +2853,8 @@ class emitter // Sets the emitter's record of the currently live GC variables // and registers. void* emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs DEBUG_ARG(BasicBlock* block = nullptr)); + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block = nullptr)); // Same as above, except the label is added and is conceptually "inline" in // the current block. Thus it extends the previous block and the emitter @@ -3135,10 +3135,10 @@ class emitter bool emitFullGCinfo; // full GC pointer maps? bool emitFullyInt; // fully interruptible code? - regMaskTP emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); + regMaskAny emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); // Gets a register mask that represent the kill set for a NoGC helper call. - regMaskTP emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); + regMaskAny emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); #if EMIT_TRACK_STACK_DEPTH unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere @@ -3193,7 +3193,7 @@ class emitter /* Liveness of stack variables, and registers */ void emitUpdateLiveGCvars(VARSET_VALARG_TP vars, BYTE* addr); - void emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr); + void emitUpdateLiveGCregs(GCtype gcType, regMaskGpr regs, BYTE* addr); #ifdef DEBUG const char* emitGetFrameReg(); @@ -3202,10 +3202,10 @@ class emitter #endif void emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr); - void emitGCregLiveSet(GCtype gcType, regMaskTP mask, BYTE* addr, bool isThis); - void emitGCregDeadUpdMask(regMaskTP, BYTE* addr); + void emitGCregLiveSet(GCtype gcType, regMaskGpr mask, BYTE* addr, bool isThis); + void emitGCregDeadUpdMask(regMaskGpr, BYTE* addr); void emitGCregDeadUpd(regNumber reg, BYTE* addr); - void emitGCregDeadSet(GCtype gcType, regMaskTP mask, BYTE* addr); + void emitGCregDeadSet(GCtype gcType, regMaskGpr mask, BYTE* addr); void emitGCvarLiveUpd(int offs, int varNum, GCtype gcType, BYTE* addr DEBUG_ARG(unsigned actualVarNum)); void emitGCvarLiveSet(int offs, GCtype gcType, BYTE* addr, ssize_t disp = -1); diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 3fa92b60d0e5b5..f34be27bb6389d 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -4670,8 +4670,8 @@ void emitter::emitIns_Call(EmitCallType callType, int argSize, emitAttr retSize, VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di /* = DebugInfo() */, regNumber ireg /* = REG_NA */, regNumber xreg /* = REG_NA */, @@ -4680,7 +4680,8 @@ void emitter::emitIns_Call(EmitCallType callType, bool isJump /* = false */) { /* Sanity check the arguments depending on callType */ - + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); assert(callType < EC_COUNT); assert((callType != EC_FUNC_TOKEN) || (addr != nullptr && ireg == REG_NA)); assert(callType != EC_INDIR_R || (addr == nullptr && ireg < REG_COUNT)); @@ -4693,7 +4694,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskTP savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; @@ -5764,8 +5765,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { int imm; BYTE* addr; - regMaskTP gcrefRegs; - regMaskTP byrefRegs; + regMaskGpr gcrefRegs; + regMaskGpr byrefRegs; case IF_T1_A: // T1_A ................ sz = SMALL_IDSC_SIZE; diff --git a/src/coreclr/jit/emitarm.h b/src/coreclr/jit/emitarm.h index 245196bfa18346..cf4754fc9d32d5 100644 --- a/src/coreclr/jit/emitarm.h +++ b/src/coreclr/jit/emitarm.h @@ -65,10 +65,10 @@ void emitDispInsHelp(instrDesc* id, private: instrDesc* emitNewInstrCallDir( - int argCnt, VARSET_VALARG_TP GCvars, regMaskTP gcrefRegs, regMaskTP byrefRegs, emitAttr retSize); + int argCnt, VARSET_VALARG_TP GCvars, regMaskGpr gcrefRegs, regMaskGpr byrefRegs, emitAttr retSize); instrDesc* emitNewInstrCallInd( - int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, regMaskTP gcrefRegs, regMaskTP byrefRegs, emitAttr retSize); + int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, regMaskGpr gcrefRegs, regMaskGpr byrefRegs, emitAttr retSize); /************************************************************************/ /* Private helpers for instruction output */ @@ -328,8 +328,8 @@ void emitIns_Call(EmitCallType callType, int argSize, emitAttr retSize, VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di = DebugInfo(), regNumber ireg = REG_NA, regNumber xreg = REG_NA, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 2391e52e5b2004..80b518e6e75a5a 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14479,8 +14479,8 @@ void emitter::emitIns_Call(EmitCallType callType, emitAttr retSize, emitAttr secondRetSize, VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di /* = DebugInfo() */, regNumber ireg /* = REG_NA */, regNumber xreg /* = REG_NA */, @@ -14489,7 +14489,8 @@ void emitter::emitIns_Call(EmitCallType callType, bool isJump /* = false */) { /* Sanity check the arguments depending on callType */ - + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); assert(callType < EC_COUNT); assert((callType != EC_FUNC_TOKEN) || (addr != nullptr && ireg == REG_NA)); assert(callType != EC_INDIR_R || (addr == nullptr && ireg < REG_COUNT)); @@ -14502,7 +14503,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskTP savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; @@ -18362,8 +18363,8 @@ BYTE* emitter::emitOutputVectorConstant( unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) { const unsigned char callInstrSize = sizeof(code_t); // 4 bytes - regMaskTP gcrefRegs; - regMaskTP byrefRegs; + regMaskGpr gcrefRegs; + regMaskGpr byrefRegs; VARSET_TP GCvars(VarSetOps::UninitVal()); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 6d40ce8116fb3e..3b0957ae19090c 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -80,16 +80,16 @@ void emitDispSvePattern(insSvePattern pattern, bool addComma); private: instrDesc* emitNewInstrCallDir(int argCnt, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSize, emitAttr secondRetSize); instrDesc* emitNewInstrCallInd(int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSize, emitAttr secondRetSize); @@ -1398,8 +1398,8 @@ void emitIns_Call(EmitCallType callType, emitAttr retSize, emitAttr secondRetSize, VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di, regNumber ireg, regNumber xreg, diff --git a/src/coreclr/jit/emitinl.h b/src/coreclr/jit/emitinl.h index 66a33b813d58fa..43e68885fac8b9 100644 --- a/src/coreclr/jit/emitinl.h +++ b/src/coreclr/jit/emitinl.h @@ -211,7 +211,7 @@ inline ssize_t emitter::emitGetInsAmdAny(const instrDesc* id) const * * Convert between a register mask and a smaller version for storage. */ -/*static*/ inline void emitter::emitEncodeCallGCregs(regMaskTP regmask, instrDesc* id) +/*static*/ inline void emitter::emitEncodeCallGCregs(regMaskGpr regmask, instrDesc* id) { unsigned encodeMask; diff --git a/src/coreclr/jit/emitpub.h b/src/coreclr/jit/emitpub.h index 674b98a7f4bacd..9a6db100265975 100644 --- a/src/coreclr/jit/emitpub.h +++ b/src/coreclr/jit/emitpub.h @@ -53,8 +53,8 @@ void emitEndProlog(); void emitCreatePlaceholderIG(insGroupPlaceholderType igType, BasicBlock* igBB, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, bool last); void emitGeneratePrologEpilog(); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index d5dc2fd9530a62..c988e871910bc8 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9469,8 +9469,8 @@ void emitter::emitIns_Call(EmitCallType callType, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di, regNumber ireg, regNumber xreg, @@ -9480,6 +9480,10 @@ void emitter::emitIns_Call(EmitCallType callType, // clang-format on { /* Sanity check the arguments depending on callType */ + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); + assert(emitComp->IsGprRegMask(gcrefRegs)); + assert(emitComp->IsGprRegMask(byrefRegs)); assert(callType < EC_COUNT); if (!emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)) @@ -9495,7 +9499,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs((signed)argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskTP savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; @@ -16329,8 +16333,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) BYTE* addr; bool recCall; - regMaskTP gcrefRegs; - regMaskTP byrefRegs; + regMaskGpr gcrefRegs; + regMaskGpr byrefRegs; /********************************************************************/ /* No operands */ diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index d842f91f06a5da..01255131e9d461 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -524,15 +524,15 @@ instrDesc* emitNewInstrAmdCns(emitAttr attr, ssize_t dsp, int cns); instrDesc* emitNewInstrCallDir(int argCnt, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)); instrDesc* emitNewInstrCallInd(int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)); void emitGetInsCns(const instrDesc* id, CnsVal* cv) const; @@ -897,8 +897,8 @@ void emitIns_Call(EmitCallType callType, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di = DebugInfo(), regNumber ireg = REG_NA, regNumber xreg = REG_NA, diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 138d4e4a5169dc..d1360970f4afc6 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -83,7 +83,7 @@ void GCInfo::gcResetForBB() * Print the changes in the gcRegGCrefSetCur sets. */ -void GCInfo::gcDspGCrefSetChanges(regMaskTP gcRegGCrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspGCrefSetChanges(regMaskAny gcRegGCrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -112,7 +112,7 @@ void GCInfo::gcDspGCrefSetChanges(regMaskTP gcRegGCrefSetNew DEBUGARG(bool force * Print the changes in the gcRegByrefSetCur sets. */ -void GCInfo::gcDspByrefSetChanges(regMaskTP gcRegByrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspByrefSetChanges(regMaskAny gcRegByrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -144,14 +144,16 @@ void GCInfo::gcDspByrefSetChanges(regMaskTP gcRegByrefSetNew DEBUGARG(bool force * GCref pointer values. */ -void GCInfo::gcMarkRegSetGCref(regMaskTP regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetGCref(regMaskGpr regMask DEBUGARG(bool forceOutput)) { + assert(compiler->IsGprRegMask(regMask)); + // This set of registers are going to hold REFs. // Make sure they were not holding BYREFs. assert((gcRegByrefSetCur & regMask) == 0); - regMaskTP gcRegByrefSetNew = gcRegByrefSetCur & ~regMask; // Clear it if set in Byref mask - regMaskTP gcRegGCrefSetNew = gcRegGCrefSetCur | regMask; // Set it in GCref mask + regMaskGpr gcRegByrefSetNew = gcRegByrefSetCur & ~regMask; // Clear it if set in Byref mask + regMaskGpr gcRegGCrefSetNew = gcRegGCrefSetCur | regMask; // Set it in GCref mask INDEBUG(gcDspGCrefSetChanges(gcRegGCrefSetNew, forceOutput)); INDEBUG(gcDspByrefSetChanges(gcRegByrefSetNew)); @@ -166,10 +168,12 @@ void GCInfo::gcMarkRegSetGCref(regMaskTP regMask DEBUGARG(bool forceOutput)) * Byref pointer values. */ -void GCInfo::gcMarkRegSetByref(regMaskTP regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput)) { - regMaskTP gcRegByrefSetNew = gcRegByrefSetCur | regMask; // Set it in Byref mask - regMaskTP gcRegGCrefSetNew = gcRegGCrefSetCur & ~regMask; // Clear it if set in GCref mask + assert(compiler->IsGprRegMask(regMask)); + + regMaskGpr gcRegByrefSetNew = gcRegByrefSetCur | regMask; // Set it in Byref mask + regMaskGpr gcRegGCrefSetNew = gcRegGCrefSetCur & ~regMask; // Clear it if set in GCref mask INDEBUG(gcDspGCrefSetChanges(gcRegGCrefSetNew)); INDEBUG(gcDspByrefSetChanges(gcRegByrefSetNew, forceOutput)); @@ -184,12 +188,12 @@ void GCInfo::gcMarkRegSetByref(regMaskTP regMask DEBUGARG(bool forceOutput)) * non-pointer values. */ -void GCInfo::gcMarkRegSetNpt(regMaskTP regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetNpt(regMaskAny regMask DEBUGARG(bool forceOutput)) { /* NOTE: don't unmark any live register variables */ - regMaskTP gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); - regMaskTP gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetMaskVars()); + regMaskAny gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); + regMaskAny gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetMaskVars()); INDEBUG(gcDspGCrefSetChanges(gcRegGCrefSetNew, forceOutput)); INDEBUG(gcDspByrefSetChanges(gcRegByrefSetNew, forceOutput)); @@ -205,7 +209,7 @@ void GCInfo::gcMarkRegSetNpt(regMaskTP regMask DEBUGARG(bool forceOutput)) void GCInfo::gcMarkRegPtrVal(regNumber reg, var_types type) { - regMaskTP regMask = genRegMask(reg); + regMaskAny regMask = genRegMask(reg); switch (type) { @@ -718,7 +722,7 @@ void GCInfo::gcRegPtrSetInit() // It is also called by LinearScan::recordVarLocationAtStartOfBB() which is in turn called by // CodeGen::genCodeForBBList() at the block boundary. -void GCInfo::gcUpdateForRegVarMove(regMaskTP srcMask, regMaskTP dstMask, LclVarDsc* varDsc) +void GCInfo::gcUpdateForRegVarMove(regMaskAny srcMask, regMaskAny dstMask, LclVarDsc* varDsc) { var_types type = varDsc->TypeGet(); bool isGCRef = (type == TYP_REF); diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index b73e8fbc68773a..b5c27557cf295c 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -89,14 +89,14 @@ class GCInfo void gcResetForBB(); - void gcMarkRegSetGCref(regMaskTP regMask DEBUGARG(bool forceOutput = false)); - void gcMarkRegSetByref(regMaskTP regMask DEBUGARG(bool forceOutput = false)); - void gcMarkRegSetNpt(regMaskTP regMask DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetGCref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetNpt(regMaskAny regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegPtrVal(regNumber reg, var_types type); #ifdef DEBUG - void gcDspGCrefSetChanges(regMaskTP gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); - void gcDspByrefSetChanges(regMaskTP gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspGCrefSetChanges(regMaskAny gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspByrefSetChanges(regMaskAny gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); #endif // DEBUG /*****************************************************************************/ @@ -107,8 +107,8 @@ class GCInfo // values. // - regMaskTP gcRegGCrefSetCur; // current regs holding GCrefs - regMaskTP gcRegByrefSetCur; // current regs holding Byrefs + regMaskGpr gcRegGCrefSetCur; // current regs holding GCrefs + regMaskGpr gcRegByrefSetCur; // current regs holding Byrefs VARSET_TP gcTrkStkPtrLcls; // set of tracked stack ptr lcls (GCref and Byref) - no args VARSET_TP gcVarPtrSetCur; // currently live part of "gcTrkStkPtrLcls" @@ -392,7 +392,7 @@ class GCInfo public: // This method updates the appropriate reg masks when a variable is moved. - void gcUpdateForRegVarMove(regMaskTP srcMask, regMaskTP dstMask, LclVarDsc* varDsc); + void gcUpdateForRegVarMove(regMaskAny srcMask, regMaskAny dstMask, LclVarDsc* varDsc); private: ReturnKind getReturnKind(); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 865829dd96c8b4..d7369dc6d13712 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -228,6 +228,7 @@ typedef unsigned __int64 regMaskOnlyOne; // registers. Once we identify all the places with `regMaskOnlyOne`, // `regMaskFloat`, `regMaskGpr`, we will revisit `regMaskAny` and try // to either: +// 0. Revisit regMaskAny and see if they should be "regMaskAny" // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together typedef unsigned __int64 regMaskAny; @@ -607,7 +608,7 @@ inline regMaskTP genRegMask(regNumber reg) * Map a register number to a floating-point register mask. */ -inline regMaskTP genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TYP_DOUBLE */)) +inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TYP_DOUBLE */)) { #if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_X86) || defined(TARGET_LOONGARCH64) || \ defined(TARGET_RISCV64) From e05cecee0f62fc506ced9656de818d62c04611bc Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 17:24:00 -0800 Subject: [PATCH 012/201] Update codegencommon.cpp and related code --- src/coreclr/jit/codegencommon.cpp | 62 +++++++++++++++---------------- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/compiler.h | 4 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 2b838c5e743a11..ca18f4aad54c79 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -314,8 +314,6 @@ void CodeGen::genPrepForCompiler() } } } - VarSetOps::AssignNoCopy(compiler, genLastLiveSet, VarSetOps::MakeEmpty(compiler)); - genLastLiveMask = RBM_NONE; #ifdef DEBUG compiler->fgBBcountAtCodegen = compiler->fgBBcount; #endif @@ -750,7 +748,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) { // TODO-Cleanup: Move the code from compUpdateLifeVar to genUpdateRegLife that updates the // gc sets - regMaskTP regMask = varDsc->lvRegMask(); + regMaskAny regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur &= ~regMask; @@ -795,7 +793,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) VarSetOps::RemoveElemD(this, codeGen->gcInfo.gcVarPtrSetCur, bornVarIndex); } codeGen->genUpdateRegLife(varDsc, true /*isBorn*/, false /*isDying*/ DEBUGARG(nullptr)); - regMaskTP regMask = varDsc->lvRegMask(); + regMaskAny regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur |= regMask; @@ -2874,8 +2872,10 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere unsigned argNum; // current argNum, always in [0..argMax-1] unsigned fixedRetBufIndex; // argNum value used by the fixed return buffer argument (ARM64) unsigned regArgNum; // index into the regArgTab[] table - regMaskTP regArgMaskLive = regState->rsCalleeRegArgMaskLiveIn; + regMaskOnlyOne regArgMaskLive = regState->rsCalleeRegArgMaskLiveIn; bool doingFloat = regState->rsIsFloat; + assert(compiler->IsOnlyOneRegMask(regArgMaskLive)); + assert((doingFloat && compiler->IsFloatRegMask(regArgMaskLive)) || compiler->IsGprRegMask(regArgMaskLive)); // We should be generating the prolog block when we are called assert(compiler->compGeneratingProlog); @@ -3636,7 +3636,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere // Compute xtraReg here when we have a float argument assert(xtraReg == REG_NA); - regMaskTP fpAvailMask; + regMaskFloat fpAvailMask; fpAvailMask = RBM_FLT_CALLEE_TRASH & ~regArgMaskLive; if (GlobalJitOptions::compFeatureHfa) @@ -3656,7 +3656,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere assert(fpAvailMask != RBM_NONE); // We pick the lowest avail register number - regMaskTP tempMask = genFindLowestBit(fpAvailMask); + regMaskFloat tempMask = genFindLowestBit(fpAvailMask); xtraReg = genRegNumFromMask(tempMask); } #if defined(TARGET_X86) @@ -3947,7 +3947,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere /* Finally take care of the remaining arguments that must be enregistered */ while (regArgMaskLive) { - regMaskTP regArgMaskLiveSave = regArgMaskLive; + regMaskOnlyOne regArgMaskLiveSave = regArgMaskLive; for (argNum = 0; argNum < argMax; argNum++) { @@ -4078,7 +4078,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere * Skip this one until its target will be free * which is guaranteed to happen since we have no circular dependencies. */ - regMaskTP destMask = genRegMask(destRegNum); + regMaskOnlyOne destMask = genRegMask(destRegNum); #ifdef TARGET_ARM // Don't process the double until both halves of the destination are clear. if (genActualType(destMemType) == TYP_DOUBLE) @@ -4523,7 +4523,7 @@ void CodeGen::genCheckUseBlockInit() if (genUseBlockInit) { - regMaskTP maskCalleeRegArgMask = intRegState.rsCalleeRegArgMaskLiveIn; + regMaskGpr maskCalleeRegArgMask = intRegState.rsCalleeRegArgMaskLiveIn; // If there is a secret stub param, don't count it, as it will no longer // be live when we do block init. @@ -4574,7 +4574,7 @@ void CodeGen::genZeroInitFltRegs(const regMaskFloat& initFltRegs, // Iterate through float/double registers and initialize them to 0 or // copy from already initialized register of the same type. - regMaskTP regMask = genRegMask(REG_FP_FIRST); + regMaskFloat regMask = genRegMask(REG_FP_FIRST); for (regNumber reg = REG_FP_FIRST; reg <= REG_FP_LAST; reg = REG_NEXT(reg), regMask <<= 1) { if (regMask & initFltRegs) @@ -5360,7 +5360,7 @@ void CodeGen::genFinalizeFrame() // We always save FP. noway_assert(isFramePointerUsed()); #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - regMaskTP okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED); + regMaskAny okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED); if (RBM_ENC_CALLEE_SAVED != 0) { regSet.rsSetRegsModified(RBM_ENC_CALLEE_SAVED); @@ -5397,7 +5397,7 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - regMaskTP maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskAny maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; #ifdef TARGET_ARMARCH if (isFramePointerUsed()) @@ -5418,8 +5418,8 @@ void CodeGen::genFinalizeFrame() #if defined(TARGET_ARM) // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? - regMaskTP maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT; - regMaskTP maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat; + regMaskFloat maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT; + regMaskGpr maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat; if ((maskPushRegsFloat != RBM_NONE) || (compiler->opts.MinOpts() && (regSet.rsMaskResvd & maskCalleeRegsPushed & RBM_OPT_RSVD))) @@ -5447,7 +5447,7 @@ void CodeGen::genFinalizeFrame() // if (maskPushRegsFloat != RBM_NONE) { - regMaskTP contiguousMask = genRegMaskFloat(REG_F16); + regMaskFloat contiguousMask = genRegMaskFloat(REG_F16); while (maskPushRegsFloat > contiguousMask) { contiguousMask <<= 2; @@ -5455,7 +5455,7 @@ void CodeGen::genFinalizeFrame() } if (maskPushRegsFloat != contiguousMask) { - regMaskTP maskExtraRegs = contiguousMask - maskPushRegsFloat; + regMaskFloat maskExtraRegs = contiguousMask - maskPushRegsFloat; maskPushRegsFloat |= maskExtraRegs; regSet.rsSetRegsModified(maskExtraRegs); maskCalleeRegsPushed |= maskExtraRegs; @@ -5645,9 +5645,9 @@ void CodeGen::genFnProlog() int GCrefHi = -INT_MAX; bool hasGCRef = false; - regMaskTP initRegs = RBM_NONE; // Registers which must be init'ed. - regMaskTP initFltRegs = RBM_NONE; // FP registers which must be init'ed. - regMaskTP initDblRegs = RBM_NONE; + regMaskGpr initRegs = RBM_NONE; // Registers which must be init'ed. + regMaskFloat initFltRegs = RBM_NONE; // FP registers which must be init'ed. + regMaskFloat initDblRegs = RBM_NONE; unsigned varNum; LclVarDsc* varDsc; @@ -5721,7 +5721,7 @@ void CodeGen::genFnProlog() if (isInReg) { regNumber regForVar = varDsc->GetRegNum(); - regMaskTP regMask = genRegMask(regForVar); + regMaskAny regMask = genRegMask(regForVar); if (!genIsValidFloatReg(regForVar)) { initRegs |= regMask; @@ -5834,9 +5834,9 @@ void CodeGen::genFnProlog() // Track if initReg holds non-zero value. Start conservative and assume it has non-zero value. // If initReg is ever set to zero, this variable is set to true and zero initializing initReg // will be skipped. - bool initRegZeroed = false; - regMaskTP excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; - regMaskTP tempMask; + bool initRegZeroed = false; + regMaskOnlyOne excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; + regMaskGpr tempMask; // We should not use the special PINVOKE registers as the initReg // since they are trashed by the jithelper call to setup the PINVOKE frame @@ -6073,7 +6073,7 @@ void CodeGen::genFnProlog() CLANG_FORMAT_COMMENT_ANCHOR; #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) - regMaskTP maskStackAlloc = RBM_NONE; + regMaskGpr maskStackAlloc = RBM_NONE; #ifdef TARGET_ARM maskStackAlloc = genStackAllocRegisterMask(compiler->compLclFrameSize + extraFrameSize, @@ -6320,7 +6320,7 @@ void CodeGen::genFnProlog() if (initRegs) { - regMaskTP regMask = 0x1; + regMaskGpr regMask = 0x1; for (regNumber reg = REG_INT_FIRST; reg <= REG_INT_LAST; reg = REG_NEXT(reg), regMask <<= 1) { @@ -6818,11 +6818,11 @@ regMaskGpr CodeGen::genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskG noway_assert(genTypeStSz(TYP_REF) == genTypeStSz(TYP_I_IMPL)); noway_assert(genTypeStSz(TYP_BYREF) == genTypeStSz(TYP_I_IMPL)); - regMaskTP pushedRegs = regs; + regMaskGpr pushedRegs = regs; for (regNumber reg = REG_INT_FIRST; regs != RBM_NONE; reg = REG_NEXT(reg)) { - regMaskTP regBit = regMaskTP(1) << reg; + regMaskGpr regBit = regMaskGpr(1) << reg; if ((regBit & regs) == RBM_NONE) continue; @@ -6895,7 +6895,7 @@ void CodeGen::genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRef // Walk the registers in the reverse order as genPushRegs() for (regNumber reg = REG_INT_LAST; regs != RBM_NONE; reg = REG_PREV(reg)) { - regMaskTP regBit = regMaskTP(1) << reg; + regMaskGpr regBit = regMaskGpr(1) << reg; if ((regBit & regs) == RBM_NONE) continue; @@ -8096,7 +8096,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) // First set the source registers as busy if they haven't been spilled. // (Note that this is just for verification that we don't have circular dependencies.) - regMaskTP busyRegs = RBM_NONE; + regMaskAny busyRegs = RBM_NONE; for (unsigned i = 0; i < regCount; ++i) { if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0) @@ -8112,7 +8112,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) regNumber targetReg = genRegCopy(treeNode, i); if (targetReg != sourceReg) { - regMaskTP targetRegMask = genRegMask(targetReg); + regMaskAny targetRegMask = genRegMask(targetReg); assert((busyRegs & targetRegMask) == 0); // Clear sourceReg from the busyRegs, and add targetReg. busyRegs &= ~genRegMask(sourceReg); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index def2c6bacb177b..56c14eacbc55fd 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6612,7 +6612,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - regMaskTP tempMask = varDsc->lvRegMask(); + regMaskAny tempMask = varDsc->lvRegMask(); regSet.RemoveMaskVars(tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 14fb7619c7bd88..4d2e393fb33559 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1005,9 +1005,9 @@ class LclVarDsc return lvIsRegCandidate() && (GetRegNum() != REG_STK); } - regMaskTP lvRegMask() const + regMaskAny lvRegMask() const { - regMaskTP regMask = RBM_NONE; + regMaskAny regMask = RBM_NONE; if (GetRegNum() != REG_STK) { if (varTypeUsesFloatReg(this)) From 82ed9aa70d140574d6f7c265a209ae9455e45603 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 17:32:10 -0800 Subject: [PATCH 013/201] Update codegeninterface.h and related code --- src/coreclr/jit/codegeninterface.h | 23 ++++++++++------------- src/coreclr/jit/codegenlinear.cpp | 4 ++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index c21ac9ae9a5cfa..9a411207ad637d 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -60,31 +60,31 @@ class CodeGenInterface } #if defined(TARGET_AMD64) - regMaskTP rbmAllFloat; - regMaskTP rbmFltCalleeTrash; + regMaskFloat rbmAllFloat; + regMaskFloat rbmFltCalleeTrash; - FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const + FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskFloat get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - regMaskTP rbmAllMask; - regMaskTP rbmMskCalleeTrash; + regMaskAny rbmAllMask; + regMaskPredicate rbmMskCalleeTrash; // Call this function after the equivalent fields in Compiler have been initialized. void CopyRegisterInfo(); - FORCEINLINE regMaskTP get_RBM_ALLMASK() const + FORCEINLINE regMaskAny get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskPredicate get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } @@ -158,11 +158,8 @@ class CodeGenInterface bool genTempLiveChg; #endif - VARSET_TP genLastLiveSet; // A one element map (genLastLiveSet-> genLastLiveMask) - regMaskTP genLastLiveMask; // these two are used in genLiveMask - - regMaskTP genGetRegMask(const LclVarDsc* varDsc); - regMaskTP genGetRegMask(GenTree* tree); + regMaskOnlyOne genGetRegMask(const LclVarDsc* varDsc); + regMaskOnlyOne genGetRegMask(GenTree* tree); void genUpdateLife(GenTree* tree); void genUpdateLife(VARSET_VALARG_TP newLife); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 11e9eccff8e700..28a0bdebbee970 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2271,8 +2271,8 @@ void CodeGen::genProduceReg(GenTree* tree) // transfer gc/byref status of src reg to dst reg void CodeGen::genTransferRegGCState(regNumber dst, regNumber src) { - regMaskTP srcMask = genRegMask(src); - regMaskTP dstMask = genRegMask(dst); + regMaskOnlyOne srcMask = genRegMask(src); + regMaskOnlyOne dstMask = genRegMask(dst); if (gcInfo.gcRegGCrefSetCur & srcMask) { From 91462c72e9ce0b145551e0fbb6957a28451cdf13 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 17:54:59 -0800 Subject: [PATCH 014/201] Update codegenxarch.cpp and related code --- src/coreclr/jit/codegenxarch.cpp | 73 ++++++++++++++++---------------- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/target.h | 1 + 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 56c14eacbc55fd..69f74916d0731d 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -2373,8 +2373,9 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) { + assert(compiler->IsGprRegMask(maskArgRegsLiveIn)); assert(compiler->compGeneratingProlog); if (frameSize == 0) @@ -5798,9 +5799,9 @@ void CodeGen::genCodeForSwap(GenTreeOp* tree) assert(varTypeUsesIntReg(type1)); regNumber oldOp1Reg = lcl1->GetRegNum(); - regMaskTP oldOp1RegMask = genRegMask(oldOp1Reg); + regMaskGpr oldOp1RegMask = genRegMask(oldOp1Reg); regNumber oldOp2Reg = lcl2->GetRegNum(); - regMaskTP oldOp2RegMask = genRegMask(oldOp2Reg); + regMaskGpr oldOp2RegMask = genRegMask(oldOp2Reg); // We don't call genUpdateVarReg because we don't have a tree node with the new register. varDsc1->SetRegNum(oldOp2Reg); @@ -6095,7 +6096,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskTP killMask = RBM_CALLEE_TRASH; + regMaskAny killMask = RBM_CALLEE_TRASH; if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); @@ -6639,7 +6640,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) #endif // Next move any un-enregistered register arguments back to their register. - regMaskTP fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. + regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { @@ -6814,14 +6815,14 @@ void CodeGen::genJmpMethod(GenTree* jmp) assert(compiler->info.compIsVarArgs); assert(firstArgVarNum != BAD_VAR_NUM); - regMaskTP remainingIntArgMask = RBM_ARG_REGS & ~fixedIntArgMask; + regMaskGpr remainingIntArgMask = RBM_ARG_REGS & ~fixedIntArgMask; if (remainingIntArgMask != RBM_NONE) { GetEmitter()->emitDisableGC(); for (int argNum = 0, argOffset = 0; argNum < MAX_REG_ARG; ++argNum) { regNumber argReg = intArgRegs[argNum]; - regMaskTP argRegMask = genRegMask(argReg); + regMaskGpr argRegMask = genRegMask(argReg); if ((remainingIntArgMask & argRegMask) != 0) { @@ -6914,7 +6915,7 @@ void CodeGen::genCompareFloat(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - regMaskTP targetRegMask = genRegMask(targetReg); + regMaskFloat targetRegMask = genRegMask(targetReg); if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetRegMask) == 0) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); @@ -7085,7 +7086,7 @@ void CodeGen::genCompareInt(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - regMaskTP targetRegMask = genRegMask(targetReg); + regMaskGpr targetRegMask = genRegMask(targetReg); if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetRegMask) == 0) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); @@ -8440,7 +8441,7 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk) regNumber simdTmpReg = REG_NA; if (putArgStk->AvailableTempRegCount() != 0) { - regMaskTP rsvdRegs = putArgStk->gtRsvdRegs; + regMaskAny rsvdRegs = putArgStk->gtRsvdRegs; if ((rsvdRegs & RBM_ALLINT) != 0) { intTmpReg = putArgStk->GetSingleTempReg(RBM_ALLINT); @@ -9143,7 +9144,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); regNumber callTarget = REG_NA; - regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskAny killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { @@ -9173,14 +9174,14 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, // If a callTargetReg has not been explicitly provided, we will use REG_DEFAULT_HELPER_CALL_TARGET, but // this is only a valid assumption if the helper call is known to kill REG_DEFAULT_HELPER_CALL_TARGET. callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; - regMaskTP callTargetMask = genRegMask(callTargetReg); + regMaskGpr callTargetMask = genRegMask(callTargetReg); noway_assert((callTargetMask & killMask) == callTargetMask); } else { // The call target must not overwrite any live variable, though it may not be in the // kill set for the call. - regMaskTP callTargetMask = genRegMask(callTargetReg); + regMaskGpr callTargetMask = genRegMask(callTargetReg); noway_assert((callTargetMask & regSet.GetMaskVars()) == RBM_NONE); } #endif @@ -9671,7 +9672,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // registers that profiler callback kills. if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { - regMaskTP thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); + regMaskAny thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); noway_assert((RBM_PROFILER_LEAVE_TRASH & thisPtrMask) == 0); } @@ -9787,8 +9788,8 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // Emit appropriate unwind. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); - regMaskTP tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; int const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; JITDUMP("--OSR--- tier0 has already saved "); @@ -9810,7 +9811,7 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // for (regNumber reg = REG_INT_LAST; tier0IntCalleeSaves != RBM_NONE; reg = REG_PREV(reg)) { - regMaskTP regBit = genRegMask(reg); + regMaskAny regBit = genRegMask(reg); if ((regBit & tier0IntCalleeSaves) != 0) { @@ -9855,7 +9856,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not // here. - regMaskTP rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -9867,11 +9868,11 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // Figure out which set of int callee saves still needs saving. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); - regMaskTP tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; - regMaskTP const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; - regMaskTP osrAdditionalIntCalleeSaves = osrIntCalleeSaves & ~tier0IntCalleeSaves; + regMaskGpr const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr osrAdditionalIntCalleeSaves = osrIntCalleeSaves & ~tier0IntCalleeSaves; JITDUMP("---OSR--- int callee saves are "); JITDUMPEXEC(dspRegMask(osrIntCalleeSaves)); @@ -9901,7 +9902,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // for (regNumber reg = REG_INT_LAST; osrAdditionalIntCalleeSaves != RBM_NONE; reg = REG_PREV(reg)) { - regMaskTP regBit = genRegMask(reg); + regMaskAny regBit = genRegMask(reg); if ((regBit & osrAdditionalIntCalleeSaves) != 0) { @@ -9936,7 +9937,7 @@ void CodeGen::genPushCalleeSavedRegisters() // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not // here. - regMaskTP rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -9966,7 +9967,7 @@ void CodeGen::genPushCalleeSavedRegisters() // and all the other code that expects it to be in this order. for (regNumber reg = REG_INT_LAST; rsPushRegs != RBM_NONE; reg = REG_PREV(reg)) { - regMaskTP regBit = genRegMask(reg); + regMaskAny regBit = genRegMask(reg); if ((regBit & rsPushRegs) != 0) { @@ -10201,10 +10202,10 @@ void CodeGen::genFnEpilog(BasicBlock* block) // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); - regMaskTP const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; - regMaskTP const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; - regMaskTP const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; + regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskGpr const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; unsigned const osrCalleeSaveSize = compiler->compCalleeRegsPushed * REGSIZE_BYTES; @@ -10733,7 +10734,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() assert(isFramePointerUsed()); assert(compiler->lvaDoneFrameLayout == Compiler::FINAL_FRAME_LAYOUT); // The frame size and offsets must be // finalized - assert(compiler->compCalleeFPRegsSavedMask != (regMaskTP)-1); // The float registers to be preserved is finalized + assert(compiler->compCalleeFPRegsSavedMask != (regMaskAny)-1); // The float registers to be preserved is finalized // Even though lvaToInitialSPRelativeOffset() depends on compLclFrameSize, // that's ok, because we're figuring out an offset in the parent frame. @@ -11191,10 +11192,10 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu void CodeGen::genPreserveCalleeSavedFltRegs(unsigned lclFrameSize) { genVzeroupperIfNeeded(false); - regMaskTP regMask = compiler->compCalleeFPRegsSavedMask; + regMaskFloat regMask = compiler->compCalleeFPRegsSavedMask; // Only callee saved floating point registers should be in regMask - assert((regMask & RBM_FLT_CALLEE_SAVED) == regMask); + assert(compiler->IsFloatRegMask(regMask)); // fast path return if (regMask == RBM_NONE) @@ -11216,7 +11217,7 @@ void CodeGen::genPreserveCalleeSavedFltRegs(unsigned lclFrameSize) for (regNumber reg = REG_FLT_CALLEE_SAVED_FIRST; regMask != RBM_NONE; reg = REG_NEXT(reg)) { - regMaskTP regBit = genRegMask(reg); + regMaskAny regBit = genRegMask(reg); if ((regBit & regMask) != 0) { // ABI requires us to preserve lower 128-bits of YMM register. @@ -11238,10 +11239,10 @@ void CodeGen::genPreserveCalleeSavedFltRegs(unsigned lclFrameSize) // funclet frames: this will be FuncletInfo.fiSpDelta. void CodeGen::genRestoreCalleeSavedFltRegs(unsigned lclFrameSize) { - regMaskTP regMask = compiler->compCalleeFPRegsSavedMask; + regMaskFloat regMask = compiler->compCalleeFPRegsSavedMask; // Only callee saved floating point registers should be in regMask - assert((regMask & RBM_FLT_CALLEE_SAVED) == regMask); + assert(compiler->IsFloatRegMask(regMask)); // fast path return if (regMask == RBM_NONE) @@ -11280,7 +11281,7 @@ void CodeGen::genRestoreCalleeSavedFltRegs(unsigned lclFrameSize) for (regNumber reg = REG_FLT_CALLEE_SAVED_FIRST; regMask != RBM_NONE; reg = REG_NEXT(reg)) { - regMaskTP regBit = genRegMask(reg); + regMaskAny regBit = genRegMask(reg); if ((regBit & regMask) != 0) { // ABI requires us to restore lower 128-bits of YMM register. diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 5fac5c8e0704c6..9b43196dab64c4 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5773,7 +5773,7 @@ void Compiler::generatePatchpointInfo() // Record callee save registers. // Currently only needed for x64. // - regMaskTP rsPushRegs = codeGen->regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskAny rsPushRegs = codeGen->regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; rsPushRegs |= RBM_FPBASE; patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs); JITDUMP("--OSR-- Tier0 callee saves: "); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 4d2e393fb33559..445f71860193b7 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10432,7 +10432,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #if defined(TARGET_XARCH) // Mask of callee saved float regs on stack. - regMaskTP compCalleeFPRegsSavedMask; + regMaskFloat compCalleeFPRegsSavedMask; #endif #ifdef TARGET_AMD64 // Quirk for VS debug-launch scenario to work: diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index d7369dc6d13712..9f07b18916adef 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -232,6 +232,7 @@ typedef unsigned __int64 regMaskOnlyOne; // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together typedef unsigned __int64 regMaskAny; +// TODO: should have `regMaskSingleReg` to represent genRegMask(regNumber) return value #else // x86 and arm typedef unsigned regMaskTP; From b9d1ed5af15c2049c0693f15b56d9ae2dff259cd Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 8 Feb 2024 17:56:36 -0800 Subject: [PATCH 015/201] Update compiler.cpp and related code --- src/coreclr/jit/compiler.cpp | 12 ++++++------ src/coreclr/jit/compiler.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 9b43196dab64c4..05c8fa2c916e26 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3382,12 +3382,12 @@ void Compiler::compInitOptions(JitFlags* jitFlags) // Make sure we copy the register info and initialize the // trash regs after the underlying fields are initialized - const regMaskTP vtCalleeTrashRegs[TYP_COUNT]{ + const regMaskAny vtCalleeTrashRegs[TYP_COUNT]{ #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP }; - memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskTP) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskAny) * TYP_COUNT); codeGen->CopyRegisterInfo(); #endif // TARGET_XARCH @@ -3631,7 +3631,7 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) // Arguments: // regs - The set of registers to display // -void Compiler::dumpRegMask(regMaskTP regs) const +void Compiler::dumpRegMask(regMaskAny regs) const { if (regs == RBM_ALLINT) { @@ -5777,7 +5777,7 @@ void Compiler::generatePatchpointInfo() rsPushRegs |= RBM_FPBASE; patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs); JITDUMP("--OSR-- Tier0 callee saves: "); - JITDUMPEXEC(dspRegMask((regMaskTP)patchpointInfo->CalleeSaveRegisters())); + JITDUMPEXEC(dspRegMask((regMaskAny)patchpointInfo->CalleeSaveRegisters())); JITDUMP("\n"); #endif @@ -9336,7 +9336,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX * cVN, dVN : Display a ValueNum (call vnPrint()). * * The following don't require a Compiler* to work: - * dRegMask : Display a regMaskTP (call dspRegMask(mask)). + * dRegMask : Display a regMaskAny (call dspRegMask(mask)). * dBlockList : Display a BasicBlockList*. * * The following find an object in the IR and return it, as well as setting a global variable with the value that can @@ -10287,7 +10287,7 @@ JITDBGAPI void __cdecl dVN(ValueNum vn) cVN(JitTls::GetCompiler(), vn); } -JITDBGAPI void __cdecl dRegMask(regMaskTP mask) +JITDBGAPI void __cdecl dRegMask(regMaskAny mask) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== dRegMask %u\n", sequenceNumber++); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 445f71860193b7..8090098a93aaa6 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10799,7 +10799,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool compJitHaltMethod(); - void dumpRegMask(regMaskTP regs) const; + void dumpRegMask(regMaskAny regs) const; #endif From 83f0f012bd9e29ba3f9ebef2b38d6d1829008f55 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 09:43:48 -0800 Subject: [PATCH 016/201] Update compiler.h and related code --- src/coreclr/jit/codegeninterface.h | 2 +- src/coreclr/jit/compiler.h | 44 +++++++++++++++--------------- src/coreclr/jit/lclvars.cpp | 2 +- src/coreclr/jit/lsra.cpp | 2 ++ src/coreclr/jit/lsra.h | 10 +++---- src/coreclr/jit/regset.h | 2 +- src/coreclr/jit/unwind.cpp | 4 ++- src/coreclr/jit/unwindarmarch.cpp | 20 ++++++++++---- 8 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index 9a411207ad637d..17db64a8c6f312 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -74,7 +74,7 @@ class CodeGenInterface #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - regMaskAny rbmAllMask; + regMaskPredicate rbmAllMask; regMaskPredicate rbmMskCalleeTrash; // Call this function after the equivalent fields in Compiler have been initialized. diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 8090098a93aaa6..3f4c32a923f755 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3866,7 +3866,7 @@ class Compiler unsigned lvaGetMaxSpillTempSize(); #ifdef TARGET_ARM - bool lvaIsPreSpilled(unsigned lclNum, regMaskTP preSpillMask); + bool lvaIsPreSpilled(unsigned lclNum, regMaskAny preSpillMask); #endif // TARGET_ARM void lvaAssignFrameOffsets(FrameLayoutState curState); void lvaFixVirtualFrameOffsets(); @@ -8422,7 +8422,7 @@ class Compiler // Gets a register mask that represent the kill set for a helper call since // not all JIT Helper calls follow the standard ABI on the target architecture. - regMaskTP compHelperCallKillSet(CorInfoHelpFunc helper); + regMaskAny compHelperCallKillSet(CorInfoHelpFunc helper); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -8476,10 +8476,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void unwindSaveReg(regNumber reg, unsigned offset); #if defined(TARGET_ARM) - void unwindPushMaskInt(regMaskTP mask); - void unwindPushMaskFloat(regMaskTP mask); - void unwindPopMaskInt(regMaskTP mask); - void unwindPopMaskFloat(regMaskTP mask); + void unwindPushMaskInt(regMaskGpr mask); + void unwindPushMaskFloat(regMaskFloat mask); + void unwindPopMaskInt(regMaskGpr mask); + void unwindPopMaskFloat(regMaskFloat mask); void unwindBranch16(); // The epilog terminates with a 16-bit branch (e.g., "bx lr") void unwindNop(unsigned codeSizeInBytes); // Generate unwind NOP code. 'codeSizeInBytes' is 2 or 4 bytes. Only // called via unwindPadding(). @@ -8554,8 +8554,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // UNIX_AMD64_ABI #elif defined(TARGET_ARM) - void unwindPushPopMaskInt(regMaskTP mask, bool useOpsize16); - void unwindPushPopMaskFloat(regMaskTP mask); + void unwindPushPopMaskInt(regMaskGpr mask, bool useOpsize16); + void unwindPushPopMaskFloat(regMaskFloat mask); #endif // TARGET_ARM @@ -8564,7 +8564,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void createCfiCode(FuncInfoDsc* func, UNATIVE_OFFSET codeOffset, UCHAR opcode, short dwarfReg, INT offset = 0); void unwindPushPopCFI(regNumber reg); void unwindBegPrologCFI(); - void unwindPushPopMaskCFI(regMaskTP regMask, bool isFloat); + void unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat); void unwindAllocStackCFI(unsigned size); void unwindSetFrameRegCFI(regNumber reg, unsigned offset); void unwindEmitFuncCFI(FuncInfoDsc* func, void* pHotCode, void* pColdCode); @@ -11072,8 +11072,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // // Users of these values need to define four accessor functions: // - // regMaskTP get_RBM_ALLFLOAT(); - // regMaskTP get_RBM_FLT_CALLEE_TRASH(); + // regMaskFloat get_RBM_ALLFLOAT(); + // regMaskFloat get_RBM_FLT_CALLEE_TRASH(); // unsigned get_CNT_CALLEE_TRASH_FLOAT(); // unsigned get_AVAILABLE_REG_COUNT(); // @@ -11082,16 +11082,16 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only // TARGET_AMD64 requires one. // - regMaskTP rbmAllFloat; - regMaskTP rbmFltCalleeTrash; + regMaskFloat rbmAllFloat; + regMaskFloat rbmFltCalleeTrash; unsigned cntCalleeTrashFloat; public: - FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const + FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskFloat get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } @@ -11110,8 +11110,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // // Users of these values need to define four accessor functions: // - // regMaskTP get_RBM_ALLMASK(); - // regMaskTP get_RBM_MSK_CALLEE_TRASH(); + // regMaskPredicate get_RBM_ALLMASK(); + // regMaskPredicate get_RBM_MSK_CALLEE_TRASH(); // unsigned get_CNT_CALLEE_TRASH_MASK(); // unsigned get_AVAILABLE_REG_COUNT(); // @@ -11120,17 +11120,17 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only // TARGET_XARCH requires one. // - regMaskTP rbmAllMask; - regMaskTP rbmMskCalleeTrash; + regMaskPredicate rbmAllMask; + regMaskPredicate rbmMskCalleeTrash; unsigned cntCalleeTrashMask; - regMaskTP varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskAny varTypeCalleeTrashRegs[TYP_COUNT]; public: - FORCEINLINE regMaskTP get_RBM_ALLMASK() const + FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskPredicate get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 1cf64495ec0ecb..5fd3001a48a471 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -5358,7 +5358,7 @@ void Compiler::lvaFixVirtualFrameOffsets() } #ifdef TARGET_ARM -bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskTP preSpillMask) +bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskAny preSpillMask) { const LclVarDsc& desc = lvaTable[lclNum]; return desc.lvIsRegArg && (preSpillMask & genRegMask(desc.GetArgReg())); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3915a2bb45a164..60c0b800464a05 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -720,6 +720,8 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_AMD64) rbmAllFloat = compiler->rbmAllFloat; rbmFltCalleeTrash = compiler->rbmFltCalleeTrash; + assert(compiler->IsGprRegMask(rbmAllFloat)); + assert(compiler->IsGprRegMask(rbmFltCalleeTrash)); #endif // TARGET_AMD64 #if defined(TARGET_XARCH) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index c0e0f5d2fdbd34..91252a30110a14 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2036,7 +2036,7 @@ class LinearScan : public LinearScanInterface regMaskTP rbmAllFloat; regMaskTP rbmFltCalleeTrash; - FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const + FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } @@ -2047,14 +2047,14 @@ class LinearScan : public LinearScanInterface #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - regMaskTP rbmAllMask; - regMaskTP rbmMskCalleeTrash; + regMaskPredicate rbmAllMask; + regMaskPredicate rbmMskCalleeTrash; - FORCEINLINE regMaskTP get_RBM_ALLMASK() const + FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskPredicate get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 02c5460197bd49..502d03e0d8aee8 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -43,7 +43,7 @@ class RegSet RegSet(Compiler* compiler, GCInfo& gcInfo); #ifdef TARGET_ARM - regMaskTP rsMaskPreSpillRegs(bool includeAlignment) const + regMaskAny rsMaskPreSpillRegs(bool includeAlignment) const { return includeAlignment ? (rsMaskPreSpillRegArg | rsMaskPreSpillAlign) : rsMaskPreSpillRegArg; } diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index a927e73c02b9f1..ef722504a0f11a 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -201,8 +201,10 @@ void Compiler::unwindBegPrologCFI() #endif // FEATURE_EH_FUNCLETS } -void Compiler::unwindPushPopMaskCFI(regMaskTP regMask, bool isFloat) +void Compiler::unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat) { + assert(IsOnlyOneRegMask(regMask)); + #if TARGET_ARM regNumber regNum = isFloat ? REG_PREV(REG_FP_LAST) : REG_INT_LAST; regMaskTP regBit = isFloat ? genRegMask(regNum) | genRegMask(REG_NEXT(regNum)) : genRegMask(regNum); diff --git a/src/coreclr/jit/unwindarmarch.cpp b/src/coreclr/jit/unwindarmarch.cpp index 445b2581ca0abb..21a2a0b43e453e 100644 --- a/src/coreclr/jit/unwindarmarch.cpp +++ b/src/coreclr/jit/unwindarmarch.cpp @@ -196,10 +196,10 @@ void Compiler::unwindEndEpilog() #if defined(TARGET_ARM) -void Compiler::unwindPushPopMaskInt(regMaskTP maskInt, bool useOpsize16) +void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) { // floating point registers cannot be specified in 'maskInt' - assert((maskInt & RBM_ALLFLOAT) == 0); + assert(IsGprRegMask(maskInt)); UnwindInfo* pu = &funCurrentFunc()->uwi; @@ -318,8 +318,10 @@ void Compiler::unwindPushPopMaskFloat(regMaskTP maskFloat) pu->AddCode(0xE0 | val); } -void Compiler::unwindPushMaskInt(regMaskTP maskInt) +void Compiler::unwindPushMaskInt(regMaskGpr maskInt) { + assert(IsGprRegMask(maskInt)); + // Only r0-r12 and lr are supported assert((maskInt & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | @@ -338,8 +340,10 @@ void Compiler::unwindPushMaskInt(regMaskTP maskInt) unwindPushPopMaskInt(maskInt, useOpsize16); } -void Compiler::unwindPushMaskFloat(regMaskTP maskFloat) +void Compiler::unwindPushMaskFloat(regMaskFloat maskFloat) { + assert(IsFloatRegMask(maskFloat)); + // Only floating point registers should be in maskFloat assert((maskFloat & RBM_ALLFLOAT) == maskFloat); @@ -354,8 +358,10 @@ void Compiler::unwindPushMaskFloat(regMaskTP maskFloat) unwindPushPopMaskFloat(maskFloat); } -void Compiler::unwindPopMaskInt(regMaskTP maskInt) +void Compiler::unwindPopMaskInt(regMaskGpr maskInt) { + assert(IsGprRegMask(maskInt)); + #if defined(FEATURE_CFI_SUPPORT) if (generateCFIUnwindCodes()) { @@ -382,8 +388,10 @@ void Compiler::unwindPopMaskInt(regMaskTP maskInt) unwindPushPopMaskInt(maskInt, useOpsize16); } -void Compiler::unwindPopMaskFloat(regMaskTP maskFloat) +void Compiler::unwindPopMaskFloat(regMaskFloat maskFloat) { + assert(IsFloatRegMask(maskFloat)); + #if defined(FEATURE_CFI_SUPPORT) if (generateCFIUnwindCodes()) { From c3797e06288bdeeaee6379b3dfb4e191ab7538e7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 09:51:04 -0800 Subject: [PATCH 017/201] Update compiler.hpp and related code --- src/coreclr/jit/compiler.hpp | 55 ++++++++---------------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index d664a57bd24143..32bed4cf267907 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -918,7 +918,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block) // Assumptions: // The mask contains one and only one register. -inline regNumber genRegNumFromMask(regMaskTP mask) +inline regNumber genRegNumFromMask(regMaskOnlyOne mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -944,7 +944,7 @@ inline regNumber genRegNumFromMask(regMaskTP mask) // the bit. // -inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask) +inline regNumber genFirstRegNumFromMaskAndToggle(regMaskAny& mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -966,7 +966,7 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask) // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(regMaskTP mask) +inline regNumber genFirstRegNumFromMask(regMaskAny mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -3302,14 +3302,14 @@ __forceinline regNumber genMapRegArgNumToRegNum(unsigned argNum, var_types type) * (for a double on ARM) is returned. */ -inline regMaskTP genMapIntRegArgNumToRegMask(unsigned argNum) +inline regMaskGpr genMapIntRegArgNumToRegMask(unsigned argNum) { assert(argNum < ArrLen(intArgMasks)); return intArgMasks[argNum]; } -inline regMaskTP genMapFloatRegArgNumToRegMask(unsigned argNum) +inline regMaskFloat genMapFloatRegArgNumToRegMask(unsigned argNum) { #ifndef TARGET_X86 assert(argNum < ArrLen(fltArgMasks)); @@ -3321,9 +3321,9 @@ inline regMaskTP genMapFloatRegArgNumToRegMask(unsigned argNum) #endif } -__forceinline regMaskTP genMapArgNumToRegMask(unsigned argNum, var_types type) +__forceinline regMaskOnlyOne genMapArgNumToRegMask(unsigned argNum, var_types type) { - regMaskTP result; + regMaskOnlyOne result; if (varTypeUsesFloatArgReg(type)) { result = genMapFloatRegArgNumToRegMask(argNum); @@ -3457,39 +3457,6 @@ inline unsigned genMapRegNumToRegArgNum(regNumber regNum, var_types type) } } -/*****************************************************************************/ -/* Return a register mask with the first 'numRegs' argument registers set. - */ - -inline regMaskTP genIntAllRegArgMask(unsigned numRegs) -{ - assert(numRegs <= MAX_REG_ARG); - - regMaskTP result = RBM_NONE; - for (unsigned i = 0; i < numRegs; i++) - { - result |= intArgMasks[i]; - } - return result; -} - -inline regMaskTP genFltAllRegArgMask(unsigned numRegs) -{ -#ifndef TARGET_X86 - assert(numRegs <= MAX_FLOAT_REG_ARG); - - regMaskTP result = RBM_NONE; - for (unsigned i = 0; i < numRegs; i++) - { - result |= fltArgMasks[i]; - } - return result; -#else - assert(!"no x86 float arg regs\n"); - return RBM_NONE; -#endif -} - /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -4464,12 +4431,12 @@ inline void* operator new[](size_t sz, Compiler* compiler, CompMemKind cmk) #ifdef DEBUG -inline void printRegMask(regMaskTP mask) +inline void printRegMask(regMaskAny mask) { printf(REG_MASK_ALL_FMT, mask); } -inline char* regMaskToString(regMaskTP mask, Compiler* context) +inline char* regMaskToString(regMaskAny mask, Compiler* context) { const size_t cchRegMask = 24; char* regmask = new (context, CMK_Unknown) char[cchRegMask]; @@ -4479,12 +4446,12 @@ inline char* regMaskToString(regMaskTP mask, Compiler* context) return regmask; } -inline void printRegMaskInt(regMaskTP mask) +inline void printRegMaskInt(regMaskGpr mask) { printf(REG_MASK_INT_FMT, (mask & RBM_ALLINT)); } -inline char* regMaskIntToString(regMaskTP mask, Compiler* context) +inline char* regMaskIntToString(regMaskGpr mask, Compiler* context) { const size_t cchRegMask = 24; char* regmask = new (context, CMK_Unknown) char[cchRegMask]; From 147891cefc2576a4bfb76105832d28d118638fac Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 09:57:33 -0800 Subject: [PATCH 018/201] Update emit.cpp and related code --- src/coreclr/jit/emit.cpp | 12 ++++++------ src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/target.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index fdc29c9a4a3af5..98ceee18a1fcf8 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3476,7 +3476,7 @@ const char* emitter::emitGetFrameReg() * Display a register set in a readable form. */ -void emitter::emitDispRegSet(regMaskTP regs) +void emitter::emitDispRegSet(regMaskAny regs) { regNumber reg; bool sp = false; @@ -3490,7 +3490,7 @@ void emitter::emitDispRegSet(regMaskTP regs) break; } - regMaskTP curReg = genRegMask(reg); + regMaskAny curReg = genRegMask(reg); if ((regs & curReg) == 0) { continue; @@ -9255,7 +9255,7 @@ void emitter::emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr) assert(needsGC(gcType)); - regMaskTP regMask = genRegMask(reg); + regMaskAny regMask = genRegMask(reg); regMaskGpr& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; regMaskGpr& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; @@ -9357,7 +9357,7 @@ void emitter::emitGCregDeadUpd(regNumber reg, BYTE* addr) return; } - regMaskTP regMask = genRegMask(reg); + regMaskAny regMask = genRegMask(reg); if ((emitThisGCrefRegs & regMask) != 0) { @@ -10056,7 +10056,7 @@ void emitter::emitStackPopLargeStk(BYTE* addr, bool isCall, unsigned char callIn // of callee-saved registers only). for (unsigned calleeSavedRegIdx = 0; calleeSavedRegIdx < CNT_CALLEE_SAVED; calleeSavedRegIdx++) { - regMaskTP calleeSavedRbm = raRbmCalleeSaveOrder[calleeSavedRegIdx]; + regMaskGpr calleeSavedRbm = raRbmCalleeSaveOrder[calleeSavedRegIdx]; if (emitThisGCrefRegs & calleeSavedRbm) { gcrefRegs |= (1 << calleeSavedRegIdx); @@ -10423,7 +10423,7 @@ regMaskAny emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) regMaskAny emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); - regMaskTP result; + regMaskAny result; switch (helper) { case CORINFO_HELP_ASSIGN_REF: diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index b87533c0e4e8f3..6bda1dad3cb8f4 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -936,7 +936,7 @@ regNumber genRegArgNext(regNumber argReg) */ const regNumber raRegCalleeSaveOrder[] = {REG_CALLEE_SAVED_ORDER}; -const regMaskTP raRbmCalleeSaveOrder[] = {RBM_CALLEE_SAVED_ORDER}; +const regMaskGpr raRbmCalleeSaveOrder[] = {RBM_CALLEE_SAVED_ORDER}; regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) { diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 9f07b18916adef..79719820e14a20 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -679,7 +679,7 @@ inline regMaskTP genRegMask(regNumber regNum, var_types type) * the current architecture. */ extern const regNumber raRegCalleeSaveOrder[CNT_CALLEE_SAVED]; -extern const regMaskTP raRbmCalleeSaveOrder[CNT_CALLEE_SAVED]; +extern const regMaskGpr raRbmCalleeSaveOrder[CNT_CALLEE_SAVED]; // This method takes a "compact" bitset of the callee-saved registers, and "expands" it to a full register mask. regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short); From 2238e41067b11b2eb878cb5810f5385ca7ea7d91 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 09:59:01 -0800 Subject: [PATCH 019/201] Update emit.h and related code --- src/coreclr/jit/emit.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index e70b1a3783ac45..0e0d7b8bf76740 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -57,7 +57,7 @@ void emitterStats(FILE* fout); void emitterStaticStats(FILE* fout); // Static stats about the emitter (data structure offsets, sizes, etc.) #endif -void printRegMaskInt(regMaskTP mask); +void printRegMaskInt(regMaskGpr mask); /*****************************************************************************/ /* Forward declarations */ @@ -2471,18 +2471,18 @@ class emitter private: #if defined(TARGET_AMD64) - regMaskTP rbmFltCalleeTrash; + regMaskFloat rbmFltCalleeTrash; - FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskFloat get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - regMaskTP rbmMskCalleeTrash; + regMaskPredicate rbmMskCalleeTrash; - FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskPredicate get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } @@ -3197,7 +3197,7 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regMaskTP regs); + void emitDispRegSet(regMaskAny regs); void emitDispVarSet(); #endif From e5e71d11ec07fd9200465785432996913a9d3c1e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 10:01:43 -0800 Subject: [PATCH 020/201] Update emitxarch.cpp and related code --- src/coreclr/jit/emitxarch.cpp | 8 ++++---- src/coreclr/jit/gcencode.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index c988e871910bc8..c2d53fa0181e1d 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -14663,7 +14663,7 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) case IF_RRW: { #ifdef DEBUG - regMaskTP regMask = genRegMask(reg); + regMaskAny regMask = genRegMask(reg); #endif if (id->idGCref()) { @@ -15006,7 +15006,7 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) // instruction, if writing a GC ref even through reading a long, will go live here. // These situations typically occur due to unsafe casting, such as with Span. - regMaskTP regMask; + regMaskAny regMask; regMask = genRegMask(reg1) | genRegMask(reg2); // r1/r2 could have been a GCREF as GCREF + int=BYREF @@ -15509,7 +15509,7 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) assert(id->idGCref() == GCT_BYREF); #ifdef DEBUG - regMaskTP regMask; + regMaskGpr regMask; regMask = genRegMask(reg); // FIXNOW review the other places and relax the assert there too @@ -17936,7 +17936,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) // The target of the 3-operand imul is implicitly encoded. Make sure // that we detected the implicit register and cleared its GC-status. - regMaskTP regMask = genRegMask(inst3opImulReg(ins)); + regMaskAny regMask = genRegMask(inst3opImulReg(ins)); assert((regMask & (emitThisGCrefRegs | emitThisByrefRegs)) == 0); } diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 96005e6057662d..25faa816b5bd69 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -4628,7 +4628,7 @@ void GCInfo::gcInfoRecordGCRegStateChange(GcInfoEncoder* gcInfoEncoder, while (regMask) { // Get hold of the next register bit. - regMaskTP tmpMask = genFindLowestBit(regMask); + regMaskGpr tmpMask = genFindLowestBit(regMask); assert(tmpMask); // Remember the new state of this register. From 7e15435ff0417af8987ae6e6153b2434bbae40f4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:10:50 -0800 Subject: [PATCH 021/201] Update gentree.cpp and related code --- src/coreclr/jit/gentree.cpp | 26 +++++++++++++------------- src/coreclr/jit/gentree.h | 12 ++++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 54c839be5989c1..82aca4b00a59f1 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1059,14 +1059,14 @@ bool GenTree::NeedsConsecutiveRegisters() const // Return Value: // Reg Mask of GenTree node. // -regMaskTP GenTree::gtGetContainedRegMask() +regMaskAny GenTree::gtGetContainedRegMask() { if (!isContained()) { return isUsedFromReg() ? gtGetRegMask() : RBM_NONE; } - regMaskTP mask = 0; + regMaskAny mask = 0; for (GenTree* operand : Operands()) { mask |= operand->gtGetContainedRegMask(); @@ -1083,9 +1083,9 @@ regMaskTP GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -regMaskTP GenTree::gtGetRegMask() const +regMaskAny GenTree::gtGetRegMask() const { - regMaskTP resultMask; + regMaskAny resultMask; if (IsMultiRegCall()) { @@ -2085,9 +2085,9 @@ void CallArgs::Remove(CallArg* arg) // Return Value: // Reg mask of gtOtherRegs of call node. // -regMaskTP GenTreeCall::GetOtherRegMask() const +regMaskAny GenTreeCall::GetOtherRegMask() const { - regMaskTP resultMask = RBM_NONE; + regMaskAny resultMask = RBM_NONE; #if FEATURE_MULTIREG_RET for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i) @@ -27067,9 +27067,9 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx) const // of return registers and wants to know the set of return registers. // // static -regMaskTP ReturnTypeDesc::GetABIReturnRegs() const +regMaskAny ReturnTypeDesc::GetABIReturnRegs() const { - regMaskTP resultMask = RBM_NONE; + regMaskAny resultMask = RBM_NONE; unsigned count = GetReturnRegCount(); for (unsigned i = 0; i < count; ++i) @@ -27094,7 +27094,7 @@ regMaskTP ReturnTypeDesc::GetABIReturnRegs() const // Return Value: // Count of available temporary registers in given set. // -unsigned GenTree::AvailableTempRegCount(regMaskTP mask /* = (regMaskTP)-1 */) const +unsigned GenTree::AvailableTempRegCount(regMaskAny mask /* = (regMaskAny)-1 */) const { return genCountBits(gtRsvdRegs & mask); } @@ -27111,9 +27111,9 @@ unsigned GenTree::AvailableTempRegCount(regMaskTP mask /* = (regMaskTP)-1 */) co // Return Value: // Available temporary register in given mask. // -regNumber GenTree::GetSingleTempReg(regMaskTP mask /* = (regMaskTP)-1 */) +regNumber GenTree::GetSingleTempReg(regMaskAny mask /* = (regMaskAny)-1 */) { - regMaskTP availableSet = gtRsvdRegs & mask; + regMaskAny availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); regNumber tempReg = genRegNumFromMask(availableSet); INDEBUG(gtRsvdRegs &= ~availableSet;) // Remove the register from the set, so it can't be used again. @@ -27132,9 +27132,9 @@ regNumber GenTree::GetSingleTempReg(regMaskTP mask /* = (regMaskTP)-1 */) // Return Value: // Available temporary register in given mask. // -regNumber GenTree::ExtractTempReg(regMaskTP mask /* = (regMaskTP)-1 */) +regNumber GenTree::ExtractTempReg(regMaskAny mask /* = (regMaskAny)-1 */) { - regMaskTP availableSet = gtRsvdRegs & mask; + regMaskAny availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); regNumber tempReg = genFirstRegNumFromMask(availableSet); gtRsvdRegs ^= genRegMask(tempReg); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5a192e68a7e54a..63b6d98fee0928 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -958,7 +958,7 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; regMaskTP gtGetRegMask() const; - regMaskTP gtGetContainedRegMask(); + regMaskAny gtGetContainedRegMask(); GenTreeFlags gtFlags; @@ -970,9 +970,9 @@ struct GenTree regMaskSmall gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskTP mask = (regMaskTP)-1) const; - regNumber GetSingleTempReg(regMaskTP mask = (regMaskTP)-1); - regNumber ExtractTempReg(regMaskTP mask = (regMaskTP)-1); + unsigned AvailableTempRegCount(regMaskAny mask = (regMaskAny)-1) const; + regNumber GetSingleTempReg(regMaskAny mask = (regMaskAny)-1); + regNumber ExtractTempReg(regMaskAny mask = (regMaskAny)-1); void SetVNsFromNode(GenTree* tree) { @@ -4319,7 +4319,7 @@ struct ReturnTypeDesc regNumber GetABIReturnReg(unsigned idx) const; // Get reg mask of ABI return registers - regMaskTP GetABIReturnRegs() const; + regMaskAny GetABIReturnRegs() const; }; class TailCallSiteInfo @@ -5124,7 +5124,7 @@ struct GenTreeCall final : public GenTree } // Get reg mask of all the valid registers of gtOtherRegs array - regMaskTP GetOtherRegMask() const; + regMaskAny GetOtherRegMask() const; GenTreeFlags GetRegSpillFlagByIdx(unsigned idx) const { From 6bbc2d2fcf2f07d0744addada48b9d0e492aa73a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:11:32 -0800 Subject: [PATCH 022/201] Update gentree.h and related code --- src/coreclr/jit/gentree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 63b6d98fee0928..7306b045010384 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -957,7 +957,7 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; - regMaskTP gtGetRegMask() const; + regMaskAny gtGetRegMask() const; regMaskAny gtGetContainedRegMask(); GenTreeFlags gtFlags; From 3e3d51a9d0c719961d536e6a78199d69e7efead6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:14:46 -0800 Subject: [PATCH 023/201] Update lclvars.cpp and related code --- src/coreclr/jit/lclvars.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 5fd3001a48a471..29b6f222832887 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -595,7 +595,7 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un } #ifdef TARGET_ARM - regMaskTP doubleAlignMask = RBM_NONE; + regMaskGpr doubleAlignMask = RBM_NONE; #endif // TARGET_ARM // Skip skipArgs arguments from the signature. @@ -770,7 +770,7 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un { break; } - regMaskTP regMask = genMapArgNumToRegMask(varDscInfo->regArgNum(TYP_INT) + ix, TYP_INT); + regMaskGpr regMask = genMapArgNumToRegMask(varDscInfo->regArgNum(TYP_INT) + ix, TYP_INT); if (cAlign == 2) { doubleAlignMask |= regMask; @@ -5556,8 +5556,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() unsigned argLcls = 0; // Take care of pre spill registers first. - regMaskTP preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); - regMaskTP tempMask = RBM_NONE; + regMaskAny preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); + regMaskAny tempMask = RBM_NONE; for (unsigned i = 0, preSpillLclNum = lclNum; i < argSigLen; ++i, ++preSpillLclNum) { if (lvaIsPreSpilled(preSpillLclNum, preSpillMask)) @@ -5785,7 +5785,7 @@ int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, // On ARM we spill the registers in codeGen->regSet.rsMaskPreSpillRegArg // in the prolog, so we have to do SetStackOffset() here // - regMaskTP regMask = genRegMask(varDsc->GetArgReg()); + regMaskAny regMask = genRegMask(varDsc->GetArgReg()); if (codeGen->regSet.rsMaskPreSpillRegArg & regMask) { // Signature: void foo(struct_8, int, struct_4) From 41959f22e2d174dad887d77bbc36c3c2b0f5a236 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:16:23 -0800 Subject: [PATCH 024/201] Update morph.cpp and related code --- src/coreclr/jit/morph.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 80a91daab4edaf..8e162962a789e5 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -1969,8 +1969,8 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call bool callIsVararg = IsVarArgs(); #ifdef TARGET_ARM - regMaskTP argSkippedRegMask = RBM_NONE; - regMaskTP fltArgSkippedRegMask = RBM_NONE; + regMaskGpr argSkippedRegMask = RBM_NONE; + regMaskFloat fltArgSkippedRegMask = RBM_NONE; #endif // TARGET_ARM #if defined(TARGET_X86) @@ -2547,7 +2547,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call { // Back-fill the register. isBackFilled = true; - regMaskTP backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); + regMaskFloat backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); fltArgSkippedRegMask &= ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask nextFltArgRegNum = genMapFloatRegNumToRegArgNum(genRegNumFromMask(backFillBitMask)); From 4b72781fbc9c04c4b79654683df6228807abf80d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:21:37 -0800 Subject: [PATCH 025/201] Update regalloc.cpp and related code --- src/coreclr/jit/regalloc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/regalloc.cpp b/src/coreclr/jit/regalloc.cpp index fa2ef10764a5a1..85aef232603adc 100644 --- a/src/coreclr/jit/regalloc.cpp +++ b/src/coreclr/jit/regalloc.cpp @@ -100,7 +100,7 @@ bool Compiler::shouldDoubleAlign( regNumber Compiler::raUpdateRegStateForArg(RegState* regState, LclVarDsc* argDsc) { regNumber inArgReg = argDsc->GetArgReg(); - regMaskTP inArgMask = genRegMask(inArgReg); + regMaskAny inArgMask = genRegMask(inArgReg); if (regState->rsIsFloat) { From 7214d7bf4826504ae7e7df3f928c537cedd130da Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:22:36 -0800 Subject: [PATCH 026/201] Update registerargconvention.h and related code --- src/coreclr/jit/registerargconvention.cpp | 2 +- src/coreclr/jit/registerargconvention.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/registerargconvention.cpp b/src/coreclr/jit/registerargconvention.cpp index f58388a39672c1..5321483f2739a1 100644 --- a/src/coreclr/jit/registerargconvention.cpp +++ b/src/coreclr/jit/registerargconvention.cpp @@ -28,7 +28,7 @@ unsigned InitVarDscInfo::allocRegArg(var_types type, unsigned numRegs /* = 1 */) // can't create a > 1 register alignment hole to back-fill. // Back-fill the register - regMaskTP backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); + regMaskFloat backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); fltArgSkippedRegMask &= ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask resultArgNum = genMapFloatRegNumToRegArgNum(genRegNumFromMask(backFillBitMask)); assert(resultArgNum < MAX_FLOAT_REG_ARG); diff --git a/src/coreclr/jit/registerargconvention.h b/src/coreclr/jit/registerargconvention.h index 858efdc9d22c6f..18bcfc85b476f5 100644 --- a/src/coreclr/jit/registerargconvention.h +++ b/src/coreclr/jit/registerargconvention.h @@ -21,7 +21,7 @@ struct InitVarDscInfo #ifdef TARGET_ARM // Support back-filling of FP parameters. This is similar to code in gtMorphArgs() that // handles arguments. - regMaskTP fltArgSkippedRegMask; + regMaskFloat fltArgSkippedRegMask; bool anyFloatStackArgs; #endif // TARGET_ARM From c7e913e0bdacfa840d5cfbb3a31e164f44442653 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:47:12 -0800 Subject: [PATCH 027/201] Update regset.cpp and related code --- src/coreclr/jit/regset.cpp | 20 +++++++------------- src/coreclr/jit/regset.h | 16 ++++++++-------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 6bda1dad3cb8f4..8ccb74defe8378 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -87,7 +87,7 @@ void RegSet::verifyRegUsed(regNumber reg) // should simply validate that the register (or registers) have // already been added to the modified set. -void RegSet::verifyRegistersUsed(regMaskTP regMask) +void RegSet::verifyRegistersUsed(regMaskAny regMask) { if (m_rsCompiler->opts.OptimizationDisabled()) { @@ -119,7 +119,7 @@ void RegSet::rsClearRegsModified() rsModifiedRegsMask = RBM_NONE; } -void RegSet::rsSetRegsModified(regMaskTP mask DEBUGARG(bool suppressDump)) +void RegSet::rsSetRegsModified(regMaskAny mask DEBUGARG(bool suppressDump)) { assert(mask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); @@ -153,7 +153,7 @@ void RegSet::rsSetRegsModified(regMaskTP mask DEBUGARG(bool suppressDump)) rsModifiedRegsMask |= mask; } -void RegSet::rsRemoveRegsModified(regMaskTP mask) +void RegSet::rsRemoveRegsModified(regMaskGpr mask) { assert(mask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); @@ -187,7 +187,7 @@ void RegSet::rsRemoveRegsModified(regMaskTP mask) rsModifiedRegsMask &= ~mask; } -void RegSet::SetMaskVars(regMaskTP newMaskVars) +void RegSet::SetMaskVars(regMaskAny newMaskVars) { #ifdef DEBUG if (m_rsCompiler->verbose) @@ -203,10 +203,10 @@ void RegSet::SetMaskVars(regMaskTP newMaskVars) m_rsCompiler->GetEmitter()->emitDispRegSet(_rsMaskVars); // deadSet = old - new - regMaskTP deadSet = _rsMaskVars & ~newMaskVars; + regMaskAny deadSet = _rsMaskVars & ~newMaskVars; // bornSet = new - old - regMaskTP bornSet = newMaskVars & ~_rsMaskVars; + regMaskAny bornSet = newMaskVars & ~_rsMaskVars; if (deadSet != RBM_NONE) { @@ -331,17 +331,11 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */) } var_types tempType = RegSet::tmpNormalizeType(treeType); - regMaskTP mask; bool floatSpill = false; if (isFloatRegType(treeType)) { floatSpill = true; - mask = genRegMaskFloat(reg ARM_ARG(treeType)); - } - else - { - mask = genRegMask(reg); } rsNeededSpillReg = true; @@ -943,7 +937,7 @@ regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) regMaskSmall res = 0; for (int i = 0; i < CNT_CALLEE_SAVED; i++) { - if ((calleeSaveMask & ((regMaskTP)1 << i)) != 0) + if ((calleeSaveMask & ((regMaskAny)1 << i)) != 0) { res |= raRbmCalleeSaveOrder[i]; } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 502d03e0d8aee8..a057adfd1650e4 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -83,11 +83,11 @@ class RegSet void rsClearRegsModified(); - void rsSetRegsModified(regMaskTP mask DEBUGARG(bool suppressDump = false)); + void rsSetRegsModified(regMaskAny mask DEBUGARG(bool suppressDump = false)); - void rsRemoveRegsModified(regMaskTP mask); + void rsRemoveRegsModified(regMaskGpr mask); - bool rsRegsModified(regMaskTP mask) const + bool rsRegsModified(regMaskGpr mask) const { assert(rsModifiedRegsMaskInitialized); return (rsModifiedRegsMask & mask) != 0; @@ -95,7 +95,7 @@ class RegSet void verifyRegUsed(regNumber reg); - void verifyRegistersUsed(regMaskTP regMask); + void verifyRegistersUsed(regMaskAny regMask); public: regMaskTP GetMaskVars() const // 'get' property function for rsMaskVars property @@ -103,14 +103,14 @@ class RegSet return _rsMaskVars; } - void SetMaskVars(regMaskTP newMaskVars); // 'put' property function for rsMaskVars property + void SetMaskVars(regMaskAny newMaskVars); // 'put' property function for rsMaskVars property - void AddMaskVars(regMaskTP addMaskVars) // union 'addMaskVars' with the rsMaskVars set + void AddMaskVars(regMaskAny addMaskVars) // union 'addMaskVars' with the rsMaskVars set { SetMaskVars(_rsMaskVars | addMaskVars); } - void RemoveMaskVars(regMaskTP removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) + void RemoveMaskVars(regMaskAny removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) { SetMaskVars(_rsMaskVars & ~removeMaskVars); } @@ -128,7 +128,7 @@ class RegSet #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 public: // TODO-Cleanup: Should be private, but Compiler uses it - regMaskTP rsMaskResvd; // mask of the registers that are reserved for special purposes (typically empty) + regMaskGpr rsMaskResvd; // mask of the registers that are reserved for special purposes (typically empty) public: // The PreSpill masks are used in LclVars.cpp #ifdef TARGET_ARM From d417ad554fab2ae26790083f03e8ca2b0afaa4c7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:50:52 -0800 Subject: [PATCH 028/201] Update regset.h and related code --- src/coreclr/jit/regset.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index a057adfd1650e4..7728dc413567c9 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -98,7 +98,7 @@ class RegSet void verifyRegistersUsed(regMaskAny regMask); public: - regMaskTP GetMaskVars() const // 'get' property function for rsMaskVars property + regMaskAny GetMaskVars() const // 'get' property function for rsMaskVars property { return _rsMaskVars; } @@ -121,7 +121,7 @@ class RegSet } private: - regMaskTP _rsMaskVars; // backing store for rsMaskVars property + regMaskAny _rsMaskVars; // backing store for rsMaskVars property #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskAny rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog @@ -132,9 +132,9 @@ class RegSet public: // The PreSpill masks are used in LclVars.cpp #ifdef TARGET_ARM - regMaskTP rsMaskPreSpillAlign; // Mask of alignment padding added to prespill to keep double aligned args + regMaskGpr rsMaskPreSpillAlign; // Mask of alignment padding added to prespill to keep double aligned args // at aligned stack addresses. - regMaskTP rsMaskPreSpillRegArg; // mask of incoming registers that are spilled at the start of the prolog + regMaskGpr rsMaskPreSpillRegArg; // mask of incoming registers that are spilled at the start of the prolog // This includes registers used to pass a struct (or part of a struct) // and all enregistered user arguments in a varargs call #endif // TARGET_ARM From dc54b20eea83b94cb44b80440c463de25ab5054e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:53:08 -0800 Subject: [PATCH 029/201] Update target.h and related code --- src/coreclr/jit/target.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 79719820e14a20..a598670417a588 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -366,7 +366,7 @@ const char* getRegName(regNumber reg); #ifdef DEBUG const char* getRegNameFloat(regNumber reg, var_types type); -extern void dspRegMask(regMaskTP regMask, size_t minSiz = 0); +extern void dspRegMask(regMaskAny regMask, size_t minSiz = 0); #endif #if CPU_HAS_BYTE_REGS @@ -381,8 +381,8 @@ inline bool isByteReg(regNumber reg) } #endif -inline regMaskTP genRegMask(regNumber reg); -inline regMaskTP genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); +inline regMaskAny genRegMask(regNumber reg); +inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); /***************************************************************************** * Return true if the register number is valid @@ -474,7 +474,7 @@ inline regNumber theFixedRetBuffReg() // theFixedRetBuffMask: // Returns the regNumber to use for the fixed return buffer // -inline regMaskTP theFixedRetBuffMask() +inline regMaskGpr theFixedRetBuffMask() { assert(hasFixedRetBuffReg()); // This predicate should be checked before calling this method #ifdef TARGET_ARM64 @@ -503,7 +503,7 @@ inline unsigned theFixedRetBuffArgNum() // Returns the full mask of all possible integer registers // Note this includes the fixed return buffer register on Arm64 // -inline regMaskTP fullIntArgRegMask() +inline regMaskGpr fullIntArgRegMask() { if (hasFixedRetBuffReg()) { @@ -588,7 +588,7 @@ inline bool floatRegCanHoldType(regNumber reg, var_types type) extern const regMaskSmall regMasks[REG_COUNT]; -inline regMaskTP genRegMask(regNumber reg) +inline regMaskAny genRegMask(regNumber reg) { assert((unsigned)reg < ArrLen(regMasks)); #ifdef TARGET_AMD64 @@ -596,7 +596,7 @@ inline regMaskTP genRegMask(regNumber reg) // (L1 latency on sandy bridge is 4 cycles for [base] and 5 for [base + index*c] ) // the reason this is AMD-only is because the x86 BE will try to get reg masks for REG_STK // and the result needs to be zero. - regMaskTP result = 1ULL << reg; + regMaskAny result = 1ULL << reg; assert(result == regMasks[reg]); return result; #else @@ -652,10 +652,10 @@ inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TY // For registers that are used in pairs, the caller will be handling // each member of the pair separately. // -inline regMaskTP genRegMask(regNumber regNum, var_types type) +inline regMaskAny genRegMask(regNumber regNum, var_types type) { #if defined(TARGET_ARM) - regMaskTP regMask = RBM_NONE; + regMaskAny regMask = RBM_NONE; if (varTypeUsesIntReg(type)) { From 5b8a1e65d166006a27e94c4e85c4ab15015e806c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:53:51 -0800 Subject: [PATCH 030/201] Update targetamd64.h and related code --- src/coreclr/jit/targetamd64.cpp | 8 ++++---- src/coreclr/jit/targetamd64.h | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/targetamd64.cpp b/src/coreclr/jit/targetamd64.cpp index 4ac48cb229fbed..2f08592da840c2 100644 --- a/src/coreclr/jit/targetamd64.cpp +++ b/src/coreclr/jit/targetamd64.cpp @@ -19,14 +19,14 @@ const Target::ArgOrder Target::g_tgtUnmanagedArgOrder = ARG_ORDER_R2L; // clang-format off #ifdef UNIX_AMD64_ABI const regNumber intArgRegs [] = { REG_EDI, REG_ESI, REG_EDX, REG_ECX, REG_R8, REG_R9 }; -const regMaskTP intArgMasks[] = { RBM_EDI, RBM_ESI, RBM_EDX, RBM_ECX, RBM_R8, RBM_R9 }; +const regMaskGpr intArgMasks[] = { RBM_EDI, RBM_ESI, RBM_EDX, RBM_ECX, RBM_R8, RBM_R9 }; const regNumber fltArgRegs [] = { REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7 }; -const regMaskTP fltArgMasks[] = { RBM_XMM0, RBM_XMM1, RBM_XMM2, RBM_XMM3, RBM_XMM4, RBM_XMM5, RBM_XMM6, RBM_XMM7 }; +const regMaskFloat fltArgMasks[] = { RBM_XMM0, RBM_XMM1, RBM_XMM2, RBM_XMM3, RBM_XMM4, RBM_XMM5, RBM_XMM6, RBM_XMM7 }; #else // !UNIX_AMD64_ABI const regNumber intArgRegs [] = { REG_ECX, REG_EDX, REG_R8, REG_R9 }; -const regMaskTP intArgMasks[] = { RBM_ECX, RBM_EDX, RBM_R8, RBM_R9 }; +const regMaskGpr intArgMasks[] = { RBM_ECX, RBM_EDX, RBM_R8, RBM_R9 }; const regNumber fltArgRegs [] = { REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3 }; -const regMaskTP fltArgMasks[] = { RBM_XMM0, RBM_XMM1, RBM_XMM2, RBM_XMM3 }; +const regMaskFloat fltArgMasks[] = { RBM_XMM0, RBM_XMM1, RBM_XMM2, RBM_XMM3 }; #endif // !UNIX_AMD64_ABI // clang-format on diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 4abe71984b57cf..635490feef6bca 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -450,9 +450,9 @@ #define REG_ARG_5 REG_R9 extern const regNumber intArgRegs [MAX_REG_ARG]; - extern const regMaskTP intArgMasks[MAX_REG_ARG]; + extern const regMaskGpr intArgMasks[MAX_REG_ARG]; extern const regNumber fltArgRegs [MAX_FLOAT_REG_ARG]; - extern const regMaskTP fltArgMasks[MAX_FLOAT_REG_ARG]; + extern const regMaskFloat fltArgMasks[MAX_FLOAT_REG_ARG]; #define RBM_ARG_0 RBM_RDI #define RBM_ARG_1 RBM_RSI @@ -473,9 +473,9 @@ #define REG_ARG_3 REG_R9 extern const regNumber intArgRegs [MAX_REG_ARG]; - extern const regMaskTP intArgMasks[MAX_REG_ARG]; + extern const regMaskGpr intArgMasks[MAX_REG_ARG]; extern const regNumber fltArgRegs [MAX_FLOAT_REG_ARG]; - extern const regMaskTP fltArgMasks[MAX_FLOAT_REG_ARG]; + extern const regMaskFloat fltArgMasks[MAX_FLOAT_REG_ARG]; #define RBM_ARG_0 RBM_ECX #define RBM_ARG_1 RBM_EDX From c79e9fe4f4b7a1b26458f849702ed6900e215e84 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:55:30 -0800 Subject: [PATCH 031/201] Update targetarm.cpp and related code --- src/coreclr/jit/targetarm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/targetarm.cpp b/src/coreclr/jit/targetarm.cpp index 8e117bae810270..3fa42f7df32284 100644 --- a/src/coreclr/jit/targetarm.cpp +++ b/src/coreclr/jit/targetarm.cpp @@ -18,10 +18,10 @@ const Target::ArgOrder Target::g_tgtUnmanagedArgOrder = ARG_ORDER_R2L; // clang-format off const regNumber intArgRegs [] = {REG_R0, REG_R1, REG_R2, REG_R3}; -const regMaskTP intArgMasks[] = {RBM_R0, RBM_R1, RBM_R2, RBM_R3}; +const regMaskGpr intArgMasks[] = {RBM_R0, RBM_R1, RBM_R2, RBM_R3}; const regNumber fltArgRegs [] = {REG_F0, REG_F1, REG_F2, REG_F3, REG_F4, REG_F5, REG_F6, REG_F7, REG_F8, REG_F9, REG_F10, REG_F11, REG_F12, REG_F13, REG_F14, REG_F15 }; -const regMaskTP fltArgMasks[] = {RBM_F0, RBM_F1, RBM_F2, RBM_F3, RBM_F4, RBM_F5, RBM_F6, RBM_F7, RBM_F8, RBM_F9, RBM_F10, RBM_F11, RBM_F12, RBM_F13, RBM_F14, RBM_F15 }; +const regMaskFloat fltArgMasks[] = {RBM_F0, RBM_F1, RBM_F2, RBM_F3, RBM_F4, RBM_F5, RBM_F6, RBM_F7, RBM_F8, RBM_F9, RBM_F10, RBM_F11, RBM_F12, RBM_F13, RBM_F14, RBM_F15 }; // clang-format on static_assert_no_msg(RBM_ALLDOUBLE == (RBM_ALLDOUBLE_HIGH >> 1)); From 9171cd8b0a98d5a4e2b5dbde66c1492665bfd16b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:55:53 -0800 Subject: [PATCH 032/201] Update targetarm.h and related code --- src/coreclr/jit/targetarm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index ac9d72cab31f6c..82000c93aa6fb7 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -277,7 +277,7 @@ #define REG_ARG_3 REG_R3 extern const regNumber intArgRegs [MAX_REG_ARG]; - extern const regMaskTP intArgMasks[MAX_REG_ARG]; + extern const regMaskGpr intArgMasks[MAX_REG_ARG]; #define RBM_ARG_0 RBM_R0 #define RBM_ARG_1 RBM_R1 @@ -288,7 +288,7 @@ #define RBM_FLTARG_REGS (RBM_F0|RBM_F1|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7|RBM_F8|RBM_F9|RBM_F10|RBM_F11|RBM_F12|RBM_F13|RBM_F14|RBM_F15) extern const regNumber fltArgRegs [MAX_FLOAT_REG_ARG]; - extern const regMaskTP fltArgMasks[MAX_FLOAT_REG_ARG]; + extern const regMaskFloat fltArgMasks[MAX_FLOAT_REG_ARG]; #define LBL_DIST_SMALL_MAX_NEG (0) #define LBL_DIST_SMALL_MAX_POS (+1020) From 0aeefe10fee39d538cb736144fabe2daf307e18b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:56:15 -0800 Subject: [PATCH 033/201] Update targetarm64.cpp and related code --- src/coreclr/jit/targetarm64.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/targetarm64.cpp b/src/coreclr/jit/targetarm64.cpp index dcec1db6c5229f..87b91b34d8f50e 100644 --- a/src/coreclr/jit/targetarm64.cpp +++ b/src/coreclr/jit/targetarm64.cpp @@ -18,10 +18,10 @@ const Target::ArgOrder Target::g_tgtUnmanagedArgOrder = ARG_ORDER_R2L; // clang-format off const regNumber intArgRegs [] = {REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, REG_R6, REG_R7}; -const regMaskTP intArgMasks[] = {RBM_R0, RBM_R1, RBM_R2, RBM_R3, RBM_R4, RBM_R5, RBM_R6, RBM_R7}; +const regMaskGpr intArgMasks[] = {RBM_R0, RBM_R1, RBM_R2, RBM_R3, RBM_R4, RBM_R5, RBM_R6, RBM_R7}; const regNumber fltArgRegs [] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, REG_V6, REG_V7 }; -const regMaskTP fltArgMasks[] = {RBM_V0, RBM_V1, RBM_V2, RBM_V3, RBM_V4, RBM_V5, RBM_V6, RBM_V7 }; +const regMaskFloat fltArgMasks[] = {RBM_V0, RBM_V1, RBM_V2, RBM_V3, RBM_V4, RBM_V5, RBM_V6, RBM_V7 }; // clang-format on #endif // TARGET_ARM64 From 9ccaabb3c7c58fff5cce26dece066eb807d349e9 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:56:38 -0800 Subject: [PATCH 034/201] Update targetarm64.h and related code --- src/coreclr/jit/targetarm64.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 3646ecb4407bf7..20376e221bf4a3 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -300,7 +300,7 @@ #define REG_ARG_7 REG_R7 extern const regNumber intArgRegs [MAX_REG_ARG]; - extern const regMaskTP intArgMasks[MAX_REG_ARG]; + extern const regMaskGpr intArgMasks[MAX_REG_ARG]; #define RBM_ARG_0 RBM_R0 #define RBM_ARG_1 RBM_R1 @@ -333,7 +333,7 @@ #define RBM_FLTARG_REGS (RBM_FLTARG_0|RBM_FLTARG_1|RBM_FLTARG_2|RBM_FLTARG_3|RBM_FLTARG_4|RBM_FLTARG_5|RBM_FLTARG_6|RBM_FLTARG_7) extern const regNumber fltArgRegs [MAX_FLOAT_REG_ARG]; - extern const regMaskTP fltArgMasks[MAX_FLOAT_REG_ARG]; + extern const regMaskFloat fltArgMasks[MAX_FLOAT_REG_ARG]; #define LBL_DIST_SMALL_MAX_NEG (-1048576) #define LBL_DIST_SMALL_MAX_POS (+1048575) From 806fc031b3290fe247ad52a389b303cd67504a22 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:56:57 -0800 Subject: [PATCH 035/201] Update targetx86.cpp and related code --- src/coreclr/jit/targetx86.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/targetx86.cpp b/src/coreclr/jit/targetx86.cpp index d5ed8b0bbf606f..9fc8577d256f2a 100644 --- a/src/coreclr/jit/targetx86.cpp +++ b/src/coreclr/jit/targetx86.cpp @@ -18,7 +18,7 @@ const Target::ArgOrder Target::g_tgtUnmanagedArgOrder = ARG_ORDER_R2L; // clang-format off const regNumber intArgRegs [] = {REG_ECX, REG_EDX}; -const regMaskTP intArgMasks[] = {RBM_ECX, RBM_EDX}; +const regMaskGpr intArgMasks[] = {RBM_ECX, RBM_EDX}; // clang-format on #endif // TARGET_X86 From 757d8f2618e276d0ff01da32a9ff76658a7f61f4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 11:57:09 -0800 Subject: [PATCH 036/201] Update targetx86.h and related code --- src/coreclr/jit/targetx86.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index 60b2f7793f435b..9287a7b3333ba3 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -320,7 +320,7 @@ #define REG_ARG_1 REG_EDX extern const regNumber intArgRegs [MAX_REG_ARG]; - extern const regMaskTP intArgMasks[MAX_REG_ARG]; + extern const regMaskGpr intArgMasks[MAX_REG_ARG]; #define RBM_ARG_0 RBM_ECX #define RBM_ARG_1 RBM_EDX From e6b33043e7dee16a1c4b707f8ea2bf55a29b5184 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 14:45:26 -0800 Subject: [PATCH 037/201] Update unwindarmarch.cpp and related code --- src/coreclr/jit/unwind.cpp | 6 +++--- src/coreclr/jit/unwindarmarch.cpp | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index ef722504a0f11a..7623b30b13b826 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -152,7 +152,7 @@ void Compiler::unwindPushPopCFI(regNumber reg) FuncInfoDsc* func = funCurrentFunc(); UNATIVE_OFFSET cbProlog = unwindGetCurrentOffset(func); - regMaskTP relOffsetMask = RBM_CALLEE_SAVED + regMaskAny relOffsetMask = RBM_CALLEE_SAVED #if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) // is excluded from the callee-save register list. @@ -207,10 +207,10 @@ void Compiler::unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat) #if TARGET_ARM regNumber regNum = isFloat ? REG_PREV(REG_FP_LAST) : REG_INT_LAST; - regMaskTP regBit = isFloat ? genRegMask(regNum) | genRegMask(REG_NEXT(regNum)) : genRegMask(regNum); + regMaskOnlyOne regBit = isFloat ? genRegMask(regNum) | genRegMask(REG_NEXT(regNum)) : genRegMask(regNum); #else regNumber regNum = isFloat ? REG_FP_LAST : REG_INT_LAST; - regMaskTP regBit = genRegMask(regNum); + regMaskOnlyOne regBit = genRegMask(regNum); #endif for (; regMask != 0 && regBit != RBM_NONE;) diff --git a/src/coreclr/jit/unwindarmarch.cpp b/src/coreclr/jit/unwindarmarch.cpp index 21a2a0b43e453e..377f29e6d6f4bc 100644 --- a/src/coreclr/jit/unwindarmarch.cpp +++ b/src/coreclr/jit/unwindarmarch.cpp @@ -213,8 +213,8 @@ void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) if ((maskInt & (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3)) == 0) { - regMaskTP matchMask = maskInt & (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7); - regMaskTP valMask = RBM_R4; + regMaskGpr matchMask = maskInt & (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7); + regMaskGpr valMask = RBM_R4; while (val < 4) { if (matchMask == valMask) @@ -253,8 +253,8 @@ void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) if (((maskInt & (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3)) == 0) && ((maskInt & (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8)) == (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8))) { - regMaskTP matchMask = maskInt & (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | RBM_R11); - regMaskTP valMask = RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8; + regMaskGpr matchMask = maskInt & (RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | RBM_R11); + regMaskGpr valMask = RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8; while (val < 4) { if (matchMask == valMask) @@ -283,10 +283,10 @@ void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) } } -void Compiler::unwindPushPopMaskFloat(regMaskTP maskFloat) -{ +void Compiler::unwindPushPopMaskFloat(regMaskFloat maskFloat) +{ // Only floating pointer registers can be specified in 'maskFloat' - assert((maskFloat & ~RBM_ALLFLOAT) == 0); + assert(IsFloatRegMask(maskFloat)); // If the maskFloat is zero there is no unwind code to emit // @@ -298,7 +298,7 @@ void Compiler::unwindPushPopMaskFloat(regMaskTP maskFloat) UnwindInfo* pu = &funCurrentFunc()->uwi; BYTE val = 0; - regMaskTP valMask = (RBM_F16 | RBM_F17); + regMaskFloat valMask = (RBM_F16 | RBM_F17); while (maskFloat != valMask) { From 48e2e328bf1f983115719ff5c04aa0d8c60bcbc1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 14:46:26 -0800 Subject: [PATCH 038/201] Update utils.cpp and related code --- src/coreclr/jit/utils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index d17d0431f9ee80..be8a3d2b0db321 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -282,7 +282,7 @@ const char* getRegNameFloat(regNumber reg, var_types type) * Displays a range of registers * -- This is a helper used by dspRegMask */ -const char* dspRegRange(regMaskTP regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) +const char* dspRegRange(regMaskAny regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) { #ifdef TARGET_XARCH assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || @@ -306,7 +306,7 @@ const char* dspRegRange(regMaskTP regMask, size_t& minSiz, const char* sep, regN for (regNumber regNum = regFirst; regNum <= regLast; regNum = REG_NEXT(regNum)) { - regMaskTP regBit = genRegMask(regNum); + regMaskAny regBit = genRegMask(regNum); if ((regMask & regBit) != 0) { @@ -435,7 +435,7 @@ const char* dspRegRange(regMaskTP regMask, size_t& minSiz, const char* sep, regN * Displays a register set. * TODO-ARM64-Cleanup: don't allow ip0, ip1 as part of a range. */ -void dspRegMask(regMaskTP regMask, size_t minSiz) +void dspRegMask(regMaskAny regMask, size_t minSiz) { const char* sep = ""; From 93f87fa9993c3f676a292d5782c637506cefe383 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 17:13:01 -0800 Subject: [PATCH 039/201] Add singleRegMask and some notes in target.h --- src/coreclr/jit/target.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a598670417a588..350d6a65991699 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -209,6 +209,8 @@ enum _regMask_enum : unsigned // In any case, we believe that is OK to freely cast between these types; no information will // be lost. +typedef unsigned __int64 singleRegMask; + #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; typedef unsigned __int64 regMaskGpr; @@ -231,8 +233,10 @@ typedef unsigned __int64 regMaskOnlyOne; // 0. Revisit regMaskAny and see if they should be "regMaskAny" // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together -typedef unsigned __int64 regMaskAny; -// TODO: should have `regMaskSingleReg` to represent genRegMask(regNumber) return value +typedef unsigned __int64 regMaskAny; //TODO: Rename this to regMaskMixed + +// TODO: For LSRA, the select() method should be regMaskOnlyOne because we will be +// allocating either GPR or Vector or Mask but not all #else // x86 and arm typedef unsigned regMaskTP; @@ -588,7 +592,7 @@ inline bool floatRegCanHoldType(regNumber reg, var_types type) extern const regMaskSmall regMasks[REG_COUNT]; -inline regMaskAny genRegMask(regNumber reg) +inline regMaskOnlyOne genRegMask(regNumber reg) { assert((unsigned)reg < ArrLen(regMasks)); #ifdef TARGET_AMD64 @@ -652,7 +656,7 @@ inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TY // For registers that are used in pairs, the caller will be handling // each member of the pair separately. // -inline regMaskAny genRegMask(regNumber regNum, var_types type) +inline regMaskOnlyOne genRegMask(regNumber regNum, var_types type) { #if defined(TARGET_ARM) regMaskAny regMask = RBM_NONE; From f2c82191117d5632108feb6ad6f74308b20884f5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 17:13:19 -0800 Subject: [PATCH 040/201] Update lsra.cpp and related code --- src/coreclr/jit/compiler.h | 7 + src/coreclr/jit/lsra.cpp | 264 +++++++++++++++++----------------- src/coreclr/jit/lsra.h | 91 ++++++------ src/coreclr/jit/lsraarm64.cpp | 63 ++++---- src/coreclr/jit/lsrabuild.cpp | 20 +-- src/coreclr/jit/lsraxarch.cpp | 10 +- 6 files changed, 237 insertions(+), 218 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3f4c32a923f755..28ad3caf242b83 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8453,6 +8453,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX return (regMask & RBM_ALLFLOAT) == regMask; } +#if defined(FEATURE_SIMD) && defined(TARGET_XARCH) + bool IsPredicateRegMask(regMaskTP regMask) + { + return (regMask & RBM_ALLMASK) == regMask; + } +#endif + bool IsOnlyOneRegMask(regMaskTP regMask) { return IsGprRegMask(regMask) != IsFloatRegMask(regMask); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 60c0b800464a05..f43845af8ee523 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -234,13 +234,13 @@ weight_t LinearScan::getWeight(RefPosition* refPos) // allRegs represents a set of registers that can // be used to allocate the specified type in any point // in time (more of a 'bank' of registers). -regMaskTP LinearScan::allRegs(RegisterType rt) +regMaskOnlyOne LinearScan::allRegs(RegisterType rt) { assert((rt != TYP_UNDEF) && (rt != TYP_STRUCT)); return *availableRegs[rt]; } -regMaskTP LinearScan::allByteRegs() +regMaskGpr LinearScan::allByteRegs() { #ifdef TARGET_X86 return availableIntRegs & RBM_BYTE_REGS; @@ -249,7 +249,7 @@ regMaskTP LinearScan::allByteRegs() #endif } -regMaskTP LinearScan::allSIMDRegs() +regMaskFloat LinearScan::allSIMDRegs() { return availableFloatRegs; } @@ -262,7 +262,7 @@ regMaskTP LinearScan::allSIMDRegs() // Return Value: // Register mask of the SSE/VEX-only SIMD registers // -regMaskTP LinearScan::lowSIMDRegs() +regMaskFloat LinearScan::lowSIMDRegs() { #if defined(TARGET_AMD64) return (availableFloatRegs & RBM_LOWFLOAT); @@ -288,15 +288,15 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo nextFixedRef[regRecord->regNum] = nextLocation; } -regMaskTP LinearScan::getMatchingConstants(regMaskTP mask, Interval* currentInterval, RefPosition* refPosition) +regMaskOnlyOne LinearScan::getMatchingConstants(regMaskAny mask, Interval* currentInterval, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); - regMaskTP candidates = (mask & m_RegistersWithConstants); - regMaskTP result = RBM_NONE; + regMaskAny candidates = (mask & m_RegistersWithConstants); + regMaskAny result = RBM_NONE; while (candidates != RBM_NONE) { regNumber regNum = genFirstRegNumFromMask(candidates); - regMaskTP candidateBit = genRegMask(regNum); + regMaskAny candidateBit = genRegMask(regNum); candidates ^= candidateBit; RegRecord* physRegRecord = getRegisterRecord(regNum); @@ -385,11 +385,13 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) // assignedReg - Assigned register for this refposition. // void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, - regMaskTP regsBusy, - regMaskTP* regsToFree, - regMaskTP* delayRegsToFree DEBUG_ARG(Interval* interval) + regMaskOnlyOne regsBusy, + regMaskOnlyOne* regsToFree, + regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { + assert(compiler->IsOnlyOneRegMask(regsBusy)); + regsInUseThisLocation |= regsBusy; if (refPosition.lastUse) { @@ -416,7 +418,7 @@ void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, // for use as internal float registers. // // Return Value: -// The set of registers (as a regMaskTP). +// The set of registers (as a regMaskFloat). // // Notes: // compFloatingPointUsed is only required to be set if it is possible that we @@ -425,7 +427,7 @@ void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, // that it will select a callee-save register. But to be safe, we restrict // the set of candidates if compFloatingPointUsed is not already set. // -regMaskTP LinearScan::internalFloatRegCandidates() +regMaskFloat LinearScan::internalFloatRegCandidates() { needNonIntegerRegisters = true; @@ -470,12 +472,12 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // New regMask that has minRegCount registers after intersection. // Otherwise returns regMaskActual. // -regMaskTP LinearScan::getConstrainedRegMask(RefPosition* refPosition, - regMaskTP regMaskActual, - regMaskTP regMaskConstraint, +regMaskAny LinearScan::getConstrainedRegMask(RefPosition* refPosition, + regMaskAny regMaskActual, + regMaskAny regMaskConstraint, unsigned minRegCount) { - regMaskTP newMask = regMaskActual & regMaskConstraint; + regMaskAny newMask = regMaskActual & regMaskConstraint; if (genCountBits(newMask) < minRegCount) { // Constrained mask does not have minimum required registers needed. @@ -484,7 +486,7 @@ regMaskTP LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { - regMaskTP busyRegs = regsBusyUntilKill | regsInUseThisLocation; + regMaskAny busyRegs = regsBusyUntilKill | regsInUseThisLocation; if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -510,7 +512,7 @@ regMaskTP LinearScan::getConstrainedRegMask(RefPosition* refPosition, // This is the method used to implement the stress options that limit // the set of registers considered for allocation. // -regMaskTP LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskTP mask) +regMaskAny LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskAny mask) { #ifdef TARGET_ARM64 if ((refPosition != nullptr) && refPosition->isLiveAtConsecutiveRegistersLoc(consecutiveRegistersLocation)) @@ -727,7 +729,7 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_XARCH) rbmAllMask = compiler->rbmAllMask; rbmMskCalleeTrash = compiler->rbmMskCalleeTrash; - memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskTP) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskAny) * TYP_COUNT); if (!compiler->canUseEvexEncoding()) { @@ -2733,7 +2735,7 @@ void LinearScan::setFrameType() // If we are using FPBASE as the frame register, we cannot also use it for // a local var. - regMaskTP removeMask = RBM_NONE; + regMaskAny removeMask = RBM_NONE; if (frameType == FT_EBP_FRAME) { removeMask |= RBM_FPBASE; @@ -2816,7 +2818,7 @@ RegisterType LinearScan::getRegisterType(Interval* currentInterval, RefPosition* { assert(refPosition->getInterval() == currentInterval); RegisterType regType = currentInterval->registerType; - regMaskTP candidates = refPosition->registerAssignment; + regMaskAny candidates = refPosition->registerAssignment; #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // The LoongArch64's ABI which the float args maybe passed by integer register // when no float register left but free integer register. @@ -2933,7 +2935,7 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, { assert(!enregisterLocalVars); regNumber foundReg; - regMaskTP foundRegBit; + regMaskAny foundRegBit; RegRecord* availablePhysRegRecord; foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) @@ -2994,7 +2996,7 @@ template regNumber LinearScan::allocateReg(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { - regMaskTP foundRegBit = + regMaskAny foundRegBit = regSelector->select(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) { @@ -3231,7 +3233,7 @@ bool LinearScan::isRefPositionActive(RefPosition* refPosition, LsraLocation refL // bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord) { - regMaskTP candidateBit = genRegMask(physRegRecord->regNum); + regMaskAny candidateBit = genRegMask(physRegRecord->regNum); LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. @@ -3449,7 +3451,7 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - regMaskTP assignedRegMask = genRegMask(regRec->regNum); + regMaskAny assignedRegMask = genRegMask(regRec->regNum); compiler->codeGen->regSet.rsSetRegsModified(assignedRegMask DEBUGARG(true)); interval->assignedReg = regRec; @@ -3971,7 +3973,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { // For each physical register that can hold a GC type, // if it is occupied by an interval of a GC type, spill that interval. - regMaskTP candidateRegs = killRefPosition->registerAssignment; + regMaskAny candidateRegs = killRefPosition->registerAssignment; INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { @@ -4061,13 +4063,13 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // The new register to use. // #ifdef DEBUG -regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskTP availableRegs) +regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskAny availableRegs) { if (targetReg != REG_STK && getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE) { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs; + regMaskAny candidateRegs = allRegs(interval->registerType) & availableRegs; regNumber firstReg = REG_NA; regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) @@ -4371,9 +4373,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } // If we are rotating register assignments at block boundaries, we want to make the // inactive registers available for the rotation. - regMaskTP inactiveRegs = RBM_NONE; + regMaskAny inactiveRegs = RBM_NONE; #endif // DEBUG - regMaskTP liveRegs = RBM_NONE; + regMaskAny liveRegs = RBM_NONE; VarSetOps::Iter iter(compiler, currentLiveVars); unsigned varIndex = 0; while (iter.NextElem(&varIndex)) @@ -4631,7 +4633,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } } #else - regMaskTP deadCandidates = ~liveRegs; + regMaskAny deadCandidates = ~liveRegs; // Only focus on actual registers present deadCandidates &= actualRegistersMask; @@ -4823,7 +4825,7 @@ void LinearScan::freeRegister(RegRecord* physRegRecord) // Arguments: // regsToFree - the mask of registers to free // -void LinearScan::freeRegisters(regMaskTP regsToFree) +void LinearScan::freeRegisters(regMaskAny regsToFree) { if (regsToFree == RBM_NONE) { @@ -4898,11 +4900,11 @@ void LinearScan::allocateRegistersMinimal() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskTP regsToFree = RBM_NONE; - regMaskTP delayRegsToFree = RBM_NONE; - regMaskTP regsToMakeInactive = RBM_NONE; - regMaskTP delayRegsToMakeInactive = RBM_NONE; - regMaskTP copyRegsToFree = RBM_NONE; + regMaskAny regsToFree = RBM_NONE; + regMaskAny delayRegsToFree = RBM_NONE; + regMaskAny regsToMakeInactive = RBM_NONE; + regMaskAny delayRegsToMakeInactive = RBM_NONE; + regMaskAny copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -4920,7 +4922,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskTP tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskAny tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -5170,7 +5172,7 @@ void LinearScan::allocateRegistersMinimal() INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_RELOAD, currentInterval, assignedRegister)); } - regMaskTP assignedRegBit = RBM_NONE; + singleRegMask assignedRegBit = RBM_NONE; bool isInRegister = false; if (assignedRegister != REG_NA) { @@ -5236,8 +5238,8 @@ void LinearScan::allocateRegistersMinimal() regNumber copyReg = assignCopyRegMinimal(¤tRefPosition); lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); // For consecutive register, although it shouldn't matter what the assigned register was, // because we have just assigned it `copyReg` and that's the one in-use, and not the @@ -5361,7 +5363,7 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskAny regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -5579,11 +5581,11 @@ void LinearScan::allocateRegisters() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskTP regsToFree = RBM_NONE; - regMaskTP delayRegsToFree = RBM_NONE; - regMaskTP regsToMakeInactive = RBM_NONE; - regMaskTP delayRegsToMakeInactive = RBM_NONE; - regMaskTP copyRegsToFree = RBM_NONE; + regMaskAny regsToFree = RBM_NONE; + regMaskAny delayRegsToFree = RBM_NONE; + regMaskAny regsToMakeInactive = RBM_NONE; + regMaskAny delayRegsToMakeInactive = RBM_NONE; + regMaskAny copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -5601,7 +5603,7 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskTP tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskAny tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -6048,7 +6050,7 @@ void LinearScan::allocateRegisters() INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_RELOAD, currentInterval, assignedRegister)); } - regMaskTP assignedRegBit = RBM_NONE; + singleRegMask assignedRegBit = RBM_NONE; bool isInRegister = false; if (assignedRegister != REG_NA) { @@ -6095,7 +6097,7 @@ void LinearScan::allocateRegisters() // it might be beneficial to keep it in this reg for PART of the lifetime if (currentInterval->isLocalVar) { - regMaskTP preferences = currentInterval->registerPreferences; + regMaskAny preferences = currentInterval->registerPreferences; bool keepAssignment = true; bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; @@ -6204,8 +6206,8 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskAny copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskAny assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -6302,8 +6304,8 @@ void LinearScan::allocateRegisters() } lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskAny copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskAny assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.needsConsecutive) @@ -6525,7 +6527,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskAny regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -8212,7 +8214,7 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { - regMaskTP initialRegMask = interval->firstRefPosition->registerAssignment; + regMaskAny initialRegMask = interval->firstRefPosition->registerAssignment; regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK : genRegNumFromMask(initialRegMask); @@ -8301,7 +8303,7 @@ void LinearScan::resolveRegisters() varDsc->lvOnFrame = false; } #ifdef DEBUG - regMaskTP registerAssignment = genRegMask(varDsc->GetRegNum()); + singleRegMask registerAssignment = genRegMask(varDsc->GetRegNum()); assert(!interval->isSpilled && !interval->isSplit); RefPosition* refPosition = interval->firstRefPosition; assert(refPosition != nullptr); @@ -8540,7 +8542,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskTP terminatorConsumedRegs) + regMaskAny terminatorConsumedRegs) { // TODO-Throughput: This would be much more efficient if we add RegToVarMaps instead of VarToRegMaps // and they would be more space-efficient as well. @@ -8548,7 +8550,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, VarToRegMap toVarToRegMap = toBlock == nullptr ? nullptr : getInVarToRegMap(toBlock->bbNum); #ifdef TARGET_ARM - regMaskTP freeRegs; + regMaskOnlyOne freeRegs; if (type == TYP_DOUBLE) { // We have to consider all float registers for TYP_DOUBLE @@ -8559,7 +8561,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, freeRegs = allRegs(type); } #else // !TARGET_ARM - regMaskTP freeRegs = allRegs(type); + regMaskOnlyOne freeRegs = allRegs(type); #endif // !TARGET_ARM #ifdef DEBUG @@ -8826,7 +8828,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // available to copy into. // Note that for this purpose we use the full live-out set, because we must ensure that // even the registers that remain the same across the edge are preserved correctly. - regMaskTP liveOutRegs = RBM_NONE; + regMaskAny liveOutRegs = RBM_NONE; VarSetOps::Iter liveOutIter(compiler, block->bbLiveOut); unsigned liveOutVarIndex = 0; while (liveOutIter.NextElem(&liveOutVarIndex)) @@ -8834,7 +8836,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) regNumber fromReg = getVarReg(outVarToRegMap, liveOutVarIndex); if (fromReg != REG_STK) { - regMaskTP fromRegMask = genRegMask(fromReg, getIntervalForLocalVar(liveOutVarIndex)->registerType); + regMaskOnlyOne fromRegMask = genRegMask(fromReg, getIntervalForLocalVar(liveOutVarIndex)->registerType); liveOutRegs |= fromRegMask; } } @@ -8846,7 +8848,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // // Note: Only switches and JCMP/JTEST (for Arm4) have input regs (and so can be fed by copies), so those // are the only block-ending branches that need special handling. - regMaskTP consumedRegs = RBM_NONE; + regMaskAny consumedRegs = RBM_NONE; if (block->KindIs(BBJ_SWITCH)) { // At this point, Lowering has transformed any non-switch-table blocks into @@ -8926,8 +8928,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) } VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; - regMaskTP sameWriteRegs = RBM_NONE; - regMaskTP diffReadRegs = RBM_NONE; + regMaskAny sameWriteRegs = RBM_NONE; + regMaskAny diffReadRegs = RBM_NONE; // For each var that may require resolution, classify them as: // - in the same register at the end of this block and at each target (no resolution needed) @@ -8983,7 +8985,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // In this case, sameToReg will be in the liveOutRegs of this block. // Similarly, if sameToReg is in sameWriteRegs, it has already been used (i.e. for a lclVar that's // live only at another target), and we can't copy another lclVar into that reg in this block. - regMaskTP sameToRegMask = + regMaskOnlyOne sameToRegMask = genRegMask(sameToReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); if (maybeSameLivePaths && (((sameToRegMask & liveOutRegs) != RBM_NONE) || ((sameToRegMask & sameWriteRegs) != RBM_NONE))) @@ -9361,7 +9363,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskTP terminatorConsumedRegs) + regMaskAny terminatorConsumedRegs) { VarToRegMap fromVarToRegMap = getOutVarToRegMap(fromBlock->bbNum); VarToRegMap toVarToRegMap; @@ -9429,9 +9431,9 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } } - regMaskTP targetRegsToDo = RBM_NONE; - regMaskTP targetRegsReady = RBM_NONE; - regMaskTP targetRegsFromStack = RBM_NONE; + regMaskAny targetRegsToDo = RBM_NONE; + regMaskAny targetRegsReady = RBM_NONE; + regMaskAny targetRegsFromStack = RBM_NONE; // The following arrays capture the location of the registers as they are moved: // - location[reg] gives the current location of the var that was originally in 'reg'. @@ -9559,11 +9561,11 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // REGISTER to REGISTER MOVES // First, find all the ones that are ready to move now - regMaskTP targetCandidates = targetRegsToDo; + regMaskAny targetCandidates = targetRegsToDo; while (targetCandidates != RBM_NONE) { regNumber targetReg = genFirstRegNumFromMask(targetCandidates); - regMaskTP targetRegMask = genRegMask(targetReg); + singleRegMask targetRegMask = genRegMask(targetReg); targetCandidates ^= targetRegMask; if (location[targetReg] == REG_NA) { @@ -9594,7 +9596,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, while (targetRegsReady != RBM_NONE) { regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); - regMaskTP targetRegMask = genRegMask(targetReg); + singleRegMask targetRegMask = genRegMask(targetReg); targetRegsToDo ^= targetRegMask; targetRegsReady ^= targetRegMask; assert(location[targetReg] != targetReg); @@ -9610,7 +9612,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, fromReg DEBUG_ARG(fromBlock) DEBUG_ARG(toBlock) DEBUG_ARG(resolveTypeName[resolveType])); sourceIntervals[sourceReg] = nullptr; location[sourceReg] = REG_NA; - regMaskTP fromRegMask = genRegMask(fromReg); + singleRegMask fromRegMask = genRegMask(fromReg); // Do we have a free targetReg? if (fromReg == sourceReg) @@ -9639,7 +9641,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber lowerHalfReg = REG_PREV(fromReg); regNumber lowerHalfSrcReg = (regNumber)source[lowerHalfReg]; regNumber lowerHalfSrcLoc = (regNumber)location[lowerHalfReg]; - regMaskTP lowerHalfRegMask = genRegMask(lowerHalfReg); + singleRegMask lowerHalfRegMask = genRegMask(lowerHalfReg); // Necessary conditions: // - There is a source register for this reg (lowerHalfSrcReg != REG_NA) // - It is currently free (lowerHalfSrcLoc == REG_NA) @@ -9665,7 +9667,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, if (targetRegsToDo != RBM_NONE) { regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); - regMaskTP targetRegMask = genRegMask(targetReg); + singleRegMask targetRegMask = genRegMask(targetReg); // Is it already there due to other moves? // If not, move it to the temp reg, OR swap it with another register @@ -9716,7 +9718,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Otherwise, we'll spill it to the stack and reload it later. if (useSwap) { - regMaskTP fromRegMask = genRegMask(fromReg); + singleRegMask fromRegMask = genRegMask(fromReg); targetRegsToDo &= ~fromRegMask; } } @@ -9724,7 +9726,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { // Look at the remaining registers from targetRegsToDo (which we expect to be relatively // small at this point) to find out what's currently in targetReg. - regMaskTP mask = targetRegsToDo; + regMaskAny mask = targetRegsToDo; while (mask != RBM_NONE && otherTargetReg == REG_NA) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(mask); @@ -9761,7 +9763,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, DEBUG_ARG(resolveTypeName[resolveType])); location[source[otherTargetReg]] = REG_STK; - regMaskTP otherTargetRegMask = genRegMask(otherTargetReg); + singleRegMask otherTargetRegMask = genRegMask(otherTargetReg); targetRegsFromStack |= otherTargetRegMask; stackToRegIntervals[otherTargetReg] = otherInterval; targetRegsToDo &= ~otherTargetRegMask; @@ -9778,7 +9780,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // value will be retrieved from STK. if (source[fromReg] != REG_NA && fromReg != otherTargetReg) { - regMaskTP fromRegMask = genRegMask(fromReg); + singleRegMask fromRegMask = genRegMask(fromReg); targetRegsReady |= fromRegMask; #ifdef TARGET_ARM if (genIsValidDoubleReg(fromReg)) @@ -11558,11 +11560,11 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) // Arguments: // regsToFree - Registers that were just freed. // -void LinearScan::verifyFreeRegisters(regMaskTP regsToFree) +void LinearScan::verifyFreeRegisters(regMaskAny regsToFree) { for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { - regMaskTP regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); // If this isn't available or if it's still waiting to be freed (i.e. it was in // delayRegsToFree and so now it's in regsToFree), then skip it. if ((regMask & allAvailableRegs & ~regsToFree) == RBM_NONE) @@ -12010,7 +12012,7 @@ void LinearScan::verifyFinalAllocation() // However, we will assert that, at resolution time, no registers contain GC refs. { DBEXEC(VERBOSE, printf(" ")); - regMaskTP candidateRegs = currentRefPosition.registerAssignment; + regMaskAny candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -12393,9 +12395,11 @@ void LinearScan::RegisterSelection::reset(Interval* interval, RefPosition* refPo // Return Values: // 'true' if there was a single register candidate available after the heuristic is applied. // -bool LinearScan::RegisterSelection::applySelection(int selectionScore, regMaskTP selectionCandidates) +bool LinearScan::RegisterSelection::applySelection(int selectionScore, regMaskOnlyOne selectionCandidates) { - regMaskTP newCandidates = candidates & selectionCandidates; + assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidates)); + + regMaskAny newCandidates = candidates & selectionCandidates; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12414,10 +12418,12 @@ bool LinearScan::RegisterSelection::applySelection(int selectionScore, regMaskTP // Return Values: // 'true' if there was a single register candidate available after the heuristic is applied. // -bool LinearScan::RegisterSelection::applySingleRegSelection(int selectionScore, regMaskTP selectionCandidate) +bool LinearScan::RegisterSelection::applySingleRegSelection(int selectionScore, regMaskOnlyOne selectionCandidate) { + assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidate)); assert(LinearScan::isSingleRegister(selectionCandidate)); - regMaskTP newCandidates = candidates & selectionCandidate; + + regMaskAny newCandidates = candidates & selectionCandidate; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12464,7 +12470,7 @@ void LinearScan::RegisterSelection::try_CONST_AVAILABLE() if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) { - regMaskTP newCandidates = candidates & matchingConstants; + regMaskAny newCandidates = candidates & matchingConstants; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12625,7 +12631,7 @@ void LinearScan::RegisterSelection::try_COVERS_FULL() calculateCoversSets(); #endif - regMaskTP newCandidates = candidates & coversFullSet & freeCandidates; + regMaskOnlyOne newCandidates = candidates & coversFullSet & freeCandidates; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12649,15 +12655,15 @@ void LinearScan::RegisterSelection::try_BEST_FIT() } #endif - regMaskTP bestFitSet = RBM_NONE; + regMaskOnlyOne bestFitSet = RBM_NONE; // If the best score includes COVERS_FULL, pick the one that's killed soonest. // If none cover the full range, the BEST_FIT is the one that's killed later. bool earliestIsBest = coversFullApplied; LsraLocation bestFitLocation = earliestIsBest ? MaxLocation : MinLocation; - for (regMaskTP bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) + for (regMaskOnlyOne bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) { regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); - regMaskTP bestFitCandidateBit = genRegMask(bestFitCandidateRegNum); + singleRegMask bestFitCandidateBit = genRegMask(bestFitCandidateRegNum); bestFitCandidates ^= bestFitCandidateBit; // Find the next RefPosition of the register. @@ -12752,11 +12758,11 @@ void LinearScan::RegisterSelection::try_REG_ORDER() // for free candidates, and doesn't make sense as anything other than the last // heuristic for free registers. unsigned lowestRegOrder = UINT_MAX; - regMaskTP lowestRegOrderBit = RBM_NONE; - for (regMaskTP regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) + regMaskOnlyOne lowestRegOrderBit = RBM_NONE; + for (regMaskOnlyOne regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); - regMaskTP regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); + singleRegMask regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; unsigned thisRegOrder = linearScan->getRegisterRecord(regOrderCandidateRegNum)->regOrder; @@ -12778,7 +12784,7 @@ void LinearScan::RegisterSelection::try_SPILL_COST() assert(!found); // The set of registers with the lowest spill weight. - regMaskTP lowestCostSpillSet = RBM_NONE; + regMaskOnlyOne lowestCostSpillSet = RBM_NONE; // Apply the SPILL_COST heuristic and eliminate regs that can't be spilled. // The spill weight for 'refPosition' (the one we're allocating now). @@ -12789,10 +12795,10 @@ void LinearScan::RegisterSelection::try_SPILL_COST() bool foundLowerSpillWeight = false; LsraLocation thisLocation = refPosition->nodeLocation; - for (regMaskTP spillCandidates = candidates; spillCandidates != RBM_NONE;) + for (regMaskOnlyOne spillCandidates = candidates; spillCandidates != RBM_NONE;) { regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); - regMaskTP spillCandidateBit = genRegMask(spillCandidateRegNum); + singleRegMask spillCandidateBit = genRegMask(spillCandidateRegNum); spillCandidates ^= spillCandidateBit; RegRecord* spillCandidateRegRecord = &linearScan->physRegs[spillCandidateRegNum]; @@ -12913,11 +12919,11 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() assert(!found); LsraLocation farthestLocation = MinLocation; - regMaskTP farthestSet = RBM_NONE; - for (regMaskTP farthestCandidates = candidates; farthestCandidates != RBM_NONE;) + regMaskOnlyOne farthestSet = RBM_NONE; + for (regMaskOnlyOne farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); - regMaskTP farthestCandidateBit = genRegMask(farthestCandidateRegNum); + singleRegMask farthestCandidateBit = genRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; // Find the next RefPosition of the register. @@ -12946,11 +12952,11 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() { assert(!found); - regMaskTP prevRegOptSet = RBM_NONE; - for (regMaskTP prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) + regMaskOnlyOne prevRegOptSet = RBM_NONE; + for (regMaskOnlyOne prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); - regMaskTP prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); + singleRegMask prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; bool foundPrevRegOptReg = true; @@ -13049,11 +13055,11 @@ void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate return; } - regMaskTP coversCandidates = candidates; + regMaskOnlyOne coversCandidates = candidates; for (; coversCandidates != RBM_NONE;) { regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); - regMaskTP coversCandidateBit = genRegMask(coversCandidateRegNum); + singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; // The register is considered unassigned if it has no assignedInterval, OR @@ -13077,11 +13083,11 @@ void LinearScan::RegisterSelection::calculateCoversSets() } preferenceSet = (candidates & preferences); - regMaskTP coversCandidates = (preferenceSet == RBM_NONE) ? candidates : preferenceSet; + regMaskOnlyOne coversCandidates = (preferenceSet == RBM_NONE) ? candidates : preferenceSet; for (; coversCandidates != RBM_NONE;) { regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); - regMaskTP coversCandidateBit = genRegMask(coversCandidateRegNum); + singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; // If we have a single candidate we don't need to compute the preference-related sets, but we @@ -13152,7 +13158,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() // Register bit selected (a single register) and REG_NA if no register was selected. // template -regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, +regMaskOnlyOne LinearScan::RegisterSelection::select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { #ifdef DEBUG @@ -13246,9 +13252,9 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, nextRelatedInterval = nullptr; // First, get the preferences for this interval - regMaskTP thisRelatedPreferences = finalRelatedInterval->getCurrentPreferences(); + regMaskOnlyOne thisRelatedPreferences = finalRelatedInterval->getCurrentPreferences(); // Now, determine if they are compatible and update the relatedPreferences that we'll consider. - regMaskTP newRelatedPreferences = thisRelatedPreferences & relatedPreferences; + regMaskOnlyOne newRelatedPreferences = thisRelatedPreferences & relatedPreferences; if (newRelatedPreferences != RBM_NONE && (!avoidByteRegs || thisRelatedPreferences != RBM_BYTE_REGS)) { // TODO-CQ: The following isFree() check doesn't account for the possibility that there's an @@ -13316,11 +13322,11 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, if (preferCalleeSave) { - regMaskTP calleeSaveCandidates = linearScan->calleeSaveRegs(currentInterval->registerType); + regMaskOnlyOne calleeSaveCandidates = linearScan->calleeSaveRegs(currentInterval->registerType); if (currentInterval->isWriteThru) { // We'll only prefer a callee-save register if it's already been used. - regMaskTP unusedCalleeSaves = + regMaskOnlyOne unusedCalleeSaves = calleeSaveCandidates & ~(linearScan->compiler->codeGen->regSet.rsGetModifiedRegsMask()); callerCalleePrefs = calleeSaveCandidates & ~unusedCalleeSaves; preferences &= ~unusedCalleeSaves; @@ -13345,7 +13351,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, found = false; // Is this a fixedReg? - regMaskTP fixedRegMask = RBM_NONE; + regMaskOnlyOne fixedRegMask = RBM_NONE; if (refPosition->isFixedRegRef) { assert(genMaxOneBit(refPosition->registerAssignment)); @@ -13361,7 +13367,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, } #ifdef DEBUG - regMaskTP inUseOrBusyRegsMask = RBM_NONE; + regMaskAny inUseOrBusyRegsMask = RBM_NONE; #endif // Eliminate candidates that are in-use or busy. @@ -13370,7 +13376,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskTP busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + regMaskAny busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; candidates &= ~busyRegs; #ifdef TARGET_ARM @@ -13392,11 +13398,11 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskTP checkConflictMask = candidates & linearScan->fixedRegs; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); - regMaskTP checkConflictBit = genRegMask(checkConflictReg); + singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; LsraLocation checkConflictLocation = linearScan->nextFixedRef[checkConflictReg]; @@ -13452,7 +13458,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, if (needsConsecutiveRegisters) { #ifdef TARGET_ARM64 - regMaskTP busyConsecutiveCandidates = RBM_NONE; + regMaskFloat busyConsecutiveCandidates = RBM_NONE; if (refPosition->isFirstRefPositionOfConsecutiveRegisters()) { freeCandidates = linearScan->getConsecutiveCandidates(candidates, refPosition, &busyConsecutiveCandidates); @@ -13486,16 +13492,16 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, // Remove the `inUseOrBusyRegsMask` from the original candidates list and find one // such range that is consecutive. Next, append that range to the `candidates`. // - regMaskTP limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask; - regMaskTP overallLimitCandidates; - regMaskTP limitConsecutiveResult = + regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask; + regMaskFloat overallLimitCandidates; + regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, &overallLimitCandidates); assert(limitConsecutiveResult != RBM_NONE); unsigned startRegister = BitOperations::BitScanForward(limitConsecutiveResult); - regMaskTP registersNeededMask = (1ULL << refPosition->regCount) - 1; + regMaskFloat registersNeededMask = (1ULL << refPosition->regCount) - 1; candidates |= (registersNeededMask << startRegister); } @@ -13617,7 +13623,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* currentInterval, // select the REG_ORDER heuristics (if there are any free candidates) or REG_NUM (if all registers // are busy). // -regMaskTP LinearScan::RegisterSelection::selectMinimal(Interval* currentInterval, +regMaskOnlyOne LinearScan::RegisterSelection::selectMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!linearScan->enregisterLocalVars); @@ -13680,7 +13686,7 @@ regMaskTP LinearScan::RegisterSelection::selectMinimal(Interval* currentInter #endif // Is this a fixedReg? - regMaskTP fixedRegMask = RBM_NONE; + regMaskOnlyOne fixedRegMask = RBM_NONE; if (refPosition->isFixedRegRef) { assert(genMaxOneBit(refPosition->registerAssignment)); @@ -13697,7 +13703,7 @@ regMaskTP LinearScan::RegisterSelection::selectMinimal(Interval* currentInter // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskTP busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + regMaskAny busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; candidates &= ~busyRegs; #ifdef TARGET_ARM @@ -13715,11 +13721,11 @@ regMaskTP LinearScan::RegisterSelection::selectMinimal(Interval* currentInter // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskTP checkConflictMask = candidates & linearScan->fixedRegs; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); - regMaskTP checkConflictBit = genRegMask(checkConflictReg); + singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; LsraLocation checkConflictLocation = linearScan->nextFixedRef[checkConflictReg]; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 91252a30110a14..c8b0772633ffa3 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -716,7 +716,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskTP terminatorConsumedRegs); + regMaskAny terminatorConsumedRegs); void resolveEdges(); @@ -808,9 +808,10 @@ class LinearScan : public LinearScanInterface return (LsraStressLimitRegs)(lsraStressMask & LSRA_LIMIT_MASK); } - regMaskTP getConstrainedRegMask(RefPosition* refPosition, - regMaskTP regMaskActual, - regMaskTP regMaskConstrain, + // TODO: Can have separate methods for each type + regMaskAny getConstrainedRegMask(RefPosition* refPosition, + regMaskAny regMaskActual, + regMaskAny regMaskConstrain, unsigned minRegCount); regMaskTP stressLimitRegs(RefPosition* refPosition, regMaskTP mask); @@ -940,7 +941,7 @@ class LinearScan : public LinearScanInterface static bool IsResolutionMove(GenTree* node); static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); - void verifyFreeRegisters(regMaskTP regsToFree); + void verifyFreeRegisters(regMaskAny regsToFree); void verifyFinalAllocation(); void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG @@ -1093,15 +1094,15 @@ class LinearScan : public LinearScanInterface // Given some tree node add refpositions for all the registers this node kills bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskTP killMask); - regMaskTP allRegs(RegisterType rt); - regMaskTP allByteRegs(); - regMaskTP allSIMDRegs(); - regMaskTP lowSIMDRegs(); - regMaskTP internalFloatRegCandidates(); + regMaskOnlyOne allRegs(RegisterType rt); + regMaskGpr allByteRegs(); + regMaskFloat allSIMDRegs(); + regMaskFloat lowSIMDRegs(); + regMaskFloat internalFloatRegCandidates(); void makeRegisterInactive(RegRecord* physRegRecord); void freeRegister(RegRecord* physRegRecord); - void freeRegisters(regMaskTP regsToFree); + void freeRegisters(regMaskAny regsToFree); // Get the type that this tree defines. var_types getDefType(GenTree* tree) @@ -1221,11 +1222,14 @@ class LinearScan : public LinearScanInterface #if defined(TARGET_ARM64) bool canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); void assignConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); - regMaskTP getConsecutiveCandidates(regMaskTP candidates, RefPosition* refPosition, regMaskTP* busyCandidates); - regMaskTP filterConsecutiveCandidates(regMaskTP candidates, + regMaskFloat getConsecutiveCandidates(regMaskFloat candidates, + RefPosition* refPosition, + regMaskFloat* busyCandidates); + regMaskFloat filterConsecutiveCandidates(regMaskFloat candidates, unsigned int registersNeeded, - regMaskTP* allConsecutiveCandidates); - regMaskTP filterConsecutiveCandidatesForSpill(regMaskTP consecutiveCandidates, unsigned int registersNeeded); + regMaskFloat* allConsecutiveCandidates); + regMaskFloat filterConsecutiveCandidatesForSpill(regMaskFloat consecutiveCandidates, + unsigned int registersNeeded); #endif // TARGET_ARM64 regMaskTP getFreeCandidates(regMaskTP candidates ARM_ARG(var_types regType)) @@ -1258,10 +1262,10 @@ class LinearScan : public LinearScanInterface // Perform register selection and update currentInterval or refPosition template - FORCEINLINE regMaskTP select(Interval* currentInterval, + FORCEINLINE singleRegMask select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); - FORCEINLINE regMaskTP selectMinimal(Interval* currentInterval, + FORCEINLINE singleRegMask selectMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); // If the register is from unassigned set such that it was not already @@ -1303,34 +1307,34 @@ class LinearScan : public LinearScanInterface RegisterType regType = RegisterType::TYP_UNKNOWN; - regMaskTP candidates; - regMaskTP preferences = RBM_NONE; + regMaskOnlyOne candidates; + regMaskOnlyOne preferences = RBM_NONE; Interval* relatedInterval = nullptr; - regMaskTP relatedPreferences = RBM_NONE; + regMaskOnlyOne relatedPreferences = RBM_NONE; LsraLocation rangeEndLocation; LsraLocation relatedLastLocation; bool preferCalleeSave = false; RefPosition* rangeEndRefPosition; RefPosition* lastRefPosition; - regMaskTP callerCalleePrefs = RBM_NONE; + regMaskOnlyOne callerCalleePrefs = RBM_NONE; LsraLocation lastLocation; - regMaskTP foundRegBit; + singleRegMask foundRegBit; - regMaskTP prevRegBit = RBM_NONE; + singleRegMask prevRegBit = RBM_NONE; // These are used in the post-selection updates, and must be set for any selection. - regMaskTP freeCandidates; - regMaskTP matchingConstants; - regMaskTP unassignedSet; + regMaskOnlyOne freeCandidates; + regMaskOnlyOne matchingConstants; + regMaskOnlyOne unassignedSet; // Compute the sets for COVERS, OWN_PREFERENCE, COVERS_RELATED, COVERS_FULL and UNASSIGNED together, // as they all require similar computation. - regMaskTP coversSet; - regMaskTP preferenceSet; - regMaskTP coversRelatedSet; - regMaskTP coversFullSet; + regMaskOnlyOne coversSet; + regMaskOnlyOne preferenceSet; + regMaskOnlyOne coversRelatedSet; + regMaskOnlyOne coversFullSet; bool coversSetsCalculated = false; bool found = false; bool skipAllocation = false; @@ -1344,8 +1348,8 @@ class LinearScan : public LinearScanInterface return (prevRegBit & preferences) == foundRegBit; } - bool applySelection(int selectionScore, regMaskTP selectionCandidates); - bool applySingleRegSelection(int selectionScore, regMaskTP selectionCandidate); + bool applySelection(int selectionScore, regMaskOnlyOne selectionCandidates); + bool applySingleRegSelection(int selectionScore, regMaskOnlyOne selectionCandidate); FORCEINLINE void calculateCoversSets(); FORCEINLINE void calculateUnassignedSets(); FORCEINLINE void reset(Interval* interval, RefPosition* refPosition); @@ -1406,7 +1410,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskTP terminatorConsumedRegs); + regMaskAny terminatorConsumedRegs); #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; @@ -1791,12 +1795,13 @@ class LinearScan : public LinearScanInterface void updateSpillCost(regNumber reg, Interval* interval); FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, - regMaskTP regsBusy, - regMaskTP* regsToFree, - regMaskTP* delayRegsToFree DEBUG_ARG(Interval* interval) + regMaskOnlyOne regsBusy, + regMaskOnlyOne* regsToFree, + regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); - regMaskTP m_RegistersWithConstants; + // TODO: This should be m_GprWithConstants, m_FloatRegsWithConstant, etc. + regMaskAny m_RegistersWithConstants; void clearConstantReg(regNumber reg, var_types regType) { m_RegistersWithConstants &= ~getRegMask(reg, regType); @@ -1811,7 +1816,7 @@ class LinearScan : public LinearScanInterface regMaskTP regMask = getRegMask(reg, regType); return (m_RegistersWithConstants & regMask) == regMask; } - regMaskTP getMatchingConstants(regMaskTP mask, Interval* currentInterval, RefPosition* refPosition); + regMaskOnlyOne getMatchingConstants(regMaskAny mask, Interval* currentInterval, RefPosition* refPosition); regMaskTP fixedRegs; LsraLocation nextFixedRef[REG_COUNT]; @@ -2073,7 +2078,7 @@ class LinearScan : public LinearScanInterface // NOTE: we currently don't need a LinearScan `this` pointer for this definition, and some callers // don't have one available, so make is static. // - static FORCEINLINE regMaskTP calleeSaveRegs(RegisterType rt) + static FORCEINLINE regMaskOnlyOne calleeSaveRegs(RegisterType rt) { static const regMaskTP varTypeCalleeSaveRegs[] = { #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) csr, @@ -2169,10 +2174,10 @@ class Interval : public Referenceable void setLocalNumber(Compiler* compiler, unsigned lclNum, LinearScan* l); // Fixed registers for which this Interval has a preference - regMaskTP registerPreferences; + regMaskOnlyOne registerPreferences; // Registers that should be avoided for this interval - regMaskTP registerAversion; + regMaskOnlyOne registerAversion; // The relatedInterval is: // - for any other interval, it is the interval to which this interval @@ -2321,7 +2326,7 @@ class Interval : public Referenceable // definitions. This method will return the current assigned register if any, or // the 'registerPreferences' otherwise. // - regMaskTP getCurrentPreferences() + regMaskOnlyOne getCurrentPreferences() { return (assignedReg == nullptr) ? registerPreferences : genRegMask(assignedReg->regNum); } @@ -2433,7 +2438,7 @@ class RefPosition // Prior to the allocation pass, registerAssignment captures the valid registers // for this RefPosition. // After the allocation pass, this contains the actual assignment - regMaskTP registerAssignment; + regMaskOnlyOne registerAssignment; RefType refType; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index ea3bc9d7fb37e0..d0c26734d6d233 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -176,9 +176,9 @@ bool LinearScan::canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition // From `candidates`, the mask of series of consecutive registers of `registersNeeded` size with just the first-bit // set. // -regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, +regMaskFloat LinearScan::filterConsecutiveCandidates(regMaskFloat candidates, unsigned int registersNeeded, - regMaskTP* allConsecutiveCandidates) + regMaskFloat* allConsecutiveCandidates) { if (BitOperations::PopCount(candidates) < registersNeeded) { @@ -187,16 +187,16 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, return RBM_NONE; } - regMaskTP currAvailableRegs = candidates; - regMaskTP overallResult = RBM_NONE; - regMaskTP consecutiveResult = RBM_NONE; + regMaskFloat currAvailableRegs = candidates; + regMaskFloat overallResult = RBM_NONE; + regMaskFloat consecutiveResult = RBM_NONE; // At this point, for 'n' registers requirement, if Rm, Rm+1, Rm+2, ..., Rm+k-1 are // available, create the mask only for Rm, Rm+1, ..., Rm+(k-n) to convey that it // is safe to assign any of those registers, but not beyond that. #define AppendConsecutiveMask(startIndex, endIndex, availableRegistersMask) \ - regMaskTP selectionStartMask = (1ULL << regAvailableStartIndex) - 1; \ - regMaskTP selectionEndMask = (1ULL << (regAvailableEndIndex - registersNeeded + 1)) - 1; \ + regMaskFloat selectionStartMask = (1ULL << regAvailableStartIndex) - 1; \ + regMaskFloat selectionEndMask = (1ULL << (regAvailableEndIndex - registersNeeded + 1)) - 1; \ consecutiveResult |= availableRegistersMask & (selectionEndMask & ~selectionStartMask); \ overallResult |= availableRegistersMask; @@ -206,10 +206,10 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, { // From LSB, find the first available register (bit `1`) regAvailableStartIndex = BitOperations::BitScanForward(static_cast(currAvailableRegs)); - regMaskTP startMask = (1ULL << regAvailableStartIndex) - 1; + regMaskFloat startMask = (1ULL << regAvailableStartIndex) - 1; // Mask all the bits that are processed from LSB thru regAvailableStart until the last `1`. - regMaskTP maskProcessed = ~(currAvailableRegs | startMask); + regMaskFloat maskProcessed = ~(currAvailableRegs | startMask); // From regAvailableStart, find the first unavailable register (bit `0`). if (maskProcessed == RBM_NONE) @@ -225,7 +225,7 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, { regAvailableEndIndex = BitOperations::BitScanForward(static_cast(maskProcessed)); } - regMaskTP endMask = (1ULL << regAvailableEndIndex) - 1; + regMaskFloat endMask = (1ULL << regAvailableEndIndex) - 1; // Anything between regAvailableStart and regAvailableEnd is the range of consecutive registers available. // If they are equal to or greater than our register requirements, then add all of them to the result. @@ -236,7 +236,7 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, currAvailableRegs &= ~endMask; } while (currAvailableRegs != RBM_NONE); - regMaskTP v0_v31_mask = RBM_V0 | RBM_V31; + regMaskFloat v0_v31_mask = RBM_V0 | RBM_V31; if ((candidates & v0_v31_mask) == v0_v31_mask) { // Finally, check for round robin case where sequence of last register @@ -260,14 +260,14 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, } case 3: { - regMaskTP v0_v30_v31_mask = RBM_V0 | RBM_V30 | RBM_V31; + regMaskFloat v0_v30_v31_mask = RBM_V0 | RBM_V30 | RBM_V31; if ((candidates & v0_v30_v31_mask) != RBM_NONE) { consecutiveResult |= RBM_V30; overallResult |= v0_v30_v31_mask; } - regMaskTP v0_v1_v31_mask = RBM_V0 | RBM_V1 | RBM_V31; + regMaskFloat v0_v1_v31_mask = RBM_V0 | RBM_V1 | RBM_V31; if ((candidates & v0_v1_v31_mask) != RBM_NONE) { consecutiveResult |= RBM_V31; @@ -277,21 +277,21 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, } case 4: { - regMaskTP v0_v29_v30_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; + regMaskFloat v0_v29_v30_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; if ((candidates & v0_v29_v30_v31_mask) != RBM_NONE) { consecutiveResult |= RBM_V29; overallResult |= v0_v29_v30_v31_mask; } - regMaskTP v0_v1_v30_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; + regMaskFloat v0_v1_v30_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; if ((candidates & v0_v1_v30_v31_mask) != RBM_NONE) { consecutiveResult |= RBM_V30; overallResult |= v0_v1_v30_v31_mask; } - regMaskTP v0_v1_v2_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; + regMaskFloat v0_v1_v2_v31_mask = RBM_V0 | RBM_V29 | RBM_V30 | RBM_V31; if ((candidates & v0_v1_v2_v31_mask) != RBM_NONE) { consecutiveResult |= RBM_V31; @@ -323,22 +323,23 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, // Returns: // Filtered candidates that needs fewer spilling. // -regMaskTP LinearScan::filterConsecutiveCandidatesForSpill(regMaskTP consecutiveCandidates, unsigned int registersNeeded) +regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consecutiveCandidates, + unsigned int registersNeeded) { assert(consecutiveCandidates != RBM_NONE); assert((registersNeeded >= 2) && (registersNeeded <= 4)); - regMaskTP consecutiveResultForBusy = RBM_NONE; - regMaskTP unprocessedRegs = consecutiveCandidates; + regMaskFloat consecutiveResultForBusy = RBM_NONE; + regMaskFloat unprocessedRegs = consecutiveCandidates; unsigned regAvailableStartIndex = 0, regAvailableEndIndex = 0; int maxSpillRegs = registersNeeded; - regMaskTP registersNeededMask = (1ULL << registersNeeded) - 1; + regMaskFloat registersNeededMask = (1ULL << registersNeeded) - 1; do { // From LSB, find the first available register (bit `1`) regAvailableStartIndex = BitOperations::BitScanForward(static_cast(unprocessedRegs)); // For the current range, find how many registers are free vs. busy - regMaskTP maskForCurRange = RBM_NONE; + regMaskFloat maskForCurRange = RBM_NONE; bool shouldCheckForRounding = false; switch (registersNeeded) { @@ -413,13 +414,13 @@ regMaskTP LinearScan::filterConsecutiveCandidatesForSpill(regMaskTP consecutiveC // allCandidates = 0x1C080D0F00000000, the consecutive register mask returned // will be 0x400000300000000. // -regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, +regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, RefPosition* refPosition, - regMaskTP* busyCandidates) + regMaskFloat* busyCandidates) { assert(compiler->info.compNeedsConsecutiveRegisters); assert(refPosition->isFirstRefPositionOfConsecutiveRegisters()); - regMaskTP freeCandidates = allCandidates & m_AvailableRegs; + regMaskFloat freeCandidates = allCandidates & m_AvailableRegs; #ifdef DEBUG if (getStressLimitRegs() != LSRA_LIMIT_NONE) @@ -431,12 +432,12 @@ regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, #endif *busyCandidates = RBM_NONE; - regMaskTP overallResult; + regMaskFloat overallResult; unsigned int registersNeeded = refPosition->regCount; if (freeCandidates != RBM_NONE) { - regMaskTP consecutiveResultForFree = + regMaskFloat consecutiveResultForFree = filterConsecutiveCandidates(freeCandidates, registersNeeded, &overallResult); if (consecutiveResultForFree != RBM_NONE) @@ -449,7 +450,7 @@ regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, regNumber firstRegNum = REG_NA; regNumber prevRegNum = REG_NA; int foundCount = 0; - regMaskTP foundRegMask = RBM_NONE; + regMaskFloat foundRegMask = RBM_NONE; RefPosition* consecutiveRefPosition = getNextConsecutiveRefPosition(refPosition); assert(consecutiveRefPosition != nullptr); @@ -488,7 +489,7 @@ regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, if (foundCount != 0) { assert(firstRegNum != REG_NA); - regMaskTP remainingRegsMask = ((1ULL << (registersNeeded - foundCount)) - 1) << (firstRegNum - 1); + regMaskFloat remainingRegsMask = ((1ULL << (registersNeeded - foundCount)) - 1) << (firstRegNum - 1); if ((overallResult & remainingRegsMask) != RBM_NONE) { @@ -528,8 +529,8 @@ regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, // try_FAR_NEXT_REF(), etc. here which would complicate things. Instead, we just go with option# 1 and select // registers based on fewer number of registers that has to be spilled. // - regMaskTP overallResultForBusy; - regMaskTP consecutiveResultForBusy = + regMaskFloat overallResultForBusy; + regMaskFloat consecutiveResultForBusy = filterConsecutiveCandidates(allCandidates, registersNeeded, &overallResultForBusy); *busyCandidates = consecutiveResultForBusy; @@ -541,7 +542,7 @@ regMaskTP LinearScan::getConsecutiveCandidates(regMaskTP allCandidates, // If there is an overlap of that with free registers, then try to find a series that will need least // registers spilling as mentioned in #1 above. - regMaskTP optimalConsecutiveResultForBusy = + regMaskFloat optimalConsecutiveResultForBusy = filterConsecutiveCandidatesForSpill(consecutiveResultForBusy, registersNeeded); if (optimalConsecutiveResultForBusy != RBM_NONE) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0c1d3f74475c8d..6cc1368da6d778 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1364,20 +1364,20 @@ RefPosition* LinearScan::buildInternalIntRegisterDefForNode(GenTree* tree, regMa // Returns: // The def RefPosition created for this internal temp. // -RefPosition* LinearScan::buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskTP internalCands) +RefPosition* LinearScan::buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskFloat internalCands) { // The candidate set should contain only float registers. - assert((internalCands & ~availableFloatRegs) == RBM_NONE); + assert(compiler->IsFloatRegMask(internalCands)); RefPosition* defRefPosition = defineNewInternalTemp(tree, FloatRegisterType, internalCands); return defRefPosition; } #if defined(FEATURE_SIMD) && defined(TARGET_XARCH) -RefPosition* LinearScan::buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskTP internalCands) +RefPosition* LinearScan::buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskPredicate internalCands) { // The candidate set should contain only float registers. - assert((internalCands & ~availableMaskRegs) == RBM_NONE); + assert(compiler->IsPredicateRegMask(internalCands)); return defineNewInternalTemp(tree, MaskRegisterType, internalCands); } @@ -3651,7 +3651,7 @@ void LinearScan::BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, } } - regMaskTP defCandidates = RBM_NONE; + regMaskOnlyOne defCandidates = RBM_NONE; var_types type = varDsc->GetRegisterType(); #ifdef TARGET_X86 @@ -3737,7 +3737,7 @@ int LinearScan::BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc) if (isMultiRegSrc) { - regMaskTP srcCandidates = RBM_NONE; + regMaskGpr srcCandidates = RBM_NONE; #ifdef TARGET_X86 var_types type = fieldVarDsc->TypeGet(); if (varTypeIsByte(type)) @@ -3848,7 +3848,7 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) else { srcCount = 1; - regMaskTP srcCandidates = RBM_NONE; + regMaskGpr srcCandidates = RBM_NONE; #ifdef TARGET_X86 var_types type = varDsc->GetRegisterType(storeLoc); if (varTypeIsByte(type)) @@ -4276,7 +4276,7 @@ int LinearScan::BuildCmp(GenTree* tree) if (!tree->TypeIs(TYP_VOID)) { - regMaskTP dstCandidates = RBM_NONE; + regMaskGpr dstCandidates = RBM_NONE; #ifdef TARGET_X86 // If the compare is used by a jump, we just need to set the condition codes. If not, then we need @@ -4300,8 +4300,8 @@ int LinearScan::BuildCmp(GenTree* tree) // int LinearScan::BuildCmpOperands(GenTree* tree) { - regMaskTP op1Candidates = RBM_NONE; - regMaskTP op2Candidates = RBM_NONE; + regMaskGpr op1Candidates = RBM_NONE; + regMaskGpr op2Candidates = RBM_NONE; GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 4fc56947478aac..6e9ba0ddeaf9ae 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -759,8 +759,8 @@ bool LinearScan::isRMWRegOper(GenTree* tree) int LinearScan::BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskTP candidates) { int srcCount = 0; - regMaskTP op1Candidates = candidates; - regMaskTP op2Candidates = candidates; + regMaskGpr op1Candidates = candidates; + regMaskGpr op2Candidates = candidates; #ifdef TARGET_X86 if (varTypeIsByte(node)) @@ -1478,7 +1478,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) // or if are but the remainder is a power of 2 and less than the // size of a register - regMaskTP regMask = availableIntRegs; + regMaskGpr regMask = availableIntRegs; #ifdef TARGET_X86 if ((size & 1) != 0) { @@ -1756,7 +1756,7 @@ int LinearScan::BuildPutArgStk(GenTreePutArgStk* putArgStk) // If we have a remainder smaller than XMM_REGSIZE_BYTES, we need an integer temp reg. if ((loadSize % XMM_REGSIZE_BYTES) != 0) { - regMaskTP regMask = availableIntRegs; + regMaskGpr regMask = availableIntRegs; #ifdef TARGET_X86 // Storing at byte granularity requires a byteable register. if ((loadSize & 1) != 0) @@ -2803,7 +2803,7 @@ int LinearScan::BuildCast(GenTreeCast* cast) const var_types srcType = genActualType(src->TypeGet()); const var_types castType = cast->gtCastType; - regMaskTP candidates = RBM_NONE; + regMaskGpr candidates = RBM_NONE; #ifdef TARGET_X86 if (varTypeIsByte(castType)) { From 8e97320f6a9f7ea95c4898e4b92e43a2fe615124 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 18:59:44 -0800 Subject: [PATCH 041/201] Update lsra.h and related code --- src/coreclr/jit/lsra.h | 178 +++++++++++++++++----------------- src/coreclr/jit/lsrabuild.cpp | 66 +++++++------ src/coreclr/jit/lsraxarch.cpp | 2 +- 3 files changed, 127 insertions(+), 119 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index c8b0772633ffa3..9b22a3a5153a18 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -615,7 +615,7 @@ class LinearScan : public LinearScanInterface // This is the main driver virtual PhaseStatus doLinearScan(); - static bool isSingleRegister(regMaskTP regMask) + static bool isSingleRegister(regMaskAny regMask) { return (genExactlyOneBit(regMask)); } @@ -771,34 +771,34 @@ class LinearScan : public LinearScanInterface #if defined(TARGET_AMD64) #ifdef UNIX_AMD64_ABI // On System V the RDI and RSI are not callee saved. Use R12 ans R13 as callee saved registers. - static const regMaskTP LsraLimitSmallIntSet = + static const regMaskGpr LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EBX | RBM_ETW_FRAMED_EBP | RBM_R12 | RBM_R13); #else // !UNIX_AMD64_ABI // On Windows Amd64 use the RDI and RSI as callee saved registers. - static const regMaskTP LsraLimitSmallIntSet = + static const regMaskGpr LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EBX | RBM_ETW_FRAMED_EBP | RBM_ESI | RBM_EDI); #endif // !UNIX_AMD64_ABI - static const regMaskTP LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); - static const regMaskTP LsraLimitUpperSimdSet = + static const regMaskFloat LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); + static const regMaskFloat LsraLimitUpperSimdSet = (RBM_XMM16 | RBM_XMM17 | RBM_XMM18 | RBM_XMM19 | RBM_XMM20 | RBM_XMM21 | RBM_XMM22 | RBM_XMM23 | RBM_XMM24 | RBM_XMM25 | RBM_XMM26 | RBM_XMM27 | RBM_XMM28 | RBM_XMM29 | RBM_XMM30 | RBM_XMM31); #elif defined(TARGET_ARM) // On ARM, we may need two registers to set up the target register for a virtual call, so we need // to have at least the maximum number of arg registers, plus 2. - static const regMaskTP LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5); - static const regMaskTP LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F16 | RBM_F17); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5); + static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F16 | RBM_F17); #elif defined(TARGET_ARM64) - static const regMaskTP LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); - static const regMaskTP LsraLimitSmallFPSet = (RBM_V0 | RBM_V1 | RBM_V2 | RBM_V8 | RBM_V9); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); + static const regMaskFloat LsraLimitSmallFPSet = (RBM_V0 | RBM_V1 | RBM_V2 | RBM_V8 | RBM_V9); #elif defined(TARGET_X86) - static const regMaskTP LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EDI); - static const regMaskTP LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EDI); + static const regMaskFloat LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); #elif defined(TARGET_LOONGARCH64) - static const regMaskTP LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); - static const regMaskTP LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); + static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); #elif defined(TARGET_RISCV64) - static const regMaskTP LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); - static const regMaskTP LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); + static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); #else #error Unsupported or unset target architecture #endif // target @@ -813,7 +813,7 @@ class LinearScan : public LinearScanInterface regMaskAny regMaskActual, regMaskAny regMaskConstrain, unsigned minRegCount); - regMaskTP stressLimitRegs(RefPosition* refPosition, regMaskTP mask); + regMaskAny stressLimitRegs(RefPosition* refPosition, regMaskAny mask); // This controls the heuristics used to select registers // These can be combined. @@ -878,7 +878,7 @@ class LinearScan : public LinearScanInterface { return (LsraBlockBoundaryLocations)(lsraStressMask & LSRA_BLOCK_BOUNDARY_MASK); } - regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskTP availableRegs); + regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskAny availableRegs); // This controls whether we always insert a GT_RELOAD instruction after a spill // Note that this can be combined with LSRA_SPILL_ALWAYS (or not) @@ -1035,14 +1035,14 @@ class LinearScan : public LinearScanInterface void insertZeroInitRefPositions(); // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions - void addRefsForPhysRegMask(regMaskTP mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); void buildRefPositionsForNode(GenTree* tree, LsraLocation loc); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskTP fpCalleeKillSet); + void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskFloat fpCalleeKillSet); void buildUpperVectorRestoreRefPosition( Interval* lclVarInterval, LsraLocation currentLoc, GenTree* node, bool isUse, unsigned multiRegIdx); #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE @@ -1072,27 +1072,27 @@ class LinearScan : public LinearScanInterface } // Helpers for getKillSetForNode(). - regMaskTP getKillSetForStoreInd(GenTreeStoreInd* tree); - regMaskTP getKillSetForShiftRotate(GenTreeOp* tree); - regMaskTP getKillSetForMul(GenTreeOp* tree); - regMaskTP getKillSetForCall(GenTreeCall* call); - regMaskTP getKillSetForModDiv(GenTreeOp* tree); - regMaskTP getKillSetForBlockStore(GenTreeBlk* blkNode); - regMaskTP getKillSetForReturn(); - regMaskTP getKillSetForProfilerHook(); + regMaskAny getKillSetForStoreInd(GenTreeStoreInd* tree); + regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); + regMaskGpr getKillSetForMul(GenTreeOp* tree); + regMaskAny getKillSetForCall(GenTreeCall* call); + regMaskGpr getKillSetForModDiv(GenTreeOp* tree); + regMaskAny getKillSetForBlockStore(GenTreeBlk* blkNode); + regMaskAny getKillSetForReturn(); + regMaskAny getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS - regMaskTP getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); + regMaskGpr getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); #endif // FEATURE_HW_INTRINSICS // Return the registers killed by the given tree node. // This is used only for an assert, and for stress, so it is only defined under DEBUG. // Otherwise, the Build methods should obtain the killMask from the appropriate method above. #ifdef DEBUG - regMaskTP getKillSetForNode(GenTree* tree); + regMaskAny getKillSetForNode(GenTree* tree); #endif // Given some tree node add refpositions for all the registers this node kills - bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskTP killMask); + bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskAny killMask); regMaskOnlyOne allRegs(RegisterType rt); regMaskGpr allByteRegs(); @@ -1120,11 +1120,11 @@ class LinearScan : public LinearScanInterface } // Managing internal registers during the BuildNode process. - RefPosition* defineNewInternalTemp(GenTree* tree, RegisterType regType, regMaskTP candidates); - RefPosition* buildInternalIntRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE); - RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE); + RefPosition* defineNewInternalTemp(GenTree* tree, RegisterType regType, regMaskOnlyOne candidates); + RefPosition* buildInternalIntRegisterDefForNode(GenTree* tree, regMaskGpr internalCands = RBM_NONE); + RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskFloat internalCands = RBM_NONE); #if defined(FEATURE_SIMD) - RefPosition* buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskTP internalCands = RBM_NONE); + RefPosition* buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskPredicate internalCands = RBM_NONE); #endif void buildInternalRegisterUses(); @@ -1161,11 +1161,11 @@ class LinearScan : public LinearScanInterface LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, - regMaskTP mask, + regMaskOnlyOne mask, unsigned multiRegIdx = 0); RefPosition* newRefPosition( - regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskTP mask); + regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskOnlyOne mask); void applyCalleeSaveHeuristics(RefPosition* rp); @@ -1232,9 +1232,9 @@ class LinearScan : public LinearScanInterface unsigned int registersNeeded); #endif // TARGET_ARM64 - regMaskTP getFreeCandidates(regMaskTP candidates ARM_ARG(var_types regType)) + regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates ARM_ARG(var_types regType)) { - regMaskTP result = candidates & m_AvailableRegs; + regMaskOnlyOne result = candidates & m_AvailableRegs; #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use register for which the odd half is // also available. @@ -1478,8 +1478,8 @@ class LinearScan : public LinearScanInterface static const int MAX_ROWS_BETWEEN_TITLES = 50; int rowCountSinceLastTitle; // Current mask of registers being printed in the dump. - regMaskTP lastDumpedRegisters; - regMaskTP registersToDump; + regMaskAny lastDumpedRegisters; + regMaskAny registersToDump; int lastUsedRegNumIndex; bool shouldDumpReg(regNumber regNum) { @@ -1662,14 +1662,14 @@ class LinearScan : public LinearScanInterface // A temporary VarToRegMap used during the resolution of critical edges. VarToRegMap sharedCriticalVarToRegMap; - PhasedVar actualRegistersMask; - PhasedVar availableIntRegs; - PhasedVar availableFloatRegs; - PhasedVar availableDoubleRegs; + PhasedVar actualRegistersMask; + PhasedVar availableIntRegs; + PhasedVar availableFloatRegs; + PhasedVar availableDoubleRegs; #if defined(TARGET_XARCH) - PhasedVar availableMaskRegs; + PhasedVar availableMaskRegs; #endif - PhasedVar* availableRegs[TYP_COUNT]; + PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, predicate #if defined(TARGET_XARCH) #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) @@ -1681,7 +1681,7 @@ class LinearScan : public LinearScanInterface // PUTARG_REG node. Tracked between the PUTARG_REG and its corresponding // CALL node and is used to avoid preferring these registers for locals // which would otherwise force a spill. - regMaskTP placedArgRegs; + regMaskAny placedArgRegs; struct PlacedLocal { @@ -1730,7 +1730,7 @@ class LinearScan : public LinearScanInterface // Register status //----------------------------------------------------------------------- - regMaskTP m_AvailableRegs; + regMaskAny m_AvailableRegs; //TODO: Should be separate for gpr, vector, predicate regNumber getRegForType(regNumber reg, var_types regType) { #ifdef TARGET_ARM @@ -1742,10 +1742,10 @@ class LinearScan : public LinearScanInterface return reg; } - regMaskTP getRegMask(regNumber reg, var_types regType) + regMaskOnlyOne getRegMask(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); - regMaskTP regMask = genRegMask(reg); + regMaskOnlyOne regMask = genRegMask(reg); #ifdef TARGET_ARM if (regType == TYP_DOUBLE) { @@ -1764,25 +1764,25 @@ class LinearScan : public LinearScanInterface bool isRegAvailable(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_AvailableRegs & regMask) == regMask; } - void setRegsInUse(regMaskTP regMask) + void setRegsInUse(regMaskAny regMask) { m_AvailableRegs &= ~regMask; } void setRegInUse(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); setRegsInUse(regMask); } - void makeRegsAvailable(regMaskTP regMask) + void makeRegsAvailable(regMaskAny regMask) { m_AvailableRegs |= regMask; } void makeRegAvailable(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); makeRegsAvailable(regMask); } @@ -1813,12 +1813,12 @@ class LinearScan : public LinearScanInterface bool isRegConstant(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_RegistersWithConstants & regMask) == regMask; } regMaskOnlyOne getMatchingConstants(regMaskAny mask, Interval* currentInterval, RefPosition* refPosition); - regMaskTP fixedRegs; + regMaskAny fixedRegs; //TODO: Should be also seperate LsraLocation nextFixedRef[REG_COUNT]; void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); LsraLocation getNextFixedRef(regNumber regNum, var_types regType) @@ -1847,15 +1847,15 @@ class LinearScan : public LinearScanInterface } weight_t spillCost[REG_COUNT]; - regMaskTP regsBusyUntilKill; - regMaskTP regsInUseThisLocation; - regMaskTP regsInUseNextLocation; + regMaskAny regsBusyUntilKill; //TODO: Likewise, probably have this global 32-bit and set it point to the specific version like gpr, vector, etc. + regMaskAny regsInUseThisLocation; + regMaskAny regsInUseNextLocation; #ifdef TARGET_ARM64 - regMaskTP consecutiveRegsInUseThisLocation; + regMaskFloat consecutiveRegsInUseThisLocation; #endif bool isRegBusy(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); return (regsBusyUntilKill & regMask) != RBM_NONE; } void setRegBusyUntilKill(regNumber reg, var_types regType) @@ -1869,7 +1869,7 @@ class LinearScan : public LinearScanInterface bool isRegInUse(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskOnlyOne regMask = getRegMask(reg, regType); return (regsInUseThisLocation & regMask) != RBM_NONE; } @@ -1936,13 +1936,13 @@ class LinearScan : public LinearScanInterface bool isCandidateMultiRegLclVar(GenTreeLclVar* lclNode); bool checkContainedOrCandidateLclVar(GenTreeLclVar* lclNode); - RefPosition* BuildUse(GenTree* operand, regMaskTP candidates = RBM_NONE, int multiRegIdx = 0); + RefPosition* BuildUse(GenTree* operand, regMaskOnlyOne candidates = RBM_NONE, int multiRegIdx = 0); void setDelayFree(RefPosition* use); - int BuildBinaryUses(GenTreeOp* node, regMaskTP candidates = RBM_NONE); - int BuildCastUses(GenTreeCast* cast, regMaskTP candidates); + int BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates = RBM_NONE); + int BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates); #ifdef TARGET_XARCH - int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskTP candidates = RBM_NONE); - inline regMaskTP BuildEvexIncompatibleMask(GenTree* tree); + int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates = RBM_NONE); + inline regMaskFloat BuildEvexIncompatibleMask(GenTree* tree); #endif // !TARGET_XARCH int BuildSelect(GenTreeOp* select); // This is the main entry point for building the RefPositions for a node. @@ -1954,18 +1954,18 @@ class LinearScan : public LinearScanInterface bool supportsSpecialPutArg(); int BuildSimple(GenTree* tree); - int BuildOperandUses(GenTree* node, regMaskTP candidates = RBM_NONE); + int BuildOperandUses(GenTree* node, regMaskOnlyOne candidates = RBM_NONE); void AddDelayFreeUses(RefPosition* refPosition, GenTree* rmwNode); int BuildDelayFreeUses(GenTree* node, GenTree* rmwNode = nullptr, - regMaskTP candidates = RBM_NONE, + regMaskOnlyOne candidates = RBM_NONE, RefPosition** useRefPosition = nullptr); - int BuildIndirUses(GenTreeIndir* indirTree, regMaskTP candidates = RBM_NONE); - int BuildAddrUses(GenTree* addr, regMaskTP candidates = RBM_NONE); + int BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates = RBM_NONE); + int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); - RefPosition* BuildDef(GenTree* tree, regMaskTP dstCandidates = RBM_NONE, int multiRegIdx = 0); - void BuildDefs(GenTree* tree, int dstCount, regMaskTP dstCandidates = RBM_NONE); - void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskTP dstCandidates, regMaskTP killMask); + RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); + void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); + void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskAny killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH @@ -2038,14 +2038,14 @@ class LinearScan : public LinearScanInterface int BuildLclHeap(GenTree* tree); #if defined(TARGET_AMD64) - regMaskTP rbmAllFloat; - regMaskTP rbmFltCalleeTrash; + regMaskFloat rbmAllFloat; + regMaskFloat rbmFltCalleeTrash; FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskFloat get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } @@ -2080,7 +2080,7 @@ class LinearScan : public LinearScanInterface // static FORCEINLINE regMaskOnlyOne calleeSaveRegs(RegisterType rt) { - static const regMaskTP varTypeCalleeSaveRegs[] = { + static const regMaskOnlyOne varTypeCalleeSaveRegs[] = { #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) csr, #include "typelist.h" #undef DEF_TP @@ -2094,16 +2094,16 @@ class LinearScan : public LinearScanInterface // Not all of the callee trash values are constant, so don't declare this as a method local static // doing so results in significantly more complex codegen and we'd rather just initialize this once // as part of initializing LSRA instead - regMaskTP varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskOnlyOne varTypeCalleeTrashRegs[TYP_COUNT]; #endif // TARGET_XARCH //------------------------------------------------------------------------ // callerSaveRegs: Get the set of caller-save registers of the given RegisterType // - FORCEINLINE regMaskTP callerSaveRegs(RegisterType rt) const + FORCEINLINE regMaskOnlyOne callerSaveRegs(RegisterType rt) const { #if !defined(TARGET_XARCH) - static const regMaskTP varTypeCalleeTrashRegs[] = { + static const regMaskOnlyOne varTypeCalleeTrashRegs[] = { #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP @@ -2130,7 +2130,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX class Interval : public Referenceable { public: - Interval(RegisterType registerType, regMaskTP registerPreferences) + Interval(RegisterType registerType, regMaskOnlyOne registerPreferences) : registerPreferences(registerPreferences) , registerAversion(RBM_NONE) , relatedInterval(nullptr) @@ -2331,7 +2331,7 @@ class Interval : public Referenceable return (assignedReg == nullptr) ? registerPreferences : genRegMask(assignedReg->regNum); } - void mergeRegisterPreferences(regMaskTP preferences) + void mergeRegisterPreferences(regMaskOnlyOne preferences) { // We require registerPreferences to have been initialized. assert(registerPreferences != RBM_NONE); @@ -2346,7 +2346,7 @@ class Interval : public Referenceable return; } - regMaskTP commonPreferences = (registerPreferences & preferences); + regMaskOnlyOne commonPreferences = (registerPreferences & preferences); if (commonPreferences != RBM_NONE) { registerPreferences = commonPreferences; @@ -2381,11 +2381,11 @@ class Interval : public Referenceable // Keep only the callee-save preferences, if not empty. // Otherwise, take the union of the preferences. - regMaskTP newPreferences = registerPreferences | preferences; + regMaskOnlyOne newPreferences = registerPreferences | preferences; if (preferCalleeSave) { - regMaskTP calleeSaveMask = (LinearScan::calleeSaveRegs(this->registerType) & newPreferences); + regMaskOnlyOne calleeSaveMask = (LinearScan::calleeSaveRegs(this->registerType) & newPreferences); if (calleeSaveMask != RBM_NONE) { newPreferences = calleeSaveMask; @@ -2400,7 +2400,7 @@ class Interval : public Referenceable // An exception is made in the case where one of the existing or new // preferences are all callee-save, in which case we "prefer" the callee-save - void updateRegisterPreferences(regMaskTP preferences) + void updateRegisterPreferences(regMaskOnlyOne preferences) { // If this interval is preferenced, that interval may have already been assigned a // register, and we want to include that in the preferences. @@ -2696,7 +2696,7 @@ class RefPosition // isFixedRefOfRegMask indicates that the RefPosition has a fixed assignment to the register // specified by the given mask - bool isFixedRefOfRegMask(regMaskTP regMask) + bool isFixedRefOfRegMask(regMaskOnlyOne regMask) { assert(genMaxOneBit(regMask)); return (registerAssignment == regMask); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 6cc1368da6d778..e29a6c3a67326b 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -520,6 +520,8 @@ void LinearScan::associateRefPosWithInterval(RefPosition* rp) RefPosition* LinearScan::newRefPosition( regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskTP mask) { + assert(compiler->IsOnlyOneRegMask(mask)); + RefPosition* newRP = newRefPositionRaw(theLocation, theTreeNode, theRefType); RegRecord* regRecord = getRegisterRecord(reg); @@ -557,9 +559,11 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, - regMaskTP mask, + regMaskOnlyOne mask, unsigned multiRegIdx /* = 0 */) { + assert(compiler->IsOnlyOneRegMask(mask)); + if (theInterval != nullptr) { if (mask == RBM_NONE) @@ -695,7 +699,7 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(regMaskTP mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); @@ -710,7 +714,7 @@ void LinearScan::addRefsForPhysRegMask(regMaskTP mask, LsraLocation currentLoc, // modified until codegen, which is too late. compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); - for (regMaskTP candidates = mask; candidates != RBM_NONE;) + for (regMaskAny candidates = mask; candidates != RBM_NONE;) { regNumber reg = genFirstRegNumFromMaskAndToggle(candidates); // This assumes that these are all "special" RefTypes that @@ -771,9 +775,9 @@ regMaskTP LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForShiftRotate(GenTreeOp* shiftNode) +regMaskGpr LinearScan::getKillSetForShiftRotate(GenTreeOp* shiftNode) { - regMaskTP killMask = RBM_NONE; + regMaskGpr killMask = RBM_NONE; #ifdef TARGET_XARCH assert(shiftNode->OperIsShiftOrRotate()); GenTree* shiftBy = shiftNode->gtGetOp2(); @@ -793,9 +797,9 @@ regMaskTP LinearScan::getKillSetForShiftRotate(GenTreeOp* shiftNode) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForMul(GenTreeOp* mulNode) +regMaskGpr LinearScan::getKillSetForMul(GenTreeOp* mulNode) { - regMaskTP killMask = RBM_NONE; + regMaskGpr killMask = RBM_NONE; #ifdef TARGET_XARCH assert(mulNode->OperIsMul()); if (!mulNode->OperIs(GT_MUL) || (((mulNode->gtFlags & GTF_UNSIGNED) != 0) && mulNode->gtOverflowEx())) @@ -814,9 +818,9 @@ regMaskTP LinearScan::getKillSetForMul(GenTreeOp* mulNode) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForModDiv(GenTreeOp* node) +regMaskGpr LinearScan::getKillSetForModDiv(GenTreeOp* node) { - regMaskTP killMask = RBM_NONE; + regMaskGpr killMask = RBM_NONE; #ifdef TARGET_XARCH assert(node->OperIs(GT_MOD, GT_DIV, GT_UMOD, GT_UDIV)); if (varTypeUsesIntReg(node->TypeGet())) @@ -891,10 +895,10 @@ regMaskTP LinearScan::getKillSetForCall(GenTreeCall* call) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) +regMaskAny LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) { assert(blkNode->OperIsStoreBlk()); - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; bool isCopyBlk = varTypeIsStruct(blkNode->Data()); switch (blkNode->gtBlkOpKind) @@ -949,17 +953,14 @@ regMaskTP LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) #ifdef FEATURE_HW_INTRINSICS //------------------------------------------------------------------------ -// getKillSetForHWIntrinsic: Determine the liveness kill set for a GT_STOREIND node. -// If the GT_STOREIND will generate a write barrier, determine the specific kill -// set required by the case-specific, platform-specific write barrier. If no -// write barrier is required, the kill set will be RBM_NONE. +// getKillSetForHWIntrinsic: Determine the liveness kill set for a GT_HWINTRINSIC node. // // Arguments: -// tree - the GT_STOREIND node +// tree - the GT_HWINTRINSIC node // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) +regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) { regMaskTP killMask = RBM_NONE; #ifdef TARGET_XARCH @@ -993,7 +994,7 @@ regMaskTP LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForReturn() +regMaskAny LinearScan::getKillSetForReturn() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_LEAVE) : RBM_NONE; @@ -1007,7 +1008,7 @@ regMaskTP LinearScan::getKillSetForReturn() // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForProfilerHook() +regMaskAny LinearScan::getKillSetForProfilerHook() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_TAILCALL) : RBM_NONE; @@ -1022,9 +1023,9 @@ regMaskTP LinearScan::getKillSetForProfilerHook() // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForNode(GenTree* tree) +regMaskAny LinearScan::getKillSetForNode(GenTree* tree) { - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; switch (tree->OperGet()) { case GT_LSH: @@ -1122,7 +1123,7 @@ regMaskTP LinearScan::getKillSetForNode(GenTree* tree) // This method can add kills even if killMask is RBM_NONE, if this tree is one of the // special cases that signals that we can't permit callee save registers to hold GC refs. -bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskTP killMask) +bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskAny killMask) { bool insertedKills = false; @@ -1325,8 +1326,10 @@ bool LinearScan::checkContainedOrCandidateLclVar(GenTreeLclVar* lclNode) // currentLoc - Location of the temp Def position // regMask - register mask of candidates for temp // -RefPosition* LinearScan::defineNewInternalTemp(GenTree* tree, RegisterType regType, regMaskTP regMask) +RefPosition* LinearScan::defineNewInternalTemp(GenTree* tree, RegisterType regType, regMaskOnlyOne regMask) { + assert(compiler->IsOnlyOneRegMask(regMask)); + Interval* current = newInterval(regType); current->isInternal = true; RefPosition* newDef = newRefPosition(current, currentLoc, RefTypeDef, tree, regMask, 0); @@ -1348,7 +1351,7 @@ RefPosition* LinearScan::defineNewInternalTemp(GenTree* tree, RegisterType regTy RefPosition* LinearScan::buildInternalIntRegisterDefForNode(GenTree* tree, regMaskTP internalCands) { // The candidate set should contain only integer registers. - assert((internalCands & ~availableIntRegs) == RBM_NONE); + assert(compiler->IsGprRegMask(internalCands)); RefPosition* defRefPosition = defineNewInternalTemp(tree, IntRegisterType, internalCands); return defRefPosition; @@ -1473,8 +1476,10 @@ Interval* LinearScan::getUpperVectorInterval(unsigned varIndex) // But we will use as a proxy any node that kills floating point registers. // (Note that some calls are masquerading as other nodes at this point so we can't just check for calls.) // -void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskTP fpCalleeKillSet) +void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskFloat fpCalleeKillSet) { + assert(compiler->IsFloatRegMask(fpCalleeKillSet)); + if ((tree != nullptr) && tree->IsCall()) { if (tree->AsCall()->IsNoReturn() || compiler->fgIsThrow(tree)) @@ -2922,9 +2927,10 @@ void setTgtPref(Interval* interval, RefPosition* tgtPrefUse) // Notes: // Adds the RefInfo for the definition to the defList. // -RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskTP dstCandidates, int multiRegIdx) +RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, int multiRegIdx) { assert(!tree->isContained()); + assert(compiler->IsOnlyOneRegMask(dstCandidates)); if (dstCandidates != RBM_NONE) { @@ -3064,7 +3070,7 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskTP dstCandidates) // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskTP dstCandidates, regMaskTP killMask) +void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskAny killMask) { assert(killMask == getKillSetForNode(tree)); @@ -3177,8 +3183,10 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Notes: // The node must not be contained, and must have been processed by buildRefPositionsForNode(). // -RefPosition* LinearScan::BuildUse(GenTree* operand, regMaskTP candidates, int multiRegIdx) +RefPosition* LinearScan::BuildUse(GenTree* operand, regMaskOnlyOne candidates, int multiRegIdx) { + assert(compiler->IsOnlyOneRegMask(candidates)); + assert(!operand->isContained()); Interval* interval; bool regOptional = operand->IsRegOptional(); @@ -3309,7 +3317,7 @@ int LinearScan::BuildAddrUses(GenTree* addr, regMaskTP candidates) // Return Value: // The number of source registers used by the *parent* of this node. // -int LinearScan::BuildOperandUses(GenTree* node, regMaskTP candidates) +int LinearScan::BuildOperandUses(GenTree* node, regMaskOnlyOne candidates) { if (!node->isContained()) { diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 6e9ba0ddeaf9ae..db6ed79da1ebde 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -3085,7 +3085,7 @@ void LinearScan::SetContainsAVXFlags(unsigned sizeOfSIMDVector /* = 0*/) // RBM_NONE if compatible with EVEX (or not a floating/SIMD register), // lowSIMDRegs() (XMM0-XMM16) otherwise. // -inline regMaskTP LinearScan::BuildEvexIncompatibleMask(GenTree* tree) +inline regMaskFloat LinearScan::BuildEvexIncompatibleMask(GenTree* tree) { #if defined(TARGET_AMD64) if (!(varTypeIsFloating(tree->gtType) || varTypeIsSIMD(tree->gtType))) From 1b24a61808922982b516d187d243beb430fa730d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:05:41 -0800 Subject: [PATCH 042/201] fix compilation error for singleRegMask --- src/coreclr/jit/lsra.cpp | 5 +++-- src/coreclr/jit/target.h | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index f43845af8ee523..3a5b356665476d 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -13158,7 +13158,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() // Register bit selected (a single register) and REG_NA if no register was selected. // template -regMaskOnlyOne LinearScan::RegisterSelection::select(Interval* currentInterval, +singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { #ifdef DEBUG @@ -13623,7 +13623,8 @@ regMaskOnlyOne LinearScan::RegisterSelection::select(Interval* currentInterva // select the REG_ORDER heuristics (if there are any free candidates) or REG_NUM (if all registers // are busy). // -regMaskOnlyOne LinearScan::RegisterSelection::selectMinimal(Interval* currentInterval, +singleRegMask LinearScan::RegisterSelection::selectMinimal( + Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!linearScan->enregisterLocalVars); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 350d6a65991699..8210519f38584e 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -209,8 +209,6 @@ enum _regMask_enum : unsigned // In any case, we believe that is OK to freely cast between these types; no information will // be lost. -typedef unsigned __int64 singleRegMask; - #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; typedef unsigned __int64 regMaskGpr; @@ -237,6 +235,8 @@ typedef unsigned __int64 regMaskAny; //TODO: Rename this to regMaskMixed // TODO: For LSRA, the select() method should be regMaskOnlyOne because we will be // allocating either GPR or Vector or Mask but not all +typedef unsigned __int64 singleRegMask; + #else // x86 and arm typedef unsigned regMaskTP; @@ -245,6 +245,7 @@ typedef unsigned regMaskTP; #define regMaskPredicate regMaskTP #define regMaskOnlyOne regMaskTP #define regMaskAny regMaskTP +#define singleRegMask regMaskTP #endif #if REGMASK_BITS == 8 From 1842bc2a23962cbf0ada94d397c60812de94c343 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:07:06 -0800 Subject: [PATCH 043/201] Update lsraarm.cpp and related code --- src/coreclr/jit/lsraarm.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 30991778868d62..424ede7a1fe234 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -194,8 +194,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskTP dstCandidates = RBM_NONE; - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -668,7 +667,7 @@ int LinearScan::BuildNode(GenTree* tree) { assert(dstCount == 1); regNumber argReg = tree->GetRegNum(); - regMaskTP argMask = RBM_NONE; + regMaskOnlyOne argMask = RBM_NONE; if (argReg != REG_COUNT) { argMask = genRegMask(argReg); From d3d81fe7166b69055f9b575540872bf4c18ee5a4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:07:48 -0800 Subject: [PATCH 044/201] Update lsraarm64.cpp and related code --- src/coreclr/jit/lsraarm64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index d0c26734d6d233..c4dcd3936754cb 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -584,7 +584,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. From 43a19d6024ceaa6d0813d41fe5db47880e18375e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:12:50 -0800 Subject: [PATCH 045/201] Update lsraarmarch.cpp and related code --- src/coreclr/jit/lsraarmarch.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 8b1305caec52ae..5b1e972787e82a 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -129,7 +129,7 @@ int LinearScan::BuildCall(GenTreeCall* call) { bool hasMultiRegRetVal = false; const ReturnTypeDesc* retTypeDesc = nullptr; - regMaskTP dstCandidates = RBM_NONE; + regMaskOnlyOne dstCandidates = RBM_NONE; int srcCount = 0; int dstCount = 0; @@ -149,7 +149,7 @@ int LinearScan::BuildCall(GenTreeCall* call) } GenTree* ctrlExpr = call->gtControlExpr; - regMaskTP ctrlExprCandidates = RBM_NONE; + regMaskGpr ctrlExprCandidates = RBM_NONE; if (call->gtCallType == CT_INDIRECT) { // either gtControlExpr != null or gtCallAddr != null. @@ -183,7 +183,7 @@ int LinearScan::BuildCall(GenTreeCall* call) { // For R2R and VSD we have stub address in REG_R2R_INDIRECT_PARAM // and will load call address into the temp register from this register. - regMaskTP candidates = RBM_NONE; + regMaskGpr candidates = RBM_NONE; if (call->IsFastTailCall()) { candidates = allRegs(TYP_INT) & RBM_INT_CALLEE_TRASH; @@ -205,7 +205,7 @@ int LinearScan::BuildCall(GenTreeCall* call) // the target. We do not handle these constraints on the same // refposition too well so we help ourselves a bit here by forcing the // null check with LR. - regMaskTP candidates = call->IsFastTailCall() ? RBM_LR : RBM_NONE; + regMaskGpr candidates = call->IsFastTailCall() ? RBM_LR : RBM_NONE; buildInternalIntRegisterDefForNode(call, candidates); } #endif // TARGET_ARM @@ -375,7 +375,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskTP killMask = getKillSetForCall(call); + regMaskAny killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -491,7 +491,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) int dstCount = argNode->gtNumRegs; regNumber argReg = argNode->GetRegNum(); - regMaskTP argMask = RBM_NONE; + regMaskAny argMask = RBM_NONE; for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); @@ -531,7 +531,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) // go into registers. for (unsigned regIndex = 0; regIndex < currentRegCount; regIndex++) { - regMaskTP sourceMask = RBM_NONE; + regMaskOnlyOne sourceMask = RBM_NONE; if (sourceRegCount < argNode->gtNumRegs) { sourceMask = genRegMask((regNumber)((unsigned)argReg + sourceRegCount)); @@ -589,9 +589,9 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) GenTree* srcAddrOrFill = nullptr; - regMaskTP dstAddrRegMask = RBM_NONE; - regMaskTP srcRegMask = RBM_NONE; - regMaskTP sizeRegMask = RBM_NONE; + regMaskGpr dstAddrRegMask = RBM_NONE; + regMaskGpr srcRegMask = RBM_NONE; + regMaskGpr sizeRegMask = RBM_NONE; if (blkNode->OperIsInitBlkOp()) { @@ -655,7 +655,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) // We don't need to materialize the struct size but we still need // a temporary register to perform the sequence of loads and stores. // We can't use the special Write Barrier registers, so exclude them from the mask - regMaskTP internalIntCandidates = + regMaskGpr internalIntCandidates = allRegs(TYP_INT) & ~(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF); buildInternalIntRegisterDefForNode(blkNode, internalIntCandidates); @@ -793,6 +793,8 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) if (!dstAddr->isContained()) { + assert(compiler->IsGprRegMask(dstAddrRegMask)); + useCount++; BuildUse(dstAddr, dstAddrRegMask); } @@ -805,6 +807,8 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) { if (!srcAddrOrFill->isContained()) { + assert(compiler->IsGprRegMask(srcRegMask)); + useCount++; BuildUse(srcAddrOrFill, srcRegMask); } @@ -816,12 +820,14 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) if (blkNode->OperIs(GT_STORE_DYN_BLK)) { + assert(compiler->IsGprRegMask(sizeRegMask)); + useCount++; BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); } buildInternalRegisterUses(); - regMaskTP killMask = getKillSetForBlockStore(blkNode); + regMaskAny killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; } From cdad3143985954a53e31f0a76e531bc1d376d1b1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:19:51 -0800 Subject: [PATCH 046/201] Update lsrabuild.cpp and related code --- src/coreclr/jit/lsrabuild.cpp | 76 ++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index e29a6c3a67326b..fbc673c75046cb 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -250,8 +250,8 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de assert(!interval->isLocalVar); RefPosition* useRefPosition = defRefPosition->nextRefPosition; - regMaskTP defRegAssignment = defRefPosition->registerAssignment; - regMaskTP useRegAssignment = useRefPosition->registerAssignment; + regMaskOnlyOne defRegAssignment = defRefPosition->registerAssignment; + regMaskOnlyOne useRegAssignment = useRefPosition->registerAssignment; RegRecord* defRegRecord = nullptr; RegRecord* useRegRecord = nullptr; regNumber defReg = REG_NA; @@ -351,7 +351,7 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de RegisterType regType = interval->registerType; assert((getRegisterType(interval, defRefPosition) == regType) && (getRegisterType(interval, useRefPosition) == regType)); - regMaskTP candidates = allRegs(regType); + regMaskOnlyOne candidates = allRegs(regType); defRefPosition->registerAssignment = candidates; defRefPosition->isFixedRegRef = false; return; @@ -422,8 +422,8 @@ void LinearScan::checkConflictingDefUse(RefPosition* useRP) // All defs must have a valid treeNode, but we check it below to be conservative. assert(defRP->treeNode != nullptr); - regMaskTP prevAssignment = defRP->registerAssignment; - regMaskTP newAssignment = (prevAssignment & useRP->registerAssignment); + regMaskOnlyOne prevAssignment = defRP->registerAssignment; + regMaskOnlyOne newAssignment = (prevAssignment & useRP->registerAssignment); if (newAssignment != RBM_NONE) { if (!isSingleRegister(newAssignment) || !theInterval->hasInterferingUses) @@ -518,7 +518,7 @@ void LinearScan::associateRefPosWithInterval(RefPosition* rp) // a new RefPosition // RefPosition* LinearScan::newRefPosition( - regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskTP mask) + regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskOnlyOne mask) { assert(compiler->IsOnlyOneRegMask(mask)); @@ -740,11 +740,11 @@ void LinearScan::addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) +regMaskAny LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) { assert(tree->OperIs(GT_STOREIND)); - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; GCInfo::WriteBarrierForm writeBarrierForm = compiler->codeGen->gcInfo.gcIsWriteBarrierCandidate(tree); if (writeBarrierForm != GCInfo::WBF_NoBarrier) @@ -840,9 +840,9 @@ regMaskGpr LinearScan::getKillSetForModDiv(GenTreeOp* node) // // Return Value: a register mask of the registers killed // -regMaskTP LinearScan::getKillSetForCall(GenTreeCall* call) +regMaskAny LinearScan::getKillSetForCall(GenTreeCall* call) { - regMaskTP killMask = RBM_CALLEE_TRASH; + regMaskAny killMask = RBM_CALLEE_TRASH; #ifdef TARGET_X86 if (compiler->compFloatingPointUsed) { @@ -962,7 +962,7 @@ regMaskAny LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) // regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) { - regMaskTP killMask = RBM_NONE; + regMaskGpr killMask = RBM_NONE; #ifdef TARGET_XARCH switch (node->GetHWIntrinsicId()) { @@ -1175,7 +1175,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo // See the "heuristics for writeThru intervals" in 'buildIntervals()'. if (!interval->isWriteThru || !isCallKill) { - regMaskTP newPreferences = allRegs(interval->registerType) & (~killMask); + regMaskOnlyOne newPreferences = allRegs(interval->registerType) & (~killMask); if (newPreferences != RBM_NONE) { @@ -1348,7 +1348,7 @@ RefPosition* LinearScan::defineNewInternalTemp(GenTree* tree, RegisterType regTy // Returns: // The def RefPosition created for this internal temp. // -RefPosition* LinearScan::buildInternalIntRegisterDefForNode(GenTree* tree, regMaskTP internalCands) +RefPosition* LinearScan::buildInternalIntRegisterDefForNode(GenTree* tree, regMaskGpr internalCands) { // The candidate set should contain only integer registers. assert(compiler->IsGprRegMask(internalCands)); @@ -1408,7 +1408,7 @@ void LinearScan::buildInternalRegisterUses() for (int i = 0; i < internalCount; i++) { RefPosition* def = internalDefs[i]; - regMaskTP mask = def->registerAssignment; + regMaskOnlyOne mask = def->registerAssignment; RefPosition* use = newRefPosition(def->getInterval(), currentLoc, RefTypeUse, def->treeNode, mask, 0); if (setInternalRegsDelayFree) { @@ -1838,7 +1838,7 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc // The use position of v02 cannot be allocated a reg since it is marked delay-reg free and // {eax,edx} are getting killed before the def of GT_DIV. For this reason, minRegCount for // the use position of v02 also needs to take into account the kill set of its consuming node. - regMaskTP killMask = getKillSetForNode(tree); + regMaskAny killMask = getKillSetForNode(tree); if (killMask != RBM_NONE) { minRegCountForRef += genCountBits(killMask); @@ -1853,8 +1853,8 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc if (newRefPosition->IsActualRef() && doReverseCallerCallee()) { Interval* interval = newRefPosition->getInterval(); - regMaskTP oldAssignment = newRefPosition->registerAssignment; - regMaskTP calleeSaveMask = calleeSaveRegs(interval->registerType); + regMaskOnlyOne oldAssignment = newRefPosition->registerAssignment; + regMaskOnlyOne calleeSaveMask = calleeSaveRegs(interval->registerType); #ifdef TARGET_ARM64 if (newRefPosition->isLiveAtConsecutiveRegistersLoc(consecutiveRegistersLocation)) { @@ -2298,7 +2298,7 @@ void LinearScan::buildIntervals() { Interval* interval = getIntervalForLocalVar(varIndex); const var_types regType = argDsc->GetRegisterType(); - regMaskTP mask = allRegs(regType); + regMaskOnlyOne mask = allRegs(regType); if (argDsc->lvIsRegArg) { // Set this interval as currently assigned to that register @@ -2492,7 +2492,7 @@ void LinearScan::buildIntervals() // into the scratch register, so it will be killed here. if (compiler->compShouldPoisonFrame() && compiler->fgFirstBBisScratch() && block == compiler->fgFirstBB) { - regMaskTP killed; + regMaskAny killed; #if defined(TARGET_XARCH) // Poisoning uses EAX for small vars and rep stosd that kills edi, ecx and eax for large vars. killed = RBM_EDI | RBM_ECX | RBM_EAX; @@ -2781,7 +2781,7 @@ void LinearScan::buildIntervals() availableRegCount = REG_INT_COUNT; } - if (availableRegCount < (sizeof(regMaskTP) * 8)) + if (availableRegCount < (sizeof(regMaskAny) * 8)) { // Mask out the bits that are between 64 ~ availableRegCount actualRegistersMask = (1ULL << availableRegCount) - 1; @@ -3017,8 +3017,10 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, i // Notes: // Adds the RefInfo for the definitions to the defList. // -void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskTP dstCandidates) +void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates) { + assert(compiler->IsOnlyOneRegMask(dstCandidates)); + bool fixedReg = false; if ((dstCount > 1) && (dstCandidates != RBM_NONE) && ((int)genCountBits(dstCandidates) == dstCount)) { @@ -3031,7 +3033,7 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskTP dstCandidates) } for (int i = 0; i < dstCount; i++) { - regMaskTP thisDstCandidates; + regMaskOnlyOne thisDstCandidates; if (fixedReg) { // In case of multi-reg call node, we have to query the i'th position return register. @@ -3138,7 +3140,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Find the registers that we should remove from the preference set because // they are occupied with argument values. - regMaskTP unpref = placedArgRegs; + regMaskAny unpref = placedArgRegs; unsigned varIndex = interval->getVarIndex(compiler); for (size_t i = 0; i < numPlacedArgLocals; i++) { @@ -3163,7 +3165,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) #endif interval->registerAversion |= unpref; - regMaskTP newPreferences = allRegs(interval->registerType) & ~unpref; + regMaskOnlyOne newPreferences = allRegs(interval->registerType) & ~unpref; interval->updateRegisterPreferences(newPreferences); } } @@ -3255,12 +3257,12 @@ RefPosition* LinearScan::BuildUse(GenTree* operand, regMaskOnlyOne candidates, i // Notes: // This method may only be used if the candidates are the same for all sources. // -int LinearScan::BuildIndirUses(GenTreeIndir* indirTree, regMaskTP candidates) +int LinearScan::BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates) { return BuildAddrUses(indirTree->Addr(), candidates); } -int LinearScan::BuildAddrUses(GenTree* addr, regMaskTP candidates) +int LinearScan::BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates) { if (!addr->isContained()) { @@ -3463,7 +3465,7 @@ void LinearScan::AddDelayFreeUses(RefPosition* useRefPosition, GenTree* rmwNode) // int LinearScan::BuildDelayFreeUses(GenTree* node, GenTree* rmwNode, - regMaskTP candidates, + regMaskOnlyOne candidates, RefPosition** useRefPositionRef) { RefPosition* use = nullptr; @@ -3560,7 +3562,7 @@ int LinearScan::BuildDelayFreeUses(GenTree* node, // The operands must already have been processed by buildRefPositionsForNode, and their // RefInfoListNodes placed in the defList. // -int LinearScan::BuildBinaryUses(GenTreeOp* node, regMaskTP candidates) +int LinearScan::BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates) { GenTree* op1 = node->gtGetOp1(); GenTree* op2 = node->gtGetOp2IfPresent(); @@ -3594,7 +3596,7 @@ int LinearScan::BuildBinaryUses(GenTreeOp* node, regMaskTP candidates) // Return Value: // The number of actual register operands. // -int LinearScan::BuildCastUses(GenTreeCast* cast, regMaskTP candidates) +int LinearScan::BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates) { GenTree* src = cast->CastOp(); @@ -3947,7 +3949,7 @@ int LinearScan::BuildReturn(GenTree* tree) #endif // !defined(TARGET_64BIT) if ((tree->TypeGet() != TYP_VOID) && !op1->isContained()) { - regMaskTP useCandidates = RBM_NONE; + regMaskOnlyOne useCandidates = RBM_NONE; #if FEATURE_MULTIREG_RET #ifdef TARGET_ARM64 @@ -3963,7 +3965,7 @@ int LinearScan::BuildReturn(GenTree* tree) // op1 has to be either a lclvar or a multi-reg returning call if ((op1->OperGet() == GT_LCL_VAR) && !op1->IsMultiRegLclVar()) { - BuildUse(op1, useCandidates); + BuildUse(op1, RBM_NONE); } else { @@ -3985,7 +3987,7 @@ int LinearScan::BuildReturn(GenTree* tree) if (srcType != dstType) { hasMismatchedRegTypes = true; - regMaskTP dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i)); + regMaskOnlyOne dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i)); if (varTypeUsesIntReg(dstType)) { @@ -4139,7 +4141,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) // To avoid redundant moves, have the argument operand computed in the // register in which the argument is passed to the call. - regMaskTP argMask = genRegMask(argReg); + singleRegMask argMask = genRegMask(argReg); RefPosition* use = BuildUse(op1, argMask); // Record that this register is occupied by a register now. @@ -4171,7 +4173,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) if (node->TypeGet() == TYP_LONG) { srcCount++; - regMaskTP argMaskHi = genRegMask(REG_NEXT(argReg)); + singleRegMask argMaskHi = genRegMask(REG_NEXT(argReg)); assert(genRegArgNext(argReg) == REG_NEXT(argReg)); use = BuildUse(op1, argMaskHi, 1); BuildDef(node, argMask, 0); @@ -4236,8 +4238,8 @@ int LinearScan::BuildGCWriteBarrier(GenTree* tree) // is an indir through an lea, we need to actually instantiate the // lea in a register assert(!addr->isContained() && !src->isContained()); - regMaskTP addrCandidates = RBM_WRITE_BARRIER_DST; - regMaskTP srcCandidates = RBM_WRITE_BARRIER_SRC; + regMaskGpr addrCandidates = RBM_WRITE_BARRIER_DST; + regMaskGpr srcCandidates = RBM_WRITE_BARRIER_SRC; #if defined(TARGET_X86) && NOGC_WRITE_BARRIERS @@ -4256,7 +4258,7 @@ int LinearScan::BuildGCWriteBarrier(GenTree* tree) BuildUse(addr, addrCandidates); BuildUse(src, srcCandidates); - regMaskTP killMask = getKillSetForStoreInd(tree->AsStoreInd()); + regMaskAny killMask = getKillSetForStoreInd(tree->AsStoreInd()); buildKillPositionsForNode(tree, currentLoc + 1, killMask); return 2; } From 7937068c86641f944476e4d8f77ca7da5a4813ea Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:28:03 -0800 Subject: [PATCH 047/201] Update lsraxarch.cpp and related code --- src/coreclr/jit/lsraxarch.cpp | 64 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index db6ed79da1ebde..b78579f2f2dbf7 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -47,7 +47,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskTP killMask = RBM_NONE; + regMaskAny killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -756,7 +756,7 @@ bool LinearScan::isRMWRegOper(GenTree* tree) } // Support for building RefPositions for RMW nodes. -int LinearScan::BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskTP candidates) +int LinearScan::BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates) { int srcCount = 0; regMaskGpr op1Candidates = candidates; @@ -765,7 +765,7 @@ int LinearScan::BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskT #ifdef TARGET_X86 if (varTypeIsByte(node)) { - regMaskTP byteCandidates = (candidates == RBM_NONE) ? allByteRegs() : (candidates & allByteRegs()); + regMaskGpr byteCandidates = (candidates == RBM_NONE) ? allByteRegs() : (candidates & allByteRegs()); if (!op1->isContained()) { assert(byteCandidates != RBM_NONE); @@ -1020,8 +1020,8 @@ int LinearScan::BuildShiftRotate(GenTree* tree) int srcCount = 0; GenTree* shiftBy = tree->gtGetOp2(); GenTree* source = tree->gtGetOp1(); - regMaskTP srcCandidates = RBM_NONE; - regMaskTP dstCandidates = RBM_NONE; + regMaskGpr srcCandidates = RBM_NONE; + regMaskGpr dstCandidates = RBM_NONE; // x64 can encode 8 bits of shift and it will use 5 or 6. (the others are masked off) // We will allow whatever can be encoded - hope you know what you are doing. @@ -1035,8 +1035,8 @@ int LinearScan::BuildShiftRotate(GenTree* tree) { // shlx (as opposed to mov+shl) instructions handles all register forms, but it does not handle contained form // for memory operand. Likewise for sarx and shrx. - srcCount += BuildOperandUses(source, srcCandidates); - srcCount += BuildOperandUses(shiftBy, srcCandidates); + srcCount += BuildOperandUses(source); + srcCount += BuildOperandUses(shiftBy); BuildDef(tree, dstCandidates); return srcCount; } @@ -1130,7 +1130,7 @@ int LinearScan::BuildCall(GenTreeCall* call) const ReturnTypeDesc* retTypeDesc = nullptr; int srcCount = 0; int dstCount = 0; - regMaskTP dstCandidates = RBM_NONE; + regMaskOnlyOne dstCandidates = RBM_NONE; assert(!call->isContained()); if (call->TypeGet() != TYP_VOID) @@ -1304,7 +1304,7 @@ int LinearScan::BuildCall(GenTreeCall* call) // set reg requirements on call target represented as control sequence. if (ctrlExpr != nullptr) { - regMaskTP ctrlExprCandidates = RBM_NONE; + regMaskGpr ctrlExprCandidates = RBM_NONE; // In case of fast tail implemented as jmp, make sure that gtControlExpr is // computed into appropriate registers. @@ -1344,7 +1344,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskTP killMask = getKillSetForCall(call); + regMaskAny killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -1370,9 +1370,9 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) GenTree* srcAddrOrFill = nullptr; - regMaskTP dstAddrRegMask = RBM_NONE; - regMaskTP srcRegMask = RBM_NONE; - regMaskTP sizeRegMask = RBM_NONE; + regMaskGpr dstAddrRegMask = RBM_NONE; + regMaskGpr srcRegMask = RBM_NONE; + regMaskGpr sizeRegMask = RBM_NONE; RefPosition* internalIntDef = nullptr; #ifdef TARGET_X86 @@ -1582,6 +1582,8 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) if (!dstAddr->isContained()) { + assert(compiler->IsGprRegMask(dstAddrRegMask)); + useCount++; BuildUse(dstAddr, dstAddrRegMask); } @@ -1594,6 +1596,8 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) { if (!srcAddrOrFill->isContained()) { + assert(compiler->IsGprRegMask(srcRegMask)); + useCount++; BuildUse(srcAddrOrFill, srcRegMask); } @@ -1605,6 +1609,8 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) if (blkNode->OperIs(GT_STORE_DYN_BLK)) { + assert(compiler->IsGprRegMask(sizeRegMask)); + useCount++; BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); } @@ -1630,7 +1636,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) #endif buildInternalRegisterUses(); - regMaskTP killMask = getKillSetForBlockStore(blkNode); + regMaskAny killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; @@ -1864,7 +1870,7 @@ int LinearScan::BuildModDiv(GenTree* tree) { GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); - regMaskTP dstCandidates = RBM_NONE; + regMaskGpr dstCandidates = RBM_NONE; int srcCount = 0; if (varTypeIsFloating(tree->TypeGet())) @@ -1918,11 +1924,12 @@ int LinearScan::BuildModDiv(GenTree* tree) srcCount = 1; } - srcCount += BuildDelayFreeUses(op2, op1, availableIntRegs & ~(RBM_RAX | RBM_RDX)); + assert(compiler->IsGprRegMask(dstCandidates)); + srcCount += BuildDelayFreeUses(op2, op1, availableIntRegs & ~(RBM_RAX | RBM_RDX)); buildInternalRegisterUses(); - regMaskTP killMask = getKillSetForModDiv(tree->AsOp()); + regMaskAny killMask = getKillSetForModDiv(tree->AsOp()); BuildDefsWithKills(tree, 1, dstCandidates, killMask); return srcCount; } @@ -2080,7 +2087,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou dstCount = 0; } - regMaskTP dstCandidates = RBM_NONE; + regMaskOnlyOne dstCandidates = RBM_NONE; if (intrinsicTree->GetOperandCount() == 0) { @@ -2662,7 +2669,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if (buildUses) { - regMaskTP op1RegCandidates = RBM_NONE; + regMaskFloat op1RegCandidates = RBM_NONE; #if defined(TARGET_AMD64) if (!isEvexCompatible) @@ -2687,7 +2694,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if (op2 != nullptr) { - regMaskTP op2RegCandidates = RBM_NONE; + regMaskFloat op2RegCandidates = RBM_NONE; #if defined(TARGET_AMD64) if (!isEvexCompatible) @@ -2733,7 +2740,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if (op3 != nullptr) { - regMaskTP op3RegCandidates = RBM_NONE; + regMaskFloat op3RegCandidates = RBM_NONE; #if defined(TARGET_AMD64) if (!isEvexCompatible) @@ -2747,7 +2754,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if (op4 != nullptr) { - regMaskTP op4RegCandidates = RBM_NONE; + regMaskFloat op4RegCandidates = RBM_NONE; #if defined(TARGET_AMD64) assert(isEvexCompatible); @@ -2853,8 +2860,7 @@ int LinearScan::BuildIndir(GenTreeIndir* indirTree) } #endif // FEATURE_SIMD - regMaskTP indirCandidates = RBM_NONE; - int srcCount = BuildIndirUses(indirTree, indirCandidates); + int srcCount = BuildIndirUses(indirTree); if (indirTree->gtOper == GT_STOREIND) { GenTree* source = indirTree->gtGetOp2(); @@ -2870,7 +2876,7 @@ int LinearScan::BuildIndir(GenTreeIndir* indirTree) } else { - regMaskTP srcCandidates = RBM_NONE; + regMaskGpr srcCandidates = RBM_NONE; #ifdef TARGET_X86 // Determine if we need byte regs for the non-mem source, if any. @@ -2910,6 +2916,7 @@ int LinearScan::BuildIndir(GenTreeIndir* indirTree) } #endif // TARGET_X86 + assert(compiler->IsGprRegMask(srcCandidates)); srcCount += BuildBinaryUses(source->AsOp(), srcCandidates); } } @@ -2977,7 +2984,7 @@ int LinearScan::BuildMul(GenTree* tree) int srcCount = BuildBinaryUses(tree->AsOp()); int dstCount = 1; - regMaskTP dstCandidates = RBM_NONE; + regMaskGpr dstCandidates = RBM_NONE; bool isUnsignedMultiply = ((tree->gtFlags & GTF_UNSIGNED) != 0); bool requiresOverflowCheck = tree->gtOverflowEx(); @@ -3032,7 +3039,10 @@ int LinearScan::BuildMul(GenTree* tree) { containedMemOp = op2; } - regMaskTP killMask = getKillSetForMul(tree->AsOp()); + + assert(compiler->IsGprRegMask(dstCandidates)); + + regMaskAny killMask = getKillSetForMul(tree->AsOp()); BuildDefsWithKills(tree, dstCount, dstCandidates, killMask); return srcCount; } From 82000a5150c579eeb3e841b5f34066e46d0131bd Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:37:08 -0800 Subject: [PATCH 048/201] Convert the return type of genRegMask() from `regMaskAny` to `singleRegMask` --- src/coreclr/jit/codegenarmarch.cpp | 2 +- src/coreclr/jit/codegencommon.cpp | 4 ++-- src/coreclr/jit/codegenxarch.cpp | 10 +++++----- src/coreclr/jit/emit.cpp | 6 +++--- src/coreclr/jit/emitxarch.cpp | 4 ++-- src/coreclr/jit/gcinfo.cpp | 2 +- src/coreclr/jit/lclvars.cpp | 2 +- src/coreclr/jit/lsra.cpp | 6 +++--- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/regalloc.cpp | 2 +- src/coreclr/jit/target.h | 9 +++++++-- src/coreclr/jit/utils.cpp | 2 +- 12 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index a18a6572957396..e81810cd8572ac 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3834,7 +3834,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - regMaskAny tempMask = genRegMask(varDsc->GetRegNum()); + singleRegMask tempMask = genRegMask(varDsc->GetRegNum()); regSet.RemoveMaskVars(tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index ca18f4aad54c79..ac4d992b72e45e 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5721,7 +5721,7 @@ void CodeGen::genFnProlog() if (isInReg) { regNumber regForVar = varDsc->GetRegNum(); - regMaskAny regMask = genRegMask(regForVar); + singleRegMask regMask = genRegMask(regForVar); if (!genIsValidFloatReg(regForVar)) { initRegs |= regMask; @@ -8112,7 +8112,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) regNumber targetReg = genRegCopy(treeNode, i); if (targetReg != sourceReg) { - regMaskAny targetRegMask = genRegMask(targetReg); + singleRegMask targetRegMask = genRegMask(targetReg); assert((busyRegs & targetRegMask) == 0); // Clear sourceReg from the busyRegs, and add targetReg. busyRegs &= ~genRegMask(sourceReg); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 69f74916d0731d..6cbaf912118d95 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9811,7 +9811,7 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // for (regNumber reg = REG_INT_LAST; tier0IntCalleeSaves != RBM_NONE; reg = REG_PREV(reg)) { - regMaskAny regBit = genRegMask(reg); + regMaskGpr regBit = genRegMask(reg); if ((regBit & tier0IntCalleeSaves) != 0) { @@ -9902,7 +9902,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // for (regNumber reg = REG_INT_LAST; osrAdditionalIntCalleeSaves != RBM_NONE; reg = REG_PREV(reg)) { - regMaskAny regBit = genRegMask(reg); + regMaskGpr regBit = genRegMask(reg); if ((regBit & osrAdditionalIntCalleeSaves) != 0) { @@ -9967,7 +9967,7 @@ void CodeGen::genPushCalleeSavedRegisters() // and all the other code that expects it to be in this order. for (regNumber reg = REG_INT_LAST; rsPushRegs != RBM_NONE; reg = REG_PREV(reg)) { - regMaskAny regBit = genRegMask(reg); + regMaskGpr regBit = genRegMask(reg); if ((regBit & rsPushRegs) != 0) { @@ -11217,7 +11217,7 @@ void CodeGen::genPreserveCalleeSavedFltRegs(unsigned lclFrameSize) for (regNumber reg = REG_FLT_CALLEE_SAVED_FIRST; regMask != RBM_NONE; reg = REG_NEXT(reg)) { - regMaskAny regBit = genRegMask(reg); + regMaskFloat regBit = genRegMask(reg); if ((regBit & regMask) != 0) { // ABI requires us to preserve lower 128-bits of YMM register. @@ -11281,7 +11281,7 @@ void CodeGen::genRestoreCalleeSavedFltRegs(unsigned lclFrameSize) for (regNumber reg = REG_FLT_CALLEE_SAVED_FIRST; regMask != RBM_NONE; reg = REG_NEXT(reg)) { - regMaskAny regBit = genRegMask(reg); + regMaskFloat regBit = genRegMask(reg); if ((regBit & regMask) != 0) { // ABI requires us to restore lower 128-bits of YMM register. diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 98ceee18a1fcf8..3b64dfb9d7e4c9 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3490,7 +3490,7 @@ void emitter::emitDispRegSet(regMaskAny regs) break; } - regMaskAny curReg = genRegMask(reg); + singleRegMask curReg = genRegMask(reg); if ((regs & curReg) == 0) { continue; @@ -9255,7 +9255,7 @@ void emitter::emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr) assert(needsGC(gcType)); - regMaskAny regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); regMaskGpr& emitThisXXrefRegs = (gcType == GCT_GCREF) ? emitThisGCrefRegs : emitThisByrefRegs; regMaskGpr& emitThisYYrefRegs = (gcType == GCT_GCREF) ? emitThisByrefRegs : emitThisGCrefRegs; @@ -9357,7 +9357,7 @@ void emitter::emitGCregDeadUpd(regNumber reg, BYTE* addr) return; } - regMaskAny regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); if ((emitThisGCrefRegs & regMask) != 0) { diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index c2d53fa0181e1d..15ec8ca1c9c472 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -14663,7 +14663,7 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) case IF_RRW: { #ifdef DEBUG - regMaskAny regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); #endif if (id->idGCref()) { @@ -17936,7 +17936,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) // The target of the 3-operand imul is implicitly encoded. Make sure // that we detected the implicit register and cleared its GC-status. - regMaskAny regMask = genRegMask(inst3opImulReg(ins)); + singleRegMask regMask = genRegMask(inst3opImulReg(ins)); assert((regMask & (emitThisGCrefRegs | emitThisByrefRegs)) == 0); } diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index d1360970f4afc6..46ac7c2acfe283 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -209,7 +209,7 @@ void GCInfo::gcMarkRegSetNpt(regMaskAny regMask DEBUGARG(bool forceOutput)) void GCInfo::gcMarkRegPtrVal(regNumber reg, var_types type) { - regMaskAny regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); switch (type) { diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 29b6f222832887..f56397f7bccd2d 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -5785,7 +5785,7 @@ int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, // On ARM we spill the registers in codeGen->regSet.rsMaskPreSpillRegArg // in the prolog, so we have to do SetStackOffset() here // - regMaskAny regMask = genRegMask(varDsc->GetArgReg()); + singleRegMask regMask = genRegMask(varDsc->GetArgReg()); if (codeGen->regSet.rsMaskPreSpillRegArg & regMask) { // Signature: void foo(struct_8, int, struct_4) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3a5b356665476d..a52d53de29f4fc 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -296,7 +296,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskAny mask, Interval* curre while (candidates != RBM_NONE) { regNumber regNum = genFirstRegNumFromMask(candidates); - regMaskAny candidateBit = genRegMask(regNum); + singleRegMask candidateBit = genRegMask(regNum); candidates ^= candidateBit; RegRecord* physRegRecord = getRegisterRecord(regNum); @@ -3233,7 +3233,7 @@ bool LinearScan::isRefPositionActive(RefPosition* refPosition, LsraLocation refL // bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord) { - regMaskAny candidateBit = genRegMask(physRegRecord->regNum); + singleRegMask candidateBit = genRegMask(physRegRecord->regNum); LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. @@ -3451,7 +3451,7 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - regMaskAny assignedRegMask = genRegMask(regRec->regNum); + singleRegMask assignedRegMask = genRegMask(regRec->regNum); compiler->codeGen->regSet.rsSetRegsModified(assignedRegMask DEBUGARG(true)); interval->assignedReg = regRec; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 9b22a3a5153a18..0dc4b0496a9c6a 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1745,7 +1745,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne getRegMask(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); - regMaskOnlyOne regMask = genRegMask(reg); + singleRegMask regMask = genRegMask(reg); #ifdef TARGET_ARM if (regType == TYP_DOUBLE) { diff --git a/src/coreclr/jit/regalloc.cpp b/src/coreclr/jit/regalloc.cpp index 85aef232603adc..cbf8c3c556c991 100644 --- a/src/coreclr/jit/regalloc.cpp +++ b/src/coreclr/jit/regalloc.cpp @@ -100,7 +100,7 @@ bool Compiler::shouldDoubleAlign( regNumber Compiler::raUpdateRegStateForArg(RegState* regState, LclVarDsc* argDsc) { regNumber inArgReg = argDsc->GetArgReg(); - regMaskAny inArgMask = genRegMask(inArgReg); + singleRegMask inArgMask = genRegMask(inArgReg); if (regState->rsIsFloat) { diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 8210519f38584e..fb434ae2b6de05 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -215,6 +215,11 @@ typedef unsigned __int64 regMaskGpr; typedef unsigned __int64 regMaskFloat; typedef unsigned __int64 regMaskPredicate; +// Design: +// 1. Reduce regMaskGpr to 32-bit +// 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskAny +// + // // We will add a "//TODO: regMaskOnlyOne" through out the code // that we know that the existing `regMaskTP` contains data that @@ -386,7 +391,7 @@ inline bool isByteReg(regNumber reg) } #endif -inline regMaskAny genRegMask(regNumber reg); +inline singleRegMask genRegMask(regNumber reg); inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); /***************************************************************************** @@ -593,7 +598,7 @@ inline bool floatRegCanHoldType(regNumber reg, var_types type) extern const regMaskSmall regMasks[REG_COUNT]; -inline regMaskOnlyOne genRegMask(regNumber reg) +inline singleRegMask genRegMask(regNumber reg) { assert((unsigned)reg < ArrLen(regMasks)); #ifdef TARGET_AMD64 diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index be8a3d2b0db321..254f53525fa88b 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -306,7 +306,7 @@ const char* dspRegRange(regMaskAny regMask, size_t& minSiz, const char* sep, reg for (regNumber regNum = regFirst; regNum <= regLast; regNum = REG_NEXT(regNum)) { - regMaskAny regBit = genRegMask(regNum); + singleRegMask regBit = genRegMask(regNum); if ((regMask & regBit) != 0) { From 515383b85dcf1eaaa8fabeb75721504edec6fd6e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:37:57 -0800 Subject: [PATCH 049/201] Rename regMaskAny to regMaskMixed --- src/coreclr/jit/codegen.h | 14 ++-- src/coreclr/jit/codegenarm.cpp | 8 +- src/coreclr/jit/codegenarm64.cpp | 16 ++-- src/coreclr/jit/codegenarmarch.cpp | 6 +- src/coreclr/jit/codegencommon.cpp | 16 ++-- src/coreclr/jit/codegeninterface.h | 2 +- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 18 ++--- src/coreclr/jit/compiler.cpp | 14 ++-- src/coreclr/jit/compiler.h | 12 +-- src/coreclr/jit/compiler.hpp | 8 +- src/coreclr/jit/emit.cpp | 8 +- src/coreclr/jit/emit.h | 6 +- src/coreclr/jit/emitarm.cpp | 2 +- src/coreclr/jit/emitarm64.cpp | 2 +- src/coreclr/jit/emitxarch.cpp | 4 +- src/coreclr/jit/gcinfo.cpp | 12 +-- src/coreclr/jit/gentree.cpp | 26 +++---- src/coreclr/jit/gentree.h | 14 ++-- src/coreclr/jit/jitgcinfo.h | 8 +- src/coreclr/jit/lclvars.cpp | 6 +- src/coreclr/jit/lsra.cpp | 120 ++++++++++++++--------------- src/coreclr/jit/lsra.h | 66 ++++++++-------- src/coreclr/jit/lsraarm.cpp | 2 +- src/coreclr/jit/lsraarm64.cpp | 2 +- src/coreclr/jit/lsraarmarch.cpp | 6 +- src/coreclr/jit/lsrabuild.cpp | 38 ++++----- src/coreclr/jit/lsraxarch.cpp | 10 +-- src/coreclr/jit/regset.cpp | 12 +-- src/coreclr/jit/regset.h | 22 +++--- src/coreclr/jit/target.h | 19 ++--- src/coreclr/jit/unwind.cpp | 2 +- src/coreclr/jit/utils.cpp | 4 +- 33 files changed, 254 insertions(+), 253 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 8b5b6f61da2063..fbabacb566aa3c 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -337,8 +337,8 @@ class CodeGen final : public CodeGenInterface void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); - void genSaveCalleeSavedRegistersHelp(regMaskAny regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); - void genRestoreCalleeSavedRegistersHelp(regMaskAny regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); + void genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); + void genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); @@ -357,7 +357,7 @@ class CodeGen final : public CodeGenInterface void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn); - void genPoisonFrame(regMaskAny bbRegLiveIn); + void genPoisonFrame(regMaskMixed bbRegLiveIn); #if defined(TARGET_ARM) @@ -385,7 +385,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskAny fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) + regMaskMixed fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer unsigned fiSpDelta; // Stack pointer delta unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP @@ -401,7 +401,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) @@ -436,7 +436,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) @@ -455,7 +455,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskAny fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 27158487ba7ab7..68402f3280bf6b 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2173,7 +2173,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskAny maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; regMaskFloat maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT; regMaskGpr maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; @@ -2490,7 +2490,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() unsigned preSpillRegArgSize = genCountBits(regSet.rsMaskPreSpillRegs(true)) * REGSIZE_BYTES; genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; - regMaskAny rsMaskSaveRegs = regSet.rsMaskCalleeSaved; + regMaskMixed rsMaskSaveRegs = regSet.rsMaskCalleeSaved; unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; @@ -2646,9 +2646,9 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu regNumber rAddr; regNumber rCnt = REG_NA; // Invalid - regMaskAny regMask; + regMaskMixed regMask; - regMaskAny availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers + regMaskMixed availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers availMask &= ~intRegState.rsCalleeRegArgMaskLiveIn; // Remove all of the incoming argument registers as they are // currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg as we will zero it and maybe use it for diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 7d08d089fde1e3..3aea34a32de3b5 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,7 +36,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskAny rsRestoreRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed rsRestoreRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; if (isFramePointerUsed()) { @@ -45,7 +45,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) rsRestoreRegs |= RBM_LR; // We must save/restore the return address (in the LR register) - regMaskAny regsToRestoreMask = rsRestoreRegs; + regMaskMixed regsToRestoreMask = rsRestoreRegs; const int totalFrameSize = genTotalFrameSize(); @@ -904,7 +904,7 @@ void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDel // The save set can contain LR in which case LR is saved along with the other callee-saved registers. // But currently Jit doesn't use frames without frame pointer on arm64. // -void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskAny regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); @@ -1019,7 +1019,7 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int sp // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskAny regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta >= 0); unsigned regsToRestoreCount = genCountBits(regsToRestoreMask); @@ -1598,7 +1598,7 @@ void CodeGen::genFuncletEpilog() } } - regMaskAny regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; + regMaskMixed regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; if ((genFuncletInfo.fiFrameType == 1) || (genFuncletInfo.fiFrameType == 2) || (genFuncletInfo.fiFrameType == 3)) { regsToRestoreMask &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end @@ -1735,7 +1735,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiFunction_CallerSP_to_FP_delta = genCallerSPtoFPdelta() - osrPad; - regMaskAny rsMaskSaveRegs = regSet.rsMaskCalleeSaved; + regMaskMixed rsMaskSaveRegs = regSet.rsMaskCalleeSaved; assert((rsMaskSaveRegs & RBM_LR) != 0); assert((rsMaskSaveRegs & RBM_FP) != 0); @@ -5102,7 +5102,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, } regMaskGpr callTargetMask = genRegMask(callTargetReg); - regMaskAny callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskMixed callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet noway_assert((callTargetMask & callKillSet) == callTargetMask); @@ -5124,7 +5124,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, false /* isJump */ ); - regMaskAny killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); } diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index e81810cd8572ac..5d95fff549b9ce 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3433,7 +3433,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskAny killMask = RBM_CALLEE_TRASH; + regMaskMixed killMask = RBM_CALLEE_TRASH; if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); @@ -3576,7 +3576,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (call->IsFastTailCall()) { - regMaskAny trashedByEpilog = RBM_CALLEE_SAVED; + regMaskMixed trashedByEpilog = RBM_CALLEE_SAVED; // The epilog may use and trash REG_GSCOOKIE_TMP_0/1. Make sure we have no // non-standard args that may be trash if this is a tailcall. @@ -4898,7 +4898,7 @@ void CodeGen::genPushCalleeSavedRegisters() intRegState.rsCalleeRegArgMaskLiveIn); #endif - regMaskAny rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index ac4d992b72e45e..7c7e1134b717c3 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -128,7 +128,7 @@ CodeGen::CodeGen(Compiler* theCompiler) : CodeGenInterface(theCompiler) #if defined(TARGET_XARCH) // Shouldn't be used before it is set in genFnProlog() - compiler->compCalleeFPRegsSavedMask = (regMaskAny)-1; + compiler->compCalleeFPRegsSavedMask = (regMaskMixed)-1; #endif // defined(TARGET_XARCH) #endif // DEBUG @@ -623,7 +623,7 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // Return Value: // Mask of register kills -- registers whose values are no longer guaranteed to be the same. // -regMaskAny Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) +regMaskMixed Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) { switch (helper) { @@ -748,7 +748,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) { // TODO-Cleanup: Move the code from compUpdateLifeVar to genUpdateRegLife that updates the // gc sets - regMaskAny regMask = varDsc->lvRegMask(); + regMaskMixed regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur &= ~regMask; @@ -793,7 +793,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) VarSetOps::RemoveElemD(this, codeGen->gcInfo.gcVarPtrSetCur, bornVarIndex); } codeGen->genUpdateRegLife(varDsc, true /*isBorn*/, false /*isDying*/ DEBUGARG(nullptr)); - regMaskAny regMask = varDsc->lvRegMask(); + regMaskMixed regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur |= regMask; @@ -5360,7 +5360,7 @@ void CodeGen::genFinalizeFrame() // We always save FP. noway_assert(isFramePointerUsed()); #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - regMaskAny okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED); + regMaskMixed okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED); if (RBM_ENC_CALLEE_SAVED != 0) { regSet.rsSetRegsModified(RBM_ENC_CALLEE_SAVED); @@ -5397,7 +5397,7 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - regMaskAny maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; #ifdef TARGET_ARMARCH if (isFramePointerUsed()) @@ -8096,7 +8096,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) // First set the source registers as busy if they haven't been spilled. // (Note that this is just for verification that we don't have circular dependencies.) - regMaskAny busyRegs = RBM_NONE; + regMaskMixed busyRegs = RBM_NONE; for (unsigned i = 0; i < regCount; ++i) { if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0) @@ -8309,7 +8309,7 @@ unsigned CodeGenInterface::getCurrentStackLevel() const // This function emits code to poison address exposed non-zero-inited local variables. We expect this function // to be called when emitting code for the scratch BB that comes right after the prolog. // The variables are poisoned using 0xcdcdcdcd. -void CodeGen::genPoisonFrame(regMaskAny regLiveIn) +void CodeGen::genPoisonFrame(regMaskMixed regLiveIn) { assert(compiler->compShouldPoisonFrame()); #if defined(TARGET_XARCH) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index 17db64a8c6f312..058cb1fa81e887 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -80,7 +80,7 @@ class CodeGenInterface // Call this function after the equivalent fields in Compiler have been initialized. void CopyRegisterInfo(); - FORCEINLINE regMaskAny get_RBM_ALLMASK() const + FORCEINLINE regMaskMixed get_RBM_ALLMASK() const { return this->rbmAllMask; } diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 28a0bdebbee970..0c461272c75692 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -201,7 +201,7 @@ void CodeGen::genCodeForBBlist() // change? We cleared them out above. Maybe we should just not clear them out, but update the ones that change // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). - regMaskAny newLiveRegSet = RBM_NONE; + regMaskMixed newLiveRegSet = RBM_NONE; regMaskGpr newRegGCrefSet = RBM_NONE; regMaskGpr newRegByrefSet = RBM_NONE; #ifdef DEBUG diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 6cbaf912118d95..8eae1aa637f954 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6096,7 +6096,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskAny killMask = RBM_CALLEE_TRASH; + regMaskMixed killMask = RBM_CALLEE_TRASH; if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); @@ -6613,7 +6613,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - regMaskAny tempMask = varDsc->lvRegMask(); + regMaskMixed tempMask = varDsc->lvRegMask(); regSet.RemoveMaskVars(tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) @@ -8441,7 +8441,7 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk) regNumber simdTmpReg = REG_NA; if (putArgStk->AvailableTempRegCount() != 0) { - regMaskAny rsvdRegs = putArgStk->gtRsvdRegs; + regMaskMixed rsvdRegs = putArgStk->gtRsvdRegs; if ((rsvdRegs & RBM_ALLINT) != 0) { intTmpReg = putArgStk->GetSingleTempReg(RBM_ALLINT); @@ -9144,7 +9144,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); regNumber callTarget = REG_NA; - regMaskAny killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { @@ -9672,7 +9672,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // registers that profiler callback kills. if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { - regMaskAny thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); + regMaskMixed thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); noway_assert((RBM_PROFILER_LEAVE_TRASH & thisPtrMask) == 0); } @@ -9788,7 +9788,7 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // Emit appropriate unwind. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; int const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; @@ -9868,7 +9868,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // Figure out which set of int callee saves still needs saving. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; regMaskGpr const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; @@ -10202,7 +10202,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskAny const tier0CalleeSaves = (regMaskAny)patchpointInfo->CalleeSaveRegisters(); + regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); regMaskGpr const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; @@ -10734,7 +10734,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() assert(isFramePointerUsed()); assert(compiler->lvaDoneFrameLayout == Compiler::FINAL_FRAME_LAYOUT); // The frame size and offsets must be // finalized - assert(compiler->compCalleeFPRegsSavedMask != (regMaskAny)-1); // The float registers to be preserved is finalized + assert(compiler->compCalleeFPRegsSavedMask != (regMaskMixed)-1); // The float registers to be preserved is finalized // Even though lvaToInitialSPRelativeOffset() depends on compLclFrameSize, // that's ok, because we're figuring out an offset in the parent frame. diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 05c8fa2c916e26..da2f67059a4317 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3382,12 +3382,12 @@ void Compiler::compInitOptions(JitFlags* jitFlags) // Make sure we copy the register info and initialize the // trash regs after the underlying fields are initialized - const regMaskAny vtCalleeTrashRegs[TYP_COUNT]{ + const regMaskMixed vtCalleeTrashRegs[TYP_COUNT]{ #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP }; - memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskAny) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskMixed) * TYP_COUNT); codeGen->CopyRegisterInfo(); #endif // TARGET_XARCH @@ -3631,7 +3631,7 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) // Arguments: // regs - The set of registers to display // -void Compiler::dumpRegMask(regMaskAny regs) const +void Compiler::dumpRegMask(regMaskMixed regs) const { if (regs == RBM_ALLINT) { @@ -5773,11 +5773,11 @@ void Compiler::generatePatchpointInfo() // Record callee save registers. // Currently only needed for x64. // - regMaskAny rsPushRegs = codeGen->regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed rsPushRegs = codeGen->regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; rsPushRegs |= RBM_FPBASE; patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs); JITDUMP("--OSR-- Tier0 callee saves: "); - JITDUMPEXEC(dspRegMask((regMaskAny)patchpointInfo->CalleeSaveRegisters())); + JITDUMPEXEC(dspRegMask((regMaskMixed)patchpointInfo->CalleeSaveRegisters())); JITDUMP("\n"); #endif @@ -9336,7 +9336,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX * cVN, dVN : Display a ValueNum (call vnPrint()). * * The following don't require a Compiler* to work: - * dRegMask : Display a regMaskAny (call dspRegMask(mask)). + * dRegMask : Display a regMaskMixed (call dspRegMask(mask)). * dBlockList : Display a BasicBlockList*. * * The following find an object in the IR and return it, as well as setting a global variable with the value that can @@ -10287,7 +10287,7 @@ JITDBGAPI void __cdecl dVN(ValueNum vn) cVN(JitTls::GetCompiler(), vn); } -JITDBGAPI void __cdecl dRegMask(regMaskAny mask) +JITDBGAPI void __cdecl dRegMask(regMaskMixed mask) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== dRegMask %u\n", sequenceNumber++); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 28ad3caf242b83..16e55077cf7550 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1005,9 +1005,9 @@ class LclVarDsc return lvIsRegCandidate() && (GetRegNum() != REG_STK); } - regMaskAny lvRegMask() const + regMaskMixed lvRegMask() const { - regMaskAny regMask = RBM_NONE; + regMaskMixed regMask = RBM_NONE; if (GetRegNum() != REG_STK) { if (varTypeUsesFloatReg(this)) @@ -3866,7 +3866,7 @@ class Compiler unsigned lvaGetMaxSpillTempSize(); #ifdef TARGET_ARM - bool lvaIsPreSpilled(unsigned lclNum, regMaskAny preSpillMask); + bool lvaIsPreSpilled(unsigned lclNum, regMaskMixed preSpillMask); #endif // TARGET_ARM void lvaAssignFrameOffsets(FrameLayoutState curState); void lvaFixVirtualFrameOffsets(); @@ -8422,7 +8422,7 @@ class Compiler // Gets a register mask that represent the kill set for a helper call since // not all JIT Helper calls follow the standard ABI on the target architecture. - regMaskAny compHelperCallKillSet(CorInfoHelpFunc helper); + regMaskMixed compHelperCallKillSet(CorInfoHelpFunc helper); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -10806,7 +10806,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool compJitHaltMethod(); - void dumpRegMask(regMaskAny regs) const; + void dumpRegMask(regMaskMixed regs) const; #endif @@ -11130,7 +11130,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX regMaskPredicate rbmAllMask; regMaskPredicate rbmMskCalleeTrash; unsigned cntCalleeTrashMask; - regMaskAny varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskMixed varTypeCalleeTrashRegs[TYP_COUNT]; public: FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 32bed4cf267907..43d62d39f94b78 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -944,7 +944,7 @@ inline regNumber genRegNumFromMask(regMaskOnlyOne mask) // the bit. // -inline regNumber genFirstRegNumFromMaskAndToggle(regMaskAny& mask) +inline regNumber genFirstRegNumFromMaskAndToggle(regMaskMixed& mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -966,7 +966,7 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskAny& mask) // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(regMaskAny mask) +inline regNumber genFirstRegNumFromMask(regMaskMixed mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -4431,12 +4431,12 @@ inline void* operator new[](size_t sz, Compiler* compiler, CompMemKind cmk) #ifdef DEBUG -inline void printRegMask(regMaskAny mask) +inline void printRegMask(regMaskMixed mask) { printf(REG_MASK_ALL_FMT, mask); } -inline char* regMaskToString(regMaskAny mask, Compiler* context) +inline char* regMaskToString(regMaskMixed mask, Compiler* context) { const size_t cchRegMask = 24; char* regmask = new (context, CMK_Unknown) char[cchRegMask]; diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 3b64dfb9d7e4c9..045e29139610fb 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3476,7 +3476,7 @@ const char* emitter::emitGetFrameReg() * Display a register set in a readable form. */ -void emitter::emitDispRegSet(regMaskAny regs) +void emitter::emitDispRegSet(regMaskMixed regs) { regNumber reg; bool sp = false; @@ -10374,7 +10374,7 @@ const char* emitter::emitOffsetToLabel(unsigned offs) // Return value: // the saved set of registers. // -regMaskAny emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) +regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? bool isNoGCHelper = emitNoGChelper(methHnd); @@ -10420,10 +10420,10 @@ regMaskAny emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) // Return Value: // Mask of GC register kills // -regMaskAny emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) +regMaskMixed emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); - regMaskAny result; + regMaskMixed result; switch (helper) { case CORINFO_HELP_ASSIGN_REF: diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 0e0d7b8bf76740..256c6079b7c681 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3135,10 +3135,10 @@ class emitter bool emitFullGCinfo; // full GC pointer maps? bool emitFullyInt; // fully interruptible code? - regMaskAny emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); + regMaskMixed emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); // Gets a register mask that represent the kill set for a NoGC helper call. - regMaskAny emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); + regMaskMixed emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); #if EMIT_TRACK_STACK_DEPTH unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere @@ -3197,7 +3197,7 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regMaskAny regs); + void emitDispRegSet(regMaskMixed regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index f34be27bb6389d..06b986a6ea971b 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -4694,7 +4694,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 80b518e6e75a5a..7eb74ce85f2cfa 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14503,7 +14503,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 15ec8ca1c9c472..e208b45956ce9d 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9499,7 +9499,7 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs((signed)argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskAny savedSet = emitGetGCRegsSavedOrModified(methHnd); + regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); gcrefRegs &= savedSet; byrefRegs &= savedSet; @@ -15006,7 +15006,7 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) // instruction, if writing a GC ref even through reading a long, will go live here. // These situations typically occur due to unsafe casting, such as with Span. - regMaskAny regMask; + regMaskMixed regMask; regMask = genRegMask(reg1) | genRegMask(reg2); // r1/r2 could have been a GCREF as GCREF + int=BYREF diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 46ac7c2acfe283..ce523b497dbfb2 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -83,7 +83,7 @@ void GCInfo::gcResetForBB() * Print the changes in the gcRegGCrefSetCur sets. */ -void GCInfo::gcDspGCrefSetChanges(regMaskAny gcRegGCrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -112,7 +112,7 @@ void GCInfo::gcDspGCrefSetChanges(regMaskAny gcRegGCrefSetNew DEBUGARG(bool forc * Print the changes in the gcRegByrefSetCur sets. */ -void GCInfo::gcDspByrefSetChanges(regMaskAny gcRegByrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspByrefSetChanges(regMaskMixed gcRegByrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -188,12 +188,12 @@ void GCInfo::gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput)) * non-pointer values. */ -void GCInfo::gcMarkRegSetNpt(regMaskAny regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetNpt(regMaskMixed regMask DEBUGARG(bool forceOutput)) { /* NOTE: don't unmark any live register variables */ - regMaskAny gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); - regMaskAny gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetMaskVars()); + regMaskMixed gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); + regMaskMixed gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetMaskVars()); INDEBUG(gcDspGCrefSetChanges(gcRegGCrefSetNew, forceOutput)); INDEBUG(gcDspByrefSetChanges(gcRegByrefSetNew, forceOutput)); @@ -722,7 +722,7 @@ void GCInfo::gcRegPtrSetInit() // It is also called by LinearScan::recordVarLocationAtStartOfBB() which is in turn called by // CodeGen::genCodeForBBList() at the block boundary. -void GCInfo::gcUpdateForRegVarMove(regMaskAny srcMask, regMaskAny dstMask, LclVarDsc* varDsc) +void GCInfo::gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, LclVarDsc* varDsc) { var_types type = varDsc->TypeGet(); bool isGCRef = (type == TYP_REF); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 82aca4b00a59f1..503e918cb92dac 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1059,14 +1059,14 @@ bool GenTree::NeedsConsecutiveRegisters() const // Return Value: // Reg Mask of GenTree node. // -regMaskAny GenTree::gtGetContainedRegMask() +regMaskMixed GenTree::gtGetContainedRegMask() { if (!isContained()) { return isUsedFromReg() ? gtGetRegMask() : RBM_NONE; } - regMaskAny mask = 0; + regMaskMixed mask = 0; for (GenTree* operand : Operands()) { mask |= operand->gtGetContainedRegMask(); @@ -1083,9 +1083,9 @@ regMaskAny GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -regMaskAny GenTree::gtGetRegMask() const +regMaskMixed GenTree::gtGetRegMask() const { - regMaskAny resultMask; + regMaskMixed resultMask; if (IsMultiRegCall()) { @@ -2085,9 +2085,9 @@ void CallArgs::Remove(CallArg* arg) // Return Value: // Reg mask of gtOtherRegs of call node. // -regMaskAny GenTreeCall::GetOtherRegMask() const +regMaskMixed GenTreeCall::GetOtherRegMask() const { - regMaskAny resultMask = RBM_NONE; + regMaskMixed resultMask = RBM_NONE; #if FEATURE_MULTIREG_RET for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i) @@ -27067,9 +27067,9 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx) const // of return registers and wants to know the set of return registers. // // static -regMaskAny ReturnTypeDesc::GetABIReturnRegs() const +regMaskMixed ReturnTypeDesc::GetABIReturnRegs() const { - regMaskAny resultMask = RBM_NONE; + regMaskMixed resultMask = RBM_NONE; unsigned count = GetReturnRegCount(); for (unsigned i = 0; i < count; ++i) @@ -27094,7 +27094,7 @@ regMaskAny ReturnTypeDesc::GetABIReturnRegs() const // Return Value: // Count of available temporary registers in given set. // -unsigned GenTree::AvailableTempRegCount(regMaskAny mask /* = (regMaskAny)-1 */) const +unsigned GenTree::AvailableTempRegCount(regMaskMixed mask /* = (regMaskMixed)-1 */) const { return genCountBits(gtRsvdRegs & mask); } @@ -27111,9 +27111,9 @@ unsigned GenTree::AvailableTempRegCount(regMaskAny mask /* = (regMaskAny)-1 */) // Return Value: // Available temporary register in given mask. // -regNumber GenTree::GetSingleTempReg(regMaskAny mask /* = (regMaskAny)-1 */) +regNumber GenTree::GetSingleTempReg(regMaskMixed mask /* = (regMaskMixed)-1 */) { - regMaskAny availableSet = gtRsvdRegs & mask; + regMaskMixed availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); regNumber tempReg = genRegNumFromMask(availableSet); INDEBUG(gtRsvdRegs &= ~availableSet;) // Remove the register from the set, so it can't be used again. @@ -27132,9 +27132,9 @@ regNumber GenTree::GetSingleTempReg(regMaskAny mask /* = (regMaskAny)-1 */) // Return Value: // Available temporary register in given mask. // -regNumber GenTree::ExtractTempReg(regMaskAny mask /* = (regMaskAny)-1 */) +regNumber GenTree::ExtractTempReg(regMaskMixed mask /* = (regMaskMixed)-1 */) { - regMaskAny availableSet = gtRsvdRegs & mask; + regMaskMixed availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); regNumber tempReg = genFirstRegNumFromMask(availableSet); gtRsvdRegs ^= genRegMask(tempReg); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 7306b045010384..96bf8c9d50cd88 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -957,8 +957,8 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; - regMaskAny gtGetRegMask() const; - regMaskAny gtGetContainedRegMask(); + regMaskMixed gtGetRegMask() const; + regMaskMixed gtGetContainedRegMask(); GenTreeFlags gtFlags; @@ -970,9 +970,9 @@ struct GenTree regMaskSmall gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskAny mask = (regMaskAny)-1) const; - regNumber GetSingleTempReg(regMaskAny mask = (regMaskAny)-1); - regNumber ExtractTempReg(regMaskAny mask = (regMaskAny)-1); + unsigned AvailableTempRegCount(regMaskMixed mask = (regMaskMixed)-1) const; + regNumber GetSingleTempReg(regMaskMixed mask = (regMaskMixed)-1); + regNumber ExtractTempReg(regMaskMixed mask = (regMaskMixed)-1); void SetVNsFromNode(GenTree* tree) { @@ -4319,7 +4319,7 @@ struct ReturnTypeDesc regNumber GetABIReturnReg(unsigned idx) const; // Get reg mask of ABI return registers - regMaskAny GetABIReturnRegs() const; + regMaskMixed GetABIReturnRegs() const; }; class TailCallSiteInfo @@ -5124,7 +5124,7 @@ struct GenTreeCall final : public GenTree } // Get reg mask of all the valid registers of gtOtherRegs array - regMaskAny GetOtherRegMask() const; + regMaskMixed GetOtherRegMask() const; GenTreeFlags GetRegSpillFlagByIdx(unsigned idx) const { diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index b5c27557cf295c..c3f41f36ec7998 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -91,12 +91,12 @@ class GCInfo void gcMarkRegSetGCref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); - void gcMarkRegSetNpt(regMaskAny regMask DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetNpt(regMaskMixed regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegPtrVal(regNumber reg, var_types type); #ifdef DEBUG - void gcDspGCrefSetChanges(regMaskAny gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); - void gcDspByrefSetChanges(regMaskAny gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspByrefSetChanges(regMaskMixed gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); #endif // DEBUG /*****************************************************************************/ @@ -392,7 +392,7 @@ class GCInfo public: // This method updates the appropriate reg masks when a variable is moved. - void gcUpdateForRegVarMove(regMaskAny srcMask, regMaskAny dstMask, LclVarDsc* varDsc); + void gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, LclVarDsc* varDsc); private: ReturnKind getReturnKind(); diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index f56397f7bccd2d..a968f163dc4429 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -5358,7 +5358,7 @@ void Compiler::lvaFixVirtualFrameOffsets() } #ifdef TARGET_ARM -bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskAny preSpillMask) +bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskMixed preSpillMask) { const LclVarDsc& desc = lvaTable[lclNum]; return desc.lvIsRegArg && (preSpillMask & genRegMask(desc.GetArgReg())); @@ -5556,8 +5556,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() unsigned argLcls = 0; // Take care of pre spill registers first. - regMaskAny preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); - regMaskAny tempMask = RBM_NONE; + regMaskMixed preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); + regMaskMixed tempMask = RBM_NONE; for (unsigned i = 0, preSpillLclNum = lclNum; i < argSigLen; ++i, ++preSpillLclNum) { if (lvaIsPreSpilled(preSpillLclNum, preSpillMask)) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index a52d53de29f4fc..efbbd3e6c0ff47 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -288,11 +288,11 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo nextFixedRef[regRecord->regNum] = nextLocation; } -regMaskOnlyOne LinearScan::getMatchingConstants(regMaskAny mask, Interval* currentInterval, RefPosition* refPosition) +regMaskOnlyOne LinearScan::getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); - regMaskAny candidates = (mask & m_RegistersWithConstants); - regMaskAny result = RBM_NONE; + regMaskMixed candidates = (mask & m_RegistersWithConstants); + regMaskMixed result = RBM_NONE; while (candidates != RBM_NONE) { regNumber regNum = genFirstRegNumFromMask(candidates); @@ -472,12 +472,12 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // New regMask that has minRegCount registers after intersection. // Otherwise returns regMaskActual. // -regMaskAny LinearScan::getConstrainedRegMask(RefPosition* refPosition, - regMaskAny regMaskActual, - regMaskAny regMaskConstraint, +regMaskMixed LinearScan::getConstrainedRegMask(RefPosition* refPosition, + regMaskMixed regMaskActual, + regMaskMixed regMaskConstraint, unsigned minRegCount) { - regMaskAny newMask = regMaskActual & regMaskConstraint; + regMaskMixed newMask = regMaskActual & regMaskConstraint; if (genCountBits(newMask) < minRegCount) { // Constrained mask does not have minimum required registers needed. @@ -486,7 +486,7 @@ regMaskAny LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { - regMaskAny busyRegs = regsBusyUntilKill | regsInUseThisLocation; + regMaskMixed busyRegs = regsBusyUntilKill | regsInUseThisLocation; if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -512,7 +512,7 @@ regMaskAny LinearScan::getConstrainedRegMask(RefPosition* refPosition, // This is the method used to implement the stress options that limit // the set of registers considered for allocation. // -regMaskAny LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskAny mask) +regMaskMixed LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskMixed mask) { #ifdef TARGET_ARM64 if ((refPosition != nullptr) && refPosition->isLiveAtConsecutiveRegistersLoc(consecutiveRegistersLocation)) @@ -729,7 +729,7 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_XARCH) rbmAllMask = compiler->rbmAllMask; rbmMskCalleeTrash = compiler->rbmMskCalleeTrash; - memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskAny) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskMixed) * TYP_COUNT); if (!compiler->canUseEvexEncoding()) { @@ -2735,7 +2735,7 @@ void LinearScan::setFrameType() // If we are using FPBASE as the frame register, we cannot also use it for // a local var. - regMaskAny removeMask = RBM_NONE; + regMaskMixed removeMask = RBM_NONE; if (frameType == FT_EBP_FRAME) { removeMask |= RBM_FPBASE; @@ -2818,7 +2818,7 @@ RegisterType LinearScan::getRegisterType(Interval* currentInterval, RefPosition* { assert(refPosition->getInterval() == currentInterval); RegisterType regType = currentInterval->registerType; - regMaskAny candidates = refPosition->registerAssignment; + regMaskMixed candidates = refPosition->registerAssignment; #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // The LoongArch64's ABI which the float args maybe passed by integer register // when no float register left but free integer register. @@ -2935,7 +2935,7 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, { assert(!enregisterLocalVars); regNumber foundReg; - regMaskAny foundRegBit; + regMaskMixed foundRegBit; RegRecord* availablePhysRegRecord; foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) @@ -2996,7 +2996,7 @@ template regNumber LinearScan::allocateReg(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { - regMaskAny foundRegBit = + regMaskMixed foundRegBit = regSelector->select(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) { @@ -3973,7 +3973,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { // For each physical register that can hold a GC type, // if it is occupied by an interval of a GC type, spill that interval. - regMaskAny candidateRegs = killRefPosition->registerAssignment; + regMaskMixed candidateRegs = killRefPosition->registerAssignment; INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { @@ -4063,13 +4063,13 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // The new register to use. // #ifdef DEBUG -regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskAny availableRegs) +regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskMixed availableRegs) { if (targetReg != REG_STK && getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE) { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskAny candidateRegs = allRegs(interval->registerType) & availableRegs; + regMaskMixed candidateRegs = allRegs(interval->registerType) & availableRegs; regNumber firstReg = REG_NA; regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) @@ -4373,9 +4373,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } // If we are rotating register assignments at block boundaries, we want to make the // inactive registers available for the rotation. - regMaskAny inactiveRegs = RBM_NONE; + regMaskMixed inactiveRegs = RBM_NONE; #endif // DEBUG - regMaskAny liveRegs = RBM_NONE; + regMaskMixed liveRegs = RBM_NONE; VarSetOps::Iter iter(compiler, currentLiveVars); unsigned varIndex = 0; while (iter.NextElem(&varIndex)) @@ -4633,7 +4633,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } } #else - regMaskAny deadCandidates = ~liveRegs; + regMaskMixed deadCandidates = ~liveRegs; // Only focus on actual registers present deadCandidates &= actualRegistersMask; @@ -4825,7 +4825,7 @@ void LinearScan::freeRegister(RegRecord* physRegRecord) // Arguments: // regsToFree - the mask of registers to free // -void LinearScan::freeRegisters(regMaskAny regsToFree) +void LinearScan::freeRegisters(regMaskMixed regsToFree) { if (regsToFree == RBM_NONE) { @@ -4900,11 +4900,11 @@ void LinearScan::allocateRegistersMinimal() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskAny regsToFree = RBM_NONE; - regMaskAny delayRegsToFree = RBM_NONE; - regMaskAny regsToMakeInactive = RBM_NONE; - regMaskAny delayRegsToMakeInactive = RBM_NONE; - regMaskAny copyRegsToFree = RBM_NONE; + regMaskMixed regsToFree = RBM_NONE; + regMaskMixed delayRegsToFree = RBM_NONE; + regMaskMixed regsToMakeInactive = RBM_NONE; + regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -4922,7 +4922,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskAny tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskMixed tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -5363,7 +5363,7 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskAny regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -5581,11 +5581,11 @@ void LinearScan::allocateRegisters() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskAny regsToFree = RBM_NONE; - regMaskAny delayRegsToFree = RBM_NONE; - regMaskAny regsToMakeInactive = RBM_NONE; - regMaskAny delayRegsToMakeInactive = RBM_NONE; - regMaskAny copyRegsToFree = RBM_NONE; + regMaskMixed regsToFree = RBM_NONE; + regMaskMixed delayRegsToFree = RBM_NONE; + regMaskMixed regsToMakeInactive = RBM_NONE; + regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -5603,7 +5603,7 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskAny tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskMixed tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -6097,7 +6097,7 @@ void LinearScan::allocateRegisters() // it might be beneficial to keep it in this reg for PART of the lifetime if (currentInterval->isLocalVar) { - regMaskAny preferences = currentInterval->registerPreferences; + regMaskMixed preferences = currentInterval->registerPreferences; bool keepAssignment = true; bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; @@ -6206,8 +6206,8 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { lastAllocatedRefPosition = ¤tRefPosition; - regMaskAny copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskAny assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -6304,8 +6304,8 @@ void LinearScan::allocateRegisters() } lastAllocatedRefPosition = ¤tRefPosition; - regMaskAny copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskAny assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.needsConsecutive) @@ -6527,7 +6527,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskAny regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -8214,7 +8214,7 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { - regMaskAny initialRegMask = interval->firstRefPosition->registerAssignment; + regMaskMixed initialRegMask = interval->firstRefPosition->registerAssignment; regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK : genRegNumFromMask(initialRegMask); @@ -8542,7 +8542,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskAny terminatorConsumedRegs) + regMaskMixed terminatorConsumedRegs) { // TODO-Throughput: This would be much more efficient if we add RegToVarMaps instead of VarToRegMaps // and they would be more space-efficient as well. @@ -8828,7 +8828,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // available to copy into. // Note that for this purpose we use the full live-out set, because we must ensure that // even the registers that remain the same across the edge are preserved correctly. - regMaskAny liveOutRegs = RBM_NONE; + regMaskMixed liveOutRegs = RBM_NONE; VarSetOps::Iter liveOutIter(compiler, block->bbLiveOut); unsigned liveOutVarIndex = 0; while (liveOutIter.NextElem(&liveOutVarIndex)) @@ -8848,7 +8848,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // // Note: Only switches and JCMP/JTEST (for Arm4) have input regs (and so can be fed by copies), so those // are the only block-ending branches that need special handling. - regMaskAny consumedRegs = RBM_NONE; + regMaskMixed consumedRegs = RBM_NONE; if (block->KindIs(BBJ_SWITCH)) { // At this point, Lowering has transformed any non-switch-table blocks into @@ -8928,8 +8928,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) } VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; - regMaskAny sameWriteRegs = RBM_NONE; - regMaskAny diffReadRegs = RBM_NONE; + regMaskMixed sameWriteRegs = RBM_NONE; + regMaskMixed diffReadRegs = RBM_NONE; // For each var that may require resolution, classify them as: // - in the same register at the end of this block and at each target (no resolution needed) @@ -9363,7 +9363,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskAny terminatorConsumedRegs) + regMaskMixed terminatorConsumedRegs) { VarToRegMap fromVarToRegMap = getOutVarToRegMap(fromBlock->bbNum); VarToRegMap toVarToRegMap; @@ -9431,9 +9431,9 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } } - regMaskAny targetRegsToDo = RBM_NONE; - regMaskAny targetRegsReady = RBM_NONE; - regMaskAny targetRegsFromStack = RBM_NONE; + regMaskMixed targetRegsToDo = RBM_NONE; + regMaskMixed targetRegsReady = RBM_NONE; + regMaskMixed targetRegsFromStack = RBM_NONE; // The following arrays capture the location of the registers as they are moved: // - location[reg] gives the current location of the var that was originally in 'reg'. @@ -9561,7 +9561,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // REGISTER to REGISTER MOVES // First, find all the ones that are ready to move now - regMaskAny targetCandidates = targetRegsToDo; + regMaskMixed targetCandidates = targetRegsToDo; while (targetCandidates != RBM_NONE) { regNumber targetReg = genFirstRegNumFromMask(targetCandidates); @@ -9726,7 +9726,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { // Look at the remaining registers from targetRegsToDo (which we expect to be relatively // small at this point) to find out what's currently in targetReg. - regMaskAny mask = targetRegsToDo; + regMaskMixed mask = targetRegsToDo; while (mask != RBM_NONE && otherTargetReg == REG_NA) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(mask); @@ -11560,7 +11560,7 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) // Arguments: // regsToFree - Registers that were just freed. // -void LinearScan::verifyFreeRegisters(regMaskAny regsToFree) +void LinearScan::verifyFreeRegisters(regMaskMixed regsToFree) { for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { @@ -12012,7 +12012,7 @@ void LinearScan::verifyFinalAllocation() // However, we will assert that, at resolution time, no registers contain GC refs. { DBEXEC(VERBOSE, printf(" ")); - regMaskAny candidateRegs = currentRefPosition.registerAssignment; + regMaskMixed candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -12399,7 +12399,7 @@ bool LinearScan::RegisterSelection::applySelection(int selectionScore, regMaskOn { assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidates)); - regMaskAny newCandidates = candidates & selectionCandidates; + regMaskMixed newCandidates = candidates & selectionCandidates; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12423,7 +12423,7 @@ bool LinearScan::RegisterSelection::applySingleRegSelection(int selectionScore, assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidate)); assert(LinearScan::isSingleRegister(selectionCandidate)); - regMaskAny newCandidates = candidates & selectionCandidate; + regMaskMixed newCandidates = candidates & selectionCandidate; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12470,7 +12470,7 @@ void LinearScan::RegisterSelection::try_CONST_AVAILABLE() if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) { - regMaskAny newCandidates = candidates & matchingConstants; + regMaskMixed newCandidates = candidates & matchingConstants; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -13367,7 +13367,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* cur } #ifdef DEBUG - regMaskAny inUseOrBusyRegsMask = RBM_NONE; + regMaskMixed inUseOrBusyRegsMask = RBM_NONE; #endif // Eliminate candidates that are in-use or busy. @@ -13376,7 +13376,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* cur // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskAny busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + regMaskMixed busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; candidates &= ~busyRegs; #ifdef TARGET_ARM @@ -13704,7 +13704,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskAny busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + regMaskMixed busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; candidates &= ~busyRegs; #ifdef TARGET_ARM diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 0dc4b0496a9c6a..b9ec004c2a55b7 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -615,7 +615,7 @@ class LinearScan : public LinearScanInterface // This is the main driver virtual PhaseStatus doLinearScan(); - static bool isSingleRegister(regMaskAny regMask) + static bool isSingleRegister(regMaskMixed regMask) { return (genExactlyOneBit(regMask)); } @@ -716,7 +716,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskAny terminatorConsumedRegs); + regMaskMixed terminatorConsumedRegs); void resolveEdges(); @@ -809,11 +809,11 @@ class LinearScan : public LinearScanInterface } // TODO: Can have separate methods for each type - regMaskAny getConstrainedRegMask(RefPosition* refPosition, - regMaskAny regMaskActual, - regMaskAny regMaskConstrain, + regMaskMixed getConstrainedRegMask(RefPosition* refPosition, + regMaskMixed regMaskActual, + regMaskMixed regMaskConstrain, unsigned minRegCount); - regMaskAny stressLimitRegs(RefPosition* refPosition, regMaskAny mask); + regMaskMixed stressLimitRegs(RefPosition* refPosition, regMaskMixed mask); // This controls the heuristics used to select registers // These can be combined. @@ -878,7 +878,7 @@ class LinearScan : public LinearScanInterface { return (LsraBlockBoundaryLocations)(lsraStressMask & LSRA_BLOCK_BOUNDARY_MASK); } - regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskAny availableRegs); + regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskMixed availableRegs); // This controls whether we always insert a GT_RELOAD instruction after a spill // Note that this can be combined with LSRA_SPILL_ALWAYS (or not) @@ -941,7 +941,7 @@ class LinearScan : public LinearScanInterface static bool IsResolutionMove(GenTree* node); static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); - void verifyFreeRegisters(regMaskAny regsToFree); + void verifyFreeRegisters(regMaskMixed regsToFree); void verifyFinalAllocation(); void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG @@ -1035,7 +1035,7 @@ class LinearScan : public LinearScanInterface void insertZeroInitRefPositions(); // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions - void addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); @@ -1072,14 +1072,14 @@ class LinearScan : public LinearScanInterface } // Helpers for getKillSetForNode(). - regMaskAny getKillSetForStoreInd(GenTreeStoreInd* tree); + regMaskMixed getKillSetForStoreInd(GenTreeStoreInd* tree); regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); regMaskGpr getKillSetForMul(GenTreeOp* tree); - regMaskAny getKillSetForCall(GenTreeCall* call); + regMaskMixed getKillSetForCall(GenTreeCall* call); regMaskGpr getKillSetForModDiv(GenTreeOp* tree); - regMaskAny getKillSetForBlockStore(GenTreeBlk* blkNode); - regMaskAny getKillSetForReturn(); - regMaskAny getKillSetForProfilerHook(); + regMaskMixed getKillSetForBlockStore(GenTreeBlk* blkNode); + regMaskMixed getKillSetForReturn(); + regMaskMixed getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS regMaskGpr getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); #endif // FEATURE_HW_INTRINSICS @@ -1088,11 +1088,11 @@ class LinearScan : public LinearScanInterface // This is used only for an assert, and for stress, so it is only defined under DEBUG. // Otherwise, the Build methods should obtain the killMask from the appropriate method above. #ifdef DEBUG - regMaskAny getKillSetForNode(GenTree* tree); + regMaskMixed getKillSetForNode(GenTree* tree); #endif // Given some tree node add refpositions for all the registers this node kills - bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskAny killMask); + bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskMixed killMask); regMaskOnlyOne allRegs(RegisterType rt); regMaskGpr allByteRegs(); @@ -1102,7 +1102,7 @@ class LinearScan : public LinearScanInterface void makeRegisterInactive(RegRecord* physRegRecord); void freeRegister(RegRecord* physRegRecord); - void freeRegisters(regMaskAny regsToFree); + void freeRegisters(regMaskMixed regsToFree); // Get the type that this tree defines. var_types getDefType(GenTree* tree) @@ -1410,7 +1410,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskAny terminatorConsumedRegs); + regMaskMixed terminatorConsumedRegs); #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; @@ -1478,8 +1478,8 @@ class LinearScan : public LinearScanInterface static const int MAX_ROWS_BETWEEN_TITLES = 50; int rowCountSinceLastTitle; // Current mask of registers being printed in the dump. - regMaskAny lastDumpedRegisters; - regMaskAny registersToDump; + regMaskMixed lastDumpedRegisters; + regMaskMixed registersToDump; int lastUsedRegNumIndex; bool shouldDumpReg(regNumber regNum) { @@ -1662,14 +1662,14 @@ class LinearScan : public LinearScanInterface // A temporary VarToRegMap used during the resolution of critical edges. VarToRegMap sharedCriticalVarToRegMap; - PhasedVar actualRegistersMask; + PhasedVar actualRegistersMask; PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; #if defined(TARGET_XARCH) PhasedVar availableMaskRegs; #endif - PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, predicate + PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, predicate #if defined(TARGET_XARCH) #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) @@ -1681,7 +1681,7 @@ class LinearScan : public LinearScanInterface // PUTARG_REG node. Tracked between the PUTARG_REG and its corresponding // CALL node and is used to avoid preferring these registers for locals // which would otherwise force a spill. - regMaskAny placedArgRegs; + regMaskMixed placedArgRegs; struct PlacedLocal { @@ -1730,7 +1730,7 @@ class LinearScan : public LinearScanInterface // Register status //----------------------------------------------------------------------- - regMaskAny m_AvailableRegs; //TODO: Should be separate for gpr, vector, predicate + regMaskMixed m_AvailableRegs; //TODO: Should be separate for gpr, vector, predicate regNumber getRegForType(regNumber reg, var_types regType) { #ifdef TARGET_ARM @@ -1767,7 +1767,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_AvailableRegs & regMask) == regMask; } - void setRegsInUse(regMaskAny regMask) + void setRegsInUse(regMaskMixed regMask) { m_AvailableRegs &= ~regMask; } @@ -1776,7 +1776,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regMask = getRegMask(reg, regType); setRegsInUse(regMask); } - void makeRegsAvailable(regMaskAny regMask) + void makeRegsAvailable(regMaskMixed regMask) { m_AvailableRegs |= regMask; } @@ -1801,7 +1801,7 @@ class LinearScan : public LinearScanInterface DEBUG_ARG(regNumber assignedReg)); // TODO: This should be m_GprWithConstants, m_FloatRegsWithConstant, etc. - regMaskAny m_RegistersWithConstants; + regMaskMixed m_RegistersWithConstants; void clearConstantReg(regNumber reg, var_types regType) { m_RegistersWithConstants &= ~getRegMask(reg, regType); @@ -1816,9 +1816,9 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_RegistersWithConstants & regMask) == regMask; } - regMaskOnlyOne getMatchingConstants(regMaskAny mask, Interval* currentInterval, RefPosition* refPosition); + regMaskOnlyOne getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition); - regMaskAny fixedRegs; //TODO: Should be also seperate + regMaskMixed fixedRegs; //TODO: Should be also seperate LsraLocation nextFixedRef[REG_COUNT]; void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); LsraLocation getNextFixedRef(regNumber regNum, var_types regType) @@ -1847,9 +1847,9 @@ class LinearScan : public LinearScanInterface } weight_t spillCost[REG_COUNT]; - regMaskAny regsBusyUntilKill; //TODO: Likewise, probably have this global 32-bit and set it point to the specific version like gpr, vector, etc. - regMaskAny regsInUseThisLocation; - regMaskAny regsInUseNextLocation; + regMaskMixed regsBusyUntilKill; //TODO: Likewise, probably have this global 32-bit and set it point to the specific version like gpr, vector, etc. + regMaskMixed regsInUseThisLocation; + regMaskMixed regsInUseNextLocation; #ifdef TARGET_ARM64 regMaskFloat consecutiveRegsInUseThisLocation; #endif @@ -1965,7 +1965,7 @@ class LinearScan : public LinearScanInterface void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); - void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskAny killMask); + void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 424ede7a1fe234..207a8f9f2e9d73 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -194,7 +194,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index c4dcd3936754cb..19c542339ed004 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -584,7 +584,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 5b1e972787e82a..19078fdf5822a2 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -375,7 +375,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskAny killMask = getKillSetForCall(call); + regMaskMixed killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -491,7 +491,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) int dstCount = argNode->gtNumRegs; regNumber argReg = argNode->GetRegNum(); - regMaskAny argMask = RBM_NONE; + regMaskMixed argMask = RBM_NONE; for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); @@ -827,7 +827,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } buildInternalRegisterUses(); - regMaskAny killMask = getKillSetForBlockStore(blkNode); + regMaskMixed killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index fbc673c75046cb..2718eccc0526a5 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -699,7 +699,7 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); @@ -714,7 +714,7 @@ void LinearScan::addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, // modified until codegen, which is too late. compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); - for (regMaskAny candidates = mask; candidates != RBM_NONE;) + for (regMaskMixed candidates = mask; candidates != RBM_NONE;) { regNumber reg = genFirstRegNumFromMaskAndToggle(candidates); // This assumes that these are all "special" RefTypes that @@ -740,11 +740,11 @@ void LinearScan::addRefsForPhysRegMask(regMaskAny mask, LsraLocation currentLoc, // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) +regMaskMixed LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) { assert(tree->OperIs(GT_STOREIND)); - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; GCInfo::WriteBarrierForm writeBarrierForm = compiler->codeGen->gcInfo.gcIsWriteBarrierCandidate(tree); if (writeBarrierForm != GCInfo::WBF_NoBarrier) @@ -840,9 +840,9 @@ regMaskGpr LinearScan::getKillSetForModDiv(GenTreeOp* node) // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForCall(GenTreeCall* call) +regMaskMixed LinearScan::getKillSetForCall(GenTreeCall* call) { - regMaskAny killMask = RBM_CALLEE_TRASH; + regMaskMixed killMask = RBM_CALLEE_TRASH; #ifdef TARGET_X86 if (compiler->compFloatingPointUsed) { @@ -895,10 +895,10 @@ regMaskAny LinearScan::getKillSetForCall(GenTreeCall* call) // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) +regMaskMixed LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) { assert(blkNode->OperIsStoreBlk()); - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; bool isCopyBlk = varTypeIsStruct(blkNode->Data()); switch (blkNode->gtBlkOpKind) @@ -994,7 +994,7 @@ regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForReturn() +regMaskMixed LinearScan::getKillSetForReturn() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_LEAVE) : RBM_NONE; @@ -1008,7 +1008,7 @@ regMaskAny LinearScan::getKillSetForReturn() // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForProfilerHook() +regMaskMixed LinearScan::getKillSetForProfilerHook() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_TAILCALL) : RBM_NONE; @@ -1023,9 +1023,9 @@ regMaskAny LinearScan::getKillSetForProfilerHook() // // Return Value: a register mask of the registers killed // -regMaskAny LinearScan::getKillSetForNode(GenTree* tree) +regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) { - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; switch (tree->OperGet()) { case GT_LSH: @@ -1123,7 +1123,7 @@ regMaskAny LinearScan::getKillSetForNode(GenTree* tree) // This method can add kills even if killMask is RBM_NONE, if this tree is one of the // special cases that signals that we can't permit callee save registers to hold GC refs. -bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskAny killMask) +bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskMixed killMask) { bool insertedKills = false; @@ -1838,7 +1838,7 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc // The use position of v02 cannot be allocated a reg since it is marked delay-reg free and // {eax,edx} are getting killed before the def of GT_DIV. For this reason, minRegCount for // the use position of v02 also needs to take into account the kill set of its consuming node. - regMaskAny killMask = getKillSetForNode(tree); + regMaskMixed killMask = getKillSetForNode(tree); if (killMask != RBM_NONE) { minRegCountForRef += genCountBits(killMask); @@ -2492,7 +2492,7 @@ void LinearScan::buildIntervals() // into the scratch register, so it will be killed here. if (compiler->compShouldPoisonFrame() && compiler->fgFirstBBisScratch() && block == compiler->fgFirstBB) { - regMaskAny killed; + regMaskMixed killed; #if defined(TARGET_XARCH) // Poisoning uses EAX for small vars and rep stosd that kills edi, ecx and eax for large vars. killed = RBM_EDI | RBM_ECX | RBM_EAX; @@ -2781,7 +2781,7 @@ void LinearScan::buildIntervals() availableRegCount = REG_INT_COUNT; } - if (availableRegCount < (sizeof(regMaskAny) * 8)) + if (availableRegCount < (sizeof(regMaskMixed) * 8)) { // Mask out the bits that are between 64 ~ availableRegCount actualRegistersMask = (1ULL << availableRegCount) - 1; @@ -3072,7 +3072,7 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandid // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskAny killMask) +void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask) { assert(killMask == getKillSetForNode(tree)); @@ -3140,7 +3140,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Find the registers that we should remove from the preference set because // they are occupied with argument values. - regMaskAny unpref = placedArgRegs; + regMaskMixed unpref = placedArgRegs; unsigned varIndex = interval->getVarIndex(compiler); for (size_t i = 0; i < numPlacedArgLocals; i++) { @@ -4258,7 +4258,7 @@ int LinearScan::BuildGCWriteBarrier(GenTree* tree) BuildUse(addr, addrCandidates); BuildUse(src, srcCandidates); - regMaskAny killMask = getKillSetForStoreInd(tree->AsStoreInd()); + regMaskMixed killMask = getKillSetForStoreInd(tree->AsStoreInd()); buildKillPositionsForNode(tree, currentLoc + 1, killMask); return 2; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index b78579f2f2dbf7..d0a1c23500f00f 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -47,7 +47,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskAny killMask = RBM_NONE; + regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -1344,7 +1344,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskAny killMask = getKillSetForCall(call); + regMaskMixed killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -1636,7 +1636,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) #endif buildInternalRegisterUses(); - regMaskAny killMask = getKillSetForBlockStore(blkNode); + regMaskMixed killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; @@ -1929,7 +1929,7 @@ int LinearScan::BuildModDiv(GenTree* tree) srcCount += BuildDelayFreeUses(op2, op1, availableIntRegs & ~(RBM_RAX | RBM_RDX)); buildInternalRegisterUses(); - regMaskAny killMask = getKillSetForModDiv(tree->AsOp()); + regMaskMixed killMask = getKillSetForModDiv(tree->AsOp()); BuildDefsWithKills(tree, 1, dstCandidates, killMask); return srcCount; } @@ -3042,7 +3042,7 @@ int LinearScan::BuildMul(GenTree* tree) assert(compiler->IsGprRegMask(dstCandidates)); - regMaskAny killMask = getKillSetForMul(tree->AsOp()); + regMaskMixed killMask = getKillSetForMul(tree->AsOp()); BuildDefsWithKills(tree, dstCount, dstCandidates, killMask); return srcCount; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 8ccb74defe8378..a1ce9f6d41b664 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -87,7 +87,7 @@ void RegSet::verifyRegUsed(regNumber reg) // should simply validate that the register (or registers) have // already been added to the modified set. -void RegSet::verifyRegistersUsed(regMaskAny regMask) +void RegSet::verifyRegistersUsed(regMaskMixed regMask) { if (m_rsCompiler->opts.OptimizationDisabled()) { @@ -119,7 +119,7 @@ void RegSet::rsClearRegsModified() rsModifiedRegsMask = RBM_NONE; } -void RegSet::rsSetRegsModified(regMaskAny mask DEBUGARG(bool suppressDump)) +void RegSet::rsSetRegsModified(regMaskMixed mask DEBUGARG(bool suppressDump)) { assert(mask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); @@ -187,7 +187,7 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) rsModifiedRegsMask &= ~mask; } -void RegSet::SetMaskVars(regMaskAny newMaskVars) +void RegSet::SetMaskVars(regMaskMixed newMaskVars) { #ifdef DEBUG if (m_rsCompiler->verbose) @@ -203,10 +203,10 @@ void RegSet::SetMaskVars(regMaskAny newMaskVars) m_rsCompiler->GetEmitter()->emitDispRegSet(_rsMaskVars); // deadSet = old - new - regMaskAny deadSet = _rsMaskVars & ~newMaskVars; + regMaskMixed deadSet = _rsMaskVars & ~newMaskVars; // bornSet = new - old - regMaskAny bornSet = newMaskVars & ~_rsMaskVars; + regMaskMixed bornSet = newMaskVars & ~_rsMaskVars; if (deadSet != RBM_NONE) { @@ -937,7 +937,7 @@ regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) regMaskSmall res = 0; for (int i = 0; i < CNT_CALLEE_SAVED; i++) { - if ((calleeSaveMask & ((regMaskAny)1 << i)) != 0) + if ((calleeSaveMask & ((regMaskMixed)1 << i)) != 0) { res |= raRbmCalleeSaveOrder[i]; } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 7728dc413567c9..21eab9d1c1040d 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -43,7 +43,7 @@ class RegSet RegSet(Compiler* compiler, GCInfo& gcInfo); #ifdef TARGET_ARM - regMaskAny rsMaskPreSpillRegs(bool includeAlignment) const + regMaskMixed rsMaskPreSpillRegs(bool includeAlignment) const { return includeAlignment ? (rsMaskPreSpillRegArg | rsMaskPreSpillAlign) : rsMaskPreSpillRegArg; } @@ -68,14 +68,14 @@ class RegSet private: bool rsNeededSpillReg; // true if this method needed to spill any registers - regMaskAny rsModifiedRegsMask; // mask of the registers modified by the current function. + regMaskMixed rsModifiedRegsMask; // mask of the registers modified by the current function. #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. #endif // DEBUG public: - regMaskAny rsGetModifiedRegsMask() const + regMaskMixed rsGetModifiedRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; @@ -83,7 +83,7 @@ class RegSet void rsClearRegsModified(); - void rsSetRegsModified(regMaskAny mask DEBUGARG(bool suppressDump = false)); + void rsSetRegsModified(regMaskMixed mask DEBUGARG(bool suppressDump = false)); void rsRemoveRegsModified(regMaskGpr mask); @@ -95,22 +95,22 @@ class RegSet void verifyRegUsed(regNumber reg); - void verifyRegistersUsed(regMaskAny regMask); + void verifyRegistersUsed(regMaskMixed regMask); public: - regMaskAny GetMaskVars() const // 'get' property function for rsMaskVars property + regMaskMixed GetMaskVars() const // 'get' property function for rsMaskVars property { return _rsMaskVars; } - void SetMaskVars(regMaskAny newMaskVars); // 'put' property function for rsMaskVars property + void SetMaskVars(regMaskMixed newMaskVars); // 'put' property function for rsMaskVars property - void AddMaskVars(regMaskAny addMaskVars) // union 'addMaskVars' with the rsMaskVars set + void AddMaskVars(regMaskMixed addMaskVars) // union 'addMaskVars' with the rsMaskVars set { SetMaskVars(_rsMaskVars | addMaskVars); } - void RemoveMaskVars(regMaskAny removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) + void RemoveMaskVars(regMaskMixed removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) { SetMaskVars(_rsMaskVars & ~removeMaskVars); } @@ -121,10 +121,10 @@ class RegSet } private: - regMaskAny _rsMaskVars; // backing store for rsMaskVars property + regMaskMixed _rsMaskVars; // backing store for rsMaskVars property #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - regMaskAny rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog + regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 public: // TODO-Cleanup: Should be private, but Compiler uses it diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index fb434ae2b6de05..801f3f61662869 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -217,7 +217,8 @@ typedef unsigned __int64 regMaskPredicate; // Design: // 1. Reduce regMaskGpr to 32-bit -// 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskAny +// 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskMixed +// 3. Revisit `regMaskMixed` and see how we can pass m // // @@ -229,14 +230,14 @@ typedef unsigned __int64 regMaskPredicate; // converted to "unsigned" typedef unsigned __int64 regMaskOnlyOne; -// `regMaskAny` tells that the mask can contain any of the gpr/vector +// `regMaskMixed` tells that the mask can contain any of the gpr/vector // registers. Once we identify all the places with `regMaskOnlyOne`, -// `regMaskFloat`, `regMaskGpr`, we will revisit `regMaskAny` and try +// `regMaskFloat`, `regMaskGpr`, we will revisit `regMaskMixed` and try // to either: -// 0. Revisit regMaskAny and see if they should be "regMaskAny" +// 0. Revisit regMaskMixed and see if they should be "regMaskMixed" // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together -typedef unsigned __int64 regMaskAny; //TODO: Rename this to regMaskMixed +typedef unsigned __int64 regMaskMixed; //TODO: Rename this to regMaskMixed // TODO: For LSRA, the select() method should be regMaskOnlyOne because we will be // allocating either GPR or Vector or Mask but not all @@ -249,7 +250,7 @@ typedef unsigned regMaskTP; #define regMaskFloat regMaskTP #define regMaskPredicate regMaskTP #define regMaskOnlyOne regMaskTP -#define regMaskAny regMaskTP +#define regMaskMixed regMaskTP #define singleRegMask regMaskTP #endif @@ -376,7 +377,7 @@ const char* getRegName(regNumber reg); #ifdef DEBUG const char* getRegNameFloat(regNumber reg, var_types type); -extern void dspRegMask(regMaskAny regMask, size_t minSiz = 0); +extern void dspRegMask(regMaskMixed regMask, size_t minSiz = 0); #endif #if CPU_HAS_BYTE_REGS @@ -606,7 +607,7 @@ inline singleRegMask genRegMask(regNumber reg) // (L1 latency on sandy bridge is 4 cycles for [base] and 5 for [base + index*c] ) // the reason this is AMD-only is because the x86 BE will try to get reg masks for REG_STK // and the result needs to be zero. - regMaskAny result = 1ULL << reg; + regMaskMixed result = 1ULL << reg; assert(result == regMasks[reg]); return result; #else @@ -665,7 +666,7 @@ inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TY inline regMaskOnlyOne genRegMask(regNumber regNum, var_types type) { #if defined(TARGET_ARM) - regMaskAny regMask = RBM_NONE; + regMaskMixed regMask = RBM_NONE; if (varTypeUsesIntReg(type)) { diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 7623b30b13b826..7bf82b6faaef81 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -152,7 +152,7 @@ void Compiler::unwindPushPopCFI(regNumber reg) FuncInfoDsc* func = funCurrentFunc(); UNATIVE_OFFSET cbProlog = unwindGetCurrentOffset(func); - regMaskAny relOffsetMask = RBM_CALLEE_SAVED + regMaskMixed relOffsetMask = RBM_CALLEE_SAVED #if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) // is excluded from the callee-save register list. diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 254f53525fa88b..5d3e7277026ee1 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -282,7 +282,7 @@ const char* getRegNameFloat(regNumber reg, var_types type) * Displays a range of registers * -- This is a helper used by dspRegMask */ -const char* dspRegRange(regMaskAny regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) +const char* dspRegRange(regMaskMixed regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) { #ifdef TARGET_XARCH assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || @@ -435,7 +435,7 @@ const char* dspRegRange(regMaskAny regMask, size_t& minSiz, const char* sep, reg * Displays a register set. * TODO-ARM64-Cleanup: don't allow ip0, ip1 as part of a range. */ -void dspRegMask(regMaskAny regMask, size_t minSiz) +void dspRegMask(regMaskMixed regMask, size_t minSiz) { const char* sep = ""; From 3fb09bb3b087c894295e38311cdb65d02ccd7169 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 19:43:30 -0800 Subject: [PATCH 050/201] Update design notes of target.h --- src/coreclr/jit/target.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 801f3f61662869..3a4f69c241e465 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -218,7 +218,8 @@ typedef unsigned __int64 regMaskPredicate; // Design: // 1. Reduce regMaskGpr to 32-bit // 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskMixed -// 3. Revisit `regMaskMixed` and see how we can pass m +// 3. Revisit `regMaskMixed` and see how we can separate them into `regMaskGpr`, `regMaskFloat`, `regMaskPredicate`. +// 4. Once #3 is solved, `regMaskOnlyOne` should be reduced to 32-bit and so can `regMaskFloat` and `regMaskPredicate` // // From cdb42bcf115b4cd7c179793678e0a54c2f3449e8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 21:16:16 -0800 Subject: [PATCH 051/201] fix the typo --- src/coreclr/jit/lsra.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index efbbd3e6c0ff47..f8bb65c3c5f8e1 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -722,8 +722,8 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_AMD64) rbmAllFloat = compiler->rbmAllFloat; rbmFltCalleeTrash = compiler->rbmFltCalleeTrash; - assert(compiler->IsGprRegMask(rbmAllFloat)); - assert(compiler->IsGprRegMask(rbmFltCalleeTrash)); + assert(compiler->IsFloatRegMask(rbmAllFloat)); + assert(compiler->IsFloatRegMask(rbmFltCalleeTrash)); #endif // TARGET_AMD64 #if defined(TARGET_XARCH) From e2cc19bd03abd160d197fae30324ce125563037c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 21:24:03 -0800 Subject: [PATCH 052/201] Update the assert in IsOnlyOneRegMask() --- src/coreclr/jit/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 16e55077cf7550..7188d46798519a 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8462,7 +8462,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool IsOnlyOneRegMask(regMaskTP regMask) { - return IsGprRegMask(regMask) != IsFloatRegMask(regMask); + return (regMask == RBM_NONE) || (IsGprRegMask(regMask) != IsFloatRegMask(regMask)); } void unwindBegProlog(); From 656629ee1f57398ba70ae95b06cc84ed677f2666 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 9 Feb 2024 21:28:45 -0800 Subject: [PATCH 053/201] Fix assert in buildUpperVectorSaveRefPositions() --- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/lsrabuild.cpp | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index b9ec004c2a55b7..08029e278e4529 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1042,7 +1042,7 @@ class LinearScan : public LinearScanInterface void buildRefPositionsForNode(GenTree* tree, LsraLocation loc); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskFloat fpCalleeKillSet); + void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskMixed fpCalleeKillSet); void buildUpperVectorRestoreRefPosition( Interval* lclVarInterval, LsraLocation currentLoc, GenTree* node, bool isUse, unsigned multiRegIdx); #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 2718eccc0526a5..77c4030c14ba74 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1476,10 +1476,8 @@ Interval* LinearScan::getUpperVectorInterval(unsigned varIndex) // But we will use as a proxy any node that kills floating point registers. // (Note that some calls are masquerading as other nodes at this point so we can't just check for calls.) // -void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskFloat fpCalleeKillSet) +void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskMixed fpCalleeKillSet) { - assert(compiler->IsFloatRegMask(fpCalleeKillSet)); - if ((tree != nullptr) && tree->IsCall()) { if (tree->AsCall()->IsNoReturn() || compiler->fgIsThrow(tree)) From ec471e5cc00a9f63807965cf215c80319f8352d2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Feb 2024 12:24:49 -0800 Subject: [PATCH 054/201] Revert to buildUpperVectorSaveRefPositions() to regMaskFloat --- src/coreclr/jit/lsra.h | 3 ++- src/coreclr/jit/lsrabuild.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 08029e278e4529..337878f2a75481 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1042,7 +1042,8 @@ class LinearScan : public LinearScanInterface void buildRefPositionsForNode(GenTree* tree, LsraLocation loc); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskMixed fpCalleeKillSet); + void buildUpperVectorSaveRefPositions(GenTree* tree, + LsraLocation currentLoc DEBUG_ARG(regMaskFloat fpCalleeKillSet)); void buildUpperVectorRestoreRefPosition( Interval* lclVarInterval, LsraLocation currentLoc, GenTree* node, bool isUse, unsigned multiRegIdx); #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 77c4030c14ba74..e401872be30dcd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1476,8 +1476,11 @@ Interval* LinearScan::getUpperVectorInterval(unsigned varIndex) // But we will use as a proxy any node that kills floating point registers. // (Note that some calls are masquerading as other nodes at this point so we can't just check for calls.) // -void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc, regMaskMixed fpCalleeKillSet) +void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, + LsraLocation currentLoc DEBUG_ARG(regMaskFloat fpCalleeKillSet)) { + assert(compiler->IsFloatRegMask(fpCalleeKillSet)); + if ((tree != nullptr) && tree->IsCall()) { if (tree->AsCall()->IsNoReturn() || compiler->fgIsThrow(tree)) @@ -3093,7 +3096,7 @@ void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne // if ((killMask & RBM_FLT_CALLEE_TRASH) != RBM_NONE) { - buildUpperVectorSaveRefPositions(tree, currentLoc + 1, killMask); + buildUpperVectorSaveRefPositions(tree, currentLoc + 1 DEBUG_ARG((killMask & RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } From ac8b5ead0b36507a37544dd6cb2873d741d2141d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Feb 2024 12:25:59 -0800 Subject: [PATCH 055/201] Change BuildDefs() to regMaskMixed --- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/lsrabuild.cpp | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 337878f2a75481..14d4a68d9ed9b1 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1965,7 +1965,7 @@ class LinearScan : public LinearScanInterface int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); - void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); + void BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates = RBM_NONE); void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask); int BuildReturn(GenTree* tree); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index e401872be30dcd..60c9ca15051bfd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3018,10 +3018,8 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, i // Notes: // Adds the RefInfo for the definitions to the defList. // -void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates) +void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates) { - assert(compiler->IsOnlyOneRegMask(dstCandidates)); - bool fixedReg = false; if ((dstCount > 1) && (dstCandidates != RBM_NONE) && ((int)genCountBits(dstCandidates) == dstCount)) { From 8e177460c98ac9def53d55eaaf3905e3b6d999f8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Feb 2024 12:26:16 -0800 Subject: [PATCH 056/201] jit format --- src/coreclr/jit/codegen.h | 25 +++-- src/coreclr/jit/codegenarm.cpp | 25 +++-- src/coreclr/jit/codegenarm64.cpp | 23 +++-- src/coreclr/jit/codegenarmarch.cpp | 10 +- src/coreclr/jit/codegencommon.cpp | 20 ++-- src/coreclr/jit/codegeninterface.h | 4 +- src/coreclr/jit/codegenlinear.cpp | 4 +- src/coreclr/jit/codegenxarch.cpp | 43 ++++---- src/coreclr/jit/compiler.h | 6 +- src/coreclr/jit/emit.cpp | 4 +- src/coreclr/jit/emit.h | 20 ++-- src/coreclr/jit/emitarm.cpp | 4 +- src/coreclr/jit/emitarm64.cpp | 8 +- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/lsra.cpp | 131 ++++++++++++------------ src/coreclr/jit/lsra.h | 129 +++++++++++------------ src/coreclr/jit/lsraarm.cpp | 8 +- src/coreclr/jit/lsraarm64.cpp | 38 +++---- src/coreclr/jit/lsraarmarch.cpp | 6 +- src/coreclr/jit/lsrabuild.cpp | 66 ++++++------ src/coreclr/jit/lsraxarch.cpp | 30 +++--- src/coreclr/jit/morph.cpp | 4 +- src/coreclr/jit/regalloc.cpp | 2 +- src/coreclr/jit/registerargconvention.h | 2 +- src/coreclr/jit/regset.cpp | 4 +- src/coreclr/jit/regset.h | 17 +-- src/coreclr/jit/target.h | 6 +- src/coreclr/jit/unwind.cpp | 16 +-- src/coreclr/jit/unwindarmarch.cpp | 4 +- 29 files changed, 336 insertions(+), 325 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index fbabacb566aa3c..7f677730bbadc4 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -385,11 +385,11 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) - unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer - unsigned fiSpDelta; // Stack pointer delta - unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP - int fiPSP_slot_CallerSP_offset; // PSP slot offset from Caller SP + regMaskMixed fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) + unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer + unsigned fiSpDelta; // Stack pointer delta + unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP + int fiPSP_slot_CallerSP_offset; // PSP slot offset from Caller SP }; FuncletFrameInfoDsc genFuncletInfo; @@ -401,7 +401,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) @@ -436,7 +436,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) @@ -455,7 +455,7 @@ class CodeGen final : public CodeGenInterface // and used by all funclet prologs and epilogs, which must all be the same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) + regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive) @@ -479,8 +479,7 @@ class CodeGen final : public CodeGenInterface #endif // TARGET_XARCH - void genZeroInitFltRegs(const regMaskFloat& initFltRegs, const regMaskFloat& initDblRegs, - const regNumber& initReg); + void genZeroInitFltRegs(const regMaskFloat& initFltRegs, const regMaskFloat& initDblRegs, const regNumber& initReg); regNumber genGetZeroReg(regNumber initReg, bool* pInitRegZeroed); @@ -679,10 +678,10 @@ class CodeGen final : public CodeGenInterface // //------------------------------------------------------------------------- - void genSinglePush(); - void genSinglePop(); + void genSinglePush(); + void genSinglePop(); regMaskGpr genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskGpr* noRefRegs); - void genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs); + void genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 68402f3280bf6b..739ac1b0032cf2 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1714,7 +1714,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // On Arm arguments are prespilled on stack, which frees r0-r3. // For generating Enter callout we would need two registers and one of them has to be r0 to pass profiler handle. // The call target register could be any free register. - regNumber argReg = REG_PROFILER_ENTER_ARG; + regNumber argReg = REG_PROFILER_ENTER_ARG; regMaskGpr argRegMask = genRegMask(argReg); assert((regSet.rsMaskPreSpillRegArg & argRegMask) != 0); @@ -1891,7 +1891,10 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, + regNumber initReg, + bool* pInitRegZeroed, + regMaskGpr maskArgRegsLiveIn) { assert(compiler->compGeneratingProlog); @@ -1964,7 +1967,7 @@ void CodeGen::genPushFltRegs(regMaskFloat regMask) void CodeGen::genPopFltRegs(regMaskFloat regMask) { - assert(regMask != 0); // Don't call uness we have some registers to pop + assert(regMask != 0); // Don't call uness we have some registers to pop assert(compiler->IsFloatRegMask(regMask)); // Only floasting point registers should be in regMask regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask)); @@ -2175,7 +2178,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) regMaskMixed maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; regMaskFloat maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT; - regMaskGpr maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; + regMaskGpr maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; // First, pop float registers @@ -2335,7 +2338,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); regMaskFloat maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat; + regMaskGpr maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat; regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPushRegsFloat); maskPushRegsInt |= maskStackAlloc; @@ -2431,7 +2434,7 @@ void CodeGen::genFuncletEpilog() assert(genFuncletInfo.fiSaveRegs & RBM_LR); regMaskFloat maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat; + regMaskGpr maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat; regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPopRegsFloat); maskPopRegsInt |= maskStackAlloc; @@ -2491,9 +2494,9 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; regMaskMixed rsMaskSaveRegs = regSet.rsMaskCalleeSaved; - unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); - unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving - unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; + unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); + unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving + unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; if (compiler->lvaMonAcquired != BAD_VAR_NUM) { saveSizeWithPSP += TARGET_POINTER_SIZE; @@ -2644,8 +2647,8 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu // // str rZero1,[rAddr] // When cnt is odd - regNumber rAddr; - regNumber rCnt = REG_NA; // Invalid + regNumber rAddr; + regNumber rCnt = REG_NA; // Invalid regMaskMixed regMask; regMaskMixed availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 3aea34a32de3b5..0c66f9e52bc956 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -929,7 +929,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int l // Save integer registers at higher addresses than floating-point registers. regMaskFloat maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; + regMaskGpr maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; if (maskSaveRegsFloat != RBM_NONE) { @@ -1019,7 +1019,9 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int sp // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, + int lowestCalleeSavedOffset, + int spDelta) { assert(spDelta >= 0); unsigned regsToRestoreCount = genCountBits(regsToRestoreMask); @@ -1046,7 +1048,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, // Save integer registers at higher addresses than floating-point registers. regMaskFloat maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; - regMaskGpr maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; + regMaskGpr maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; // Restore in the opposite order of saving. @@ -1371,7 +1373,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); regMaskFloat maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat; + regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat; // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskSaveRegsInt & RBM_LR) != 0); @@ -4336,7 +4338,7 @@ void CodeGen::genCodeForSwap(GenTreeOp* tree) regNumber oldOp1Reg = lcl1->GetRegNum(); regNumber oldOp2Reg = lcl2->GetRegNum(); - regMaskGpr oldOp1RegMask = genRegMask(oldOp1Reg); + regMaskGpr oldOp1RegMask = genRegMask(oldOp1Reg); regMaskGpr oldOp2RegMask = genRegMask(oldOp2Reg); assert(compiler->IsGprRegMask(oldOp1RegMask)); @@ -5101,7 +5103,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; } - regMaskGpr callTargetMask = genRegMask(callTargetReg); + regMaskGpr callTargetMask = genRegMask(callTargetReg); regMaskMixed callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet @@ -5547,7 +5549,10 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, + regNumber initReg, + bool* pInitRegZeroed, + regMaskGpr maskArgRegsLiveIn) { assert(compiler->compGeneratingProlog); assert(compiler->IsGprRegMask(maskArgRegsLiveIn)); @@ -5610,8 +5615,8 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni availMask &= ~maskArgRegsLiveIn; // Remove all of the incoming argument registers as they are currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg - regNumber rOffset = initReg; - regNumber rLimit; + regNumber rOffset = initReg; + regNumber rLimit; regMaskGpr tempMask; // We pick the next lowest register number for rLimit diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 5d95fff549b9ce..db01f11d7f719a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3850,8 +3850,8 @@ void CodeGen::genJmpMethod(GenTree* jmp) #endif // Next move any un-enregistered register arguments back to their register. - regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. - unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. + regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. + unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { varDsc = compiler->lvaGetDesc(varNum); @@ -4088,7 +4088,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) GetEmitter()->emitDisableGC(); for (int argNum = 0, argOffset = 0; argNum < MAX_REG_ARG; ++argNum) { - regNumber argReg = intArgRegs[argNum]; + regNumber argReg = intArgRegs[argNum]; regMaskGpr argRegMask = genRegMask(argReg); if ((remainingIntArgMask & argRegMask) != 0) @@ -4947,7 +4947,7 @@ void CodeGen::genPushCalleeSavedRegisters() #if defined(TARGET_ARM) regMaskFloat maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat; + regMaskGpr maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat; maskPushRegsInt |= genStackAllocRegisterMask(compiler->compLclFrameSize, maskPushRegsFloat); @@ -5054,7 +5054,7 @@ void CodeGen::genPushCalleeSavedRegisters() int offset; // This will be the starting place for saving the callee-saved registers, in increasing order. regMaskFloat maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat; + regMaskGpr maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat; #ifdef DEBUG if (verbose) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7c7e1134b717c3..7a23df0407f9aa 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -2868,12 +2868,12 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere } #endif - unsigned argMax; // maximum argNum value plus 1, (including the RetBuffArg) - unsigned argNum; // current argNum, always in [0..argMax-1] - unsigned fixedRetBufIndex; // argNum value used by the fixed return buffer argument (ARM64) - unsigned regArgNum; // index into the regArgTab[] table + unsigned argMax; // maximum argNum value plus 1, (including the RetBuffArg) + unsigned argNum; // current argNum, always in [0..argMax-1] + unsigned fixedRetBufIndex; // argNum value used by the fixed return buffer argument (ARM64) + unsigned regArgNum; // index into the regArgTab[] table regMaskOnlyOne regArgMaskLive = regState->rsCalleeRegArgMaskLiveIn; - bool doingFloat = regState->rsIsFloat; + bool doingFloat = regState->rsIsFloat; assert(compiler->IsOnlyOneRegMask(regArgMaskLive)); assert((doingFloat && compiler->IsFloatRegMask(regArgMaskLive)) || compiler->IsGprRegMask(regArgMaskLive)); @@ -3657,7 +3657,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere // We pick the lowest avail register number regMaskFloat tempMask = genFindLowestBit(fpAvailMask); - xtraReg = genRegNumFromMask(tempMask); + xtraReg = genRegNumFromMask(tempMask); } #if defined(TARGET_X86) // This case shouldn't occur on x86 since NYI gets converted to an assert @@ -5419,7 +5419,7 @@ void CodeGen::genFinalizeFrame() #if defined(TARGET_ARM) // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? regMaskFloat maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat; + regMaskGpr maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat; if ((maskPushRegsFloat != RBM_NONE) || (compiler->opts.MinOpts() && (regSet.rsMaskResvd & maskCalleeRegsPushed & RBM_OPT_RSVD))) @@ -5645,7 +5645,7 @@ void CodeGen::genFnProlog() int GCrefHi = -INT_MAX; bool hasGCRef = false; - regMaskGpr initRegs = RBM_NONE; // Registers which must be init'ed. + regMaskGpr initRegs = RBM_NONE; // Registers which must be init'ed. regMaskFloat initFltRegs = RBM_NONE; // FP registers which must be init'ed. regMaskFloat initDblRegs = RBM_NONE; @@ -5720,7 +5720,7 @@ void CodeGen::genFnProlog() if (isInReg) { - regNumber regForVar = varDsc->GetRegNum(); + regNumber regForVar = varDsc->GetRegNum(); singleRegMask regMask = genRegMask(regForVar); if (!genIsValidFloatReg(regForVar)) { @@ -5836,7 +5836,7 @@ void CodeGen::genFnProlog() // will be skipped. bool initRegZeroed = false; regMaskOnlyOne excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; - regMaskGpr tempMask; + regMaskGpr tempMask; // We should not use the special PINVOKE registers as the initReg // since they are trashed by the jithelper call to setup the PINVOKE frame diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index 058cb1fa81e887..fd17d4d0da4f00 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -37,8 +37,8 @@ class emitter; struct RegState { regMaskOnlyOne rsCalleeRegArgMaskLiveIn; // mask of register arguments (live on entry to method) - unsigned rsCalleeRegArgCount; // total number of incoming register arguments of this kind (int or float) - bool rsIsFloat; // true for float argument registers, false for integer argument registers + unsigned rsCalleeRegArgCount; // total number of incoming register arguments of this kind (int or float) + bool rsIsFloat; // true for float argument registers, false for integer argument registers }; //-------------------- CodeGenInterface --------------------------------- diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 0c461272c75692..11a33cef744cc5 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -202,8 +202,8 @@ void CodeGen::genCodeForBBlist() // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). regMaskMixed newLiveRegSet = RBM_NONE; - regMaskGpr newRegGCrefSet = RBM_NONE; - regMaskGpr newRegByrefSet = RBM_NONE; + regMaskGpr newRegGCrefSet = RBM_NONE; + regMaskGpr newRegByrefSet = RBM_NONE; #ifdef DEBUG VARSET_TP removedGCVars(VarSetOps::MakeEmpty(compiler)); VARSET_TP addedGCVars(VarSetOps::MakeEmpty(compiler)); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 8eae1aa637f954..ef90563bb1471c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -126,7 +126,7 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg) } } - regNumber regGSCheck; + regNumber regGSCheck; regMaskGpr regMaskGSCheck = RBM_NONE; if (!pushReg) @@ -2373,7 +2373,10 @@ void CodeGen::genEstablishFramePointer(int delta, bool reportUnwindData) // Return value: // None // -void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn) +void CodeGen::genAllocLclFrame(unsigned frameSize, + regNumber initReg, + bool* pInitRegZeroed, + regMaskGpr maskArgRegsLiveIn) { assert(compiler->IsGprRegMask(maskArgRegsLiveIn)); assert(compiler->compGeneratingProlog); @@ -5798,9 +5801,9 @@ void CodeGen::genCodeForSwap(GenTreeOp* tree) // FP swap is not yet implemented (and should have NYI'd in LSRA) assert(varTypeUsesIntReg(type1)); - regNumber oldOp1Reg = lcl1->GetRegNum(); + regNumber oldOp1Reg = lcl1->GetRegNum(); regMaskGpr oldOp1RegMask = genRegMask(oldOp1Reg); - regNumber oldOp2Reg = lcl2->GetRegNum(); + regNumber oldOp2Reg = lcl2->GetRegNum(); regMaskGpr oldOp2RegMask = genRegMask(oldOp2Reg); // We don't call genUpdateVarReg because we don't have a tree node with the new register. @@ -6640,8 +6643,8 @@ void CodeGen::genJmpMethod(GenTree* jmp) #endif // Next move any un-enregistered register arguments back to their register. - regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. - unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. + regMaskGpr fixedIntArgMask = RBM_NONE; // tracks the int arg regs occupying fixed args in case of a vararg method. + unsigned firstArgVarNum = BAD_VAR_NUM; // varNum of the first argument in case of a vararg method. for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { varDsc = compiler->lvaGetDesc(varNum); @@ -6821,7 +6824,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) GetEmitter()->emitDisableGC(); for (int argNum = 0, argOffset = 0; argNum < MAX_REG_ARG; ++argNum) { - regNumber argReg = intArgRegs[argNum]; + regNumber argReg = intArgRegs[argNum]; regMaskGpr argRegMask = genRegMask(argReg); if ((remainingIntArgMask & argRegMask) != 0) @@ -9143,8 +9146,8 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); - regNumber callTarget = REG_NA; - regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regNumber callTarget = REG_NA; + regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { @@ -9173,7 +9176,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, { // If a callTargetReg has not been explicitly provided, we will use REG_DEFAULT_HELPER_CALL_TARGET, but // this is only a valid assumption if the helper call is known to kill REG_DEFAULT_HELPER_CALL_TARGET. - callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; + callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; regMaskGpr callTargetMask = genRegMask(callTargetReg); noway_assert((callTargetMask & killMask) == callTargetMask); } @@ -9788,7 +9791,7 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // Emit appropriate unwind. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); + regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; int const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; @@ -9868,7 +9871,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // Figure out which set of int callee saves still needs saving. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); + regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; regMaskGpr const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; @@ -10203,14 +10206,14 @@ void CodeGen::genFnEpilog(BasicBlock* block) PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); - regMaskGpr const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; - regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; - regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; - unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; - unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; - unsigned const osrCalleeSaveSize = compiler->compCalleeRegsPushed * REGSIZE_BYTES; - unsigned const osrFramePointerSize = isFramePointerUsed() ? REGSIZE_BYTES : 0; - unsigned const osrAdjust = + regMaskGpr const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; + unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; + unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; + unsigned const osrCalleeSaveSize = compiler->compCalleeRegsPushed * REGSIZE_BYTES; + unsigned const osrFramePointerSize = isFramePointerUsed() ? REGSIZE_BYTES : 0; + unsigned const osrAdjust = tier0FrameSize - tier0IntCalleeSaveUsedSize + osrCalleeSaveSize + osrFramePointerSize; JITDUMP("OSR epilog adjust factors: tier0 frame %u, tier0 callee saves -%u, osr callee saves %u, osr " diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 7188d46798519a..56d4fe018b69dd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11091,7 +11091,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // regMaskFloat rbmAllFloat; regMaskFloat rbmFltCalleeTrash; - unsigned cntCalleeTrashFloat; + unsigned cntCalleeTrashFloat; public: FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const @@ -11129,8 +11129,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // regMaskPredicate rbmAllMask; regMaskPredicate rbmMskCalleeTrash; - unsigned cntCalleeTrashMask; - regMaskMixed varTypeCalleeTrashRegs[TYP_COUNT]; + unsigned cntCalleeTrashMask; + regMaskMixed varTypeCalleeTrashRegs[TYP_COUNT]; public: FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 045e29139610fb..d07d5b9f09caae 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -2891,7 +2891,7 @@ bool emitter::emitNoGChelper(CORINFO_METHOD_HANDLE methHnd) */ void* emitter::emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskGpr gcrefRegs, + regMaskGpr gcrefRegs, regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block)) { /* Create a new IG if the current one is non-empty */ @@ -8954,7 +8954,7 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskGpr regs, BYTE* addr) do { regMaskGpr bit = genFindLowestBit(chg); - regNumber reg = genRegNumFromMask(bit); + regNumber reg = genRegNumFromMask(bit); if (life & bit) { diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 256c6079b7c681..1112249c7b0f7e 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2135,11 +2135,11 @@ class emitter { instrDescCGCA() = delete; - VARSET_TP idcGCvars; // ... updated GC vars or - ssize_t idcDisp; // ... big addrmode disp + VARSET_TP idcGCvars; // ... updated GC vars or + ssize_t idcDisp; // ... big addrmode disp regMaskGpr idcGcrefRegs; // ... gcref registers regMaskGpr idcByrefRegs; // ... byref registers - unsigned idcArgCnt; // ... lots of args or (<0 ==> caller pops args) + unsigned idcArgCnt; // ... lots of args or (<0 ==> caller pops args) #if MULTIREG_HAS_SECOND_GC_RET // This method handle the GC-ness of the second register in a 2 register returned struct on System V. @@ -2250,8 +2250,8 @@ class emitter VARSET_TP debugPrevGCrefVars; VARSET_TP debugThisGCrefVars; regPtrDsc* debugPrevRegPtrDsc; - regMaskGpr debugPrevGCrefRegs; - regMaskGpr debugPrevByrefRegs; + regMaskGpr debugPrevGCrefRegs; + regMaskGpr debugPrevByrefRegs; void emitDispInsIndent(); void emitDispGCDeltaTitle(const char* title); void emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMaskGpr curRegs); @@ -2645,11 +2645,11 @@ class emitter // in that tracking. See emitSavIG(): the important use of ByrefRegs is commented // out, and GCrefRegs is always saved. - VARSET_TP emitPrevGCrefVars; + VARSET_TP emitPrevGCrefVars; regMaskGpr emitPrevGCrefRegs; regMaskGpr emitPrevByrefRegs; - VARSET_TP emitInitGCrefVars; + VARSET_TP emitInitGCrefVars; regMaskGpr emitInitGCrefRegs; regMaskGpr emitInitByrefRegs; @@ -2668,7 +2668,7 @@ class emitter // really the only one used; the others seem to be calculated, but not // used due to bugs. - VARSET_TP emitThisGCrefVars; + VARSET_TP emitThisGCrefVars; regMaskGpr emitThisGCrefRegs; // Current set of registers holding GC references regMaskGpr emitThisByrefRegs; // Current set of registers holding BYREF references @@ -2853,7 +2853,7 @@ class emitter // Sets the emitter's record of the currently live GC variables // and registers. void* emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskGpr gcrefRegs, + regMaskGpr gcrefRegs, regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block = nullptr)); // Same as above, except the label is added and is conceptually "inline" in @@ -3197,7 +3197,7 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regMaskMixed regs); + void emitDispRegSet(regMaskMixed regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 06b986a6ea971b..dcae9cc0f22cbb 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -5763,8 +5763,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) switch (fmt) { - int imm; - BYTE* addr; + int imm; + BYTE* addr; regMaskGpr gcrefRegs; regMaskGpr byrefRegs; diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 7eb74ce85f2cfa..c00b6b895edb88 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14479,8 +14479,8 @@ void emitter::emitIns_Call(EmitCallType callType, emitAttr retSize, emitAttr secondRetSize, VARSET_VALARG_TP ptrVars, - regMaskGpr gcrefRegs, - regMaskGpr byrefRegs, + regMaskGpr gcrefRegs, + regMaskGpr byrefRegs, const DebugInfo& di /* = DebugInfo() */, regNumber ireg /* = REG_NA */, regNumber xreg /* = REG_NA */, @@ -18363,8 +18363,8 @@ BYTE* emitter::emitOutputVectorConstant( unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) { const unsigned char callInstrSize = sizeof(code_t); // 4 bytes - regMaskGpr gcrefRegs; - regMaskGpr byrefRegs; + regMaskGpr gcrefRegs; + regMaskGpr byrefRegs; VARSET_TP GCvars(VarSetOps::UninitVal()); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 96bf8c9d50cd88..f3578f1488bebf 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -970,7 +970,7 @@ struct GenTree regMaskSmall gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskMixed mask = (regMaskMixed)-1) const; + unsigned AvailableTempRegCount(regMaskMixed mask = (regMaskMixed)-1) const; regNumber GetSingleTempReg(regMaskMixed mask = (regMaskMixed)-1); regNumber ExtractTempReg(regMaskMixed mask = (regMaskMixed)-1); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index f8bb65c3c5f8e1..219ca330d29379 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -295,7 +295,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskMixed mask, Interval* cur regMaskMixed result = RBM_NONE; while (candidates != RBM_NONE) { - regNumber regNum = genFirstRegNumFromMask(candidates); + regNumber regNum = genFirstRegNumFromMask(candidates); singleRegMask candidateBit = genRegMask(regNum); candidates ^= candidateBit; @@ -384,9 +384,9 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) // interval - Interval of Refposition. // assignedReg - Assigned register for this refposition. // -void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - regMaskOnlyOne* regsToFree, +void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + regMaskOnlyOne* regsToFree, regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { @@ -473,9 +473,9 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // Otherwise returns regMaskActual. // regMaskMixed LinearScan::getConstrainedRegMask(RefPosition* refPosition, - regMaskMixed regMaskActual, - regMaskMixed regMaskConstraint, - unsigned minRegCount) + regMaskMixed regMaskActual, + regMaskMixed regMaskConstraint, + unsigned minRegCount) { regMaskMixed newMask = regMaskActual & regMaskConstraint; if (genCountBits(newMask) < minRegCount) @@ -2818,7 +2818,7 @@ RegisterType LinearScan::getRegisterType(Interval* currentInterval, RefPosition* { assert(refPosition->getInterval() == currentInterval); RegisterType regType = currentInterval->registerType; - regMaskMixed candidates = refPosition->registerAssignment; + regMaskMixed candidates = refPosition->registerAssignment; #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // The LoongArch64's ABI which the float args maybe passed by integer register // when no float register left but free integer register. @@ -2934,9 +2934,9 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!enregisterLocalVars); - regNumber foundReg; - regMaskMixed foundRegBit; - RegRecord* availablePhysRegRecord; + regNumber foundReg; + regMaskMixed foundRegBit; + RegRecord* availablePhysRegRecord; foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) { @@ -3233,8 +3233,8 @@ bool LinearScan::isRefPositionActive(RefPosition* refPosition, LsraLocation refL // bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord) { - singleRegMask candidateBit = genRegMask(physRegRecord->regNum); - LsraLocation refLocation = refPosition->nodeLocation; + singleRegMask candidateBit = genRegMask(physRegRecord->regNum); + LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. assert(!isRegBusy(physRegRecord->regNum, current->registerType)); @@ -4070,8 +4070,8 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. regMaskMixed candidateRegs = allRegs(interval->registerType) & availableRegs; - regNumber firstReg = REG_NA; - regNumber newReg = REG_NA; + regNumber firstReg = REG_NA; + regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -4375,7 +4375,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // inactive registers available for the rotation. regMaskMixed inactiveRegs = RBM_NONE; #endif // DEBUG - regMaskMixed liveRegs = RBM_NONE; + regMaskMixed liveRegs = RBM_NONE; VarSetOps::Iter iter(compiler, currentLiveVars); unsigned varIndex = 0; while (iter.NextElem(&varIndex)) @@ -4900,11 +4900,11 @@ void LinearScan::allocateRegistersMinimal() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskMixed regsToFree = RBM_NONE; - regMaskMixed delayRegsToFree = RBM_NONE; - regMaskMixed regsToMakeInactive = RBM_NONE; - regMaskMixed delayRegsToMakeInactive = RBM_NONE; - regMaskMixed copyRegsToFree = RBM_NONE; + regMaskMixed regsToFree = RBM_NONE; + regMaskMixed delayRegsToFree = RBM_NONE; + regMaskMixed regsToMakeInactive = RBM_NONE; + regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -5173,7 +5173,7 @@ void LinearScan::allocateRegistersMinimal() } singleRegMask assignedRegBit = RBM_NONE; - bool isInRegister = false; + bool isInRegister = false; if (assignedRegister != REG_NA) { isInRegister = true; @@ -5237,7 +5237,7 @@ void LinearScan::allocateRegistersMinimal() { regNumber copyReg = assignCopyRegMinimal(¤tRefPosition); - lastAllocatedRefPosition = ¤tRefPosition; + lastAllocatedRefPosition = ¤tRefPosition; regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); @@ -5362,7 +5362,7 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) @@ -5581,11 +5581,11 @@ void LinearScan::allocateRegisters() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskMixed regsToFree = RBM_NONE; - regMaskMixed delayRegsToFree = RBM_NONE; - regMaskMixed regsToMakeInactive = RBM_NONE; - regMaskMixed delayRegsToMakeInactive = RBM_NONE; - regMaskMixed copyRegsToFree = RBM_NONE; + regMaskMixed regsToFree = RBM_NONE; + regMaskMixed delayRegsToFree = RBM_NONE; + regMaskMixed regsToMakeInactive = RBM_NONE; + regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -6051,7 +6051,7 @@ void LinearScan::allocateRegisters() } singleRegMask assignedRegBit = RBM_NONE; - bool isInRegister = false; + bool isInRegister = false; if (assignedRegister != REG_NA) { isInRegister = true; @@ -6098,8 +6098,8 @@ void LinearScan::allocateRegisters() if (currentInterval->isLocalVar) { regMaskMixed preferences = currentInterval->registerPreferences; - bool keepAssignment = true; - bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; + bool keepAssignment = true; + bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; // Will the assigned register cover the lifetime? If not, does it at least // meet the preferences for the next RefPosition? @@ -6205,7 +6205,7 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { - lastAllocatedRefPosition = ¤tRefPosition; + lastAllocatedRefPosition = ¤tRefPosition; regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); @@ -6303,7 +6303,7 @@ void LinearScan::allocateRegisters() copyReg = assignCopyReg(¤tRefPosition); } - lastAllocatedRefPosition = ¤tRefPosition; + lastAllocatedRefPosition = ¤tRefPosition; regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); @@ -6526,7 +6526,7 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) @@ -8215,7 +8215,7 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { regMaskMixed initialRegMask = interval->firstRefPosition->registerAssignment; - regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) + regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK : genRegNumFromMask(initialRegMask); @@ -8542,7 +8542,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskMixed terminatorConsumedRegs) + regMaskMixed terminatorConsumedRegs) { // TODO-Throughput: This would be much more efficient if we add RegToVarMaps instead of VarToRegMaps // and they would be more space-efficient as well. @@ -8828,7 +8828,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // available to copy into. // Note that for this purpose we use the full live-out set, because we must ensure that // even the registers that remain the same across the edge are preserved correctly. - regMaskMixed liveOutRegs = RBM_NONE; + regMaskMixed liveOutRegs = RBM_NONE; VarSetOps::Iter liveOutIter(compiler, block->bbLiveOut); unsigned liveOutVarIndex = 0; while (liveOutIter.NextElem(&liveOutVarIndex)) @@ -8927,9 +8927,9 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) } } - VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; - regMaskMixed sameWriteRegs = RBM_NONE; - regMaskMixed diffReadRegs = RBM_NONE; + VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; + regMaskMixed sameWriteRegs = RBM_NONE; + regMaskMixed diffReadRegs = RBM_NONE; // For each var that may require resolution, classify them as: // - in the same register at the end of this block and at each target (no resolution needed) @@ -9363,7 +9363,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskMixed terminatorConsumedRegs) + regMaskMixed terminatorConsumedRegs) { VarToRegMap fromVarToRegMap = getOutVarToRegMap(fromBlock->bbNum); VarToRegMap toVarToRegMap; @@ -9564,7 +9564,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regMaskMixed targetCandidates = targetRegsToDo; while (targetCandidates != RBM_NONE) { - regNumber targetReg = genFirstRegNumFromMask(targetCandidates); + regNumber targetReg = genFirstRegNumFromMask(targetCandidates); singleRegMask targetRegMask = genRegMask(targetReg); targetCandidates ^= targetRegMask; if (location[targetReg] == REG_NA) @@ -9595,7 +9595,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { while (targetRegsReady != RBM_NONE) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); + regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); singleRegMask targetRegMask = genRegMask(targetReg); targetRegsToDo ^= targetRegMask; targetRegsReady ^= targetRegMask; @@ -9638,9 +9638,9 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { // We may have freed up the other half of a double where the lower half // was already free. - regNumber lowerHalfReg = REG_PREV(fromReg); - regNumber lowerHalfSrcReg = (regNumber)source[lowerHalfReg]; - regNumber lowerHalfSrcLoc = (regNumber)location[lowerHalfReg]; + regNumber lowerHalfReg = REG_PREV(fromReg); + regNumber lowerHalfSrcReg = (regNumber)source[lowerHalfReg]; + regNumber lowerHalfSrcLoc = (regNumber)location[lowerHalfReg]; singleRegMask lowerHalfRegMask = genRegMask(lowerHalfReg); // Necessary conditions: // - There is a source register for this reg (lowerHalfSrcReg != REG_NA) @@ -9666,7 +9666,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } if (targetRegsToDo != RBM_NONE) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); + regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); singleRegMask targetRegMask = genRegMask(targetReg); // Is it already there due to other moves? @@ -12662,7 +12662,7 @@ void LinearScan::RegisterSelection::try_BEST_FIT() LsraLocation bestFitLocation = earliestIsBest ? MaxLocation : MinLocation; for (regMaskOnlyOne bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) { - regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); + regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); singleRegMask bestFitCandidateBit = genRegMask(bestFitCandidateRegNum); bestFitCandidates ^= bestFitCandidateBit; @@ -12757,11 +12757,11 @@ void LinearScan::RegisterSelection::try_REG_ORDER() // This will always result in a single candidate. That is, it is the tie-breaker // for free candidates, and doesn't make sense as anything other than the last // heuristic for free registers. - unsigned lowestRegOrder = UINT_MAX; + unsigned lowestRegOrder = UINT_MAX; regMaskOnlyOne lowestRegOrderBit = RBM_NONE; for (regMaskOnlyOne regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { - regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); + regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); singleRegMask regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; @@ -12797,7 +12797,7 @@ void LinearScan::RegisterSelection::try_SPILL_COST() for (regMaskOnlyOne spillCandidates = candidates; spillCandidates != RBM_NONE;) { - regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); + regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); singleRegMask spillCandidateBit = genRegMask(spillCandidateRegNum); spillCandidates ^= spillCandidateBit; @@ -12918,11 +12918,11 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() { assert(!found); - LsraLocation farthestLocation = MinLocation; + LsraLocation farthestLocation = MinLocation; regMaskOnlyOne farthestSet = RBM_NONE; for (regMaskOnlyOne farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { - regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); + regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); singleRegMask farthestCandidateBit = genRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; @@ -12955,7 +12955,7 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() regMaskOnlyOne prevRegOptSet = RBM_NONE; for (regMaskOnlyOne prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { - regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); + regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); singleRegMask prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; @@ -13058,7 +13058,7 @@ void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate regMaskOnlyOne coversCandidates = candidates; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13082,11 +13082,11 @@ void LinearScan::RegisterSelection::calculateCoversSets() return; } - preferenceSet = (candidates & preferences); + preferenceSet = (candidates & preferences); regMaskOnlyOne coversCandidates = (preferenceSet == RBM_NONE) ? candidates : preferenceSet; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13158,8 +13158,8 @@ void LinearScan::RegisterSelection::calculateCoversSets() // Register bit selected (a single register) and REG_NA if no register was selected. // template -singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval, - RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) +singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { #ifdef DEBUG *registerScore = NONE; @@ -13401,7 +13401,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* cur regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; @@ -13492,9 +13492,9 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* cur // Remove the `inUseOrBusyRegsMask` from the original candidates list and find one // such range that is consecutive. Next, append that range to the `candidates`. // - regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask; + regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask; regMaskFloat overallLimitCandidates; - regMaskFloat limitConsecutiveResult = + regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, &overallLimitCandidates); assert(limitConsecutiveResult != RBM_NONE); @@ -13624,8 +13624,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* cur // are busy). // singleRegMask LinearScan::RegisterSelection::selectMinimal( - Interval* currentInterval, - RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) + Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!linearScan->enregisterLocalVars); #ifdef DEBUG @@ -13725,7 +13724,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 14d4a68d9ed9b1..8b6f2dac377c5f 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -716,7 +716,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskMixed terminatorConsumedRegs); + regMaskMixed terminatorConsumedRegs); void resolveEdges(); @@ -785,19 +785,19 @@ class LinearScan : public LinearScanInterface #elif defined(TARGET_ARM) // On ARM, we may need two registers to set up the target register for a virtual call, so we need // to have at least the maximum number of arg registers, plus 2. - static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5); static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F16 | RBM_F17); #elif defined(TARGET_ARM64) - static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); static const regMaskFloat LsraLimitSmallFPSet = (RBM_V0 | RBM_V1 | RBM_V2 | RBM_V8 | RBM_V9); #elif defined(TARGET_X86) - static const regMaskGpr LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EDI); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EDI); static const regMaskFloat LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); #elif defined(TARGET_LOONGARCH64) - static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); #elif defined(TARGET_RISCV64) - static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); + static const regMaskGpr LsraLimitSmallIntSet = (RBM_T1 | RBM_T3 | RBM_A0 | RBM_A1 | RBM_T0); static const regMaskFloat LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F8 | RBM_F9); #else #error Unsupported or unset target architecture @@ -810,9 +810,9 @@ class LinearScan : public LinearScanInterface // TODO: Can have separate methods for each type regMaskMixed getConstrainedRegMask(RefPosition* refPosition, - regMaskMixed regMaskActual, - regMaskMixed regMaskConstrain, - unsigned minRegCount); + regMaskMixed regMaskActual, + regMaskMixed regMaskConstrain, + unsigned minRegCount); regMaskMixed stressLimitRegs(RefPosition* refPosition, regMaskMixed mask); // This controls the heuristics used to select registers @@ -1096,7 +1096,7 @@ class LinearScan : public LinearScanInterface bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskMixed killMask); regMaskOnlyOne allRegs(RegisterType rt); - regMaskGpr allByteRegs(); + regMaskGpr allByteRegs(); regMaskFloat allSIMDRegs(); regMaskFloat lowSIMDRegs(); regMaskFloat internalFloatRegCandidates(); @@ -1158,12 +1158,12 @@ class LinearScan : public LinearScanInterface RefPosition* newRefPositionRaw(LsraLocation nodeLocation, GenTree* treeNode, RefType refType); - RefPosition* newRefPosition(Interval* theInterval, - LsraLocation theLocation, - RefType theRefType, - GenTree* theTreeNode, - regMaskOnlyOne mask, - unsigned multiRegIdx = 0); + RefPosition* newRefPosition(Interval* theInterval, + LsraLocation theLocation, + RefType theRefType, + GenTree* theTreeNode, + regMaskOnlyOne mask, + unsigned multiRegIdx = 0); RefPosition* newRefPosition( regNumber reg, LsraLocation theLocation, RefType theRefType, GenTree* theTreeNode, regMaskOnlyOne mask); @@ -1224,13 +1224,12 @@ class LinearScan : public LinearScanInterface bool canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); void assignConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); regMaskFloat getConsecutiveCandidates(regMaskFloat candidates, - RefPosition* refPosition, - regMaskFloat* busyCandidates); + RefPosition* refPosition, + regMaskFloat* busyCandidates); regMaskFloat filterConsecutiveCandidates(regMaskFloat candidates, - unsigned int registersNeeded, - regMaskFloat* allConsecutiveCandidates); - regMaskFloat filterConsecutiveCandidatesForSpill(regMaskFloat consecutiveCandidates, - unsigned int registersNeeded); + unsigned int registersNeeded, + regMaskFloat* allConsecutiveCandidates); + regMaskFloat filterConsecutiveCandidatesForSpill(regMaskFloat consecutiveCandidates, unsigned int registersNeeded); #endif // TARGET_ARM64 regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates ARM_ARG(var_types regType)) @@ -1263,11 +1262,11 @@ class LinearScan : public LinearScanInterface // Perform register selection and update currentInterval or refPosition template - FORCEINLINE singleRegMask select(Interval* currentInterval, - RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); + FORCEINLINE singleRegMask select(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); - FORCEINLINE singleRegMask selectMinimal(Interval* currentInterval, - RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); + FORCEINLINE singleRegMask selectMinimal(Interval* currentInterval, + RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); // If the register is from unassigned set such that it was not already // assigned to the current interval @@ -1310,16 +1309,16 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne candidates; regMaskOnlyOne preferences = RBM_NONE; - Interval* relatedInterval = nullptr; + Interval* relatedInterval = nullptr; regMaskOnlyOne relatedPreferences = RBM_NONE; - LsraLocation rangeEndLocation; - LsraLocation relatedLastLocation; - bool preferCalleeSave = false; - RefPosition* rangeEndRefPosition; - RefPosition* lastRefPosition; + LsraLocation rangeEndLocation; + LsraLocation relatedLastLocation; + bool preferCalleeSave = false; + RefPosition* rangeEndRefPosition; + RefPosition* lastRefPosition; regMaskOnlyOne callerCalleePrefs = RBM_NONE; - LsraLocation lastLocation; + LsraLocation lastLocation; singleRegMask foundRegBit; @@ -1336,11 +1335,11 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne preferenceSet; regMaskOnlyOne coversRelatedSet; regMaskOnlyOne coversFullSet; - bool coversSetsCalculated = false; - bool found = false; - bool skipAllocation = false; - bool coversFullApplied = false; - bool constAvailableApplied = false; + bool coversSetsCalculated = false; + bool found = false; + bool skipAllocation = false; + bool coversFullApplied = false; + bool constAvailableApplied = false; // If the selected register is already assigned to the current internal FORCEINLINE bool isAlreadyAssigned() @@ -1349,7 +1348,7 @@ class LinearScan : public LinearScanInterface return (prevRegBit & preferences) == foundRegBit; } - bool applySelection(int selectionScore, regMaskOnlyOne selectionCandidates); + bool applySelection(int selectionScore, regMaskOnlyOne selectionCandidates); bool applySingleRegSelection(int selectionScore, regMaskOnlyOne selectionCandidate); FORCEINLINE void calculateCoversSets(); FORCEINLINE void calculateUnassignedSets(); @@ -1411,7 +1410,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskMixed terminatorConsumedRegs); + regMaskMixed terminatorConsumedRegs); #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; @@ -1481,7 +1480,7 @@ class LinearScan : public LinearScanInterface // Current mask of registers being printed in the dump. regMaskMixed lastDumpedRegisters; regMaskMixed registersToDump; - int lastUsedRegNumIndex; + int lastUsedRegNumIndex; bool shouldDumpReg(regNumber regNum) { return (registersToDump & genRegMask(regNum)) != 0; @@ -1662,15 +1661,16 @@ class LinearScan : public LinearScanInterface VarToRegMap* outVarToRegMaps; // A temporary VarToRegMap used during the resolution of critical edges. - VarToRegMap sharedCriticalVarToRegMap; + VarToRegMap sharedCriticalVarToRegMap; PhasedVar actualRegistersMask; - PhasedVar availableIntRegs; + PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; #if defined(TARGET_XARCH) PhasedVar availableMaskRegs; #endif - PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, predicate + PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, + // predicate #if defined(TARGET_XARCH) #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) @@ -1719,7 +1719,7 @@ class LinearScan : public LinearScanInterface #if defined(TARGET_AMD64) static const var_types LargeVectorSaveType = TYP_SIMD16; #elif defined(TARGET_ARM64) - static const var_types LargeVectorSaveType = TYP_DOUBLE; + static const var_types LargeVectorSaveType = TYP_DOUBLE; #endif // !defined(TARGET_AMD64) && !defined(TARGET_ARM64) // Set of large vector (TYP_SIMD32 on AVX) variables. VARSET_TP largeVectorVars; @@ -1731,7 +1731,7 @@ class LinearScan : public LinearScanInterface // Register status //----------------------------------------------------------------------- - regMaskMixed m_AvailableRegs; //TODO: Should be separate for gpr, vector, predicate + regMaskMixed m_AvailableRegs; // TODO: Should be separate for gpr, vector, predicate regNumber getRegForType(regNumber reg, var_types regType) { #ifdef TARGET_ARM @@ -1745,7 +1745,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne getRegMask(regNumber reg, var_types regType) { - reg = getRegForType(reg, regType); + reg = getRegForType(reg, regType); singleRegMask regMask = genRegMask(reg); #ifdef TARGET_ARM if (regType == TYP_DOUBLE) @@ -1795,13 +1795,13 @@ class LinearScan : public LinearScanInterface void clearSpillCost(regNumber reg, var_types regType); void updateSpillCost(regNumber reg, Interval* interval); - FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - regMaskOnlyOne* regsToFree, + FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + regMaskOnlyOne* regsToFree, regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); - // TODO: This should be m_GprWithConstants, m_FloatRegsWithConstant, etc. + // TODO: This should be m_GprWithConstants, m_FloatRegsWithConstant, etc. regMaskMixed m_RegistersWithConstants; void clearConstantReg(regNumber reg, var_types regType) { @@ -1813,13 +1813,13 @@ class LinearScan : public LinearScanInterface } bool isRegConstant(regNumber reg, var_types regType) { - reg = getRegForType(reg, regType); + reg = getRegForType(reg, regType); regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_RegistersWithConstants & regMask) == regMask; } regMaskOnlyOne getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition); - regMaskMixed fixedRegs; //TODO: Should be also seperate + regMaskMixed fixedRegs; // TODO: Should be also seperate LsraLocation nextFixedRef[REG_COUNT]; void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); LsraLocation getNextFixedRef(regNumber regNum, var_types regType) @@ -1848,7 +1848,8 @@ class LinearScan : public LinearScanInterface } weight_t spillCost[REG_COUNT]; - regMaskMixed regsBusyUntilKill; //TODO: Likewise, probably have this global 32-bit and set it point to the specific version like gpr, vector, etc. + regMaskMixed regsBusyUntilKill; // TODO: Likewise, probably have this global 32-bit and set it point to the specific + // version like gpr, vector, etc. regMaskMixed regsInUseThisLocation; regMaskMixed regsInUseNextLocation; #ifdef TARGET_ARM64 @@ -1939,10 +1940,10 @@ class LinearScan : public LinearScanInterface RefPosition* BuildUse(GenTree* operand, regMaskOnlyOne candidates = RBM_NONE, int multiRegIdx = 0); void setDelayFree(RefPosition* use); - int BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates = RBM_NONE); - int BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates); + int BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates = RBM_NONE); + int BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates); #ifdef TARGET_XARCH - int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates = RBM_NONE); + int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates = RBM_NONE); inline regMaskFloat BuildEvexIncompatibleMask(GenTree* tree); #endif // !TARGET_XARCH int BuildSelect(GenTreeOp* select); @@ -1957,16 +1958,16 @@ class LinearScan : public LinearScanInterface int BuildSimple(GenTree* tree); int BuildOperandUses(GenTree* node, regMaskOnlyOne candidates = RBM_NONE); void AddDelayFreeUses(RefPosition* refPosition, GenTree* rmwNode); - int BuildDelayFreeUses(GenTree* node, - GenTree* rmwNode = nullptr, - regMaskOnlyOne candidates = RBM_NONE, - RefPosition** useRefPosition = nullptr); - int BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates = RBM_NONE); - int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); + int BuildDelayFreeUses(GenTree* node, + GenTree* rmwNode = nullptr, + regMaskOnlyOne candidates = RBM_NONE, + RefPosition** useRefPosition = nullptr); + int BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates = RBM_NONE); + int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); void BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates = RBM_NONE); - void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask); + void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 207a8f9f2e9d73..6ebfaa1d023bc8 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -192,10 +192,10 @@ int LinearScan::BuildShiftLongCarry(GenTree* tree) int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; + int srcCount; + int dstCount = 0; regMaskMixed killMask = RBM_NONE; - bool isLocalDefUse = false; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); @@ -666,7 +666,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_BITCAST: { assert(dstCount == 1); - regNumber argReg = tree->GetRegNum(); + regNumber argReg = tree->GetRegNum(); regMaskOnlyOne argMask = RBM_NONE; if (argReg != REG_COUNT) { diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 19c542339ed004..c9c2fb9105f0ee 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -177,8 +177,8 @@ bool LinearScan::canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition // set. // regMaskFloat LinearScan::filterConsecutiveCandidates(regMaskFloat candidates, - unsigned int registersNeeded, - regMaskFloat* allConsecutiveCandidates) + unsigned int registersNeeded, + regMaskFloat* allConsecutiveCandidates) { if (BitOperations::PopCount(candidates) < registersNeeded) { @@ -187,16 +187,16 @@ regMaskFloat LinearScan::filterConsecutiveCandidates(regMaskFloat candidates, return RBM_NONE; } - regMaskFloat currAvailableRegs = candidates; - regMaskFloat overallResult = RBM_NONE; - regMaskFloat consecutiveResult = RBM_NONE; + regMaskFloat currAvailableRegs = candidates; + regMaskFloat overallResult = RBM_NONE; + regMaskFloat consecutiveResult = RBM_NONE; // At this point, for 'n' registers requirement, if Rm, Rm+1, Rm+2, ..., Rm+k-1 are // available, create the mask only for Rm, Rm+1, ..., Rm+(k-n) to convey that it // is safe to assign any of those registers, but not beyond that. #define AppendConsecutiveMask(startIndex, endIndex, availableRegistersMask) \ - regMaskFloat selectionStartMask = (1ULL << regAvailableStartIndex) - 1; \ - regMaskFloat selectionEndMask = (1ULL << (regAvailableEndIndex - registersNeeded + 1)) - 1; \ + regMaskFloat selectionStartMask = (1ULL << regAvailableStartIndex) - 1; \ + regMaskFloat selectionEndMask = (1ULL << (regAvailableEndIndex - registersNeeded + 1)) - 1; \ consecutiveResult |= availableRegistersMask & (selectionEndMask & ~selectionStartMask); \ overallResult |= availableRegistersMask; @@ -330,8 +330,8 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec assert((registersNeeded >= 2) && (registersNeeded <= 4)); regMaskFloat consecutiveResultForBusy = RBM_NONE; regMaskFloat unprocessedRegs = consecutiveCandidates; - unsigned regAvailableStartIndex = 0, regAvailableEndIndex = 0; - int maxSpillRegs = registersNeeded; + unsigned regAvailableStartIndex = 0, regAvailableEndIndex = 0; + int maxSpillRegs = registersNeeded; regMaskFloat registersNeededMask = (1ULL << registersNeeded) - 1; do { @@ -340,7 +340,7 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec // For the current range, find how many registers are free vs. busy regMaskFloat maskForCurRange = RBM_NONE; - bool shouldCheckForRounding = false; + bool shouldCheckForRounding = false; switch (registersNeeded) { case 2: @@ -414,8 +414,8 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec // allCandidates = 0x1C080D0F00000000, the consecutive register mask returned // will be 0x400000300000000. // -regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, - RefPosition* refPosition, +regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, + RefPosition* refPosition, regMaskFloat* busyCandidates) { assert(compiler->info.compNeedsConsecutiveRegisters); @@ -447,9 +447,9 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, // register out of the `consecutiveResult` is available for the first RefPosition, then just use // that. This will avoid unnecessary copies. - regNumber firstRegNum = REG_NA; - regNumber prevRegNum = REG_NA; - int foundCount = 0; + regNumber firstRegNum = REG_NA; + regNumber prevRegNum = REG_NA; + int foundCount = 0; regMaskFloat foundRegMask = RBM_NONE; RefPosition* consecutiveRefPosition = getNextConsecutiveRefPosition(refPosition); @@ -529,7 +529,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, // try_FAR_NEXT_REF(), etc. here which would complicate things. Instead, we just go with option# 1 and select // registers based on fewer number of registers that has to be spilled. // - regMaskFloat overallResultForBusy; + regMaskFloat overallResultForBusy; regMaskFloat consecutiveResultForBusy = filterConsecutiveCandidates(allCandidates, registersNeeded, &overallResultForBusy); @@ -582,10 +582,10 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; + int srcCount; + int dstCount = 0; regMaskMixed killMask = RBM_NONE; - bool isLocalDefUse = false; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 19078fdf5822a2..a9190b7a0f975c 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -129,7 +129,7 @@ int LinearScan::BuildCall(GenTreeCall* call) { bool hasMultiRegRetVal = false; const ReturnTypeDesc* retTypeDesc = nullptr; - regMaskOnlyOne dstCandidates = RBM_NONE; + regMaskOnlyOne dstCandidates = RBM_NONE; int srcCount = 0; int dstCount = 0; @@ -148,7 +148,7 @@ int LinearScan::BuildCall(GenTreeCall* call) } } - GenTree* ctrlExpr = call->gtControlExpr; + GenTree* ctrlExpr = call->gtControlExpr; regMaskGpr ctrlExprCandidates = RBM_NONE; if (call->gtCallType == CT_INDIRECT) { @@ -490,7 +490,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) // Registers for split argument corresponds to source int dstCount = argNode->gtNumRegs; - regNumber argReg = argNode->GetRegNum(); + regNumber argReg = argNode->GetRegNum(); regMaskMixed argMask = RBM_NONE; for (unsigned i = 0; i < argNode->gtNumRegs; i++) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 60c9ca15051bfd..3b3b2f733a9acd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -249,15 +249,15 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de { assert(!interval->isLocalVar); - RefPosition* useRefPosition = defRefPosition->nextRefPosition; - regMaskOnlyOne defRegAssignment = defRefPosition->registerAssignment; + RefPosition* useRefPosition = defRefPosition->nextRefPosition; + regMaskOnlyOne defRegAssignment = defRefPosition->registerAssignment; regMaskOnlyOne useRegAssignment = useRefPosition->registerAssignment; - RegRecord* defRegRecord = nullptr; - RegRecord* useRegRecord = nullptr; - regNumber defReg = REG_NA; - regNumber useReg = REG_NA; - bool defRegConflict = ((defRegAssignment & useRegAssignment) == RBM_NONE); - bool useRegConflict = defRegConflict; + RegRecord* defRegRecord = nullptr; + RegRecord* useRegRecord = nullptr; + regNumber defReg = REG_NA; + regNumber useReg = REG_NA; + bool defRegConflict = ((defRegAssignment & useRegAssignment) == RBM_NONE); + bool useRegConflict = defRegConflict; // If the useRefPosition is a "delayRegFree", we can't change the registerAssignment // on it, or we will fail to ensure that the fixedReg is busy at the time the target @@ -351,7 +351,7 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de RegisterType regType = interval->registerType; assert((getRegisterType(interval, defRefPosition) == regType) && (getRegisterType(interval, useRefPosition) == regType)); - regMaskOnlyOne candidates = allRegs(regType); + regMaskOnlyOne candidates = allRegs(regType); defRefPosition->registerAssignment = candidates; defRefPosition->isFixedRegRef = false; return; @@ -555,12 +555,12 @@ RefPosition* LinearScan::newRefPosition( // Return Value: // a new RefPosition // -RefPosition* LinearScan::newRefPosition(Interval* theInterval, - LsraLocation theLocation, - RefType theRefType, - GenTree* theTreeNode, - regMaskOnlyOne mask, - unsigned multiRegIdx /* = 0 */) +RefPosition* LinearScan::newRefPosition(Interval* theInterval, + LsraLocation theLocation, + RefType theRefType, + GenTree* theTreeNode, + regMaskOnlyOne mask, + unsigned multiRegIdx /* = 0 */) { assert(compiler->IsOnlyOneRegMask(mask)); @@ -1407,9 +1407,9 @@ void LinearScan::buildInternalRegisterUses() assert(internalCount <= MaxInternalCount); for (int i = 0; i < internalCount; i++) { - RefPosition* def = internalDefs[i]; - regMaskOnlyOne mask = def->registerAssignment; - RefPosition* use = newRefPosition(def->getInterval(), currentLoc, RefTypeUse, def->treeNode, mask, 0); + RefPosition* def = internalDefs[i]; + regMaskOnlyOne mask = def->registerAssignment; + RefPosition* use = newRefPosition(def->getInterval(), currentLoc, RefTypeUse, def->treeNode, mask, 0); if (setInternalRegsDelayFree) { use->delayRegFree = true; @@ -1853,7 +1853,7 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc newRefPosition->minRegCandidateCount = minRegCountForRef; if (newRefPosition->IsActualRef() && doReverseCallerCallee()) { - Interval* interval = newRefPosition->getInterval(); + Interval* interval = newRefPosition->getInterval(); regMaskOnlyOne oldAssignment = newRefPosition->registerAssignment; regMaskOnlyOne calleeSaveMask = calleeSaveRegs(interval->registerType); #ifdef TARGET_ARM64 @@ -2299,7 +2299,7 @@ void LinearScan::buildIntervals() { Interval* interval = getIntervalForLocalVar(varIndex); const var_types regType = argDsc->GetRegisterType(); - regMaskOnlyOne mask = allRegs(regType); + regMaskOnlyOne mask = allRegs(regType); if (argDsc->lvIsRegArg) { // Set this interval as currently assigned to that register @@ -3140,7 +3140,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Find the registers that we should remove from the preference set because // they are occupied with argument values. regMaskMixed unpref = placedArgRegs; - unsigned varIndex = interval->getVarIndex(compiler); + unsigned varIndex = interval->getVarIndex(compiler); for (size_t i = 0; i < numPlacedArgLocals; i++) { if (placedArgLocals[i].VarIndex == varIndex) @@ -3462,10 +3462,10 @@ void LinearScan::AddDelayFreeUses(RefPosition* useRefPosition, GenTree* rmwNode) // Return Value: // The number of source registers used by the *parent* of this node. // -int LinearScan::BuildDelayFreeUses(GenTree* node, - GenTree* rmwNode, - regMaskOnlyOne candidates, - RefPosition** useRefPositionRef) +int LinearScan::BuildDelayFreeUses(GenTree* node, + GenTree* rmwNode, + regMaskOnlyOne candidates, + RefPosition** useRefPositionRef) { RefPosition* use = nullptr; GenTree* addr = nullptr; @@ -3661,7 +3661,7 @@ void LinearScan::BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, } regMaskOnlyOne defCandidates = RBM_NONE; - var_types type = varDsc->GetRegisterType(); + var_types type = varDsc->GetRegisterType(); #ifdef TARGET_X86 if (varTypeIsByte(type)) @@ -3856,7 +3856,7 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) } else { - srcCount = 1; + srcCount = 1; regMaskGpr srcCandidates = RBM_NONE; #ifdef TARGET_X86 var_types type = varDsc->GetRegisterType(storeLoc); @@ -3985,8 +3985,8 @@ int LinearScan::BuildReturn(GenTree* tree) RegisterType dstType = regType(retTypeDesc.GetReturnRegType(i)); if (srcType != dstType) { - hasMismatchedRegTypes = true; - regMaskOnlyOne dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i)); + hasMismatchedRegTypes = true; + regMaskOnlyOne dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i)); if (varTypeUsesIntReg(dstType)) { @@ -4140,8 +4140,8 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) // To avoid redundant moves, have the argument operand computed in the // register in which the argument is passed to the call. - singleRegMask argMask = genRegMask(argReg); - RefPosition* use = BuildUse(op1, argMask); + singleRegMask argMask = genRegMask(argReg); + RefPosition* use = BuildUse(op1, argMask); // Record that this register is occupied by a register now. placedArgRegs |= argMask; @@ -4311,8 +4311,8 @@ int LinearScan::BuildCmpOperands(GenTree* tree) { regMaskGpr op1Candidates = RBM_NONE; regMaskGpr op2Candidates = RBM_NONE; - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2(); + GenTree* op1 = tree->gtGetOp1(); + GenTree* op2 = tree->gtGetOp2(); #ifdef TARGET_X86 bool needByteRegs = false; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index d0a1c23500f00f..ef55d125321c53 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -45,10 +45,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; + int srcCount; + int dstCount = 0; regMaskMixed killMask = RBM_NONE; - bool isLocalDefUse = false; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); @@ -758,7 +758,7 @@ bool LinearScan::isRMWRegOper(GenTree* tree) // Support for building RefPositions for RMW nodes. int LinearScan::BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates) { - int srcCount = 0; + int srcCount = 0; regMaskGpr op1Candidates = candidates; regMaskGpr op2Candidates = candidates; @@ -1017,9 +1017,9 @@ int LinearScan::BuildShiftRotate(GenTree* tree) // For shift operations, we need that the number // of bits moved gets stored in CL in case // the number of bits to shift is not a constant. - int srcCount = 0; - GenTree* shiftBy = tree->gtGetOp2(); - GenTree* source = tree->gtGetOp1(); + int srcCount = 0; + GenTree* shiftBy = tree->gtGetOp2(); + GenTree* source = tree->gtGetOp1(); regMaskGpr srcCandidates = RBM_NONE; regMaskGpr dstCandidates = RBM_NONE; @@ -1130,7 +1130,7 @@ int LinearScan::BuildCall(GenTreeCall* call) const ReturnTypeDesc* retTypeDesc = nullptr; int srcCount = 0; int dstCount = 0; - regMaskOnlyOne dstCandidates = RBM_NONE; + regMaskOnlyOne dstCandidates = RBM_NONE; assert(!call->isContained()); if (call->TypeGet() != TYP_VOID) @@ -1610,7 +1610,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) if (blkNode->OperIs(GT_STORE_DYN_BLK)) { assert(compiler->IsGprRegMask(sizeRegMask)); - + useCount++; BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); } @@ -1868,10 +1868,10 @@ int LinearScan::BuildLclHeap(GenTree* tree) // int LinearScan::BuildModDiv(GenTree* tree) { - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2(); + GenTree* op1 = tree->gtGetOp1(); + GenTree* op2 = tree->gtGetOp2(); regMaskGpr dstCandidates = RBM_NONE; - int srcCount = 0; + int srcCount = 0; if (varTypeIsFloating(tree->TypeGet())) { @@ -2860,7 +2860,7 @@ int LinearScan::BuildIndir(GenTreeIndir* indirTree) } #endif // FEATURE_SIMD - int srcCount = BuildIndirUses(indirTree); + int srcCount = BuildIndirUses(indirTree); if (indirTree->gtOper == GT_STOREIND) { GenTree* source = indirTree->gtGetOp2(); @@ -2982,8 +2982,8 @@ int LinearScan::BuildMul(GenTree* tree) return BuildSimple(tree); } - int srcCount = BuildBinaryUses(tree->AsOp()); - int dstCount = 1; + int srcCount = BuildBinaryUses(tree->AsOp()); + int dstCount = 1; regMaskGpr dstCandidates = RBM_NONE; bool isUnsignedMultiply = ((tree->gtFlags & GTF_UNSIGNED) != 0); diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 8e162962a789e5..66101ad7a615ca 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -1969,7 +1969,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call bool callIsVararg = IsVarArgs(); #ifdef TARGET_ARM - regMaskGpr argSkippedRegMask = RBM_NONE; + regMaskGpr argSkippedRegMask = RBM_NONE; regMaskFloat fltArgSkippedRegMask = RBM_NONE; #endif // TARGET_ARM @@ -2546,7 +2546,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call (size == 1)) // The size to back-fill is one float register { // Back-fill the register. - isBackFilled = true; + isBackFilled = true; regMaskFloat backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); fltArgSkippedRegMask &= ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask diff --git a/src/coreclr/jit/regalloc.cpp b/src/coreclr/jit/regalloc.cpp index cbf8c3c556c991..9e1d1cf3e95598 100644 --- a/src/coreclr/jit/regalloc.cpp +++ b/src/coreclr/jit/regalloc.cpp @@ -99,7 +99,7 @@ bool Compiler::shouldDoubleAlign( // by linear scan. (It is not shared for System V AMD64 platform.) regNumber Compiler::raUpdateRegStateForArg(RegState* regState, LclVarDsc* argDsc) { - regNumber inArgReg = argDsc->GetArgReg(); + regNumber inArgReg = argDsc->GetArgReg(); singleRegMask inArgMask = genRegMask(inArgReg); if (regState->rsIsFloat) diff --git a/src/coreclr/jit/registerargconvention.h b/src/coreclr/jit/registerargconvention.h index 18bcfc85b476f5..214a22cddc5236 100644 --- a/src/coreclr/jit/registerargconvention.h +++ b/src/coreclr/jit/registerargconvention.h @@ -22,7 +22,7 @@ struct InitVarDscInfo // Support back-filling of FP parameters. This is similar to code in gtMorphArgs() that // handles arguments. regMaskFloat fltArgSkippedRegMask; - bool anyFloatStackArgs; + bool anyFloatStackArgs; #endif // TARGET_ARM #if defined(TARGET_ARM) || defined(TARGET_RISCV64) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index a1ce9f6d41b664..6a65636fb6ba4d 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -330,7 +330,7 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */) treeType = tree->TypeGet(); } - var_types tempType = RegSet::tmpNormalizeType(treeType); + var_types tempType = RegSet::tmpNormalizeType(treeType); bool floatSpill = false; if (isFloatRegType(treeType)) @@ -929,7 +929,7 @@ regNumber genRegArgNext(regNumber argReg) * register numbers and corresponding bitmaps. */ -const regNumber raRegCalleeSaveOrder[] = {REG_CALLEE_SAVED_ORDER}; +const regNumber raRegCalleeSaveOrder[] = {REG_CALLEE_SAVED_ORDER}; const regMaskGpr raRbmCalleeSaveOrder[] = {RBM_CALLEE_SAVED_ORDER}; regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 21eab9d1c1040d..f9218bbbe6e57a 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -67,7 +67,7 @@ class RegSet // private: - bool rsNeededSpillReg; // true if this method needed to spill any registers + bool rsNeededSpillReg; // true if this method needed to spill any registers regMaskMixed rsModifiedRegsMask; // mask of the registers modified by the current function. #ifdef DEBUG @@ -110,7 +110,8 @@ class RegSet SetMaskVars(_rsMaskVars | addMaskVars); } - void RemoveMaskVars(regMaskMixed removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) + void RemoveMaskVars(regMaskMixed removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset + // DiffD) { SetMaskVars(_rsMaskVars & ~removeMaskVars); } @@ -125,19 +126,19 @@ class RegSet #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog -#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 +#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 -public: // TODO-Cleanup: Should be private, but Compiler uses it +public: // TODO-Cleanup: Should be private, but Compiler uses it regMaskGpr rsMaskResvd; // mask of the registers that are reserved for special purposes (typically empty) public: // The PreSpill masks are used in LclVars.cpp #ifdef TARGET_ARM regMaskGpr rsMaskPreSpillAlign; // Mask of alignment padding added to prespill to keep double aligned args - // at aligned stack addresses. + // at aligned stack addresses. regMaskGpr rsMaskPreSpillRegArg; // mask of incoming registers that are spilled at the start of the prolog - // This includes registers used to pass a struct (or part of a struct) - // and all enregistered user arguments in a varargs call -#endif // TARGET_ARM + // This includes registers used to pass a struct (or part of a struct) + // and all enregistered user arguments in a varargs call +#endif // TARGET_ARM private: //------------------------------------------------------------------------- diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 3a4f69c241e465..951b14036ff1b1 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -220,7 +220,7 @@ typedef unsigned __int64 regMaskPredicate; // 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskMixed // 3. Revisit `regMaskMixed` and see how we can separate them into `regMaskGpr`, `regMaskFloat`, `regMaskPredicate`. // 4. Once #3 is solved, `regMaskOnlyOne` should be reduced to 32-bit and so can `regMaskFloat` and `regMaskPredicate` -// +// // // We will add a "//TODO: regMaskOnlyOne" through out the code @@ -238,7 +238,7 @@ typedef unsigned __int64 regMaskOnlyOne; // 0. Revisit regMaskMixed and see if they should be "regMaskMixed" // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together -typedef unsigned __int64 regMaskMixed; //TODO: Rename this to regMaskMixed +typedef unsigned __int64 regMaskMixed; // TODO: Rename this to regMaskMixed // TODO: For LSRA, the select() method should be regMaskOnlyOne because we will be // allocating either GPR or Vector or Mask but not all @@ -690,7 +690,7 @@ inline regMaskOnlyOne genRegMask(regNumber regNum, var_types type) * These arrays list the callee-saved register numbers (and bitmaps, respectively) for * the current architecture. */ -extern const regNumber raRegCalleeSaveOrder[CNT_CALLEE_SAVED]; +extern const regNumber raRegCalleeSaveOrder[CNT_CALLEE_SAVED]; extern const regMaskGpr raRbmCalleeSaveOrder[CNT_CALLEE_SAVED]; // This method takes a "compact" bitset of the callee-saved registers, and "expands" it to a full register mask. diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 7bf82b6faaef81..64025b6a1de5de 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -154,14 +154,14 @@ void Compiler::unwindPushPopCFI(regNumber reg) regMaskMixed relOffsetMask = RBM_CALLEE_SAVED #if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED - // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) - // is excluded from the callee-save register list. - // Make sure the register gets PUSH unwind info in this case, - // since it is pushed as a frame register. - | RBM_FPBASE + // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) + // is excluded from the callee-save register list. + // Make sure the register gets PUSH unwind info in this case, + // since it is pushed as a frame register. + | RBM_FPBASE #endif #if defined(TARGET_ARM) - | RBM_R11 | RBM_LR | RBM_PC + | RBM_R11 | RBM_LR | RBM_PC #endif ; @@ -206,10 +206,10 @@ void Compiler::unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat) assert(IsOnlyOneRegMask(regMask)); #if TARGET_ARM - regNumber regNum = isFloat ? REG_PREV(REG_FP_LAST) : REG_INT_LAST; + regNumber regNum = isFloat ? REG_PREV(REG_FP_LAST) : REG_INT_LAST; regMaskOnlyOne regBit = isFloat ? genRegMask(regNum) | genRegMask(REG_NEXT(regNum)) : genRegMask(regNum); #else - regNumber regNum = isFloat ? REG_FP_LAST : REG_INT_LAST; + regNumber regNum = isFloat ? REG_FP_LAST : REG_INT_LAST; regMaskOnlyOne regBit = genRegMask(regNum); #endif diff --git a/src/coreclr/jit/unwindarmarch.cpp b/src/coreclr/jit/unwindarmarch.cpp index 377f29e6d6f4bc..7a8b51573eac71 100644 --- a/src/coreclr/jit/unwindarmarch.cpp +++ b/src/coreclr/jit/unwindarmarch.cpp @@ -284,7 +284,7 @@ void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) } void Compiler::unwindPushPopMaskFloat(regMaskFloat maskFloat) -{ +{ // Only floating pointer registers can be specified in 'maskFloat' assert(IsFloatRegMask(maskFloat)); @@ -297,7 +297,7 @@ void Compiler::unwindPushPopMaskFloat(regMaskFloat maskFloat) UnwindInfo* pu = &funCurrentFunc()->uwi; - BYTE val = 0; + BYTE val = 0; regMaskFloat valMask = (RBM_F16 | RBM_F17); while (maskFloat != valMask) From 0659741b5b6c0d7882b4f758b1dc58b66f3caa4b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 12 Feb 2024 14:29:29 -0800 Subject: [PATCH 057/201] Convert m_RegistersWithConstants to regMaskOnlyOne --- src/coreclr/jit/clrjit.natvis | 14 ++++++++++-- src/coreclr/jit/lsra.cpp | 4 ++-- src/coreclr/jit/lsra.h | 41 ++++++++++++++++++++++++++++++----- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/clrjit.natvis b/src/coreclr/jit/clrjit.natvis index 95dd3dc305689b..450ce2017517e7 100644 --- a/src/coreclr/jit/clrjit.natvis +++ b/src/coreclr/jit/clrjit.natvis @@ -138,9 +138,9 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u reg = reg >> 1 - this->m_RegistersWithConstants + this->m_RegistersWithConstants[0] - + ((regNumber)regIndex),en @@ -148,6 +148,16 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u reg = reg >> 1 + this->m_RegistersWithConstants[1] + + + + + ((regNumber)regIndex),en + regIndex++ + reg = reg >> 1 + + diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 219ca330d29379..ddffcd8b1dee0a 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -291,8 +291,8 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo regMaskOnlyOne LinearScan::getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); - regMaskMixed candidates = (mask & m_RegistersWithConstants); - regMaskMixed result = RBM_NONE; + regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regTypeIndex(currentInterval->registerType)]); + regMaskOnlyOne result = RBM_NONE; while (candidates != RBM_NONE) { regNumber regNum = genFirstRegNumFromMask(candidates); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 8b6f2dac377c5f..4c1fd8b39dc2bc 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -64,6 +64,26 @@ RegisterType regType(T type) } } +template +unsigned regTypeIndex(T type) +{ + if (varTypeUsesIntReg(type)) + { + return 0; + } +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + else if (varTypeUsesMaskReg(type)) + { + return 2; + } +#endif // TARGET_XARCH && FEATURE_SIMD + else + { + assert(varTypeUsesFloatReg(type)); + return 1; + } +} + //------------------------------------------------------------------------ // useFloatReg: Check if the given var_type should be allocated to a FloatRegisterType // @@ -1760,7 +1780,12 @@ class LinearScan : public LinearScanInterface void resetAvailableRegs() { m_AvailableRegs = allAvailableRegs; - m_RegistersWithConstants = RBM_NONE; + m_RegistersWithConstants[0] = RBM_NONE; + m_RegistersWithConstants[1] = RBM_NONE; +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + m_RegistersWithConstants[2] = RBM_NONE; +#endif + } bool isRegAvailable(regNumber reg, var_types regType) @@ -1801,21 +1826,25 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); - // TODO: This should be m_GprWithConstants, m_FloatRegsWithConstant, etc. - regMaskMixed m_RegistersWithConstants; +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + regMaskOnlyOne m_RegistersWithConstants[3]; +#else + regMaskOnlyOne m_RegistersWithConstants[2]; +#endif + void clearConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants &= ~getRegMask(reg, regType); + m_RegistersWithConstants[regTypeIndex(regType)] &= ~getRegMask(reg, regType); } void setConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants |= getRegMask(reg, regType); + m_RegistersWithConstants[regTypeIndex(regType)] |= getRegMask(reg, regType); } bool isRegConstant(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); regMaskOnlyOne regMask = getRegMask(reg, regType); - return (m_RegistersWithConstants & regMask) == regMask; + return (m_RegistersWithConstants[regTypeIndex(regType)] & regMask) == regMask; } regMaskOnlyOne getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition); From 15135f835f3955ab079e8662bfb60b581d257224 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 13 Feb 2024 11:51:11 -0800 Subject: [PATCH 058/201] temp: Convert m_AvailableRegs to regMaskOnlyOne --- src/coreclr/jit/clrjit.natvis | 14 +++++++-- src/coreclr/jit/lsra.cpp | 34 +++++++++++----------- src/coreclr/jit/lsra.h | 54 ++++++++++++++++++++++++++++------- src/coreclr/jit/lsraarm64.cpp | 10 +++---- 4 files changed, 78 insertions(+), 34 deletions(-) diff --git a/src/coreclr/jit/clrjit.natvis b/src/coreclr/jit/clrjit.natvis index 450ce2017517e7..9a124a59497dd6 100644 --- a/src/coreclr/jit/clrjit.natvis +++ b/src/coreclr/jit/clrjit.natvis @@ -128,9 +128,9 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u block = block->Next() - this->m_AvailableRegs + this->m_AvailableRegs[0] - + ((regNumber)regIndex),en @@ -138,6 +138,16 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u reg = reg >> 1 + this->m_AvailableRegs[1] + + + + + ((regNumber)regIndex),en + regIndex++ + reg = reg >> 1 + + this->m_RegistersWithConstants[0] diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ddffcd8b1dee0a..fe781c2ee9b468 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2935,7 +2935,7 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, { assert(!enregisterLocalVars); regNumber foundReg; - regMaskMixed foundRegBit; + singleRegMask foundRegBit; RegRecord* availablePhysRegRecord; foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) @@ -2996,7 +2996,7 @@ template regNumber LinearScan::allocateReg(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { - regMaskMixed foundRegBit = + singleRegMask foundRegBit = regSelector->select(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) { @@ -3973,7 +3973,9 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { // For each physical register that can hold a GC type, // if it is occupied by an interval of a GC type, spill that interval. - regMaskMixed candidateRegs = killRefPosition->registerAssignment; + regMaskGpr candidateRegs = killRefPosition->registerAssignment; + assert(compiler->IsGprRegMask(candidateRegs)); + INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { @@ -4026,6 +4028,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) template void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { + //TODO: Save current_ConstantRegs and current_AvailableRegs assert(currentBlock != nullptr); markBlockVisited(currentBlock); @@ -4047,6 +4050,7 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { resetAllRegistersState(); } + // TODO: Restore current_ConstantRegs and current_AvailableRegs } //------------------------------------------------------------------------ @@ -4902,8 +4906,8 @@ void LinearScan::allocateRegistersMinimal() LsraLocation prevLocation = MinLocation; regMaskMixed regsToFree = RBM_NONE; regMaskMixed delayRegsToFree = RBM_NONE; - regMaskMixed regsToMakeInactive = RBM_NONE; - regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskOnlyOne regsToMakeInactive = RBM_NONE; + regMaskOnlyOne delayRegsToMakeInactive = RBM_NONE; regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -4922,7 +4926,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskMixed tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskOnlyOne tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -4935,7 +4939,6 @@ void LinearScan::allocateRegistersMinimal() makeRegsAvailable(regsToMakeInactive); // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. - regsToMakeInactive |= delayRegsToMakeInactive; regsToMakeInactive = delayRegsToMakeInactive; delayRegsToMakeInactive = RBM_NONE; } @@ -5363,7 +5366,7 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -5583,8 +5586,8 @@ void LinearScan::allocateRegisters() LsraLocation prevLocation = MinLocation; regMaskMixed regsToFree = RBM_NONE; regMaskMixed delayRegsToFree = RBM_NONE; - regMaskMixed regsToMakeInactive = RBM_NONE; - regMaskMixed delayRegsToMakeInactive = RBM_NONE; + regMaskOnlyOne regsToMakeInactive = RBM_NONE; + regMaskOnlyOne delayRegsToMakeInactive = RBM_NONE; regMaskMixed copyRegsToFree = RBM_NONE; regsInUseThisLocation = RBM_NONE; regsInUseNextLocation = RBM_NONE; @@ -5603,7 +5606,7 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskMixed tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + regMaskOnlyOne tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); while (tempRegsToMakeInactive != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); @@ -5616,7 +5619,6 @@ void LinearScan::allocateRegisters() makeRegsAvailable(regsToMakeInactive); // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. - regsToMakeInactive |= delayRegsToMakeInactive; regsToMakeInactive = delayRegsToMakeInactive; delayRegsToMakeInactive = RBM_NONE; } @@ -6527,7 +6529,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskMixed regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -13477,7 +13479,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // refpositions. assert((refPosition->refType == RefTypeUpperVectorRestore) || (genCountBits(candidates) == 1)); - freeCandidates = candidates & linearScan->m_AvailableRegs; + freeCandidates = candidates & linearScan->m_AvailableRegs[regTypeIndex(currentInterval->registerType)]; } if ((freeCandidates == RBM_NONE) && (candidates == RBM_NONE)) @@ -13522,7 +13524,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval return RBM_NONE; } - freeCandidates = linearScan->getFreeCandidates(candidates ARM_ARG(regType)); + freeCandidates = linearScan->getFreeCandidates(candidates, regType); } // If no free candidates, then double check if refPosition is an actual ref. @@ -13756,7 +13758,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( return RBM_NONE; } - freeCandidates = linearScan->getFreeCandidates(candidates ARM_ARG(regType)); + freeCandidates = linearScan->getFreeCandidates(candidates, regType); if (freeCandidates != RBM_NONE) { diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 4c1fd8b39dc2bc..96d991775396fb 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -64,6 +64,7 @@ RegisterType regType(T type) } } +// TODO: If there are lot of callers of RegisterType, simplify it. template unsigned regTypeIndex(T type) { @@ -1252,15 +1253,15 @@ class LinearScan : public LinearScanInterface regMaskFloat filterConsecutiveCandidatesForSpill(regMaskFloat consecutiveCandidates, unsigned int registersNeeded); #endif // TARGET_ARM64 - regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates ARM_ARG(var_types regType)) + regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates, var_types regType) { - regMaskOnlyOne result = candidates & m_AvailableRegs; + regMaskOnlyOne result = candidates & m_AvailableRegs[regTypeIndex(regType)]; #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use register for which the odd half is // also available. if (regType == TYP_DOUBLE) { - result &= (m_AvailableRegs >> 1); + result &= (m_AvailableRegs[1] >> 1); } #endif // TARGET_ARM return result; @@ -1751,7 +1752,16 @@ class LinearScan : public LinearScanInterface // Register status //----------------------------------------------------------------------- - regMaskMixed m_AvailableRegs; // TODO: Should be separate for gpr, vector, predicate + //TODO: One option is to also just have another current_AvailableREgs that + // gets reset for every refposition we are processing depending on the + // register type. That wawy we do not have to query and fetch the appropriate + // entry again and agin. +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + regMaskOnlyOne m_AvailableRegs[3]; +#else + regMaskOnlyOne m_AvailableRegs[2]; +#endif + regNumber getRegForType(regNumber reg, var_types regType) { #ifdef TARGET_ARM @@ -1779,7 +1789,11 @@ class LinearScan : public LinearScanInterface void resetAvailableRegs() { - m_AvailableRegs = allAvailableRegs; + m_AvailableRegs[0] = availableIntRegs; + m_AvailableRegs[1] = availableFloatRegs; +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + m_AvailableRegs[2] = availableMaskRegs; +#endif m_RegistersWithConstants[0] = RBM_NONE; m_RegistersWithConstants[1] = RBM_NONE; #if defined(TARGET_XARCH) && defined(FEATURE_SIMD) @@ -1788,28 +1802,42 @@ class LinearScan : public LinearScanInterface } - bool isRegAvailable(regNumber reg, var_types regType) + bool isRegAvailable(regNumber reg, var_types regType) // only used in asserts { regMaskOnlyOne regMask = getRegMask(reg, regType); - return (m_AvailableRegs & regMask) == regMask; + return (m_AvailableRegs[regTypeIndex(regType)] & regMask) == regMask; } void setRegsInUse(regMaskMixed regMask) { - m_AvailableRegs &= ~regMask; + //TODO: Fix this later. + m_AvailableRegs[0] &= ~(regMask & ~RBM_ALLFLOAT); + m_AvailableRegs[1] &= ~(regMask & RBM_ALLFLOAT); +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + m_AvailableRegs[2] &= ~(regMask & RBM_ALLMASK); +#endif } void setRegInUse(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - setRegsInUse(regMask); + m_AvailableRegs[regTypeIndex(regType)] &= ~regMask; } void makeRegsAvailable(regMaskMixed regMask) { - m_AvailableRegs |= regMask; + // TODO: This will be just `regMask` + makeRegAvailable(regMask & ~RBM_ALLFLOAT, IntRegisterType); + makeRegAvailable(regMask & RBM_ALLFLOAT, FloatRegisterType); +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) + makeRegAvailable(regMask & RBM_ALLMASK, MaskRegisterType); +#endif } void makeRegAvailable(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - makeRegsAvailable(regMask); + makeRegAvailable(regMask, regType); + } + void makeRegAvailable(regMaskOnlyOne regMask, var_types regType) + { + m_AvailableRegs[regTypeIndex(regType)] |= regMask; } void clearAllNextIntervalRef(); @@ -1834,6 +1862,10 @@ class LinearScan : public LinearScanInterface void clearConstantReg(regNumber reg, var_types regType) { + //TODO: If we decide to have curr_RegistersWithConstants, then here we will + // just operate on curr_RegistersWithConstants and assert + // assert(m_RegistersWithConstants[regTypeIndex(regType)] == curr_RegistersWithConstants); + // but we will have to make sure that we save it back too?? m_RegistersWithConstants[regTypeIndex(regType)] &= ~getRegMask(reg, regType); } void setConstantReg(regNumber reg, var_types regType) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index c9c2fb9105f0ee..e617bbb62d4202 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -364,7 +364,7 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec } maskForCurRange |= (registersNeededMask << regAvailableStartIndex); - maskForCurRange &= m_AvailableRegs; + maskForCurRange &= m_AvailableRegs[1]; if (maskForCurRange != RBM_NONE) { @@ -420,7 +420,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { assert(compiler->info.compNeedsConsecutiveRegisters); assert(refPosition->isFirstRefPositionOfConsecutiveRegisters()); - regMaskFloat freeCandidates = allCandidates & m_AvailableRegs; + regMaskFloat freeCandidates = allCandidates & m_AvailableRegs[1]; #ifdef DEBUG if (getStressLimitRegs() != LSRA_LIMIT_NONE) @@ -536,7 +536,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, *busyCandidates = consecutiveResultForBusy; // Check if we can further check better registers amoung consecutiveResultForBusy. - if ((m_AvailableRegs & overallResultForBusy) != RBM_NONE) + if ((m_AvailableRegs[1] & overallResultForBusy) != RBM_NONE) { // `overallResultForBusy` contains the mask of entire series that can be the consecutive candidates. // If there is an overlap of that with free registers, then try to find a series that will need least @@ -549,13 +549,13 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { *busyCandidates = optimalConsecutiveResultForBusy; } - else if ((m_AvailableRegs & consecutiveResultForBusy) != RBM_NONE) + else if ((m_AvailableRegs[1] & consecutiveResultForBusy) != RBM_NONE) { // We did not find free consecutive candidates, however we found some registers among the // `allCandidates` that are mix of free and busy. Since `busyCandidates` just has bit set for first // register of such series, return the mask that starts with free register, if possible. The busy // registers will be spilled during assignment of subsequent RefPosition. - *busyCandidates = (m_AvailableRegs & consecutiveResultForBusy); + *busyCandidates = (m_AvailableRegs[1] & consecutiveResultForBusy); } } From 4c6cfdacff8b0dc2b9d2c842c64b259c6a744ccd Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 13 Feb 2024 12:04:41 -0800 Subject: [PATCH 059/201] Add a TODO to convert m_Regs to separate variables --- src/coreclr/jit/lsra.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 96d991775396fb..dbc511c15c2066 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -66,7 +66,7 @@ RegisterType regType(T type) // TODO: If there are lot of callers of RegisterType, simplify it. template -unsigned regTypeIndex(T type) +FORCEINLINE unsigned regTypeIndex(T type) { if (varTypeUsesIntReg(type)) { @@ -1757,7 +1757,7 @@ class LinearScan : public LinearScanInterface // register type. That wawy we do not have to query and fetch the appropriate // entry again and agin. #if defined(TARGET_XARCH) && defined(FEATURE_SIMD) - regMaskOnlyOne m_AvailableRegs[3]; + regMaskOnlyOne m_AvailableRegs[3]; // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. #else regMaskOnlyOne m_AvailableRegs[2]; #endif @@ -1855,7 +1855,7 @@ class LinearScan : public LinearScanInterface DEBUG_ARG(regNumber assignedReg)); #if defined(TARGET_XARCH) && defined(FEATURE_SIMD) - regMaskOnlyOne m_RegistersWithConstants[3]; + regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, m_FloatRegistersWithConstant, etc. #else regMaskOnlyOne m_RegistersWithConstants[2]; #endif From e08561bc53aa7957a448febc5e87548d5d3758c5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 16 Feb 2024 12:10:56 -0800 Subject: [PATCH 060/201] Handle various `regsModified` and `RBM_` to return AllRegsMask --- src/coreclr/inc/patchpointinfo.h | 14 +- src/coreclr/jit/codegen.h | 11 +- src/coreclr/jit/codegenarm.cpp | 51 ++++--- src/coreclr/jit/codegenarm64.cpp | 100 +++++++------ src/coreclr/jit/codegenarmarch.cpp | 49 ++++--- src/coreclr/jit/codegencommon.cpp | 108 +++++++------- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 60 ++++---- src/coreclr/jit/compiler.cpp | 29 ++-- src/coreclr/jit/compiler.h | 9 +- src/coreclr/jit/emit.cpp | 36 ++--- src/coreclr/jit/emit.h | 4 +- src/coreclr/jit/emitarm.cpp | 6 +- src/coreclr/jit/emitarm64.cpp | 6 +- src/coreclr/jit/emitxarch.cpp | 6 +- src/coreclr/jit/emitxarch.h | 4 + src/coreclr/jit/instr.cpp | 36 ++--- src/coreclr/jit/lsra.cpp | 82 ++++++++++- src/coreclr/jit/lsra.h | 45 +++--- src/coreclr/jit/lsraarm.cpp | 11 +- src/coreclr/jit/lsraarm64.cpp | 17 ++- src/coreclr/jit/lsraarmarch.cpp | 4 +- src/coreclr/jit/lsrabuild.cpp | 138 +++++++++++------- src/coreclr/jit/lsraxarch.cpp | 25 ++-- src/coreclr/jit/regset.cpp | 226 +++++++++++++++++++++++++---- src/coreclr/jit/regset.h | 70 +++++++-- src/coreclr/jit/target.h | 122 +++++++++++++++- src/coreclr/jit/targetamd64.h | 23 +-- src/coreclr/jit/targetarm.h | 25 ++-- src/coreclr/jit/targetarm64.h | 22 +-- src/coreclr/jit/targetx86.h | 24 +-- src/coreclr/jit/utils.cpp | 20 ++- src/coreclr/jit/vartype.h | 4 +- 33 files changed, 954 insertions(+), 435 deletions(-) diff --git a/src/coreclr/inc/patchpointinfo.h b/src/coreclr/inc/patchpointinfo.h index 02b9fd89f338ae..27346a3c4aeea3 100644 --- a/src/coreclr/inc/patchpointinfo.h +++ b/src/coreclr/inc/patchpointinfo.h @@ -37,7 +37,7 @@ struct PatchpointInfo // Initialize void Initialize(unsigned localCount, int totalFrameSize) { - m_calleeSaveRegisters = 0; + m_calleeSaveGprRegisters = 0; m_totalFrameSize = totalFrameSize; m_numberOfLocals = localCount; m_genericContextArgOffset = -1; @@ -49,7 +49,7 @@ struct PatchpointInfo // Copy void Copy(const PatchpointInfo* original) { - m_calleeSaveRegisters = original->m_calleeSaveRegisters; + m_calleeSaveGprRegisters = original->m_calleeSaveGprRegisters; m_genericContextArgOffset = original->m_genericContextArgOffset; m_keptAliveThisOffset = original->m_keptAliveThisOffset; m_securityCookieOffset = original->m_securityCookieOffset; @@ -163,14 +163,14 @@ struct PatchpointInfo // Callee save registers saved by the original method. // Includes all saves that must be restored (eg includes pushed RBP on x64). // - uint64_t CalleeSaveRegisters() const + uint64_t CalleeSaveGprRegisters() const { - return m_calleeSaveRegisters; + return m_calleeSaveGprRegisters; } - void SetCalleeSaveRegisters(uint64_t registerMask) + void SetCalleeSaveGprRegisters(uint64_t gprRegs) { - m_calleeSaveRegisters = registerMask; + m_calleeSaveGprRegisters = gprRegs; } private: @@ -180,7 +180,7 @@ struct PatchpointInfo EXPOSURE_MASK = 0x1 }; - uint64_t m_calleeSaveRegisters; + uint64_t m_calleeSaveGprRegisters; unsigned m_numberOfLocals; int m_totalFrameSize; int m_genericContextArgOffset; diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 7f677730bbadc4..6ab4f5daf91b39 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -337,8 +337,8 @@ class CodeGen final : public CodeGenInterface void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); - void genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); - void genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); + void genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); + void genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); @@ -385,7 +385,8 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR) + regMaskGpr fiSaveGprRegs; // Set of GPR registers saved in the funclet prolog (includes LR) + regMaskFloat fiSaveFloatRegs; // Set of Float registers saved in the funclet prolog (includes LR) unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer unsigned fiSpDelta; // Stack pointer delta unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP @@ -401,7 +402,9 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskMixed fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR) + regMaskGpr fiSaveGprRegs; // Set of callee-saved GPR registers saved in the funclet prolog (includes LR) + regMaskFloat fiSaveFloatRegs; // Set of callee-saved float registers saved in the funclet prolog (includes LR) + regMaskFloat fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog (includes LR) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 739ac1b0032cf2..6f1e32f550e178 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1663,7 +1663,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, else { GetEmitter()->emitIns_R_AI(INS_ldr, EA_PTR_DSP_RELOC, callTargetReg, (ssize_t)pAddr); - regSet.verifyRegUsed(callTargetReg); + regSet.verifyGprRegUsed(callTargetReg); } GetEmitter()->emitIns_Call(emitter::EC_INDIR_R, compiler->eeFindHelper(helper), @@ -1685,7 +1685,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, ); } - regSet.verifyRegistersUsed(RBM_CALLEE_TRASH); + regSet.verifyRegistersUsed(AllRegsMask_CALLEE_TRASH); } #ifdef PROFILING_SUPPORTED @@ -1721,7 +1721,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) if (compiler->compProfilerMethHndIndirected) { GetEmitter()->emitIns_R_AI(INS_ldr, EA_PTR_DSP_RELOC, argReg, (ssize_t)compiler->compProfilerMethHnd); - regSet.verifyRegUsed(argReg); + regSet.verifyGprRegUsed(argReg); } else { @@ -1820,7 +1820,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // profiler handle. Therefore, r0 is moved to REG_PROFILER_RETURN_SCRATCH as per contract. GetEmitter()->emitIns_Mov(INS_mov, attr, REG_PROFILER_RET_SCRATCH, REG_R0, /* canSkip */ false); genTransferRegGCState(REG_PROFILER_RET_SCRATCH, REG_R0); - regSet.verifyRegUsed(REG_PROFILER_RET_SCRATCH); + regSet.verifyGprRegUsed(REG_PROFILER_RET_SCRATCH); } if (compiler->compProfilerMethHndIndirected) @@ -1833,7 +1833,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) } gcInfo.gcMarkRegSetNpt(RBM_R0); - regSet.verifyRegUsed(REG_R0); + regSet.verifyGprRegUsed(REG_R0); genEmitHelperCall(helper, 0, // argSize @@ -1844,7 +1844,9 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) { GetEmitter()->emitIns_Mov(INS_mov, attr, REG_R0, REG_PROFILER_RET_SCRATCH, /* canSkip */ false); genTransferRegGCState(REG_R0, REG_PROFILER_RET_SCRATCH); - gcInfo.gcMarkRegSetNpt(RBM_PROFILER_RET_SCRATCH); + assert(compiler->IsGprRegMask(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs)); + assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs == RBM_NONE); + gcInfo.gcMarkRegSetNpt(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs); } } @@ -1930,9 +1932,9 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, genInstrWithConstant(INS_sub, EA_PTRSIZE, REG_STACK_PROBE_HELPER_ARG, REG_SPBASE, frameSize, INS_FLAGS_DONT_CARE, REG_STACK_PROBE_HELPER_ARG); - regSet.verifyRegUsed(REG_STACK_PROBE_HELPER_ARG); + regSet.verifyGprRegUsed(REG_STACK_PROBE_HELPER_ARG); genEmitHelperCall(CORINFO_HELP_STACK_PROBE, 0, EA_UNKNOWN, REG_STACK_PROBE_HELPER_CALL_TARGET); - regSet.verifyRegUsed(REG_STACK_PROBE_HELPER_CALL_TARGET); + regSet.verifyGprRegUsed(REG_STACK_PROBE_HELPER_CALL_TARGET); compiler->unwindPadding(); GetEmitter()->emitIns_Mov(INS_mov, EA_PTRSIZE, REG_SPBASE, REG_STACK_PROBE_HELPER_ARG, /* canSkip */ false); @@ -2176,9 +2178,8 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskMixed maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; - regMaskFloat maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT; - regMaskGpr maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat; + regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedFloatRegsMask(); + regMaskGpr maskPopRegsInt = regSet.rsGetModifiedGprRegsMask(); // First, pop float registers @@ -2337,8 +2338,8 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); - regMaskFloat maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat; + regMaskFloat maskPushRegsFloat = genFuncletInfo.fiSaveFloatRegs; + regMaskGpr maskPushRegsInt = genFuncletInfo.fiSaveGprRegs; regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPushRegsFloat); maskPushRegsInt |= maskStackAlloc; @@ -2391,7 +2392,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) // This is the first block of a filter GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, REG_R1, REG_R1, genFuncletInfo.fiPSP_slot_CallerSP_offset); - regSet.verifyRegUsed(REG_R1); + regSet.verifyGprRegUsed(REG_R1); GetEmitter()->emitIns_R_R_I(INS_str, EA_PTRSIZE, REG_R1, REG_SPBASE, genFuncletInfo.fiPSP_slot_SP_offset); GetEmitter()->emitIns_R_R_I(INS_sub, EA_PTRSIZE, REG_FPBASE, REG_R1, genFuncletInfo.fiFunctionCallerSPtoFPdelta); @@ -2401,7 +2402,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) // This is a non-filter funclet GetEmitter()->emitIns_R_R_I(INS_add, EA_PTRSIZE, REG_R3, REG_FPBASE, genFuncletInfo.fiFunctionCallerSPtoFPdelta); - regSet.verifyRegUsed(REG_R3); + regSet.verifyGprRegUsed(REG_R3); GetEmitter()->emitIns_R_R_I(INS_str, EA_PTRSIZE, REG_R3, REG_SPBASE, genFuncletInfo.fiPSP_slot_SP_offset); } } @@ -2431,10 +2432,10 @@ void CodeGen::genFuncletEpilog() bool unwindStarted = false; /* The saved regs info saves the LR register. We need to pop the PC register to return */ - assert(genFuncletInfo.fiSaveRegs & RBM_LR); + assert(genFuncletInfo.fiSaveGprRegs & RBM_LR); - regMaskFloat maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat; + regMaskFloat maskPopRegsFloat = genFuncletInfo.fiSaveFloatRegs; + regMaskGpr maskPopRegsInt = genFuncletInfo.fiSaveGprRegs; regMaskGpr maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPopRegsFloat); maskPopRegsInt |= maskStackAlloc; @@ -2489,12 +2490,13 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() // of adding the number of callee-saved regs to CallerSP, we add 1 for lr and 1 for r11 // (plus the "pre spill regs"). Note that we assume r12 and r13 aren't saved // (also assumed in genFnProlog()). - assert((regSet.rsMaskCalleeSaved & (RBM_R12 | RBM_R13)) == 0); + assert((regSet.rsGprMaskCalleeSaved & (RBM_R12 | RBM_R13)) == 0); unsigned preSpillRegArgSize = genCountBits(regSet.rsMaskPreSpillRegs(true)) * REGSIZE_BYTES; genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; - regMaskMixed rsMaskSaveRegs = regSet.rsMaskCalleeSaved; - unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); + regMaskMixed rsGprMaskSaveRegs = regSet.rsGprMaskCalleeSaved; + regMaskMixed rsFloatMaskSaveRegs = regSet.rsFloatMaskCalleeSaved; + unsigned saveRegsCount = genCountBits(rsGprMaskSaveRegs) + genCountBits(rsFloatMaskSaveRegs); unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; if (compiler->lvaMonAcquired != BAD_VAR_NUM) @@ -2514,7 +2516,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() /* Now save it for future use */ - genFuncletInfo.fiSaveRegs = rsMaskSaveRegs; + genFuncletInfo.fiSaveGprRegs = rsGprMaskSaveRegs; + genFuncletInfo.fiSaveFloatRegs = rsFloatMaskSaveRegs; genFuncletInfo.fiSpDelta = spDelta; genFuncletInfo.fiPSP_slot_SP_offset = PSP_slot_SP_offset; genFuncletInfo.fiPSP_slot_CallerSP_offset = PSP_slot_CallerSP_offset; @@ -2526,7 +2529,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf("Funclet prolog / epilog info\n"); printf(" Function CallerSP-to-FP delta: %d\n", genFuncletInfo.fiFunctionCallerSPtoFPdelta); printf(" Save regs: "); - dspRegMask(rsMaskSaveRegs); + dspRegMask(rsGprMaskSaveRegs, rsFloatMaskSaveRegs); printf("\n"); printf(" SP delta: %d\n", genFuncletInfo.fiSpDelta); printf(" PSP slot SP offset: %d\n", genFuncletInfo.fiPSP_slot_SP_offset); @@ -2651,7 +2654,7 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu regNumber rCnt = REG_NA; // Invalid regMaskMixed regMask; - regMaskMixed availMask = regSet.rsGetModifiedRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers + regMaskMixed availMask = regSet.rsGetModifiedGprRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers availMask &= ~intRegState.rsCalleeRegArgMaskLiveIn; // Remove all of the incoming argument registers as they are // currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg as we will zero it and maybe use it for diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 0c66f9e52bc956..f9550f9cebd775 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,16 +36,15 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskMixed rsRestoreRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskMixed rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskMixed rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; if (isFramePointerUsed()) { - rsRestoreRegs |= RBM_FPBASE; + rsRestoreGprRegs |= RBM_FPBASE; } - rsRestoreRegs |= RBM_LR; // We must save/restore the return address (in the LR register) - - regMaskMixed regsToRestoreMask = rsRestoreRegs; + rsRestoreGprRegs |= RBM_LR; // We must save/restore the return address (in the LR register) const int totalFrameSize = genTotalFrameSize(); @@ -71,7 +70,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) compiler->unwindSetFrameReg(REG_FPBASE, 0); } - regsToRestoreMask &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and post-index SP. + rsRestoreGprRegs &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and post-index SP. break; } @@ -91,7 +90,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) compiler->unwindSetFrameReg(REG_FPBASE, SPtoFPdelta); } - regsToRestoreMask &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and post-index SP. + rsRestoreGprRegs &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and post-index SP. break; } @@ -104,7 +103,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) JITDUMP(" calleeSaveSpDelta=%d\n", calleeSaveSpDelta); - regsToRestoreMask &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and (hopefully) post-index SP. + rsRestoreGprRegs &= ~(RBM_FP | RBM_LR); // We'll restore FP/LR at the end, and (hopefully) post-index SP. int remainingFrameSz = totalFrameSize - calleeSaveSpDelta; assert(remainingFrameSz > 0); @@ -203,7 +202,8 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) } JITDUMP(" calleeSaveSpOffset=%d, calleeSaveSpDelta=%d\n", calleeSaveSpOffset, calleeSaveSpDelta); - genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, calleeSaveSpOffset, calleeSaveSpDelta); + genRestoreCalleeSavedRegistersHelp(AllRegsMask(rsRestoreGprRegs, rsRestoreFloatRegs), calleeSaveSpOffset, + calleeSaveSpDelta); switch (frameType) { @@ -904,12 +904,18 @@ void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDel // The save set can contain LR in which case LR is saved along with the other callee-saved registers. // But currently Jit doesn't use frames without frame pointer on arm64. // -void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); - unsigned regsToSaveCount = genCountBits(regsToSaveMask); + regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs; + regMaskTP maskSaveRegsInt = regsToSaveMask.gprRegs; + + assert(compiler->IsGprRegMask(maskSaveRegsInt)); + assert(compiler->IsFloatRegMask(maskSaveRegsFloat)); + + unsigned regsToSaveCount = genCountBits(maskSaveRegsFloat) + genCountBits(maskSaveRegsInt); if (regsToSaveCount == 0) { if (spDelta != 0) @@ -927,10 +933,6 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskMixed regsToSaveMask, int l assert(regsToSaveCount <= genCountBits(RBM_CALLEE_SAVED | RBM_FP | RBM_LR)); // Save integer registers at higher addresses than floating-point registers. - - regMaskFloat maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; - if (maskSaveRegsFloat != RBM_NONE) { genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset); @@ -1019,12 +1021,19 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int sp // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, +void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) { assert(spDelta >= 0); - unsigned regsToRestoreCount = genCountBits(regsToRestoreMask); + + regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs; + regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs; + + assert(compiler->IsGprRegMask(maskRestoreRegsInt)); + assert(compiler->IsFloatRegMask(maskRestoreRegsFloat)); + + unsigned regsToRestoreCount = genCountBits(maskRestoreRegsInt) + genCountBits(maskRestoreRegsFloat); if (regsToRestoreCount == 0) { if (spDelta != 0) @@ -1047,11 +1056,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskMixed regsToRestoreMask, // Save integer registers at higher addresses than floating-point registers. - regMaskFloat maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; - regMaskGpr maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; - // Restore in the opposite order of saving. - if (maskRestoreRegsInt != RBM_NONE) { int spIntDelta = (maskRestoreRegsFloat != RBM_NONE) ? 0 : spDelta; // should we delay the SP adjustment? @@ -1372,8 +1377,8 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindBegProlog(); - regMaskFloat maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat; + regMaskFloat maskSaveRegsFloat = genFuncletInfo.fiSaveFloatRegs; + regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveGprRegs; // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskSaveRegsInt & RBM_LR) != 0); @@ -1490,7 +1495,8 @@ void CodeGen::genFuncletProlog(BasicBlock* block) int lowestCalleeSavedOffset = genFuncletInfo.fiSP_to_CalleeSave_delta + genFuncletInfo.fiSpDelta2; // We haven't done the second adjustment of SP yet (if any) - genSaveCalleeSavedRegistersHelp(maskSaveRegsInt | maskSaveRegsFloat, lowestCalleeSavedOffset, 0); + + genSaveCalleeSavedRegistersHelp(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat), lowestCalleeSavedOffset, 0); if ((genFuncletInfo.fiFrameType == 3) || (genFuncletInfo.fiFrameType == 5)) { @@ -1526,7 +1532,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) // function) genInstrWithConstant(INS_ldr, EA_PTRSIZE, REG_R1, REG_R1, genFuncletInfo.fiCallerSP_to_PSP_slot_delta, REG_R2, false); - regSet.verifyRegUsed(REG_R1); + regSet.verifyGprRegUsed(REG_R1); // Store the PSP value (aka CallerSP) genInstrWithConstant(INS_str, EA_PTRSIZE, REG_R1, REG_SPBASE, genFuncletInfo.fiSP_to_PSP_slot_delta, REG_R2, @@ -1543,7 +1549,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) // compute the CallerSP, given the frame pointer. x3 is scratch. genInstrWithConstant(INS_add, EA_PTRSIZE, REG_R3, REG_FPBASE, -genFuncletInfo.fiFunction_CallerSP_to_FP_delta, REG_R2, false); - regSet.verifyRegUsed(REG_R3); + regSet.verifyGprRegUsed(REG_R3); genInstrWithConstant(INS_str, EA_PTRSIZE, REG_R3, REG_SPBASE, genFuncletInfo.fiSP_to_PSP_slot_delta, REG_R2, false); @@ -1576,8 +1582,8 @@ void CodeGen::genFuncletEpilog() unwindStarted = true; } - regMaskFloat maskRestoreRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT; - regMaskGpr maskRestoreRegsInt = genFuncletInfo.fiSaveRegs & ~maskRestoreRegsFloat; + regMaskFloat maskRestoreRegsFloat = genFuncletInfo.fiSaveFloatRegs; + regMaskGpr maskRestoreRegsInt = genFuncletInfo.fiSaveGprRegs; // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskRestoreRegsInt & RBM_LR) != 0); @@ -1603,10 +1609,11 @@ void CodeGen::genFuncletEpilog() regMaskMixed regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; if ((genFuncletInfo.fiFrameType == 1) || (genFuncletInfo.fiFrameType == 2) || (genFuncletInfo.fiFrameType == 3)) { - regsToRestoreMask &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end + maskRestoreRegsInt &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end } int lowestCalleeSavedOffset = genFuncletInfo.fiSP_to_CalleeSave_delta + genFuncletInfo.fiSpDelta2; - genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, lowestCalleeSavedOffset, 0); + genRestoreCalleeSavedRegistersHelp(AllRegsMask(maskRestoreRegsInt, maskRestoreRegsFloat), lowestCalleeSavedOffset, + 0); if (genFuncletInfo.fiFrameType == 1) { @@ -1737,11 +1744,18 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiFunction_CallerSP_to_FP_delta = genCallerSPtoFPdelta() - osrPad; - regMaskMixed rsMaskSaveRegs = regSet.rsMaskCalleeSaved; - assert((rsMaskSaveRegs & RBM_LR) != 0); - assert((rsMaskSaveRegs & RBM_FP) != 0); + regMaskGpr rsMaskSaveGprRegs = regSet.rsGprMaskCalleeSaved; + regMaskFloat rsMaskSaveFloatRegs = regSet.rsFloatMaskCalleeSaved; + regMaskPredicate rsMaskSavePredicateRegs = RBM_NONE; +#ifdef HAS_PREDICATE_REGS + rsMaskSavePredicateRegs = regSet.rsPredicateMaskCalleeSaved; +#endif + + assert((rsMaskSaveGprRegs & RBM_LR) != 0); + assert((rsMaskSaveGprRegs & RBM_FP) != 0); - unsigned saveRegsCount = genCountBits(rsMaskSaveRegs); + unsigned saveRegsCount = + genCountBits(rsMaskSaveGprRegs) + genCountBits(rsMaskSaveFloatRegs) + genCountBits(rsMaskSavePredicateRegs); unsigned saveRegsPlusPSPSize = saveRegsCount * REGSIZE_BYTES + PSPSize; if (compiler->info.compIsVarArgs) { @@ -1860,7 +1874,9 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() /* Now save it for future use */ - genFuncletInfo.fiSaveRegs = rsMaskSaveRegs; + genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; + genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; + genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; genFuncletInfo.fiSP_to_FPLR_save_delta = SP_to_FPLR_save_delta; genFuncletInfo.fiSP_to_PSP_slot_delta = SP_to_PSP_slot_delta; genFuncletInfo.fiSP_to_CalleeSave_delta = SP_to_PSP_slot_delta + PSPSize; @@ -1872,7 +1888,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf("\n"); printf("Funclet prolog / epilog info\n"); printf(" Save regs: "); - dspRegMask(genFuncletInfo.fiSaveRegs); + dspRegMask( + AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs, genFuncletInfo.fiSavePredicateRegs)); printf("\n"); if (compiler->opts.IsOSR()) { @@ -5104,10 +5121,10 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, } regMaskGpr callTargetMask = genRegMask(callTargetReg); - regMaskMixed callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet - noway_assert((callTargetMask & callKillSet) == callTargetMask); + noway_assert((callTargetMask & callKillSet.gprRegs) == callTargetMask); callTarget = callTargetReg; @@ -5126,7 +5143,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, false /* isJump */ ); - regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); } @@ -5426,6 +5443,7 @@ void CodeGen::genStoreLclTypeSimd12(GenTreeLclVarCommon* treeNode) void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) { assert(compiler->compGeneratingProlog); + assert(genIsValidIntReg(initReg)); if (!compiler->compIsProfilerHookNeeded()) { @@ -5449,7 +5467,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) genEmitHelperCall(CORINFO_HELP_PROF_FCN_ENTER, 0, EA_UNKNOWN); - if ((genRegMask(initReg) & RBM_PROFILER_ENTER_TRASH) != RBM_NONE) + if ((genRegMask(initReg) & AllRegsMask_PROFILER_ENTER_TRASH.gprRegs) != RBM_NONE) { *pInitRegZeroed = false; } @@ -5611,7 +5629,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, // until this is complete since the tickles could cause a stack overflow, and we need to be able to crawl // the stack afterward (which means the stack pointer needs to be known). - regMaskGpr availMask = RBM_ALLINT & (regSet.rsGetModifiedRegsMask() | ~RBM_INT_CALLEE_SAVED); + regMaskGpr availMask = regSet.rsGetModifiedGprRegsMask() | ~RBM_INT_CALLEE_SAVED; availMask &= ~maskArgRegsLiveIn; // Remove all of the incoming argument registers as they are currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index db01f11d7f719a..94832804dfbcb5 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3433,15 +3433,16 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskMixed killMask = RBM_CALLEE_TRASH; + AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); killMask = compiler->compHelperCallKillSet(helpFunc); } - assert((gcInfo.gcRegGCrefSetCur & killMask) == 0); - assert((gcInfo.gcRegByrefSetCur & killMask) == 0); + assert((gcInfo.gcRegGCrefSetCur & killMask.gprRegs) == 0); + assert((gcInfo.gcRegByrefSetCur & killMask.gprRegs) == 0); #endif var_types returnType = call->TypeGet(); @@ -4898,7 +4899,8 @@ void CodeGen::genPushCalleeSavedRegisters() intRegState.rsCalleeRegArgMaskLiveIn); #endif - regMaskMixed rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + regMaskGpr rsPushGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskFloat rsPushFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -4909,7 +4911,7 @@ void CodeGen::genPushCalleeSavedRegisters() // On ARM we push the FP (frame-pointer) here along with all other callee saved registers if (isFramePointerUsed()) - rsPushRegs |= RBM_FPBASE; + rsPushGprRegs |= RBM_FPBASE; // // It may be possible to skip pushing/popping lr for leaf methods. However, such optimization would require @@ -4930,24 +4932,26 @@ void CodeGen::genPushCalleeSavedRegisters() // Given the limited benefit from this optimization (<10k for CoreLib NGen image), the extra complexity // is not worth it. // - rsPushRegs |= RBM_LR; // We must save the return address (in the LR register) + rsPushGprRegs |= RBM_LR; // We must save the return address (in the LR register) - regSet.rsMaskCalleeSaved = rsPushRegs; + regSet.rsGprMaskCalleeSaved = rsPushGprRegs; + regSet.rsFloatMaskCalleeSaved = rsPushFloatRegs; #ifdef DEBUG - if (compiler->compCalleeRegsPushed != genCountBits(rsPushRegs)) + unsigned pushRegsCnt = genCountBits(rsPushGprRegs) + genCountBits(rsPushFloatRegs); + if (compiler->compCalleeRegsPushed != pushRegsCnt) { printf("Error: unexpected number of callee-saved registers to push. Expected: %d. Got: %d ", - compiler->compCalleeRegsPushed, genCountBits(rsPushRegs)); - dspRegMask(rsPushRegs); + compiler->compCalleeRegsPushed, pushRegsCnt); + dspRegMask(rsPushGprRegs, rsPushFloatRegs); printf("\n"); - assert(compiler->compCalleeRegsPushed == genCountBits(rsPushRegs)); + assert(compiler->compCalleeRegsPushed == pushRegsCnt); } #endif // DEBUG #if defined(TARGET_ARM) - regMaskFloat maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat; + regMaskFloat maskPushRegsFloat = rsPushFloatRegs; + regMaskGpr maskPushRegsInt = rsPushGprRegs; maskPushRegsInt |= genStackAllocRegisterMask(compiler->compLclFrameSize, maskPushRegsFloat); @@ -5053,17 +5057,13 @@ void CodeGen::genPushCalleeSavedRegisters() int offset; // This will be the starting place for saving the callee-saved registers, in increasing order. - regMaskFloat maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT; - regMaskGpr maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat; + regMaskFloat maskSaveRegsFloat = rsPushFloatRegs; + regMaskGpr maskSaveRegsInt = rsPushGprRegs; #ifdef DEBUG if (verbose) { - printf("Save float regs: "); - dspRegMask(maskSaveRegsFloat); - printf("\n"); - printf("Save int regs: "); - dspRegMask(maskSaveRegsInt); + dspRegMask(maskSaveRegsInt, maskSaveRegsFloat); printf("\n"); } #endif // DEBUG @@ -5308,7 +5308,8 @@ void CodeGen::genPushCalleeSavedRegisters() const int calleeSaveSpOffset = offset; JITDUMP(" offset=%d, calleeSaveSpDelta=%d\n", offset, calleeSaveSpDelta); - genSaveCalleeSavedRegistersHelp(maskSaveRegsInt | maskSaveRegsFloat, offset, -calleeSaveSpDelta); + + genSaveCalleeSavedRegistersHelp(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat), offset, -calleeSaveSpDelta); offset += genCountBits(maskSaveRegsInt | maskSaveRegsFloat) * REGSIZE_BYTES; @@ -5546,7 +5547,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) } if (jmpEpilog || - genStackAllocRegisterMask(compiler->compLclFrameSize, regSet.rsGetModifiedRegsMask() & RBM_FLT_CALLEE_SAVED) == + genStackAllocRegisterMask(compiler->compLclFrameSize, regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) { genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); @@ -5667,7 +5668,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) if (addrInfo.accessType == IAT_PVALUE) { GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, indCallReg, indCallReg, 0); - regSet.verifyRegUsed(indCallReg); + regSet.verifyGprRegUsed(indCallReg); } break; @@ -5681,7 +5682,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) indCallReg = REG_R12; addr = NULL; - regSet.verifyRegUsed(indCallReg); + regSet.verifyGprRegUsed(indCallReg); break; } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7a23df0407f9aa..7f692ed038e34b 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -623,25 +623,25 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // Return Value: // Mask of register kills -- registers whose values are no longer guaranteed to be the same. // -regMaskMixed Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) +AllRegsMask Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) { switch (helper) { case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: - return RBM_CALLEE_TRASH_WRITEBARRIER; + return AllRegsMask_CALLEE_TRASH_WRITEBARRIER; case CORINFO_HELP_ASSIGN_BYREF: - return RBM_CALLEE_TRASH_WRITEBARRIER_BYREF; + return AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF; case CORINFO_HELP_PROF_FCN_ENTER: - return RBM_PROFILER_ENTER_TRASH; + return AllRegsMask_PROFILER_ENTER_TRASH; case CORINFO_HELP_PROF_FCN_LEAVE: - return RBM_PROFILER_LEAVE_TRASH; + return AllRegsMask_PROFILER_LEAVE_TRASH; case CORINFO_HELP_PROF_FCN_TAILCALL: - return RBM_PROFILER_TAILCALL_TRASH; + return AllRegsMask_PROFILER_TAILCALL_TRASH; #ifdef TARGET_X86 case CORINFO_HELP_ASSIGN_REF_EAX: @@ -657,20 +657,20 @@ regMaskMixed Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) case CORINFO_HELP_CHECKED_ASSIGN_REF_EBP: case CORINFO_HELP_CHECKED_ASSIGN_REF_ESI: case CORINFO_HELP_CHECKED_ASSIGN_REF_EDI: - return RBM_EDX; + return GprRegsMask(RBM_EDX); #endif case CORINFO_HELP_STOP_FOR_GC: - return RBM_STOP_FOR_GC_TRASH; + return AllRegsMask_STOP_FOR_GC_TRASH; case CORINFO_HELP_INIT_PINVOKE_FRAME: - return RBM_INIT_PINVOKE_FRAME_TRASH; + return AllRegsMask_INIT_PINVOKE_FRAME_TRASH; case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - return RBM_VALIDATE_INDIRECT_CALL_TRASH; + return AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; default: - return RBM_CALLEE_TRASH; + return AllRegsMask_CALLEE_TRASH; } } @@ -3811,7 +3811,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, srcRegNum, /* canSkip */ false); - regSet.verifyRegUsed(destRegNum); + regSet.verifyRegUsed(destRegNum, destMemType); /* mark 'src' as processed */ noway_assert(srcReg < argMax); @@ -3862,7 +3862,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, xtraReg, /* canSkip */ false); - regSet.verifyRegUsed(destRegNum); + regSet.verifyRegUsed(destRegNum, destMemType); /* mark the beginning register as processed */ regArgTab[srcReg].processed = true; @@ -4312,7 +4312,7 @@ void CodeGen::genEnregisterIncomingStackArgs() GetEmitter()->emitIns_R_S(ins_Load(regType), emitTypeSize(regType), regNum, varNum, 0); #endif // !TARGET_LOONGARCH64 - regSet.verifyRegUsed(regNum); + regSet.verifyRegUsed(regNum, regType); } } @@ -4540,11 +4540,11 @@ void CodeGen::genCheckUseBlockInit() // int forceSpillRegCount = genCountBits(maskCalleeRegArgMask & ~regSet.rsMaskPreSpillRegs(false)) - 1; if (forceSpillRegCount > 0) - regSet.rsSetRegsModified(RBM_R4); + regSet.rsSetGprRegsModified(RBM_R4); if (forceSpillRegCount > 1) - regSet.rsSetRegsModified(RBM_R5); + regSet.rsSetGprRegsModified(RBM_R5); if (forceSpillRegCount > 2) - regSet.rsSetRegsModified(RBM_R6); + regSet.rsSetGprRegsModified(RBM_R6); #endif // TARGET_ARM } } @@ -5326,7 +5326,7 @@ void CodeGen::genFinalizeFrame() // registers (ebx, esi, edi). So, we need to make sure all the callee-saved registers // actually get saved. - regSet.rsSetRegsModified(RBM_INT_CALLEE_SAVED); + regSet.rsSetGprRegsModified(RBM_INT_CALLEE_SAVED); } #endif // TARGET_X86 @@ -5334,14 +5334,14 @@ void CodeGen::genFinalizeFrame() // Make sure that callee-saved registers used by call to a stack probing helper generated are pushed on stack. if (compiler->compLclFrameSize >= compiler->eeGetPageSize()) { - regSet.rsSetRegsModified(RBM_STACK_PROBE_HELPER_ARG | RBM_STACK_PROBE_HELPER_CALL_TARGET | + regSet.rsSetGprRegsModified(RBM_STACK_PROBE_HELPER_ARG | RBM_STACK_PROBE_HELPER_CALL_TARGET | RBM_STACK_PROBE_HELPER_TRASH); } // If there are any reserved registers, add them to the modified set. if (regSet.rsMaskResvd != RBM_NONE) { - regSet.rsSetRegsModified(regSet.rsMaskResvd); + regSet.rsSetGprRegsModified(regSet.rsMaskResvd); } #endif // TARGET_ARM @@ -5349,7 +5349,11 @@ void CodeGen::genFinalizeFrame() if (verbose) { printf("Modified regs: "); - dspRegMask(regSet.rsGetModifiedRegsMask()); + dspRegMask(AllRegsMask(regSet.rsGetModifiedGprRegsMask(), regSet.rsGetModifiedFloatRegsMask() +#ifdef HAS_PREDICATE_REGS + ,regSet.rsGetModifiedPredicateRegsMask() +#endif + )); printf("\n"); } #endif // DEBUG @@ -5360,15 +5364,19 @@ void CodeGen::genFinalizeFrame() // We always save FP. noway_assert(isFramePointerUsed()); #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - regMaskMixed okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED); if (RBM_ENC_CALLEE_SAVED != 0) { - regSet.rsSetRegsModified(RBM_ENC_CALLEE_SAVED); + regSet.rsSetGprRegsModified(RBM_ENC_CALLEE_SAVED); } - noway_assert((regSet.rsGetModifiedRegsMask() & ~okRegs) == 0); + noway_assert( + (regSet.rsGetModifiedGprRegsMask() & ~(RBM_INT_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED)) == 0); + noway_assert((regSet.rsGetModifiedFloatRegsMask() & ~RBM_FLT_CALLEE_TRASH) == 0); +#ifdef TARGET_AMD64 + noway_assert((regSet.rsGetModifiedPredicateRegsMask() & ~RBM_MSK_CALLEE_TRASH) == 0); +#endif // TARGET_AMD64 #else // !TARGET_AMD64 && !TARGET_ARM64 // On x86 we save all callee saved regs so the saved reg area size is consistent - regSet.rsSetRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); + regSet.rsSetGprRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); #endif // !TARGET_AMD64 && !TARGET_ARM64 } @@ -5376,14 +5384,14 @@ void CodeGen::genFinalizeFrame() if (compiler->compMethodRequiresPInvokeFrame()) { noway_assert(isFramePointerUsed()); // Setup of Pinvoke frame currently requires an EBP style frame - regSet.rsSetRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); + regSet.rsSetGprRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); } #ifdef UNIX_AMD64_ABI // On Unix x64 we also save R14 and R15 for ELT profiler hook generation. if (compiler->compIsProfilerHookNeeded()) { - regSet.rsSetRegsModified(RBM_PROFILER_ENTER_ARG_0 | RBM_PROFILER_ENTER_ARG_1); + regSet.rsSetGprRegsModified(RBM_PROFILER_ENTER_ARG_0 | RBM_PROFILER_ENTER_ARG_1); } #endif @@ -5397,14 +5405,19 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - regMaskMixed maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; + //regMaskMixed maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; +#ifdef TARGET_ARM + // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? +#endif + regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; #ifdef TARGET_ARMARCH if (isFramePointerUsed()) { // For a FP based frame we have to push/pop the FP register // - maskCalleeRegsPushed |= RBM_FPBASE; + maskPushRegsInt |= RBM_FPBASE; // This assert check that we are not using REG_FP // as both the frame pointer and as a codegen register @@ -5414,15 +5427,11 @@ void CodeGen::genFinalizeFrame() // we always push LR. See genPushCalleeSavedRegisters // - maskCalleeRegsPushed |= RBM_LR; + maskPushRegsInt |= RBM_LR; #if defined(TARGET_ARM) - // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? - regMaskFloat maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT; - regMaskGpr maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat; - if ((maskPushRegsFloat != RBM_NONE) || - (compiler->opts.MinOpts() && (regSet.rsMaskResvd & maskCalleeRegsPushed & RBM_OPT_RSVD))) + (compiler->opts.MinOpts() && (regSet.rsMaskResvd & maskPushRegsInt & RBM_OPT_RSVD))) { // Here we try to keep stack double-aligned before the vpush if ((genCountBits(regSet.rsMaskPreSpillRegs(true) | maskPushRegsInt) % 2) != 0) @@ -5435,10 +5444,9 @@ void CodeGen::genFinalizeFrame() if (extraPushedReg < REG_R11) { maskPushRegsInt |= genRegMask(extraPushedReg); - regSet.rsSetRegsModified(genRegMask(extraPushedReg)); + regSet.rsSetGprRegsModified(genRegMask(extraPushedReg)); } } - maskCalleeRegsPushed = maskPushRegsInt | maskPushRegsFloat; } // We currently only expect to push/pop consecutive FP registers @@ -5457,8 +5465,8 @@ void CodeGen::genFinalizeFrame() { regMaskFloat maskExtraRegs = contiguousMask - maskPushRegsFloat; maskPushRegsFloat |= maskExtraRegs; - regSet.rsSetRegsModified(maskExtraRegs); - maskCalleeRegsPushed |= maskExtraRegs; + regSet.rsSetFloatRegsModified(maskExtraRegs); + maskPushRegsFloat |= maskExtraRegs; } } #endif // TARGET_ARM @@ -5468,8 +5476,8 @@ void CodeGen::genFinalizeFrame() // Compute the count of callee saved float regs saved on stack. // On Amd64 we push only integer regs. Callee saved float (xmm6-xmm31) // regs are stack allocated and preserved in their stack locations. - compiler->compCalleeFPRegsSavedMask = maskCalleeRegsPushed & RBM_FLT_CALLEE_SAVED; - maskCalleeRegsPushed &= ~RBM_FLT_CALLEE_SAVED; + compiler->compCalleeFPRegsSavedMask = maskPushRegsFloat & RBM_FLT_CALLEE_SAVED; + maskPushRegsFloat &= ~RBM_FLT_CALLEE_SAVED; #endif // defined(TARGET_XARCH) #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) @@ -5477,7 +5485,7 @@ void CodeGen::genFinalizeFrame() { // For a FP based frame we have to push/pop the FP register // - maskCalleeRegsPushed |= RBM_FPBASE; + maskPushRegsInt |= RBM_FPBASE; // This assert check that we are not using REG_FP // as both the frame pointer and as a codegen register @@ -5486,16 +5494,16 @@ void CodeGen::genFinalizeFrame() } // we always push RA. See genPushCalleeSavedRegisters - maskCalleeRegsPushed |= RBM_RA; + maskPushRegsInt |= RBM_RA; #endif // TARGET_LOONGARCH64 || TARGET_RISCV64 - compiler->compCalleeRegsPushed = genCountBits(maskCalleeRegsPushed); + compiler->compCalleeRegsPushed = genCountBits(maskPushRegsInt) + genCountBits(maskPushRegsFloat); #ifdef DEBUG if (verbose) { printf("Callee-saved registers pushed: %d ", compiler->compCalleeRegsPushed); - dspRegMask(maskCalleeRegsPushed); + dspRegMask(maskPushRegsInt, maskPushRegsFloat); //TODO: Should this also have maskPushRegsPredicate printf("\n"); } #endif // DEBUG @@ -5835,7 +5843,7 @@ void CodeGen::genFnProlog() // If initReg is ever set to zero, this variable is set to true and zero initializing initReg // will be skipped. bool initRegZeroed = false; - regMaskOnlyOne excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; + regMaskGpr excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; regMaskGpr tempMask; // We should not use the special PINVOKE registers as the initReg @@ -5889,7 +5897,7 @@ void CodeGen::genFnProlog() // If they aren't available we use one of the caller-saved integer registers. else { - tempMask = regSet.rsGetModifiedRegsMask() & RBM_ALLINT & ~excludeMask & ~regSet.rsMaskResvd; + tempMask = regSet.rsGetModifiedGprRegsMask() & RBM_ALLINT & ~excludeMask & ~regSet.rsMaskResvd; if (tempMask != RBM_NONE) { // We pick the lowest register number @@ -6077,7 +6085,7 @@ void CodeGen::genFnProlog() #ifdef TARGET_ARM maskStackAlloc = genStackAllocRegisterMask(compiler->compLclFrameSize + extraFrameSize, - regSet.rsGetModifiedRegsMask() & RBM_FLT_CALLEE_SAVED); + regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED); #endif // TARGET_ARM if (maskStackAlloc == RBM_NONE) @@ -6102,7 +6110,7 @@ void CodeGen::genFnProlog() if (compiler->compLocallocUsed) { GetEmitter()->emitIns_Mov(INS_mov, EA_4BYTE, REG_SAVED_LOCALLOC_SP, REG_SPBASE, /* canSkip */ false); - regSet.verifyRegUsed(REG_SAVED_LOCALLOC_SP); + regSet.verifyGprRegUsed(REG_SAVED_LOCALLOC_SP); compiler->unwindSetFrameReg(REG_SAVED_LOCALLOC_SP, 0); } #endif // TARGET_ARMARCH @@ -6413,7 +6421,7 @@ void CodeGen::genFnProlog() // MOV EAX, GetEmitter()->emitIns_R_S(ins_Load(TYP_I_IMPL), EA_PTRSIZE, REG_EAX, compiler->info.compArgsCount - 1, 0); - regSet.verifyRegUsed(REG_EAX); + regSet.verifyGprRegUsed(REG_EAX); // MOV EAX, [EAX] GetEmitter()->emitIns_R_AR(ins_Load(TYP_I_IMPL), EA_PTRSIZE, REG_EAX, REG_EAX, 0); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 11a33cef744cc5..5d3fa9234ddb33 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -60,7 +60,7 @@ void CodeGen::genInitializeRegisterState() regNumber reg = varDsc->GetRegNum(); if (genIsValidIntReg(reg)) { - regSet.verifyRegUsed(reg); + regSet.verifyGprRegUsed(reg); } } } diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ef90563bb1471c..c654a4c4a49c3d 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -75,7 +75,7 @@ void CodeGen::genSetGSSecurityCookie(regNumber initReg, bool* pInitRegZeroed) // mov eax, dword ptr [compiler->gsGlobalSecurityCookieAddr] // mov dword ptr [frame.GSSecurityCookie], eax GetEmitter()->emitIns_R_AI(INS_mov, EA_PTR_DSP_RELOC, REG_EAX, (ssize_t)compiler->gsGlobalSecurityCookieAddr); - regSet.verifyRegUsed(REG_EAX); + regSet.verifyGprRegUsed(REG_EAX); GetEmitter()->emitIns_S_R(INS_mov, EA_PTRSIZE, REG_EAX, compiler->lvaGSSecurityCookie, 0); if (initReg == REG_EAX) { @@ -1142,7 +1142,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) { var_types op1Type = op1->TypeGet(); inst_Mov(op1Type, targetReg, op1reg, /* canSkip */ false); - regSet.verifyRegUsed(targetReg); + regSet.verifyRegUsed(targetReg, targetType); gcInfo.gcMarkRegPtrVal(targetReg, op1Type); dst = treeNode; src = op2; @@ -2422,7 +2422,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, } GetEmitter()->emitIns_R_AR(INS_lea, EA_PTRSIZE, REG_STACK_PROBE_HELPER_ARG, REG_SPBASE, spOffset); - regSet.verifyRegUsed(REG_STACK_PROBE_HELPER_ARG); + regSet.verifyGprRegUsed(REG_STACK_PROBE_HELPER_ARG); genEmitHelperCall(CORINFO_HELP_STACK_PROBE, 0, EA_UNKNOWN); @@ -2440,7 +2440,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, RBM_NONE); GetEmitter()->emitIns_R_AR(INS_lea, EA_PTRSIZE, REG_STACK_PROBE_HELPER_ARG, REG_SPBASE, -(int)frameSize); - regSet.verifyRegUsed(REG_STACK_PROBE_HELPER_ARG); + regSet.verifyGprRegUsed(REG_STACK_PROBE_HELPER_ARG); genEmitHelperCall(CORINFO_HELP_STACK_PROBE, 0, EA_UNKNOWN); @@ -6099,15 +6099,16 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - regMaskMixed killMask = RBM_CALLEE_TRASH; + AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + if (call->IsHelperCall()) { CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd); killMask = compiler->compHelperCallKillSet(helpFunc); } - assert((gcInfo.gcRegGCrefSetCur & killMask) == 0); - assert((gcInfo.gcRegByrefSetCur & killMask) == 0); + assert((gcInfo.gcRegGCrefSetCur & killMask.gprRegs) == 0); + assert((gcInfo.gcRegByrefSetCur & killMask.gprRegs) == 0); #endif var_types returnType = call->TypeGet(); @@ -9147,7 +9148,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); regNumber callTarget = REG_NA; - regMaskMixed killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { @@ -9178,7 +9179,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, // this is only a valid assumption if the helper call is known to kill REG_DEFAULT_HELPER_CALL_TARGET. callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; regMaskGpr callTargetMask = genRegMask(callTargetReg); - noway_assert((callTargetMask & killMask) == callTargetMask); + noway_assert((callTargetMask & killMask.gprRegs) == callTargetMask); } else { @@ -9791,12 +9792,11 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // Emit appropriate unwind. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); - regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; int const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; JITDUMP("--OSR--- tier0 has already saved "); - JITDUMPEXEC(dspRegMask(tier0IntCalleeSaves)); + JITDUMPEXEC(dspRegMask(tier0IntCalleeSaves, RBM_NONE)); JITDUMP("\n"); // We must account for the Tier0 callee saves. @@ -9859,7 +9859,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not // here. - regMaskGpr rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr rsPushRegs = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -9871,18 +9871,17 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // Figure out which set of int callee saves still needs saving. // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); - regMaskGpr tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; regMaskGpr const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr osrAdditionalIntCalleeSaves = osrIntCalleeSaves & ~tier0IntCalleeSaves; JITDUMP("---OSR--- int callee saves are "); - JITDUMPEXEC(dspRegMask(osrIntCalleeSaves)); + JITDUMPEXEC(dspRegMask(osrIntCalleeSaves, RBM_NONE)); JITDUMP("; tier0 already saved "); - JITDUMPEXEC(dspRegMask(tier0IntCalleeSaves)); + JITDUMPEXEC(dspRegMask(tier0IntCalleeSaves, RBM_NONE)); JITDUMP("; so only saving "); - JITDUMPEXEC(dspRegMask(osrAdditionalIntCalleeSaves)); + JITDUMPEXEC(dspRegMask(osrAdditionalIntCalleeSaves, RBM_NONE)); JITDUMP("\n"); // These remaining callee saves will be stored in the Tier0 callee save area @@ -9898,7 +9897,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // The tier0 frame is always an RBP frame, so the OSR method should never need to save RBP. // - assert((tier0CalleeSaves & RBM_FPBASE) == RBM_FPBASE); + assert((tier0IntCalleeSaves & RBM_FPBASE) == RBM_FPBASE); assert((osrAdditionalIntCalleeSaves & RBM_FPBASE) == RBM_NONE); // The OSR method must use MOVs to save additional callee saves. @@ -9940,7 +9939,7 @@ void CodeGen::genPushCalleeSavedRegisters() // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not // here. - regMaskGpr rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPushRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -9960,7 +9959,7 @@ void CodeGen::genPushCalleeSavedRegisters() { printf("Error: unexpected number of callee-saved registers to push. Expected: %d. Got: %d ", compiler->compCalleeRegsPushed, genCountBits(rsPushRegs)); - dspRegMask(rsPushRegs); + dspRegMask(rsPushRegs, RBM_NONE); printf("\n"); assert(compiler->compCalleeRegsPushed == genCountBits(rsPushRegs)); } @@ -9998,9 +9997,9 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // if (doesSupersetOfNormalPops) { - regMaskGpr rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr rsPopRegs = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr tier0CalleeSaves = - ((regMaskGpr)compiler->info.compPatchpointInfo->CalleeSaveRegisters()) & RBM_OSR_INT_CALLEE_SAVED; + ((regMaskGpr)compiler->info.compPatchpointInfo->CalleeSaveGprRegisters()) & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr additionalCalleeSaves = rsPopRegs & ~tier0CalleeSaves; // Registers saved by the OSR prolog. @@ -10018,7 +10017,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // Registers saved by a normal prolog // - regMaskGpr rsPopRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPopRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; const unsigned popCount = genPopCalleeSavedRegistersFromMask(rsPopRegs); noway_assert(compiler->compCalleeRegsPushed == popCount); } @@ -10205,9 +10204,8 @@ void CodeGen::genFnEpilog(BasicBlock* block) // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskMixed const tier0CalleeSaves = (regMaskMixed)patchpointInfo->CalleeSaveRegisters(); - regMaskGpr const tier0IntCalleeSaves = tier0CalleeSaves & RBM_OSR_INT_CALLEE_SAVED; - regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; @@ -10234,7 +10232,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) if ((frameSize == TARGET_POINTER_SIZE) && !compiler->compJmpOpUsed) { inst_RV(INS_pop, REG_ECX, TYP_I_IMPL); - regSet.verifyRegUsed(REG_ECX); + regSet.verifyGprRegUsed(REG_ECX); } else #endif // TARGET_X86 @@ -10322,7 +10320,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) { // "pop ecx" will make ESP point to the callee-saved registers inst_RV(INS_pop, REG_ECX, TYP_I_IMPL); - regSet.verifyRegUsed(REG_ECX); + regSet.verifyGprRegUsed(REG_ECX); } #endif // TARGET_X86 else @@ -10461,7 +10459,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) indCallReg = REG_RAX; addr = nullptr; instGen_Set_Reg_To_Imm(EA_HANDLE_CNS_RELOC, indCallReg, (ssize_t)addrInfo.addr); - regSet.verifyRegUsed(indCallReg); + regSet.verifyGprRegUsed(indCallReg); } } else @@ -10678,7 +10676,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block) GetEmitter()->emitIns_R_AR(INS_mov, EA_PTRSIZE, REG_FPBASE, REG_ARG_0, genFuncletInfo.fiPSP_slot_InitialSP_offset); - regSet.verifyRegUsed(REG_FPBASE); + regSet.verifyGprRegUsed(REG_FPBASE); GetEmitter()->emitIns_AR_R(INS_mov, EA_PTRSIZE, REG_FPBASE, REG_SPBASE, genFuncletInfo.fiPSP_slot_InitialSP_offset); diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index da2f67059a4317..0f9f6baf7291d3 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3624,40 +3624,45 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) return rejectThisPromo; } -//------------------------------------------------------------------------ +void Compiler::dumpRegMask(regMaskOnlyOne mask) const +{ + + dumpRegMask(AllRegsMask(mask, mask, mask)); +} + //------------------------------------------------------------------------ // dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of // a potentially large number of registers. // // Arguments: // regs - The set of registers to display // -void Compiler::dumpRegMask(regMaskMixed regs) const +void Compiler::dumpRegMask(AllRegsMask mask) const { - if (regs == RBM_ALLINT) + if (mask.gprRegs == RBM_ALLINT) { printf("[allInt]"); } - else if (regs == (RBM_ALLINT & ~RBM_FPBASE)) + else if (mask.gprRegs == (RBM_ALLINT & ~RBM_FPBASE)) { printf("[allIntButFP]"); } - else if (regs == RBM_ALLFLOAT) + else if (mask.floatRegs == RBM_ALLFLOAT) { printf("[allFloat]"); } - else if (regs == RBM_ALLDOUBLE) + else if (mask.floatRegs == RBM_ALLDOUBLE) { printf("[allDouble]"); } #ifdef TARGET_XARCH - else if (regs == RBM_ALLMASK) + else if (mask.predicateRegs == RBM_ALLMASK) { printf("[allMask]"); } #endif // TARGET_XARCH else { - dspRegMask(regs); + dspRegMask(mask); } } @@ -5773,11 +5778,11 @@ void Compiler::generatePatchpointInfo() // Record callee save registers. // Currently only needed for x64. // - regMaskMixed rsPushRegs = codeGen->regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; - rsPushRegs |= RBM_FPBASE; - patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs); + regMaskGpr rsPushGprRegs = codeGen->regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + rsPushGprRegs |= RBM_FPBASE; + patchpointInfo->SetCalleeSaveGprRegisters(rsPushGprRegs); JITDUMP("--OSR-- Tier0 callee saves: "); - JITDUMPEXEC(dspRegMask((regMaskMixed)patchpointInfo->CalleeSaveRegisters())); + JITDUMPEXEC(dspRegMask(patchpointInfo->CalleeSaveGprRegisters(), RBM_NONE)); JITDUMP("\n"); #endif diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 56d4fe018b69dd..d26348874fa71e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8422,7 +8422,7 @@ class Compiler // Gets a register mask that represent the kill set for a helper call since // not all JIT Helper calls follow the standard ABI on the target architecture. - regMaskMixed compHelperCallKillSet(CorInfoHelpFunc helper); + AllRegsMask compHelperCallKillSet(CorInfoHelpFunc helper); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -8453,12 +8453,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX return (regMask & RBM_ALLFLOAT) == regMask; } -#if defined(FEATURE_SIMD) && defined(TARGET_XARCH) +#ifdef HAS_PREDICATE_REGS bool IsPredicateRegMask(regMaskTP regMask) { return (regMask & RBM_ALLMASK) == regMask; } -#endif +#endif // HAS_PREDICATE_REGS bool IsOnlyOneRegMask(regMaskTP regMask) { @@ -10806,7 +10806,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool compJitHaltMethod(); - void dumpRegMask(regMaskMixed regs) const; + void dumpRegMask(regMaskOnlyOne mask) const; + void dumpRegMask(AllRegsMask mask) const; #endif diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index d07d5b9f09caae..a6b267ed813997 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3867,12 +3867,12 @@ void emitter::emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMask if (removedRegs != RBM_NONE) { printf(" -"); - dspRegMask(removedRegs); + dspRegMask(removedRegs, RBM_NONE); } if (addedRegs != RBM_NONE) { printf(" +"); - dspRegMask(addedRegs); + dspRegMask(addedRegs, RBM_NONE); } printf("\n"); } @@ -10374,7 +10374,7 @@ const char* emitter::emitOffsetToLabel(unsigned offs) // Return value: // the saved set of registers. // -regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) +AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? bool isNoGCHelper = emitNoGChelper(methHnd); @@ -10383,7 +10383,7 @@ regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd); // Get the set of registers that this call kills and remove it from the saved set. - regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc); + regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc).gprRegs; #ifdef DEBUG if (emitComp->verbose) @@ -10394,12 +10394,12 @@ regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd printf("\n"); } #endif - return savedSet; + return GprRegsMask(savedSet); } else { // This is the saved set of registers after a normal call. - return RBM_CALLEE_SAVED; + return AllRegsMask_CALLEE_SAVED; // TODO: We should have this as a constant. } } @@ -10412,7 +10412,7 @@ regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd // address registers as killed for liveness purposes, since their values change. However, they still are // valid byref pointers after the call, so the dst/src address registers are NOT reported as killed here. // -// Note: This list may not be complete and defaults to the default RBM_CALLEE_TRASH_NOGC registers. +// Note: This list may not be complete and defaults to the default AllRegsMask_CALLEE_TRASH_NOGC registers. // // Arguments: // helper - The helper being inquired about @@ -10420,52 +10420,52 @@ regMaskMixed emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd // Return Value: // Mask of GC register kills // -regMaskMixed emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) +AllRegsMask emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); - regMaskMixed result; + AllRegsMask result; switch (helper) { case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: - result = RBM_CALLEE_GCTRASH_WRITEBARRIER; + result = AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; break; case CORINFO_HELP_ASSIGN_BYREF: - result = RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF; + result = AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; break; #if !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) case CORINFO_HELP_PROF_FCN_ENTER: - result = RBM_PROFILER_ENTER_TRASH; + result = AllRegsMask_PROFILER_ENTER_TRASH; break; case CORINFO_HELP_PROF_FCN_LEAVE: #if defined(TARGET_ARM) // profiler scratch remains gc live - result = RBM_PROFILER_LEAVE_TRASH & ~RBM_PROFILER_RET_SCRATCH; + result = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; #else - result = RBM_PROFILER_LEAVE_TRASH; + result = AllRegsMask_PROFILER_LEAVE_TRASH; #endif break; case CORINFO_HELP_PROF_FCN_TAILCALL: - result = RBM_PROFILER_TAILCALL_TRASH; + result = AllRegsMask_PROFILER_TAILCALL_TRASH; break; #endif // !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) #if defined(TARGET_X86) case CORINFO_HELP_INIT_PINVOKE_FRAME: - result = RBM_INIT_PINVOKE_FRAME_TRASH; + result = AllRegsMask_INIT_PINVOKE_FRAME_TRASH; break; #endif // defined(TARGET_X86) case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - result = RBM_VALIDATE_INDIRECT_CALL_TRASH; + result = AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; break; default: - result = RBM_CALLEE_TRASH_NOGC; + result = AllRegsMask_CALLEE_TRASH_NOGC; break; } diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 1112249c7b0f7e..bb2347568afb22 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3135,10 +3135,10 @@ class emitter bool emitFullGCinfo; // full GC pointer maps? bool emitFullyInt; // fully interruptible code? - regMaskMixed emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); + AllRegsMask emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); // Gets a register mask that represent the kill set for a NoGC helper call. - regMaskMixed emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); + AllRegsMask emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); #if EMIT_TRACK_STACK_DEPTH unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index dcae9cc0f22cbb..8260eb1e54b55a 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -4694,9 +4694,9 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet; - byrefRegs &= savedSet; + AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet.gprRegs; + byrefRegs &= savedSet.gprRegs; #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index c00b6b895edb88..b45a2443e2d03a 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14503,9 +14503,9 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet; - byrefRegs &= savedSet; + AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet.gprRegs; + byrefRegs &= savedSet.gprRegs; #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index e208b45956ce9d..e1b5b20ac6786b 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9499,9 +9499,9 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs((signed)argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - regMaskMixed savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet; - byrefRegs &= savedSet; + AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet.gprRegs; + byrefRegs &= savedSet.gprRegs; #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 01255131e9d461..913f9d7ba74f67 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -10,21 +10,25 @@ public: inline static bool isGeneralRegister(regNumber reg) { + // TODO: This assert will no longer be true return (reg <= REG_INT_LAST); } inline static bool isFloatReg(regNumber reg) { + // TODO: This assert will no longer be true return (reg >= REG_FP_FIRST && reg <= REG_FP_LAST); } inline static bool isDoubleReg(regNumber reg) { + // TODO: This assert will no longer be true return isFloatReg(reg); } inline static bool isMaskReg(regNumber reg) { + // TODO: This assert will no longer be true return (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST); } diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index caee21bff8d869..c7b4dcb05f9b51 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1684,12 +1684,12 @@ instruction CodeGen::ins_Move_Extend(var_types srcType, bool srcInReg) return ins; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) if (varTypeUsesMaskReg(srcType)) { return INS_kmovq_msk; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // TARGET_XARCH && HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(srcType)); @@ -1834,12 +1834,12 @@ instruction CodeGenInterface::ins_Load(var_types srcType, bool aligned /*=false* return ins; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) if (varTypeUsesMaskReg(srcType)) { return INS_kmovq_msk; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // TARGET_XARCH && HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(srcType)); @@ -1918,12 +1918,12 @@ instruction CodeGen::ins_Copy(var_types dstType) #endif } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) if (varTypeUsesMaskReg(dstType)) { return INS_kmovq_msk; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // TARGET_XARCH && HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(dstType)); @@ -1995,13 +1995,13 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) return ins_Copy(dstType); } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) if (genIsValidMaskReg(srcReg)) { // mask to int return INS_kmovq_gpr; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // TARGET_XARCH && HAS_PREDICATE_REGS // float to int assert(genIsValidFloatReg(srcReg)); @@ -2027,7 +2027,7 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) #endif } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS if (varTypeUsesMaskReg(dstType)) { if (genIsValidMaskReg(srcReg)) @@ -2036,11 +2036,13 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) return ins_Copy(dstType); } +#ifdef TARGET_XARCH // mask to int assert(genIsValidIntOrFakeReg(srcReg)); return INS_kmovq_gpr; +#endif // TARGET_XARCH } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(dstType)); @@ -2142,12 +2144,12 @@ instruction CodeGenInterface::ins_Store(var_types dstType, bool aligned /*=false return ins; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) if (varTypeUsesMaskReg(dstType)) { return INS_kmovq_msk; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // TARGET_XARCH && HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(dstType)); @@ -2229,13 +2231,13 @@ instruction CodeGenInterface::ins_StoreFromSrc(regNumber srcReg, var_types dstTy return ins_Store(dstType, aligned); } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS if (genIsValidMaskReg(srcReg)) { // mask to int, treat as mask so it works on 32-bit return ins_Store(TYP_MASK, aligned); } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS // float to int, treat as float to float assert(genIsValidFloatReg(srcReg)); @@ -2259,7 +2261,7 @@ instruction CodeGenInterface::ins_StoreFromSrc(regNumber srcReg, var_types dstTy return ins_Store(dstType, aligned); } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS if (varTypeUsesMaskReg(dstType)) { if (genIsValidMaskReg(srcReg)) @@ -2272,7 +2274,7 @@ instruction CodeGenInterface::ins_StoreFromSrc(regNumber srcReg, var_types dstTy assert(genIsValidIntOrFakeReg(srcReg)); return ins_Store(dstType, aligned); } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS assert(varTypeUsesFloatReg(dstType)); @@ -2583,7 +2585,7 @@ void CodeGen::instGen_Set_Reg_To_Zero(emitAttr size, regNumber reg, insFlags fla #error "Unknown TARGET" #endif - regSet.verifyRegUsed(reg); + regSet.verifyGprRegUsed(reg); } /*****************************************************************************/ diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index fe781c2ee9b468..03c373d2d6a97a 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -3451,8 +3451,27 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - singleRegMask assignedRegMask = genRegMask(regRec->regNum); - compiler->codeGen->regSet.rsSetRegsModified(assignedRegMask DEBUGARG(true)); + regNumber reg = regRec->regNum; + singleRegMask assignedRegMask = genRegMask(reg); + + // TODO: genIsValid* is heavy operation because it does 2 comparisons, but unfortunately + // we cannot use `varTypeRegister[interval->registerType] == VTR_INT` because sometimes + // we do `vmovd xmm1, rcx` where we assign gpr for interval of type TYP_SIMD8 + if (genIsValidIntReg(reg)) + { + compiler->codeGen->regSet.rsSetGprRegsModified(assignedRegMask DEBUGARG(true)); + } +#ifdef HAS_PREDICATE_REGS + else if (genIsValidMaskReg(reg)) + { + compiler->codeGen->regSet.rsSetPredicateRegsModified(assignedRegMask DEBUGARG(true)); + } +#endif + else + { + assert(genIsValidFloatReg(reg)); + compiler->codeGen->regSet.rsSetFloatRegsModified(assignedRegMask DEBUGARG(true)); + } interval->assignedReg = regRec; checkAndAssignInterval(regRec, interval); @@ -7968,7 +7987,7 @@ void LinearScan::resolveRegisters() { // If the localVar is in a register, it must be in a register that is not trashed by // the current node (otherwise it would have already been spilled). - assert((genRegMask(localVarInterval->physReg) & getKillSetForNode(treeNode)) == RBM_NONE); + assert((genRegMask(localVarInterval->physReg) & getKillSetForNode(treeNode).GetRegTypeMask(interval->registerType)) == RBM_NONE); // If we have allocated a register to spill it to, we will use that; otherwise, we will // spill it to the stack. We can use as a temp register any non-arg caller-save register. currentRefPosition->referent->recentRefPosition = currentRefPosition; @@ -9804,7 +9823,15 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } else { - compiler->codeGen->regSet.rsSetRegsModified(genRegMask(tempReg) DEBUGARG(true)); + if (emitter::isFloatReg(tempReg)) + { + compiler->codeGen->regSet.rsSetFloatRegsModified(genRegMask(tempReg) DEBUGARG(true)); + } + else + { + assert(emitter::isGeneralRegister(tempReg)); + compiler->codeGen->regSet.rsSetGprRegsModified(genRegMask(tempReg) DEBUGARG(true)); + } #ifdef TARGET_ARM if (sourceIntervals[fromReg]->registerType == TYP_DOUBLE) { @@ -13324,12 +13351,12 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval if (preferCalleeSave) { - regMaskOnlyOne calleeSaveCandidates = linearScan->calleeSaveRegs(currentInterval->registerType); + regMaskOnlyOne calleeSaveCandidates = linearScan->calleeSaveRegs(regType); if (currentInterval->isWriteThru) { // We'll only prefer a callee-save register if it's already been used. regMaskOnlyOne unusedCalleeSaves = - calleeSaveCandidates & ~(linearScan->compiler->codeGen->regSet.rsGetModifiedRegsMask()); + calleeSaveCandidates & ~linearScan->compiler->codeGen->regSet.rsGetModifiedRegsMask(regType); callerCalleePrefs = calleeSaveCandidates & ~unusedCalleeSaves; preferences &= ~unusedCalleeSaves; } @@ -13802,3 +13829,46 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( assert(found && isSingleRegister(candidates)); return candidates; } + + +bool operator==(const AllRegsMask& first, const AllRegsMask& second) +{ + return first.gprRegs == second.gprRegs && first.floatRegs == second.floatRegs +#ifdef HAS_PREDICATE_REGS + && first.predicateRegs == second.predicateRegs +#endif + ; +} + +AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second) +{ + AllRegsMask result(first.gprRegs & second.gprRegs, first.floatRegs & second.floatRegs +#ifdef HAS_PREDICATE_REGS + , + first.predicateRegs & second.predicateRegs +#endif + ); + return result; +} + +AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second) +{ + AllRegsMask result(first.gprRegs | second.gprRegs, first.floatRegs | second.floatRegs +#ifdef HAS_PREDICATE_REGS + , + first.predicateRegs | second.predicateRegs +#endif + ); + return result; +} + +AllRegsMask operator~(const AllRegsMask& first) +{ + AllRegsMask result(~first.gprRegs, ~first.floatRegs +#ifdef HAS_PREDICATE_REGS + , + ~first.predicateRegs +#endif + ); + return result; +} diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index dbc511c15c2066..4f76141070efbd 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -51,12 +51,12 @@ RegisterType regType(T type) { return IntRegisterType; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS else if (varTypeUsesMaskReg(type)) { return MaskRegisterType; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS else { assert(varTypeUsesFloatReg(type)); @@ -72,12 +72,12 @@ FORCEINLINE unsigned regTypeIndex(T type) { return 0; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS else if (varTypeUsesMaskReg(type)) { return 2; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS else { assert(varTypeUsesFloatReg(type)); @@ -519,13 +519,13 @@ class RegRecord : public Referenceable { registerType = FloatRegisterType; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS else { assert(emitter::isMaskReg(reg)); registerType = MaskRegisterType; } -#endif +#endif // HAS_PREDICATE_REGS regNum = reg; isCalleeSave = ((RBM_CALLEE_SAVED & genRegMask(reg)) != 0); } @@ -1056,7 +1056,8 @@ class LinearScan : public LinearScanInterface void insertZeroInitRefPositions(); // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions - void addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(regMaskOnlyOne mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); @@ -1094,14 +1095,14 @@ class LinearScan : public LinearScanInterface } // Helpers for getKillSetForNode(). - regMaskMixed getKillSetForStoreInd(GenTreeStoreInd* tree); + AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); regMaskGpr getKillSetForMul(GenTreeOp* tree); - regMaskMixed getKillSetForCall(GenTreeCall* call); + AllRegsMask getKillSetForCall(GenTreeCall* call); regMaskGpr getKillSetForModDiv(GenTreeOp* tree); - regMaskMixed getKillSetForBlockStore(GenTreeBlk* blkNode); - regMaskMixed getKillSetForReturn(); - regMaskMixed getKillSetForProfilerHook(); + AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); + AllRegsMask getKillSetForReturn(); + AllRegsMask getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS regMaskGpr getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); #endif // FEATURE_HW_INTRINSICS @@ -1110,11 +1111,11 @@ class LinearScan : public LinearScanInterface // This is used only for an assert, and for stress, so it is only defined under DEBUG. // Otherwise, the Build methods should obtain the killMask from the appropriate method above. #ifdef DEBUG - regMaskMixed getKillSetForNode(GenTree* tree); + AllRegsMask getKillSetForNode(GenTree* tree); #endif // Given some tree node add refpositions for all the registers this node kills - bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskMixed killMask); + bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask); regMaskOnlyOne allRegs(RegisterType rt); regMaskGpr allByteRegs(); @@ -1756,11 +1757,11 @@ class LinearScan : public LinearScanInterface // gets reset for every refposition we are processing depending on the // register type. That wawy we do not have to query and fetch the appropriate // entry again and agin. -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS regMaskOnlyOne m_AvailableRegs[3]; // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. #else regMaskOnlyOne m_AvailableRegs[2]; -#endif +#endif // HAS_PREDICATE_REGS regNumber getRegForType(regNumber reg, var_types regType) { @@ -1791,12 +1792,12 @@ class LinearScan : public LinearScanInterface { m_AvailableRegs[0] = availableIntRegs; m_AvailableRegs[1] = availableFloatRegs; -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS m_AvailableRegs[2] = availableMaskRegs; #endif m_RegistersWithConstants[0] = RBM_NONE; m_RegistersWithConstants[1] = RBM_NONE; -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS m_RegistersWithConstants[2] = RBM_NONE; #endif @@ -1812,7 +1813,7 @@ class LinearScan : public LinearScanInterface //TODO: Fix this later. m_AvailableRegs[0] &= ~(regMask & ~RBM_ALLFLOAT); m_AvailableRegs[1] &= ~(regMask & RBM_ALLFLOAT); -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS m_AvailableRegs[2] &= ~(regMask & RBM_ALLMASK); #endif } @@ -1826,7 +1827,7 @@ class LinearScan : public LinearScanInterface // TODO: This will be just `regMask` makeRegAvailable(regMask & ~RBM_ALLFLOAT, IntRegisterType); makeRegAvailable(regMask & RBM_ALLFLOAT, FloatRegisterType); -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS makeRegAvailable(regMask & RBM_ALLMASK, MaskRegisterType); #endif } @@ -1854,7 +1855,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, m_FloatRegistersWithConstant, etc. #else regMaskOnlyOne m_RegistersWithConstants[2]; @@ -2028,7 +2029,7 @@ class LinearScan : public LinearScanInterface void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); void BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates = RBM_NONE); - void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask); + void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 6ebfaa1d023bc8..f12f5b5728d1a2 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -194,7 +194,6 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -366,14 +365,16 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_RETURNTRAP: + { // this just turns into a compare of its child with an int // + a conditional call srcCount = 1; assert(dstCount == 0); BuildUse(tree->gtGetOp1()); - killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_MUL: if (tree->gtOverflow()) @@ -421,7 +422,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, RBM_NONE); + BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); break; case GT_LONG: @@ -466,10 +467,12 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_RETURN: + { srcCount = BuildReturn(tree); - killMask = getKillSetForReturn(); + AllRegsMask killMask = getKillSetForReturn(); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_RETFILT: assert(dstCount == 0); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index e617bbb62d4202..a98d07d7712689 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -584,7 +584,6 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -664,17 +663,19 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_PROF_HOOK: + { srcCount = 0; assert(dstCount == 0); - killMask = getKillSetForProfilerHook(); + AllRegsMask killMask = getKillSetForProfilerHook(); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_START_PREEMPTGC: // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, RBM_NONE); + BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); break; case GT_CNS_DBL: @@ -737,10 +738,12 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_RETURN: - srcCount = BuildReturn(tree); - killMask = getKillSetForReturn(); + { + srcCount = BuildReturn(tree); + AllRegsMask killMask = getKillSetForReturn(); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_RETFILT: assert(dstCount == 0); @@ -824,14 +827,16 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_RETURNTRAP: + { // this just turns into a compare of its child with an int // + a conditional call BuildUse(tree->gtGetOp1()); srcCount = 1; assert(dstCount == 0); - killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_MOD: case GT_UMOD: diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index a9190b7a0f975c..344db97a39545d 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -375,7 +375,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskMixed killMask = getKillSetForCall(call); + AllRegsMask killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -827,7 +827,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } buildInternalRegisterUses(); - regMaskMixed killMask = getKillSetForBlockStore(blkNode); + AllRegsMask killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 3b3b2f733a9acd..9d5d4f2d39b365 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -699,9 +699,40 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); + if (mask.gprRegs != RBM_NONE) + { + compiler->codeGen->regSet.rsSetGprRegsModified(mask.gprRegs DEBUGARG(true)); + addRefsForPhysRegMask(mask.gprRegs, currentLoc, refType, isLastUse); + } + if (mask.floatRegs != RBM_NONE) + { + compiler->codeGen->regSet.rsSetFloatRegsModified(mask.floatRegs DEBUGARG(true)); + addRefsForPhysRegMask(mask.floatRegs, currentLoc, refType, isLastUse); + } +#ifdef HAS_PREDICATE_REGS + if (mask.predicateRegs != RBM_NONE) + { + compiler->codeGen->regSet.rsSetPredicateRegsModified(mask.predicateRegs DEBUGARG(true)); + addRefsForPhysRegMask(mask.predicateRegs, currentLoc, refType, isLastUse); + } +#endif // HAS_PREDICATE_REGS +} + +//------------------------------------------------------------------------ +// addRefsForPhysRegMask: Adds RefPositions of the given type for specific registers in 'mask'. +// +// Arguments: +// mask - the mask (set) of registers - either gpr/float/predicate +// currentLoc - the location at which they should be added +// refType - the type of refposition +// isLastUse - true IFF this is a last use of the register +// +void LinearScan::addRefsForPhysRegMask(regMaskOnlyOne mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +{ + assert(compiler->IsOnlyOneRegMask(mask)); // The mask identifies a set of registers that will be used during // codegen. Mark these as modified here, so when we do final frame @@ -712,9 +743,7 @@ void LinearScan::addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLo // CORINFO_HELP_ASSIGN_BYREF helper, which kills callee-saved RSI and // RDI, if LSRA doesn't assign RSI/RDI, they wouldn't get marked as // modified until codegen, which is too late. - compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); - - for (regMaskMixed candidates = mask; candidates != RBM_NONE;) + for (regMaskOnlyOne candidates = mask; candidates != RBM_NONE;) { regNumber reg = genFirstRegNumFromMaskAndToggle(candidates); // This assumes that these are all "special" RefTypes that @@ -740,11 +769,11 @@ void LinearScan::addRefsForPhysRegMask(regMaskMixed mask, LsraLocation currentLo // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) +AllRegsMask LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) { assert(tree->OperIs(GT_STOREIND)); - regMaskMixed killMask = RBM_NONE; + AllRegsMask killMask; GCInfo::WriteBarrierForm writeBarrierForm = compiler->codeGen->gcInfo.gcIsWriteBarrierCandidate(tree); if (writeBarrierForm != GCInfo::WBF_NoBarrier) @@ -755,7 +784,7 @@ regMaskMixed LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) // the allocated register for the `data` operand. However, all the (x86) optimized // helpers have the same kill set: EDX. And note that currently, only x86 can return // `true` for genUseOptimizedWriteBarriers(). - killMask = RBM_CALLEE_TRASH_NOGC; + killMask = AllRegsMask_CALLEE_TRASH_NOGC; } else { @@ -840,9 +869,10 @@ regMaskGpr LinearScan::getKillSetForModDiv(GenTreeOp* node) // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForCall(GenTreeCall* call) +AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) { - regMaskMixed killMask = RBM_CALLEE_TRASH; + AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + #ifdef TARGET_X86 if (compiler->compFloatingPointUsed) { @@ -865,16 +895,15 @@ regMaskMixed LinearScan::getKillSetForCall(GenTreeCall* call) // if there is no FP used, we can ignore the FP kills if (!compiler->compFloatingPointUsed) { -#if defined(TARGET_XARCH) - killMask &= ~(RBM_FLT_CALLEE_TRASH | RBM_MSK_CALLEE_TRASH); -#else - killMask &= ~RBM_FLT_CALLEE_TRASH; -#endif // TARGET_XARCH + killMask.floatRegs &= ~RBM_FLT_CALLEE_TRASH; +#if defined(TARGET_XARCH) || defined(HAS_PREDICATE_REGS) + killMask.predicateRegs &= ~RBM_MSK_CALLEE_TRASH; +#endif // TARGET_XARCH || HAS_PREDICATE_REGS } #ifdef TARGET_ARM if (call->IsVirtualStub()) { - killMask |= compiler->virtualStubParamInfo->GetRegMask(); + killMask.gprRegs |= compiler->virtualStubParamInfo->GetRegMask(); } #else // !TARGET_ARM // Verify that the special virtual stub call registers are in the kill mask. @@ -882,7 +911,7 @@ regMaskMixed LinearScan::getKillSetForCall(GenTreeCall* call) // they are already in the RBM_CALLEE_TRASH set, // and we don't want to introduce extra checks and calls in this hot function. assert(!call->IsVirtualStub() || - ((killMask & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); + ((killMask.gprRegs & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); #endif // !TARGET_ARM return killMask; } @@ -895,10 +924,10 @@ regMaskMixed LinearScan::getKillSetForCall(GenTreeCall* call) // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) +AllRegsMask LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) { assert(blkNode->OperIsStoreBlk()); - regMaskMixed killMask = RBM_NONE; + AllRegsMask killMask; bool isCopyBlk = varTypeIsStruct(blkNode->Data()); switch (blkNode->gtBlkOpKind) @@ -928,7 +957,7 @@ regMaskMixed LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) if (isCopyBlk) { // rep movs kills RCX, RDI and RSI - killMask = RBM_RCX | RBM_RDI | RBM_RSI; + killMask.gprRegs = RBM_RCX | RBM_RDI | RBM_RSI; } else { @@ -936,7 +965,7 @@ regMaskMixed LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) // (Note that the Data() node, if not constant, will be assigned to // RCX, but it's find that this kills it, as the value is not available // after this node in any case.) - killMask = RBM_RDI | RBM_RCX; + killMask.gprRegs = RBM_RDI | RBM_RCX; } break; #endif @@ -994,10 +1023,10 @@ regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForReturn() +AllRegsMask LinearScan::getKillSetForReturn() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_LEAVE) - : RBM_NONE; + : AllRegsMask(); } //------------------------------------------------------------------------ @@ -1008,10 +1037,10 @@ regMaskMixed LinearScan::getKillSetForReturn() // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForProfilerHook() +AllRegsMask LinearScan::getKillSetForProfilerHook() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_TAILCALL) - : RBM_NONE; + : AllRegsMask(); } #ifdef DEBUG @@ -1023,9 +1052,9 @@ regMaskMixed LinearScan::getKillSetForProfilerHook() // // Return Value: a register mask of the registers killed // -regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) +AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) { - regMaskMixed killMask = RBM_NONE; + AllRegsMask killMask; switch (tree->OperGet()) { case GT_LSH: @@ -1037,7 +1066,7 @@ regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) case GT_LSH_HI: case GT_RSH_LO: #endif - killMask = getKillSetForShiftRotate(tree->AsOp()); + killMask.gprRegs = getKillSetForShiftRotate(tree->AsOp()); break; case GT_MUL: @@ -1045,14 +1074,14 @@ regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) #if !defined(TARGET_64BIT) || defined(TARGET_ARM64) case GT_MUL_LONG: #endif - killMask = getKillSetForMul(tree->AsOp()); + killMask.gprRegs = getKillSetForMul(tree->AsOp()); break; case GT_MOD: case GT_DIV: case GT_UMOD: case GT_UDIV: - killMask = getKillSetForModDiv(tree->AsOp()); + killMask.gprRegs = getKillSetForModDiv(tree->AsOp()); break; case GT_STORE_BLK: @@ -1088,7 +1117,7 @@ regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: - killMask = getKillSetForHWIntrinsic(tree->AsHWIntrinsic()); + killMask.gprRegs = getKillSetForHWIntrinsic(tree->AsHWIntrinsic()); break; #endif // FEATURE_HW_INTRINSICS @@ -1123,11 +1152,17 @@ regMaskMixed LinearScan::getKillSetForNode(GenTree* tree) // This method can add kills even if killMask is RBM_NONE, if this tree is one of the // special cases that signals that we can't permit callee save registers to hold GC refs. -bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, regMaskMixed killMask) +bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask) { + assert(compiler->IsGprRegMask(killMask.gprRegs)); + assert(compiler->IsFloatRegMask(killMask.floatRegs)); +#ifdef HAS_PREDICATE_REGS + assert(compiler->IsPredicateRegMask(killMask.predicateRegs)); +#endif // HAS_PREDICATE_REGS + bool insertedKills = false; - if (killMask != RBM_NONE) + if (!killMask.IsEmpty()) { addRefsForPhysRegMask(killMask, currentLoc, RefTypeKill, true); @@ -1162,7 +1197,9 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo continue; } Interval* interval = getIntervalForLocalVar(varIndex); - const bool isCallKill = ((killMask == RBM_INT_CALLEE_TRASH) || (killMask == RBM_CALLEE_TRASH)); + regMaskOnlyOne regsKillMask = killMask.GetRegTypeMask(interval->registerType); + + const bool isCallKill = ((regsKillMask == RBM_INT_CALLEE_TRASH) || (regsKillMask == RBM_CALLEE_TRASH)); if (isCallKill) { @@ -1175,7 +1212,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo // See the "heuristics for writeThru intervals" in 'buildIntervals()'. if (!interval->isWriteThru || !isCallKill) { - regMaskOnlyOne newPreferences = allRegs(interval->registerType) & (~killMask); + regMaskOnlyOne newPreferences = allRegs(interval->registerType) & (~regsKillMask); if (newPreferences != RBM_NONE) { @@ -1183,7 +1220,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo { // Update the register aversion as long as this is not write-thru vars for // reason mentioned above. - interval->registerAversion |= killMask; + interval->registerAversion |= regsKillMask; } interval->updateRegisterPreferences(newPreferences); } @@ -1839,10 +1876,10 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc // The use position of v02 cannot be allocated a reg since it is marked delay-reg free and // {eax,edx} are getting killed before the def of GT_DIV. For this reason, minRegCount for // the use position of v02 also needs to take into account the kill set of its consuming node. - regMaskMixed killMask = getKillSetForNode(tree); - if (killMask != RBM_NONE) + AllRegsMask killMask = getKillSetForNode(tree); + if (!killMask.IsEmpty()) { - minRegCountForRef += genCountBits(killMask); + minRegCountForRef += killMask.Count(); } } else if ((newRefPosition->refType) == RefTypeDef && (newRefPosition->getInterval()->isSpecialPutArg)) @@ -2493,13 +2530,14 @@ void LinearScan::buildIntervals() // into the scratch register, so it will be killed here. if (compiler->compShouldPoisonFrame() && compiler->fgFirstBBisScratch() && block == compiler->fgFirstBB) { - regMaskMixed killed; + AllRegsMask killed; #if defined(TARGET_XARCH) // Poisoning uses EAX for small vars and rep stosd that kills edi, ecx and eax for large vars. - killed = RBM_EDI | RBM_ECX | RBM_EAX; + killed.gprRegs = RBM_EDI | RBM_ECX | RBM_EAX; #else // Poisoning uses REG_SCRATCH for small vars and memset helper for big vars. - killed = genRegMask(REG_SCRATCH) | compiler->compHelperCallKillSet(CORINFO_HELP_MEMSET); + killed = compiler->compHelperCallKillSet(CORINFO_HELP_MEMSET); + killed.gprRegs |= genRegMask(REG_SCRATCH); #endif addRefsForPhysRegMask(killed, currentLoc + 1, RefTypeKill, true); currentLoc += 2; @@ -2722,12 +2760,12 @@ void LinearScan::buildIntervals() { calleeSaveCount = CNT_CALLEE_ENREG; } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS else if (varTypeUsesMaskReg(interval->registerType)) { calleeSaveCount = CNT_CALLEE_SAVED_MASK; } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS else { assert(varTypeUsesFloatReg(interval->registerType)); @@ -3071,14 +3109,14 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidat // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, regMaskMixed killMask) +void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask) { assert(killMask == getKillSetForNode(tree)); // Call this even when killMask is RBM_NONE, as we have to check for some special cases buildKillPositionsForNode(tree, currentLoc + 1, killMask); - if (killMask != RBM_NONE) + if (!killMask.IsEmpty()) { #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE // Build RefPositions to account for the fact that, even in a callee-save register, the upper half of any large @@ -3092,9 +3130,9 @@ void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne // RefPositions in that case. // This must be done after the kills, so that we know which large vectors are still live. // - if ((killMask & RBM_FLT_CALLEE_TRASH) != RBM_NONE) + if ((killMask.floatRegs & RBM_FLT_CALLEE_TRASH) != RBM_NONE) { - buildUpperVectorSaveRefPositions(tree, currentLoc + 1 DEBUG_ARG((killMask & RBM_FLT_CALLEE_TRASH))); + buildUpperVectorSaveRefPositions(tree, currentLoc + 1 DEBUG_ARG((killMask.floatRegs & RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } @@ -3992,12 +4030,12 @@ int LinearScan::BuildReturn(GenTree* tree) { buildInternalIntRegisterDefForNode(tree, dstRegMask); } -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS else if (varTypeUsesMaskReg(dstType)) { buildInternalMaskRegisterDefForNode(tree, dstRegMask); } -#endif // TARGET_XARCH && FEATURE_SIMD +#endif // HAS_PREDICATE_REGS else { assert(varTypeUsesFloatReg(dstType)); @@ -4257,7 +4295,7 @@ int LinearScan::BuildGCWriteBarrier(GenTree* tree) BuildUse(addr, addrCandidates); BuildUse(src, srcCandidates); - regMaskMixed killMask = getKillSetForStoreInd(tree->AsStoreInd()); + AllRegsMask killMask = getKillSetForStoreInd(tree->AsStoreInd()); buildKillPositionsForNode(tree, currentLoc + 1, killMask); return 2; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index ef55d125321c53..af7968e9a12093 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -47,7 +47,6 @@ int LinearScan::BuildNode(GenTree* tree) assert(!tree->isContained()); int srcCount; int dstCount = 0; - regMaskMixed killMask = RBM_NONE; bool isLocalDefUse = false; // Reset the build-related members of LinearScan. @@ -138,15 +137,17 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, RBM_NONE); + BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); break; case GT_PROF_HOOK: + { srcCount = 0; assert(dstCount == 0); - killMask = getKillSetForProfilerHook(); + AllRegsMask killMask = getKillSetForProfilerHook(); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_CNS_INT: case GT_CNS_LNG: @@ -188,10 +189,12 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_RETURN: + { srcCount = BuildReturn(tree); - killMask = getKillSetForReturn(); + AllRegsMask killMask = getKillSetForReturn(); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); break; + } case GT_RETFILT: assert(dstCount == 0); @@ -295,7 +298,7 @@ int LinearScan::BuildNode(GenTree* tree) RefPosition* internalDef = buildInternalIntRegisterDefForNode(tree); srcCount = BuildOperandUses(tree->gtGetOp1()); buildInternalRegisterUses(); - killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildDefsWithKills(tree, 0, RBM_NONE, killMask); } break; @@ -1100,7 +1103,7 @@ int LinearScan::BuildShiftRotate(GenTree* tree) if (!shiftBy->isContained()) { srcCount += BuildDelayFreeUses(shiftBy, source, RBM_RCX); - buildKillPositionsForNode(tree, currentLoc + 1, RBM_RCX); + buildKillPositionsForNode(tree, currentLoc + 1, GprRegsMask(RBM_RCX)); } BuildDef(tree, dstCandidates); } @@ -1109,7 +1112,7 @@ int LinearScan::BuildShiftRotate(GenTree* tree) if (!shiftBy->isContained()) { srcCount += BuildOperandUses(shiftBy, RBM_RCX); - buildKillPositionsForNode(tree, currentLoc + 1, RBM_RCX); + buildKillPositionsForNode(tree, currentLoc + 1, GprRegsMask(RBM_RCX)); } } return srcCount; @@ -1344,7 +1347,7 @@ int LinearScan::BuildCall(GenTreeCall* call) buildInternalRegisterUses(); // Now generate defs and kills. - regMaskMixed killMask = getKillSetForCall(call); + AllRegsMask killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); // No args are placed in registers anymore. @@ -1636,7 +1639,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) #endif buildInternalRegisterUses(); - regMaskMixed killMask = getKillSetForBlockStore(blkNode); + AllRegsMask killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); return useCount; @@ -1929,7 +1932,7 @@ int LinearScan::BuildModDiv(GenTree* tree) srcCount += BuildDelayFreeUses(op2, op1, availableIntRegs & ~(RBM_RAX | RBM_RDX)); buildInternalRegisterUses(); - regMaskMixed killMask = getKillSetForModDiv(tree->AsOp()); + AllRegsMask killMask(getKillSetForModDiv(tree->AsOp()), RBM_NONE); BuildDefsWithKills(tree, 1, dstCandidates, killMask); return srcCount; } @@ -3042,7 +3045,7 @@ int LinearScan::BuildMul(GenTree* tree) assert(compiler->IsGprRegMask(dstCandidates)); - regMaskMixed killMask = getKillSetForMul(tree->AsOp()); + AllRegsMask killMask(getKillSetForMul(tree->AsOp()), RBM_NONE); BuildDefsWithKills(tree, dstCount, dstCandidates, killMask); return srcCount; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 6a65636fb6ba4d..05a6e8c56602f3 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -45,7 +45,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ //------------------------------------------------------------------------ -// verifyRegUsed: verify that the register is marked as used. +// verifyGprRegUsed: verify that the GPR register is marked as used. // // Arguments: // reg - The register to verify. @@ -61,12 +61,132 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // This method is intended to be called during code generation, and // should simply validate that the register (or registers) have // already been added to the modified set. +// +void RegSet::verifyGprRegUsed(regNumber reg) +{ + assert(emitter::isGeneralRegister(reg)); + rsSetGprRegsModified(genRegMask(reg)); +} + +//------------------------------------------------------------------------ +// verifyFloatRegUsed: verify that the Float register is marked as used. +// +// Arguments: +// reg - The register to verify. +// +// Return Value: +// None. +// +// Assumptions: +// The caller must have ensured that the register is already marked +// as used. +// +// Notes: +// This method is intended to be called during code generation, and +// should simply validate that the register (or registers) have +// already been added to the modified set. +// +void RegSet::verifyFloatRegUsed(regNumber reg) +{ + assert(emitter::isFloatReg(reg)); + rsSetFloatRegsModified(genRegMask(reg)); +} + +#ifdef HAS_PREDICATE_REGS +//------------------------------------------------------------------------ +// verifyPredicateRegUsed: verify that the mask register is marked as used. +// +// Arguments: +// reg - The register to verify. +// +// Return Value: +// None. +// +// Assumptions: +// The caller must have ensured that the register is already marked +// as used. +// +// Notes: +// This method is intended to be called during code generation, and +// should simply validate that the register (or registers) have +// already been added to the modified set. +void RegSet::verifyPredicateRegUsed(regNumber reg) +{ + assert(emitter::isMaskReg(reg)); + rsSetPredicateRegsModified(genRegMask(reg)); +} +#endif // HAS_PREDICATE_REGS + +//------------------------------------------------------------------------ +// verifyRegUsed: verify that the mask register is marked as used. +// +// Arguments: +// reg - The register to verify. +// +// Return Value: +// None. +// +// Assumptions: +// The caller must have ensured that the register is already marked +// as used. +// +// Notes: +// This method is intended to be called during code generation, and +// should simply validate that the register (or registers) have +// already been added to the modified set. +void RegSet::verifyRegUsed(regNumber reg, var_types type) +{ + if (varTypeUsesIntReg(type)) + { + verifyGprRegUsed(reg); + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeUsesMaskReg(type)) + { + verifyPredicateRegUsed(reg); + } +#endif // HAS_PREDICATE_REGS + else + { + assert(varTypeUsesFloatReg(type)); + verifyFloatRegUsed(reg); + } +} +//------------------------------------------------------------------------ +// verifyRegUsed: verify that the mask register is marked as used. +// +// Arguments: +// reg - The register to verify. +// +// Return Value: +// None. +// +// Assumptions: +// The caller must have ensured that the register is already marked +// as used. +// +// Notes: +// This method is intended to be called during code generation, and +// should simply validate that the register (or registers) have +// already been added to the modified set. void RegSet::verifyRegUsed(regNumber reg) { - // TODO-Cleanup: we need to identify the places where the register - // is not marked as used when this is called. - rsSetRegsModified(genRegMask(reg)); + if (emitter::isGeneralRegister(reg)) + { + verifyGprRegUsed(reg); + } +#ifdef HAS_PREDICATE_REGS + else if (emitter::isMaskReg(reg)) + { + verifyPredicateRegUsed(reg); + } +#endif // HAS_PREDICATE_REGS + else + { + assert(emitter::isFloatReg(reg)); + verifyFloatRegUsed(reg); + } } //------------------------------------------------------------------------ @@ -87,21 +207,36 @@ void RegSet::verifyRegUsed(regNumber reg) // should simply validate that the register (or registers) have // already been added to the modified set. -void RegSet::verifyRegistersUsed(regMaskMixed regMask) +void RegSet::verifyRegistersUsed(AllRegsMask regs) { if (m_rsCompiler->opts.OptimizationDisabled()) { return; } - if (regMask == RBM_NONE) + // TODO-Cleanup: + // We need to identify the places where the register + // is not marked as used when this is called. + // + // See https://github.com/dotnet/runtime/issues/10411 and + // https://github.com/dotnet/coreclr/pull/18230 on why we call + // rsSetGprRegsModified() instead of assert(rsRegsModified()) + if (regs.gprRegs != RBM_NONE) { - return; + rsSetGprRegsModified(regs.gprRegs); } - // TODO-Cleanup: we need to identify the places where the registers - // are not marked as used when this is called. - rsSetRegsModified(regMask); + if (regs.floatRegs != RBM_NONE) + { + rsSetFloatRegsModified(regs.floatRegs); + } + +#ifdef HAS_PREDICATE_REGS + if (regs.predicateRegs != RBM_NONE) + { + rsSetPredicateRegsModified(regs.predicateRegs); + } +#endif } void RegSet::rsClearRegsModified() @@ -116,14 +251,40 @@ void RegSet::rsClearRegsModified() rsModifiedRegsMaskInitialized = true; #endif // DEBUG - rsModifiedRegsMask = RBM_NONE; + rsModifiedGprRegsMask = RBM_NONE; + rsModifiedFloatRegsMask = RBM_NONE; + rsModifiedPredicateRegsMask = RBM_NONE; } -void RegSet::rsSetRegsModified(regMaskMixed mask DEBUGARG(bool suppressDump)) +void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { - assert(mask != RBM_NONE); + assert(m_rsCompiler->IsGprRegMask(mask)); + rsSetRegsModified(rsModifiedGprRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_INT_CALLEE_SAVED)); +} + +void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump)) +{ + assert(m_rsCompiler->IsFloatRegMask(mask)); + rsSetRegsModified(rsModifiedFloatRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_FLT_CALLEE_SAVED)); +} + +#ifdef HAS_PREDICATE_REGS +void RegSet::rsSetPredicateRegsModified(regMaskPredicate mask DEBUGARG(bool suppressDump)) +{ + assert(m_rsCompiler->IsPredicateRegMask(mask)); + rsSetRegsModified(rsModifiedPredicateRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_MSK_CALLEE_SAVED)); +} +#endif // HAS_PREDICATE_REGS + +void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) DEBUGARG(regMaskOnlyOne calleeSaveMask)) +{ + assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); +#ifdef DEBUG + + regMaskOnlyOne newMask = (trackingMask | modifiedMask); + // We can't update the modified registers set after final frame layout (that is, during code // generation and after). Ignore prolog and epilog generation: they call register tracking to // modify rbp, for example, even in functions that use rbp as a frame pointer. Make sure normal @@ -131,60 +292,58 @@ void RegSet::rsSetRegsModified(regMaskMixed mask DEBUGARG(bool suppressDump)) // Frame layout is only affected by callee-saved registers, so only ensure that callee-saved // registers aren't modified after final frame layout. assert((m_rsCompiler->lvaDoneFrameLayout < Compiler::FINAL_FRAME_LAYOUT) || m_rsCompiler->compGeneratingProlog || - m_rsCompiler->compGeneratingEpilog || - (((rsModifiedRegsMask | mask) & RBM_CALLEE_SAVED) == (rsModifiedRegsMask & RBM_CALLEE_SAVED))); - -#ifdef DEBUG + m_rsCompiler->compGeneratingEpilog || ((newMask & calleeSaveMask) == (trackingMask & calleeSaveMask))); if (m_rsCompiler->verbose && !suppressDump) { - if (rsModifiedRegsMask != (rsModifiedRegsMask | mask)) + if (trackingMask != newMask) { printf("Marking regs modified: "); - dspRegMask(mask); + dspRegMask(modifiedMask); printf(" ("); - dspRegMask(rsModifiedRegsMask); + dspRegMask(trackingMask); printf(" => "); - dspRegMask(rsModifiedRegsMask | mask); + dspRegMask(newMask); printf(")\n"); } } #endif // DEBUG - rsModifiedRegsMask |= mask; + trackingMask |= modifiedMask; } void RegSet::rsRemoveRegsModified(regMaskGpr mask) { assert(mask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); + assert(m_rsCompiler->IsGprRegMask(mask)); // See comment in rsSetRegsModified(). assert((m_rsCompiler->lvaDoneFrameLayout < Compiler::FINAL_FRAME_LAYOUT) || m_rsCompiler->compGeneratingProlog || m_rsCompiler->compGeneratingEpilog || - (((rsModifiedRegsMask & ~mask) & RBM_CALLEE_SAVED) == (rsModifiedRegsMask & RBM_CALLEE_SAVED))); + (((rsModifiedGprRegsMask & ~mask) & RBM_CALLEE_SAVED) == (rsModifiedGprRegsMask & RBM_CALLEE_SAVED))); #ifdef DEBUG if (m_rsCompiler->verbose) { printf("Removing modified regs: "); - dspRegMask(mask); - if (rsModifiedRegsMask == (rsModifiedRegsMask & ~mask)) + dspRegMask(mask, RBM_NONE); + if (rsModifiedGprRegsMask == (rsModifiedGprRegsMask & ~mask)) { printf(" (unchanged)"); } else { printf(" ("); - dspRegMask(rsModifiedRegsMask); + dspRegMask(rsModifiedGprRegsMask, RBM_NONE); printf(" => "); - dspRegMask(rsModifiedRegsMask & ~mask); + dspRegMask(rsModifiedGprRegsMask & ~mask, RBM_NONE); printf(")"); } printf("\n"); } #endif // DEBUG - rsModifiedRegsMask &= ~mask; + rsModifiedGprRegsMask &= ~mask; } void RegSet::SetMaskVars(regMaskMixed newMaskVars) @@ -247,9 +406,16 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) : m_rsCompiler(compiler), m_r rsMaskResvd = RBM_NONE; -#if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) +#if defined(TARGET_ARMARCH) + rsGprMaskCalleeSaved = RBM_NONE; + rsFloatMaskCalleeSaved = RBM_NONE; +#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) rsMaskCalleeSaved = RBM_NONE; -#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 +#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_RISCV64 + +#ifdef HAS_PREDICATE_REGS + rsPredicateMaskCalleeSaved = RBM_NONE; +#endif // HAS_PREDICATE_REGS #ifdef TARGET_ARM rsMaskPreSpillRegArg = RBM_NONE; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index f9218bbbe6e57a..4dd287f2442373 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -68,34 +68,79 @@ class RegSet private: bool rsNeededSpillReg; // true if this method needed to spill any registers - regMaskMixed rsModifiedRegsMask; // mask of the registers modified by the current function. + regMaskGpr rsModifiedGprRegsMask; // mask of the registers modified by the current function. + regMaskFloat rsModifiedFloatRegsMask; // mask of the registers modified by the current function. + regMaskPredicate rsModifiedPredicateRegsMask; // mask of the registers modified by the current function. + FORCEINLINE void rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) + DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)); #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. #endif // DEBUG public: - regMaskMixed rsGetModifiedRegsMask() const + + regMaskGpr rsGetModifiedRegsMask(var_types type) const + { + if (varTypeUsesIntReg(type)) + { + return rsGetModifiedGprRegsMask(); + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeUsesMaskReg(type)) + { + return rsGetModifiedPredicateRegsMask(); + } +#endif // HAS_PREDICATE_REGS + else + { + assert(varTypeUsesFloatReg(type)); + return rsGetModifiedFloatRegsMask(); + } + } + + regMaskGpr rsGetModifiedGprRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedRegsMask; + return rsModifiedGprRegsMask; } - void rsClearRegsModified(); + regMaskFloat rsGetModifiedFloatRegsMask() const + { + assert(rsModifiedRegsMaskInitialized); + return rsModifiedFloatRegsMask; + } - void rsSetRegsModified(regMaskMixed mask DEBUGARG(bool suppressDump = false)); + regMaskPredicate rsGetModifiedPredicateRegsMask() const + { + assert(rsModifiedRegsMaskInitialized); + return rsModifiedPredicateRegsMask; + } + + void rsClearRegsModified(); + void rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump = false)); + void rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump = false)); +#ifdef HAS_PREDICATE_REGS + void rsSetPredicateRegsModified(regMaskPredicate mask DEBUGARG(bool suppressDump = false)); +#endif // HAS_PREDICATE_REGS void rsRemoveRegsModified(regMaskGpr mask); bool rsRegsModified(regMaskGpr mask) const { assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & mask) != 0; + return (rsModifiedGprRegsMask & mask) != 0; } void verifyRegUsed(regNumber reg); + void verifyRegUsed(regNumber reg, var_types type); + void verifyGprRegUsed(regNumber reg); + void verifyFloatRegUsed(regNumber reg); +#ifdef HAS_PREDICATE_REGS + void verifyPredicateRegUsed(regNumber reg); +#endif // HAS_PREDICATE_REGS - void verifyRegistersUsed(regMaskMixed regMask); + void verifyRegistersUsed(AllRegsMask mask); public: regMaskMixed GetMaskVars() const // 'get' property function for rsMaskVars property @@ -124,9 +169,16 @@ class RegSet private: regMaskMixed _rsMaskVars; // backing store for rsMaskVars property -#if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) +#if defined(TARGET_ARMARCH) + regMaskGpr rsGprMaskCalleeSaved; + regMaskFloat rsFloatMaskCalleeSaved; +#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog -#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 +#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_LOONGARCH64 + +#ifdef HAS_PREDICATE_REGS + regMaskPredicate rsPredicateMaskCalleeSaved; +#endif public: // TODO-Cleanup: Should be private, but Compiler uses it regMaskGpr rsMaskResvd; // mask of the registers that are reserved for special purposes (typically empty) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 951b14036ff1b1..6d634bea2e0083 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -198,6 +198,10 @@ enum _regMask_enum : unsigned #define AVAILABLE_REG_COUNT get_AVAILABLE_REG_COUNT() +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#define HAS_PREDICATE_REGS +#endif + /*****************************************************************************/ // TODO-Cleanup: The types defined below are mildly confusing: why are there both? @@ -253,7 +257,114 @@ typedef unsigned regMaskTP; #define regMaskOnlyOne regMaskTP #define regMaskMixed regMaskTP #define singleRegMask regMaskTP +#endif // defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) + +unsigned genCountBits(uint64_t bits); + +typedef struct _regMaskAll +{ + regMaskGpr gprRegs; + regMaskFloat floatRegs; +#ifdef HAS_PREDICATE_REGS + regMaskPredicate predicateRegs; +#endif // HAS_PREDICATE_REGS +// +// _regMaskAll(regMaskGpr _gprRegMask) +// : gprRegs(_gprRegMask) +// , floatRegs(RBM_NONE) +//#ifdef HAS_PREDICATE_REGS +// , predicateRegs(RBM_NONE) +//#endif +// { +// } +// +// _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask) : gprRegs(_gprRegMask), floatRegs(_floatRegMask) +//#ifdef HAS_PREDICATE_REGS +// , predicateRegs(RBM_NONE) +//#endif +// { +// +// } + + _regMaskAll(regMaskGpr _gprRegMask, + regMaskFloat _floatRegMask, + regMaskPredicate _predicateRegMask = RBM_NONE) + { + gprRegs = _gprRegMask; + floatRegs = _floatRegMask; +#ifdef HAS_PREDICATE_REGS + predicateRegs = _predicateRegMask; +#endif // HAS_PREDICATE_REGS + + } + +// // Useful to create an entity from `mask` when the caller +// // doesn't know what is the type of mask but knows that it +// // is of "onlyone" type. +// _regMaskAll(regMaskOnlyOne mask) +// { +// gprRegs = mask; +// floatRegs = mask; +//#ifdef HAS_PREDICATE_REGS +// predicateRegs = mask; +//#endif +// } + + _regMaskAll() + { + gprRegs = RBM_NONE; + floatRegs = RBM_NONE; +#ifdef HAS_PREDICATE_REGS + predicateRegs = RBM_NONE; #endif + } + + bool IsEmpty() + { + return (gprRegs == RBM_NONE) && (floatRegs == RBM_NONE) +#ifdef HAS_PREDICATE_REGS + && (predicateRegs == RBM_NONE) +#endif // HAS_PREDICATE_REGS + ; + } + + unsigned Count() + { + return genCountBits(gprRegs) + genCountBits(floatRegs) +#ifdef HAS_PREDICATE_REGS + + genCountBits(predicateRegs) +#endif // HAS_PREDICATE_REGS + ; + } + + regMaskOnlyOne GetRegTypeMask(var_types type) + { + if (varTypeRegister[type] == VTR_INT) + { + return gprRegs; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + return predicateRegs; + } +#endif + else + { + assert(varTypeRegister[type] == VTR_FLOAT); + return floatRegs; + } + } +} AllRegsMask; + +#define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) +#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) + +bool operator==(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator~(const AllRegsMask& first); + #if REGMASK_BITS == 8 typedef unsigned char regMaskSmall; @@ -309,6 +420,14 @@ typedef unsigned char regNumberSmall; #error Unsupported or unset target architecture #endif +#ifdef HAS_PREDICATE_REGS + #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) + #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) +#else + #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED) + #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH) +#endif + #ifdef TARGET_XARCH #define JMP_DIST_SMALL_MAX_NEG (-128) @@ -378,7 +497,8 @@ const char* getRegName(regNumber reg); #ifdef DEBUG const char* getRegNameFloat(regNumber reg, var_types type); -extern void dspRegMask(regMaskMixed regMask, size_t minSiz = 0); +extern void dspRegMask(regMaskOnlyOne mask, size_t minSiz = 0); +extern void dspRegMask(AllRegsMask mask, size_t minSiz = 0); #endif #if CPU_HAS_BYTE_REGS diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 635490feef6bca..b13284bd160ac1 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -198,19 +198,19 @@ #define REG_WRITE_BARRIER_SRC REG_ARG_1 #define RBM_WRITE_BARRIER_SRC RBM_ARG_1 - #define RBM_CALLEE_TRASH_NOGC RBM_CALLEE_TRASH + #define AllRegsMask_CALLEE_TRASH_NOGC AllRegsMask_CALLEE_TRASH // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_TRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_TRASH_NOGC) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF (GprRegsMask(RBM_RSI | RBM_RDI) | AllRegsMask_CALLEE_TRASH_NOGC) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_CALLEE_TRASH_NOGC & ~(RBM_RDI | RBM_RSI)) + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF (AllRegsMask_CALLEE_TRASH_NOGC & GprRegsMask(~(RBM_RDI | RBM_RSI))) // We have two register classifications // * callee trash: aka volatile or caller saved @@ -513,9 +513,9 @@ // The registers trashed by profiler enter/leave/tailcall hook // See vm\amd64\asmhelpers.asm for more details. - #define RBM_PROFILER_ENTER_TRASH RBM_CALLEE_TRASH + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_CALLEE_TRASH - #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_LEAVE_TRASH // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper. #ifdef UNIX_AMD64_ABI @@ -524,18 +524,21 @@ // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. // STOP_FOR_GC helper preserves all the 4 possible return registers. - #define RBM_STOP_FOR_GC_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) + #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) + #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #else // See vm\amd64\asmhelpers.asm for more details. - #define RBM_STOP_FOR_GC_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH & AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET) #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) + #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET)) #endif // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define RBM_INIT_PINVOKE_FRAME_TRASH RBM_CALLEE_TRASH + #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX)) + #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_RAX diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 82000c93aa6fb7..09a68fd49c964f 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -165,20 +165,21 @@ #define REG_WRITE_BARRIER_SRC_BYREF REG_ARG_1 #define RBM_WRITE_BARRIER_SRC_BYREF RBM_ARG_1 - #define RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + #define _RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_WRITEBARRIER + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_WRITEBARRIER // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_CALLEE_TRASH_NOGC + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R4 @@ -219,15 +220,15 @@ #define REG_PROFILER_ENTER_ARG REG_R0 #define RBM_PROFILER_ENTER_ARG RBM_R0 #define REG_PROFILER_RET_SCRATCH REG_R2 - #define RBM_PROFILER_RET_SCRATCH RBM_R2 + #define AllRegsMask_PROFILER_RET_SCRATCH GprRegsMask(RBM_R2) // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. - #define RBM_PROFILER_ENTER_TRASH RBM_NONE + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must // consider it killed by the return. - #define RBM_PROFILER_LEAVE_TRASH RBM_PROFILER_RET_SCRATCH - #define RBM_PROFILER_TAILCALL_TRASH RBM_NONE + #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_RET_SCRATCH + #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask() // Which register are int and long values returned in ? #define REG_INTRET REG_R0 @@ -244,12 +245,12 @@ // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). // See vm\arm\amshelpers.asm for more details. - #define RBM_STOP_FOR_GC_TRASH (RBM_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11|RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7)) + #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11), ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define RBM_INIT_PINVOKE_FRAME_TRASH (RBM_CALLEE_TRASH | RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH) + #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) - #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH) + #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R0 #define REG_FPBASE REG_R11 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 20376e221bf4a3..6e4719dc22eca3 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -176,20 +176,21 @@ #define REG_WRITE_BARRIER_SRC_BYREF REG_R13 #define RBM_WRITE_BARRIER_SRC_BYREF RBM_R13 - #define RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) + #define _RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) + #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_R14|RBM_CALLEE_TRASH_NOGC) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_CALLEE_TRASH_NOGC + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R15 @@ -237,9 +238,9 @@ #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 // The registers trashed by profiler enter/leave/tailcall hook - #define RBM_PROFILER_ENTER_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FLTARG_REGS|RBM_FP)) - #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FLTARG_REGS|RBM_FP)) - #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + #define AllRegsMask_PROFILER_ENTER_TRASH (AllRegsMask_CALLEE_TRASH | AllRegsMask(~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP), ~RBM_FLTARG_REGS)) + #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH + #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH // Which register are int and long values returned in ? #define REG_INTRET REG_R0 @@ -254,12 +255,13 @@ #define RBM_DOUBLERET RBM_V0 // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - #define RBM_STOP_FOR_GC_TRASH RBM_CALLEE_TRASH + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define RBM_INIT_PINVOKE_FRAME_TRASH RBM_CALLEE_TRASH + #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15)) + #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15 #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index 9287a7b3333ba3..2654e6605abe5d 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -222,7 +222,7 @@ #define RBM_OPTIMIZED_WRITE_BARRIER_SRC (RBM_EAX|RBM_ECX|RBM_EBX|RBM_ESI|RBM_EDI) #endif // NOGC_WRITE_BARRIERS - #define RBM_CALLEE_TRASH_NOGC RBM_EDX + #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(RBM_EDX) // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. // Note that x86 normally emits an optimized (source-register-specific) write barrier, but can emit @@ -230,20 +230,20 @@ CLANG_FORMAT_COMMENT_ANCHOR; #ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS - #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_EAX | RBM_EDX) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_EAX | RBM_EDX) #else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - #define RBM_CALLEE_TRASH_WRITEBARRIER RBM_CALLEE_TRASH + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH #endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_EDX + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER GprRegsMask(RBM_EDX) // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_ESI | RBM_EDI | RBM_ECX) + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX) // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_ECX + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ECX) // GenericPInvokeCalliHelper unmanaged target parameter #define REG_PINVOKE_TARGET_PARAM REG_EAX @@ -291,13 +291,13 @@ #define RBM_DOUBLERET RBM_NONE // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - #define RBM_STOP_FOR_GC_TRASH RBM_CALLEE_TRASH + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. - #define RBM_INIT_PINVOKE_FRAME_TRASH (RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) + #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) - #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~RBM_ECX) + #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_ECX #define REG_FPBASE REG_EBP @@ -329,9 +329,9 @@ // The registers trashed by profiler enter/leave/tailcall hook // See vm\i386\asmhelpers.asm for more details. - #define RBM_PROFILER_ENTER_TRASH RBM_NONE - #define RBM_PROFILER_LEAVE_TRASH RBM_NONE - #define RBM_PROFILER_TAILCALL_TRASH (RBM_CALLEE_TRASH & ~RBM_ARG_REGS) + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() + #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask() + #define AllRegsMask_PROFILER_TAILCALL_TRASH (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)) // What sort of reloc do we use for [disp32] address mode #define IMAGE_REL_BASED_DISP32 IMAGE_REL_BASED_HIGHLOW diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 5d3e7277026ee1..35552cb83eca4d 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -430,22 +430,34 @@ const char* dspRegRange(regMaskMixed regMask, size_t& minSiz, const char* sep, r return sep; } +void dspRegMask(regMaskOnlyOne mask, size_t minSiz) +{ + dspRegMask(AllRegsMask(mask, mask +#ifdef HAS_PREDICATE_REGS + , mask +#endif + ), minSiz); +} + /***************************************************************************** * * Displays a register set. * TODO-ARM64-Cleanup: don't allow ip0, ip1 as part of a range. */ -void dspRegMask(regMaskMixed regMask, size_t minSiz) +void dspRegMask(AllRegsMask mask, size_t minSiz) { + //TODO: Need to fix all the callers where we don't know if the input is gpr/float but is of type `regMaskOnlyOne`. + // For now, I am just making `floatMask` as optional and default to RBM_NONE so we don't have to deal with + // lot of build errors. const char* sep = ""; printf("["); - sep = dspRegRange(regMask, minSiz, sep, REG_INT_FIRST, REG_INT_LAST); - sep = dspRegRange(regMask, minSiz, sep, REG_FP_FIRST, REG_FP_LAST); + sep = dspRegRange(mask.gprRegs, minSiz, sep, REG_INT_FIRST, REG_INT_LAST); + sep = dspRegRange(mask.floatRegs, minSiz, sep, REG_FP_FIRST, REG_FP_LAST); #ifdef TARGET_XARCH - sep = dspRegRange(regMask, minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); + sep = dspRegRange(mask.predicateRegs, minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); #endif // TARGET_XARCH printf("]"); diff --git a/src/coreclr/jit/vartype.h b/src/coreclr/jit/vartype.h index 27dd5b3329574f..df0169aba89dce 100644 --- a/src/coreclr/jit/vartype.h +++ b/src/coreclr/jit/vartype.h @@ -85,9 +85,9 @@ inline bool varTypeIsSIMD(T vt) template inline bool varTypeIsMask(T vt) { -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) +#ifdef HAS_PREDICATE_REGS return (TypeGet(vt) == TYP_MASK); -#else // FEATURE_SIMD +#else return false; #endif } From b1cad44f294a42f1238e0c2a7ac4cf04d7076864 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 16 Feb 2024 14:26:26 -0800 Subject: [PATCH 061/201] Update genPoisonFrame() --- src/coreclr/jit/codegen.h | 2 +- src/coreclr/jit/codegencommon.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 6ab4f5daf91b39..f77027ab10fd90 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -357,7 +357,7 @@ class CodeGen final : public CodeGenInterface void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn); - void genPoisonFrame(regMaskMixed bbRegLiveIn); + void genPoisonFrame(regMaskGpr bbRegLiveIn); #if defined(TARGET_ARM) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7f692ed038e34b..1c4957f920c143 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8317,7 +8317,7 @@ unsigned CodeGenInterface::getCurrentStackLevel() const // This function emits code to poison address exposed non-zero-inited local variables. We expect this function // to be called when emitting code for the scratch BB that comes right after the prolog. // The variables are poisoned using 0xcdcdcdcd. -void CodeGen::genPoisonFrame(regMaskMixed regLiveIn) +void CodeGen::genPoisonFrame(regMaskGpr regLiveIn) { assert(compiler->compShouldPoisonFrame()); #if defined(TARGET_XARCH) From dabf8f592911ee1ecd8d2c15c0a5247be59f8608 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 14:13:23 -0800 Subject: [PATCH 062/201] Fix some of the reserved registers and related code paths --- src/coreclr/jit/codegenarm.cpp | 8 ++-- src/coreclr/jit/codegenarm64.cpp | 5 +-- src/coreclr/jit/codegenarmarch.cpp | 30 ++++++++++--- src/coreclr/jit/codegencommon.cpp | 68 ++++++++++++++++++++++++------ src/coreclr/jit/codegeninterface.h | 2 +- src/coreclr/jit/codegenlinear.cpp | 4 +- src/coreclr/jit/codegenxarch.cpp | 10 +++-- src/coreclr/jit/compiler.h | 31 +++++++++++++- src/coreclr/jit/emit.cpp | 4 -- src/coreclr/jit/emit.h | 4 -- src/coreclr/jit/gcinfo.cpp | 4 +- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/jitgcinfo.h | 2 +- src/coreclr/jit/regset.h | 3 +- src/coreclr/jit/target.h | 10 +---- 15 files changed, 133 insertions(+), 54 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 6f1e32f550e178..d61a3f22d2203d 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2494,8 +2494,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() unsigned preSpillRegArgSize = genCountBits(regSet.rsMaskPreSpillRegs(true)) * REGSIZE_BYTES; genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; - regMaskMixed rsGprMaskSaveRegs = regSet.rsGprMaskCalleeSaved; - regMaskMixed rsFloatMaskSaveRegs = regSet.rsFloatMaskCalleeSaved; + regMaskGpr rsGprMaskSaveRegs = regSet.rsGprMaskCalleeSaved; + regMaskFloat rsFloatMaskSaveRegs = regSet.rsFloatMaskCalleeSaved; unsigned saveRegsCount = genCountBits(rsGprMaskSaveRegs) + genCountBits(rsFloatMaskSaveRegs); unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; @@ -2652,9 +2652,9 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu regNumber rAddr; regNumber rCnt = REG_NA; // Invalid - regMaskMixed regMask; + regMaskGpr regMask; - regMaskMixed availMask = regSet.rsGetModifiedGprRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers + regMaskGpr availMask = regSet.rsGetModifiedGprRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers availMask &= ~intRegState.rsCalleeRegArgMaskLiveIn; // Remove all of the incoming argument registers as they are // currently live availMask &= ~genRegMask(initReg); // Remove the pre-calculated initReg as we will zero it and maybe use it for diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 17646eebba6cf7..d1f0bc4050f334 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,8 +36,8 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskMixed rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; - regMaskMixed rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; + regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskFloat rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; if (isFramePointerUsed()) { @@ -1606,7 +1606,6 @@ void CodeGen::genFuncletEpilog() } } - regMaskMixed regsToRestoreMask = maskRestoreRegsInt | maskRestoreRegsFloat; if ((genFuncletInfo.fiFrameType == 1) || (genFuncletInfo.fiFrameType == 2) || (genFuncletInfo.fiFrameType == 3)) { maskRestoreRegsInt &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 0c0fd47cedaf53..8220fd92b71960 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3577,14 +3577,18 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (call->IsFastTailCall()) { - regMaskMixed trashedByEpilog = RBM_CALLEE_SAVED; + regMaskGpr trashedGprByEpilog = RBM_INT_CALLEE_SAVED; + regMaskFloat trashedFloatByEpilog = RBM_FLT_CALLEE_SAVED; +#if HAS_PREDICATE_REGS + regMaskPredicate trashedPredicateByEpilog = RBM_MSK_CALLEE_SAVED; +#endif // HAS_PREDICATE_REGS // The epilog may use and trash REG_GSCOOKIE_TMP_0/1. Make sure we have no // non-standard args that may be trash if this is a tailcall. if (compiler->getNeedsGSSecurityCookie()) { - trashedByEpilog |= genRegMask(REG_GSCOOKIE_TMP_0); - trashedByEpilog |= genRegMask(REG_GSCOOKIE_TMP_1); + trashedGprByEpilog |= genRegMask(REG_GSCOOKIE_TMP_0); + trashedGprByEpilog |= genRegMask(REG_GSCOOKIE_TMP_1); } for (CallArg& arg : call->gtArgs.Args()) @@ -3592,13 +3596,29 @@ void CodeGen::genCallInstruction(GenTreeCall* call) for (unsigned j = 0; j < arg.AbiInfo.NumRegs; j++) { regNumber reg = arg.AbiInfo.GetRegNum(j); - if ((trashedByEpilog & genRegMask(reg)) != 0) + if ((trashedGprByEpilog & genRegMask(reg)) != 0) { JITDUMP("Tail call node:\n"); DISPTREE(call); - JITDUMP("Register used: %s\n", getRegName(reg)); + JITDUMP("Gpr Register used: %s\n", getRegName(reg)); assert(!"Argument to tailcall may be trashed by epilog"); } + else if ((trashedFloatByEpilog & genRegMask(reg)) != 0) + { + JITDUMP("Tail call node:\n"); + DISPTREE(call); + JITDUMP("Float Register used: %s\n", getRegName(reg)); + assert(!"Argument to tailcall may be trashed by epilog"); + } +#if HAS_PREDICATE_REGS + else if ((trashedPredicateByEpilog & genRegMask(reg)) != 0) + { + JITDUMP("Tail call node:\n"); + DISPTREE(call); + JITDUMP("Mask Register used: %s\n", getRegName(reg)); + assert(!"Argument to tailcall may be trashed by epilog"); + } +#endif // HAS_PREDICATE_REGS } } } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 1c4957f920c143..cf927fd3156626 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -128,7 +128,7 @@ CodeGen::CodeGen(Compiler* theCompiler) : CodeGenInterface(theCompiler) #if defined(TARGET_XARCH) // Shouldn't be used before it is set in genFnProlog() - compiler->compCalleeFPRegsSavedMask = (regMaskMixed)-1; + compiler->compCalleeFPRegsSavedMask = (regMaskFloat)-1; #endif // defined(TARGET_XARCH) #endif // DEBUG @@ -748,7 +748,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) { // TODO-Cleanup: Move the code from compUpdateLifeVar to genUpdateRegLife that updates the // gc sets - regMaskMixed regMask = varDsc->lvRegMask(); + regMaskOnlyOne regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur &= ~regMask; @@ -793,7 +793,7 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife) VarSetOps::RemoveElemD(this, codeGen->gcInfo.gcVarPtrSetCur, bornVarIndex); } codeGen->genUpdateRegLife(varDsc, true /*isBorn*/, false /*isDying*/ DEBUGARG(nullptr)); - regMaskMixed regMask = varDsc->lvRegMask(); + regMaskOnlyOne regMask = varDsc->lvRegMask(); if (isGCRef) { codeGen->gcInfo.gcRegGCrefSetCur |= regMask; @@ -5405,7 +5405,6 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - //regMaskMixed maskCalleeRegsPushed = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; #ifdef TARGET_ARM // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? #endif @@ -8104,28 +8103,73 @@ void CodeGen::genRegCopy(GenTree* treeNode) // First set the source registers as busy if they haven't been spilled. // (Note that this is just for verification that we don't have circular dependencies.) - regMaskMixed busyRegs = RBM_NONE; +#ifdef DEBUG + AllRegsMask busyRegs; for (unsigned i = 0; i < regCount; ++i) { if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0) { - busyRegs |= genRegMask(op1->GetRegByIndex(i)); + regNumber reg = op1->GetRegByIndex(i); + //regMaskOnlyOne regMask = genRegMask(); + if (genIsValidIntReg(reg)) + { + busyRegs.gprRegs |= genRegMask(reg); + } + else if (genIsValidFloatReg(reg)) + { + busyRegs.floatRegs |= genRegMask(reg); + } +#ifdef HAS_PREDICATE_REGS + else if (genIsValidMaskReg(reg)) + { + busyRegs.predicateRegs |= genRegMask(reg); + } +#endif // HAS_PREDICATE_REGS } } +#endif // DEBUG + for (unsigned i = 0; i < regCount; ++i) { regNumber sourceReg = op1->GetRegByIndex(i); // genRegCopy will consume the source register, perform any required reloads, // and will return either the register copied to, or the original register if there's no copy. regNumber targetReg = genRegCopy(treeNode, i); - if (targetReg != sourceReg) + +#ifdef DEBUG + +#define DO_VALIDATION(regType) \ +if (targetReg != sourceReg) \ +{ \ + singleRegMask targetRegMask = genRegMask(targetReg); \ + assert((busyRegs.regType & targetRegMask) == 0); \ + busyRegs.regType &= ~genRegMask(sourceReg); \ +} \ +busyRegs.regType |= genRegMask(targetReg); + + if (genIsValidIntReg(targetReg)) { - singleRegMask targetRegMask = genRegMask(targetReg); - assert((busyRegs & targetRegMask) == 0); - // Clear sourceReg from the busyRegs, and add targetReg. - busyRegs &= ~genRegMask(sourceReg); + DO_VALIDATION(gprRegs) } - busyRegs |= genRegMask(targetReg); + else if (genIsValidFloatReg(targetReg)) + { + DO_VALIDATION(floatRegs) + } +#ifdef HAS_PREDICATE_REGS + else if (genIsValidMaskReg(targetReg)) + { + DO_VALIDATION(predicateRegs) + } +#endif // HAS_PREDICATE_REGS + //if (targetReg != sourceReg) + //{ + // singleRegMask targetRegMask = genRegMask(targetReg); + // assert((busyRegs & targetRegMask) == 0); + // // Clear sourceReg from the busyRegs, and add targetReg. + // busyRegs &= ~genRegMask(sourceReg); + //} + //busyRegs |= genRegMask(targetReg); +#endif // DEBUG } return; } diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index fd17d4d0da4f00..ec527efbb5b0c5 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -80,7 +80,7 @@ class CodeGenInterface // Call this function after the equivalent fields in Compiler have been initialized. void CopyRegisterInfo(); - FORCEINLINE regMaskMixed get_RBM_ALLMASK() const + FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const { return this->rbmAllMask; } diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 5d3fa9234ddb33..9f09e70b4baa5f 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -201,7 +201,7 @@ void CodeGen::genCodeForBBlist() // change? We cleared them out above. Maybe we should just not clear them out, but update the ones that change // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). - regMaskMixed newLiveRegSet = RBM_NONE; + AllRegsMask newLiveRegSet; regMaskGpr newRegGCrefSet = RBM_NONE; regMaskGpr newRegByrefSet = RBM_NONE; #ifdef DEBUG @@ -216,7 +216,7 @@ void CodeGen::genCodeForBBlist() if (varDsc->lvIsInReg()) { - newLiveRegSet |= varDsc->lvRegMask(); + varDsc->lvRegMask(newLiveRegSet); if (varDsc->lvType == TYP_REF) { newRegGCrefSet |= varDsc->lvRegMask(); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 289390919b7126..53b000ba31dcc8 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6619,7 +6619,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - regMaskMixed tempMask = varDsc->lvRegMask(); + regMaskOnlyOne tempMask = varDsc->lvRegMask(); regSet.RemoveMaskVars(tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) @@ -8447,7 +8447,9 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk) regNumber simdTmpReg = REG_NA; if (putArgStk->AvailableTempRegCount() != 0) { - regMaskMixed rsvdRegs = putArgStk->gtRsvdRegs; + regMaskOnlyOne rsvdRegs = putArgStk->gtRsvdRegs; + assert(compiler->IsOnlyOneRegMask(rsvdRegs)); + if ((rsvdRegs & RBM_ALLINT) != 0) { intTmpReg = putArgStk->GetSingleTempReg(RBM_ALLINT); @@ -9678,7 +9680,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // registers that profiler callback kills. if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { - regMaskMixed thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); + regMaskOnlyOne thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); noway_assert((RBM_PROFILER_LEAVE_TRASH & thisPtrMask) == 0); } @@ -10737,7 +10739,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() assert(isFramePointerUsed()); assert(compiler->lvaDoneFrameLayout == Compiler::FINAL_FRAME_LAYOUT); // The frame size and offsets must be // finalized - assert(compiler->compCalleeFPRegsSavedMask != (regMaskMixed)-1); // The float registers to be preserved is finalized + assert(compiler->compCalleeFPRegsSavedMask != (regMaskFloat)-1); // The float registers to be preserved is finalized // Even though lvaToInitialSPRelativeOffset() depends on compLclFrameSize, // that's ok, because we're figuring out an offset in the parent frame. diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 08c629a0485d8b..19aee8041bc5da 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1005,9 +1005,9 @@ class LclVarDsc return lvIsRegCandidate() && (GetRegNum() != REG_STK); } - regMaskMixed lvRegMask() const + regMaskOnlyOne lvRegMask() const { - regMaskMixed regMask = RBM_NONE; + regMaskOnlyOne regMask = RBM_NONE; if (GetRegNum() != REG_STK) { if (varTypeUsesFloatReg(this)) @@ -1028,6 +1028,33 @@ class LclVarDsc return regMask; } + regMaskOnlyOne lvRegMask(AllRegsMask& allRegsMask) const + { + regMaskOnlyOne regMask = RBM_NONE; + if (GetRegNum() != REG_STK) + { + if (varTypeUsesFloatReg(this)) + { + regMask = genRegMaskFloat(GetRegNum() ARM_ARG(TypeGet())); + allRegsMask.floatRegs = regMask; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeUsesMaskReg(this)) + { + regMask = genRegMask(GetRegNum()); + allRegsMask.predicateRegs = regMask; + } +#endif + else + { + assert(varTypeUsesIntReg(this)); + regMask = genRegMask(GetRegNum()); + allRegsMask.gprRegs = regMask; + } + } + return regMask; + } + //----------------------------------------------------------------------------- // AllFieldDeathFlags: Get a bitset of flags that represents all fields dying. // diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index a6b267ed813997..587f3a802812ab 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -273,10 +273,8 @@ void emitterStaticStats() fprintf(fout, "Offset / size of igLoopBackEdge = %3zu / %2zu\n", offsetof(insGroup, igLoopBackEdge), sizeof(igDummy->igLoopBackEdge)); #endif // FEATURE_LOOP_ALIGN -#if !(REGMASK_BITS <= 32) fprintf(fout, "Offset / size of igGCregs = %3zu / %2zu\n", offsetof(insGroup, igGCregs), sizeof(igDummy->igGCregs)); -#endif // !(REGMASK_BITS <= 32) fprintf(fout, "Offset / size of igData = %3zu / %2zu\n", offsetof(insGroup, igData), sizeof(igDummy->igData)); fprintf(fout, "Offset / size of igPhData = %3zu / %2zu\n", offsetof(insGroup, igPhData), @@ -289,10 +287,8 @@ void emitterStaticStats() fprintf(fout, "Offset / size of igStkLvl = %3zu / %2zu\n", offsetof(insGroup, igStkLvl), sizeof(igDummy->igStkLvl)); #endif // EMIT_TRACK_STACK_DEPTH -#if REGMASK_BITS <= 32 fprintf(fout, "Offset / size of igGCregs = %3zu / %2zu\n", offsetof(insGroup, igGCregs), sizeof(igDummy->igGCregs)); -#endif // REGMASK_BITS <= 32 fprintf(fout, "Offset / size of igInsCnt = %3zu / %2zu\n", offsetof(insGroup, igInsCnt), sizeof(igDummy->igInsCnt)); fprintf(fout, "\n"); diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index bb2347568afb22..a7b9e2f32c0c48 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -321,9 +321,7 @@ struct insGroup // Try to do better packing based on how large regMaskSmall is (8, 16, or 64 bits). CLANG_FORMAT_COMMENT_ANCHOR; -#if !(REGMASK_BITS <= 32) regMaskSmall igGCregs; // set of registers with live GC refs -#endif // !(REGMASK_BITS <= 32) union { BYTE* igData; // addr of instruction descriptors @@ -340,9 +338,7 @@ struct insGroup unsigned igStkLvl; // stack level on entry #endif // EMIT_TRACK_STACK_DEPTH -#if REGMASK_BITS <= 32 regMaskSmall igGCregs; // set of registers with live GC refs -#endif // REGMASK_BITS <= 32 unsigned char igInsCnt; // # of instructions in this group diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index f391aa0072cbd5..0c9320345eb13e 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -188,8 +188,10 @@ void GCInfo::gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput)) * non-pointer values. */ -void GCInfo::gcMarkRegSetNpt(regMaskMixed regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput)) { + assert(compiler->IsOnlyOneRegMask(regMask)); + /* NOTE: don't unmark any live register variables */ regMaskMixed gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 37794f62963b42..3f6c10a79ee4f4 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -968,7 +968,7 @@ struct GenTree ValueNumPair gtVNPair; - regMaskSmall gtRsvdRegs; // set of fixed trashed registers + regMaskOnlyOne gtRsvdRegs; // set of fixed trashed registers unsigned AvailableTempRegCount(regMaskMixed mask = (regMaskMixed)-1) const; regNumber GetSingleTempReg(regMaskMixed mask = (regMaskMixed)-1); diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index c3f41f36ec7998..027c1f166fdc73 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -91,7 +91,7 @@ class GCInfo void gcMarkRegSetGCref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); - void gcMarkRegSetNpt(regMaskMixed regMask DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegPtrVal(regNumber reg, var_types type); #ifdef DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 4dd287f2442373..b9fca9ee176f20 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -168,9 +168,10 @@ class RegSet private: regMaskMixed _rsMaskVars; // backing store for rsMaskVars property + AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property #if defined(TARGET_ARMARCH) - regMaskGpr rsGprMaskCalleeSaved; + regMaskGpr rsGprMaskCalleeSaved; // TODO: Can use AllRegsMask here as well regMaskFloat rsFloatMaskCalleeSaved; #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6d634bea2e0083..cf6edde2cfb713 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -366,15 +366,7 @@ AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator~(const AllRegsMask& first); -#if REGMASK_BITS == 8 -typedef unsigned char regMaskSmall; -#define REG_MASK_INT_FMT "%02X" -#define REG_MASK_ALL_FMT "%02X" -#elif REGMASK_BITS == 16 -typedef unsigned short regMaskSmall; -#define REG_MASK_INT_FMT "%04X" -#define REG_MASK_ALL_FMT "%04X" -#elif REGMASK_BITS == 32 +#if REGMASK_BITS == 32 typedef unsigned regMaskSmall; #define REG_MASK_INT_FMT "%08X" #define REG_MASK_ALL_FMT "%08X" From b3febfa8a6e62a4f21b2ae9989b4bc3ec2e27c99 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 16:41:46 -0800 Subject: [PATCH 063/201] Add AllRegsMask for *VarRegs() method --- src/coreclr/jit/codegenarmarch.cpp | 14 ++--- src/coreclr/jit/codegencommon.cpp | 6 +-- src/coreclr/jit/codegenlinear.cpp | 21 ++++---- src/coreclr/jit/codegenxarch.cpp | 14 ++--- src/coreclr/jit/compiler.cpp | 8 +-- src/coreclr/jit/compiler.h | 31 +---------- src/coreclr/jit/compiler.hpp | 9 +++- src/coreclr/jit/emit.cpp | 64 +++++++++++++++-------- src/coreclr/jit/emit.h | 6 +-- src/coreclr/jit/emitarm.cpp | 8 +-- src/coreclr/jit/emitarm64.cpp | 4 +- src/coreclr/jit/emitxarch.cpp | 8 +-- src/coreclr/jit/gcinfo.cpp | 19 +++---- src/coreclr/jit/jitgcinfo.h | 6 ++- src/coreclr/jit/lclvars.cpp | 7 +-- src/coreclr/jit/lsra.cpp | 2 +- src/coreclr/jit/regset.cpp | 18 +++---- src/coreclr/jit/regset.h | 84 ++++++++++++++++++++++++++---- src/coreclr/jit/target.h | 28 +++++++--- 19 files changed, 221 insertions(+), 136 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 8220fd92b71960..ab9664bd766618 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3902,7 +3902,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). singleRegMask tempMask = genRegMask(varDsc->GetRegNum()); - regSet.RemoveMaskVars(tempMask); + regSet.RemoveMaskVars(varDsc->TypeGet(), tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) { @@ -3990,7 +3990,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // expecting it. Therefore manually update life of argReg. Note that GT_JMP marks the end of // the basic block and after which reg life and gc info will be recomputed for the new block // in genCodeForBBList(). - regSet.AddMaskVars(genRegMask(argReg)); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(argReg)); gcInfo.gcMarkRegPtrVal(argReg, loadType); if (compiler->lvaIsMultiregStruct(varDsc, compiler->info.compIsVarArgs)) @@ -4002,7 +4002,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) loadSize = emitActualTypeSize(loadType); GetEmitter()->emitIns_R_S(ins_Load(loadType), loadSize, argRegNext, varNum, TARGET_POINTER_SIZE); - regSet.AddMaskVars(genRegMask(argRegNext)); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(argRegNext)); gcInfo.gcMarkRegPtrVal(argRegNext, loadType); } @@ -4100,7 +4100,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) GetEmitter()->emitIns_R_S(ins_Load(loadType), loadSize, slotReg, varNum, ofs); } - regSet.AddMaskVars(genRegMask(slotReg)); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(slotReg)); gcInfo.gcMarkRegPtrVal(slotReg, loadType); if (genIsValidIntReg(slotReg) && compiler->info.compIsVarArgs) { @@ -4119,7 +4119,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) GetEmitter()->emitIns_R_S(ins_Load(loadType), emitTypeSize(loadType), argReg, varNum, 0); } - regSet.AddMaskVars(genRegMask(argReg)); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(argReg)); gcInfo.gcMarkRegPtrVal(argReg, loadType); if (genIsValidIntReg(argReg) && compiler->info.compIsVarArgs) @@ -5558,10 +5558,10 @@ void CodeGen::genFnEpilog(BasicBlock* block) dumpConvertedVarSet(compiler, gcInfo.gcVarPtrSetCur); printf(", gcRegGCrefSetCur="); printRegMaskInt(gcInfo.gcRegGCrefSetCur); - GetEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur); + GetEmitter()->emitDispGprRegSet(gcInfo.gcRegGCrefSetCur); printf(", gcRegByrefSetCur="); printRegMaskInt(gcInfo.gcRegByrefSetCur); - GetEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur); + GetEmitter()->emitDispGprRegSet(gcInfo.gcRegByrefSetCur); printf("\n"); } #endif // DEBUG diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index cf927fd3156626..2b0b5bf8bc54bd 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -579,15 +579,15 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // We'd like to be able to assert the following, however if we are walking // through a qmark/colon tree, we may encounter multiple last-use nodes. // assert((regSet.GetMaskVars() & regMask) == regMask); - regSet.RemoveMaskVars(regMask); + regSet.RemoveMaskVars(varDsc->TypeGet(), regMask); } else { // If this is going live, the register must not have a variable in it, except // in the case of an exception or "spill at single-def" variable, which may be already treated // as live in the register. - assert(varDsc->IsAlwaysAliveInMemory() || ((regSet.GetMaskVars() & regMask) == 0)); - regSet.AddMaskVars(regMask); + assert(varDsc->IsAlwaysAliveInMemory() || ((regSet.GetMaskVars(varDsc->TypeGet()) & regMask) == 0)); + regSet.AddMaskVars(varDsc->TypeGet(), regMask); } } diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 9f09e70b4baa5f..44270ffbfa1a84 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -216,7 +216,8 @@ void CodeGen::genCodeForBBlist() if (varDsc->lvIsInReg()) { - varDsc->lvRegMask(newLiveRegSet); + newLiveRegSet.AddRegTypeMask(varDsc->TypeGet(), varDsc->lvRegMask()); + if (varDsc->lvType == TYP_REF) { newRegGCrefSet |= varDsc->lvRegMask(); @@ -390,7 +391,7 @@ void CodeGen::genCodeForBBlist() // We cannot emit this code in the prolog as it might make the prolog too large. if (compiler->compShouldPoisonFrame() && compiler->fgBBisScratch(block)) { - genPoisonFrame(newLiveRegSet); + genPoisonFrame(newLiveRegSet.gprRegs); } // Traverse the block in linear order, generating code for each node as we @@ -481,7 +482,7 @@ void CodeGen::genCodeForBBlist() /* Make sure we didn't bungle pointer register tracking */ regMaskGpr ptrRegs = gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur; - regMaskGpr nonVarPtrRegs = ptrRegs & ~regSet.GetMaskVars(); + regMaskGpr nonVarPtrRegs = ptrRegs & ~regSet.GetGprMaskVars(); // If return is a GC-type, clear it. Note that if a common // epilog is generated (genReturnBB) it has a void return @@ -499,14 +500,14 @@ void CodeGen::genCodeForBBlist() if (nonVarPtrRegs) { printf("Regset after " FMT_BB " gcr=", block->bbNum); - printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.GetMaskVars()); - compiler->GetEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.GetMaskVars()); + printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.GetGprMaskVars()); + compiler->GetEmitter()->emitDispGprRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.GetGprMaskVars()); printf(", byr="); - printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.GetMaskVars()); - compiler->GetEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur & ~regSet.GetMaskVars()); + printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.GetGprMaskVars()); + compiler->GetEmitter()->emitDispGprRegSet(gcInfo.gcRegByrefSetCur & ~regSet.GetGprMaskVars()); printf(", regVars="); - printRegMaskInt(regSet.GetMaskVars()); - compiler->GetEmitter()->emitDispRegSet(regSet.GetMaskVars()); + printRegMaskInt(regSet.GetGprMaskVars()); + compiler->GetEmitter()->emitDispGprRegSet(regSet.GetGprMaskVars()); printf("\n"); } @@ -1123,7 +1124,7 @@ void CodeGen::genUnspillLocal( } #endif // DEBUG - regSet.AddMaskVars(genGetRegMask(varDsc)); + regSet.AddMaskVars(varDsc->TypeGet(), genGetRegMask(varDsc)); } gcInfo.gcMarkRegPtrVal(regNum, type); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 53b000ba31dcc8..b3e682d01740a5 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6620,7 +6620,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). regMaskOnlyOne tempMask = varDsc->lvRegMask(); - regSet.RemoveMaskVars(tempMask); + regSet.RemoveMaskVars(varDsc->TypeGet(), tempMask); gcInfo.gcMarkRegSetNpt(tempMask); if (compiler->lvaIsGCTracked(varDsc)) { @@ -6696,7 +6696,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) if (type0 != TYP_UNKNOWN) { GetEmitter()->emitIns_R_S(ins_Load(type0), emitTypeSize(type0), varDsc->GetArgReg(), varNum, offset0); - regSet.SetMaskVars(regSet.GetMaskVars() | genRegMask(varDsc->GetArgReg())); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(varDsc->GetArgReg())); gcInfo.gcMarkRegPtrVal(varDsc->GetArgReg(), type0); } @@ -6704,7 +6704,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) { GetEmitter()->emitIns_R_S(ins_Load(type1), emitTypeSize(type1), varDsc->GetOtherArgReg(), varNum, offset1); - regSet.SetMaskVars(regSet.GetMaskVars() | genRegMask(varDsc->GetOtherArgReg())); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(varDsc->GetOtherArgReg())); gcInfo.gcMarkRegPtrVal(varDsc->GetOtherArgReg(), type1); } @@ -6751,7 +6751,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // expecting it. Therefore manually update life of argReg. Note that GT_JMP marks the end of the // basic block and after which reg life and gc info will be recomputed for the new block in // genCodeForBBList(). - regSet.AddMaskVars(genRegMask(argReg)); + regSet.AddMaskVars(varDsc->TypeGet(), genRegMask(argReg)); gcInfo.gcMarkRegPtrVal(argReg, loadType); if (compiler->lvaIsGCTracked(varDsc)) { @@ -9190,7 +9190,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, // The call target must not overwrite any live variable, though it may not be in the // kill set for the call. regMaskGpr callTargetMask = genRegMask(callTargetReg); - noway_assert((callTargetMask & regSet.GetMaskVars()) == RBM_NONE); + noway_assert((callTargetMask & regSet.GetGprMaskVars()) == RBM_NONE); } #endif @@ -10136,10 +10136,10 @@ void CodeGen::genFnEpilog(BasicBlock* block) dumpConvertedVarSet(compiler, gcInfo.gcVarPtrSetCur); printf(", gcRegGCrefSetCur="); printRegMaskInt(gcInfo.gcRegGCrefSetCur); - GetEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur); + GetEmitter()->emitDispGprRegSet(gcInfo.gcRegGCrefSetCur); printf(", gcRegByrefSetCur="); printRegMaskInt(gcInfo.gcRegByrefSetCur); - GetEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur); + GetEmitter()->emitDispGprRegSet(gcInfo.gcRegByrefSetCur); printf("\n"); } #endif diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 87b8302f2c5833..5ba6fd6df8e044 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3383,12 +3383,12 @@ void Compiler::compInitOptions(JitFlags* jitFlags) // Make sure we copy the register info and initialize the // trash regs after the underlying fields are initialized - const regMaskMixed vtCalleeTrashRegs[TYP_COUNT]{ + const regMaskOnlyOne vtCalleeTrashRegs[TYP_COUNT]{ #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP }; - memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskMixed) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskOnlyOne) * TYP_COUNT); codeGen->CopyRegisterInfo(); #endif // TARGET_XARCH @@ -9342,7 +9342,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX * cVN, dVN : Display a ValueNum (call vnPrint()). * * The following don't require a Compiler* to work: - * dRegMask : Display a regMaskMixed (call dspRegMask(mask)). + * dRegMask : Display a regMaskOnlyOne (call dspRegMask(mask)). * dBlockList : Display a BasicBlockList*. * * The following find an object in the IR and return it, as well as setting a global variable with the value that can @@ -10288,7 +10288,7 @@ JITDBGAPI void __cdecl dVN(ValueNum vn) cVN(JitTls::GetCompiler(), vn); } -JITDBGAPI void __cdecl dRegMask(regMaskMixed mask) +JITDBGAPI void __cdecl dRegMask(regMaskOnlyOne mask) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== dRegMask %u\n", sequenceNumber++); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 19aee8041bc5da..e63b3065951e12 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1028,33 +1028,6 @@ class LclVarDsc return regMask; } - regMaskOnlyOne lvRegMask(AllRegsMask& allRegsMask) const - { - regMaskOnlyOne regMask = RBM_NONE; - if (GetRegNum() != REG_STK) - { - if (varTypeUsesFloatReg(this)) - { - regMask = genRegMaskFloat(GetRegNum() ARM_ARG(TypeGet())); - allRegsMask.floatRegs = regMask; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(this)) - { - regMask = genRegMask(GetRegNum()); - allRegsMask.predicateRegs = regMask; - } -#endif - else - { - assert(varTypeUsesIntReg(this)); - regMask = genRegMask(GetRegNum()); - allRegsMask.gprRegs = regMask; - } - } - return regMask; - } - //----------------------------------------------------------------------------- // AllFieldDeathFlags: Get a bitset of flags that represents all fields dying. // @@ -3906,7 +3879,7 @@ class Compiler unsigned lvaGetMaxSpillTempSize(); #ifdef TARGET_ARM - bool lvaIsPreSpilled(unsigned lclNum, regMaskMixed preSpillMask); + bool lvaIsPreSpilled(unsigned lclNum, regMaskGpr preSpillMask); #endif // TARGET_ARM void lvaAssignFrameOffsets(FrameLayoutState curState); void lvaFixVirtualFrameOffsets(); @@ -11185,7 +11158,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX regMaskPredicate rbmAllMask; regMaskPredicate rbmMskCalleeTrash; unsigned cntCalleeTrashMask; - regMaskMixed varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskOnlyOne varTypeCalleeTrashRegs[TYP_COUNT]; public: FORCEINLINE regMaskPredicate get_RBM_ALLMASK() const diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 54b3e5a19b43f3..43a472b12a8fbf 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4445,9 +4445,16 @@ inline void* operator new[](size_t sz, Compiler* compiler, CompMemKind cmk) #ifdef DEBUG -inline void printRegMask(regMaskMixed mask) +inline void printRegMask(AllRegsMask mask) { + printf(REG_MASK_ALL_FMT, mask.gprRegs); + printf(" "); + printf(REG_MASK_ALL_FMT, mask.floatRegs); + +#ifdef HAS_PREDICATE_REGS + printf(" "); printf(REG_MASK_ALL_FMT, mask); +#endif // HAS_PREDICATE_REGS } inline char* regMaskToString(regMaskMixed mask, Compiler* context) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 587f3a802812ab..2a77f777d676f6 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -2919,10 +2919,10 @@ void* emitter::emitAddLabel(VARSET_VALARG_TP GCvars, dumpConvertedVarSet(emitComp, GCvars); printf(", gcrefRegs="); printRegMaskInt(gcrefRegs); - emitDispRegSet(gcrefRegs); + emitDispGprRegSet(gcrefRegs); printf(", byrefRegs="); printRegMaskInt(byrefRegs); - emitDispRegSet(byrefRegs); + emitDispGprRegSet(byrefRegs); printf("\n"); } #endif @@ -3472,14 +3472,12 @@ const char* emitter::emitGetFrameReg() * Display a register set in a readable form. */ -void emitter::emitDispRegSet(regMaskMixed regs) +void emitter::emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs) { regNumber reg; bool sp = false; - printf(" {"); - - for (reg = REG_FIRST; reg < ACTUAL_REG_COUNT; reg = REG_NEXT(reg)) + for (reg = firstReg; reg <= lastReg; reg = REG_NEXT(reg)) { if (regs == RBM_NONE) { @@ -3505,7 +3503,31 @@ void emitter::emitDispRegSet(regMaskMixed regs) printf("%s", emitRegName(reg)); } +} + +void emitter::emitDispGprRegSet(regMaskGpr regs) +{ + emitDispRegSet(REG_INT_FIRST, REG_INT_LAST, regs); +} + +void emitter::emitDispRegSet(AllRegsMask regs) +{ + printf(" {"); + if (regs.gprRegs != RBM_NONE) + { + emitDispGprRegSet(regs.gprRegs); + } + if (regs.floatRegs != RBM_NONE) + { + emitDispRegSet(REG_FP_FIRST, REG_FP_LAST, regs.floatRegs); + } +#ifdef HAS_PREDICATE_REGS + if (regs.predicateRegs != RBM_NONE) + { + emitDispRegSet(REG_MASK_FIRST, REG_MASK_LAST, regs.predicateRegs); + } +#endif printf("}"); } @@ -4098,10 +4120,10 @@ void emitter::emitDispIG(insGroup* ig, bool displayFunc, bool displayInstruction dumpConvertedVarSet(emitComp, igPh->igPhData->igPhPrevGCrefVars); printf(", PrevGCrefRegs="); printRegMaskInt(igPh->igPhData->igPhPrevGCrefRegs); - emitDispRegSet(igPh->igPhData->igPhPrevGCrefRegs); + emitDispGprRegSet(igPh->igPhData->igPhPrevGCrefRegs); printf(", PrevByrefRegs="); printRegMaskInt(igPh->igPhData->igPhPrevByrefRegs); - emitDispRegSet(igPh->igPhData->igPhPrevByrefRegs); + emitDispGprRegSet(igPh->igPhData->igPhPrevByrefRegs); printf("\n"); printf("%*s; InitGCVars=%s ", strlen(buff), "", @@ -4109,10 +4131,10 @@ void emitter::emitDispIG(insGroup* ig, bool displayFunc, bool displayInstruction dumpConvertedVarSet(emitComp, igPh->igPhData->igPhInitGCrefVars); printf(", InitGCrefRegs="); printRegMaskInt(igPh->igPhData->igPhInitGCrefRegs); - emitDispRegSet(igPh->igPhData->igPhInitGCrefRegs); + emitDispGprRegSet(igPh->igPhData->igPhInitGCrefRegs); printf(", InitByrefRegs="); printRegMaskInt(igPh->igPhData->igPhInitByrefRegs); - emitDispRegSet(igPh->igPhData->igPhInitByrefRegs); + emitDispGprRegSet(igPh->igPhData->igPhInitByrefRegs); printf("\n"); assert(!(ig->igFlags & IGF_GC_VARS)); @@ -4148,7 +4170,7 @@ void emitter::emitDispIG(insGroup* ig, bool displayFunc, bool displayInstruction { printf("%sgcrefRegs=", separator); printRegMaskInt(ig->igGCregs); - emitDispRegSet(ig->igGCregs); + emitDispGprRegSet(ig->igGCregs); separator = ", "; } @@ -4156,7 +4178,7 @@ void emitter::emitDispIG(insGroup* ig, bool displayFunc, bool displayInstruction { printf("%sbyrefRegs=", separator); printRegMaskInt(ig->igByrefRegs()); - emitDispRegSet(ig->igByrefRegs()); + emitDispGprRegSet(ig->igByrefRegs()); separator = ", "; } @@ -4252,26 +4274,26 @@ void emitter::emitDispGCinfo() dumpConvertedVarSet(emitComp, emitPrevGCrefVars); printf("\n emitPrevGCrefRegs(0x%p)=", dspPtr(&emitPrevGCrefRegs)); printRegMaskInt(emitPrevGCrefRegs); - emitDispRegSet(emitPrevGCrefRegs); + emitDispGprRegSet(emitPrevGCrefRegs); printf("\n emitPrevByrefRegs(0x%p)=", dspPtr(&emitPrevByrefRegs)); printRegMaskInt(emitPrevByrefRegs); - emitDispRegSet(emitPrevByrefRegs); + emitDispGprRegSet(emitPrevByrefRegs); printf("\n emitInitGCrefVars "); dumpConvertedVarSet(emitComp, emitInitGCrefVars); printf("\n emitInitGCrefRegs(0x%p)=", dspPtr(&emitInitGCrefRegs)); printRegMaskInt(emitInitGCrefRegs); - emitDispRegSet(emitInitGCrefRegs); + emitDispGprRegSet(emitInitGCrefRegs); printf("\n emitInitByrefRegs(0x%p)=", dspPtr(&emitInitByrefRegs)); printRegMaskInt(emitInitByrefRegs); - emitDispRegSet(emitInitByrefRegs); + emitDispGprRegSet(emitInitByrefRegs); printf("\n emitThisGCrefVars "); dumpConvertedVarSet(emitComp, emitThisGCrefVars); printf("\n emitThisGCrefRegs(0x%p)=", dspPtr(&emitThisGCrefRegs)); printRegMaskInt(emitThisGCrefRegs); - emitDispRegSet(emitThisGCrefRegs); + emitDispGprRegSet(emitThisGCrefRegs); printf("\n emitThisByrefRegs(0x%p)=", dspPtr(&emitThisByrefRegs)); printRegMaskInt(emitThisByrefRegs); - emitDispRegSet(emitThisByrefRegs); + emitDispGprRegSet(emitThisByrefRegs); printf("\n\n"); } @@ -8810,12 +8832,12 @@ void emitter::emitRecordGCcall(BYTE* codePos, unsigned char callInstrSize) emitDispVarSet(); printf(", gcrefRegs="); printRegMaskInt(emitThisGCrefRegs); - emitDispRegSet(emitThisGCrefRegs); + emitDispGprRegSet(emitThisGCrefRegs); // printRegMaskInt(emitThisGCrefRegs & ~RBM_INTRET & RBM_CALLEE_SAVED); // only display callee-saved // emitDispRegSet (emitThisGCrefRegs & ~RBM_INTRET & RBM_CALLEE_SAVED); // only display callee-saved printf(", byrefRegs="); printRegMaskInt(emitThisByrefRegs); - emitDispRegSet(emitThisByrefRegs); + emitDispGprRegSet(emitThisByrefRegs); // printRegMaskInt(emitThisByrefRegs & ~RBM_INTRET & RBM_CALLEE_SAVED); // only display callee-saved // emitDispRegSet (emitThisByrefRegs & ~RBM_INTRET & RBM_CALLEE_SAVED); // only display callee-saved printf("\n"); @@ -10386,7 +10408,7 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { printf("NoGC Call: savedSet="); printRegMaskInt(savedSet); - emitDispRegSet(savedSet); + emitDispGprRegSet(savedSet); printf("\n"); } #endif diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index a7b9e2f32c0c48..5d89bb89b26c91 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -338,8 +338,6 @@ struct insGroup unsigned igStkLvl; // stack level on entry #endif // EMIT_TRACK_STACK_DEPTH - regMaskSmall igGCregs; // set of registers with live GC refs - unsigned char igInsCnt; // # of instructions in this group VARSET_VALRET_TP igGCvars() const @@ -3193,7 +3191,9 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regMaskMixed regs); + void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); + void emitDispGprRegSet(regMaskGpr regs); + void emitDispRegSet(AllRegsMask regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 8260eb1e54b55a..59de9f6cec6ca1 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -4705,10 +4705,10 @@ void emitter::emitIns_Call(EmitCallType callType, dumpConvertedVarSet(emitComp, ptrVars); printf(", gcrefRegs="); printRegMaskInt(gcrefRegs); - emitDispRegSet(gcrefRegs); + emitDispGprRegSet(gcrefRegs); printf(", byrefRegs="); printRegMaskInt(byrefRegs); - emitDispRegSet(byrefRegs); + emitDispGprRegSet(byrefRegs); printf("\n"); } #endif @@ -6668,11 +6668,11 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) printf("Before emitOutputInstr for id->idDebugOnlyInfo()->idNum=0x%02x\n", id->idDebugOnlyInfo()->idNum); printf(" emitThisGCrefRegs(0x%p)=", dspPtr(&emitThisGCrefRegs)); printRegMaskInt(emitThisGCrefRegs); - emitDispRegSet(emitThisGCrefRegs); + emitDispGprRegSet(emitThisGCrefRegs); printf("\n"); printf(" emitThisByrefRegs(0x%p)=", dspPtr(&emitThisByrefRegs)); printRegMaskInt(emitThisByrefRegs); - emitDispRegSet(emitThisByrefRegs); + emitDispGprRegSet(emitThisByrefRegs); printf("\n"); } diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 04a3cce10e5339..6c3dd5768c4f79 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -16254,10 +16254,10 @@ void emitter::emitIns_Call(EmitCallType callType, dumpConvertedVarSet(emitComp, ptrVars); printf(", gcrefRegs="); printRegMaskInt(gcrefRegs); - emitDispRegSet(gcrefRegs); + emitDispGprRegSet(gcrefRegs); printf(", byrefRegs="); printRegMaskInt(byrefRegs); - emitDispRegSet(byrefRegs); + emitDispGprRegSet(byrefRegs); printf("\n"); } #endif diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 0fbaabcf4eecab..56d120e9f0e2b4 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9529,10 +9529,10 @@ void emitter::emitIns_Call(EmitCallType callType, dumpConvertedVarSet(emitComp, ptrVars); printf(", gcrefRegs="); printRegMaskInt(gcrefRegs); - emitDispRegSet(gcrefRegs); + emitDispGprRegSet(gcrefRegs); printf(", byrefRegs="); printRegMaskInt(byrefRegs); - emitDispRegSet(byrefRegs); + emitDispGprRegSet(byrefRegs); printf("\n"); } #endif @@ -17923,11 +17923,11 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) printf("Before emitOutputInstr for id->idDebugOnlyInfo()->idNum=0x%02x\n", id->idDebugOnlyInfo()->idNum); printf(" emitThisGCrefRegs(0x%p)=", emitComp->dspPtr(&emitThisGCrefRegs)); printRegMaskInt(emitThisGCrefRegs); - emitDispRegSet(emitThisGCrefRegs); + emitDispGprRegSet(emitThisGCrefRegs); printf("\n"); printf(" emitThisByrefRegs(0x%p)=", emitComp->dspPtr(&emitThisByrefRegs)); printRegMaskInt(emitThisByrefRegs); - emitDispRegSet(emitThisByrefRegs); + emitDispGprRegSet(emitThisByrefRegs); printf("\n"); } diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 0c9320345eb13e..2e48c95a2726e7 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -83,7 +83,7 @@ void GCInfo::gcResetForBB() * Print the changes in the gcRegGCrefSetCur sets. */ -void GCInfo::gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspGCrefSetChanges(regMaskGpr gcRegGCrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -97,11 +97,11 @@ void GCInfo::gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool fo else { printRegMaskInt(gcRegGCrefSetCur); - compiler->GetEmitter()->emitDispRegSet(gcRegGCrefSetCur); + compiler->GetEmitter()->emitDispGprRegSet(gcRegGCrefSetCur); printf(" => "); } printRegMaskInt(gcRegGCrefSetNew); - compiler->GetEmitter()->emitDispRegSet(gcRegGCrefSetNew); + compiler->GetEmitter()->emitDispGprRegSet(gcRegGCrefSetNew); printf("\n"); } } @@ -112,7 +112,7 @@ void GCInfo::gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool fo * Print the changes in the gcRegByrefSetCur sets. */ -void GCInfo::gcDspByrefSetChanges(regMaskMixed gcRegByrefSetNew DEBUGARG(bool forceOutput)) +void GCInfo::gcDspByrefSetChanges(regMaskGpr gcRegByrefSetNew DEBUGARG(bool forceOutput)) { if (compiler->verbose) { @@ -126,11 +126,11 @@ void GCInfo::gcDspByrefSetChanges(regMaskMixed gcRegByrefSetNew DEBUGARG(bool fo else { printRegMaskInt(gcRegByrefSetCur); - compiler->GetEmitter()->emitDispRegSet(gcRegByrefSetCur); + compiler->GetEmitter()->emitDispGprRegSet(gcRegByrefSetCur); printf(" => "); } printRegMaskInt(gcRegByrefSetNew); - compiler->GetEmitter()->emitDispRegSet(gcRegByrefSetNew); + compiler->GetEmitter()->emitDispGprRegSet(gcRegByrefSetNew); printf("\n"); } } @@ -194,8 +194,8 @@ void GCInfo::gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput)) /* NOTE: don't unmark any live register variables */ - regMaskMixed gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetMaskVars()); - regMaskMixed gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetMaskVars()); + regMaskGpr gcRegByrefSetNew = gcRegByrefSetCur & ~(regMask & ~regSet->GetGprMaskVars()); + regMaskGpr gcRegGCrefSetNew = gcRegGCrefSetCur & ~(regMask & ~regSet->GetGprMaskVars()); INDEBUG(gcDspGCrefSetChanges(gcRegGCrefSetNew, forceOutput)); INDEBUG(gcDspByrefSetChanges(gcRegByrefSetNew, forceOutput)); @@ -705,6 +705,7 @@ void GCInfo::gcRegPtrSetInit() #endif // JIT32_GCENCODER +#if 0 //------------------------------------------------------------------------ // gcUpdateForRegVarMove: Update the masks when a variable is moved // @@ -771,6 +772,6 @@ void GCInfo::gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, L VarSetOps::AddElemD(compiler, gcVarPtrSetCur, varDsc->lvVarIndex); } } - +#endif /*****************************************************************************/ /*****************************************************************************/ diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index 027c1f166fdc73..eadd14db3a8241 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -95,8 +95,8 @@ class GCInfo void gcMarkRegPtrVal(regNumber reg, var_types type); #ifdef DEBUG - void gcDspGCrefSetChanges(regMaskMixed gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); - void gcDspByrefSetChanges(regMaskMixed gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspGCrefSetChanges(regMaskGpr gcRegGCrefSetNew DEBUGARG(bool forceOutput = false)); + void gcDspByrefSetChanges(regMaskGpr gcRegByrefSetNew DEBUGARG(bool forceOutput = false)); #endif // DEBUG /*****************************************************************************/ @@ -390,9 +390,11 @@ class GCInfo #endif // JIT32_GCENCODER #endif // DUMP_GC_TABLES +#if 0 public: // This method updates the appropriate reg masks when a variable is moved. void gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, LclVarDsc* varDsc); +#endif private: ReturnKind getReturnKind(); diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 4225b95a129586..8dd7cf68d22332 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -5354,8 +5354,9 @@ void Compiler::lvaFixVirtualFrameOffsets() } #ifdef TARGET_ARM -bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskMixed preSpillMask) +bool Compiler::lvaIsPreSpilled(unsigned lclNum, regMaskGpr preSpillMask) { + assert(IsGprRegMask(preSpillMask)); const LclVarDsc& desc = lvaTable[lclNum]; return desc.lvIsRegArg && (preSpillMask & genRegMask(desc.GetArgReg())); } @@ -5552,8 +5553,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() unsigned argLcls = 0; // Take care of pre spill registers first. - regMaskMixed preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); - regMaskMixed tempMask = RBM_NONE; + regMaskGpr preSpillMask = codeGen->regSet.rsMaskPreSpillRegs(false); + regMaskGpr tempMask = RBM_NONE; for (unsigned i = 0, preSpillLclNum = lclNum; i < argSigLen; ++i, ++preSpillLclNum) { if (lvaIsPreSpilled(preSpillLclNum, preSpillMask)) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index f568f38f7590c4..5c3f644a28be07 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4092,7 +4092,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskMixed candidateRegs = allRegs(interval->registerType) & availableRegs; + regMaskOnlyOne candidateRegs = allRegs(interval->registerType) & availableRegs; regNumber firstReg = REG_NA; regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 05a6e8c56602f3..37d60c3198d8f6 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -346,34 +346,34 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) rsModifiedGprRegsMask &= ~mask; } -void RegSet::SetMaskVars(regMaskMixed newMaskVars) +void RegSet::SetMaskVars(AllRegsMask newMaskVars) { #ifdef DEBUG if (m_rsCompiler->verbose) { printf("\t\t\t\t\t\t\tLive regs: "); - if (_rsMaskVars == newMaskVars) + if (_rsAllMaskVars == newMaskVars) { printf("(unchanged) "); } else { - printRegMask(_rsMaskVars); - m_rsCompiler->GetEmitter()->emitDispRegSet(_rsMaskVars); + printRegMask(_rsAllMaskVars); + m_rsCompiler->GetEmitter()->emitDispRegSet(_rsAllMaskVars); // deadSet = old - new - regMaskMixed deadSet = _rsMaskVars & ~newMaskVars; + AllRegsMask deadSet = _rsAllMaskVars & ~newMaskVars; // bornSet = new - old - regMaskMixed bornSet = newMaskVars & ~_rsMaskVars; + AllRegsMask bornSet = newMaskVars & ~_rsAllMaskVars; - if (deadSet != RBM_NONE) + if (!deadSet.IsEmpty()) { printf(" -"); m_rsCompiler->GetEmitter()->emitDispRegSet(deadSet); } - if (bornSet != RBM_NONE) + if (!bornSet.IsEmpty()) { printf(" +"); m_rsCompiler->GetEmitter()->emitDispRegSet(bornSet); @@ -387,7 +387,7 @@ void RegSet::SetMaskVars(regMaskMixed newMaskVars) } #endif // DEBUG - _rsMaskVars = newMaskVars; + _rsAllMaskVars = newMaskVars; } /*****************************************************************************/ diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index b9fca9ee176f20..df0ced5567e606 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -43,7 +43,7 @@ class RegSet RegSet(Compiler* compiler, GCInfo& gcInfo); #ifdef TARGET_ARM - regMaskMixed rsMaskPreSpillRegs(bool includeAlignment) const + regMaskGpr rsMaskPreSpillRegs(bool includeAlignment) const { return includeAlignment ? (rsMaskPreSpillRegArg | rsMaskPreSpillAlign) : rsMaskPreSpillRegArg; } @@ -143,31 +143,93 @@ class RegSet void verifyRegistersUsed(AllRegsMask mask); public: - regMaskMixed GetMaskVars() const // 'get' property function for rsMaskVars property + regMaskOnlyOne GetMaskVars(var_types type) const // 'get' property function for rsMaskVars property { - return _rsMaskVars; + if (varTypeRegister[type] == VTR_INT) + { + return _rsAllMaskVars.gprRegs; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + return _rsAllMaskVars.predicateRegs; + } +#endif + else + { + assert(varTypeRegister[type] == VTR_FLOAT); + return _rsAllMaskVars.floatRegs; + } + } + + regMaskGpr GetGprMaskVars() const // 'get' property function for rsMaskVars property + { + return _rsAllMaskVars.gprRegs; } - void SetMaskVars(regMaskMixed newMaskVars); // 'put' property function for rsMaskVars property + void SetMaskVars(AllRegsMask newMaskVars); // 'put' property function for rsMaskVars property - void AddMaskVars(regMaskMixed addMaskVars) // union 'addMaskVars' with the rsMaskVars set + void AddMaskVars(var_types type, regMaskOnlyOne addMaskVars) // union 'addMaskVars' with the rsMaskVars set { - SetMaskVars(_rsMaskVars | addMaskVars); + AllRegsMask newMask = _rsAllMaskVars; + if (varTypeRegister[type] == VTR_INT) + { + newMask.gprRegs |= addMaskVars; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + // TODO: If we never hit this assert, then just convert _rsAllMaskVars + // to regMaskGpr + assert(false); + newMask.predicateRegs |= addMaskVars; + } +#endif + else + { + // TODO: If we never hit this assert, then just convert _rsAllMaskVars + // to regMaskGpr + assert(false); + assert(varTypeRegister[type] == VTR_FLOAT); + newMask.floatRegs |= addMaskVars; + } + SetMaskVars(newMask); } - void RemoveMaskVars(regMaskMixed removeMaskVars) // remove 'removeMaskVars' from the rsMaskVars set (like bitset - // DiffD) + // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) + void RemoveMaskVars(var_types type, regMaskOnlyOne removeMaskVars) { - SetMaskVars(_rsMaskVars & ~removeMaskVars); + AllRegsMask newMask = _rsAllMaskVars; + if (varTypeRegister[type] == VTR_INT) + { + newMask.gprRegs = ~removeMaskVars; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + // TODO: If we never hit this assert, then just convert _rsAllMaskVars + // to regMaskGpr + assert(false); + newMask.predicateRegs = ~removeMaskVars; + } +#endif + else + { + // TODO: If we never hit this assert, then just convert _rsAllMaskVars + // to regMaskGpr + assert(false); + assert(varTypeRegister[type] == VTR_FLOAT); + newMask.floatRegs = ~removeMaskVars; + } + SetMaskVars(newMask); } void ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. { - _rsMaskVars = RBM_NONE; + _rsAllMaskVars = AllRegsMask(); } private: - regMaskMixed _rsMaskVars; // backing store for rsMaskVars property AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property #if defined(TARGET_ARMARCH) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index cf6edde2cfb713..6088394478bedf 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -337,6 +337,24 @@ typedef struct _regMaskAll ; } + void AddRegTypeMask(var_types type, regMaskOnlyOne maskToAdd) + { + if (varTypeRegister[type] == VTR_INT) + { + gprRegs |= maskToAdd; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + predicateRegs |= maskToAdd; + } +#endif + else + { + assert(varTypeRegister[type] == VTR_FLOAT); + floatRegs |= maskToAdd; + } + } regMaskOnlyOne GetRegTypeMask(var_types type) { if (varTypeRegister[type] == VTR_INT) @@ -720,7 +738,7 @@ inline singleRegMask genRegMask(regNumber reg) // (L1 latency on sandy bridge is 4 cycles for [base] and 5 for [base + index*c] ) // the reason this is AMD-only is because the x86 BE will try to get reg masks for REG_STK // and the result needs to be zero. - regMaskMixed result = 1ULL << reg; + singleRegMask result = 1ULL << reg; assert(result == regMasks[reg]); return result; #else @@ -779,19 +797,17 @@ inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TY inline regMaskOnlyOne genRegMask(regNumber regNum, var_types type) { #if defined(TARGET_ARM) - regMaskMixed regMask = RBM_NONE; - if (varTypeUsesIntReg(type)) { - regMask = genRegMask(regNum); + return genRegMask(regNum); } else { assert(varTypeUsesFloatReg(type)); - regMask = genRegMaskFloat(regNum, type); + return genRegMaskFloat(regNum, type); } - return regMask; + return RBM_NONE; #else return genRegMask(regNum); #endif From 16cdcf91c5a5f75d914a7037d59c71117107b6c2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 16:44:24 -0800 Subject: [PATCH 064/201] Remove regMaskMixed from emitxarch.cpp --- src/coreclr/jit/compiler.hpp | 10 ---------- src/coreclr/jit/emitxarch.cpp | 3 ++- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 43a472b12a8fbf..de84bc046b84e5 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4457,16 +4457,6 @@ inline void printRegMask(AllRegsMask mask) #endif // HAS_PREDICATE_REGS } -inline char* regMaskToString(regMaskMixed mask, Compiler* context) -{ - const size_t cchRegMask = 24; - char* regmask = new (context, CMK_Unknown) char[cchRegMask]; - - sprintf_s(regmask, cchRegMask, REG_MASK_ALL_FMT, mask); - - return regmask; -} - inline void printRegMaskInt(regMaskGpr mask) { printf(REG_MASK_INT_FMT, (mask & RBM_ALLINT)); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 56d120e9f0e2b4..f5202a007bdf6b 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -15026,8 +15026,9 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) // instruction, if writing a GC ref even through reading a long, will go live here. // These situations typically occur due to unsafe casting, such as with Span. - regMaskMixed regMask; + regMaskGpr regMask; regMask = genRegMask(reg1) | genRegMask(reg2); + assert(emitComp->IsGprRegMask(regMask)); // r1/r2 could have been a GCREF as GCREF + int=BYREF // or BYREF+/-int=BYREF From 2849c8d7232050b64ec6e72521c8a6bb28ca22d6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 16:45:57 -0800 Subject: [PATCH 065/201] Remove regMaskMixed from *gcinfo.* --- src/coreclr/jit/gcinfo.cpp | 2 +- src/coreclr/jit/jitgcinfo.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 2e48c95a2726e7..de1726149d187b 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -722,7 +722,7 @@ void GCInfo::gcRegPtrSetInit() // It is also called by LinearScan::recordVarLocationAtStartOfBB() which is in turn called by // CodeGen::genCodeForBBList() at the block boundary. -void GCInfo::gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, LclVarDsc* varDsc) +void GCInfo::gcUpdateForRegVarMove(regMaskOnlyOne srcMask, regMaskOnlyOne dstMask, LclVarDsc* varDsc) { var_types type = varDsc->TypeGet(); bool isGCRef = (type == TYP_REF); diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index eadd14db3a8241..d513bfe9bdc567 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -393,7 +393,7 @@ class GCInfo #if 0 public: // This method updates the appropriate reg masks when a variable is moved. - void gcUpdateForRegVarMove(regMaskMixed srcMask, regMaskMixed dstMask, LclVarDsc* varDsc); + void gcUpdateForRegVarMove(regMaskOnlyOne srcMask, regMaskOnlyOne dstMask, LclVarDsc* varDsc); #endif private: From eeb173a4fb8f25d44af653057e03ebc5f3c1ded0 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 21:32:01 -0800 Subject: [PATCH 066/201] Remove regMaskMixed from TempReg() --- src/coreclr/jit/gentree.cpp | 12 +++++++++--- src/coreclr/jit/gentree.h | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 602c7ae28db81a..98d76554b4592d 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27339,8 +27339,10 @@ regMaskMixed ReturnTypeDesc::GetABIReturnRegs() const // Return Value: // Count of available temporary registers in given set. // -unsigned GenTree::AvailableTempRegCount(regMaskMixed mask /* = (regMaskMixed)-1 */) const +unsigned GenTree::AvailableTempRegCount(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) const { + assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); + return genCountBits(gtRsvdRegs & mask); } @@ -27356,8 +27358,10 @@ unsigned GenTree::AvailableTempRegCount(regMaskMixed mask /* = (regMaskMixed)-1 // Return Value: // Available temporary register in given mask. // -regNumber GenTree::GetSingleTempReg(regMaskMixed mask /* = (regMaskMixed)-1 */) +regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskMixed)-1 */) { + assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); + regMaskMixed availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); regNumber tempReg = genRegNumFromMask(availableSet); @@ -27377,8 +27381,10 @@ regNumber GenTree::GetSingleTempReg(regMaskMixed mask /* = (regMaskMixed)-1 */) // Return Value: // Available temporary register in given mask. // -regNumber GenTree::ExtractTempReg(regMaskMixed mask /* = (regMaskMixed)-1 */) +regNumber GenTree::ExtractTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) { + assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); + regMaskMixed availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); regNumber tempReg = genFirstRegNumFromMask(availableSet); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 3f6c10a79ee4f4..4d1cc3d2278e90 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -970,9 +970,9 @@ struct GenTree regMaskOnlyOne gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskMixed mask = (regMaskMixed)-1) const; - regNumber GetSingleTempReg(regMaskMixed mask = (regMaskMixed)-1); - regNumber ExtractTempReg(regMaskMixed mask = (regMaskMixed)-1); + unsigned AvailableTempRegCount(regMaskOnlyOne mask = (regMaskOnlyOne)-1) const; + regNumber GetSingleTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); + regNumber ExtractTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); void SetVNsFromNode(GenTree* tree) { From 500b78a6783cc49d34ef7f996387a3de9fd26783 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 21:41:32 -0800 Subject: [PATCH 067/201] Handled unwind.cpp --- src/coreclr/jit/codegenarmarch.cpp | 4 ++-- src/coreclr/jit/unwind.cpp | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index ab9664bd766618..85490e74e7234b 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3579,7 +3579,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { regMaskGpr trashedGprByEpilog = RBM_INT_CALLEE_SAVED; regMaskFloat trashedFloatByEpilog = RBM_FLT_CALLEE_SAVED; -#if HAS_PREDICATE_REGS +#ifdef HAS_PREDICATE_REGS regMaskPredicate trashedPredicateByEpilog = RBM_MSK_CALLEE_SAVED; #endif // HAS_PREDICATE_REGS @@ -3610,7 +3610,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) JITDUMP("Float Register used: %s\n", getRegName(reg)); assert(!"Argument to tailcall may be trashed by epilog"); } -#if HAS_PREDICATE_REGS +#ifdef HAS_PREDICATE_REGS else if ((trashedPredicateByEpilog & genRegMask(reg)) != 0) { JITDUMP("Tail call node:\n"); diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 64025b6a1de5de..359c606bb147cf 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -152,7 +152,6 @@ void Compiler::unwindPushPopCFI(regNumber reg) FuncInfoDsc* func = funCurrentFunc(); UNATIVE_OFFSET cbProlog = unwindGetCurrentOffset(func); - regMaskMixed relOffsetMask = RBM_CALLEE_SAVED #if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) // is excluded from the callee-save register list. @@ -172,7 +171,14 @@ void Compiler::unwindPushPopCFI(regNumber reg) assert(reg < REG_FP_FIRST); createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, REGSIZE_BYTES); #endif - if (relOffsetMask & genRegMask(reg)) + + singleRegMask mask = genRegMask(reg); + if ((emitter::isIntegerRegister(reg) && (RBM_INT_CALLEE_SAVED & mask)) || + (emitter::isFloatReg(reg) && (RBM_FLT_CALLEE_SAVED & mask)) +#ifdef HAS_PREDICATE_REGS + || (emitter::isPredicateRegister(reg) && (RBM_MSK_CALLEE_SAVED & mask)) +#endif // HAS_PREDICATE_REGS + ) { createCfiCode(func, cbProlog, CFI_REL_OFFSET, mapRegNumToDwarfReg(reg)); } From ae53e67fbbde5ca2da7bfdb8ae81e28b5f9f6c98 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Feb 2024 21:59:11 -0800 Subject: [PATCH 068/201] Remove regMaskMixed from misc places --- src/coreclr/jit/gentree.cpp | 6 +++--- src/coreclr/jit/lsra.cpp | 2 +- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/utils.cpp | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 98d76554b4592d..c39b94f7f24535 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27358,11 +27358,11 @@ unsigned GenTree::AvailableTempRegCount(regMaskOnlyOne mask /* = (regMaskOnlyOne // Return Value: // Available temporary register in given mask. // -regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskMixed)-1 */) +regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) { assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); - regMaskMixed availableSet = gtRsvdRegs & mask; + regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); regNumber tempReg = genRegNumFromMask(availableSet); INDEBUG(gtRsvdRegs &= ~availableSet;) // Remove the register from the set, so it can't be used again. @@ -27385,7 +27385,7 @@ regNumber GenTree::ExtractTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */ { assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); - regMaskMixed availableSet = gtRsvdRegs & mask; + regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); regNumber tempReg = genFirstRegNumFromMask(availableSet); gtRsvdRegs ^= genRegMask(tempReg); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 5c3f644a28be07..57217f81148bd7 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -12512,7 +12512,7 @@ void LinearScan::RegisterSelection::try_CONST_AVAILABLE() if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) { - regMaskMixed newCandidates = candidates & matchingConstants; + regMaskOnlyOne newCandidates = candidates & matchingConstants; if (newCandidates != RBM_NONE) { candidates = newCandidates; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 4f76141070efbd..2e82ebdcb71228 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -636,7 +636,7 @@ class LinearScan : public LinearScanInterface // This is the main driver virtual PhaseStatus doLinearScan(); - static bool isSingleRegister(regMaskMixed regMask) + static bool isSingleRegister(regMaskOnlyOne regMask) { return (genExactlyOneBit(regMask)); } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 37d60c3198d8f6..5cd191c6398601 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -1103,7 +1103,7 @@ regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) regMaskSmall res = 0; for (int i = 0; i < CNT_CALLEE_SAVED; i++) { - if ((calleeSaveMask & ((regMaskMixed)1 << i)) != 0) + if ((calleeSaveMask & ((regMaskOnlyOne)1 << i)) != 0) { res |= raRbmCalleeSaveOrder[i]; } diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index aa43707e42310a..75cf9c162d862e 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -282,9 +282,9 @@ const char* getRegNameFloat(regNumber reg, var_types type) * Displays a range of registers * -- This is a helper used by dspRegMask */ -const char* dspRegRange(regMaskMixed regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) +const char* dspRegRange(regMaskOnlyOne regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) { -#ifdef TARGET_XARCH +#ifdef HAS_PREDICATE_REGS assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || ((regFirst == REG_FP_FIRST) && (regLast == REG_FP_LAST)) || ((regFirst == REG_MASK_FIRST) && (regLast == REG_MASK_LAST))); From 03eab41efa4fadb0d998808facbb481321c31fa6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 1 Mar 2024 16:36:26 -0800 Subject: [PATCH 069/201] fix unwind build error --- src/coreclr/jit/gentree.cpp | 6 ----- src/coreclr/jit/unwind.cpp | 44 +++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index c39b94f7f24535..01068b77f5d1da 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27341,8 +27341,6 @@ regMaskMixed ReturnTypeDesc::GetABIReturnRegs() const // unsigned GenTree::AvailableTempRegCount(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) const { - assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); - return genCountBits(gtRsvdRegs & mask); } @@ -27360,8 +27358,6 @@ unsigned GenTree::AvailableTempRegCount(regMaskOnlyOne mask /* = (regMaskOnlyOne // regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) { - assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); - regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); regNumber tempReg = genRegNumFromMask(availableSet); @@ -27383,8 +27379,6 @@ regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 // regNumber GenTree::ExtractTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */) { - assert((mask == RBM_NONE) || ((mask & RBM_ALLFLOAT) == mask) != ((mask & ~RBM_ALLFLOAT) == mask)); - regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); regNumber tempReg = genFirstRegNumFromMask(availableSet); diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 359c606bb147cf..e2b8c57af0e0ff 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -152,17 +152,8 @@ void Compiler::unwindPushPopCFI(regNumber reg) FuncInfoDsc* func = funCurrentFunc(); UNATIVE_OFFSET cbProlog = unwindGetCurrentOffset(func); -#if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED - // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) - // is excluded from the callee-save register list. - // Make sure the register gets PUSH unwind info in this case, - // since it is pushed as a frame register. - | RBM_FPBASE -#endif -#if defined(TARGET_ARM) - | RBM_R11 | RBM_LR | RBM_PC -#endif - ; + regMaskOnlyOne mask = genRegMask(reg); + #if defined(TARGET_ARM) createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, @@ -172,13 +163,34 @@ void Compiler::unwindPushPopCFI(regNumber reg) createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, REGSIZE_BYTES); #endif - singleRegMask mask = genRegMask(reg); - if ((emitter::isIntegerRegister(reg) && (RBM_INT_CALLEE_SAVED & mask)) || - (emitter::isFloatReg(reg) && (RBM_FLT_CALLEE_SAVED & mask)) + bool shouldCreateCfiCode = false; + if (emitter::isGeneralRegister(reg)) + { + +#if defined(UNIX_AMD64_ABI) && ETW_EBP_FRAMED + // In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP) + // is excluded from the callee-save register list. + // Make sure the register gets PUSH unwind info in this case, + // since it is pushed as a frame register. + mask |= RBM_FPBASE; +#endif +#if defined(TARGET_ARM) + mask |= RBM_R11 | RBM_LR | RBM_PC; +#endif + shouldCreateCfiCode = (RBM_INT_CALLEE_SAVED & mask); + } + else if (emitter::isFloatReg(reg)) + { + shouldCreateCfiCode = (RBM_FLT_CALLEE_SAVED & mask); + } #ifdef HAS_PREDICATE_REGS - || (emitter::isPredicateRegister(reg) && (RBM_MSK_CALLEE_SAVED & mask)) + else if (emitter::isMaskReg(reg) && (RBM_MSK_CALLEE_SAVED & mask)) + { + shouldCreateCfiCode = (RBM_MSK_CALLEE_SAVED & mask); + } #endif // HAS_PREDICATE_REGS - ) + + if (shouldCreateCfiCode) { createCfiCode(func, cbProlog, CFI_REL_OFFSET, mapRegNumToDwarfReg(reg)); } From 729debfdcf4486df03a7d882cf15de7e1de5fda4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 4 Mar 2024 10:14:23 -0800 Subject: [PATCH 070/201] fix the handling of RemoveMaskVars() --- src/coreclr/jit/regset.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index df0ced5567e606..e9f21a983a7aa2 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -187,9 +187,6 @@ class RegSet #endif else { - // TODO: If we never hit this assert, then just convert _rsAllMaskVars - // to regMaskGpr - assert(false); assert(varTypeRegister[type] == VTR_FLOAT); newMask.floatRegs |= addMaskVars; } @@ -202,7 +199,7 @@ class RegSet AllRegsMask newMask = _rsAllMaskVars; if (varTypeRegister[type] == VTR_INT) { - newMask.gprRegs = ~removeMaskVars; + newMask.gprRegs &= ~removeMaskVars; } #ifdef HAS_PREDICATE_REGS else if (varTypeRegister[type] == VTR_MASK) @@ -210,16 +207,13 @@ class RegSet // TODO: If we never hit this assert, then just convert _rsAllMaskVars // to regMaskGpr assert(false); - newMask.predicateRegs = ~removeMaskVars; + newMask.predicateRegs &= ~removeMaskVars; } #endif else { - // TODO: If we never hit this assert, then just convert _rsAllMaskVars - // to regMaskGpr - assert(false); assert(varTypeRegister[type] == VTR_FLOAT); - newMask.floatRegs = ~removeMaskVars; + newMask.floatRegs &= ~removeMaskVars; } SetMaskVars(newMask); } From 951e9c66a8ce9a0d82e577610464ccd3e004efa4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 4 Mar 2024 17:18:51 -0800 Subject: [PATCH 071/201] Fix makeRegAvailable and setRegInUse for registerType/reg inconsistency --- src/coreclr/jit/lsra.h | 104 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 2e82ebdcb71228..19431e066ec608 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1806,8 +1806,31 @@ class LinearScan : public LinearScanInterface bool isRegAvailable(regNumber reg, var_types regType) // only used in asserts { regMaskOnlyOne regMask = getRegMask(reg, regType); - return (m_AvailableRegs[regTypeIndex(regType)] & regMask) == regMask; + + #ifdef TARGET_ARM64 + if (emitter::isGeneralRegisterOrZR(reg)) +#else + if (emitter::isGeneralRegister(reg)) +#endif + { + return (m_AvailableRegs[0] & regMask) == regMask; + } + else if (emitter::isFloatReg(reg)) + { + return (m_AvailableRegs[1] & regMask) == regMask; + } + + else + { +#ifdef HAS_PREDICATE_REGS + assert(emitter::isMaskReg(reg)); + return (m_AvailableRegs[2] & regMask) == regMask; +#else + unreached(); +#endif + } } + void setRegsInUse(regMaskMixed regMask) { //TODO: Fix this later. @@ -1816,28 +1839,97 @@ class LinearScan : public LinearScanInterface #ifdef HAS_PREDICATE_REGS m_AvailableRegs[2] &= ~(regMask & RBM_ALLMASK); #endif + + assert(compiler->IsGprRegMask(m_AvailableRegs[0])); + assert(compiler->IsFloatRegMask(m_AvailableRegs[1])); } void setRegInUse(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - m_AvailableRegs[regTypeIndex(regType)] &= ~regMask; + +#ifdef TARGET_ARM64 + if (emitter::isGeneralRegisterOrZR(reg)) +#else + if (emitter::isGeneralRegister(reg)) +#endif + { + m_AvailableRegs[0] &= ~regMask; + } + else if (emitter::isFloatReg(reg)) + { + m_AvailableRegs[1] &= ~regMask; + } +#ifdef HAS_PREDICATE_REGS + else + { + assert(emitter::isMaskReg(reg)); + m_AvailableRegs[2] &= ~regMask; + } +#endif } void makeRegsAvailable(regMaskMixed regMask) { // TODO: This will be just `regMask` - makeRegAvailable(regMask & ~RBM_ALLFLOAT, IntRegisterType); - makeRegAvailable(regMask & RBM_ALLFLOAT, FloatRegisterType); + m_AvailableRegs[0] |= regMask & ~RBM_ALLFLOAT; + m_AvailableRegs[1] |= regMask & RBM_ALLFLOAT; #ifdef HAS_PREDICATE_REGS - makeRegAvailable(regMask & RBM_ALLMASK, MaskRegisterType); + m_AvailableRegs[2] |= regMask & RBM_ALLMASK; #endif + } void makeRegAvailable(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - makeRegAvailable(regMask, regType); +#ifdef TARGET_ARM64 + // TODO: Convert it in to a lookup array that maps register number + // to '0', '1' or '2'. So something like this: + // rax - 0 + // rbx - 0 + // ... + // xmm0 - 1 + // here it will be, m_AvailableRegs[regTypeIndex[reg]] = + if (emitter::isGeneralRegisterOrZR(reg)) +#else + if (emitter::isGeneralRegister(reg)) +#endif + { + m_AvailableRegs[0] |= regMask; + } + else if (emitter::isFloatReg(reg)) + { + m_AvailableRegs[1] |= regMask; + } + + else + { +#ifdef HAS_PREDICATE_REGS + assert(emitter::isMaskReg(reg)); + m_AvailableRegs[2] |= regMask; +#else + unreached(); +#endif + } } + void makeRegAvailable(regMaskOnlyOne regMask, var_types regType) { +#ifdef DEBUG + if (varTypeUsesIntReg(regType)) + { + assert(compiler->IsGprRegMask(regMask)); + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeUsesMaskReg(regType)) + { + assert(compiler->IsPredicateRegMask(regMask)); + } +#endif // HAS_PREDICATE_REGS + else + { + assert(varTypeUsesFloatReg(regType)); + assert(compiler->IsFloatRegMask(regMask)); + } +#endif m_AvailableRegs[regTypeIndex(regType)] |= regMask; } From c66b485dda0bc903fb1af03fc9bf0ed4897a1885 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 4 Mar 2024 17:41:02 -0800 Subject: [PATCH 072/201] fix some printing the dumps --- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/emit.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index de84bc046b84e5..0f9531ace38ac4 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4453,7 +4453,7 @@ inline void printRegMask(AllRegsMask mask) #ifdef HAS_PREDICATE_REGS printf(" "); - printf(REG_MASK_ALL_FMT, mask); + printf(REG_MASK_ALL_FMT, mask.predicateRegs); #endif // HAS_PREDICATE_REGS } diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 2a77f777d676f6..93c4adeb545dfe 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3474,6 +3474,8 @@ const char* emitter::emitGetFrameReg() void emitter::emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs) { + printf(" {"); + regNumber reg; bool sp = false; @@ -3503,6 +3505,7 @@ void emitter::emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyO printf("%s", emitRegName(reg)); } + printf("}"); } void emitter::emitDispGprRegSet(regMaskGpr regs) @@ -3512,8 +3515,6 @@ void emitter::emitDispGprRegSet(regMaskGpr regs) void emitter::emitDispRegSet(AllRegsMask regs) { - printf(" {"); - if (regs.gprRegs != RBM_NONE) { emitDispGprRegSet(regs.gprRegs); @@ -3528,7 +3529,6 @@ void emitter::emitDispRegSet(AllRegsMask regs) emitDispRegSet(REG_MASK_FIRST, REG_MASK_LAST, regs.predicateRegs); } #endif - printf("}"); } /***************************************************************************** From 2017d5265965a42c3111b12b85971a5bc56d347f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 6 Mar 2024 12:05:24 -0800 Subject: [PATCH 073/201] Add support for BuildKills/BuildDefs/BuildCall --- src/coreclr/jit/gentree.cpp | 23 ++++- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/lsra.h | 11 ++- src/coreclr/jit/lsraarm.cpp | 8 +- src/coreclr/jit/lsraarm64.cpp | 8 +- src/coreclr/jit/lsraarmarch.cpp | 56 +++++++++-- src/coreclr/jit/lsrabuild.cpp | 170 +++++++++++++++++++++++++------- src/coreclr/jit/lsraxarch.cpp | 42 +++++--- 8 files changed, 249 insertions(+), 71 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 01068b77f5d1da..09dabb35b318b3 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27312,14 +27312,31 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx) const // of return registers and wants to know the set of return registers. // // static -regMaskMixed ReturnTypeDesc::GetABIReturnRegs() const +AllRegsMask ReturnTypeDesc::GetABIReturnRegs() const { - regMaskMixed resultMask = RBM_NONE; + AllRegsMask resultMask; unsigned count = GetReturnRegCount(); for (unsigned i = 0; i < count; ++i) { - resultMask |= genRegMask(GetABIReturnReg(i)); + regNumber reg = GetABIReturnReg(i); + if ((reg >= REG_INT_FIRST) && (reg <= REG_INT_LAST)) + { + resultMask.gprRegs |= genRegMask(reg); + } + else if ((reg >= REG_FP_FIRST) && (reg <= REG_FP_LAST)) + { + resultMask.floatRegs |= genRegMask(reg); + } + else + { +#ifdef HAS_PREDICATE_REGS + assert((reg >= REG_MASK_FIRST) && (reg <= REG_MASK_LAST)); + resultMask.predicateRegs |= genRegMask(reg); +#else + unreached(); +#endif + } } return resultMask; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 4d1cc3d2278e90..e3a28bf4e0fba0 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4330,7 +4330,7 @@ struct ReturnTypeDesc regNumber GetABIReturnReg(unsigned idx) const; // Get reg mask of ABI return registers - regMaskMixed GetABIReturnRegs() const; + AllRegsMask GetABIReturnRegs() const; }; class TailCallSiteInfo diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 19431e066ec608..5232708cc110ab 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2120,8 +2120,15 @@ class LinearScan : public LinearScanInterface int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); - void BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates = RBM_NONE); - void BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask); + void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); + void BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates); + void BuildKills(GenTree* tree, AllRegsMask killMask); +#ifdef TARGET_ARMARCH + void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask); +#else + void BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllRegsMask killMask); +#endif + void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index f12f5b5728d1a2..238e1caf1f1742 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -372,7 +372,7 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == 0); BuildUse(tree->gtGetOp1()); AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -422,7 +422,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); + BuildKills(tree, AllRegsMask()); break; case GT_LONG: @@ -470,7 +470,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = BuildReturn(tree); AllRegsMask killMask = getKillSetForReturn(); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -694,7 +694,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = 0; } - BuildDefs(tree, dstCount, argMask); + BuildDefs(tree, dstCount, argMask); // This is regMaskGpr } break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index a98d07d7712689..f9ea3344f93274 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -667,7 +667,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 0; assert(dstCount == 0); AllRegsMask killMask = getKillSetForProfilerHook(); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -675,7 +675,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); + BuildKills(tree, AllRegsMask()); break; case GT_CNS_DBL: @@ -741,7 +741,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = BuildReturn(tree); AllRegsMask killMask = getKillSetForReturn(); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -834,7 +834,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 1; assert(dstCount == 0); AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 2a007385f6f00d..ccb72211cddb15 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -129,7 +129,6 @@ int LinearScan::BuildCall(GenTreeCall* call) { bool hasMultiRegRetVal = false; const ReturnTypeDesc* retTypeDesc = nullptr; - regMaskOnlyOne dstCandidates = RBM_NONE; int srcCount = 0; int dstCount = 0; @@ -210,10 +209,11 @@ int LinearScan::BuildCall(GenTreeCall* call) } #endif // TARGET_ARM + // Set destination candidates for return value of the call. + AllRegsMask dstReturnCandidates; + regMaskOnlyOne dstCandidates = RBM_NONE; RegisterType registerType = call->TypeGet(); -// Set destination candidates for return value of the call. - #ifdef TARGET_ARM if (call->IsHelperCall(compiler, CORINFO_HELP_INIT_PINVOKE_FRAME)) { @@ -226,7 +226,8 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(); + dstReturnCandidates = retTypeDesc->GetABIReturnRegs(); + assert((int)dstReturnCandidates.Count() == dstCount); } else if (varTypeUsesFloatArgReg(registerType)) { @@ -391,7 +392,23 @@ int LinearScan::BuildCall(GenTreeCall* call) // Now generate defs and kills. AllRegsMask killMask = getKillSetForCall(call); - BuildDefsWithKills(call, dstCount, dstCandidates, killMask); + if (dstCount > 0) + { + if (hasMultiRegRetVal) + { + assert(dstReturnCandidates.Count() > 0); + BuildCallDefsWithKills(call, dstCount, dstReturnCandidates, killMask); + } + else + { + assert(dstCount == 1); + BuildDefWithKills(call, dstCandidates, killMask); + } + } + else + { + BuildKills(call, killMask); + } // No args are placed in registers anymore. placedArgRegs = RBM_NONE; @@ -399,6 +416,31 @@ int LinearScan::BuildCall(GenTreeCall* call) return srcCount; } +//------------------------------------------------------------------------ +// BuildDefWithKills: Build one RefTypeDef RefPositions for the given node, +// as well as kills as specified by the given mask. +// +// Arguments: +// tree - The call node that defines a register +// dstCandidates - The candidate registers for the definition +// killMask - The mask of registers killed by this node +// +// Notes: +// Adds the RefInfo for the definitions to the defList. +// The def and kill functionality is folded into a single method so that the +// save and restores of upper vector registers can be bracketed around the def. +// +void LinearScan::BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask) +{ + assert(!tree->AsCall()->HasMultiRegRetVal()); + assert((int)genCountBits(dstCandidates) == 1); + assert(compiler->IsOnlyOneRegMask(dstCandidates)); + + // Build the kill RefPositions + BuildKills(tree, killMask); + BuildDef(tree, dstCandidates); +} + //------------------------------------------------------------------------ // BuildPutArgStk: Set the NodeInfo for a GT_PUTARG_STK node // @@ -583,7 +625,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) } } buildInternalRegisterUses(); - BuildDefs(argNode, dstCount, argMask); + BuildDefs(argNode, dstCount, argMask); // this should only be regMaskGpr return srcCount; } @@ -843,7 +885,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalRegisterUses(); AllRegsMask killMask = getKillSetForBlockStore(blkNode); - BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); + BuildKills(blkNode, killMask); return useCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 9d5d4f2d39b365..39ff19b2e287b6 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1507,7 +1507,7 @@ Interval* LinearScan::getUpperVectorInterval(unsigned varIndex) // currentLoc - The location of the current node // fpCalleeKillSet - The set of registers killed by this node. // -// Notes: This is called by BuildDefsWithKills for any node that kills registers in the +// Notes: This is called by BuildCallDefsWithKills for any node that kills registers in the // RBM_FLT_CALLEE_TRASH set. We actually need to find any calls that kill the upper-half // of the callee-save vector registers. // But we will use as a proxy any node that kills floating point registers. @@ -3046,7 +3046,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, i } //------------------------------------------------------------------------ -// BuildDef: Build one or more RefTypeDef RefPositions for the given node +// BuildDef: Build one or more RefTypeDef RefPositions for the given call node // // Arguments: // tree - The node that defines a register @@ -3056,60 +3056,90 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, i // Notes: // Adds the RefInfo for the definitions to the defList. // -void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskMixed dstCandidates) +void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates) { - bool fixedReg = false; - if ((dstCount > 1) && (dstCandidates != RBM_NONE) && ((int)genCountBits(dstCandidates) == dstCount)) - { - fixedReg = true; - } - const ReturnTypeDesc* retTypeDesc = nullptr; - if (tree->IsMultiRegCall()) - { - retTypeDesc = tree->AsCall()->GetReturnTypeDesc(); - } + assert(dstCount > 0); + assert((int)dstCandidates.Count() == dstCount); + assert(tree->IsMultiRegCall()); + + const ReturnTypeDesc* retTypeDesc = tree->AsCall()->GetReturnTypeDesc(); + assert(retTypeDesc != nullptr); + for (int i = 0; i < dstCount; i++) { - regMaskOnlyOne thisDstCandidates; - if (fixedReg) + singleRegMask thisDstCandidates; + // In case of multi-reg call node, we have to query the i'th position return register. + // For all other cases of multi-reg definitions, the registers must be in sequential order. + regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i); + thisDstCandidates = genRegMask(thisReg); + + if (emitter::isGeneralRegister(thisReg)) { - // In case of multi-reg call node, we have to query the i'th position return register. - // For all other cases of multi-reg definitions, the registers must be in sequential order. - if (retTypeDesc != nullptr) - { - thisDstCandidates = genRegMask(tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i)); - assert((dstCandidates & thisDstCandidates) != RBM_NONE); - } - else - { - thisDstCandidates = genFindLowestBit(dstCandidates); - } - dstCandidates &= ~thisDstCandidates; + assert((dstCandidates.gprRegs & thisDstCandidates) != RBM_NONE); + dstCandidates.gprRegs &= ~thisDstCandidates; + } + else if (emitter::isFloatReg(thisReg)) + { + assert((dstCandidates.floatRegs & thisDstCandidates) != RBM_NONE); + dstCandidates.floatRegs &= ~thisDstCandidates; } +#ifdef HAS_PREDICATE_REGS else { - thisDstCandidates = dstCandidates; + assert(emitter::isMaskReg(thisReg)); + assert((dstCandidates.predicateRegs & thisDstCandidates) != RBM_NONE); + dstCandidates.predicateRegs &= ~thisDstCandidates; } +#endif // HAS_PREDICATE_REGS + BuildDef(tree, thisDstCandidates, i); } } //------------------------------------------------------------------------ -// BuildDef: Build one or more RefTypeDef RefPositions for the given node, -// as well as kills as specified by the given mask. +// BuildDef: Build one or more RefTypeDef RefPositions for the given node // // Arguments: // tree - The node that defines a register // dstCount - The number of registers defined by the node -// dstCandidates - The candidate registers for the definition -// killMask - The mask of registers killed by this node +// dstCandidates - the candidate registers for the definition // // Notes: // Adds the RefInfo for the definitions to the defList. -// The def and kill functionality is folded into a single method so that the -// save and restores of upper vector registers can be bracketed around the def. +// Also, the `dstCandidates` is assumed to be of "onlyOne" type. If there are +// both gpr and float registers, use `BuildDefs` that takes `AllRegsMask` +// +void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates) +{ + assert(dstCount > 0); + assert(compiler->IsOnlyOneRegMask(dstCandidates)); + + if ((dstCandidates == RBM_NONE) || ((int)genCountBits(dstCandidates) != dstCount)) + { + // This is not fixedReg case, so just create definitions based on dstCandidates + for (int i = 0; i < dstCount; i++) + { + BuildDef(tree, dstCandidates, i); + } + return; + } + + for (int i = 0; i < dstCount; i++) + { + regMaskOnlyOne thisDstCandidates = genFindLowestBit(dstCandidates); + BuildDef(tree, thisDstCandidates, i); + dstCandidates &= ~thisDstCandidates; + } +} + +//------------------------------------------------------------------------ +// BuildDef: Build Kills RefPositions as specified by the given mask. +// +// Arguments: +// tree - The node that defines a register +// killMask - The mask of registers killed by this node // -void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask) +void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) { assert(killMask == getKillSetForNode(tree)); @@ -3132,13 +3162,77 @@ void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskOnlyOne // if ((killMask.floatRegs & RBM_FLT_CALLEE_TRASH) != RBM_NONE) { - buildUpperVectorSaveRefPositions(tree, currentLoc + 1 DEBUG_ARG((killMask.floatRegs & RBM_FLT_CALLEE_TRASH))); + buildUpperVectorSaveRefPositions(tree, + currentLoc + 1 DEBUG_ARG((killMask.floatRegs & RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } +} + + +#ifndef TARGET_ARMARCH +//------------------------------------------------------------------------ +// BuildDefWithKills: Build one or two (for 32-bit) RefTypeDef RefPositions for the given node, +// as well as kills as specified by the given mask. +// +// Arguments: +// tree - The call node that defines a register +// dstCandidates - The candidate registers for the definition +// killMask - The mask of registers killed by this node +// +// Notes: +// Adds the RefInfo for the definitions to the defList. +// The def and kill functionality is folded into a single method so that the +// save and restores of upper vector registers can be bracketed around the def. +// +void LinearScan::BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllRegsMask killMask) +{ + // Build the kill RefPositions + BuildKills(tree, killMask); + +#ifdef TARGET_64BIT + // For 64 bits, + assert((int)genCountBits(dstCandidates) <= 1); + BuildDef(tree, dstCandidates); +#else + if ((int)genCountBits(dstCandidates) <= 1) + { + BuildDef(tree, dstCandidates); + } + else + { + assert((int)genCountBits(dstCandidates) == 2); + BuildDefs(tree, 2, dstCandidates); + } +#endif // TARGET_64BIT +} +#endif + +//------------------------------------------------------------------------ +// BuildCallDefsWithKills: Build one or more RefTypeDef RefPositions for the given node, +// as well as kills as specified by the given mask. +// +// Arguments: +// tree - The node that defines a register +// dstCount - The number of registers defined by the node +// dstCandidates - The candidate registers for the definition +// killMask - The mask of registers killed by this node +// +// Notes: +// Adds the RefInfo for the definitions to the defList. +// The def and kill functionality is folded into a single method so that the +// save and restores of upper vector registers can be bracketed around the def. +// +void LinearScan::BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask) +{ + assert(dstCount > 0); + assert(!dstCandidates.IsEmpty()); + + // Build the kill RefPositions + BuildKills(tree, killMask); - // Now, create the Def(s) - BuildDefs(tree, dstCount, dstCandidates); + // And then the Def(s) + BuildCallDefs(tree, dstCount, dstCandidates); } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index ea3ddf74f9cfbc..df48e80c4ff3af 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -137,7 +137,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildDefsWithKills(tree, 0, RBM_NONE, AllRegsMask()); + BuildKills(tree, AllRegsMask()); break; case GT_PROF_HOOK: @@ -145,7 +145,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 0; assert(dstCount == 0); AllRegsMask killMask = getKillSetForProfilerHook(); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -192,7 +192,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = BuildReturn(tree); AllRegsMask killMask = getKillSetForReturn(); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); break; } @@ -299,7 +299,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = BuildOperandUses(tree->gtGetOp1()); buildInternalRegisterUses(); AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildDefsWithKills(tree, 0, RBM_NONE, killMask); + BuildKills(tree, killMask); } break; @@ -1133,7 +1133,6 @@ int LinearScan::BuildCall(GenTreeCall* call) const ReturnTypeDesc* retTypeDesc = nullptr; int srcCount = 0; int dstCount = 0; - regMaskOnlyOne dstCandidates = RBM_NONE; assert(!call->isContained()); if (call->TypeGet() != TYP_VOID) @@ -1157,7 +1156,9 @@ int LinearScan::BuildCall(GenTreeCall* call) ctrlExpr = call->gtCallAddr; } - RegisterType registerType = regType(call); + AllRegsMask dstReturnCandidates; + regMaskOnlyOne dstCandidates = RBM_NONE; + RegisterType registerType = regType(call); // Set destination candidates for return value of the call. CLANG_FORMAT_COMMENT_ANCHOR; @@ -1175,8 +1176,8 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(); - assert((int)genCountBits(dstCandidates) == dstCount); + dstReturnCandidates = retTypeDesc->GetABIReturnRegs(); + assert((int)dstReturnCandidates.Count() == dstCount); } else if (varTypeUsesFloatReg(registerType)) { @@ -1358,7 +1359,24 @@ int LinearScan::BuildCall(GenTreeCall* call) // Now generate defs and kills. AllRegsMask killMask = getKillSetForCall(call); - BuildDefsWithKills(call, dstCount, dstCandidates, killMask); + + if (dstCount > 0) + { + if (hasMultiRegRetVal) + { + assert(dstReturnCandidates.Count() > 0); + BuildCallDefsWithKills(call, dstCount, dstReturnCandidates, killMask); + } + else + { + assert(dstCount == 1); + BuildDefWithKills(call, dstCandidates, killMask); + } + } + else + { + BuildKills(call, killMask); + } // No args are placed in registers anymore. placedArgRegs = RBM_NONE; @@ -1650,7 +1668,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalRegisterUses(); AllRegsMask killMask = getKillSetForBlockStore(blkNode); - BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); + BuildKills(blkNode, killMask); return useCount; } @@ -1943,7 +1961,7 @@ int LinearScan::BuildModDiv(GenTree* tree) buildInternalRegisterUses(); AllRegsMask killMask(getKillSetForModDiv(tree->AsOp()), RBM_NONE); - BuildDefsWithKills(tree, 1, dstCandidates, killMask); + BuildDefWithKills(tree, dstCandidates, killMask); return srcCount; } @@ -3064,7 +3082,7 @@ int LinearScan::BuildMul(GenTree* tree) assert(compiler->IsGprRegMask(dstCandidates)); AllRegsMask killMask(getKillSetForMul(tree->AsOp()), RBM_NONE); - BuildDefsWithKills(tree, dstCount, dstCandidates, killMask); + BuildDefWithKills(tree, dstCandidates, killMask); return srcCount; } From 90883b896b027939a1c58a17fe720d893b75724f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 6 Mar 2024 12:11:00 -0800 Subject: [PATCH 074/201] gentree: gtGetRegMask() and updating Npt --- src/coreclr/jit/codegenarm.cpp | 2 +- src/coreclr/jit/codegenarm64.cpp | 4 +- src/coreclr/jit/codegenarmarch.cpp | 20 +++--- src/coreclr/jit/codegencommon.cpp | 8 +-- src/coreclr/jit/codegenlinear.cpp | 33 ++++++---- src/coreclr/jit/codegenxarch.cpp | 33 +++++----- src/coreclr/jit/compiler.hpp | 22 +++++++ src/coreclr/jit/gcinfo.cpp | 41 ++++++++++++- src/coreclr/jit/gentree.cpp | 29 +++++---- src/coreclr/jit/gentree.h | 6 +- src/coreclr/jit/jitgcinfo.h | 4 +- src/coreclr/jit/lsra.cpp | 52 ++++++++++++++++ src/coreclr/jit/morph.cpp | 9 +++ src/coreclr/jit/target.h | 99 ++++++++++++++++-------------- 14 files changed, 249 insertions(+), 113 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index d61a3f22d2203d..0585ff5d2a5b42 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1832,7 +1832,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) instGen_Set_Reg_To_Imm(EA_PTRSIZE, REG_R0, (ssize_t)compiler->compProfilerMethHnd); } - gcInfo.gcMarkRegSetNpt(RBM_R0); + gcInfo.gcMarkGprRegNpt(REG_R0); regSet.verifyGprRegUsed(REG_R0); genEmitHelperCall(helper, diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index d1f0bc4050f334..da36dd768bd10e 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -3962,7 +3962,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) instGen_MemoryBarrier(); - gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask().gprRegs); } if (targetReg != REG_NA) @@ -4112,7 +4112,7 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) instGen_MemoryBarrier(); - gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask().gprRegs); } if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsSigned(treeNode->TypeGet())) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 85490e74e7234b..7f9c116d48875d 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1748,7 +1748,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) GetEmitter()->emitIns_R_R_I(INS_add, emitActualTypeSize(node), node->GetRegNum(), node->GetRegNum(), node->gtElemOffset); - gcInfo.gcMarkRegSetNpt(base->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(base->gtGetRegMask().gprRegs); genProduceReg(node); } @@ -3305,7 +3305,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) #endif inst_JMP(EJ_ne, loop); - gcInfo.gcMarkRegSetNpt(genRegMask(dstReg)); + gcInfo.gcMarkGprRegNpt(dstReg); } } @@ -3517,7 +3517,7 @@ void CodeGen::genCall(GenTreeCall* call) // However, for minopts or debuggable code, we keep it live to support managed return value debugging. if ((call->gtNext == nullptr) && !compiler->opts.MinOpts() && !compiler->opts.compDbgCode) { - gcInfo.gcMarkRegSetNpt(RBM_INTRET); + gcInfo.gcMarkGprRegNpt(REG_INTRET); } } @@ -3847,6 +3847,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { varDsc = compiler->lvaGetDesc(varNum); + regNumber varReg = varDsc->GetRegNum(); if (varDsc->lvPromoted) { @@ -3857,17 +3858,17 @@ void CodeGen::genJmpMethod(GenTree* jmp) } noway_assert(varDsc->lvIsParam); - if (varDsc->lvIsRegArg && (varDsc->GetRegNum() != REG_STK)) + if (varDsc->lvIsRegArg && (varReg != REG_STK)) { // Skip reg args which are already in its right register for jmp call. // If not, we will spill such args to their stack locations. // // If we need to generate a tail call profiler hook, then spill all // arg regs to free them up for the callback. - if (!compiler->compIsProfilerHookNeeded() && (varDsc->GetRegNum() == varDsc->GetArgReg())) + if (!compiler->compIsProfilerHookNeeded() && (varReg == varDsc->GetArgReg())) continue; } - else if (varDsc->GetRegNum() == REG_STK) + else if (varReg == REG_STK) { // Skip args which are currently living in stack. continue; @@ -3876,7 +3877,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // If we came here it means either a reg argument not in the right register or // a stack argument currently living in a register. In either case the following // assert should hold. - assert(varDsc->GetRegNum() != REG_STK); + assert(varReg != REG_STK); assert(varDsc->IsEnregisterableLcl()); var_types storeType = varDsc->GetStackSlotHomeType(); emitAttr storeSize = emitActualTypeSize(storeType); @@ -3901,9 +3902,8 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Update lvRegNum life and GC info to indicate lvRegNum is dead and varDsc stack slot is going live. // Note that we cannot modify varDsc->GetRegNum() here because another basic block may not be expecting it. // Therefore manually update life of varDsc->GetRegNum(). - singleRegMask tempMask = genRegMask(varDsc->GetRegNum()); - regSet.RemoveMaskVars(varDsc->TypeGet(), tempMask); - gcInfo.gcMarkRegSetNpt(tempMask); + regSet.RemoveMaskVars(varDsc->TypeGet(), genRegMask(varReg)); + gcInfo.gcMarkRegNpt(varReg); if (compiler->lvaIsGCTracked(varDsc)) { VarSetOps::AddElemD(compiler, gcInfo.gcVarPtrSetCur, varNum); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 2b0b5bf8bc54bd..23b765a73beeda 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -7710,7 +7710,7 @@ void CodeGen::genReturn(GenTree* treeNode) if (compiler->compMethodReturnsRetBufAddr()) { - gcInfo.gcMarkRegSetNpt(genRegMask(REG_INTRET)); + gcInfo.gcMarkGprRegNpt(REG_INTRET); } else { @@ -7718,7 +7718,7 @@ void CodeGen::genReturn(GenTree* treeNode) { if (varTypeIsGC(retTypeDesc.GetReturnRegType(i))) { - gcInfo.gcMarkRegSetNpt(genRegMask(retTypeDesc.GetABIReturnReg(i))); + gcInfo.gcMarkRegNpt(retTypeDesc.GetABIReturnReg(i)); } } } @@ -8208,7 +8208,7 @@ busyRegs.regType |= genRegMask(targetReg); // The old location is dying genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1)); - gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum())); + gcInfo.gcMarkRegNpt(op1->GetRegNum()); genUpdateVarReg(varDsc, treeNode); @@ -8275,7 +8275,7 @@ regNumber CodeGen::genRegCopy(GenTree* treeNode, unsigned multiRegIndex) { // The old location is dying genUpdateRegLife(fieldVarDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1)); - gcInfo.gcMarkRegSetNpt(genRegMask(sourceReg)); + gcInfo.gcMarkRegNpt(sourceReg); genUpdateVarReg(fieldVarDsc, treeNode); // Report the home change for this variable diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 44270ffbfa1a84..c0a1058e5f7023 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -938,7 +938,14 @@ void CodeGen::genSpillVar(GenTree* tree) // Remove the live var from the register. genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(tree)); - gcInfo.gcMarkRegSetNpt(varDsc->lvRegMask()); + if (varTypeUsesIntReg(varDsc)) + { + // TYP_STRUCT are also VTR_INT and can return vector registers. + // Make sure that we pass the register, so Npt will be called + // only if the `reg` is Gpr. + regNumber reg = varDsc->GetRegNum(); + gcInfo.gcMarkRegNpt(reg); + } if (VarSetOps::IsMember(compiler, gcInfo.gcTrkStkPtrLcls, varDsc->lvVarIndex)) { @@ -1455,11 +1462,11 @@ regNumber CodeGen::genConsumeReg(GenTree* tree, unsigned multiRegIndex) if (fldVarDsc->GetRegNum() == REG_STK) { // We have loaded this into a register only temporarily - gcInfo.gcMarkRegSetNpt(genRegMask(reg)); + gcInfo.gcMarkRegNpt(reg); } else if (lcl->IsLastUse(multiRegIndex)) { - gcInfo.gcMarkRegSetNpt(genRegMask(fldVarDsc->GetRegNum())); + gcInfo.gcMarkRegNpt(fldVarDsc->GetRegNum()); } } else @@ -1467,7 +1474,7 @@ regNumber CodeGen::genConsumeReg(GenTree* tree, unsigned multiRegIndex) regNumber regAtIndex = tree->GetRegByIndex(multiRegIndex); if (regAtIndex != REG_NA) { - gcInfo.gcMarkRegSetNpt(genRegMask(regAtIndex)); + gcInfo.gcMarkRegNpt(regAtIndex); } } return reg; @@ -1534,11 +1541,11 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) if (varDsc->GetRegNum() == REG_STK) { // We have loaded this into a register only temporarily - gcInfo.gcMarkRegSetNpt(genRegMask(tree->GetRegNum())); + gcInfo.gcMarkRegNpt(tree->GetRegNum()); } else if ((tree->gtFlags & GTF_VAR_DEATH) != 0) { - gcInfo.gcMarkRegSetNpt(genRegMask(varDsc->GetRegNum())); + gcInfo.gcMarkRegNpt(varDsc->GetRegNum()); } } else if (tree->gtSkipReloadOrCopy()->IsMultiRegLclVar()) @@ -1564,17 +1571,17 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) if (fldVarDsc->GetRegNum() == REG_STK) { // We have loaded this into a register only temporarily - gcInfo.gcMarkRegSetNpt(genRegMask(reg)); + gcInfo.gcMarkRegNpt(reg); } else if (lcl->IsLastUse(i)) { - gcInfo.gcMarkRegSetNpt(genRegMask(fldVarDsc->GetRegNum())); + gcInfo.gcMarkRegNpt(fldVarDsc->GetRegNum()); } } } else { - gcInfo.gcMarkRegSetNpt(tree->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(tree->gtGetRegMask().gprRegs); } genCheckConsumeNode(tree); @@ -1858,7 +1865,7 @@ void CodeGen::genConsumeArgSplitStruct(GenTreePutArgSplit* putArgNode) genUnspillRegIfNeeded(putArgNode); - gcInfo.gcMarkRegSetNpt(putArgNode->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(putArgNode->gtGetRegMask().gprRegs); genCheckConsumeNode(putArgNode); } @@ -2171,14 +2178,14 @@ void CodeGen::genProduceReg(GenTree* tree) { regNumber reg = tree->GetRegByIndex(i); regSet.rsSpillTree(reg, tree, i); - gcInfo.gcMarkRegSetNpt(genRegMask(reg)); + gcInfo.gcMarkRegNpt(reg); } } } else { regSet.rsSpillTree(tree->GetRegNum(), tree); - gcInfo.gcMarkRegSetNpt(genRegMask(tree->GetRegNum())); + gcInfo.gcMarkRegNpt(tree->GetRegNum()); } tree->gtFlags |= GTF_SPILLED; @@ -2285,7 +2292,7 @@ void CodeGen::genTransferRegGCState(regNumber dst, regNumber src) } else { - gcInfo.gcMarkRegSetNpt(dstMask); + gcInfo.gcMarkRegNpt(dst); } } diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index b3e682d01740a5..4370deecd65899 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -974,7 +974,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* treeNode) { emit->emitIns(INS_cdq, size); // the cdq instruction writes RDX, So clear the gcInfo for RDX - gcInfo.gcMarkRegSetNpt(RBM_RDX); + gcInfo.gcMarkGprRegNpt(REG_RDX); } // Perform the 'targetType' (64-bit or 32-bit) divide instruction @@ -1583,7 +1583,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // If there is a conflict then swap the condition anyway. LSRA should have // ensured the other way around has no conflict. - if ((trueVal->gtGetContainedRegMask() & genRegMask(dstReg)) != 0) + if ((trueVal->gtGetContainedRegMask() & dstReg) != 0) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1594,7 +1594,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // There may also be a conflict with the falseVal in case this is an AND // condition. Once again, after swapping there should be no conflict as // ensured by LSRA. - if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask() & genRegMask(dstReg)) != 0) + if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask() & dstReg) != 0) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1604,13 +1604,13 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) inst_RV_TT(INS_mov, emitTypeSize(select), dstReg, falseVal); assert(!trueVal->isContained() || trueVal->isUsedFromMemory()); - assert((trueVal->gtGetContainedRegMask() & genRegMask(dstReg)) == 0); + assert((trueVal->gtGetContainedRegMask() & dstReg) == 0); inst_RV_TT(JumpKindToCmov(desc.jumpKind1), emitTypeSize(select), dstReg, trueVal); if (desc.oper == GT_AND) { assert(falseVal->isUsedFromReg()); - assert((falseVal->gtGetContainedRegMask() & genRegMask(dstReg)) == 0); + assert((falseVal->gtGetContainedRegMask() & dstReg) == 0); inst_RV_TT(JumpKindToCmov(emitter::emitReverseJumpKind(desc.jumpKind2)), emitTypeSize(select), dstReg, falseVal); } @@ -3402,7 +3402,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) GetEmitter()->emitIns_R_I(INS_sub, EA_PTRSIZE, offsetReg, TARGET_POINTER_SIZE); inst_JMP(EJ_jne, loop); - gcInfo.gcMarkRegSetNpt(genRegMask(dstReg)); + gcInfo.gcMarkGprRegNpt(dstReg); } } @@ -4310,8 +4310,8 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) // Clear the gcInfo for RSI and RDI. // While we normally update GC info prior to the last instruction that uses them, // these actually live into the helper call. - gcInfo.gcMarkRegSetNpt(RBM_RSI); - gcInfo.gcMarkRegSetNpt(RBM_RDI); + gcInfo.gcMarkGprRegNpt(REG_RSI); + gcInfo.gcMarkGprRegNpt(REG_RDI); } #ifdef TARGET_AMD64 @@ -4504,7 +4504,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* node) GetEmitter()->emitIns_AR_R(INS_cmpxchg, size, tmpReg, addr->GetRegNum(), 0); inst_JMP(EJ_jne, loop); - gcInfo.gcMarkRegSetNpt(genRegMask(addr->GetRegNum())); + gcInfo.gcMarkGprRegNpt(addr->GetRegNum()); genProduceReg(node); } return; @@ -5435,7 +5435,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) GetEmitter()->emitIns_R_ARX(INS_lea, emitTypeSize(node->TypeGet()), dstReg, baseReg, tmpReg, scale, static_cast(node->gtElemOffset)); - gcInfo.gcMarkRegSetNpt(base->gtGetRegMask()); + gcInfo.gcMarkRegSetNpt(base->gtGetRegMask().gprRegs); genProduceReg(node); } @@ -6191,7 +6191,7 @@ void CodeGen::genCall(GenTreeCall* call) // However, for minopts or debuggable code, we keep it live to support managed return value debugging. if ((call->gtNext == nullptr) && compiler->opts.OptimizationEnabled()) { - gcInfo.gcMarkRegSetNpt(RBM_INTRET); + gcInfo.gcMarkGprRegNpt(REG_INTRET); } #if defined(DEBUG) && defined(TARGET_X86) @@ -6621,7 +6621,10 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Therefore manually update life of varDsc->GetRegNum(). regMaskOnlyOne tempMask = varDsc->lvRegMask(); regSet.RemoveMaskVars(varDsc->TypeGet(), tempMask); - gcInfo.gcMarkRegSetNpt(tempMask); + if (varTypeUsesIntReg(varDsc)) + { + gcInfo.gcMarkRegSetNpt(tempMask); + } if (compiler->lvaIsGCTracked(varDsc)) { #ifdef DEBUG @@ -6921,8 +6924,7 @@ void CodeGen::genCompareFloat(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - regMaskFloat targetRegMask = genRegMask(targetReg); - if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetRegMask) == 0) + if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetReg) == 0) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); targetType = TYP_UBYTE; // just a tip for inst_SETCC that movzx is not needed @@ -7092,8 +7094,7 @@ void CodeGen::genCompareInt(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - regMaskGpr targetRegMask = genRegMask(targetReg); - if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetRegMask) == 0) + if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetReg) == 0) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); targetType = TYP_UBYTE; // just a tip for inst_SETCC that movzx is not needed diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 0f9531ace38ac4..7154d4c3745cfb 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -990,6 +990,28 @@ inline regNumber genFirstRegNumFromMask(regMaskMixed mask) return regNum; } +inline AllRegsMask createRegMask(regNumber reg) +{ + AllRegsMask result; + if (emitter::isGeneralRegister(reg)) + { + result.gprRegs = genRegMask(reg); + } + else if (emitter::isFloatReg(reg)) + { + result.floatRegs |= genRegMask(reg); + } + else + { +#ifdef HAS_PREDICATE_REGS + result.predicateRegs |= genRegMask(reg); +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } + return result; +} + /***************************************************************************** * * Return the size in bytes of the given type. diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index de1726149d187b..47974fdcb8bd95 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -182,15 +182,46 @@ void GCInfo::gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput)) gcRegGCrefSetCur = gcRegGCrefSetNew; } + +/***************************************************************************** + * + * Mark the gpr register as holding non-pointer values. + * + */ + +void GCInfo::gcMarkGprRegNpt(regNumber reg DEBUGARG(bool forceOutput)) +{ + assert(emitter::isGeneralRegister(reg)); + gcMarkRegSetNpt(genRegMask(reg) DEBUGARG(forceOutput)); +} + +/***************************************************************************** + * + * Mark the register as holding non-pointer values. + * + */ + +void GCInfo::gcMarkRegNpt(regNumber reg DEBUGARG(bool forceOutput)) +{ + if (!emitter::isGeneralRegister(reg)) + { + return; + } + + gcMarkRegSetNpt(genRegMask(reg) DEBUGARG(forceOutput)); +} + /***************************************************************************** * * Mark the set of registers given by the specified mask as holding * non-pointer values. */ -void GCInfo::gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput)) +void GCInfo::gcMarkRegSetNpt(regMaskGpr regMask DEBUGARG(bool forceOutput)) { - assert(compiler->IsOnlyOneRegMask(regMask)); + // We only care about gpr registers because those are the ones that hold + // gc pointers. + assert(compiler->IsGprRegMask(regMask)); /* NOTE: don't unmark any live register variables */ @@ -211,8 +242,12 @@ void GCInfo::gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput)) void GCInfo::gcMarkRegPtrVal(regNumber reg, var_types type) { - singleRegMask regMask = genRegMask(reg); + if (!emitter::isGeneralRegister(reg)) + { + return; + } + singleRegMask regMask = genRegMask(reg); switch (type) { case TYP_REF: diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 09dabb35b318b3..305fdde6ac0dc4 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1059,14 +1059,14 @@ bool GenTree::NeedsConsecutiveRegisters() const // Return Value: // Reg Mask of GenTree node. // -regMaskMixed GenTree::gtGetContainedRegMask() +AllRegsMask GenTree::gtGetContainedRegMask() { if (!isContained()) { - return isUsedFromReg() ? gtGetRegMask() : RBM_NONE; + return isUsedFromReg() ? gtGetRegMask() : AllRegsMask(); } - regMaskMixed mask = 0; + AllRegsMask mask; for (GenTree* operand : Operands()) { mask |= operand->gtGetContainedRegMask(); @@ -1083,13 +1083,13 @@ regMaskMixed GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -regMaskMixed GenTree::gtGetRegMask() const +AllRegsMask GenTree::gtGetRegMask() const { - regMaskMixed resultMask; + AllRegsMask resultMask; if (IsMultiRegCall()) { - resultMask = genRegMask(GetRegNum()); + resultMask = createRegMask(GetRegNum()); resultMask |= AsCall()->GetOtherRegMask(); } else if (IsCopyOrReloadOfMultiRegCall()) @@ -1102,13 +1102,12 @@ regMaskMixed GenTree::gtGetRegMask() const const GenTreeCall* call = copyOrReload->gtGetOp1()->AsCall(); const unsigned regCount = call->GetReturnTypeDesc()->GetReturnRegCount(); - resultMask = RBM_NONE; for (unsigned i = 0; i < regCount; ++i) { regNumber reg = copyOrReload->GetRegNumByIdx(i); if (reg != REG_NA) { - resultMask |= genRegMask(reg); + resultMask |= reg; } } } @@ -1118,18 +1117,17 @@ regMaskMixed GenTree::gtGetRegMask() const const GenTreePutArgSplit* splitArg = AsPutArgSplit(); const unsigned regCount = splitArg->gtNumRegs; - resultMask = RBM_NONE; for (unsigned i = 0; i < regCount; ++i) { regNumber reg = splitArg->GetRegNumByIdx(i); assert(reg != REG_NA); - resultMask |= genRegMask(reg); + resultMask |= reg; } } #endif // FEATURE_ARG_SPLIT else { - resultMask = genRegMask(GetRegNum()); + resultMask = createRegMask(GetRegNum()); } return resultMask; @@ -2154,16 +2152,17 @@ bool GenTreeCall::NeedsVzeroupper(Compiler* comp) // Return Value: // Reg mask of gtOtherRegs of call node. // -regMaskMixed GenTreeCall::GetOtherRegMask() const +AllRegsMask GenTreeCall::GetOtherRegMask() const { - regMaskMixed resultMask = RBM_NONE; + AllRegsMask resultMask; #if FEATURE_MULTIREG_RET for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i) { - if (gtOtherRegs[i] != REG_NA) + regNumber otherReg = (regNumber)gtOtherRegs[i]; + if (otherReg != REG_NA) { - resultMask |= genRegMask((regNumber)gtOtherRegs[i]); + resultMask |= otherReg; continue; } break; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index e3a28bf4e0fba0..045c036b9a5294 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -957,8 +957,8 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; - regMaskMixed gtGetRegMask() const; - regMaskMixed gtGetContainedRegMask(); + AllRegsMask gtGetRegMask() const; + AllRegsMask gtGetContainedRegMask(); GenTreeFlags gtFlags; @@ -5139,7 +5139,7 @@ struct GenTreeCall final : public GenTree #endif // TARGET_XARCH // Get reg mask of all the valid registers of gtOtherRegs array - regMaskMixed GetOtherRegMask() const; + AllRegsMask GetOtherRegMask() const; GenTreeFlags GetRegSpillFlagByIdx(unsigned idx) const { diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index d513bfe9bdc567..ed4572f3776a82 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -91,7 +91,9 @@ class GCInfo void gcMarkRegSetGCref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); - void gcMarkRegSetNpt(regMaskOnlyOne regMask DEBUGARG(bool forceOutput = false)); + void gcMarkGprRegNpt(regNumber reg DEBUGARG(bool forceOutput = false)); + void gcMarkRegNpt(regNumber reg DEBUGARG(bool forceOutput = false)); + void gcMarkRegSetNpt(regMaskGpr regMask DEBUGARG(bool forceOutput = false)); void gcMarkRegPtrVal(regNumber reg, var_types type); #ifdef DEBUG diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 57217f81148bd7..306903ea4a4a23 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -13864,6 +13864,26 @@ AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second) return result; } +regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg) +{ + if (emitter::isGeneralRegister(reg)) + { + return first.gprRegs & genRegMask(reg); + } + else if (emitter::isFloatReg(reg)) + { + return first.floatRegs & genRegMask(reg); + } + else + { +#ifdef HAS_PREDICATE_REGS + return first.predicateRegs & genRegMask(reg); +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} + AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second) { AllRegsMask result(first.gprRegs | second.gprRegs, first.floatRegs | second.floatRegs @@ -13875,6 +13895,38 @@ AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second) return result; } +AllRegsMask operator|=(AllRegsMask& first, const regNumber reg) +{ + if (emitter::isGeneralRegister(reg)) + { + first.gprRegs |= genRegMask(reg); + } + else if (emitter::isFloatReg(reg)) + { + first.floatRegs |= genRegMask(reg); + } + else + { +#ifdef HAS_PREDICATE_REGS + first.predicateRegs |= genRegMask(reg); +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } + return first; +} + +AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second) +{ + first.gprRegs |= second.gprRegs; + first.floatRegs |= second.floatRegs; + +#ifdef HAS_PREDICATE_REGS + first.predicateRegs |= second.predicateRegs; +#endif + return first; +} + AllRegsMask operator~(const AllRegsMask& first) { AllRegsMask result(~first.gprRegs, ~first.floatRegs diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4ca6d57a5304f5..3ffc129ed27c5a 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2706,6 +2706,11 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call } } + //if ((structIntRegs > 0) && (structFloatRegs > 0)) + //{ + // assert(!"Different numbers"); + //} + isRegArg = ((nextFltArgRegNum + structFloatRegs) <= MAX_FLOAT_REG_ARG) && ((intArgRegNum + structIntRegs) <= MAX_REG_ARG); } @@ -2817,6 +2822,10 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call ++structFloatRegs; } } + //if ((structIntRegs > 0) && (structFloatRegs > 0)) + //{ + // assert(!"Different numbers"); + //} } #endif // defined(UNIX_AMD64_ABI) else diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6088394478bedf..677f830674f324 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -261,6 +261,8 @@ typedef unsigned regMaskTP; unsigned genCountBits(uint64_t bits); +//struct AllRegsMask; + typedef struct _regMaskAll { regMaskGpr gprRegs; @@ -268,47 +270,45 @@ typedef struct _regMaskAll #ifdef HAS_PREDICATE_REGS regMaskPredicate predicateRegs; #endif // HAS_PREDICATE_REGS -// -// _regMaskAll(regMaskGpr _gprRegMask) -// : gprRegs(_gprRegMask) -// , floatRegs(RBM_NONE) -//#ifdef HAS_PREDICATE_REGS -// , predicateRegs(RBM_NONE) -//#endif -// { -// } -// -// _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask) : gprRegs(_gprRegMask), floatRegs(_floatRegMask) -//#ifdef HAS_PREDICATE_REGS -// , predicateRegs(RBM_NONE) -//#endif -// { -// -// } - - _regMaskAll(regMaskGpr _gprRegMask, - regMaskFloat _floatRegMask, - regMaskPredicate _predicateRegMask = RBM_NONE) + // + // _regMaskAll(regMaskGpr _gprRegMask) + // : gprRegs(_gprRegMask) + // , floatRegs(RBM_NONE) + // #ifdef HAS_PREDICATE_REGS + // , predicateRegs(RBM_NONE) + // #endif + // { + // } + // + // _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask) : gprRegs(_gprRegMask), + // floatRegs(_floatRegMask) + // #ifdef HAS_PREDICATE_REGS + // , predicateRegs(RBM_NONE) + // #endif + // { + // + // } + + _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask, regMaskPredicate _predicateRegMask = RBM_NONE) { - gprRegs = _gprRegMask; - floatRegs = _floatRegMask; + gprRegs = _gprRegMask; + floatRegs = _floatRegMask; #ifdef HAS_PREDICATE_REGS predicateRegs = _predicateRegMask; #endif // HAS_PREDICATE_REGS - } -// // Useful to create an entity from `mask` when the caller -// // doesn't know what is the type of mask but knows that it -// // is of "onlyone" type. -// _regMaskAll(regMaskOnlyOne mask) -// { -// gprRegs = mask; -// floatRegs = mask; -//#ifdef HAS_PREDICATE_REGS -// predicateRegs = mask; -//#endif -// } + // // Useful to create an entity from `mask` when the caller + // // doesn't know what is the type of mask but knows that it + // // is of "onlyone" type. + // _regMaskAll(regMaskOnlyOne mask) + // { + // gprRegs = mask; + // floatRegs = mask; + // #ifdef HAS_PREDICATE_REGS + // predicateRegs = mask; + // #endif + // } _regMaskAll() { @@ -323,7 +323,7 @@ typedef struct _regMaskAll { return (gprRegs == RBM_NONE) && (floatRegs == RBM_NONE) #ifdef HAS_PREDICATE_REGS - && (predicateRegs == RBM_NONE) + && (predicateRegs == RBM_NONE) #endif // HAS_PREDICATE_REGS ; } @@ -332,12 +332,12 @@ typedef struct _regMaskAll { return genCountBits(gprRegs) + genCountBits(floatRegs) #ifdef HAS_PREDICATE_REGS - + genCountBits(predicateRegs) + + genCountBits(predicateRegs) #endif // HAS_PREDICATE_REGS - ; + ; } - void AddRegTypeMask(var_types type, regMaskOnlyOne maskToAdd) + void AddRegTypeMask(var_types type, regMaskOnlyOne maskToAdd) { if (varTypeRegister[type] == VTR_INT) { @@ -373,17 +373,16 @@ typedef struct _regMaskAll return floatRegs; } } + //void AllRegsMask& operator|=(const AllRegsMask& second); + //{ + // gprRegs + // return *this; + //} } AllRegsMask; #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) #define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) -bool operator==(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator~(const AllRegsMask& first); - - #if REGMASK_BITS == 32 typedef unsigned regMaskSmall; #define REG_MASK_INT_FMT "%08X" @@ -397,6 +396,16 @@ typedef unsigned __int64 regMaskSmall; typedef _regNumber_enum regNumber; typedef unsigned char regNumberSmall; +bool operator==(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); +regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); +AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); +// AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); +AllRegsMask operator~(const AllRegsMask& first); +// inline AllRegsMask createRegMask(regNumber reg) + /*****************************************************************************/ #ifdef DEBUG From 931001dd2ce796e83210b958e21e15d2e13a2d25 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 7 Mar 2024 17:12:57 -0800 Subject: [PATCH 075/201] Remove regMaskMixed from lsraarmarch.cpp --- src/coreclr/jit/lsraarmarch.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index ccb72211cddb15..b9dce8d7121330 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -548,13 +548,14 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) int dstCount = argNode->gtNumRegs; regNumber argReg = argNode->GetRegNum(); - regMaskMixed argMask = RBM_NONE; + regMaskGpr argMask = RBM_NONE; for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); argMask |= genRegMask(thisArgReg); argNode->SetRegNumByIdx(thisArgReg, i); } + assert(compiler->IsGprRegMask(argMask)); if (src->OperGet() == GT_FIELD_LIST) { From bda4c0798181630b8f7b87cae6e72468c889493a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 7 Mar 2024 22:13:58 -0800 Subject: [PATCH 076/201] Remove regMaskMixed lsra: resolveEdge, tempRegForResolution --- src/coreclr/jit/codegenlinear.cpp | 9 ++-- src/coreclr/jit/lsra.cpp | 87 ++++++++++++++++++++++++------- src/coreclr/jit/lsra.h | 4 +- src/coreclr/jit/target.h | 12 +++-- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index c0a1058e5f7023..8f1dab36331a06 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -216,15 +216,18 @@ void CodeGen::genCodeForBBlist() if (varDsc->lvIsInReg()) { - newLiveRegSet.AddRegTypeMask(varDsc->TypeGet(), varDsc->lvRegMask()); + regMaskOnlyOne varRegMask = varDsc->lvRegMask(); + assert(compiler->IsOnlyOneRegMask(varRegMask)); + + newLiveRegSet.AddRegTypeMask(varRegMask, varDsc->TypeGet()); if (varDsc->lvType == TYP_REF) { - newRegGCrefSet |= varDsc->lvRegMask(); + newRegGCrefSet |= varRegMask; } else if (varDsc->lvType == TYP_BYREF) { - newRegByrefSet |= varDsc->lvRegMask(); + newRegByrefSet |= varRegMask; } if (!varDsc->IsAlwaysAliveInMemory()) { diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 306903ea4a4a23..ea489904b28870 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -8563,7 +8563,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskMixed terminatorConsumedRegs) + regMaskOnlyOne terminatorConsumedRegs) { // TODO-Throughput: This would be much more efficient if we add RegToVarMaps instead of VarToRegMaps // and they would be more space-efficient as well. @@ -8652,10 +8652,26 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, } else { + regMaskOnlyOne calleeTrashMask = RBM_NONE; + + if (varTypeRegister[type] == VTR_INT) + { + calleeTrashMask = RBM_INT_CALLEE_TRASH; + assert(compiler->IsGprRegMask(terminatorConsumedRegs)); + } + else if (varTypeRegister[type] == VTR_FLOAT) + { + calleeTrashMask = RBM_FLT_CALLEE_TRASH; + assert(compiler->IsFloatRegMask(terminatorConsumedRegs)); + } + else + { + unreached(); + } // Prefer a callee-trashed register if possible to prevent new prolog/epilog saves/restores. - if ((freeRegs & RBM_CALLEE_TRASH) != 0) + if ((freeRegs & calleeTrashMask) != 0) { - freeRegs &= RBM_CALLEE_TRASH; + freeRegs &= calleeTrashMask; } regNumber tempReg = genRegNumFromMask(genFindLowestBit(freeRegs)); @@ -8849,7 +8865,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // available to copy into. // Note that for this purpose we use the full live-out set, because we must ensure that // even the registers that remain the same across the edge are preserved correctly. - regMaskMixed liveOutRegs = RBM_NONE; + AllRegsMask liveOutRegs; VarSetOps::Iter liveOutIter(compiler, block->bbLiveOut); unsigned liveOutVarIndex = 0; while (liveOutIter.NextElem(&liveOutVarIndex)) @@ -8857,8 +8873,9 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) regNumber fromReg = getVarReg(outVarToRegMap, liveOutVarIndex); if (fromReg != REG_STK) { - regMaskOnlyOne fromRegMask = genRegMask(fromReg, getIntervalForLocalVar(liveOutVarIndex)->registerType); - liveOutRegs |= fromRegMask; + var_types varType = getIntervalForLocalVar(liveOutVarIndex)->registerType; + regMaskOnlyOne fromRegMask = genRegMask(fromReg, varType); + liveOutRegs.AddRegTypeMask(fromRegMask, varType); } } @@ -8869,7 +8886,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // // Note: Only switches and JCMP/JTEST (for Arm4) have input regs (and so can be fed by copies), so those // are the only block-ending branches that need special handling. - regMaskMixed consumedRegs = RBM_NONE; + regMaskGpr consumedRegs = RBM_NONE; if (block->KindIs(BBJ_SWITCH)) { // At this point, Lowering has transformed any non-switch-table blocks into @@ -8963,9 +8980,10 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) } } + assert(compiler->IsGprRegMask(consumedRegs)); // If this fails, then we will have to use AllRegsMask for consumedRegs VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; - regMaskMixed sameWriteRegs = RBM_NONE; - regMaskMixed diffReadRegs = RBM_NONE; + AllRegsMask sameWriteRegs; + AllRegsMask diffReadRegs; // For each var that may require resolution, classify them as: // - in the same register at the end of this block and at each target (no resolution needed) @@ -9017,17 +9035,20 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // We only need to check for these cases if sameToReg is an actual register (not REG_STK). if (sameToReg != REG_NA && sameToReg != REG_STK) { + var_types outVarRegType = getIntervalForLocalVar(outResolutionSetVarIndex)->registerType; + // If there's a path on which this var isn't live, it may use the original value in sameToReg. // In this case, sameToReg will be in the liveOutRegs of this block. // Similarly, if sameToReg is in sameWriteRegs, it has already been used (i.e. for a lclVar that's // live only at another target), and we can't copy another lclVar into that reg in this block. - regMaskOnlyOne sameToRegMask = - genRegMask(sameToReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); - if (maybeSameLivePaths && - (((sameToRegMask & liveOutRegs) != RBM_NONE) || ((sameToRegMask & sameWriteRegs) != RBM_NONE))) + regMaskOnlyOne sameToRegMask = genRegMask(sameToReg, outVarRegType); + + if (maybeSameLivePaths && (liveOutRegs.IsRegNumInMask(sameToReg, outVarRegType) || + sameWriteRegs.IsRegNumInMask(sameToReg, outVarRegType) != RBM_NONE)) { sameToReg = REG_NA; } + // If this register is busy because it is used by a switch table at the end of the block // (or for Arm64, it is consumed by JCMP), we can't do the copy in this block since we can't // insert it after the switch (or for Arm64, can't insert and overwrite the operand/source @@ -9066,7 +9087,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) VarSetOps::AddElemD(compiler, diffResolutionSet, outResolutionSetVarIndex); if (fromReg != REG_STK) { - diffReadRegs |= genRegMask(fromReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); + diffReadRegs.AddRegNumInMask(fromReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); } } else if (sameToReg != fromReg) @@ -9075,14 +9096,14 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) setVarReg(sameVarToRegMap, outResolutionSetVarIndex, sameToReg); if (sameToReg != REG_STK) { - sameWriteRegs |= genRegMask(sameToReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); + sameWriteRegs.AddRegNumInMask(sameToReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); } } } if (!VarSetOps::IsEmpty(compiler, sameResolutionSet)) { - if ((sameWriteRegs & diffReadRegs) != RBM_NONE) + if (!((sameWriteRegs & diffReadRegs).IsEmpty())) { // We cannot split the "same" and "diff" regs if the "same" set writes registers // that must be read by the "diff" set. (Note that when these are done as a "batch" @@ -9397,8 +9418,10 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskMixed terminatorConsumedRegs) + regMaskGpr terminatorConsumedRegs) { + assert(compiler->IsGprRegMask(terminatorConsumedRegs)); + VarToRegMap fromVarToRegMap = getOutVarToRegMap(fromBlock->bbNum); VarToRegMap toVarToRegMap; if (resolveType == ResolveSharedCritical) @@ -9453,7 +9476,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { #ifdef TARGET_ARM // Try to reserve a double register for TYP_DOUBLE and use it for TYP_FLOAT too if available. - tempRegDbl = getTempRegForResolution(fromBlock, toBlock, TYP_DOUBLE, liveSet, terminatorConsumedRegs); + tempRegDbl = getTempRegForResolution(fromBlock, toBlock, TYP_DOUBLE, liveSet, RBM_NONE); if (tempRegDbl != REG_NA) { tempRegFlt = tempRegDbl; @@ -9461,7 +9484,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, else #endif // TARGET_ARM { - tempRegFlt = getTempRegForResolution(fromBlock, toBlock, TYP_FLOAT, liveSet, terminatorConsumedRegs); + tempRegFlt = getTempRegForResolution(fromBlock, toBlock, TYP_FLOAT, liveSet, RBM_NONE); } } @@ -13937,3 +13960,29 @@ AllRegsMask operator~(const AllRegsMask& first) ); return result; } + +void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) +{ + regMaskOnlyOne regMask = genRegMask(reg, type); + AddRegTypeMask(regMask, type); +} + +bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) +{ + if (emitter::isGeneralRegister(reg)) + { + return (gprRegs & genRegMask(reg)) != RBM_NONE; + } + else if (emitter::isFloatReg(reg)) + { + return (floatRegs & genRegMask(reg, type)) != RBM_NONE; + } + else + { +#ifdef HAS_PREDICATE_REGS + return (predicateRegs & genRegMask(reg)) != RBM_NONE; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 5232708cc110ab..f88628d5ab6321 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -737,7 +737,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskMixed terminatorConsumedRegs); + regMaskGpr terminatorConsumedRegs); void resolveEdges(); @@ -1432,7 +1432,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskMixed terminatorConsumedRegs); + regMaskOnlyOne terminatorConsumedRegs); #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 677f830674f324..7c09b8a421a5a8 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -261,7 +261,8 @@ typedef unsigned regMaskTP; unsigned genCountBits(uint64_t bits); -//struct AllRegsMask; +typedef _regNumber_enum regNumber; +typedef unsigned char regNumberSmall; typedef struct _regMaskAll { @@ -337,7 +338,10 @@ typedef struct _regMaskAll ; } - void AddRegTypeMask(var_types type, regMaskOnlyOne maskToAdd) + void AddRegNumInMask(regNumber reg, var_types type); + bool IsRegNumInMask(regNumber reg, var_types type); + + void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) { if (varTypeRegister[type] == VTR_INT) { @@ -373,6 +377,7 @@ typedef struct _regMaskAll return floatRegs; } } + //void AllRegsMask& operator|=(const AllRegsMask& second); //{ // gprRegs @@ -393,9 +398,6 @@ typedef unsigned __int64 regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif -typedef _regNumber_enum regNumber; -typedef unsigned char regNumberSmall; - bool operator==(const AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); From 2cddfdb8f4b4821854a99383fccc6f8f1acc354c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 7 Mar 2024 22:30:44 -0800 Subject: [PATCH 077/201] Remove regMaskMixed - lsra:getConstrainedRegMask() --- src/coreclr/jit/lsra.cpp | 46 ++++++++++++++++++++++++++++++++-------- src/coreclr/jit/lsra.h | 8 +++---- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ea489904b28870..81bd4e9ad21821 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -472,12 +472,15 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // New regMask that has minRegCount registers after intersection. // Otherwise returns regMaskActual. // -regMaskMixed LinearScan::getConstrainedRegMask(RefPosition* refPosition, - regMaskMixed regMaskActual, - regMaskMixed regMaskConstraint, +regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, + regMaskOnlyOne regMaskActual, + regMaskOnlyOne regMaskConstraint, unsigned minRegCount) { - regMaskMixed newMask = regMaskActual & regMaskConstraint; + assert(compiler->IsOnlyOneRegMask(regMaskActual)); + assert(compiler->IsOnlyOneRegMask(regMaskConstraint)); + + regMaskOnlyOne newMask = regMaskActual & regMaskConstraint; if (genCountBits(newMask) < minRegCount) { // Constrained mask does not have minimum required registers needed. @@ -486,7 +489,7 @@ regMaskMixed LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { - regMaskMixed busyRegs = regsBusyUntilKill | regsInUseThisLocation; + regMaskOnlyOne busyRegs = regsBusyUntilKill | regsInUseThisLocation; if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -512,8 +515,10 @@ regMaskMixed LinearScan::getConstrainedRegMask(RefPosition* refPosition, // This is the method used to implement the stress options that limit // the set of registers considered for allocation. // -regMaskMixed LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskMixed mask) +regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask) { + assert(compiler->IsOnlyOneRegMask(mask)); + #ifdef TARGET_ARM64 if ((refPosition != nullptr) && refPosition->isLiveAtConsecutiveRegistersLoc(consecutiveRegistersLocation)) { @@ -529,20 +534,43 @@ regMaskMixed LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskMixed { // The refPosition could be null, for example when called // by getTempRegForResolution(). - int minRegCount = (refPosition != nullptr) ? refPosition->minRegCandidateCount : 1; + int minRegCount = (refPosition != nullptr) ? refPosition->minRegCandidateCount : 1; + RegisterType regType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType + : refPosition->getReg()->registerType; + regMaskOnlyOne calleeSaved = RBM_NONE; + regMaskOnlyOne calleeTrash = RBM_NONE; + if (regType == IntRegisterType) + { + calleeSaved = RBM_INT_CALLEE_SAVED; + calleeTrash = RBM_INT_CALLEE_TRASH; + } + else if (regType == FloatRegisterType) + { + calleeSaved = RBM_FLT_CALLEE_SAVED; + calleeTrash = RBM_FLT_CALLEE_TRASH; + } + else + { +#ifdef HAS_PREDICATE_REGS + calleeSaved = RBM_MSK_CALLEE_SAVED; + calleeTrash = RBM_MSK_CALLEE_TRASH; +#else + unreached(); +#endif + } switch (getStressLimitRegs()) { case LSRA_LIMIT_CALLEE: if (!compiler->opts.compDbgEnC) { - mask = getConstrainedRegMask(refPosition, mask, RBM_CALLEE_SAVED, minRegCount); + mask = getConstrainedRegMask(refPosition, mask, calleeSaved, minRegCount); } break; case LSRA_LIMIT_CALLER: { - mask = getConstrainedRegMask(refPosition, mask, RBM_CALLEE_TRASH, minRegCount); + mask = getConstrainedRegMask(refPosition, mask, calleeTrash, minRegCount); } break; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index f88628d5ab6321..8e132a78556da0 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -830,11 +830,11 @@ class LinearScan : public LinearScanInterface } // TODO: Can have separate methods for each type - regMaskMixed getConstrainedRegMask(RefPosition* refPosition, - regMaskMixed regMaskActual, - regMaskMixed regMaskConstrain, + regMaskOnlyOne getConstrainedRegMask(RefPosition* refPosition, + regMaskOnlyOne regMaskActual, + regMaskOnlyOne regMaskConstrain, unsigned minRegCount); - regMaskMixed stressLimitRegs(RefPosition* refPosition, regMaskMixed mask); + regMaskOnlyOne stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask); // This controls the heuristics used to select registers // These can be combined. From a353db4984f31c2afc88815dc59d9a3c9513bb3b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 11:59:44 -0700 Subject: [PATCH 078/201] Remove regMaskMixed - lsra:freeRegister --- src/coreclr/jit/lsra.cpp | 401 +++++++++++++++++++++++----------- src/coreclr/jit/lsra.h | 52 ++--- src/coreclr/jit/lsrabuild.cpp | 25 +-- src/coreclr/jit/target.h | 38 ++++ src/coreclr/jit/targetamd64.h | 2 + src/coreclr/jit/targetarm.h | 1 + src/coreclr/jit/targetarm64.h | 2 + src/coreclr/jit/targetx86.h | 2 + 8 files changed, 353 insertions(+), 170 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 81bd4e9ad21821..8e0291f91c6afc 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -386,30 +386,31 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) // void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, regMaskOnlyOne regsBusy, - regMaskOnlyOne* regsToFree, - regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) + AllRegsMask* regsToFree, + AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { assert(compiler->IsOnlyOneRegMask(regsBusy)); + RegisterType regType = interval->registerType; - regsInUseThisLocation |= regsBusy; + regsInUseThisLocation.AddRegTypeMask(regsBusy, regType); if (refPosition.lastUse) { if (refPosition.delayRegFree) { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED, interval, assignedReg)); - *delayRegsToFree |= regsBusy; - regsInUseNextLocation |= regsBusy; + delayRegsToFree->AddRegTypeMask(regsBusy, regType); + regsInUseNextLocation.AddRegTypeMask(regsBusy, regType); } else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE, interval, assignedReg)); - *regsToFree |= regsBusy; + regsToFree->AddRegTypeMask(regsBusy, regType); } } else if (refPosition.delayRegFree) { - regsInUseNextLocation |= regsBusy; + regsInUseNextLocation.AddRegTypeMask(regsBusy, regType); } } @@ -473,6 +474,7 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // Otherwise returns regMaskActual. // regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, + RegisterType regType, regMaskOnlyOne regMaskActual, regMaskOnlyOne regMaskConstraint, unsigned minRegCount) @@ -489,7 +491,21 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { - regMaskOnlyOne busyRegs = regsBusyUntilKill | regsInUseThisLocation; + regMaskOnlyOne busyRegs = RBM_NONE; + if (regType == IntRegisterType) + { + busyRegs = regsBusyUntilKill.gprRegs | regsInUseThisLocation.gprRegs; + } + else if (regType == FloatRegisterType) + { + busyRegs = regsBusyUntilKill.floatRegs | regsInUseThisLocation.floatRegs; + } + else + { +#ifdef HAS_PREDICATE_REGS + busyRegs = regsBusyUntilKill.predicateRegs | regsInUseThisLocation.predicateRegs; +#endif + } if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -564,24 +580,24 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly case LSRA_LIMIT_CALLEE: if (!compiler->opts.compDbgEnC) { - mask = getConstrainedRegMask(refPosition, mask, calleeSaved, minRegCount); + mask = getConstrainedRegMask(refPosition, regType, mask, calleeSaved, minRegCount); } break; case LSRA_LIMIT_CALLER: { - mask = getConstrainedRegMask(refPosition, mask, calleeTrash, minRegCount); + mask = getConstrainedRegMask(refPosition, regType, mask, calleeTrash, minRegCount); } break; case LSRA_LIMIT_SMALL_SET: if ((mask & LsraLimitSmallIntSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, mask, LsraLimitSmallIntSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitSmallIntSet, minRegCount); } else if ((mask & LsraLimitSmallFPSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, mask, LsraLimitSmallFPSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitSmallFPSet, minRegCount); } break; @@ -589,7 +605,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly case LSRA_LIMIT_UPPER_SIMD_SET: if ((mask & LsraLimitUpperSimdSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, mask, LsraLimitUpperSimdSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitUpperSimdSet, minRegCount); } break; #endif @@ -4114,13 +4130,33 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // The new register to use. // #ifdef DEBUG -regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskMixed availableRegs) +regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, AllRegsMask availableRegs) { if (targetReg != REG_STK && getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE) { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskOnlyOne candidateRegs = allRegs(interval->registerType) & availableRegs; + regMaskOnlyOne candidateRegs = RBM_NONE; + + regMaskOnlyOne allRegsMask = allRegs(interval->registerType); + + if (interval->registerType == IntRegisterType) + { + candidateRegs = allRegsMask & availableRegs.gprRegs; + } + else if (interval->registerType == FloatRegisterType) + { + candidateRegs = allRegsMask & availableRegs.floatRegs; + } + else + { +#ifdef HAS_PREDICATE_REGS + candidateRegs = allRegsMask & availableRegs.predicateRegs; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } + regNumber firstReg = REG_NA; regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) @@ -4424,9 +4460,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } // If we are rotating register assignments at block boundaries, we want to make the // inactive registers available for the rotation. - regMaskMixed inactiveRegs = RBM_NONE; + AllRegsMask inactiveRegs; #endif // DEBUG - regMaskMixed liveRegs = RBM_NONE; + AllRegsMask liveRegs; VarSetOps::Iter iter(compiler, currentLiveVars); unsigned varIndex = 0; while (iter.NextElem(&varIndex)) @@ -4521,7 +4557,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assert(targetReg != REG_STK); assert(interval->assignedReg != nullptr && interval->assignedReg->regNum == targetReg && interval->assignedReg->assignedInterval == interval); - liveRegs |= getRegMask(targetReg, interval->registerType); + liveRegs.AddRegNumInMask(targetReg, interval->registerType); continue; } } @@ -4551,8 +4587,8 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // likely to match other assignments this way. targetReg = interval->physReg; interval->isActive = true; - liveRegs |= getRegMask(targetReg, interval->registerType); - INDEBUG(inactiveRegs |= genRegMask(targetReg)); + liveRegs.AddRegNumInMask(targetReg, interval->registerType); + INDEBUG(inactiveRegs |= targetReg); setVarReg(inVarToRegMap, varIndex, targetReg); } else @@ -4563,7 +4599,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (targetReg != REG_STK) { RegRecord* targetRegRecord = getRegisterRecord(targetReg); - liveRegs |= getRegMask(targetReg, interval->registerType); + liveRegs.AddRegNumInMask(targetReg, interval->registerType); if (!allocationPassComplete) { updateNextIntervalRef(targetReg, interval); @@ -4600,7 +4636,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) RegRecord* anotherHalfRegRec = findAnotherHalfRegRec(targetRegRecord); // Use TYP_FLOAT to get the regmask of just the half reg. - liveRegs &= ~getRegMask(anotherHalfRegRec->regNum, TYP_FLOAT); + liveRegs.floatRegs &= ~getRegMask(anotherHalfRegRec->regNum, TYP_FLOAT); } #endif // TARGET_ARM @@ -4627,7 +4663,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { RegRecord* physRegRecord = getRegisterRecord(reg); - if ((liveRegs & genRegMask(reg)) == 0) + if ((liveRegs & reg) == 0) { makeRegAvailable(reg, physRegRecord->registerType); Interval* assignedInterval = physRegRecord->assignedInterval; @@ -4684,41 +4720,46 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) } } #else - regMaskMixed deadCandidates = ~liveRegs; + AllRegsMask deadCandidates = ~liveRegs; // Only focus on actual registers present deadCandidates &= actualRegistersMask; - while (deadCandidates != RBM_NONE) + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) { - regNumber reg = genFirstRegNumFromMaskAndToggle(deadCandidates); - RegRecord* physRegRecord = getRegisterRecord(reg); - - makeRegAvailable(reg, physRegRecord->registerType); - Interval* assignedInterval = physRegRecord->assignedInterval; + regMaskOnlyOne deadRegMask = deadCandidates[rType]; - if (assignedInterval != nullptr) + while (deadRegMask != RBM_NONE) { - assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); + RegRecord* physRegRecord = getRegisterRecord(reg); + + makeRegAvailable(reg, physRegRecord->registerType); + Interval* assignedInterval = physRegRecord->assignedInterval; - if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) + if (assignedInterval != nullptr) { - assignedInterval->isActive = false; - if (assignedInterval->getNextRefPosition() == nullptr) + assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + + if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) { - unassignPhysReg(physRegRecord, nullptr); + assignedInterval->isActive = false; + if (assignedInterval->getNextRefPosition() == nullptr) + { + unassignPhysReg(physRegRecord, nullptr); + } + if (!assignedInterval->IsUpperVector()) + { + inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; + } } - if (!assignedInterval->IsUpperVector()) + else { - inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; + // This interval may still be active, but was in another register in an + // intervening block. + clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); } } - else - { - // This interval may still be active, but was in another register in an - // intervening block. - clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); - } } } #endif // TARGET_ARM @@ -4876,28 +4917,34 @@ void LinearScan::freeRegister(RegRecord* physRegRecord) // Arguments: // regsToFree - the mask of registers to free // -void LinearScan::freeRegisters(regMaskMixed regsToFree) +void LinearScan::freeRegisters(AllRegsMask regsToFree) { - if (regsToFree == RBM_NONE) + if (regsToFree.IsEmpty()) { return; } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FREE_REGS)); makeRegsAvailable(regsToFree); - while (regsToFree != RBM_NONE) + + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(regsToFree); + regMaskOnlyOne freeMask = regsToFree[rType]; - RegRecord* regRecord = getRegisterRecord(nextReg); -#ifdef TARGET_ARM - if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) + while (freeMask != RBM_NONE) { - assert(genIsValidDoubleReg(nextReg)); - regsToFree ^= genRegMask(regNumber(nextReg + 1)); - } + regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); + + RegRecord* regRecord = getRegisterRecord(nextReg); +#ifdef TARGET_ARM + if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) + { + assert(genIsValidDoubleReg(nextReg)); + freeMask ^= genRegMask(regNumber(nextReg + 1)); + } #endif - freeRegister(regRecord); + freeRegister(regRecord); + } } } @@ -4951,13 +4998,13 @@ void LinearScan::allocateRegistersMinimal() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskMixed regsToFree = RBM_NONE; - regMaskMixed delayRegsToFree = RBM_NONE; - regMaskOnlyOne regsToMakeInactive = RBM_NONE; - regMaskOnlyOne delayRegsToMakeInactive = RBM_NONE; - regMaskMixed copyRegsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; - regsInUseNextLocation = RBM_NONE; + AllRegsMask regsToFree; + AllRegsMask delayRegsToFree; + AllRegsMask regsToMakeInactive; + AllRegsMask delayRegsToMakeInactive; + AllRegsMask copyRegsToFree ; + regsInUseThisLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask(); // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -4973,13 +5020,17 @@ void LinearScan::allocateRegistersMinimal() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskOnlyOne tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - while (tempRegsToMakeInactive != RBM_NONE) + AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); - RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); - makeRegisterInactive(regRecord); + regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; + while (tempRegsMaskInactive != RBM_NONE) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsMaskInactive); + RegRecord* regRecord = getRegisterRecord(nextReg); + clearSpillCost(regRecord->regNum, regRecord->registerType); + makeRegisterInactive(regRecord); + } } if (currentRefPosition.nodeLocation > prevLocation) { @@ -4987,7 +5038,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = RBM_NONE; + delayRegsToMakeInactive = AllRegsMask(); } #ifdef DEBUG @@ -5044,24 +5095,24 @@ void LinearScan::allocateRegistersMinimal() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = RBM_NONE; + copyRegsToFree = AllRegsMask(); regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = RBM_NONE; - if ((regsToFree | delayRegsToFree) != RBM_NONE) + regsInUseNextLocation = AllRegsMask(); + if (!((regsToFree | delayRegsToFree).IsEmpty())) { freeRegisters(regsToFree); - if ((currentLocation > (prevLocation + 1)) && (delayRegsToFree != RBM_NONE)) + if ((currentLocation > (prevLocation + 1)) && (!delayRegsToFree.IsEmpty())) { // We should never see a delayReg that is delayed until a Location that has no RefPosition // (that would be the RefPosition that it was supposed to interfere with). assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; + delayRegsToFree = AllRegsMask(); + regsInUseThisLocation = AllRegsMask(); } regsToFree = delayRegsToFree; - delayRegsToFree = RBM_NONE; + delayRegsToFree = AllRegsMask(); #ifdef DEBUG verifyFreeRegisters(regsToFree); @@ -5103,9 +5154,9 @@ void LinearScan::allocateRegistersMinimal() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; - regsInUseNextLocation = RBM_NONE; + regsToFree = AllRegsMask(); + regsInUseThisLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask(); handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -5161,7 +5212,7 @@ void LinearScan::allocateRegistersMinimal() } #endif // TARGET_ARM } - regsInUseThisLocation |= currentRefPosition.registerAssignment; + regsInUseThisLocation.AddRegNumInMask(regRecord->regNum, regRecord->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); } else @@ -5300,7 +5351,7 @@ void LinearScan::allocateRegistersMinimal() &delayRegsToFree DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree |= copyRegMask; + copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); } // For tree temp (non-localVar) interval, we will need an explicit move. @@ -5315,7 +5366,7 @@ void LinearScan::allocateRegistersMinimal() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -5414,15 +5465,15 @@ void LinearScan::allocateRegistersMinimal() { assignedRegBit = genRegMask(assignedRegister); regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); - regsInUseThisLocation |= regMask; + regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); if (currentRefPosition.delayRegFree) { - regsInUseNextLocation |= regMask; + regsInUseNextLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); } currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree &= ~regMask; // we'll set it again later if it's dead + regsToFree.RemoveRegNumInMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -5443,11 +5494,11 @@ void LinearScan::allocateRegistersMinimal() { if (currentRefPosition.delayRegFree) { - delayRegsToMakeInactive |= regMask; + delayRegsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); } else { - regsToMakeInactive |= regMask; + regsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); } // TODO-Cleanup: this makes things consistent with previous, and will enable preferences // to be propagated, but it seems less than ideal. @@ -5466,13 +5517,13 @@ void LinearScan::allocateRegistersMinimal() { if (currentRefPosition.delayRegFree) { - delayRegsToFree |= regMask; + delayRegsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED)); } else { - regsToFree |= regMask; + regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE)); } @@ -5631,13 +5682,13 @@ void LinearScan::allocateRegisters() BasicBlock* currentBlock = nullptr; LsraLocation prevLocation = MinLocation; - regMaskMixed regsToFree = RBM_NONE; - regMaskMixed delayRegsToFree = RBM_NONE; - regMaskOnlyOne regsToMakeInactive = RBM_NONE; - regMaskOnlyOne delayRegsToMakeInactive = RBM_NONE; - regMaskMixed copyRegsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; - regsInUseNextLocation = RBM_NONE; + AllRegsMask regsToFree; + AllRegsMask delayRegsToFree; + AllRegsMask regsToMakeInactive; + AllRegsMask delayRegsToMakeInactive; + AllRegsMask copyRegsToFree; + regsInUseThisLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask(); // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5653,13 +5704,17 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. - regMaskOnlyOne tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - while (tempRegsToMakeInactive != RBM_NONE) + AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsToMakeInactive); - RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); - makeRegisterInactive(regRecord); + regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; + while (tempRegsMaskInactive != RBM_NONE) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsMaskInactive); + RegRecord* regRecord = getRegisterRecord(nextReg); + clearSpillCost(regRecord->regNum, regRecord->registerType); + makeRegisterInactive(regRecord); + } } if (currentRefPosition.nodeLocation > prevLocation) { @@ -5667,7 +5722,7 @@ void LinearScan::allocateRegisters() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = RBM_NONE; + delayRegsToMakeInactive = AllRegsMask(); } #ifdef DEBUG @@ -5722,30 +5777,30 @@ void LinearScan::allocateRegisters() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = RBM_NONE; + copyRegsToFree = AllRegsMask(); regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = RBM_NONE; + regsInUseNextLocation = AllRegsMask(); #ifdef TARGET_ARM64 if (hasConsecutiveRegister) { consecutiveRegsInUseThisLocation = RBM_NONE; } #endif - if ((regsToFree | delayRegsToFree) != RBM_NONE) + if (!((regsToFree | delayRegsToFree).IsEmpty())) { freeRegisters(regsToFree); - if ((currentLocation > (prevLocation + 1)) && (delayRegsToFree != RBM_NONE)) + if ((currentLocation > (prevLocation + 1)) && !delayRegsToFree.IsEmpty()) { // We should never see a delayReg that is delayed until a Location that has no RefPosition // (that would be the RefPosition that it was supposed to interfere with). assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; + delayRegsToFree = AllRegsMask(); + regsInUseThisLocation = AllRegsMask(); } regsToFree = delayRegsToFree; - delayRegsToFree = RBM_NONE; + delayRegsToFree = AllRegsMask(); #ifdef DEBUG verifyFreeRegisters(regsToFree); #endif @@ -5804,9 +5859,9 @@ void LinearScan::allocateRegisters() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = RBM_NONE; - regsInUseThisLocation = RBM_NONE; - regsInUseNextLocation = RBM_NONE; + regsToFree = AllRegsMask(); + regsInUseThisLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask(); handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -5869,7 +5924,7 @@ void LinearScan::allocateRegisters() } #endif // TARGET_ARM } - regsInUseThisLocation |= currentRefPosition.registerAssignment; + regsInUseThisLocation.AddRegTypeMask(currentRefPosition.registerAssignment, regRecord->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); } else @@ -6279,7 +6334,7 @@ void LinearScan::allocateRegisters() DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree |= copyRegMask; + copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6387,7 +6442,7 @@ void LinearScan::allocateRegisters() &delayRegsToFree DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree |= copyRegMask; + copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6408,7 +6463,7 @@ void LinearScan::allocateRegisters() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -6576,16 +6631,15 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); - regsInUseThisLocation |= regMask; + regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); if (currentRefPosition.delayRegFree) { - regsInUseNextLocation |= regMask; + regsInUseNextLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); } currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree &= ~regMask; // we'll set it again later if it's dead + regsToFree.RemoveRegNumInMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -6622,11 +6676,11 @@ void LinearScan::allocateRegisters() { if (currentRefPosition.delayRegFree) { - delayRegsToMakeInactive |= regMask; + delayRegsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); } else { - regsToMakeInactive |= regMask; + regsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); } // TODO-Cleanup: this makes things consistent with previous, and will enable preferences // to be propagated, but it seems less than ideal. @@ -6645,13 +6699,13 @@ void LinearScan::allocateRegisters() { if (currentRefPosition.delayRegFree) { - delayRegsToFree |= regMask; + delayRegsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED)); } else { - regsToFree |= regMask; + regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE)); } @@ -11423,7 +11477,7 @@ void LinearScan::dumpRegRecords() #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE printf("%c", activeChar); } - else if ((genRegMask(regNum) & regsBusyUntilKill) != RBM_NONE) + else if ((regsBusyUntilKill & regNum) != RBM_NONE) { printf(columnFormatArray, "Busy"); } @@ -11653,14 +11707,28 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) // Arguments: // regsToFree - Registers that were just freed. // -void LinearScan::verifyFreeRegisters(regMaskMixed regsToFree) +void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) { + regMaskOnlyOne regsMaskToFree = regsToFree.gprRegs; + regMaskOnlyOne availableRegsMask = availableIntRegs; for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { + if (reg >= REG_FP_FIRST && reg <= REG_FP_LAST) + { + regsMaskToFree = regsToFree.floatRegs; + availableRegsMask = availableFloatRegs; + } +#ifdef HAS_PREDICATE_REGS + else if (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST) + { + regsMaskToFree = regsToFree.predicateRegs; + availableRegsMask = availableMaskRegs; + } +#endif singleRegMask regMask = genRegMask(reg); // If this isn't available or if it's still waiting to be freed (i.e. it was in // delayRegsToFree and so now it's in regsToFree), then skip it. - if ((regMask & allAvailableRegs & ~regsToFree) == RBM_NONE) + if ((regMask & availableRegsMask & ~regsMaskToFree) == RBM_NONE) { continue; } @@ -13796,8 +13864,23 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskMixed busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates &= ~busyRegs; + AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + if (regType == IntRegisterType) + { + candidates &= ~busyRegs.gprRegs; + } + else if (regType == FloatRegisterType) + { + candidates &= ~busyRegs.floatRegs; + } + else + { +#ifdef HAS_PREDICATE_REGS + candidates &= ~busyRegs.predicateRegs; +#else + unreached(); +#endif + } #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -13807,7 +13890,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM @@ -13978,6 +14061,17 @@ AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second) return first; } +AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second) +{ + first.gprRegs &= second.gprRegs; + first.floatRegs &= second.floatRegs; + +#ifdef HAS_PREDICATE_REGS + first.predicateRegs &= second.predicateRegs; +#endif + return first; +} + AllRegsMask operator~(const AllRegsMask& first) { AllRegsMask result(~first.gprRegs, ~first.floatRegs @@ -13995,6 +14089,51 @@ void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) AddRegTypeMask(regMask, type); } +void AllRegsMask::RemoveRegNumInMask(regNumber reg, var_types type) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg, type); + + if (emitter::isGeneralRegister(reg)) + { + gprRegs &= ~regMaskToRemove; + } + else if (emitter::isFloatReg(reg)) + { + floatRegs &= ~regMaskToRemove; + } + else + { +#ifdef HAS_PREDICATE_REGS + predicateRegs &= ~regMaskToRemove; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} + +void AllRegsMask::RemoveRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg); + + if (emitter::isGeneralRegister(reg)) + { + gprRegs &= ~regMaskToRemove; + } + else if (emitter::isFloatReg(reg)) + { + floatRegs &= ~regMaskToRemove; + } + else + { +#ifdef HAS_PREDICATE_REGS + predicateRegs &= ~regMaskToRemove; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} + + bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) { if (emitter::isGeneralRegister(reg)) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 8e132a78556da0..e068ee656d0fcf 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -831,6 +831,7 @@ class LinearScan : public LinearScanInterface // TODO: Can have separate methods for each type regMaskOnlyOne getConstrainedRegMask(RefPosition* refPosition, + RegisterType regType, regMaskOnlyOne regMaskActual, regMaskOnlyOne regMaskConstrain, unsigned minRegCount); @@ -899,7 +900,7 @@ class LinearScan : public LinearScanInterface { return (LsraBlockBoundaryLocations)(lsraStressMask & LSRA_BLOCK_BOUNDARY_MASK); } - regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, regMaskMixed availableRegs); + regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, AllRegsMask availableRegs); // This controls whether we always insert a GT_RELOAD instruction after a spill // Note that this can be combined with LSRA_SPILL_ALWAYS (or not) @@ -962,7 +963,7 @@ class LinearScan : public LinearScanInterface static bool IsResolutionMove(GenTree* node); static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); - void verifyFreeRegisters(regMaskMixed regsToFree); + void verifyFreeRegisters(AllRegsMask regsToFree); void verifyFinalAllocation(); void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG @@ -1125,7 +1126,7 @@ class LinearScan : public LinearScanInterface void makeRegisterInactive(RegRecord* physRegRecord); void freeRegister(RegRecord* physRegRecord); - void freeRegisters(regMaskMixed regsToFree); + void freeRegisters(AllRegsMask regsToFree); // Get the type that this tree defines. var_types getDefType(GenTree* tree) @@ -1684,7 +1685,7 @@ class LinearScan : public LinearScanInterface // A temporary VarToRegMap used during the resolution of critical edges. VarToRegMap sharedCriticalVarToRegMap; - PhasedVar actualRegistersMask; + PhasedVar actualRegistersMask; PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; @@ -1831,17 +1832,14 @@ class LinearScan : public LinearScanInterface } } - void setRegsInUse(regMaskMixed regMask) + void setRegsInUse(AllRegsMask regMask) { //TODO: Fix this later. - m_AvailableRegs[0] &= ~(regMask & ~RBM_ALLFLOAT); - m_AvailableRegs[1] &= ~(regMask & RBM_ALLFLOAT); + m_AvailableRegs[0] &= regMask.gprRegs; + m_AvailableRegs[1] &= regMask.floatRegs; #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] &= ~(regMask & RBM_ALLMASK); + m_AvailableRegs[2] &= regMask.predicateRegs; #endif - - assert(compiler->IsGprRegMask(m_AvailableRegs[0])); - assert(compiler->IsFloatRegMask(m_AvailableRegs[1])); } void setRegInUse(regNumber reg, var_types regType) { @@ -1867,14 +1865,17 @@ class LinearScan : public LinearScanInterface } #endif } - void makeRegsAvailable(regMaskMixed regMask) + void makeRegsAvailable(AllRegsMask regMask) { // TODO: This will be just `regMask` - m_AvailableRegs[0] |= regMask & ~RBM_ALLFLOAT; - m_AvailableRegs[1] |= regMask & RBM_ALLFLOAT; + m_AvailableRegs[0] |= regMask.gprRegs; + m_AvailableRegs[1] |= regMask.floatRegs; #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] |= regMask & RBM_ALLMASK; + m_AvailableRegs[2] |= regMask.predicateRegs; + assert(compiler->IsPredicateRegMask(m_AvailableRegs[2])); #endif + assert(compiler->IsGprRegMask(m_AvailableRegs[0])); + assert(compiler->IsFloatRegMask(m_AvailableRegs[1])); } void makeRegAvailable(regNumber reg, var_types regType) @@ -1943,8 +1944,8 @@ class LinearScan : public LinearScanInterface FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, regMaskOnlyOne regsBusy, - regMaskOnlyOne* regsToFree, - regMaskOnlyOne* delayRegsToFree DEBUG_ARG(Interval* interval) + AllRegsMask* regsToFree, + AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); #ifdef HAS_PREDICATE_REGS @@ -2002,37 +2003,36 @@ class LinearScan : public LinearScanInterface } weight_t spillCost[REG_COUNT]; - regMaskMixed regsBusyUntilKill; // TODO: Likewise, probably have this global 32-bit and set it point to the specific + AllRegsMask regsBusyUntilKill; // TODO: Likewise, probably have this global 32-bit and set it point to the specific // version like gpr, vector, etc. - regMaskMixed regsInUseThisLocation; - regMaskMixed regsInUseNextLocation; + AllRegsMask regsInUseThisLocation; + AllRegsMask regsInUseNextLocation; #ifdef TARGET_ARM64 regMaskFloat consecutiveRegsInUseThisLocation; #endif bool isRegBusy(regNumber reg, var_types regType) { - regMaskOnlyOne regMask = getRegMask(reg, regType); - return (regsBusyUntilKill & regMask) != RBM_NONE; + return regsBusyUntilKill.IsRegNumInMask(reg, regType); } void setRegBusyUntilKill(regNumber reg, var_types regType) { - regsBusyUntilKill |= getRegMask(reg, regType); + regsBusyUntilKill.AddRegNumInMask(reg, regType); } void clearRegBusyUntilKill(regNumber reg) { - regsBusyUntilKill &= ~genRegMask(reg); + regsBusyUntilKill.RemoveRegNumInMask(reg); } bool isRegInUse(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - return (regsInUseThisLocation & regMask) != RBM_NONE; + return regsInUseThisLocation.IsRegNumInMask(reg, regType); } void resetRegState() { resetAvailableRegs(); - regsBusyUntilKill = RBM_NONE; + regsBusyUntilKill = AllRegsMask();; } bool conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 39ff19b2e287b6..6fa299503d1b2d 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1906,7 +1906,7 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc #endif // TARGET_ARM64 { newRefPosition->registerAssignment = - getConstrainedRegMask(newRefPosition, oldAssignment, calleeSaveMask, minRegCountForRef); + getConstrainedRegMask(newRefPosition, interval->registerType, oldAssignment, calleeSaveMask, minRegCountForRef); } if ((newRefPosition->registerAssignment != oldAssignment) && (newRefPosition->refType == RefTypeUse) && @@ -2304,8 +2304,8 @@ void LinearScan::buildIntervals() RegState* floatRegState = &compiler->codeGen->floatRegState; intRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; floatRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; - regsInUseThisLocation = RBM_NONE; - regsInUseNextLocation = RBM_NONE; + regsInUseThisLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask(); for (unsigned int varIndex = 0; varIndex < compiler->lvaTrackedCount; varIndex++) { @@ -2817,18 +2817,17 @@ void LinearScan::buildIntervals() needNonIntegerRegisters |= compiler->compFloatingPointUsed; if (!needNonIntegerRegisters) { - availableRegCount = REG_INT_COUNT; + availableRegCount = REG_INT_COUNT; + availableFloatRegs = RBM_NONE; + availableDoubleRegs = RBM_NONE; + //availableMaskRegs = RBM_NONE; // Is this also needed? } - if (availableRegCount < (sizeof(regMaskMixed) * 8)) - { - // Mask out the bits that are between 64 ~ availableRegCount - actualRegistersMask = (1ULL << availableRegCount) - 1; - } - else - { - actualRegistersMask = ~RBM_NONE; - } + actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs +#ifdef HAS_PREDICATE_REGS + , availableMaskRegs +#endif // HAS_PREDICATE_REGS + ); #ifdef DEBUG // Make sure we don't have any blocks that were not visited diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 7c09b8a421a5a8..27507c5894c869 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -200,6 +200,9 @@ enum _regMask_enum : unsigned #if defined(TARGET_XARCH) && defined(FEATURE_SIMD) #define HAS_PREDICATE_REGS +#define REGISTER_TYPE_COUNT 3 +#else +#define REGISTER_TYPE_COUNT 2 #endif /*****************************************************************************/ @@ -320,6 +323,15 @@ typedef struct _regMaskAll #endif } + void Clear() + { + gprRegs = RBM_NONE; + floatRegs = RBM_NONE; +#ifdef HAS_PREDICATE_REGS + predicateRegs = RBM_NONE; +#endif + } + bool IsEmpty() { return (gprRegs == RBM_NONE) && (floatRegs == RBM_NONE) @@ -338,9 +350,13 @@ typedef struct _regMaskAll ; } + // Rename this to AddRegNum void AddRegNumInMask(regNumber reg, var_types type); + void RemoveRegNumInMask(regNumber reg); + void RemoveRegNumInMask(regNumber reg, var_types type); bool IsRegNumInMask(regNumber reg, var_types type); + // Rename this to AddRegMask() void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) { if (varTypeRegister[type] == VTR_INT) @@ -378,6 +394,26 @@ typedef struct _regMaskAll } } + regMaskOnlyOne& operator[](int index) + { + if (index == 0) + { + return gprRegs; + } + else if (index == 1) + { + return floatRegs; + } + else + { +#ifdef HAS_PREDICATE_REGS + return predicateRegs; +#else + unreached(); +#endif + } + } + //void AllRegsMask& operator|=(const AllRegsMask& second); //{ // gprRegs @@ -403,11 +439,13 @@ AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& s regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); // AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); AllRegsMask operator~(const AllRegsMask& first); // inline AllRegsMask createRegMask(regNumber reg) + /*****************************************************************************/ #ifdef DEBUG diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index b13284bd160ac1..a3b0a6849b7d3b 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -87,10 +87,12 @@ #define RBM_ALLDOUBLE RBM_ALLFLOAT #define REG_FP_FIRST REG_XMM0 #define REG_FP_LAST REG_XMM31 + #define REG_FP_COUNT (REG_FP_LAST - REG_FP_FIRST + 1) #define FIRST_FP_ARGREG REG_XMM0 #define REG_MASK_FIRST REG_K0 #define REG_MASK_LAST REG_K7 + #define REG_MASK_COUNT (REG_MASK_LAST - REG_MASK_FIRST + 1) #define RBM_ALLMASK_INIT (0) #define RBM_ALLMASK_EVEX (RBM_K1 | RBM_K2 | RBM_K3 | RBM_K4 | RBM_K5 | RBM_K6 | RBM_K7) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 09a68fd49c964f..96a5d7c767781c 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -46,6 +46,7 @@ #define REG_FP_FIRST REG_F0 #define REG_FP_LAST REG_F31 + #define REG_FP_COUNT (REG_FP_LAST - REG_FP_FIRST + 1) #define FIRST_FP_ARGREG REG_F0 #define LAST_FP_ARGREG REG_F15 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 6e4719dc22eca3..c2b23d5cc46efb 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -48,10 +48,12 @@ #define REG_FP_FIRST REG_V0 #define REG_FP_LAST REG_V31 + #define REG_FP_COUNT (REG_FP_LAST - REG_FP_FIRST + 1) #define FIRST_FP_ARGREG REG_V0 #define LAST_FP_ARGREG REG_V15 #define REG_PREDICATE_FIRST REG_P0 #define REG_PREDICATE_LAST REG_P15 + #define REG_MASK_COUNT (REG_PREDICATE_LAST - REG_PREDICATE_FIRST + 1) #define REG_PREDICATE_LOW_LAST REG_P7 // Some instructions can only use the first half of the predicate registers. #define REG_PREDICATE_HIGH_FIRST REG_P8 // Similarly, some instructions can only use the second half of the predicate registers. #define REG_PREDICATE_HIGH_LAST REG_P15 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index 2654e6605abe5d..01fc35692c07fe 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -70,9 +70,11 @@ #define REG_FP_FIRST REG_XMM0 #define REG_FP_LAST REG_XMM7 + #define REG_FP_COUNT (REG_FP_LAST - REG_FP_FIRST + 1) #define REG_MASK_FIRST REG_K0 #define REG_MASK_LAST REG_K7 + #define REG_MASK_COUNT (REG_MASK_LAST - REG_MASK_FIRST) #define CNT_MASK_REGS 8 #define FIRST_FP_ARGREG REG_XMM0 From a767406bec2a950fc54649bfd182b5e5db7b048d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 19:08:04 -0700 Subject: [PATCH 079/201] Remove regMaskMixed - lsra:fixedRegs, registersToDump, etc. --- src/coreclr/jit/lsra.cpp | 57 +++++++++++++++++++++++++---------- src/coreclr/jit/lsra.h | 9 +++--- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/target.h | 24 +++++++++++++++ 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 8e0291f91c6afc..7c75383d6ee925 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -278,12 +278,12 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo if (nextRefPosition == nullptr) { nextLocation = MaxLocation; - fixedRegs &= ~genRegMask(regRecord->regNum); + fixedRegs[regTypeIndex(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); } else { nextLocation = nextRefPosition->nodeLocation; - fixedRegs |= genRegMask(regRecord->regNum); + fixedRegs[regTypeIndex(regRecord->registerType)] |= genRegMask(regRecord->regNum); } nextFixedRef[regRecord->regNum] = nextLocation; } @@ -4988,7 +4988,7 @@ void LinearScan::allocateRegistersMinimal() "--------------------\n"); // Start with a small set of commonly used registers, so that we don't keep having to print a new title. // Include all the arg regs, as they may already have values assigned to them. - registersToDump = LsraLimitSmallIntSet | LsraLimitSmallFPSet | RBM_ARG_REGS; + registersToDump = AllRegsMask(LsraLimitSmallIntSet | RBM_ARG_REGS, LsraLimitSmallFPSet, RBM_NONE); dumpRegRecordHeader(); // Now print an empty "RefPosition", since we complete the dump of the regs at the beginning of the loop. printf(indentFormat, ""); @@ -5652,7 +5652,7 @@ void LinearScan::allocateRegisters() updateNextIntervalRef(reg, interval); updateSpillCost(reg, interval); setRegInUse(reg, interval->registerType); - INDEBUG(registersToDump |= getRegMask(reg, interval->registerType)); + INDEBUG(registersToDump.AddRegNumInMask(reg, interval->registerType)); } } else @@ -5672,7 +5672,7 @@ void LinearScan::allocateRegisters() "--------------------\n"); // Start with a small set of commonly used registers, so that we don't keep having to print a new title. // Include all the arg regs, as they may already have values assigned to them. - registersToDump = LsraLimitSmallIntSet | LsraLimitSmallFPSet | RBM_ARG_REGS; + registersToDump = AllRegsMask(LsraLimitSmallIntSet | RBM_ARG_REGS, LsraLimitSmallFPSet , RBM_NONE); dumpRegRecordHeader(); // Now print an empty "RefPosition", since we complete the dump of the regs at the beginning of the loop. printf(indentFormat, ""); @@ -11046,7 +11046,7 @@ void LinearScan::dumpLsraAllocationEvent( } if ((interval != nullptr) && (reg != REG_NA) && (reg != REG_STK)) { - registersToDump |= getRegMask(reg, interval->registerType); + registersToDump.AddRegNumInMask(reg, interval->registerType); dumpRegRecordTitleIfNeeded(); } @@ -11385,7 +11385,7 @@ void LinearScan::dumpRegRecordHeader() regColumnWidth + 1); // Print a "title row" including the legend and the reg names. - lastDumpedRegisters = RBM_NONE; + lastDumpedRegisters = AllRegsMask(); dumpRegRecordTitleIfNeeded(); } @@ -11397,7 +11397,7 @@ void LinearScan::dumpRegRecordTitleIfNeeded() int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_FP_LAST : REG_INT_LAST; for (int regNumIndex = 0; regNumIndex <= lastRegNumIndex; regNumIndex++) { - if ((registersToDump & genRegMask((regNumber)regNumIndex)) != 0) + if (registersToDump.IsRegNumInMask((regNumber)regNumIndex)) { lastUsedRegNumIndex = regNumIndex; } @@ -13528,7 +13528,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval } #ifdef DEBUG - regMaskMixed inUseOrBusyRegsMask = RBM_NONE; + AllRegsMask inUseOrBusyRegsMask; #endif // Eliminate candidates that are in-use or busy. @@ -13537,8 +13537,8 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // TODO-CQ: We assign same registerAssignment to UPPER_RESTORE and the next USE. // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. - regMaskMixed busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates &= ~busyRegs; + AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; + candidates = busyRegs.ApplyTo(candidates, regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -13548,7 +13548,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM @@ -13559,7 +13559,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regTypeIndex(regType)]; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -13573,7 +13573,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval { candidates &= ~checkConflictBit; #ifdef DEBUG - inUseOrBusyRegsMask |= checkConflictBit; + inUseOrBusyRegsMask |= checkConflictReg; #endif } } @@ -13653,7 +13653,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // Remove the `inUseOrBusyRegsMask` from the original candidates list and find one // such range that is consecutive. Next, append that range to the `candidates`. // - regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask; + regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs; regMaskFloat overallLimitCandidates; regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, @@ -13897,7 +13897,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regTypeIndex(regType)]; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -13987,6 +13987,11 @@ bool operator==(const AllRegsMask& first, const AllRegsMask& second) ; } +bool operator!=(const AllRegsMask& first, const AllRegsMask& second) +{ + return !(first == second); +} + AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second) { AllRegsMask result(first.gprRegs & second.gprRegs, first.floatRegs & second.floatRegs @@ -14153,3 +14158,23 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) #endif // HAS_PREDICATE_REGS } } + +bool AllRegsMask::IsRegNumInMask(regNumber reg) +{ + if (emitter::isGeneralRegister(reg)) + { + return (gprRegs & genRegMask(reg)) != RBM_NONE; + } + else if (emitter::isFloatReg(reg)) + { + return (floatRegs & genRegMask(reg)) != RBM_NONE; + } + else + { +#ifdef HAS_PREDICATE_REGS + return (predicateRegs & genRegMask(reg)) != RBM_NONE; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index e068ee656d0fcf..e69efde796e28e 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1501,12 +1501,12 @@ class LinearScan : public LinearScanInterface static const int MAX_ROWS_BETWEEN_TITLES = 50; int rowCountSinceLastTitle; // Current mask of registers being printed in the dump. - regMaskMixed lastDumpedRegisters; - regMaskMixed registersToDump; + AllRegsMask lastDumpedRegisters; + AllRegsMask registersToDump; int lastUsedRegNumIndex; bool shouldDumpReg(regNumber regNum) { - return (registersToDump & genRegMask(regNum)) != 0; + return registersToDump.IsRegNumInMask(regNum); } void dumpRegRecordHeader(); @@ -1950,8 +1950,10 @@ class LinearScan : public LinearScanInterface #ifdef HAS_PREDICATE_REGS regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, m_FloatRegistersWithConstant, etc. + regMaskOnlyOne fixedRegs[3]; #else regMaskOnlyOne m_RegistersWithConstants[2]; + regMaskOnlyOne fixedRegs[2]; #endif void clearConstantReg(regNumber reg, var_types regType) @@ -1974,7 +1976,6 @@ class LinearScan : public LinearScanInterface } regMaskOnlyOne getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition); - regMaskMixed fixedRegs; // TODO: Should be also seperate LsraLocation nextFixedRef[REG_COUNT]; void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); LsraLocation getNextFixedRef(regNumber regNum, var_types regType) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 6fa299503d1b2d..ee58e7f0fd7866 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2344,7 +2344,7 @@ void LinearScan::buildIntervals() assert(inArgReg < REG_COUNT); mask = genRegMask(inArgReg); assignPhysReg(inArgReg, interval); - INDEBUG(registersToDump |= getRegMask(inArgReg, interval->registerType)); + INDEBUG(registersToDump.AddRegNumInMask(inArgReg, interval->registerType)); } RefPosition* pos = newRefPosition(interval, MinLocation, RefTypeParamDef, nullptr, mask); pos->setRegOptional(true); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 27507c5894c869..a839575b5479b0 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -355,6 +355,7 @@ typedef struct _regMaskAll void RemoveRegNumInMask(regNumber reg); void RemoveRegNumInMask(regNumber reg, var_types type); bool IsRegNumInMask(regNumber reg, var_types type); + bool IsRegNumInMask(regNumber reg); // Rename this to AddRegMask() void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) @@ -375,6 +376,28 @@ typedef struct _regMaskAll floatRegs |= maskToAdd; } } + + regMaskOnlyOne ApplyTo(regMaskOnlyOne dstMask, var_types type) + { + if (varTypeRegister[type] == VTR_INT) + { + dstMask &= gprRegs; + } +#ifdef HAS_PREDICATE_REGS + else if (varTypeRegister[type] == VTR_MASK) + { + dstMask &= predicateRegs; + } +#endif + else + { + assert(varTypeRegister[type] == VTR_FLOAT); + dstMask &= floatRegs; + } + + return dstMask; + } + regMaskOnlyOne GetRegTypeMask(var_types type) { if (varTypeRegister[type] == VTR_INT) @@ -435,6 +458,7 @@ typedef unsigned __int64 regMaskSmall; #endif bool operator==(const AllRegsMask& first, const AllRegsMask& second); +bool operator!=(const AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); From fba4c09b92400f47ba77fbf4d0400a6f7b84d957 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 19:50:34 -0700 Subject: [PATCH 080/201] Fix some typo issues --- src/coreclr/jit/lsra.cpp | 6 +++--- src/coreclr/jit/lsra.h | 8 ++++---- src/coreclr/jit/lsrabuild.cpp | 4 ++-- src/coreclr/jit/target.h | 21 --------------------- 4 files changed, 9 insertions(+), 30 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 7c75383d6ee925..7974a56112a52f 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5021,7 +5021,7 @@ void LinearScan::allocateRegistersMinimal() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType) + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) { regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; while (tempRegsMaskInactive != RBM_NONE) @@ -5705,7 +5705,7 @@ void LinearScan::allocateRegisters() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType) + for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) { regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; while (tempRegsMaskInactive != RBM_NONE) @@ -13538,7 +13538,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates = busyRegs.ApplyTo(candidates, regType); + candidates &= ~busyRegs.GetRegTypeMask(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index e69efde796e28e..b2243006b5a4b8 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1835,10 +1835,10 @@ class LinearScan : public LinearScanInterface void setRegsInUse(AllRegsMask regMask) { //TODO: Fix this later. - m_AvailableRegs[0] &= regMask.gprRegs; - m_AvailableRegs[1] &= regMask.floatRegs; + m_AvailableRegs[0] &= ~regMask.gprRegs; + m_AvailableRegs[1] &= ~regMask.floatRegs; #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] &= regMask.predicateRegs; + m_AvailableRegs[2] &= ~regMask.predicateRegs; #endif } void setRegInUse(regNumber reg, var_types regType) @@ -1850,7 +1850,7 @@ class LinearScan : public LinearScanInterface #else if (emitter::isGeneralRegister(reg)) #endif - { + { m_AvailableRegs[0] &= ~regMask; } else if (emitter::isFloatReg(reg)) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ee58e7f0fd7866..b686517018d819 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2818,8 +2818,8 @@ void LinearScan::buildIntervals() if (!needNonIntegerRegisters) { availableRegCount = REG_INT_COUNT; - availableFloatRegs = RBM_NONE; - availableDoubleRegs = RBM_NONE; + availableFloatRegs.OverrideAssign(RBM_NONE); + availableDoubleRegs.OverrideAssign(RBM_NONE); //availableMaskRegs = RBM_NONE; // Is this also needed? } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a839575b5479b0..71d35b028672c6 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -377,27 +377,6 @@ typedef struct _regMaskAll } } - regMaskOnlyOne ApplyTo(regMaskOnlyOne dstMask, var_types type) - { - if (varTypeRegister[type] == VTR_INT) - { - dstMask &= gprRegs; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) - { - dstMask &= predicateRegs; - } -#endif - else - { - assert(varTypeRegister[type] == VTR_FLOAT); - dstMask &= floatRegs; - } - - return dstMask; - } - regMaskOnlyOne GetRegTypeMask(var_types type) { if (varTypeRegister[type] == VTR_INT) From 7401f00b8877ea6e50c722cb32822a23d002163b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 19:54:13 -0700 Subject: [PATCH 081/201] Remove regMaskMixed: getMatchingConstants --- src/coreclr/jit/lsra.cpp | 2 +- src/coreclr/jit/lsra.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 7974a56112a52f..652aa250f88418 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -288,7 +288,7 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo nextFixedRef[regRecord->regNum] = nextLocation; } -regMaskOnlyOne LinearScan::getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition) +regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regTypeIndex(currentInterval->registerType)]); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index b2243006b5a4b8..bce4a5a0268691 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1974,7 +1974,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regMask = getRegMask(reg, regType); return (m_RegistersWithConstants[regTypeIndex(regType)] & regMask) == regMask; } - regMaskOnlyOne getMatchingConstants(regMaskMixed mask, Interval* currentInterval, RefPosition* refPosition); + regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); LsraLocation nextFixedRef[REG_COUNT]; void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); From 0d6a0bfe6fa7afd54d830cc510d1731c7299c2d2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 20:07:27 -0700 Subject: [PATCH 082/201] Remove regMaskMixed: placedArgRegs --- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 17 +++++++++-------- src/coreclr/jit/lsraxarch.cpp | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index bce4a5a0268691..6339f7d06f57dd 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1705,7 +1705,7 @@ class LinearScan : public LinearScanInterface // PUTARG_REG node. Tracked between the PUTARG_REG and its corresponding // CALL node and is used to avoid preferring these registers for locals // which would otherwise force a spill. - regMaskMixed placedArgRegs; + AllRegsMask placedArgRegs; struct PlacedLocal { diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index b9dce8d7121330..d5d2bb6bc436c7 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -411,7 +411,7 @@ int LinearScan::BuildCall(GenTreeCall* call) } // No args are placed in registers anymore. - placedArgRegs = RBM_NONE; + placedArgRegs = AllRegsMask(); numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index b686517018d819..65a1c89b8b4205 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2411,7 +2411,7 @@ void LinearScan::buildIntervals() } numPlacedArgLocals = 0; - placedArgRegs = RBM_NONE; + placedArgRegs = AllRegsMask(); BasicBlock* predBlock = nullptr; BasicBlock* prevBlock = nullptr; @@ -3255,7 +3255,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // _after_ the call, then we are going to prefer callee-saved registers for // such local anyway, so there is no need to look at such local uses. // - if (placedArgRegs == RBM_NONE) + if (placedArgRegs.IsEmpty()) { return; } @@ -3270,7 +3270,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Find the registers that we should remove from the preference set because // they are occupied with argument values. - regMaskMixed unpref = placedArgRegs; + AllRegsMask unpref = placedArgRegs; unsigned varIndex = interval->getVarIndex(compiler); for (size_t i = 0; i < numPlacedArgLocals; i++) { @@ -3278,11 +3278,11 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) { // This local's value is going to be available in this register so // keep it in the preferences. - unpref &= ~genRegMask(placedArgLocals[i].Reg); + unpref.RemoveRegNumInMask(placedArgLocals[i].Reg); } } - if (unpref != RBM_NONE) + if (!unpref.IsEmpty()) { #ifdef DEBUG if (VERBOSE) @@ -3294,8 +3294,9 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) } #endif - interval->registerAversion |= unpref; - regMaskOnlyOne newPreferences = allRegs(interval->registerType) & ~unpref; + regMaskOnlyOne unprefRegMask = unpref.GetRegTypeMask(interval->registerType); + interval->registerAversion |= unprefRegMask; + regMaskOnlyOne newPreferences = allRegs(interval->registerType) & ~unprefRegMask; interval->updateRegisterPreferences(newPreferences); } } @@ -4275,7 +4276,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) RefPosition* use = BuildUse(op1, argMask); // Record that this register is occupied by a register now. - placedArgRegs |= argMask; + placedArgRegs |= argReg; if (supportsSpecialPutArg() && isCandidateLocalRef(op1) && ((op1->gtFlags & GTF_VAR_DEATH) == 0)) { diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index df48e80c4ff3af..f4e1615dfa0b65 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1379,7 +1379,7 @@ int LinearScan::BuildCall(GenTreeCall* call) } // No args are placed in registers anymore. - placedArgRegs = RBM_NONE; + placedArgRegs = AllRegsMask(); numPlacedArgLocals = 0; return srcCount; } From 91ae489bddab396ec1c3315989d39be89c169c85 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 20:09:51 -0700 Subject: [PATCH 083/201] Remove regMaskMixed from lsra.h --- src/coreclr/jit/lsra.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 6339f7d06f57dd..1c255616f92c46 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1692,7 +1692,7 @@ class LinearScan : public LinearScanInterface #if defined(TARGET_XARCH) PhasedVar availableMaskRegs; #endif - PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, + PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, // predicate #if defined(TARGET_XARCH) From 3d3938e4e2538808ee14adcc49f70426ba43fd91 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 20:18:00 -0700 Subject: [PATCH 084/201] Remove regMaskMixed : lsra.cpp mostly --- src/coreclr/jit/lsra.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 652aa250f88418..ddb4440f37e238 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2779,7 +2779,7 @@ void LinearScan::setFrameType() // If we are using FPBASE as the frame register, we cannot also use it for // a local var. - regMaskMixed removeMask = RBM_NONE; + regMaskGpr removeMask = RBM_NONE; if (frameType == FT_EBP_FRAME) { removeMask |= RBM_FPBASE; @@ -2862,7 +2862,7 @@ RegisterType LinearScan::getRegisterType(Interval* currentInterval, RefPosition* { assert(refPosition->getInterval() == currentInterval); RegisterType regType = currentInterval->registerType; - regMaskMixed candidates = refPosition->registerAssignment; + regMaskOnlyOne candidates = refPosition->registerAssignment; #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // The LoongArch64's ABI which the float args maybe passed by integer register // when no float register left but free integer register. @@ -6201,7 +6201,7 @@ void LinearScan::allocateRegisters() // it might be beneficial to keep it in this reg for PART of the lifetime if (currentInterval->isLocalVar) { - regMaskMixed preferences = currentInterval->registerPreferences; + regMaskOnlyOne preferences = currentInterval->registerPreferences; bool keepAssignment = true; bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; @@ -6310,8 +6310,8 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { lastAllocatedRefPosition = ¤tRefPosition; - regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -6408,8 +6408,8 @@ void LinearScan::allocateRegisters() } lastAllocatedRefPosition = ¤tRefPosition; - regMaskMixed copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskMixed assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.needsConsecutive) @@ -8317,7 +8317,7 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { - regMaskMixed initialRegMask = interval->firstRefPosition->registerAssignment; + regMaskOnlyOne initialRegMask = interval->firstRefPosition->registerAssignment; regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK : genRegNumFromMask(initialRegMask); @@ -12173,7 +12173,7 @@ void LinearScan::verifyFinalAllocation() // However, we will assert that, at resolution time, no registers contain GC refs. { DBEXEC(VERBOSE, printf(" ")); - regMaskMixed candidateRegs = currentRefPosition.registerAssignment; + regMaskOnlyOne candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -12560,7 +12560,7 @@ bool LinearScan::RegisterSelection::applySelection(int selectionScore, regMaskOn { assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidates)); - regMaskMixed newCandidates = candidates & selectionCandidates; + regMaskOnlyOne newCandidates = candidates & selectionCandidates; if (newCandidates != RBM_NONE) { candidates = newCandidates; @@ -12584,7 +12584,7 @@ bool LinearScan::RegisterSelection::applySingleRegSelection(int selectionScore, assert(linearScan->compiler->IsOnlyOneRegMask(selectionCandidate)); assert(LinearScan::isSingleRegister(selectionCandidate)); - regMaskMixed newCandidates = candidates & selectionCandidate; + regMaskOnlyOne newCandidates = candidates & selectionCandidate; if (newCandidates != RBM_NONE) { candidates = newCandidates; From f2b172dbe8ddbfe84d33e09663504ba871bec6a5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 20:53:08 -0700 Subject: [PATCH 085/201] Remove regMaskMixed: lsra.cpp/resolveEdge/targetRegsToDo --- src/coreclr/jit/compiler.hpp | 74 ++++++++++++++++++++++++++++++++++++ src/coreclr/jit/lsra.cpp | 55 ++++++++++++++++++--------- src/coreclr/jit/target.h | 1 + 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 7154d4c3745cfb..bab35bbf63c360 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -969,6 +969,80 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskMixed& mask) return regNum; } +//------------------------------------------------------------------------------ +// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a +// register number and also toggle the bit in the `mask`. +// Arguments: +// mask - the register mask +// +// Return Value: +// The number of the first register contained in the mask and updates the `mask` to toggle +// the bit. +// + +inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) +{ + assert(!mask.IsEmpty()); // Must have one bit set, so can't have a mask of zero + + /* Convert the mask to a register number */ + regNumber regNum; + + if (mask.gprRegs != RBM_NONE) + { + regNum = (regNumber)BitOperations::BitScanForward(mask.gprRegs); + mask.gprRegs ^= genRegMask(regNum); + } +#ifdef HAS_PREDICATE_REGS + else if (mask.predicateRegs != RBM_NONE) + { + regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs); + mask.predicateRegs ^= genRegMask(regNum); + } +#endif + else + { + regNum = (regNumber)BitOperations::BitScanForward(mask.floatRegs); + mask.floatRegs ^= genRegMask(regNum); + } + + return regNum; +} + +//------------------------------------------------------------------------------ +// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. +// +// Arguments: +// mask - the register mask +// +// Return Value: +// The number of the first register contained in the mask. +// + +inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) +{ + assert(!mask.IsEmpty()); // Must have one bit set, so can't have a mask of zero + + /* Convert the mask to a register number */ + regNumber regNum; + + if (mask.gprRegs != RBM_NONE) + { + regNum = (regNumber)BitOperations::BitScanForward(mask.gprRegs); + } +#ifdef HAS_PREDICATE_REGS + else if (mask.predicateRegs != RBM_NONE) + { + regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs); + } +#endif + else + { + regNum = (regNumber)BitOperations::BitScanForward(mask.floatRegs); + } + + return regNum; +} + //------------------------------------------------------------------------------ // genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. // diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ddb4440f37e238..95dc2345246525 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -9570,7 +9570,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } } - regMaskMixed targetRegsToDo = RBM_NONE; + AllRegsMask targetRegsToDo; regMaskMixed targetRegsReady = RBM_NONE; regMaskMixed targetRegsFromStack = RBM_NONE; @@ -9693,19 +9693,19 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, location[fromReg] = (regNumberSmall)fromReg; source[toReg] = (regNumberSmall)fromReg; sourceIntervals[fromReg] = interval; - targetRegsToDo |= genRegMask(toReg); + targetRegsToDo |= toReg; } } // REGISTER to REGISTER MOVES // First, find all the ones that are ready to move now - regMaskMixed targetCandidates = targetRegsToDo; - while (targetCandidates != RBM_NONE) + AllRegsMask targetCandidates = targetRegsToDo; + while (!targetCandidates.IsEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetCandidates); + regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetCandidates); singleRegMask targetRegMask = genRegMask(targetReg); - targetCandidates ^= targetRegMask; + targetCandidates ^= targetReg; if (location[targetReg] == REG_NA) { #ifdef TARGET_ARM @@ -9730,14 +9730,14 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } // Perform reg to reg moves - while (targetRegsToDo != RBM_NONE) + while (!targetRegsToDo.IsEmpty()) { while (targetRegsReady != RBM_NONE) { regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); singleRegMask targetRegMask = genRegMask(targetReg); - targetRegsToDo ^= targetRegMask; - targetRegsReady ^= targetRegMask; + targetRegsToDo ^= targetReg; + targetRegsReady ^= targetReg; assert(location[targetReg] != targetReg); assert(targetReg < REG_COUNT); regNumber sourceReg = (regNumber)source[targetReg]; @@ -9803,7 +9803,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } } } - if (targetRegsToDo != RBM_NONE) + if (!targetRegsToDo.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); singleRegMask targetRegMask = genRegMask(targetReg); @@ -9814,7 +9814,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber fromReg = (regNumber)location[sourceReg]; if (targetReg == fromReg) { - targetRegsToDo &= ~targetRegMask; + targetRegsToDo.RemoveRegNumInMask(targetReg); } else { @@ -9857,16 +9857,15 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Otherwise, we'll spill it to the stack and reload it later. if (useSwap) { - singleRegMask fromRegMask = genRegMask(fromReg); - targetRegsToDo &= ~fromRegMask; + targetRegsToDo.RemoveRegNumInMask(fromReg); } } else { // Look at the remaining registers from targetRegsToDo (which we expect to be relatively // small at this point) to find out what's currently in targetReg. - regMaskMixed mask = targetRegsToDo; - while (mask != RBM_NONE && otherTargetReg == REG_NA) + AllRegsMask mask = targetRegsToDo; + while (!mask.IsEmpty() && otherTargetReg == REG_NA) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(mask); if (location[source[nextReg]] == targetReg) @@ -9905,7 +9904,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, singleRegMask otherTargetRegMask = genRegMask(otherTargetReg); targetRegsFromStack |= otherTargetRegMask; stackToRegIntervals[otherTargetReg] = otherInterval; - targetRegsToDo &= ~otherTargetRegMask; + targetRegsToDo.RemoveRegNumInMask(otherTargetReg); // Now, move the interval that is going to targetReg. addResolution(block, insertionPoint, sourceIntervals[sourceReg], targetReg, @@ -9937,7 +9936,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, #endif // TARGET_ARM } } - targetRegsToDo &= ~targetRegMask; + targetRegsToDo.RemoveRegNumInMask(targetReg); } else { @@ -14077,6 +14076,28 @@ AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second) return first; } + +AllRegsMask operator^=(AllRegsMask& first, const regNumber reg) +{ + if (emitter::isGeneralRegister(reg)) + { + first.gprRegs ^= genRegMask(reg); + } + else if (emitter::isFloatReg(reg)) + { + first.floatRegs ^= genRegMask(reg); + } + else + { +#ifdef HAS_PREDICATE_REGS + first.predicateRegs ^= genRegMask(reg); +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } + return first; +} + AllRegsMask operator~(const AllRegsMask& first) { AllRegsMask result(~first.gprRegs, ~first.floatRegs diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 71d35b028672c6..41e01bf9bbbb91 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -444,6 +444,7 @@ AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& s AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second); AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); +AllRegsMask operator^=(AllRegsMask& first, const regNumber reg); // AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); AllRegsMask operator~(const AllRegsMask& first); // inline AllRegsMask createRegMask(regNumber reg) From 0f0c60e0480bb3157b34c4d844c33b30f96fc3c2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 20:57:15 -0700 Subject: [PATCH 086/201] Remove regMaskMixed: lsra.cpp/resolveEdge/targetRegsReady --- src/coreclr/jit/lsra.cpp | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 95dc2345246525..c5538d0ef94ee3 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -9571,7 +9571,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } AllRegsMask targetRegsToDo; - regMaskMixed targetRegsReady = RBM_NONE; + AllRegsMask targetRegsReady; regMaskMixed targetRegsFromStack = RBM_NONE; // The following arrays capture the location of the registers as they are moved: @@ -9704,7 +9704,6 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, while (!targetCandidates.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetCandidates); - singleRegMask targetRegMask = genRegMask(targetReg); targetCandidates ^= targetReg; if (location[targetReg] == REG_NA) { @@ -9718,13 +9717,13 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber anotherHalfRegNum = REG_NEXT(targetReg); if (location[anotherHalfRegNum] == REG_NA) { - targetRegsReady |= targetRegMask; + targetRegsReady |= targetReg; } } else #endif // TARGET_ARM { - targetRegsReady |= targetRegMask; + targetRegsReady |= targetReg; } } } @@ -9732,10 +9731,9 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Perform reg to reg moves while (!targetRegsToDo.IsEmpty()) { - while (targetRegsReady != RBM_NONE) + while (!targetRegsReady.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); - singleRegMask targetRegMask = genRegMask(targetReg); targetRegsToDo ^= targetReg; targetRegsReady ^= targetReg; assert(location[targetReg] != targetReg); @@ -9758,7 +9756,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { if (source[fromReg] != REG_NA && ((targetRegsFromStack & fromRegMask) != fromRegMask)) { - targetRegsReady |= fromRegMask; + targetRegsReady |= fromReg; #ifdef TARGET_ARM if (genIsValidDoubleReg(fromReg)) { @@ -9769,7 +9767,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber upperHalfReg = REG_NEXT(fromReg); if ((otherInterval->registerType == TYP_DOUBLE) && (location[upperHalfReg] != REG_NA)) { - targetRegsReady &= ~fromRegMask; + targetRegsReady.RemoveRegNumInMask(fromReg); } } } @@ -9791,13 +9789,13 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // lowerHalfRegMask) if ((lowerHalfSrcReg != REG_NA) && (lowerHalfSrcLoc == REG_NA) && (sourceIntervals[lowerHalfSrcReg] != nullptr) && - ((targetRegsReady & lowerHalfRegMask) == RBM_NONE) && + !targetRegsReady.IsRegNumInMask(lowerHalfReg) && ((targetRegsFromStack & lowerHalfRegMask) != lowerHalfRegMask)) { // This must be a double interval, otherwise it would be in targetRegsReady, or already // completed. assert(sourceIntervals[lowerHalfSrcReg]->registerType == TYP_DOUBLE); - targetRegsReady |= lowerHalfRegMask; + targetRegsReady |= lowerHalfReg; } #endif // TARGET_ARM } @@ -9806,7 +9804,6 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, if (!targetRegsToDo.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); - singleRegMask targetRegMask = genRegMask(targetReg); // Is it already there due to other moves? // If not, move it to the temp reg, OR swap it with another register @@ -9918,8 +9915,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // value will be retrieved from STK. if (source[fromReg] != REG_NA && fromReg != otherTargetReg) { - singleRegMask fromRegMask = genRegMask(fromReg); - targetRegsReady |= fromRegMask; + targetRegsReady |= fromReg; #ifdef TARGET_ARM if (genIsValidDoubleReg(fromReg)) { @@ -9930,7 +9926,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber upperHalfReg = REG_NEXT(fromReg); if ((otherInterval->registerType == TYP_DOUBLE) && (location[upperHalfReg] != REG_NA)) { - targetRegsReady &= ~fromRegMask; + targetRegsReady.RemoveRegNumInMask(fromReg); } } #endif // TARGET_ARM @@ -9968,7 +9964,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, DEBUG_ARG(resolveTypeName[resolveType])); location[targetReg] = (regNumberSmall)tempReg; } - targetRegsReady |= targetRegMask; + targetRegsReady |= targetReg; } } } From 2fbef7965e6c4534ab3aac3b33249220b76a91c0 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 21:01:30 -0700 Subject: [PATCH 087/201] Remove regMaskMixed: lsra.cpp/resolveEdge/targetRegsFromStack --- src/coreclr/jit/lsra.cpp | 41 ++++++++++++++++++++++++++++++---------- src/coreclr/jit/target.h | 1 + 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index c5538d0ef94ee3..6e0652b0e73ce1 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -9572,7 +9572,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, AllRegsMask targetRegsToDo; AllRegsMask targetRegsReady; - regMaskMixed targetRegsFromStack = RBM_NONE; + AllRegsMask targetRegsFromStack; // The following arrays capture the location of the registers as they are moved: // - location[reg] gives the current location of the var that was originally in 'reg'. @@ -9679,7 +9679,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, if (fromReg == REG_STK) { stackToRegIntervals[toReg] = interval; - targetRegsFromStack |= genRegMask(toReg); + targetRegsFromStack |= toReg; } else if (toReg == REG_STK) { @@ -9754,7 +9754,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Do we have a free targetReg? if (fromReg == sourceReg) { - if (source[fromReg] != REG_NA && ((targetRegsFromStack & fromRegMask) != fromRegMask)) + if (source[fromReg] != REG_NA && !targetRegsFromStack.IsOnlyRegNumInMask(fromReg)) { targetRegsReady |= fromReg; #ifdef TARGET_ARM @@ -9790,7 +9790,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, if ((lowerHalfSrcReg != REG_NA) && (lowerHalfSrcLoc == REG_NA) && (sourceIntervals[lowerHalfSrcReg] != nullptr) && !targetRegsReady.IsRegNumInMask(lowerHalfReg) && - ((targetRegsFromStack & lowerHalfRegMask) != lowerHalfRegMask)) + !targetRegsFromStack.IsOnlyRegNumInMask(lowerHalfReg)) { // This must be a double interval, otherwise it would be in targetRegsReady, or already // completed. @@ -9898,8 +9898,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, DEBUG_ARG(resolveTypeName[resolveType])); location[source[otherTargetReg]] = REG_STK; - singleRegMask otherTargetRegMask = genRegMask(otherTargetReg); - targetRegsFromStack |= otherTargetRegMask; + targetRegsFromStack |= otherTargetReg; stackToRegIntervals[otherTargetReg] = otherInterval; targetRegsToDo.RemoveRegNumInMask(otherTargetReg); @@ -9972,7 +9971,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Finally, perform stack to reg moves // All the target regs will be empty at this point - while (targetRegsFromStack != RBM_NONE) + while (!targetRegsFromStack.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetRegsFromStack); @@ -14178,18 +14177,40 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) bool AllRegsMask::IsRegNumInMask(regNumber reg) { + regMaskOnlyOne regMask = genRegMask(reg); if (emitter::isGeneralRegister(reg)) { - return (gprRegs & genRegMask(reg)) != RBM_NONE; + return (gprRegs & regMask) != RBM_NONE; } else if (emitter::isFloatReg(reg)) { - return (floatRegs & genRegMask(reg)) != RBM_NONE; + return (floatRegs & regMask) != RBM_NONE; } else { #ifdef HAS_PREDICATE_REGS - return (predicateRegs & genRegMask(reg)) != RBM_NONE; + return (predicateRegs & regMask) != RBM_NONE; +#else + unreached(); +#endif // HAS_PREDICATE_REGS + } +} + +bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMask = genRegMask(reg); + if (emitter::isGeneralRegister(reg)) + { + return (gprRegs & regMask) == regMask; + } + else if (emitter::isFloatReg(reg)) + { + return (floatRegs & regMask) == regMask; + } + else + { +#ifdef HAS_PREDICATE_REGS + return (predicateRegs & regMask) == regMask; #else unreached(); #endif // HAS_PREDICATE_REGS diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 41e01bf9bbbb91..c404afc468debe 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -356,6 +356,7 @@ typedef struct _regMaskAll void RemoveRegNumInMask(regNumber reg, var_types type); bool IsRegNumInMask(regNumber reg, var_types type); bool IsRegNumInMask(regNumber reg); + bool IsOnlyRegNumInMask(regNumber reg); // Rename this to AddRegMask() void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) From dbb9be34ecec1d4155b765bf9740368461e668a2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 21:04:15 -0700 Subject: [PATCH 088/201] Complete the remaining ones --- src/coreclr/jit/compiler.hpp | 4 ++-- src/coreclr/jit/lsra.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index bab35bbf63c360..274f6d6cc9d25f 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -957,7 +957,7 @@ inline regNumber genRegNumFromMask(regMaskOnlyOne mask) // the bit. // -inline regNumber genFirstRegNumFromMaskAndToggle(regMaskMixed& mask) +inline regNumber genFirstRegNumFromMaskAndToggle(regMaskOnlyOne& mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -1053,7 +1053,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(regMaskMixed mask) +inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 6e0652b0e73ce1..676bfaef00507c 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -773,7 +773,7 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_XARCH) rbmAllMask = compiler->rbmAllMask; rbmMskCalleeTrash = compiler->rbmMskCalleeTrash; - memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskMixed) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskOnlyOne) * TYP_COUNT); if (!compiler->canUseEvexEncoding()) { From 138862253f28f3d7ce8da8972bc93b31851d2ae7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 22:05:37 -0700 Subject: [PATCH 089/201] merge errors --- src/coreclr/jit/codegeninterface.h | 3 +++ src/coreclr/jit/lsra.cpp | 8 +++++--- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/lsraxarch.cpp | 2 +- src/coreclr/jit/targetamd64.h | 6 ++++++ 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index ec527efbb5b0c5..7cc532e4fc1f72 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -158,6 +158,9 @@ class CodeGenInterface bool genTempLiveChg; #endif + VARSET_TP genLastLiveSet; // A one element map (genLastLiveSet-> genLastLiveMask) + regMaskTP genLastLiveMask; // these two are used in genLiveMask + regMaskOnlyOne genGetRegMask(const LclVarDsc* varDsc); regMaskOnlyOne genGetRegMask(GenTree* tree); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 1bdc411e2038c3..ceccf0e1f402b4 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5233,7 +5233,8 @@ void LinearScan::allocateRegistersMinimal() #ifdef SWIFT_SUPPORT if (currentRefPosition.delayRegFree) { - regsInUseNextLocation |= currentRefPosition.registerAssignment; + regsInUseNextLocation.AddRegTypeMask(currentRefPosition.registerAssignment, + regRecord->registerType); } #endif // SWIFT_SUPPORT } @@ -5952,7 +5953,8 @@ void LinearScan::allocateRegisters() #ifdef SWIFT_SUPPORT if (currentRefPosition.delayRegFree) { - regsInUseNextLocation |= currentRefPosition.registerAssignment; + regsInUseNextLocation.AddRegTypeMask(currentRefPosition.registerAssignment, + regRecord->registerType); } #endif // SWIFT_SUPPORT } @@ -6100,7 +6102,7 @@ void LinearScan::allocateRegisters() updateSpillCost(assignedRegister, currentInterval); } - regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, nullptr, assignedRegister)); currentRefPosition.registerAssignment = RBM_NONE; diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 2038bb0bd49a64..7f9b054fd9d65a 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -414,7 +414,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (call->HasSwiftErrorHandling()) { // Tree is a Swift call with error handling; error register should have been killed - assert((killMask & RBM_SWIFT_ERROR) != 0); + assert((killMask.gprRegs & RBM_SWIFT_ERROR) != 0); // After a Swift call that might throw returns, we expect the error register to be consumed // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 99a2bd267211b2..f403e2e63e8746 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -920,7 +920,7 @@ AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) // so don't use the register post-call until it is consumed by SwiftError. if (call->HasSwiftErrorHandling()) { - killMask |= RBM_SWIFT_ERROR; + killMask.gprRegs |= RBM_SWIFT_ERROR; } #endif // SWIFT_SUPPORT diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index d49021c3929598..3d1e79e851499c 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1395,7 +1395,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (call->HasSwiftErrorHandling()) { // Tree is a Swift call with error handling; error register should have been killed - assert((killMask & RBM_SWIFT_ERROR) != 0); + assert((killMask.gprRegs & RBM_SWIFT_ERROR) != 0); // After a Swift call that might throw returns, we expect the error register to be consumed // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 5318a0de0668ff..b2a68bdaad26c6 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -211,9 +211,15 @@ // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_RAX | RBM_RCX) + #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) +// Registers killed by CORINFO_HELP_ASSIGN_BYREF. + #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF) + // We have two register classifications // * callee trash: aka volatile or caller saved // * callee saved: aka non-volatile From 9c4942df40832bc8115e18ccda1512d0c71340d3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 10 Mar 2024 22:06:04 -0700 Subject: [PATCH 090/201] jit format --- src/coreclr/jit/codegen.h | 5 +- src/coreclr/jit/codegenarm.cpp | 18 ++-- src/coreclr/jit/codegenarm64.cpp | 24 ++--- src/coreclr/jit/codegenarmarch.cpp | 12 +-- src/coreclr/jit/codegencommon.cpp | 57 ++++++------ src/coreclr/jit/codegeninterface.h | 4 +- src/coreclr/jit/codegenlinear.cpp | 4 +- src/coreclr/jit/codegenxarch.cpp | 22 ++--- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/emit.h | 4 +- src/coreclr/jit/gcinfo.cpp | 5 +- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/lsra.cpp | 139 +++++++++++++++-------------- src/coreclr/jit/lsra.h | 64 ++++++------- src/coreclr/jit/lsraarm.cpp | 8 +- src/coreclr/jit/lsraarm64.cpp | 6 +- src/coreclr/jit/lsraarmarch.cpp | 8 +- src/coreclr/jit/lsrabuild.cpp | 25 +++--- src/coreclr/jit/lsraxarch.cpp | 12 +-- src/coreclr/jit/morph.cpp | 4 +- src/coreclr/jit/regset.cpp | 10 ++- src/coreclr/jit/regset.h | 16 ++-- src/coreclr/jit/target.h | 23 +++-- src/coreclr/jit/unwind.cpp | 1 - src/coreclr/jit/utils.cpp | 8 +- 25 files changed, 249 insertions(+), 234 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index a0e4b27d8b21ac..023be30a61111a 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -385,7 +385,7 @@ class CodeGen final : public CodeGenInterface // same. struct FuncletFrameInfoDsc { - regMaskGpr fiSaveGprRegs; // Set of GPR registers saved in the funclet prolog (includes LR) + regMaskGpr fiSaveGprRegs; // Set of GPR registers saved in the funclet prolog (includes LR) regMaskFloat fiSaveFloatRegs; // Set of Float registers saved in the funclet prolog (includes LR) unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer unsigned fiSpDelta; // Stack pointer delta @@ -404,7 +404,8 @@ class CodeGen final : public CodeGenInterface { regMaskGpr fiSaveGprRegs; // Set of callee-saved GPR registers saved in the funclet prolog (includes LR) regMaskFloat fiSaveFloatRegs; // Set of callee-saved float registers saved in the funclet prolog (includes LR) - regMaskFloat fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog (includes LR) + regMaskFloat fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog + // (includes LR) int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 0e5bafca8f31d5..59a580589c4af4 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2472,11 +2472,11 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() unsigned preSpillRegArgSize = genCountBits(regSet.rsMaskPreSpillRegs(true)) * REGSIZE_BYTES; genFuncletInfo.fiFunctionCallerSPtoFPdelta = preSpillRegArgSize + 2 * REGSIZE_BYTES; - regMaskGpr rsGprMaskSaveRegs = regSet.rsGprMaskCalleeSaved; - regMaskFloat rsFloatMaskSaveRegs = regSet.rsFloatMaskCalleeSaved; + regMaskGpr rsGprMaskSaveRegs = regSet.rsGprMaskCalleeSaved; + regMaskFloat rsFloatMaskSaveRegs = regSet.rsFloatMaskCalleeSaved; unsigned saveRegsCount = genCountBits(rsGprMaskSaveRegs) + genCountBits(rsFloatMaskSaveRegs); - unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving - unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; + unsigned saveRegsSize = saveRegsCount * REGSIZE_BYTES; // bytes of regs we're saving + unsigned saveSizeWithPSP = saveRegsSize + REGSIZE_BYTES /* PSP sym */; if (compiler->lvaMonAcquired != BAD_VAR_NUM) { saveSizeWithPSP += TARGET_POINTER_SIZE; @@ -2494,8 +2494,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() /* Now save it for future use */ - genFuncletInfo.fiSaveGprRegs = rsGprMaskSaveRegs; - genFuncletInfo.fiSaveFloatRegs = rsFloatMaskSaveRegs; + genFuncletInfo.fiSaveGprRegs = rsGprMaskSaveRegs; + genFuncletInfo.fiSaveFloatRegs = rsFloatMaskSaveRegs; genFuncletInfo.fiSpDelta = spDelta; genFuncletInfo.fiPSP_slot_SP_offset = PSP_slot_SP_offset; genFuncletInfo.fiPSP_slot_CallerSP_offset = PSP_slot_CallerSP_offset; @@ -2628,9 +2628,9 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu // // str rZero1,[rAddr] // When cnt is odd - regNumber rAddr; - regNumber rCnt = REG_NA; // Invalid - regMaskGpr regMask; + regNumber rAddr; + regNumber rCnt = REG_NA; // Invalid + regMaskGpr regMask; regMaskGpr availMask = regSet.rsGetModifiedGprRegsMask() | RBM_INT_CALLEE_TRASH; // Set of available registers availMask &= ~intRegState.rsCalleeRegArgMaskLiveIn; // Remove all of the incoming argument registers as they are diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 30158936b2aef3..29ae6737ab4b5a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,7 +36,7 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskFloat rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; if (isFramePointerUsed()) @@ -1022,12 +1022,12 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int sp // None. void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, - int lowestCalleeSavedOffset, - int spDelta) + int lowestCalleeSavedOffset, + int spDelta) { assert(spDelta >= 0); - regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs; + regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs; regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs; assert(compiler->IsGprRegMask(maskRestoreRegsInt)); @@ -1743,8 +1743,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiFunction_CallerSP_to_FP_delta = genCallerSPtoFPdelta() - osrPad; - regMaskGpr rsMaskSaveGprRegs = regSet.rsGprMaskCalleeSaved; - regMaskFloat rsMaskSaveFloatRegs = regSet.rsFloatMaskCalleeSaved; + regMaskGpr rsMaskSaveGprRegs = regSet.rsGprMaskCalleeSaved; + regMaskFloat rsMaskSaveFloatRegs = regSet.rsFloatMaskCalleeSaved; regMaskPredicate rsMaskSavePredicateRegs = RBM_NONE; #ifdef HAS_PREDICATE_REGS rsMaskSavePredicateRegs = regSet.rsPredicateMaskCalleeSaved; @@ -1873,9 +1873,9 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() /* Now save it for future use */ - genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; - genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; - genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; + genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; + genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; + genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; genFuncletInfo.fiSP_to_FPLR_save_delta = SP_to_FPLR_save_delta; genFuncletInfo.fiSP_to_PSP_slot_delta = SP_to_PSP_slot_delta; genFuncletInfo.fiSP_to_CalleeSave_delta = SP_to_PSP_slot_delta + PSPSize; @@ -1887,8 +1887,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf("\n"); printf("Funclet prolog / epilog info\n"); printf(" Save regs: "); - dspRegMask( - AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs, genFuncletInfo.fiSavePredicateRegs)); + dspRegMask(AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs, + genFuncletInfo.fiSavePredicateRegs)); printf("\n"); if (compiler->opts.IsOSR()) { @@ -5132,7 +5132,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; } - regMaskGpr callTargetMask = genRegMask(callTargetReg); + regMaskGpr callTargetMask = genRegMask(callTargetReg); AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 92ec9f50a38f3c..fb6ac55adf6f09 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3526,7 +3526,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (call->IsFastTailCall()) { - regMaskGpr trashedGprByEpilog = RBM_INT_CALLEE_SAVED; + regMaskGpr trashedGprByEpilog = RBM_INT_CALLEE_SAVED; regMaskFloat trashedFloatByEpilog = RBM_FLT_CALLEE_SAVED; #ifdef HAS_PREDICATE_REGS regMaskPredicate trashedPredicateByEpilog = RBM_MSK_CALLEE_SAVED; @@ -3795,7 +3795,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // are not frequent. for (varNum = 0; varNum < compiler->info.compArgsCount; varNum++) { - varDsc = compiler->lvaGetDesc(varNum); + varDsc = compiler->lvaGetDesc(varNum); regNumber varReg = varDsc->GetRegNum(); if (varDsc->lvPromoted) @@ -4902,7 +4902,7 @@ void CodeGen::genPushCalleeSavedRegisters() intRegState.rsCalleeRegArgMaskLiveIn); #endif - regMaskGpr rsPushGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPushGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskFloat rsPushFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; #if ETW_EBP_FRAMED @@ -4937,7 +4937,7 @@ void CodeGen::genPushCalleeSavedRegisters() // rsPushGprRegs |= RBM_LR; // We must save the return address (in the LR register) - regSet.rsGprMaskCalleeSaved = rsPushGprRegs; + regSet.rsGprMaskCalleeSaved = rsPushGprRegs; regSet.rsFloatMaskCalleeSaved = rsPushFloatRegs; #ifdef DEBUG @@ -5550,8 +5550,8 @@ void CodeGen::genFnEpilog(BasicBlock* block) } if (jmpEpilog || - genStackAllocRegisterMask(compiler->compLclFrameSize, regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == - RBM_NONE) + genStackAllocRegisterMask(compiler->compLclFrameSize, + regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) { genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 3c59047050dd10..6862307b352a86 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5328,7 +5328,7 @@ void CodeGen::genFinalizeFrame() if (compiler->compLclFrameSize >= compiler->eeGetPageSize()) { regSet.rsSetGprRegsModified(RBM_STACK_PROBE_HELPER_ARG | RBM_STACK_PROBE_HELPER_CALL_TARGET | - RBM_STACK_PROBE_HELPER_TRASH); + RBM_STACK_PROBE_HELPER_TRASH); } // If there are any reserved registers, add them to the modified set. @@ -5344,9 +5344,10 @@ void CodeGen::genFinalizeFrame() printf("Modified regs: "); dspRegMask(AllRegsMask(regSet.rsGetModifiedGprRegsMask(), regSet.rsGetModifiedFloatRegsMask() #ifdef HAS_PREDICATE_REGS - ,regSet.rsGetModifiedPredicateRegsMask() + , + regSet.rsGetModifiedPredicateRegsMask() #endif - )); + )); printf("\n"); } #endif // DEBUG @@ -5399,7 +5400,7 @@ void CodeGen::genFinalizeFrame() #endif #ifdef TARGET_ARM - // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? +// TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? #endif regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; @@ -5495,7 +5496,7 @@ void CodeGen::genFinalizeFrame() if (verbose) { printf("Callee-saved registers pushed: %d ", compiler->compCalleeRegsPushed); - dspRegMask(maskPushRegsInt, maskPushRegsFloat); //TODO: Should this also have maskPushRegsPredicate + dspRegMask(maskPushRegsInt, maskPushRegsFloat); // TODO: Should this also have maskPushRegsPredicate printf("\n"); } #endif // DEBUG @@ -5834,9 +5835,9 @@ void CodeGen::genFnProlog() // Track if initReg holds non-zero value. Start conservative and assume it has non-zero value. // If initReg is ever set to zero, this variable is set to true and zero initializing initReg // will be skipped. - bool initRegZeroed = false; + bool initRegZeroed = false; regMaskGpr excludeMask = intRegState.rsCalleeRegArgMaskLiveIn; - regMaskGpr tempMask; + regMaskGpr tempMask; // We should not use the special PINVOKE registers as the initReg // since they are trashed by the jithelper call to setup the PINVOKE frame @@ -8188,16 +8189,16 @@ void CodeGen::genRegCopy(GenTree* treeNode) unsigned regCount = op1->GetMultiRegCount(compiler); assert(regCount <= MAX_MULTIREG_COUNT); - // First set the source registers as busy if they haven't been spilled. - // (Note that this is just for verification that we don't have circular dependencies.) +// First set the source registers as busy if they haven't been spilled. +// (Note that this is just for verification that we don't have circular dependencies.) #ifdef DEBUG AllRegsMask busyRegs; for (unsigned i = 0; i < regCount; ++i) { if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0) { - regNumber reg = op1->GetRegByIndex(i); - //regMaskOnlyOne regMask = genRegMask(); + regNumber reg = op1->GetRegByIndex(i); + // regMaskOnlyOne regMask = genRegMask(); if (genIsValidIntReg(reg)) { busyRegs.gprRegs |= genRegMask(reg); @@ -8225,13 +8226,17 @@ void CodeGen::genRegCopy(GenTree* treeNode) #ifdef DEBUG -#define DO_VALIDATION(regType) \ -if (targetReg != sourceReg) \ -{ \ - singleRegMask targetRegMask = genRegMask(targetReg); \ - assert((busyRegs.regType & targetRegMask) == 0); \ - busyRegs.regType &= ~genRegMask(sourceReg); \ -} \ +#define DO_VALIDATION(regType) \ + \ +if(targetReg != sourceReg) \ + \ +{ \ + singleRegMask targetRegMask = genRegMask(targetReg); \ + assert((busyRegs.regType & targetRegMask) == 0); \ + busyRegs.regType &= ~genRegMask(sourceReg); \ + \ +} \ + \ busyRegs.regType |= genRegMask(targetReg); if (genIsValidIntReg(targetReg)) @@ -8248,14 +8253,14 @@ busyRegs.regType |= genRegMask(targetReg); DO_VALIDATION(predicateRegs) } #endif // HAS_PREDICATE_REGS - //if (targetReg != sourceReg) - //{ - // singleRegMask targetRegMask = genRegMask(targetReg); - // assert((busyRegs & targetRegMask) == 0); - // // Clear sourceReg from the busyRegs, and add targetReg. - // busyRegs &= ~genRegMask(sourceReg); - //} - //busyRegs |= genRegMask(targetReg); + // if (targetReg != sourceReg) +//{ +// singleRegMask targetRegMask = genRegMask(targetReg); +// assert((busyRegs & targetRegMask) == 0); +// // Clear sourceReg from the busyRegs, and add targetReg. +// busyRegs &= ~genRegMask(sourceReg); +//} +// busyRegs |= genRegMask(targetReg); #endif // DEBUG } return; diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index 7cc532e4fc1f72..6d28e6a251b5c6 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -158,8 +158,8 @@ class CodeGenInterface bool genTempLiveChg; #endif - VARSET_TP genLastLiveSet; // A one element map (genLastLiveSet-> genLastLiveMask) - regMaskTP genLastLiveMask; // these two are used in genLiveMask + VARSET_TP genLastLiveSet; // A one element map (genLastLiveSet-> genLastLiveMask) + regMaskTP genLastLiveMask; // these two are used in genLiveMask regMaskOnlyOne genGetRegMask(const LclVarDsc* varDsc); regMaskOnlyOne genGetRegMask(GenTree* tree); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 6fab00509bbda2..18785c8bd59333 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -202,8 +202,8 @@ void CodeGen::genCodeForBBlist() // here. That would require handling the changes in recordVarLocationsAtStartOfBB(). AllRegsMask newLiveRegSet; - regMaskGpr newRegGCrefSet = RBM_NONE; - regMaskGpr newRegByrefSet = RBM_NONE; + regMaskGpr newRegGCrefSet = RBM_NONE; + regMaskGpr newRegByrefSet = RBM_NONE; #ifdef DEBUG VARSET_TP removedGCVars(VarSetOps::MakeEmpty(compiler)); VARSET_TP addedGCVars(VarSetOps::MakeEmpty(compiler)); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index aaf214147b24b9..7edeb0c178c459 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9077,8 +9077,8 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); - regNumber callTarget = REG_NA; - AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + regNumber callTarget = REG_NA; + AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { @@ -9721,7 +9721,7 @@ void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() // Figure out which set of int callee saves was already saved by Tier0. // Emit appropriate unwind. // - PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; + PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; regMaskGpr tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; int const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; @@ -9800,7 +9800,7 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() // Figure out which set of int callee saves still needs saving. // - PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; + PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; regMaskGpr tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; unsigned const tier0IntCalleeSaveUsedSize = genCountBits(tier0IntCalleeSaves) * REGSIZE_BYTES; regMaskGpr const osrIntCalleeSaves = rsPushRegs & RBM_OSR_INT_CALLEE_SAVED; @@ -10135,13 +10135,13 @@ void CodeGen::genFnEpilog(BasicBlock* block) PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; regMaskGpr const tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; - regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; - regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; - unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; - unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; - unsigned const osrCalleeSaveSize = compiler->compCalleeRegsPushed * REGSIZE_BYTES; - unsigned const osrFramePointerSize = isFramePointerUsed() ? REGSIZE_BYTES : 0; - unsigned const osrAdjust = + regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; + regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; + unsigned const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; + unsigned const tier0IntCalleeSaveUsedSize = genCountBits(allIntCalleeSaves) * REGSIZE_BYTES; + unsigned const osrCalleeSaveSize = compiler->compCalleeRegsPushed * REGSIZE_BYTES; + unsigned const osrFramePointerSize = isFramePointerUsed() ? REGSIZE_BYTES : 0; + unsigned const osrAdjust = tier0FrameSize - tier0IntCalleeSaveUsedSize + osrCalleeSaveSize + osrFramePointerSize; JITDUMP("OSR epilog adjust factors: tier0 frame %u, tier0 callee saves -%u, osr callee saves %u, osr " diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 5702c8b1bb2a0b..a9d8792bb224c5 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3634,7 +3634,7 @@ void Compiler::dumpRegMask(regMaskOnlyOne mask) const dumpRegMask(AllRegsMask(mask, mask, mask)); } - //------------------------------------------------------------------------ +//------------------------------------------------------------------------ // dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of // a potentially large number of registers. // diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index e3d95c1cf476a5..7171cd9bc21ebf 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3202,8 +3202,8 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); - void emitDispGprRegSet(regMaskGpr regs); + void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); + void emitDispGprRegSet(regMaskGpr regs); void emitDispRegSet(AllRegsMask regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 47974fdcb8bd95..0cdbd5e94a15ce 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -182,7 +182,6 @@ void GCInfo::gcMarkRegSetByref(regMaskGpr regMask DEBUGARG(bool forceOutput)) gcRegGCrefSetCur = gcRegGCrefSetNew; } - /***************************************************************************** * * Mark the gpr register as holding non-pointer values. @@ -198,7 +197,7 @@ void GCInfo::gcMarkGprRegNpt(regNumber reg DEBUGARG(bool forceOutput)) /***************************************************************************** * * Mark the register as holding non-pointer values. - * + * */ void GCInfo::gcMarkRegNpt(regNumber reg DEBUGARG(bool forceOutput)) @@ -807,6 +806,6 @@ void GCInfo::gcUpdateForRegVarMove(regMaskOnlyOne srcMask, regMaskOnlyOne dstMas VarSetOps::AddElemD(compiler, gcVarPtrSetCur, varDsc->lvVarIndex); } } -#endif +#endif /*****************************************************************************/ /*****************************************************************************/ diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 0cef65a07d60c3..5adf5339bdf9b4 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -967,7 +967,7 @@ struct GenTree regMaskOnlyOne gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskOnlyOne mask = (regMaskOnlyOne)-1) const; + unsigned AvailableTempRegCount(regMaskOnlyOne mask = (regMaskOnlyOne)-1) const; regNumber GetSingleTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); regNumber ExtractTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ceccf0e1f402b4..8dd00c666e413a 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -278,7 +278,7 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo if (nextRefPosition == nullptr) { nextLocation = MaxLocation; - fixedRegs[regTypeIndex(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); + fixedRegs[regTypeIndex(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); } else { @@ -288,7 +288,9 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo nextFixedRef[regRecord->regNum] = nextLocation; } -regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition) +regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, + Interval* currentInterval, + RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regTypeIndex(currentInterval->registerType)]); @@ -384,9 +386,9 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) // interval - Interval of Refposition. // assignedReg - Assigned register for this refposition. // -void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - AllRegsMask* regsToFree, +void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + AllRegsMask* regsToFree, AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { @@ -473,11 +475,11 @@ RegRecord* LinearScan::getRegisterRecord(regNumber regNum) // New regMask that has minRegCount registers after intersection. // Otherwise returns regMaskActual. // -regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, - RegisterType regType, +regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, + RegisterType regType, regMaskOnlyOne regMaskActual, regMaskOnlyOne regMaskConstraint, - unsigned minRegCount) + unsigned minRegCount) { assert(compiler->IsOnlyOneRegMask(regMaskActual)); assert(compiler->IsOnlyOneRegMask(regMaskConstraint)); @@ -550,11 +552,11 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly { // The refPosition could be null, for example when called // by getTempRegForResolution(). - int minRegCount = (refPosition != nullptr) ? refPosition->minRegCandidateCount : 1; - RegisterType regType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType - : refPosition->getReg()->registerType; - regMaskOnlyOne calleeSaved = RBM_NONE; - regMaskOnlyOne calleeTrash = RBM_NONE; + int minRegCount = (refPosition != nullptr) ? refPosition->minRegCandidateCount : 1; + RegisterType regType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType + : refPosition->getReg()->registerType; + regMaskOnlyOne calleeSaved = RBM_NONE; + regMaskOnlyOne calleeTrash = RBM_NONE; if (regType == IntRegisterType) { calleeSaved = RBM_INT_CALLEE_SAVED; @@ -2876,7 +2878,7 @@ bool copyOrMoveRegInUse(RefPosition* ref, LsraLocation loc) RegisterType LinearScan::getRegisterType(Interval* currentInterval, RefPosition* refPosition) { assert(refPosition->getInterval() == currentInterval); - RegisterType regType = currentInterval->registerType; + RegisterType regType = currentInterval->registerType; regMaskOnlyOne candidates = refPosition->registerAssignment; #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // The LoongArch64's ABI which the float args maybe passed by integer register @@ -2993,9 +2995,9 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!enregisterLocalVars); - regNumber foundReg; + regNumber foundReg; singleRegMask foundRegBit; - RegRecord* availablePhysRegRecord; + RegRecord* availablePhysRegRecord; foundRegBit = regSelector->selectMinimal(currentInterval, refPosition DEBUG_ARG(registerScore)); if (foundRegBit == RBM_NONE) { @@ -3510,7 +3512,7 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - regNumber reg = regRec->regNum; + regNumber reg = regRec->regNum; singleRegMask assignedRegMask = genRegMask(reg); // TODO: genIsValid* is heavy operation because it does 2 comparisons, but unfortunately @@ -4106,7 +4108,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) template void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { - //TODO: Save current_ConstantRegs and current_AvailableRegs + // TODO: Save current_ConstantRegs and current_AvailableRegs assert(currentBlock != nullptr); markBlockVisited(currentBlock); @@ -4172,8 +4174,8 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar #endif // HAS_PREDICATE_REGS } - regNumber firstReg = REG_NA; - regNumber newReg = REG_NA; + regNumber firstReg = REG_NA; + regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -4477,7 +4479,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // inactive registers available for the rotation. AllRegsMask inactiveRegs; #endif // DEBUG - AllRegsMask liveRegs; + AllRegsMask liveRegs; VarSetOps::Iter iter(compiler, currentLiveVars); unsigned varIndex = 0; while (iter.NextElem(&varIndex)) @@ -4754,7 +4756,8 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (assignedInterval != nullptr) { - assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + assert(assignedInterval->isLocalVar || assignedInterval->isConstant || + assignedInterval->IsUpperVector()); if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) { @@ -5012,12 +5015,12 @@ void LinearScan::allocateRegistersMinimal() BasicBlock* currentBlock = nullptr; - LsraLocation prevLocation = MinLocation; - AllRegsMask regsToFree; - AllRegsMask delayRegsToFree; - AllRegsMask regsToMakeInactive; - AllRegsMask delayRegsToMakeInactive; - AllRegsMask copyRegsToFree ; + LsraLocation prevLocation = MinLocation; + AllRegsMask regsToFree; + AllRegsMask delayRegsToFree; + AllRegsMask regsToMakeInactive; + AllRegsMask delayRegsToMakeInactive; + AllRegsMask copyRegsToFree; regsInUseThisLocation = AllRegsMask(); regsInUseNextLocation = AllRegsMask(); @@ -5486,7 +5489,7 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); if (currentRefPosition.delayRegFree) @@ -5496,7 +5499,8 @@ void LinearScan::allocateRegistersMinimal() currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumInMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead + regsToFree.RemoveRegNumInMask(assignedRegister, + currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -5695,7 +5699,7 @@ void LinearScan::allocateRegisters() "--------------------\n"); // Start with a small set of commonly used registers, so that we don't keep having to print a new title. // Include all the arg regs, as they may already have values assigned to them. - registersToDump = AllRegsMask(LsraLimitSmallIntSet | RBM_ARG_REGS, LsraLimitSmallFPSet , RBM_NONE); + registersToDump = AllRegsMask(LsraLimitSmallIntSet | RBM_ARG_REGS, LsraLimitSmallFPSet, RBM_NONE); dumpRegRecordHeader(); // Now print an empty "RefPosition", since we complete the dump of the regs at the beginning of the loop. printf(indentFormat, ""); @@ -5704,12 +5708,12 @@ void LinearScan::allocateRegisters() BasicBlock* currentBlock = nullptr; - LsraLocation prevLocation = MinLocation; - AllRegsMask regsToFree; - AllRegsMask delayRegsToFree; - AllRegsMask regsToMakeInactive; - AllRegsMask delayRegsToMakeInactive; - AllRegsMask copyRegsToFree; + LsraLocation prevLocation = MinLocation; + AllRegsMask regsToFree; + AllRegsMask delayRegsToFree; + AllRegsMask regsToMakeInactive; + AllRegsMask delayRegsToMakeInactive; + AllRegsMask copyRegsToFree; regsInUseThisLocation = AllRegsMask(); regsInUseNextLocation = AllRegsMask(); @@ -5800,7 +5804,7 @@ void LinearScan::allocateRegisters() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = AllRegsMask(); + copyRegsToFree = AllRegsMask(); regsInUseThisLocation = regsInUseNextLocation; regsInUseNextLocation = AllRegsMask(); #ifdef TARGET_ARM64 @@ -6286,8 +6290,8 @@ void LinearScan::allocateRegisters() if (currentInterval->isLocalVar) { regMaskOnlyOne preferences = currentInterval->registerPreferences; - bool keepAssignment = true; - bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; + bool keepAssignment = true; + bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; // Will the assigned register cover the lifetime? If not, does it at least // meet the preferences for the next RefPosition? @@ -6393,9 +6397,10 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { - lastAllocatedRefPosition = ¤tRefPosition; - regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + lastAllocatedRefPosition = ¤tRefPosition; + regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskOnlyOne assignedRegMask = + getRegMask(assignedRegister, currentInterval->registerType); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -6491,7 +6496,7 @@ void LinearScan::allocateRegisters() copyReg = assignCopyReg(¤tRefPosition); } - lastAllocatedRefPosition = ¤tRefPosition; + lastAllocatedRefPosition = ¤tRefPosition; regMaskOnlyOne copyRegMask = getRegMask(copyReg, currentInterval->registerType); regMaskOnlyOne assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); @@ -6714,7 +6719,7 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); if (currentRefPosition.delayRegFree) { @@ -6723,7 +6728,8 @@ void LinearScan::allocateRegisters() currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumInMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead + regsToFree.RemoveRegNumInMask(assignedRegister, + currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -8153,7 +8159,8 @@ void LinearScan::resolveRegisters() { // If the localVar is in a register, it must be in a register that is not trashed by // the current node (otherwise it would have already been spilled). - assert((genRegMask(localVarInterval->physReg) & getKillSetForNode(treeNode).GetRegTypeMask(interval->registerType)) == RBM_NONE); + assert((genRegMask(localVarInterval->physReg) & + getKillSetForNode(treeNode).GetRegTypeMask(interval->registerType)) == RBM_NONE); // If we have allocated a register to spill it to, we will use that; otherwise, we will // spill it to the stack. We can use as a temp register any non-arg caller-save register. currentRefPosition->referent->recentRefPosition = currentRefPosition; @@ -8410,7 +8417,7 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { regMaskOnlyOne initialRegMask = interval->firstRefPosition->registerAssignment; - regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) + regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK : genRegNumFromMask(initialRegMask); @@ -8737,7 +8744,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskOnlyOne terminatorConsumedRegs) + regMaskOnlyOne terminatorConsumedRegs) { // TODO-Throughput: This would be much more efficient if we add RegToVarMaps instead of VarToRegMaps // and they would be more space-efficient as well. @@ -9039,7 +9046,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // available to copy into. // Note that for this purpose we use the full live-out set, because we must ensure that // even the registers that remain the same across the edge are preserved correctly. - AllRegsMask liveOutRegs; + AllRegsMask liveOutRegs; VarSetOps::Iter liveOutIter(compiler, block->bbLiveOut); unsigned liveOutVarIndex = 0; while (liveOutIter.NextElem(&liveOutVarIndex)) @@ -9154,8 +9161,9 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) } } - assert(compiler->IsGprRegMask(consumedRegs)); // If this fails, then we will have to use AllRegsMask for consumedRegs - VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; + assert(compiler->IsGprRegMask(consumedRegs)); // If this fails, then we will have to use AllRegsMask for + // consumedRegs + VarToRegMap sameVarToRegMap = sharedCriticalVarToRegMap; AllRegsMask sameWriteRegs; AllRegsMask diffReadRegs; @@ -9270,7 +9278,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) setVarReg(sameVarToRegMap, outResolutionSetVarIndex, sameToReg); if (sameToReg != REG_STK) { - sameWriteRegs.AddRegNumInMask(sameToReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); + sameWriteRegs.AddRegNumInMask(sameToReg, + getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); } } } @@ -9663,7 +9672,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } AllRegsMask targetRegsToDo; - AllRegsMask targetRegsReady; + AllRegsMask targetRegsReady; AllRegsMask targetRegsFromStack; // The following arrays capture the location of the registers as they are moved: @@ -9825,7 +9834,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { while (!targetRegsReady.IsEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); + regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); targetRegsToDo ^= targetReg; targetRegsReady ^= targetReg; assert(location[targetReg] != targetReg); @@ -9895,7 +9904,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } if (!targetRegsToDo.IsEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); + regNumber targetReg = genFirstRegNumFromMask(targetRegsToDo); // Is it already there due to other moves? // If not, move it to the temp reg, OR swap it with another register @@ -11801,13 +11810,13 @@ void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) { if (reg >= REG_FP_FIRST && reg <= REG_FP_LAST) { - regsMaskToFree = regsToFree.floatRegs; + regsMaskToFree = regsToFree.floatRegs; availableRegsMask = availableFloatRegs; } #ifdef HAS_PREDICATE_REGS else if (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST) { - regsMaskToFree = regsToFree.predicateRegs; + regsMaskToFree = regsToFree.predicateRegs; availableRegsMask = availableMaskRegs; } #endif @@ -13624,7 +13633,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates &= ~busyRegs.GetRegTypeMask(regType); + candidates &= ~busyRegs.GetRegTypeMask(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -13739,7 +13748,8 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // Remove the `inUseOrBusyRegsMask` from the original candidates list and find one // such range that is consecutive. Next, append that range to the `candidates`. // - regMaskFloat limitCandidatesForConsecutive = refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs; + regMaskFloat limitCandidatesForConsecutive = + refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs; regMaskFloat overallLimitCandidates; regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, @@ -14063,7 +14073,6 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( return candidates; } - bool operator==(const AllRegsMask& first, const AllRegsMask& second) { return first.gprRegs == second.gprRegs && first.floatRegs == second.floatRegs @@ -14085,7 +14094,7 @@ AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second) , first.predicateRegs & second.predicateRegs #endif - ); + ); return result; } @@ -14116,7 +14125,7 @@ AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second) , first.predicateRegs | second.predicateRegs #endif - ); + ); return result; } @@ -14163,7 +14172,6 @@ AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second) return first; } - AllRegsMask operator^=(AllRegsMask& first, const regNumber reg) { if (emitter::isGeneralRegister(reg)) @@ -14192,7 +14200,7 @@ AllRegsMask operator~(const AllRegsMask& first) , ~first.predicateRegs #endif - ); + ); return result; } @@ -14246,7 +14254,6 @@ void AllRegsMask::RemoveRegNumInMask(regNumber reg) } } - bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) { if (emitter::isGeneralRegister(reg)) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 25e8e492f6fb7a..896c946c4a7db3 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -737,7 +737,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, ResolveType resolveType, VARSET_VALARG_TP liveSet, - regMaskGpr terminatorConsumedRegs); + regMaskGpr terminatorConsumedRegs); void resolveEdges(); @@ -831,10 +831,10 @@ class LinearScan : public LinearScanInterface // TODO: Can have separate methods for each type regMaskOnlyOne getConstrainedRegMask(RefPosition* refPosition, - RegisterType regType, + RegisterType regType, regMaskOnlyOne regMaskActual, regMaskOnlyOne regMaskConstrain, - unsigned minRegCount); + unsigned minRegCount); regMaskOnlyOne stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask); // This controls the heuristics used to select registers @@ -1096,14 +1096,14 @@ class LinearScan : public LinearScanInterface } // Helpers for getKillSetForNode(). - AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); + AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); regMaskGpr getKillSetForMul(GenTreeOp* tree); - AllRegsMask getKillSetForCall(GenTreeCall* call); + AllRegsMask getKillSetForCall(GenTreeCall* call); regMaskGpr getKillSetForModDiv(GenTreeOp* tree); AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); - AllRegsMask getKillSetForReturn(); - AllRegsMask getKillSetForProfilerHook(); + AllRegsMask getKillSetForReturn(); + AllRegsMask getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS regMaskGpr getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); #endif // FEATURE_HW_INTRINSICS @@ -1433,7 +1433,7 @@ class LinearScan : public LinearScanInterface BasicBlock* toBlock, var_types type, VARSET_VALARG_TP sharedCriticalLiveSet, - regMaskOnlyOne terminatorConsumedRegs); + regMaskOnlyOne terminatorConsumedRegs); #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; @@ -1503,7 +1503,7 @@ class LinearScan : public LinearScanInterface // Current mask of registers being printed in the dump. AllRegsMask lastDumpedRegisters; AllRegsMask registersToDump; - int lastUsedRegNumIndex; + int lastUsedRegNumIndex; bool shouldDumpReg(regNumber regNum) { return registersToDump.IsRegNumInMask(regNum); @@ -1685,7 +1685,7 @@ class LinearScan : public LinearScanInterface // A temporary VarToRegMap used during the resolution of critical edges. VarToRegMap sharedCriticalVarToRegMap; - PhasedVar actualRegistersMask; + PhasedVar actualRegistersMask; PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; @@ -1693,7 +1693,7 @@ class LinearScan : public LinearScanInterface PhasedVar availableMaskRegs; #endif PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, - // predicate + // predicate #if defined(TARGET_XARCH) #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) @@ -1750,14 +1750,14 @@ class LinearScan : public LinearScanInterface VARSET_TP largeVectorCalleeSaveCandidateVars; #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE - //----------------------------------------------------------------------- - // Register status - //----------------------------------------------------------------------- +//----------------------------------------------------------------------- +// Register status +//----------------------------------------------------------------------- - //TODO: One option is to also just have another current_AvailableREgs that - // gets reset for every refposition we are processing depending on the - // register type. That wawy we do not have to query and fetch the appropriate - // entry again and agin. +// TODO: One option is to also just have another current_AvailableREgs that +// gets reset for every refposition we are processing depending on the +// register type. That wawy we do not have to query and fetch the appropriate +// entry again and agin. #ifdef HAS_PREDICATE_REGS regMaskOnlyOne m_AvailableRegs[3]; // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. #else @@ -1801,14 +1801,13 @@ class LinearScan : public LinearScanInterface #ifdef HAS_PREDICATE_REGS m_RegistersWithConstants[2] = RBM_NONE; #endif - } bool isRegAvailable(regNumber reg, var_types regType) // only used in asserts { regMaskOnlyOne regMask = getRegMask(reg, regType); - #ifdef TARGET_ARM64 +#ifdef TARGET_ARM64 if (emitter::isGeneralRegisterOrZR(reg)) #else if (emitter::isGeneralRegister(reg)) @@ -1834,7 +1833,7 @@ class LinearScan : public LinearScanInterface void setRegsInUse(AllRegsMask regMask) { - //TODO: Fix this later. + // TODO: Fix this later. m_AvailableRegs[0] &= ~regMask.gprRegs; m_AvailableRegs[1] &= ~regMask.floatRegs; #ifdef HAS_PREDICATE_REGS @@ -1850,7 +1849,7 @@ class LinearScan : public LinearScanInterface #else if (emitter::isGeneralRegister(reg)) #endif - { + { m_AvailableRegs[0] &= ~regMask; } else if (emitter::isFloatReg(reg)) @@ -1876,7 +1875,6 @@ class LinearScan : public LinearScanInterface #endif assert(compiler->IsGprRegMask(m_AvailableRegs[0])); assert(compiler->IsFloatRegMask(m_AvailableRegs[1])); - } void makeRegAvailable(regNumber reg, var_types regType) { @@ -1888,7 +1886,7 @@ class LinearScan : public LinearScanInterface // rbx - 0 // ... // xmm0 - 1 - // here it will be, m_AvailableRegs[regTypeIndex[reg]] = + // here it will be, m_AvailableRegs[regTypeIndex[reg]] = if (emitter::isGeneralRegisterOrZR(reg)) #else if (emitter::isGeneralRegister(reg)) @@ -1942,14 +1940,15 @@ class LinearScan : public LinearScanInterface void clearSpillCost(regNumber reg, var_types regType); void updateSpillCost(regNumber reg, Interval* interval); - FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - AllRegsMask* regsToFree, + FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + AllRegsMask* regsToFree, AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); #ifdef HAS_PREDICATE_REGS - regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, m_FloatRegistersWithConstant, etc. + regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, + // m_FloatRegistersWithConstant, etc. regMaskOnlyOne fixedRegs[3]; #else regMaskOnlyOne m_RegistersWithConstants[2]; @@ -1958,7 +1957,7 @@ class LinearScan : public LinearScanInterface void clearConstantReg(regNumber reg, var_types regType) { - //TODO: If we decide to have curr_RegistersWithConstants, then here we will + // TODO: If we decide to have curr_RegistersWithConstants, then here we will // just operate on curr_RegistersWithConstants and assert // assert(m_RegistersWithConstants[regTypeIndex(regType)] == curr_RegistersWithConstants); // but we will have to make sure that we save it back too?? @@ -2005,7 +2004,7 @@ class LinearScan : public LinearScanInterface weight_t spillCost[REG_COUNT]; AllRegsMask regsBusyUntilKill; // TODO: Likewise, probably have this global 32-bit and set it point to the specific - // version like gpr, vector, etc. + // version like gpr, vector, etc. AllRegsMask regsInUseThisLocation; AllRegsMask regsInUseNextLocation; #ifdef TARGET_ARM64 @@ -2033,7 +2032,8 @@ class LinearScan : public LinearScanInterface void resetRegState() { resetAvailableRegs(); - regsBusyUntilKill = AllRegsMask();; + regsBusyUntilKill = AllRegsMask(); + ; } bool conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition); @@ -2129,7 +2129,7 @@ class LinearScan : public LinearScanInterface #else void BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllRegsMask killMask); #endif - void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); + void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 9d7c5ff3c11c81..aea80f705aede2 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -192,9 +192,9 @@ int LinearScan::BuildShiftLongCarry(GenTree* tree) int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; - bool isLocalDefUse = false; + int srcCount; + int dstCount = 0; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); @@ -468,7 +468,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { - srcCount = BuildReturn(tree); + srcCount = BuildReturn(tree); AllRegsMask killMask = getKillSetForReturn(); BuildKills(tree, killMask); break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 1e3f94a8e226b7..95005e81478043 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -582,9 +582,9 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; - bool isLocalDefUse = false; + int srcCount; + int dstCount = 0; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 7f9b054fd9d65a..2bd41d09930484 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -210,9 +210,9 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // TARGET_ARM // Set destination candidates for return value of the call. - AllRegsMask dstReturnCandidates; + AllRegsMask dstReturnCandidates; regMaskOnlyOne dstCandidates = RBM_NONE; - RegisterType registerType = call->TypeGet(); + RegisterType registerType = call->TypeGet(); #ifdef TARGET_ARM if (call->IsHelperCall(compiler, CORINFO_HELP_INIT_PINVOKE_FRAME)) @@ -570,7 +570,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) // Registers for split argument corresponds to source int dstCount = argNode->gtNumRegs; - regNumber argReg = argNode->GetRegNum(); + regNumber argReg = argNode->GetRegNum(); regMaskGpr argMask = RBM_NONE; for (unsigned i = 0; i < argNode->gtNumRegs; i++) { @@ -889,7 +889,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - assert(compiler->IsGprRegMask(sizeRegMask)); + assert(compiler->IsGprRegMask(sizeRegMask)); buildInternalRegisterUses(); AllRegsMask killMask = getKillSetForBlockStore(blkNode); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index f403e2e63e8746..e6e891dc80596b 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -911,8 +911,8 @@ AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) // We don't just add them unconditionally to the killMask because for most architectures // they are already in the RBM_CALLEE_TRASH set, // and we don't want to introduce extra checks and calls in this hot function. - assert(!call->IsVirtualStub() || - ((killMask.gprRegs & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); + assert(!call->IsVirtualStub() || ((killMask.gprRegs & compiler->virtualStubParamInfo->GetRegMask()) == + compiler->virtualStubParamInfo->GetRegMask())); #endif // !TARGET_ARM #ifdef SWIFT_SUPPORT @@ -1194,7 +1194,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo { continue; } - Interval* interval = getIntervalForLocalVar(varIndex); + Interval* interval = getIntervalForLocalVar(varIndex); regMaskOnlyOne regsKillMask = killMask.GetRegTypeMask(interval->registerType); const bool isCallKill = ((regsKillMask == RBM_INT_CALLEE_TRASH) || (regsKillMask == RBM_CALLEE_TRASH)); @@ -1932,7 +1932,8 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc #endif // TARGET_ARM64 { newRefPosition->registerAssignment = - getConstrainedRegMask(newRefPosition, interval->registerType, oldAssignment, calleeSaveMask, minRegCountForRef); + getConstrainedRegMask(newRefPosition, interval->registerType, oldAssignment, calleeSaveMask, + minRegCountForRef); } if ((newRefPosition->registerAssignment != oldAssignment) && (newRefPosition->refType == RefTypeUse) && @@ -2843,17 +2844,18 @@ void LinearScan::buildIntervals() needNonIntegerRegisters |= compiler->compFloatingPointUsed; if (!needNonIntegerRegisters) { - availableRegCount = REG_INT_COUNT; + availableRegCount = REG_INT_COUNT; availableFloatRegs.OverrideAssign(RBM_NONE); availableDoubleRegs.OverrideAssign(RBM_NONE); - //availableMaskRegs = RBM_NONE; // Is this also needed? + // availableMaskRegs = RBM_NONE; // Is this also needed? } actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs #ifdef HAS_PREDICATE_REGS - , availableMaskRegs + , + availableMaskRegs #endif // HAS_PREDICATE_REGS - ); + ); #ifdef DEBUG // Make sure we don't have any blocks that were not visited @@ -3194,7 +3196,6 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) } } - #ifndef TARGET_ARMARCH //------------------------------------------------------------------------ // BuildDefWithKills: Build one or two (for 32-bit) RefTypeDef RefPositions for the given node, @@ -3229,7 +3230,7 @@ void LinearScan::BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllR assert((int)genCountBits(dstCandidates) == 2); BuildDefs(tree, 2, dstCandidates); } -#endif // TARGET_64BIT +#endif // TARGET_64BIT } #endif @@ -3297,7 +3298,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) // Find the registers that we should remove from the preference set because // they are occupied with argument values. AllRegsMask unpref = placedArgRegs; - unsigned varIndex = interval->getVarIndex(compiler); + unsigned varIndex = interval->getVarIndex(compiler); for (size_t i = 0; i < numPlacedArgLocals; i++) { if (placedArgLocals[i].VarIndex == varIndex) @@ -3831,7 +3832,7 @@ void LinearScan::BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, defCandidates = allRegs(type); } #else - defCandidates = allRegs(type); + defCandidates = allRegs(type); #endif // TARGET_X86 RefPosition* def = newRefPosition(varDefInterval, currentLoc + 1, RefTypeDef, storeLoc, defCandidates, index); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 3d1e79e851499c..37f109dc0a5b16 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -45,9 +45,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; - int dstCount = 0; - bool isLocalDefUse = false; + int srcCount; + int dstCount = 0; + bool isLocalDefUse = false; // Reset the build-related members of LinearScan. clearBuildState(); @@ -190,7 +190,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { - srcCount = BuildReturn(tree); + srcCount = BuildReturn(tree); AllRegsMask killMask = getKillSetForReturn(); BuildKills(tree, killMask); break; @@ -1169,7 +1169,7 @@ int LinearScan::BuildCall(GenTreeCall* call) ctrlExpr = call->gtCallAddr; } - AllRegsMask dstReturnCandidates; + AllRegsMask dstReturnCandidates; regMaskOnlyOne dstCandidates = RBM_NONE; RegisterType registerType = regType(call); @@ -1658,7 +1658,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - assert(compiler->IsGprRegMask(sizeRegMask)); + assert(compiler->IsGprRegMask(sizeRegMask)); #ifdef TARGET_X86 // If we require a byte register on x86, we may run into an over-constrained situation diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4519f36c96e5df..3a55d0bccead8d 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2714,7 +2714,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call } } - //if ((structIntRegs > 0) && (structFloatRegs > 0)) + // if ((structIntRegs > 0) && (structFloatRegs > 0)) //{ // assert(!"Different numbers"); //} @@ -2830,7 +2830,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call ++structFloatRegs; } } - //if ((structIntRegs > 0) && (structFloatRegs > 0)) + // if ((structIntRegs > 0) && (structFloatRegs > 0)) //{ // assert(!"Different numbers"); //} diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 5cd191c6398601..4c82a7982c299f 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -251,8 +251,8 @@ void RegSet::rsClearRegsModified() rsModifiedRegsMaskInitialized = true; #endif // DEBUG - rsModifiedGprRegsMask = RBM_NONE; - rsModifiedFloatRegsMask = RBM_NONE; + rsModifiedGprRegsMask = RBM_NONE; + rsModifiedFloatRegsMask = RBM_NONE; rsModifiedPredicateRegsMask = RBM_NONE; } @@ -276,7 +276,9 @@ void RegSet::rsSetPredicateRegsModified(regMaskPredicate mask DEBUGARG(bool supp } #endif // HAS_PREDICATE_REGS -void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) DEBUGARG(regMaskOnlyOne calleeSaveMask)) +void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, + regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) + DEBUGARG(regMaskOnlyOne calleeSaveMask)) { assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); @@ -407,7 +409,7 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) : m_rsCompiler(compiler), m_r rsMaskResvd = RBM_NONE; #if defined(TARGET_ARMARCH) - rsGprMaskCalleeSaved = RBM_NONE; + rsGprMaskCalleeSaved = RBM_NONE; rsFloatMaskCalleeSaved = RBM_NONE; #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) rsMaskCalleeSaved = RBM_NONE; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index e9f21a983a7aa2..1ff03637b97f84 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -67,11 +67,12 @@ class RegSet // private: - bool rsNeededSpillReg; // true if this method needed to spill any registers - regMaskGpr rsModifiedGprRegsMask; // mask of the registers modified by the current function. - regMaskFloat rsModifiedFloatRegsMask; // mask of the registers modified by the current function. + bool rsNeededSpillReg; // true if this method needed to spill any registers + regMaskGpr rsModifiedGprRegsMask; // mask of the registers modified by the current function. + regMaskFloat rsModifiedFloatRegsMask; // mask of the registers modified by the current function. regMaskPredicate rsModifiedPredicateRegsMask; // mask of the registers modified by the current function. - FORCEINLINE void rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) + FORCEINLINE void rsSetRegsModified(regMaskOnlyOne& trackingMask, + regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)); #ifdef DEBUG @@ -79,7 +80,6 @@ class RegSet #endif // DEBUG public: - regMaskGpr rsGetModifiedRegsMask(var_types type) const { if (varTypeUsesIntReg(type)) @@ -194,7 +194,7 @@ class RegSet } // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) - void RemoveMaskVars(var_types type, regMaskOnlyOne removeMaskVars) + void RemoveMaskVars(var_types type, regMaskOnlyOne removeMaskVars) { AllRegsMask newMask = _rsAllMaskVars; if (varTypeRegister[type] == VTR_INT) @@ -227,11 +227,11 @@ class RegSet AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property #if defined(TARGET_ARMARCH) - regMaskGpr rsGprMaskCalleeSaved; // TODO: Can use AllRegsMask here as well + regMaskGpr rsGprMaskCalleeSaved; // TODO: Can use AllRegsMask here as well regMaskFloat rsFloatMaskCalleeSaved; #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog -#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_LOONGARCH64 +#endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_LOONGARCH64 #ifdef HAS_PREDICATE_REGS regMaskPredicate rsPredicateMaskCalleeSaved; diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index c404afc468debe..fb49e93e346cc5 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -253,7 +253,7 @@ typedef unsigned __int64 singleRegMask; #else // x86 and arm -typedef unsigned regMaskTP; +typedef unsigned regMaskTP; #define regMaskGpr regMaskTP #define regMaskFloat regMaskTP #define regMaskPredicate regMaskTP @@ -417,7 +417,7 @@ typedef struct _regMaskAll } } - //void AllRegsMask& operator|=(const AllRegsMask& second); + // void AllRegsMask& operator|=(const AllRegsMask& second); //{ // gprRegs // return *this; @@ -437,20 +437,19 @@ typedef unsigned __int64 regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif -bool operator==(const AllRegsMask& first, const AllRegsMask& second); -bool operator!=(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); -regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); -AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); -AllRegsMask operator^=(AllRegsMask& first, const regNumber reg); +bool operator==(const AllRegsMask& first, const AllRegsMask& second); +bool operator!=(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); +regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); +AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second); +AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); +AllRegsMask operator^=(AllRegsMask& first, const regNumber reg); // AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); AllRegsMask operator~(const AllRegsMask& first); // inline AllRegsMask createRegMask(regNumber reg) - /*****************************************************************************/ #ifdef DEBUG diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index e2b8c57af0e0ff..4cf4215bb0a373 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -154,7 +154,6 @@ void Compiler::unwindPushPopCFI(regNumber reg) regMaskOnlyOne mask = genRegMask(reg); - #if defined(TARGET_ARM) createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, reg >= REG_FP_FIRST ? 2 * REGSIZE_BYTES : REGSIZE_BYTES); diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 75cf9c162d862e..b0a0939efdae06 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -434,9 +434,11 @@ void dspRegMask(regMaskOnlyOne mask, size_t minSiz) { dspRegMask(AllRegsMask(mask, mask #ifdef HAS_PREDICATE_REGS - , mask + , + mask #endif - ), minSiz); + ), + minSiz); } /***************************************************************************** @@ -446,7 +448,7 @@ void dspRegMask(regMaskOnlyOne mask, size_t minSiz) */ void dspRegMask(AllRegsMask mask, size_t minSiz) { - //TODO: Need to fix all the callers where we don't know if the input is gpr/float but is of type `regMaskOnlyOne`. + // TODO: Need to fix all the callers where we don't know if the input is gpr/float but is of type `regMaskOnlyOne`. // For now, I am just making `floatMask` as optional and default to RBM_NONE so we don't have to deal with // lot of build errors. const char* sep = ""; From 15c482f0818e26870198fa59745e3f39e8b218c2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 11 Mar 2024 07:14:15 -0700 Subject: [PATCH 091/201] fix some build errors and failures --- src/coreclr/jit/lsra.cpp | 17 +++++++++-------- src/coreclr/jit/lsra.h | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 8dd00c666e413a..d0d8465f32a7d2 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -389,11 +389,10 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, regMaskOnlyOne regsBusy, AllRegsMask* regsToFree, - AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) + AllRegsMask* delayRegsToFree, RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { assert(compiler->IsOnlyOneRegMask(regsBusy)); - RegisterType regType = interval->registerType; regsInUseThisLocation.AddRegTypeMask(regsBusy, regType); if (refPosition.lastUse) @@ -5374,7 +5373,8 @@ void LinearScan::allocateRegistersMinimal() // happened to be restored in assignedReg, we would need assignedReg to stay alive because // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); + &delayRegsToFree, currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); @@ -6419,7 +6419,8 @@ void LinearScan::allocateRegisters() // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree DEBUG_ARG(currentInterval) + &delayRegsToFree, + currentInterval->registerType DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { @@ -6528,7 +6529,9 @@ void LinearScan::allocateRegisters() // happened to be restored in assignedReg, we would need assignedReg to stay alive because // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree DEBUG_ARG(currentInterval) DEBUG_ARG(assignedRegister)); + &delayRegsToFree, + currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); @@ -9805,7 +9808,6 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, while (!targetCandidates.IsEmpty()) { regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetCandidates); - targetCandidates ^= targetReg; if (location[targetReg] == REG_NA) { #ifdef TARGET_ARM @@ -9834,9 +9836,8 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { while (!targetRegsReady.IsEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); + regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetRegsReady); targetRegsToDo ^= targetReg; - targetRegsReady ^= targetReg; assert(location[targetReg] != targetReg); assert(targetReg < REG_COUNT); regNumber sourceReg = (regNumber)source[targetReg]; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 896c946c4a7db3..63f4a9cd99e4cc 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1943,7 +1943,8 @@ class LinearScan : public LinearScanInterface FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, regMaskOnlyOne regsBusy, AllRegsMask* regsToFree, - AllRegsMask* delayRegsToFree DEBUG_ARG(Interval* interval) + AllRegsMask* delayRegsToFree, + RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); #ifdef HAS_PREDICATE_REGS From 37b6cce9c7e5dd81c071f2b82e560c76a005ed61 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 06:14:01 -0700 Subject: [PATCH 092/201] Remove the assert(false) --- src/coreclr/jit/regset.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 1ff03637b97f84..47a9248d476013 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -179,9 +179,6 @@ class RegSet #ifdef HAS_PREDICATE_REGS else if (varTypeRegister[type] == VTR_MASK) { - // TODO: If we never hit this assert, then just convert _rsAllMaskVars - // to regMaskGpr - assert(false); newMask.predicateRegs |= addMaskVars; } #endif @@ -204,9 +201,6 @@ class RegSet #ifdef HAS_PREDICATE_REGS else if (varTypeRegister[type] == VTR_MASK) { - // TODO: If we never hit this assert, then just convert _rsAllMaskVars - // to regMaskGpr - assert(false); newMask.predicateRegs &= ~removeMaskVars; } #endif From e89a3ee67ed2ac226387445d037d2232833d9aaf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 06:14:38 -0700 Subject: [PATCH 093/201] Pass dstCount for BuildDefWithKills() --- src/coreclr/jit/lsra.h | 4 +++- src/coreclr/jit/lsrabuild.cpp | 10 ++++++---- src/coreclr/jit/lsraxarch.cpp | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 63f4a9cd99e4cc..01f07ffddcee75 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2128,7 +2128,9 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARMARCH void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask); #else - void BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllRegsMask killMask); + void BuildDefWithKills(GenTree* tree, + int dstCount, regMaskOnlyOne dstCandidates, + AllRegsMask killMask); #endif void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index e6e891dc80596b..de781d9d73b5dc 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3211,23 +3211,25 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefWithKills(GenTree* tree, regMaskGpr dstCandidates, AllRegsMask killMask) +void LinearScan::BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask) { + assert(compiler->IsOnlyOneRegMask(dstCandidates)); + // Build the kill RefPositions BuildKills(tree, killMask); #ifdef TARGET_64BIT // For 64 bits, - assert((int)genCountBits(dstCandidates) <= 1); + assert(dstCount == 1); BuildDef(tree, dstCandidates); #else - if ((int)genCountBits(dstCandidates) <= 1) + if (dstCount == 1) { BuildDef(tree, dstCandidates); } else { - assert((int)genCountBits(dstCandidates) == 2); + assert(dstCount == 2); BuildDefs(tree, 2, dstCandidates); } #endif // TARGET_64BIT diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 37f109dc0a5b16..f844e374645398 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1383,7 +1383,7 @@ int LinearScan::BuildCall(GenTreeCall* call) else { assert(dstCount == 1); - BuildDefWithKills(call, dstCandidates, killMask); + BuildDefWithKills(call, dstCount, dstCandidates, killMask); } } else @@ -1975,7 +1975,7 @@ int LinearScan::BuildModDiv(GenTree* tree) buildInternalRegisterUses(); AllRegsMask killMask(getKillSetForModDiv(tree->AsOp()), RBM_NONE); - BuildDefWithKills(tree, dstCandidates, killMask); + BuildDefWithKills(tree, 1, dstCandidates, killMask); return srcCount; } @@ -3096,7 +3096,7 @@ int LinearScan::BuildMul(GenTree* tree) assert(compiler->IsGprRegMask(dstCandidates)); AllRegsMask killMask(getKillSetForMul(tree->AsOp()), RBM_NONE); - BuildDefWithKills(tree, dstCandidates, killMask); + BuildDefWithKills(tree, dstCount, dstCandidates, killMask); return srcCount; } From a2c99a0ed8420155c1e0bc42251bb1562afdb8ee Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 06:14:50 -0700 Subject: [PATCH 094/201] Fix dspRegMask for compilation errors: --- src/coreclr/jit/codegenarm.cpp | 2 +- src/coreclr/jit/codegenarmarch.cpp | 6 +++++- src/coreclr/jit/codegencommon.cpp | 9 ++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 59a580589c4af4..71a1ce6e8e7519 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2507,7 +2507,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf("Funclet prolog / epilog info\n"); printf(" Function CallerSP-to-FP delta: %d\n", genFuncletInfo.fiFunctionCallerSPtoFPdelta); printf(" Save regs: "); - dspRegMask(rsGprMaskSaveRegs, rsFloatMaskSaveRegs); + dspRegMask(AllRegsMask(rsGprMaskSaveRegs, rsFloatMaskSaveRegs)); printf("\n"); printf(" SP delta: %d\n", genFuncletInfo.fiSpDelta); printf(" PSP slot SP offset: %d\n", genFuncletInfo.fiPSP_slot_SP_offset); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index fb6ac55adf6f09..fc5819fd3160c9 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4946,7 +4946,11 @@ void CodeGen::genPushCalleeSavedRegisters() { printf("Error: unexpected number of callee-saved registers to push. Expected: %d. Got: %d ", compiler->compCalleeRegsPushed, pushRegsCnt); - dspRegMask(rsPushGprRegs, rsPushFloatRegs); + dspRegMask(AllRegsMask(rsPushGprRegs, rsPushFloatRegs +#ifdef HAS_PREDICATE_REGS + ,RBM_NONE +#endif + )); printf("\n"); assert(compiler->compCalleeRegsPushed == pushRegsCnt); } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 6862307b352a86..e4fcbd71181daf 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5404,6 +5404,9 @@ void CodeGen::genFinalizeFrame() #endif regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; +#ifdef HAS_PREDICATE_REGS + regMaskPredicate maskPushRegsPredicate = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; +#endif #ifdef TARGET_ARMARCH if (isFramePointerUsed()) @@ -5496,7 +5499,11 @@ void CodeGen::genFinalizeFrame() if (verbose) { printf("Callee-saved registers pushed: %d ", compiler->compCalleeRegsPushed); - dspRegMask(maskPushRegsInt, maskPushRegsFloat); // TODO: Should this also have maskPushRegsPredicate + dspRegMask(AllRegsMask(maskPushRegsInt, maskPushRegsFloat +#ifdef HAS_PREDICATE_REGS + ,maskPushRegsPredicate +#endif + )); printf("\n"); } #endif // DEBUG From 3c808715c5f1d81713b2a6a11ca493deeaf509ae Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 09:32:33 -0700 Subject: [PATCH 095/201] fix some more asserts --- src/coreclr/jit/lsra.cpp | 30 ++++++++++++++++-------------- src/coreclr/jit/regset.h | 18 +++++++++--------- src/coreclr/jit/target.h | 12 ++++++------ 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index d0d8465f32a7d2..6fcaf6f8534412 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -493,17 +493,18 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { regMaskOnlyOne busyRegs = RBM_NONE; - if (regType == IntRegisterType) + if (varTypeUsesIntReg(regType)) { busyRegs = regsBusyUntilKill.gprRegs | regsInUseThisLocation.gprRegs; } - else if (regType == FloatRegisterType) + else if (varTypeUsesFloatReg(regType)) { busyRegs = regsBusyUntilKill.floatRegs | regsInUseThisLocation.floatRegs; } else { #ifdef HAS_PREDICATE_REGS + assert(varTypeUsesMaskReg(regType)); busyRegs = regsBusyUntilKill.predicateRegs | regsInUseThisLocation.predicateRegs; #endif } @@ -556,12 +557,12 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly : refPosition->getReg()->registerType; regMaskOnlyOne calleeSaved = RBM_NONE; regMaskOnlyOne calleeTrash = RBM_NONE; - if (regType == IntRegisterType) + if (varTypeUsesIntReg(regType)) { calleeSaved = RBM_INT_CALLEE_SAVED; calleeTrash = RBM_INT_CALLEE_TRASH; } - else if (regType == FloatRegisterType) + else if (varTypeUsesFloatReg(regType)) { calleeSaved = RBM_FLT_CALLEE_SAVED; calleeTrash = RBM_FLT_CALLEE_TRASH; @@ -569,6 +570,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly else { #ifdef HAS_PREDICATE_REGS + assert(varTypeUsesMaskReg(regType)); calleeSaved = RBM_MSK_CALLEE_SAVED; calleeTrash = RBM_MSK_CALLEE_TRASH; #else @@ -4155,18 +4157,20 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regMaskOnlyOne candidateRegs = RBM_NONE; regMaskOnlyOne allRegsMask = allRegs(interval->registerType); + RegisterType regType = interval->registerType; - if (interval->registerType == IntRegisterType) + if (varTypeUsesIntReg(regType)) { candidateRegs = allRegsMask & availableRegs.gprRegs; } - else if (interval->registerType == FloatRegisterType) + else if (varTypeUsesFloatReg(regType)) { candidateRegs = allRegsMask & availableRegs.floatRegs; } else { #ifdef HAS_PREDICATE_REGS + assert(varTypeUsesMaskReg(regType)); candidateRegs = allRegsMask & availableRegs.predicateRegs; #else unreached(); @@ -8838,20 +8842,17 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, { regMaskOnlyOne calleeTrashMask = RBM_NONE; - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { calleeTrashMask = RBM_INT_CALLEE_TRASH; assert(compiler->IsGprRegMask(terminatorConsumedRegs)); } - else if (varTypeRegister[type] == VTR_FLOAT) + else { + assert(varTypeUsesFloatReg(type)); calleeTrashMask = RBM_FLT_CALLEE_TRASH; assert(compiler->IsFloatRegMask(terminatorConsumedRegs)); } - else - { - unreached(); - } // Prefer a callee-trashed register if possible to prevent new prolog/epilog saves/restores. if ((freeRegs & calleeTrashMask) != 0) { @@ -13962,17 +13963,18 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - if (regType == IntRegisterType) + if (varTypeUsesIntReg(regType)) { candidates &= ~busyRegs.gprRegs; } - else if (regType == FloatRegisterType) + else if (varTypeUsesFloatReg(regType)) { candidates &= ~busyRegs.floatRegs; } else { #ifdef HAS_PREDICATE_REGS + assert(varTypeUsesMaskReg(regType)); candidates &= ~busyRegs.predicateRegs; #else unreached(); diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 47a9248d476013..a2590d3b04ce85 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -145,19 +145,19 @@ class RegSet public: regMaskOnlyOne GetMaskVars(var_types type) const // 'get' property function for rsMaskVars property { - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { return _rsAllMaskVars.gprRegs; } #ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) + else if (varTypeUsesMaskReg(type)) { return _rsAllMaskVars.predicateRegs; } #endif else { - assert(varTypeRegister[type] == VTR_FLOAT); + assert(varTypeUsesFloatReg(type)); return _rsAllMaskVars.floatRegs; } } @@ -172,19 +172,19 @@ class RegSet void AddMaskVars(var_types type, regMaskOnlyOne addMaskVars) // union 'addMaskVars' with the rsMaskVars set { AllRegsMask newMask = _rsAllMaskVars; - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { newMask.gprRegs |= addMaskVars; } #ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) + else if (varTypeUsesMaskReg(type)) { newMask.predicateRegs |= addMaskVars; } #endif else { - assert(varTypeRegister[type] == VTR_FLOAT); + assert(varTypeUsesFloatReg(type)); newMask.floatRegs |= addMaskVars; } SetMaskVars(newMask); @@ -194,19 +194,19 @@ class RegSet void RemoveMaskVars(var_types type, regMaskOnlyOne removeMaskVars) { AllRegsMask newMask = _rsAllMaskVars; - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { newMask.gprRegs &= ~removeMaskVars; } #ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) + else if (varTypeUsesMaskReg(type)) { newMask.predicateRegs &= ~removeMaskVars; } #endif else { - assert(varTypeRegister[type] == VTR_FLOAT); + assert(varTypeUsesFloatReg(type)); newMask.floatRegs &= ~removeMaskVars; } SetMaskVars(newMask); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index fb49e93e346cc5..44cc03ef3c9ad4 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -361,38 +361,38 @@ typedef struct _regMaskAll // Rename this to AddRegMask() void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) { - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { gprRegs |= maskToAdd; } #ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) + else if (varTypeUsesMaskReg(type)) { predicateRegs |= maskToAdd; } #endif else { - assert(varTypeRegister[type] == VTR_FLOAT); + assert(varTypeUsesFloatReg(type)); floatRegs |= maskToAdd; } } regMaskOnlyOne GetRegTypeMask(var_types type) { - if (varTypeRegister[type] == VTR_INT) + if (varTypeUsesIntReg(type)) { return gprRegs; } #ifdef HAS_PREDICATE_REGS - else if (varTypeRegister[type] == VTR_MASK) + else if (varTypeUsesMaskReg(type)) { return predicateRegs; } #endif else { - assert(varTypeRegister[type] == VTR_FLOAT); + assert(varTypeUsesFloatReg(type)); return floatRegs; } } From 9e597bcbccdde5f8cf31fbc8d67d0d27afe4bf67 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 10:30:58 -0700 Subject: [PATCH 096/201] fix arm issue --- src/coreclr/jit/codegenarm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 71a1ce6e8e7519..370142da448c71 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2156,8 +2156,8 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedFloatRegsMask(); - regMaskGpr maskPopRegsInt = regSet.rsGetModifiedGprRegsMask(); + regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; + regMaskGpr maskPopRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; // First, pop float registers From badf863ccf65ac1733c5e2671dea83d4705fa80e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 10:31:16 -0700 Subject: [PATCH 097/201] jit format --- src/coreclr/jit/codegenarmarch.cpp | 5 +++-- src/coreclr/jit/codegencommon.cpp | 5 +++-- src/coreclr/jit/lsra.cpp | 17 ++++++++--------- src/coreclr/jit/lsra.h | 6 ++---- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index fc5819fd3160c9..84260c04b08f68 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4948,9 +4948,10 @@ void CodeGen::genPushCalleeSavedRegisters() compiler->compCalleeRegsPushed, pushRegsCnt); dspRegMask(AllRegsMask(rsPushGprRegs, rsPushFloatRegs #ifdef HAS_PREDICATE_REGS - ,RBM_NONE + , + RBM_NONE #endif - )); + )); printf("\n"); assert(compiler->compCalleeRegsPushed == pushRegsCnt); } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index e4fcbd71181daf..8061bc4e638c68 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5501,9 +5501,10 @@ void CodeGen::genFinalizeFrame() printf("Callee-saved registers pushed: %d ", compiler->compCalleeRegsPushed); dspRegMask(AllRegsMask(maskPushRegsInt, maskPushRegsFloat #ifdef HAS_PREDICATE_REGS - ,maskPushRegsPredicate + , + maskPushRegsPredicate #endif - )); + )); printf("\n"); } #endif // DEBUG diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 6fcaf6f8534412..0e8cc246f16014 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -389,7 +389,8 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, regMaskOnlyOne regsBusy, AllRegsMask* regsToFree, - AllRegsMask* delayRegsToFree, RegisterType regType DEBUG_ARG(Interval* interval) + AllRegsMask* delayRegsToFree, + RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { assert(compiler->IsOnlyOneRegMask(regsBusy)); @@ -4157,7 +4158,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regMaskOnlyOne candidateRegs = RBM_NONE; regMaskOnlyOne allRegsMask = allRegs(interval->registerType); - RegisterType regType = interval->registerType; + RegisterType regType = interval->registerType; if (varTypeUsesIntReg(regType)) { @@ -5378,7 +5379,7 @@ void LinearScan::allocateRegistersMinimal() // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, &delayRegsToFree, currentInterval->registerType DEBUG_ARG(currentInterval) - DEBUG_ARG(assignedRegister)); + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); @@ -6423,9 +6424,8 @@ void LinearScan::allocateRegisters() // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree, - currentInterval->registerType DEBUG_ARG(currentInterval) - DEBUG_ARG(assignedRegister)); + &delayRegsToFree, currentInterval->registerType DEBUG_ARG( + currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); @@ -6533,9 +6533,8 @@ void LinearScan::allocateRegisters() // happened to be restored in assignedReg, we would need assignedReg to stay alive because // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree, - currentInterval->registerType DEBUG_ARG(currentInterval) - DEBUG_ARG(assignedRegister)); + &delayRegsToFree, currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 01f07ffddcee75..5e3117ef1b35b4 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1944,7 +1944,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regsBusy, AllRegsMask* regsToFree, AllRegsMask* delayRegsToFree, - RegisterType regType DEBUG_ARG(Interval* interval) + RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); #ifdef HAS_PREDICATE_REGS @@ -2128,9 +2128,7 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARMARCH void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask); #else - void BuildDefWithKills(GenTree* tree, - int dstCount, regMaskOnlyOne dstCandidates, - AllRegsMask killMask); + void BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask); #endif void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); From 27e6bdf50e75d7ef734ff5ff5e2ca87f0c1a6fd1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 12 Mar 2024 11:43:56 -0700 Subject: [PATCH 098/201] fix the asmdiff for windows/x64 --- src/coreclr/jit/lsrabuild.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index de781d9d73b5dc..03fa9e536bfa7a 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1196,8 +1196,8 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo } Interval* interval = getIntervalForLocalVar(varIndex); regMaskOnlyOne regsKillMask = killMask.GetRegTypeMask(interval->registerType); - - const bool isCallKill = ((regsKillMask == RBM_INT_CALLEE_TRASH) || (regsKillMask == RBM_CALLEE_TRASH)); + const bool isCallKill = + (killMask.gprRegs == RBM_INT_CALLEE_TRASH) || (killMask == AllRegsMask_CALLEE_TRASH); if (isCallKill) { From e7ee9606f55d0e3bad5725625bbbbc02622602a6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 13 Mar 2024 07:52:41 -0700 Subject: [PATCH 099/201] fix the JITStressRegs AV --- src/coreclr/jit/lsra.cpp | 40 ++++++++++++++++++++++++---------------- src/coreclr/jit/lsra.h | 2 +- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 0e8cc246f16014..0aac97b76a46c4 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -525,7 +525,9 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, // them based on the current stress options. // // Arguments: -// mask - The current mask of register candidates for a node +// refPosition - The refPosition for which we want to stress the register +// mask - The current mask of register candidates for a node +// regtype - The registerType // // Return Value: // A possibly-modified mask, based on the value of DOTNET_JitStressRegs. @@ -534,7 +536,7 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, // This is the method used to implement the stress options that limit // the set of registers considered for allocation. // -regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask) +regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask, RegisterType regtype) { assert(compiler->IsOnlyOneRegMask(mask)); @@ -553,17 +555,23 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly { // The refPosition could be null, for example when called // by getTempRegForResolution(). - int minRegCount = (refPosition != nullptr) ? refPosition->minRegCandidateCount : 1; - RegisterType regType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType - : refPosition->getReg()->registerType; + int minRegCount = 1; + if (refPosition != nullptr) + { + minRegCount = refPosition->minRegCandidateCount; + RegisterType currRegType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType + : refPosition->getReg()->registerType; + assert(regtype == currRegType); + } + regMaskOnlyOne calleeSaved = RBM_NONE; regMaskOnlyOne calleeTrash = RBM_NONE; - if (varTypeUsesIntReg(regType)) + if (varTypeUsesIntReg(regtype)) { calleeSaved = RBM_INT_CALLEE_SAVED; calleeTrash = RBM_INT_CALLEE_TRASH; } - else if (varTypeUsesFloatReg(regType)) + else if (varTypeUsesFloatReg(regtype)) { calleeSaved = RBM_FLT_CALLEE_SAVED; calleeTrash = RBM_FLT_CALLEE_TRASH; @@ -571,7 +579,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly else { #ifdef HAS_PREDICATE_REGS - assert(varTypeUsesMaskReg(regType)); + assert(varTypeUsesMaskReg(regtype)); calleeSaved = RBM_MSK_CALLEE_SAVED; calleeTrash = RBM_MSK_CALLEE_TRASH; #else @@ -584,24 +592,24 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly case LSRA_LIMIT_CALLEE: if (!compiler->opts.compDbgEnC) { - mask = getConstrainedRegMask(refPosition, regType, mask, calleeSaved, minRegCount); + mask = getConstrainedRegMask(refPosition, regtype, mask, calleeSaved, minRegCount); } break; case LSRA_LIMIT_CALLER: { - mask = getConstrainedRegMask(refPosition, regType, mask, calleeTrash, minRegCount); + mask = getConstrainedRegMask(refPosition, regtype, mask, calleeTrash, minRegCount); } break; case LSRA_LIMIT_SMALL_SET: if ((mask & LsraLimitSmallIntSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitSmallIntSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regtype, mask, LsraLimitSmallIntSet, minRegCount); } else if ((mask & LsraLimitSmallFPSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitSmallFPSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regtype, mask, LsraLimitSmallFPSet, minRegCount); } break; @@ -609,7 +617,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly case LSRA_LIMIT_UPPER_SIMD_SET: if ((mask & LsraLimitUpperSimdSet) != RBM_NONE) { - mask = getConstrainedRegMask(refPosition, regType, mask, LsraLimitUpperSimdSet, minRegCount); + mask = getConstrainedRegMask(refPosition, regtype, mask, LsraLimitUpperSimdSet, minRegCount); } break; #endif @@ -8778,7 +8786,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, return REG_NA; } #endif // DEBUG - INDEBUG(freeRegs = stressLimitRegs(nullptr, freeRegs)); + INDEBUG(freeRegs = stressLimitRegs(nullptr, freeRegs, type)); freeRegs &= ~terminatorConsumedRegs; @@ -13474,7 +13482,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval } #ifdef DEBUG - candidates = linearScan->stressLimitRegs(refPosition, candidates); + candidates = linearScan->stressLimitRegs(refPosition, candidates, regType); #endif assert(candidates != RBM_NONE); @@ -13931,7 +13939,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( } #ifdef DEBUG - candidates = linearScan->stressLimitRegs(refPosition, candidates); + candidates = linearScan->stressLimitRegs(refPosition, candidates, regType); #endif assert(candidates != RBM_NONE); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 5e3117ef1b35b4..d851f34381f548 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -835,7 +835,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne regMaskActual, regMaskOnlyOne regMaskConstrain, unsigned minRegCount); - regMaskOnlyOne stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask); + regMaskOnlyOne stressLimitRegs(RefPosition* refPosition, regMaskOnlyOne mask, RegisterType regType); // This controls the heuristics used to select registers // These can be combined. From 85ed5a6a3b85c9e6ce56b715e657fb3d46ec369f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 13 Mar 2024 08:14:49 -0700 Subject: [PATCH 100/201] Introduce rbmAllFloats in emit* --- src/coreclr/jit/emit.cpp | 1 + src/coreclr/jit/emit.h | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 6f9345af2d35a2..ccae552fb049ce 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -747,6 +747,7 @@ void emitter::emitBegCG(Compiler* comp, COMP_HANDLE cmpHandle) #endif #if defined(TARGET_AMD64) + rbmAllFloat = emitComp->rbmAllFloat; rbmFltCalleeTrash = emitComp->rbmFltCalleeTrash; #endif // TARGET_AMD64 diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 7171cd9bc21ebf..fea9ec4df6c3c2 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2476,7 +2476,13 @@ class emitter private: #if defined(TARGET_AMD64) - regMaskFloat rbmFltCalleeTrash; + regMaskFloat rbmAllFloat; + regMaskFloat rbmFltCalleeTrash; + + FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const + { + return this->rbmAllFloat; + } FORCEINLINE regMaskFloat get_RBM_FLT_CALLEE_TRASH() const { From 9199913303f3f36bfa025e9664cf749a57504310 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 13 Mar 2024 13:20:39 -0700 Subject: [PATCH 101/201] fix the masks creation --- src/coreclr/jit/target.h | 4 +++- src/coreclr/jit/targetamd64.h | 8 ++++---- src/coreclr/jit/targetarm.h | 2 +- src/coreclr/jit/targetarm64.h | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 44cc03ef3c9ad4..6ec02aac026de6 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -425,7 +425,9 @@ typedef struct _regMaskAll } AllRegsMask; #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) -#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) +#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs)3 + +#define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask((gprRegs & ~RBM_ALLFLOAT), (floatRegs & RBM_ALLFLOAT)) #if REGMASK_BITS == 32 typedef unsigned regMaskSmall; diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index b2a68bdaad26c6..f2bb0ba0fece89 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -532,14 +532,14 @@ // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. // STOP_FOR_GC helper preserves all the 4 possible return registers. - #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) + #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) - #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) + #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #else // See vm\amd64\asmhelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH & AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET) + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET) #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) - #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET)) + #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET)) #endif // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 96a5d7c767781c..50c325a027db21 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -246,7 +246,7 @@ // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). // See vm\arm\amshelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11), ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) + #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11), ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 9e74fb927793ef..4e1fe9534e8294 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -240,7 +240,7 @@ #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 // The registers trashed by profiler enter/leave/tailcall hook - #define AllRegsMask_PROFILER_ENTER_TRASH (AllRegsMask_CALLEE_TRASH | AllRegsMask(~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP), ~RBM_FLTARG_REGS)) + #define AllRegsMask_PROFILER_ENTER_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP), ~RBM_FLTARG_REGS)) #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH From ab4ec811b3974f00c63e812c69df7f3d244d62e6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 15 Mar 2024 23:03:58 -0700 Subject: [PATCH 102/201] Condition free code: reduce TP from 5% -> 3% --- src/coreclr/jit/codegenarm.cpp | 6 +- src/coreclr/jit/codegenarm64.cpp | 16 +- src/coreclr/jit/codegenarmarch.cpp | 6 +- src/coreclr/jit/codegencommon.cpp | 58 +--- src/coreclr/jit/codegenlinear.cpp | 6 +- src/coreclr/jit/codegenxarch.cpp | 22 +- src/coreclr/jit/compiler.cpp | 10 +- src/coreclr/jit/compiler.hpp | 99 ++---- src/coreclr/jit/emit.cpp | 17 +- src/coreclr/jit/emitarm.cpp | 4 +- src/coreclr/jit/emitarm64.cpp | 8 +- src/coreclr/jit/emitloongarch64.cpp | 2 +- src/coreclr/jit/emitriscv64.cpp | 2 +- src/coreclr/jit/emitxarch.cpp | 10 +- src/coreclr/jit/gentree.cpp | 99 ++++-- src/coreclr/jit/gentree.h | 1 + src/coreclr/jit/lsra.cpp | 431 +++++++++++--------------- src/coreclr/jit/lsra.h | 56 ++-- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 90 ++---- src/coreclr/jit/lsraxarch.cpp | 2 +- src/coreclr/jit/register.h | 138 ++++----- src/coreclr/jit/registerarm.h | 100 +++--- src/coreclr/jit/registerarm64.h | 136 ++++---- src/coreclr/jit/registerloongarch64.h | 132 ++++---- src/coreclr/jit/registerriscv64.h | 132 ++++---- src/coreclr/jit/regset.cpp | 23 +- src/coreclr/jit/regset.h | 52 +--- src/coreclr/jit/target.h | 228 +++++--------- src/coreclr/jit/utils.cpp | 34 +- 30 files changed, 853 insertions(+), 1069 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 370142da448c71..9dc136e3a0ab33 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1822,9 +1822,9 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) { GetEmitter()->emitIns_Mov(INS_mov, attr, REG_R0, REG_PROFILER_RET_SCRATCH, /* canSkip */ false); genTransferRegGCState(REG_R0, REG_PROFILER_RET_SCRATCH); - assert(compiler->IsGprRegMask(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs)); - assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs == RBM_NONE); - gcInfo.gcMarkRegSetNpt(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs); + assert(compiler->IsGprRegMask(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs())); + assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs() == RBM_NONE); + gcInfo.gcMarkRegSetNpt(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs()); } } diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index d8dda334a8c340..2f35f511415be1 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -909,8 +909,8 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); - regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs; - regMaskTP maskSaveRegsInt = regsToSaveMask.gprRegs; + regMaskTP maskSaveRegsInt = regsToSaveMask.gprRegs(); + regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs(); assert(compiler->IsGprRegMask(maskSaveRegsInt)); assert(compiler->IsFloatRegMask(maskSaveRegsFloat)); @@ -1027,8 +1027,8 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, { assert(spDelta >= 0); - regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs; - regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs; + regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs(); + regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs(); assert(compiler->IsGprRegMask(maskRestoreRegsInt)); assert(compiler->IsFloatRegMask(maskRestoreRegsFloat)); @@ -3971,7 +3971,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) instGen_MemoryBarrier(); - gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(addr->gtGetGprRegMask()); } if (targetReg != REG_NA) @@ -4121,7 +4121,7 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) instGen_MemoryBarrier(); - gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(addr->gtGetGprRegMask()); } if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsSigned(treeNode->TypeGet())) @@ -5137,7 +5137,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet - noway_assert((callTargetMask & callKillSet.gprRegs) == callTargetMask); + noway_assert((callTargetMask & callKillSet.gprRegs()) == callTargetMask); callTarget = callTargetReg; @@ -5480,7 +5480,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) genEmitHelperCall(CORINFO_HELP_PROF_FCN_ENTER, 0, EA_UNKNOWN); - if ((genRegMask(initReg) & AllRegsMask_PROFILER_ENTER_TRASH.gprRegs) != RBM_NONE) + if (AllRegsMask_PROFILER_ENTER_TRASH.IsRegNumInMask(initReg)) { *pInitRegZeroed = false; } diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 84260c04b08f68..405a541c66c6c5 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1753,7 +1753,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) GetEmitter()->emitIns_R_R_I(INS_add, emitActualTypeSize(node), node->GetRegNum(), node->GetRegNum(), node->gtElemOffset); - gcInfo.gcMarkRegSetNpt(base->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(base->gtGetGprRegMask()); genProduceReg(node); } @@ -3390,8 +3390,8 @@ void CodeGen::genCall(GenTreeCall* call) killMask = compiler->compHelperCallKillSet(helpFunc); } - assert((gcInfo.gcRegGCrefSetCur & killMask.gprRegs) == 0); - assert((gcInfo.gcRegByrefSetCur & killMask.gprRegs) == 0); + assert(!killMask.IsGprMaskPresent(gcInfo.gcRegGCrefSetCur)); + assert(!killMask.IsGprMaskPresent(gcInfo.gcRegByrefSetCur)); #endif var_types returnType = call->TypeGet(); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 8061bc4e638c68..e83a311df187d5 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8206,21 +8206,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) if ((op1->GetRegSpillFlagByIdx(i) & GTF_SPILLED) == 0) { regNumber reg = op1->GetRegByIndex(i); - // regMaskOnlyOne regMask = genRegMask(); - if (genIsValidIntReg(reg)) - { - busyRegs.gprRegs |= genRegMask(reg); - } - else if (genIsValidFloatReg(reg)) - { - busyRegs.floatRegs |= genRegMask(reg); - } -#ifdef HAS_PREDICATE_REGS - else if (genIsValidMaskReg(reg)) - { - busyRegs.predicateRegs |= genRegMask(reg); - } -#endif // HAS_PREDICATE_REGS + busyRegs.AddRegNumInMask(reg); } } #endif // DEBUG @@ -8234,41 +8220,13 @@ void CodeGen::genRegCopy(GenTree* treeNode) #ifdef DEBUG -#define DO_VALIDATION(regType) \ - \ -if(targetReg != sourceReg) \ - \ -{ \ - singleRegMask targetRegMask = genRegMask(targetReg); \ - assert((busyRegs.regType & targetRegMask) == 0); \ - busyRegs.regType &= ~genRegMask(sourceReg); \ - \ -} \ - \ -busyRegs.regType |= genRegMask(targetReg); - - if (genIsValidIntReg(targetReg)) - { - DO_VALIDATION(gprRegs) - } - else if (genIsValidFloatReg(targetReg)) - { - DO_VALIDATION(floatRegs) - } -#ifdef HAS_PREDICATE_REGS - else if (genIsValidMaskReg(targetReg)) - { - DO_VALIDATION(predicateRegs) - } -#endif // HAS_PREDICATE_REGS - // if (targetReg != sourceReg) -//{ -// singleRegMask targetRegMask = genRegMask(targetReg); -// assert((busyRegs & targetRegMask) == 0); -// // Clear sourceReg from the busyRegs, and add targetReg. -// busyRegs &= ~genRegMask(sourceReg); -//} -// busyRegs |= genRegMask(targetReg); + if (targetReg != sourceReg) + { + singleRegMask targetRegMask = genRegMask(targetReg); + assert((busyRegs.GetMaskForRegNum(targetReg) & targetRegMask) == 0); + busyRegs.RemoveRegNumFromMask(sourceReg); + } + busyRegs.AddRegNumInMask(targetReg); #endif // DEBUG } return; diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 18785c8bd59333..6a4b7c5d37face 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -394,7 +394,7 @@ void CodeGen::genCodeForBBlist() // We cannot emit this code in the prolog as it might make the prolog too large. if (compiler->compShouldPoisonFrame() && compiler->fgBBisScratch(block)) { - genPoisonFrame(newLiveRegSet.gprRegs); + genPoisonFrame(newLiveRegSet.gprRegs()); } // Traverse the block in linear order, generating code for each node as we @@ -1584,7 +1584,7 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) } else { - gcInfo.gcMarkRegSetNpt(tree->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(tree->gtGetGprRegMask()); } genCheckConsumeNode(tree); @@ -1868,7 +1868,7 @@ void CodeGen::genConsumeArgSplitStruct(GenTreePutArgSplit* putArgNode) genUnspillRegIfNeeded(putArgNode); - gcInfo.gcMarkRegSetNpt(putArgNode->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(putArgNode->gtGetGprRegMask()); genCheckConsumeNode(putArgNode); } diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 7edeb0c178c459..e447e1be4c29e6 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1583,7 +1583,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // If there is a conflict then swap the condition anyway. LSRA should have // ensured the other way around has no conflict. - if ((trueVal->gtGetContainedRegMask() & dstReg) != 0) + if (trueVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1594,7 +1594,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // There may also be a conflict with the falseVal in case this is an AND // condition. Once again, after swapping there should be no conflict as // ensured by LSRA. - if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask() & dstReg) != 0) + if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) != 0) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1604,13 +1604,13 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) inst_RV_TT(INS_mov, emitTypeSize(select), dstReg, falseVal); assert(!trueVal->isContained() || trueVal->isUsedFromMemory()); - assert((trueVal->gtGetContainedRegMask() & dstReg) == 0); + assert((trueVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) == 0); inst_RV_TT(JumpKindToCmov(desc.jumpKind1), emitTypeSize(select), dstReg, trueVal); if (desc.oper == GT_AND) { assert(falseVal->isUsedFromReg()); - assert((falseVal->gtGetContainedRegMask() & dstReg) == 0); + assert((falseVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) == 0); inst_RV_TT(JumpKindToCmov(emitter::emitReverseJumpKind(desc.jumpKind2)), emitTypeSize(select), dstReg, falseVal); } @@ -5360,7 +5360,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) GetEmitter()->emitIns_R_ARX(INS_lea, emitTypeSize(node->TypeGet()), dstReg, baseReg, tmpReg, scale, static_cast(node->gtElemOffset)); - gcInfo.gcMarkRegSetNpt(base->gtGetRegMask().gprRegs); + gcInfo.gcMarkRegSetNpt(base->gtGetGprRegMask()); genProduceReg(node); } @@ -6034,8 +6034,8 @@ void CodeGen::genCall(GenTreeCall* call) killMask = compiler->compHelperCallKillSet(helpFunc); } - assert((gcInfo.gcRegGCrefSetCur & killMask.gprRegs) == 0); - assert((gcInfo.gcRegByrefSetCur & killMask.gprRegs) == 0); + assert(!killMask.IsGprMaskPresent(gcInfo.gcRegGCrefSetCur)); + assert(!killMask.IsGprMaskPresent(gcInfo.gcRegByrefSetCur)); #endif var_types returnType = call->TypeGet(); @@ -6849,7 +6849,8 @@ void CodeGen::genCompareFloat(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetReg) == 0) + AllRegsMask containedRegMask = op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask(); + if (!containedRegMask.IsRegNumInMask(targetReg)) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); targetType = TYP_UBYTE; // just a tip for inst_SETCC that movzx is not needed @@ -7019,7 +7020,8 @@ void CodeGen::genCompareInt(GenTree* treeNode) if ((targetReg != REG_NA) && (op1->GetRegNum() != targetReg) && (op2->GetRegNum() != targetReg) && !varTypeIsByte(targetType)) { - if (((op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask()) & targetReg) == 0) + AllRegsMask containedRegMask = op1->gtGetContainedRegMask() | op2->gtGetContainedRegMask(); + if (!containedRegMask.IsRegNumInMask(targetReg)) { instGen_Set_Reg_To_Zero(emitTypeSize(TYP_I_IMPL), targetReg); targetType = TYP_UBYTE; // just a tip for inst_SETCC that movzx is not needed @@ -9109,7 +9111,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, // this is only a valid assumption if the helper call is known to kill REG_DEFAULT_HELPER_CALL_TARGET. callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; regMaskGpr callTargetMask = genRegMask(callTargetReg); - noway_assert((callTargetMask & killMask.gprRegs) == callTargetMask); + noway_assert((callTargetMask & killMask.gprRegs()) == callTargetMask); } else { diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 48ca7296bdd3b6..98179a4d2b455a 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3643,24 +3643,24 @@ void Compiler::dumpRegMask(regMaskOnlyOne mask) const // void Compiler::dumpRegMask(AllRegsMask mask) const { - if (mask.gprRegs == RBM_ALLINT) + if (mask.gprRegs() == RBM_ALLINT) { printf("[allInt]"); } - else if (mask.gprRegs == (RBM_ALLINT & ~RBM_FPBASE)) + else if (mask.gprRegs() == (RBM_ALLINT & ~RBM_FPBASE)) { printf("[allIntButFP]"); } - else if (mask.floatRegs == RBM_ALLFLOAT) + else if (mask.floatRegs() == RBM_ALLFLOAT) { printf("[allFloat]"); } - else if (mask.floatRegs == RBM_ALLDOUBLE) + else if (mask.floatRegs() == RBM_ALLDOUBLE) { printf("[allDouble]"); } #ifdef TARGET_XARCH - else if (mask.predicateRegs == RBM_ALLMASK) + else if (mask.predicateRegs() == RBM_ALLMASK) { printf("[allMask]"); } diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 3169aea04d5274..8b98dd9b9eaf9a 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -970,41 +970,40 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskOnlyOne& mask) } //------------------------------------------------------------------------------ -// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a -// register number and also toggle the bit in the `mask`. +// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. +// // Arguments: // mask - the register mask // // Return Value: -// The number of the first register contained in the mask and updates the `mask` to toggle -// the bit. +// The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) +inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) { assert(!mask.IsEmpty()); // Must have one bit set, so can't have a mask of zero /* Convert the mask to a register number */ regNumber regNum; - if (mask.gprRegs != RBM_NONE) - { - regNum = (regNumber)BitOperations::BitScanForward(mask.gprRegs); - mask.gprRegs ^= genRegMask(regNum); - } + regMaskTP gprOrFloatMask = mask.GetGprFloatCombinedMask(); + #ifdef HAS_PREDICATE_REGS - else if (mask.predicateRegs != RBM_NONE) + // Only check this condition if there are predicate register support + // is present. + // If not, then gprOrFloatMask should be non-empty, and we will hit the + // above assert of IsEmpty() anyway. + if (gprOrFloatMask != RBM_NONE) { - regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs); - mask.predicateRegs ^= genRegMask(regNum); + regNum = (regNumber)BitOperations::BitScanForward(gprOrFloatMask); } -#endif else { - regNum = (regNumber)BitOperations::BitScanForward(mask.floatRegs); - mask.floatRegs ^= genRegMask(regNum); + regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs()); } - +#else + regNum = (regNumber)BitOperations::BitScanForward(gprOrFloatMask); +#endif return regNum; } @@ -1018,74 +1017,35 @@ inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) +inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) { - assert(!mask.IsEmpty()); // Must have one bit set, so can't have a mask of zero + assert(mask != 0); // Must have one bit set, so can't have a mask of zero /* Convert the mask to a register number */ - regNumber regNum; - if (mask.gprRegs != RBM_NONE) - { - regNum = (regNumber)BitOperations::BitScanForward(mask.gprRegs); - } -#ifdef HAS_PREDICATE_REGS - else if (mask.predicateRegs != RBM_NONE) - { - regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs); - } -#endif - else - { - regNum = (regNumber)BitOperations::BitScanForward(mask.floatRegs); - } + regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); return regNum; } //------------------------------------------------------------------------------ -// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. -// +// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a +// register number and also toggle the bit in the `mask`. // Arguments: // mask - the register mask // // Return Value: -// The number of the first register contained in the mask. +// The number of the first register contained in the mask and updates the `mask` to toggle +// the bit. // -inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) +inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) { - assert(mask != 0); // Must have one bit set, so can't have a mask of zero - - /* Convert the mask to a register number */ - - regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); - + regNumber regNum = genFirstRegNumFromMask(mask); + mask ^= regNum; return regNum; } -inline AllRegsMask createRegMask(regNumber reg) -{ - AllRegsMask result; - if (emitter::isGeneralRegister(reg)) - { - result.gprRegs = genRegMask(reg); - } - else if (emitter::isFloatReg(reg)) - { - result.floatRegs |= genRegMask(reg); - } - else - { -#ifdef HAS_PREDICATE_REGS - result.predicateRegs |= genRegMask(reg); -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } - return result; -} - /***************************************************************************** * * Return the size in bytes of the given type. @@ -1237,6 +1197,7 @@ inline const char* varTypeGCstring(var_types type) /*****************************************************************************/ const char* varTypeName(var_types); +const int regIndexForRegister(regNumber reg); /*****************************************************************************/ // Helpers to pull little-endian values out of a byte stream. @@ -4547,13 +4508,13 @@ inline void* operator new[](size_t sz, Compiler* compiler, CompMemKind cmk) inline void printRegMask(AllRegsMask mask) { - printf(REG_MASK_ALL_FMT, mask.gprRegs); + printf(REG_MASK_ALL_FMT, mask.gprRegs()); printf(" "); - printf(REG_MASK_ALL_FMT, mask.floatRegs); + printf(REG_MASK_ALL_FMT, mask.floatRegs()); #ifdef HAS_PREDICATE_REGS printf(" "); - printf(REG_MASK_ALL_FMT, mask.predicateRegs); + printf(REG_MASK_ALL_FMT, mask.predicateRegs()); #endif // HAS_PREDICATE_REGS } diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index ccae552fb049ce..25a2c7cc32fad9 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3516,19 +3516,10 @@ void emitter::emitDispGprRegSet(regMaskGpr regs) void emitter::emitDispRegSet(AllRegsMask regs) { - if (regs.gprRegs != RBM_NONE) - { - emitDispGprRegSet(regs.gprRegs); - } - if (regs.floatRegs != RBM_NONE) - { - emitDispRegSet(REG_FP_FIRST, REG_FP_LAST, regs.floatRegs); - } + emitDispGprRegSet(regs.gprRegs()); + emitDispGprRegSet(regs.floatRegs()); #ifdef HAS_PREDICATE_REGS - if (regs.predicateRegs != RBM_NONE) - { - emitDispRegSet(REG_MASK_FIRST, REG_MASK_LAST, regs.predicateRegs); - } + emitDispGprRegSet(regs.predicateRegs()); #endif } @@ -10403,7 +10394,7 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd); // Get the set of registers that this call kills and remove it from the saved set. - regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc).gprRegs; + regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc).gprRegs(); #ifdef DEBUG if (emitComp->verbose) diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 59de9f6cec6ca1..0221ca80681cc3 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -4695,8 +4695,8 @@ void emitter::emitIns_Call(EmitCallType callType, // Trim out any callee-trashed registers from the live set. AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet.gprRegs; - byrefRegs &= savedSet.gprRegs; + gcrefRegs &= savedSet.gprRegs(); + byrefRegs &= savedSet.gprRegs(); #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 36fd91d47c66d9..1437d3ad9987d5 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -3300,13 +3300,13 @@ emitAttr emitter::emitInsLoadStoreSize(instrDesc* id) // clang-format off static const char * const xRegNames[] = { - #define REGDEF(name, rnum, mask, xname, wname) xname, + #define REGDEF(name, rnum, mask, xname, wname, regTypeTag) xname, #include "register.h" }; static const char * const wRegNames[] = { - #define REGDEF(name, rnum, mask, xname, wname) wname, + #define REGDEF(name, rnum, mask, xname, wname, regTypeTag) wname, #include "register.h" }; @@ -18413,8 +18413,8 @@ void emitter::emitIns_Call(EmitCallType callType, // Trim out any callee-trashed registers from the live set. AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet.gprRegs; - byrefRegs &= savedSet.gprRegs; + gcrefRegs &= savedSet.gprRegs(); + byrefRegs &= savedSet.gprRegs(); #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 6aaf00a973c8a7..8686aa924134aa 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -3889,7 +3889,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) // clang-format off static const char* const RegNames[] = { - #define REGDEF(name, rnum, mask, sname) sname, + #define REGDEF(name, rnum, mask, sname, regTypeTag) sname, #include "register.h" }; // clang-format on diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index 9e9f9004170964..0aa69c592fc186 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -3415,7 +3415,7 @@ void emitter::emitDispIllegalInstruction(code_t instructionCode) // clang-format off static const char* const RegNames[] = { - #define REGDEF(name, rnum, mask, sname) sname, + #define REGDEF(name, rnum, mask, sname, regTypeTag) sname, #include "register.h" }; // clang-format on diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 472287ddfc7ac4..b3a518a9017430 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9519,8 +9519,8 @@ void emitter::emitIns_Call(EmitCallType callType, // Trim out any callee-trashed registers from the live set. AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet.gprRegs; - byrefRegs &= savedSet.gprRegs; + gcrefRegs &= savedSet.gprRegs(); + byrefRegs &= savedSet.gprRegs(); #ifdef DEBUG if (EMIT_GC_VERBOSE) @@ -10222,7 +10222,7 @@ const char* emitter::emitRegName(regNumber reg, emitAttr attr, bool varName) con const char* emitter::emitXMMregName(unsigned reg) const { static const char* const regNames[] = { -#define REGDEF(name, rnum, mask, sname) "x" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "x" sname, #include "register.h" }; @@ -10240,7 +10240,7 @@ const char* emitter::emitXMMregName(unsigned reg) const const char* emitter::emitYMMregName(unsigned reg) const { static const char* const regNames[] = { -#define REGDEF(name, rnum, mask, sname) "y" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "y" sname, #include "register.h" }; @@ -10258,7 +10258,7 @@ const char* emitter::emitYMMregName(unsigned reg) const const char* emitter::emitZMMregName(unsigned reg) const { static const char* const regNames[] = { -#define REGDEF(name, rnum, mask, sname) "z" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "z" sname, #include "register.h" }; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 40fabe734d79d2..8d56c2e324d2a7 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1081,13 +1081,15 @@ AllRegsMask GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // +//TODO: All the callers of gtGetRegMask() are just interested in gpr +// so, this can just return gpr mask and if none exist, just return RBM_NONE AllRegsMask GenTree::gtGetRegMask() const { AllRegsMask resultMask; if (IsMultiRegCall()) { - resultMask = createRegMask(GetRegNum()); + resultMask = AllRegsMask(GetRegNum()); resultMask |= AsCall()->GetOtherRegMask(); } else if (IsCopyOrReloadOfMultiRegCall()) @@ -1125,7 +1127,82 @@ AllRegsMask GenTree::gtGetRegMask() const #endif // FEATURE_ARG_SPLIT else { - resultMask = createRegMask(GetRegNum()); + resultMask = AllRegsMask(GetRegNum()); + } + + return resultMask; +} + +//--------------------------------------------------------------- +// gtGetRegMask: Get the gpr reg mask of the node. +// +// Arguments: +// None +// +// Return Value: +// Reg Mask of GenTree node. +// +// Note: This method would populate the reg mask with only the GPR registers. +regMaskGpr GenTree::gtGetGprRegMask() const +{ + regMaskGpr resultMask = RBM_NONE; + + if (IsMultiRegCall()) + { + regNumber reg = GetRegNum(); + resultMask |= -static_cast(!regIndexForRegister(reg)) & genRegMask(reg); + +#if FEATURE_MULTIREG_RET + const GenTreeCall* call = AsCall(); + for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i) + { + regNumber otherReg = (regNumber)call->gtOtherRegs[i]; + if (otherReg != REG_NA) + { + resultMask |= -static_cast(!regIndexForRegister(otherReg)) & genRegMask(otherReg); + continue; + } + break; + } +#endif + } + else if (IsCopyOrReloadOfMultiRegCall()) + { + // A multi-reg copy or reload, will have valid regs for only those + // positions that need to be copied or reloaded. Hence we need + // to consider only those registers for computing reg mask. + + const GenTreeCopyOrReload* copyOrReload = AsCopyOrReload(); + const GenTreeCall* call = copyOrReload->gtGetOp1()->AsCall(); + const unsigned regCount = call->GetReturnTypeDesc()->GetReturnRegCount(); + + for (unsigned i = 0; i < regCount; ++i) + { + regNumber reg = copyOrReload->GetRegNumByIdx(i); + if (reg != REG_NA) + { + resultMask |= -static_cast(!regIndexForRegister(reg)) & genRegMask(reg); + } + } + } +#if FEATURE_ARG_SPLIT + else if (compFeatureArgSplit() && OperIsPutArgSplit()) + { + const GenTreePutArgSplit* splitArg = AsPutArgSplit(); + const unsigned regCount = splitArg->gtNumRegs; + + for (unsigned i = 0; i < regCount; ++i) + { + regNumber reg = splitArg->GetRegNumByIdx(i); + assert(reg != REG_NA); + resultMask |= -static_cast(!regIndexForRegister(reg)) & genRegMask(reg); + } + } +#endif // FEATURE_ARG_SPLIT + else + { + regNumber reg = GetRegNum(); + resultMask |= -static_cast(!regIndexForRegister(reg)) & genRegMask(reg); } return resultMask; @@ -27188,23 +27265,7 @@ AllRegsMask ReturnTypeDesc::GetABIReturnRegs() const for (unsigned i = 0; i < count; ++i) { regNumber reg = GetABIReturnReg(i); - if ((reg >= REG_INT_FIRST) && (reg <= REG_INT_LAST)) - { - resultMask.gprRegs |= genRegMask(reg); - } - else if ((reg >= REG_FP_FIRST) && (reg <= REG_FP_LAST)) - { - resultMask.floatRegs |= genRegMask(reg); - } - else - { -#ifdef HAS_PREDICATE_REGS - assert((reg >= REG_MASK_FIRST) && (reg <= REG_MASK_LAST)); - resultMask.predicateRegs |= genRegMask(reg); -#else - unreached(); -#endif - } + resultMask.AddRegNumInMask(reg); } return resultMask; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5adf5339bdf9b4..3fe98308713636 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -955,6 +955,7 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; AllRegsMask gtGetRegMask() const; + regMaskGpr gtGetGprRegMask() const; AllRegsMask gtGetContainedRegMask(); GenTreeFlags gtFlags; diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 658d60bdb610df..a4a8db8d17d1f2 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -278,12 +278,12 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo if (nextRefPosition == nullptr) { nextLocation = MaxLocation; - fixedRegs[regTypeIndex(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); + fixedRegs[regIndexForType(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); } else { nextLocation = nextRefPosition->nodeLocation; - fixedRegs[regTypeIndex(regRecord->registerType)] |= genRegMask(regRecord->regNum); + fixedRegs[regIndexForType(regRecord->registerType)] |= genRegMask(regRecord->regNum); } nextFixedRef[regRecord->regNum] = nextLocation; } @@ -293,7 +293,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); - regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regTypeIndex(currentInterval->registerType)]); + regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regIndexForType(currentInterval->registerType)]); regMaskOnlyOne result = RBM_NONE; while (candidates != RBM_NONE) { @@ -494,21 +494,7 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { regMaskOnlyOne busyRegs = RBM_NONE; - if (varTypeUsesIntReg(regType)) - { - busyRegs = regsBusyUntilKill.gprRegs | regsInUseThisLocation.gprRegs; - } - else if (varTypeUsesFloatReg(regType)) - { - busyRegs = regsBusyUntilKill.floatRegs | regsInUseThisLocation.floatRegs; - } - else - { -#ifdef HAS_PREDICATE_REGS - assert(varTypeUsesMaskReg(regType)); - busyRegs = regsBusyUntilKill.predicateRegs | regsInUseThisLocation.predicateRegs; -#endif - } + busyRegs = regsBusyUntilKill.OrMask(regsInUseThisLocation, regType); if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -4163,28 +4149,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskOnlyOne candidateRegs = RBM_NONE; - regMaskOnlyOne allRegsMask = allRegs(interval->registerType); - RegisterType regType = interval->registerType; - - if (varTypeUsesIntReg(regType)) - { - candidateRegs = allRegsMask & availableRegs.gprRegs; - } - else if (varTypeUsesFloatReg(regType)) - { - candidateRegs = allRegsMask & availableRegs.floatRegs; - } - else - { -#ifdef HAS_PREDICATE_REGS - assert(varTypeUsesMaskReg(regType)); - candidateRegs = allRegsMask & availableRegs.predicateRegs; -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } + regMaskOnlyOne allRegsMask = allRegs(interval->registerType); + RegisterType regType = interval->registerType; + regMaskOnlyOne candidateRegs = availableRegs.GetRegTypeMask(regType); regNumber firstReg = REG_NA; regNumber newReg = REG_NA; @@ -4665,7 +4633,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) RegRecord* anotherHalfRegRec = findAnotherHalfRegRec(targetRegRecord); // Use TYP_FLOAT to get the regmask of just the half reg. - liveRegs.floatRegs &= ~getRegMask(anotherHalfRegRec->regNum, TYP_FLOAT); + liveRegs.RemoveRegNumFromMask(anotherHalfRegRec->regNum, TYP_FLOAT); } #endif // TARGET_ARM @@ -4692,7 +4660,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { RegRecord* physRegRecord = getRegisterRecord(reg); - if ((liveRegs & reg) == 0) + if (!liveRegs.IsRegNumInMask(reg)) { makeRegAvailable(reg, physRegRecord->registerType); Interval* assignedInterval = physRegRecord->assignedInterval; @@ -5512,7 +5480,7 @@ void LinearScan::allocateRegistersMinimal() currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumInMask(assignedRegister, + regsToFree.RemoveRegNumFromMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. @@ -6742,7 +6710,7 @@ void LinearScan::allocateRegisters() currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumInMask(assignedRegister, + regsToFree.RemoveRegNumFromMask(assignedRegister, currentInterval->registerType); // we'll set it again later if it's dead // If this interval is dead, free the register. @@ -9877,7 +9845,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber upperHalfReg = REG_NEXT(fromReg); if ((otherInterval->registerType == TYP_DOUBLE) && (location[upperHalfReg] != REG_NA)) { - targetRegsReady.RemoveRegNumInMask(fromReg); + targetRegsReady.RemoveRegNumFromMask(fromReg); } } } @@ -9921,7 +9889,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber fromReg = (regNumber)location[sourceReg]; if (targetReg == fromReg) { - targetRegsToDo.RemoveRegNumInMask(targetReg); + targetRegsToDo.RemoveRegNumFromMask(targetReg); } else { @@ -9964,7 +9932,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Otherwise, we'll spill it to the stack and reload it later. if (useSwap) { - targetRegsToDo.RemoveRegNumInMask(fromReg); + targetRegsToDo.RemoveRegNumFromMask(fromReg); } } else @@ -10010,7 +9978,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, targetRegsFromStack |= otherTargetReg; stackToRegIntervals[otherTargetReg] = otherInterval; - targetRegsToDo.RemoveRegNumInMask(otherTargetReg); + targetRegsToDo.RemoveRegNumFromMask(otherTargetReg); // Now, move the interval that is going to targetReg. addResolution(block, insertionPoint, sourceIntervals[sourceReg], targetReg, @@ -10035,13 +10003,13 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regNumber upperHalfReg = REG_NEXT(fromReg); if ((otherInterval->registerType == TYP_DOUBLE) && (location[upperHalfReg] != REG_NA)) { - targetRegsReady.RemoveRegNumInMask(fromReg); + targetRegsReady.RemoveRegNumFromMask(fromReg); } } #endif // TARGET_ARM } } - targetRegsToDo.RemoveRegNumInMask(targetReg); + targetRegsToDo.RemoveRegNumFromMask(targetReg); } else { @@ -11581,7 +11549,7 @@ void LinearScan::dumpRegRecords() #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE printf("%c", activeChar); } - else if ((regsBusyUntilKill & regNum) != RBM_NONE) + else if (regsBusyUntilKill.IsRegNumInMask(regNum)) { printf(columnFormatArray, "Busy"); } @@ -11813,19 +11781,19 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) // void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) { - regMaskOnlyOne regsMaskToFree = regsToFree.gprRegs; + regMaskOnlyOne regsMaskToFree = regsToFree.gprRegs(); regMaskOnlyOne availableRegsMask = availableIntRegs; for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; reg = REG_NEXT(reg)) { if (reg >= REG_FP_FIRST && reg <= REG_FP_LAST) { - regsMaskToFree = regsToFree.floatRegs; + regsMaskToFree = regsToFree.floatRegs(); availableRegsMask = availableFloatRegs; } #ifdef HAS_PREDICATE_REGS else if (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST) { - regsMaskToFree = regsToFree.predicateRegs; + regsMaskToFree = regsToFree.predicateRegs(); availableRegsMask = availableMaskRegs; } #endif @@ -13652,7 +13620,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs.floatRegs & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs() & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM @@ -13663,7 +13631,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regTypeIndex(regType)]; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regIndexForType(regType)]; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -13742,7 +13710,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // refpositions. assert((refPosition->refType == RefTypeUpperVectorRestore) || (genCountBits(candidates) == 1)); - freeCandidates = candidates & linearScan->m_AvailableRegs[regTypeIndex(currentInterval->registerType)]; + freeCandidates = candidates & linearScan->m_AvailableRegs[regIndexForType(currentInterval->registerType)]; } if ((freeCandidates == RBM_NONE) && (candidates == RBM_NONE)) @@ -13758,7 +13726,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // such range that is consecutive. Next, append that range to the `candidates`. // regMaskFloat limitCandidatesForConsecutive = - refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs; + refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs(); regMaskFloat overallLimitCandidates; regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, @@ -13970,23 +13938,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - if (varTypeUsesIntReg(regType)) - { - candidates &= ~busyRegs.gprRegs; - } - else if (varTypeUsesFloatReg(regType)) - { - candidates &= ~busyRegs.floatRegs; - } - else - { -#ifdef HAS_PREDICATE_REGS - assert(varTypeUsesMaskReg(regType)); - candidates &= ~busyRegs.predicateRegs; -#else - unreached(); -#endif - } + candidates &= ~busyRegs.GetRegTypeMask(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -13996,14 +13948,14 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs.floatRegs & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs() & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regTypeIndex(regType)]; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regIndexForType(regType)]; while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -14083,245 +14035,234 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( return candidates; } -bool operator==(const AllRegsMask& first, const AllRegsMask& second) +void AllRegsMask::Clear() { - return first.gprRegs == second.gprRegs && first.floatRegs == second.floatRegs + registers[0] = RBM_NONE; + registers[1] = RBM_NONE; #ifdef HAS_PREDICATE_REGS - && first.predicateRegs == second.predicateRegs + registers[2] = RBM_NONE; #endif - ; } -bool operator!=(const AllRegsMask& first, const AllRegsMask& second) +bool AllRegsMask::IsEmpty() { - return !(first == second); + return (gprRegs() | floatRegs() +#ifdef HAS_PREDICATE_REGS + | predicateRegs() +#endif // HAS_PREDICATE_REGS + ) == RBM_NONE; } -AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second) +unsigned AllRegsMask::Count() { - AllRegsMask result(first.gprRegs & second.gprRegs, first.floatRegs & second.floatRegs + return genCountBits(gprRegs()) + genCountBits(floatRegs()) #ifdef HAS_PREDICATE_REGS - , - first.predicateRegs & second.predicateRegs -#endif - ); - return result; + + genCountBits(predicateRegs()) +#endif // HAS_PREDICATE_REGS + ; } -regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg) +regMaskOnlyOne AllRegsMask::operator[](int index) const { - if (emitter::isGeneralRegister(reg)) - { - return first.gprRegs & genRegMask(reg); - } - else if (emitter::isFloatReg(reg)) - { - return first.floatRegs & genRegMask(reg); - } - else - { -#ifdef HAS_PREDICATE_REGS - return first.predicateRegs & genRegMask(reg); -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } + assert(index <= REGISTER_TYPE_COUNT); + return registers[index]; } -AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second) +regMaskOnlyOne& AllRegsMask::operator[](int index) { - AllRegsMask result(first.gprRegs | second.gprRegs, first.floatRegs | second.floatRegs + assert(index <= REGISTER_TYPE_COUNT); + return registers[index]; +} + +void AllRegsMask::operator|=(const _regMaskAll& other) +{ + // TODO: Can we optimize to reintrepret_cast + // Something like https://godbolt.org/z/1KevT8Edh + registers[0] |= other.gprRegs(); + registers[1] |= other.floatRegs(); #ifdef HAS_PREDICATE_REGS - , - first.predicateRegs | second.predicateRegs + registers[2] |= other.predicateRegs(); #endif - ); - return result; } -AllRegsMask operator|=(AllRegsMask& first, const regNumber reg) +void AllRegsMask::operator&=(const _regMaskAll& other) { - if (emitter::isGeneralRegister(reg)) - { - first.gprRegs |= genRegMask(reg); - } - else if (emitter::isFloatReg(reg)) - { - first.floatRegs |= genRegMask(reg); - } - else - { + // TODO: Can we optimize to reintrepret_cast + // Something like https://godbolt.org/z/1KevT8Edh + registers[0] &= other.gprRegs(); + registers[1] &= other.floatRegs(); #ifdef HAS_PREDICATE_REGS - first.predicateRegs |= genRegMask(reg); -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } - return first; + registers[2] &= other.predicateRegs(); +#endif } -AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second) +void AllRegsMask::operator|=(const regNumber reg) { - first.gprRegs |= second.gprRegs; - first.floatRegs |= second.floatRegs; + registers[regIndexForRegister(reg)] |= genRegMask(reg); +} + +void AllRegsMask::operator^=(const regNumber reg) +{ + registers[regIndexForRegister(reg)] ^= genRegMask(reg); +} +_regMaskAll AllRegsMask::operator~() +{ + return _regMaskAll(~gprRegs(), ~floatRegs() #ifdef HAS_PREDICATE_REGS - first.predicateRegs |= second.predicateRegs; + , + ~predicateRegs() #endif - return first; + ); } -AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second) +bool AllRegsMask::operator==(const AllRegsMask& other) { - first.gprRegs &= second.gprRegs; - first.floatRegs &= second.floatRegs; - + return ((gprRegs() == other.gprRegs()) && (floatRegs() == other.floatRegs()) #ifdef HAS_PREDICATE_REGS - first.predicateRegs &= second.predicateRegs; + && (predicateRegs() == other.predicateRegs()) #endif - return first; + ); } -AllRegsMask operator^=(AllRegsMask& first, const regNumber reg) +bool AllRegsMask::operator!=(const AllRegsMask& other) { - if (emitter::isGeneralRegister(reg)) - { - first.gprRegs ^= genRegMask(reg); - } - else if (emitter::isFloatReg(reg)) - { - first.floatRegs ^= genRegMask(reg); - } - else - { + return !(*this == other); +} + +_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) +{ + return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() #ifdef HAS_PREDICATE_REGS - first.predicateRegs ^= genRegMask(reg); -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } - return first; + , + predicateRegs() & other.predicateRegs() +#endif + ); } -AllRegsMask operator~(const AllRegsMask& first) +_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) { - AllRegsMask result(~first.gprRegs, ~first.floatRegs + return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() #ifdef HAS_PREDICATE_REGS - , - ~first.predicateRegs + , + predicateRegs() | other.predicateRegs() #endif - ); + ); +} + +_regMaskAll AllRegsMask::operator&(const regNumber reg) +{ + _regMaskAll result = *this; + result[regIndexForRegister(reg)] &= genRegMask(reg); return result; } +// TODO: Can have version of AddGprMask(regMaskOnlyOne maskToAdd) and AddFloatMask(regMaskOnlyOne maskToAdd) +void AllRegsMask::AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) +{ + registers[regIndexForType(type)] |= maskToAdd; +} + +void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) +{ + registers[0] |= maskToAdd; +} + +void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) +{ + registers[1] |= maskToAdd; +} + +// Adds reg only if it is gpr register +void AllRegsMask::AddGprRegInMask(regNumber reg) +{ + int regIndex = regIndexForRegister(reg); + registers[0] |= -static_cast(!regIndex) & genRegMask(reg); +} + +void AllRegsMask::AddRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMask = genRegMask(reg); + registers[regIndexForRegister(reg)] |= regMask; +} + void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) { regMaskOnlyOne regMask = genRegMask(reg, type); AddRegTypeMask(regMask, type); } -void AllRegsMask::RemoveRegNumInMask(regNumber reg, var_types type) +void AllRegsMask::RemoveRegNumFromMask(regNumber reg, var_types type) { regMaskOnlyOne regMaskToRemove = genRegMask(reg, type); - - if (emitter::isGeneralRegister(reg)) - { - gprRegs &= ~regMaskToRemove; - } - else if (emitter::isFloatReg(reg)) - { - floatRegs &= ~regMaskToRemove; - } - else - { -#ifdef HAS_PREDICATE_REGS - predicateRegs &= ~regMaskToRemove; -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } + registers[regIndexForType(type)] &= ~regMaskToRemove; } -void AllRegsMask::RemoveRegNumInMask(regNumber reg) +void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { regMaskOnlyOne regMaskToRemove = genRegMask(reg); + registers[regIndexForRegister(reg)] &= ~regMaskToRemove; +} - if (emitter::isGeneralRegister(reg)) - { - gprRegs &= ~regMaskToRemove; - } - else if (emitter::isFloatReg(reg)) - { - floatRegs &= ~regMaskToRemove; - } - else - { -#ifdef HAS_PREDICATE_REGS - predicateRegs &= ~regMaskToRemove; -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } +void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) +{ + registers[regIndexForType(type)] &= ~regMaskToRemove; } bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) { - if (emitter::isGeneralRegister(reg)) - { - return (gprRegs & genRegMask(reg)) != RBM_NONE; - } - else if (emitter::isFloatReg(reg)) - { - return (floatRegs & genRegMask(reg, type)) != RBM_NONE; - } - else - { -#ifdef HAS_PREDICATE_REGS - return (predicateRegs & genRegMask(reg)) != RBM_NONE; -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } + regMaskOnlyOne regMaskToCheck = genRegMask(reg, type); + return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; } bool AllRegsMask::IsRegNumInMask(regNumber reg) { - regMaskOnlyOne regMask = genRegMask(reg); - if (emitter::isGeneralRegister(reg)) - { - return (gprRegs & regMask) != RBM_NONE; - } - else if (emitter::isFloatReg(reg)) - { - return (floatRegs & regMask) != RBM_NONE; - } - else - { -#ifdef HAS_PREDICATE_REGS - return (predicateRegs & regMask) != RBM_NONE; -#else - unreached(); -#endif // HAS_PREDICATE_REGS - } + regMaskOnlyOne regMaskToCheck = genRegMask(reg); + return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; +} + +bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const +{ + return (gprRegs() & maskToCheck) != RBM_NONE; +} + +bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const +{ + return (floatRegs() & maskToCheck) != RBM_NONE; } bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) { regMaskOnlyOne regMask = genRegMask(reg); - if (emitter::isGeneralRegister(reg)) - { - return (gprRegs & regMask) == regMask; - } - else if (emitter::isFloatReg(reg)) - { - return (floatRegs & regMask) == regMask; - } - else - { -#ifdef HAS_PREDICATE_REGS - return (predicateRegs & regMask) == regMask; + return (registers[regIndexForRegister(reg)] & regMask) == regMask; +} + +regMaskOnlyOne AllRegsMask::GetRegTypeMask(var_types type) const +{ + return registers[regIndexForType(type)]; +} + +regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const +{ + return registers[regIndexForRegister(reg)]; +} + +regMaskTP AllRegsMask::GetGprFloatCombinedMask() const +{ +#ifdef TARGET_64BIT + return ((regMaskTP)floatRegs() << 32) | gprRegs(); #else - unreached(); -#endif // HAS_PREDICATE_REGS - } + return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); +#endif +} + +bool AllRegsMask::IsGprOrFloatPresent() const +{ + return (gprRegs() | floatRegs()) != RBM_NONE; +} + +regMaskOnlyOne AllRegsMask::OrMask(const _regMaskAll& other, var_types regType) const +{ + int regIndex = regIndexForType(regType); + return registers[regIndex] | other[regIndex]; } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 24304ba669564b..71d44abc6edd7e 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -66,23 +66,14 @@ RegisterType regType(T type) // TODO: If there are lot of callers of RegisterType, simplify it. template -FORCEINLINE unsigned regTypeIndex(T type) +FORCEINLINE int regIndexForType(T vt) { - if (varTypeUsesIntReg(type)) - { - return 0; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - return 2; - } -#endif // HAS_PREDICATE_REGS - else - { - assert(varTypeUsesFloatReg(type)); - return 1; - } + int type = varTypeRegister[TypeGet(vt)]; + assert(type <= REGISTER_TYPE_COUNT); +#ifndef HAS_PREDICATE_REGS + assert(type != VTR_MASK); +#endif + return (type - 1); } //------------------------------------------------------------------------ @@ -1257,7 +1248,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates, var_types regType) { - regMaskOnlyOne result = candidates & m_AvailableRegs[regTypeIndex(regType)]; + regMaskOnlyOne result = candidates & m_AvailableRegs[regIndexForType(regType)]; #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use register for which the odd half is // also available. @@ -1759,7 +1750,10 @@ class LinearScan : public LinearScanInterface // register type. That wawy we do not have to query and fetch the appropriate // entry again and agin. #ifdef HAS_PREDICATE_REGS - regMaskOnlyOne m_AvailableRegs[3]; // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. + // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. + // TODO: Check why this can't be AllRegsMask + regMaskOnlyOne m_AvailableRegs[3]; + #else regMaskOnlyOne m_AvailableRegs[2]; #endif // HAS_PREDICATE_REGS @@ -1834,10 +1828,10 @@ class LinearScan : public LinearScanInterface void setRegsInUse(AllRegsMask regMask) { // TODO: Fix this later. - m_AvailableRegs[0] &= ~regMask.gprRegs; - m_AvailableRegs[1] &= ~regMask.floatRegs; + m_AvailableRegs[0] &= ~regMask.gprRegs(); + m_AvailableRegs[1] &= ~regMask.floatRegs(); #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] &= ~regMask.predicateRegs; + m_AvailableRegs[2] &= ~regMask.predicateRegs(); #endif } void setRegInUse(regNumber reg, var_types regType) @@ -1867,10 +1861,10 @@ class LinearScan : public LinearScanInterface void makeRegsAvailable(AllRegsMask regMask) { // TODO: This will be just `regMask` - m_AvailableRegs[0] |= regMask.gprRegs; - m_AvailableRegs[1] |= regMask.floatRegs; + m_AvailableRegs[0] |= regMask.gprRegs(); + m_AvailableRegs[1] |= regMask.floatRegs(); #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] |= regMask.predicateRegs; + m_AvailableRegs[2] |= regMask.predicateRegs(); assert(compiler->IsPredicateRegMask(m_AvailableRegs[2])); #endif assert(compiler->IsGprRegMask(m_AvailableRegs[0])); @@ -1886,7 +1880,7 @@ class LinearScan : public LinearScanInterface // rbx - 0 // ... // xmm0 - 1 - // here it will be, m_AvailableRegs[regTypeIndex[reg]] = + // here it will be, m_AvailableRegs[regIndexForType[reg]] = if (emitter::isGeneralRegisterOrZR(reg)) #else if (emitter::isGeneralRegister(reg)) @@ -1929,7 +1923,7 @@ class LinearScan : public LinearScanInterface assert(compiler->IsFloatRegMask(regMask)); } #endif - m_AvailableRegs[regTypeIndex(regType)] |= regMask; + m_AvailableRegs[regIndexForType(regType)] |= regMask; } void clearAllNextIntervalRef(); @@ -1960,19 +1954,19 @@ class LinearScan : public LinearScanInterface { // TODO: If we decide to have curr_RegistersWithConstants, then here we will // just operate on curr_RegistersWithConstants and assert - // assert(m_RegistersWithConstants[regTypeIndex(regType)] == curr_RegistersWithConstants); + // assert(m_RegistersWithConstants[regIndexForType(regType)] == curr_RegistersWithConstants); // but we will have to make sure that we save it back too?? - m_RegistersWithConstants[regTypeIndex(regType)] &= ~getRegMask(reg, regType); + m_RegistersWithConstants[regIndexForType(regType)] &= ~getRegMask(reg, regType); } void setConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants[regTypeIndex(regType)] |= getRegMask(reg, regType); + m_RegistersWithConstants[regIndexForType(regType)] |= getRegMask(reg, regType); } bool isRegConstant(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); regMaskOnlyOne regMask = getRegMask(reg, regType); - return (m_RegistersWithConstants[regTypeIndex(regType)] & regMask) == regMask; + return (m_RegistersWithConstants[regIndexForType(regType)] & regMask) == regMask; } regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); @@ -2021,7 +2015,7 @@ class LinearScan : public LinearScanInterface } void clearRegBusyUntilKill(regNumber reg) { - regsBusyUntilKill.RemoveRegNumInMask(reg); + regsBusyUntilKill.RemoveRegNumFromMask(reg); } bool isRegInUse(regNumber reg, var_types regType) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 2bd41d09930484..4fe934868edf88 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -414,7 +414,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (call->HasSwiftErrorHandling()) { // Tree is a Swift call with error handling; error register should have been killed - assert((killMask.gprRegs & RBM_SWIFT_ERROR) != 0); + assert((killMask.gprRegs() & RBM_SWIFT_ERROR) != 0); // After a Swift call that might throw returns, we expect the error register to be consumed // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 03fa9e536bfa7a..606eed1a00297a 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -703,23 +703,17 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) void LinearScan::addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); - if (mask.gprRegs != RBM_NONE) - { - compiler->codeGen->regSet.rsSetGprRegsModified(mask.gprRegs DEBUGARG(true)); - addRefsForPhysRegMask(mask.gprRegs, currentLoc, refType, isLastUse); - } - if (mask.floatRegs != RBM_NONE) - { - compiler->codeGen->regSet.rsSetFloatRegsModified(mask.floatRegs DEBUGARG(true)); - addRefsForPhysRegMask(mask.floatRegs, currentLoc, refType, isLastUse); - } + + compiler->codeGen->regSet.rsSetGprRegsModified(mask.gprRegs() DEBUGARG(true)); + compiler->codeGen->regSet.rsSetFloatRegsModified(mask.floatRegs() DEBUGARG(true)); + + addRefsForPhysRegMask(mask.gprRegs(), currentLoc, refType, isLastUse); + addRefsForPhysRegMask(mask.floatRegs(), currentLoc, refType, isLastUse); + #ifdef HAS_PREDICATE_REGS - if (mask.predicateRegs != RBM_NONE) - { - compiler->codeGen->regSet.rsSetPredicateRegsModified(mask.predicateRegs DEBUGARG(true)); - addRefsForPhysRegMask(mask.predicateRegs, currentLoc, refType, isLastUse); - } -#endif // HAS_PREDICATE_REGS + compiler->codeGen->regSet.rsSetPredicateRegsModified(mask.predicateRegs() DEBUGARG(true)); + addRefsForPhysRegMask(mask.predicateRegs(), currentLoc, refType, isLastUse); +#endif } //------------------------------------------------------------------------ @@ -896,22 +890,22 @@ AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) // if there is no FP used, we can ignore the FP kills if (!compiler->compFloatingPointUsed) { - killMask.floatRegs &= ~RBM_FLT_CALLEE_TRASH; + killMask.RemoveRegTypeFromMask(RBM_FLT_CALLEE_TRASH, TYP_FLOAT); #if defined(TARGET_XARCH) || defined(HAS_PREDICATE_REGS) - killMask.predicateRegs &= ~RBM_MSK_CALLEE_TRASH; + killMask.RemoveRegTypeFromMask(RBM_MSK_CALLEE_TRASH, TYP_MASK); #endif // TARGET_XARCH || HAS_PREDICATE_REGS } #ifdef TARGET_ARM if (call->IsVirtualStub()) { - killMask.gprRegs |= compiler->virtualStubParamInfo->GetRegMask(); + killMask.AddGprRegMask(compiler->virtualStubParamInfo->GetRegMask()); } #else // !TARGET_ARM // Verify that the special virtual stub call registers are in the kill mask. // We don't just add them unconditionally to the killMask because for most architectures // they are already in the RBM_CALLEE_TRASH set, // and we don't want to introduce extra checks and calls in this hot function. - assert(!call->IsVirtualStub() || ((killMask.gprRegs & compiler->virtualStubParamInfo->GetRegMask()) == + assert(!call->IsVirtualStub() || ((killMask.gprRegs() & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); #endif // !TARGET_ARM @@ -920,7 +914,7 @@ AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) // so don't use the register post-call until it is consumed by SwiftError. if (call->HasSwiftErrorHandling()) { - killMask.gprRegs |= RBM_SWIFT_ERROR; + killMask.AddGprRegMask(RBM_SWIFT_ERROR); } #endif // SWIFT_SUPPORT @@ -956,7 +950,7 @@ AllRegsMask LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) if (isCopyBlk) { // rep movs kills RCX, RDI and RSI - killMask.gprRegs = RBM_RCX | RBM_RDI | RBM_RSI; + killMask.AddGprRegMask(RBM_RCX | RBM_RDI | RBM_RSI); } else { @@ -964,7 +958,7 @@ AllRegsMask LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) // (Note that the Data() node, if not constant, will be assigned to // RCX, but it's find that this kills it, as the value is not available // after this node in any case.) - killMask.gprRegs = RBM_RDI | RBM_RCX; + killMask.AddGprRegMask(RBM_RDI | RBM_RCX); } break; #endif @@ -1065,7 +1059,7 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) case GT_LSH_HI: case GT_RSH_LO: #endif - killMask.gprRegs = getKillSetForShiftRotate(tree->AsOp()); + killMask.AddGprRegMask(getKillSetForShiftRotate(tree->AsOp())); break; case GT_MUL: @@ -1073,14 +1067,14 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) #if !defined(TARGET_64BIT) || defined(TARGET_ARM64) case GT_MUL_LONG: #endif - killMask.gprRegs = getKillSetForMul(tree->AsOp()); + killMask.AddGprRegMask(getKillSetForMul(tree->AsOp())); break; case GT_MOD: case GT_DIV: case GT_UMOD: case GT_UDIV: - killMask.gprRegs = getKillSetForModDiv(tree->AsOp()); + killMask.AddGprRegMask(getKillSetForModDiv(tree->AsOp())); break; case GT_STORE_BLK: @@ -1115,7 +1109,7 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: - killMask.gprRegs = getKillSetForHWIntrinsic(tree->AsHWIntrinsic()); + killMask.AddGprRegMask(getKillSetForHWIntrinsic(tree->AsHWIntrinsic())); break; #endif // FEATURE_HW_INTRINSICS @@ -1152,10 +1146,10 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask) { - assert(compiler->IsGprRegMask(killMask.gprRegs)); - assert(compiler->IsFloatRegMask(killMask.floatRegs)); + assert(compiler->IsGprRegMask(killMask.gprRegs())); + assert(compiler->IsFloatRegMask(killMask.floatRegs())); #ifdef HAS_PREDICATE_REGS - assert(compiler->IsPredicateRegMask(killMask.predicateRegs)); + assert(compiler->IsPredicateRegMask(killMask.predicateRegs())); #endif // HAS_PREDICATE_REGS bool insertedKills = false; @@ -1197,7 +1191,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo Interval* interval = getIntervalForLocalVar(varIndex); regMaskOnlyOne regsKillMask = killMask.GetRegTypeMask(interval->registerType); const bool isCallKill = - (killMask.gprRegs == RBM_INT_CALLEE_TRASH) || (killMask == AllRegsMask_CALLEE_TRASH); + (killMask.gprRegs() == RBM_INT_CALLEE_TRASH) || (killMask == AllRegsMask_CALLEE_TRASH); if (isCallKill) { @@ -2560,11 +2554,11 @@ void LinearScan::buildIntervals() AllRegsMask killed; #if defined(TARGET_XARCH) // Poisoning uses EAX for small vars and rep stosd that kills edi, ecx and eax for large vars. - killed.gprRegs = RBM_EDI | RBM_ECX | RBM_EAX; + killed.AddGprRegMask(RBM_EDI | RBM_ECX | RBM_EAX); #else // Poisoning uses REG_SCRATCH for small vars and memset helper for big vars. killed = compiler->compHelperCallKillSet(CORINFO_HELP_NATIVE_MEMSET); - killed.gprRegs |= genRegMask(REG_SCRATCH); + killed.AddRegNumInMask(REG_SCRATCH); #endif addRefsForPhysRegMask(killed, currentLoc + 1, RefTypeKill, true); currentLoc += 2; @@ -3094,32 +3088,14 @@ void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandi for (int i = 0; i < dstCount; i++) { - singleRegMask thisDstCandidates; // In case of multi-reg call node, we have to query the i'th position return register. // For all other cases of multi-reg definitions, the registers must be in sequential order. regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i); - thisDstCandidates = genRegMask(thisReg); - if (emitter::isGeneralRegister(thisReg)) - { - assert((dstCandidates.gprRegs & thisDstCandidates) != RBM_NONE); - dstCandidates.gprRegs &= ~thisDstCandidates; - } - else if (emitter::isFloatReg(thisReg)) - { - assert((dstCandidates.floatRegs & thisDstCandidates) != RBM_NONE); - dstCandidates.floatRegs &= ~thisDstCandidates; - } -#ifdef HAS_PREDICATE_REGS - else - { - assert(emitter::isMaskReg(thisReg)); - assert((dstCandidates.predicateRegs & thisDstCandidates) != RBM_NONE); - dstCandidates.predicateRegs &= ~thisDstCandidates; - } -#endif // HAS_PREDICATE_REGS + assert(dstCandidates.IsRegNumInMask(thisReg)); + dstCandidates.RemoveRegNumFromMask(thisReg); - BuildDef(tree, thisDstCandidates, i); + BuildDef(tree, genRegMask(thisReg), i); } } @@ -3187,10 +3163,10 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) // RefPositions in that case. // This must be done after the kills, so that we know which large vectors are still live. // - if ((killMask.floatRegs & RBM_FLT_CALLEE_TRASH) != RBM_NONE) + if ((killMask.IsFloatMaskPresent(RBM_FLT_CALLEE_TRASH))) { buildUpperVectorSaveRefPositions(tree, - currentLoc + 1 DEBUG_ARG((killMask.floatRegs & RBM_FLT_CALLEE_TRASH))); + currentLoc + 1 DEBUG_ARG((killMask.floatRegs() & RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } @@ -3307,7 +3283,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) { // This local's value is going to be available in this register so // keep it in the preferences. - unpref.RemoveRegNumInMask(placedArgLocals[i].Reg); + unpref.RemoveRegNumFromMask(placedArgLocals[i].Reg); } } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index f844e374645398..ee5cb02061e441 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1395,7 +1395,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (call->HasSwiftErrorHandling()) { // Tree is a Swift call with error handling; error register should have been killed - assert((killMask.gprRegs & RBM_SWIFT_ERROR) != 0); + assert((killMask.gprRegs() & RBM_SWIFT_ERROR) != 0); // After a Swift call that might throw returns, we expect the error register to be consumed // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed diff --git a/src/coreclr/jit/register.h b/src/coreclr/jit/register.h index 43454e96ec64e4..2f84c8c40acbf6 100644 --- a/src/coreclr/jit/register.h +++ b/src/coreclr/jit/register.h @@ -16,15 +16,15 @@ #if defined(TARGET_X86) /* -REGDEF(name, rnum, mask, sname) */ -REGDEF(EAX, 0, 0x01, "eax" ) -REGDEF(ECX, 1, 0x02, "ecx" ) -REGDEF(EDX, 2, 0x04, "edx" ) -REGDEF(EBX, 3, 0x08, "ebx" ) -REGDEF(ESP, 4, 0x10, "esp" ) -REGDEF(EBP, 5, 0x20, "ebp" ) -REGDEF(ESI, 6, 0x40, "esi" ) -REGDEF(EDI, 7, 0x80, "edi" ) +REGDEF(name, rnum, mask, sname, regTypeTag) */ +REGDEF(EAX, 0, 0x01, "eax", 0) +REGDEF(ECX, 1, 0x02, "ecx", 0) +REGDEF(EDX, 2, 0x04, "edx", 0) +REGDEF(EBX, 3, 0x08, "ebx", 0) +REGDEF(ESP, 4, 0x10, "esp", 0) +REGDEF(EBP, 5, 0x20, "ebp", 0) +REGDEF(ESI, 6, 0x40, "esi", 0) +REGDEF(EDI, 7, 0x80, "edi", 0) REGALIAS(RAX, EAX) REGALIAS(RCX, ECX) REGALIAS(RDX, EDX) @@ -37,23 +37,23 @@ REGALIAS(RDI, EDI) #else // !defined(TARGET_X86) /* -REGDEF(name, rnum, mask, sname) */ -REGDEF(RAX, 0, 0x0001, "rax" ) -REGDEF(RCX, 1, 0x0002, "rcx" ) -REGDEF(RDX, 2, 0x0004, "rdx" ) -REGDEF(RBX, 3, 0x0008, "rbx" ) -REGDEF(RSP, 4, 0x0010, "rsp" ) -REGDEF(RBP, 5, 0x0020, "rbp" ) -REGDEF(RSI, 6, 0x0040, "rsi" ) -REGDEF(RDI, 7, 0x0080, "rdi" ) -REGDEF(R8, 8, 0x0100, "r8" ) -REGDEF(R9, 9, 0x0200, "r9" ) -REGDEF(R10, 10, 0x0400, "r10" ) -REGDEF(R11, 11, 0x0800, "r11" ) -REGDEF(R12, 12, 0x1000, "r12" ) -REGDEF(R13, 13, 0x2000, "r13" ) -REGDEF(R14, 14, 0x4000, "r14" ) -REGDEF(R15, 15, 0x8000, "r15" ) +REGDEF(name, rnum, mask, sname, regTypeTag) */ +REGDEF(RAX, 0, 0x0001, "rax", 0) +REGDEF(RCX, 1, 0x0002, "rcx", 0) +REGDEF(RDX, 2, 0x0004, "rdx", 0) +REGDEF(RBX, 3, 0x0008, "rbx", 0) +REGDEF(RSP, 4, 0x0010, "rsp", 0) +REGDEF(RBP, 5, 0x0020, "rbp", 0) +REGDEF(RSI, 6, 0x0040, "rsi", 0) +REGDEF(RDI, 7, 0x0080, "rdi", 0) +REGDEF(R8, 8, 0x0100, "r8" , 0) +REGDEF(R9, 9, 0x0200, "r9" , 0) +REGDEF(R10, 10, 0x0400, "r10", 0) +REGDEF(R11, 11, 0x0800, "r11", 0) +REGDEF(R12, 12, 0x1000, "r12", 0) +REGDEF(R13, 13, 0x2000, "r13", 0) +REGDEF(R14, 14, 0x4000, "r14", 0) +REGDEF(R15, 15, 0x8000, "r15", 0) REGALIAS(EAX, RAX) REGALIAS(ECX, RCX) @@ -83,55 +83,55 @@ REGALIAS(EDI, RDI) #endif // !TARGET_AMD64 -REGDEF(XMM0, 0+XMMBASE, XMMMASK(0), "mm0" ) -REGDEF(XMM1, 1+XMMBASE, XMMMASK(1), "mm1" ) -REGDEF(XMM2, 2+XMMBASE, XMMMASK(2), "mm2" ) -REGDEF(XMM3, 3+XMMBASE, XMMMASK(3), "mm3" ) -REGDEF(XMM4, 4+XMMBASE, XMMMASK(4), "mm4" ) -REGDEF(XMM5, 5+XMMBASE, XMMMASK(5), "mm5" ) -REGDEF(XMM6, 6+XMMBASE, XMMMASK(6), "mm6" ) -REGDEF(XMM7, 7+XMMBASE, XMMMASK(7), "mm7" ) +REGDEF(XMM0, 0+XMMBASE, XMMMASK(0), "mm0", 1) +REGDEF(XMM1, 1+XMMBASE, XMMMASK(1), "mm1", 1) +REGDEF(XMM2, 2+XMMBASE, XMMMASK(2), "mm2", 1) +REGDEF(XMM3, 3+XMMBASE, XMMMASK(3), "mm3", 1) +REGDEF(XMM4, 4+XMMBASE, XMMMASK(4), "mm4", 1) +REGDEF(XMM5, 5+XMMBASE, XMMMASK(5), "mm5", 1) +REGDEF(XMM6, 6+XMMBASE, XMMMASK(6), "mm6", 1) +REGDEF(XMM7, 7+XMMBASE, XMMMASK(7), "mm7", 1) #ifdef TARGET_AMD64 -REGDEF(XMM8, 8+XMMBASE, XMMMASK(8), "mm8" ) -REGDEF(XMM9, 9+XMMBASE, XMMMASK(9), "mm9" ) -REGDEF(XMM10, 10+XMMBASE, XMMMASK(10), "mm10" ) -REGDEF(XMM11, 11+XMMBASE, XMMMASK(11), "mm11" ) -REGDEF(XMM12, 12+XMMBASE, XMMMASK(12), "mm12" ) -REGDEF(XMM13, 13+XMMBASE, XMMMASK(13), "mm13" ) -REGDEF(XMM14, 14+XMMBASE, XMMMASK(14), "mm14" ) -REGDEF(XMM15, 15+XMMBASE, XMMMASK(15), "mm15" ) - -REGDEF(XMM16, 16+XMMBASE, XMMMASK(16), "mm16" ) -REGDEF(XMM17, 17+XMMBASE, XMMMASK(17), "mm17" ) -REGDEF(XMM18, 18+XMMBASE, XMMMASK(18), "mm18" ) -REGDEF(XMM19, 19+XMMBASE, XMMMASK(19), "mm19" ) -REGDEF(XMM20, 20+XMMBASE, XMMMASK(20), "mm20" ) -REGDEF(XMM21, 21+XMMBASE, XMMMASK(21), "mm21" ) -REGDEF(XMM22, 22+XMMBASE, XMMMASK(22), "mm22" ) -REGDEF(XMM23, 23+XMMBASE, XMMMASK(23), "mm23" ) - -REGDEF(XMM24, 24+XMMBASE, XMMMASK(24), "mm24" ) -REGDEF(XMM25, 25+XMMBASE, XMMMASK(25), "mm25" ) -REGDEF(XMM26, 26+XMMBASE, XMMMASK(26), "mm26" ) -REGDEF(XMM27, 27+XMMBASE, XMMMASK(27), "mm27" ) -REGDEF(XMM28, 28+XMMBASE, XMMMASK(28), "mm28" ) -REGDEF(XMM29, 29+XMMBASE, XMMMASK(29), "mm29" ) -REGDEF(XMM30, 30+XMMBASE, XMMMASK(30), "mm30" ) -REGDEF(XMM31, 31+XMMBASE, XMMMASK(31), "mm31" ) +REGDEF(XMM8, 8+XMMBASE, XMMMASK(8), "mm8", 1) +REGDEF(XMM9, 9+XMMBASE, XMMMASK(9), "mm9", 1) +REGDEF(XMM10, 10+XMMBASE, XMMMASK(10), "mm10", 1) +REGDEF(XMM11, 11+XMMBASE, XMMMASK(11), "mm11", 1) +REGDEF(XMM12, 12+XMMBASE, XMMMASK(12), "mm12", 1) +REGDEF(XMM13, 13+XMMBASE, XMMMASK(13), "mm13", 1) +REGDEF(XMM14, 14+XMMBASE, XMMMASK(14), "mm14", 1) +REGDEF(XMM15, 15+XMMBASE, XMMMASK(15), "mm15", 1) + +REGDEF(XMM16, 16+XMMBASE, XMMMASK(16), "mm16", 1) +REGDEF(XMM17, 17+XMMBASE, XMMMASK(17), "mm17", 1) +REGDEF(XMM18, 18+XMMBASE, XMMMASK(18), "mm18", 1) +REGDEF(XMM19, 19+XMMBASE, XMMMASK(19), "mm19", 1) +REGDEF(XMM20, 20+XMMBASE, XMMMASK(20), "mm20", 1) +REGDEF(XMM21, 21+XMMBASE, XMMMASK(21), "mm21", 1) +REGDEF(XMM22, 22+XMMBASE, XMMMASK(22), "mm22", 1) +REGDEF(XMM23, 23+XMMBASE, XMMMASK(23), "mm23", 1) + +REGDEF(XMM24, 24+XMMBASE, XMMMASK(24), "mm24", 1) +REGDEF(XMM25, 25+XMMBASE, XMMMASK(25), "mm25", 1) +REGDEF(XMM26, 26+XMMBASE, XMMMASK(26), "mm26", 1) +REGDEF(XMM27, 27+XMMBASE, XMMMASK(27), "mm27", 1) +REGDEF(XMM28, 28+XMMBASE, XMMMASK(28), "mm28", 1) +REGDEF(XMM29, 29+XMMBASE, XMMMASK(29), "mm29", 1) +REGDEF(XMM30, 30+XMMBASE, XMMMASK(30), "mm30", 1) +REGDEF(XMM31, 31+XMMBASE, XMMMASK(31), "mm31", 1) #endif // !TARGET_AMD64 -REGDEF(K0, 0+KBASE, KMASK(0), "k0" ) -REGDEF(K1, 1+KBASE, KMASK(1), "k1" ) -REGDEF(K2, 2+KBASE, KMASK(2), "k2" ) -REGDEF(K3, 3+KBASE, KMASK(3), "k3" ) -REGDEF(K4, 4+KBASE, KMASK(4), "k4" ) -REGDEF(K5, 5+KBASE, KMASK(5), "k5" ) -REGDEF(K6, 6+KBASE, KMASK(6), "k6" ) -REGDEF(K7, 7+KBASE, KMASK(7), "k7" ) +REGDEF(K0, 0+KBASE, KMASK(0), "k0", 2) +REGDEF(K1, 1+KBASE, KMASK(1), "k1", 2) +REGDEF(K2, 2+KBASE, KMASK(2), "k2", 2) +REGDEF(K3, 3+KBASE, KMASK(3), "k3", 2) +REGDEF(K4, 4+KBASE, KMASK(4), "k4", 2) +REGDEF(K5, 5+KBASE, KMASK(5), "k5", 2) +REGDEF(K6, 6+KBASE, KMASK(6), "k6", 2) +REGDEF(K7, 7+KBASE, KMASK(7), "k7", 2) -REGDEF(STK, 8+KBASE, 0x0000, "STK" ) +REGDEF(STK, 8+KBASE, 0x0000, "STK", -1) #elif defined(TARGET_ARM) #include "registerarm.h" diff --git a/src/coreclr/jit/registerarm.h b/src/coreclr/jit/registerarm.h index ad70eaa211ccf6..678c0ea3fb0035 100644 --- a/src/coreclr/jit/registerarm.h +++ b/src/coreclr/jit/registerarm.h @@ -13,59 +13,59 @@ #endif /* -REGDEF(name, rnum, mask, sname) */ -REGDEF(R0, 0, 0x0001, "r0" ) -REGDEF(R1, 1, 0x0002, "r1" ) -REGDEF(R2, 2, 0x0004, "r2" ) -REGDEF(R3, 3, 0x0008, "r3" ) -REGDEF(R4, 4, 0x0010, "r4" ) -REGDEF(R5, 5, 0x0020, "r5" ) -REGDEF(R6, 6, 0x0040, "r6" ) -REGDEF(R7, 7, 0x0080, "r7" ) -REGDEF(R8, 8, 0x0100, "r8" ) -REGDEF(R9, 9, 0x0200, "r9" ) -REGDEF(R10, 10, 0x0400, "r10" ) -REGDEF(R11, 11, 0x0800, "r11" ) -REGDEF(R12, 12, 0x1000, "r12" ) -REGDEF(SP, 13, 0x2000, "sp" ) -REGDEF(LR, 14, 0x4000, "lr" ) -REGDEF(PC, 15, 0x8000, "pc" ) +REGDEF(name, rnum, mask, sname, regTypeTag) */ +REGDEF(R0, 0, 0x0001, "r0", 0) +REGDEF(R1, 1, 0x0002, "r1", 0) +REGDEF(R2, 2, 0x0004, "r2", 0) +REGDEF(R3, 3, 0x0008, "r3", 0) +REGDEF(R4, 4, 0x0010, "r4", 0) +REGDEF(R5, 5, 0x0020, "r5", 0) +REGDEF(R6, 6, 0x0040, "r6", 0) +REGDEF(R7, 7, 0x0080, "r7", 0) +REGDEF(R8, 8, 0x0100, "r8", 0) +REGDEF(R9, 9, 0x0200, "r9", 0) +REGDEF(R10, 10, 0x0400, "r10",0) +REGDEF(R11, 11, 0x0800, "r11",0) +REGDEF(R12, 12, 0x1000, "r12",0) +REGDEF(SP, 13, 0x2000, "sp", 0) +REGDEF(LR, 14, 0x4000, "lr", 0) +REGDEF(PC, 15, 0x8000, "pc", 0) #define FPBASE 16 #define VFPMASK(x) (((__int64)1) << (x+FPBASE)) -REGDEF(F0, 0+FPBASE, VFPMASK(0), "f0") -REGDEF(F1, 1+FPBASE, VFPMASK(1), "f1") -REGDEF(F2, 2+FPBASE, VFPMASK(2), "f2") -REGDEF(F3, 3+FPBASE, VFPMASK(3), "f3") -REGDEF(F4, 4+FPBASE, VFPMASK(4), "f4") -REGDEF(F5, 5+FPBASE, VFPMASK(5), "f5") -REGDEF(F6, 6+FPBASE, VFPMASK(6), "f6") -REGDEF(F7, 7+FPBASE, VFPMASK(7), "f7") -REGDEF(F8, 8+FPBASE, VFPMASK(8), "f8") -REGDEF(F9, 9+FPBASE, VFPMASK(9), "f9") -REGDEF(F10, 10+FPBASE, VFPMASK(10), "f10") -REGDEF(F11, 11+FPBASE, VFPMASK(11), "f11") -REGDEF(F12, 12+FPBASE, VFPMASK(12), "f12") -REGDEF(F13, 13+FPBASE, VFPMASK(13), "f13") -REGDEF(F14, 14+FPBASE, VFPMASK(14), "f14") -REGDEF(F15, 15+FPBASE, VFPMASK(15), "f15") -REGDEF(F16, 16+FPBASE, VFPMASK(16), "f16") -REGDEF(F17, 17+FPBASE, VFPMASK(17), "f17") -REGDEF(F18, 18+FPBASE, VFPMASK(18), "f18") -REGDEF(F19, 19+FPBASE, VFPMASK(19), "f19") -REGDEF(F20, 20+FPBASE, VFPMASK(20), "f20") -REGDEF(F21, 21+FPBASE, VFPMASK(21), "f21") -REGDEF(F22, 22+FPBASE, VFPMASK(22), "f22") -REGDEF(F23, 23+FPBASE, VFPMASK(23), "f23") -REGDEF(F24, 24+FPBASE, VFPMASK(24), "f24") -REGDEF(F25, 25+FPBASE, VFPMASK(25), "f25") -REGDEF(F26, 26+FPBASE, VFPMASK(26), "f26") -REGDEF(F27, 27+FPBASE, VFPMASK(27), "f27") -REGDEF(F28, 28+FPBASE, VFPMASK(28), "f28") -REGDEF(F29, 29+FPBASE, VFPMASK(29), "f29") -REGDEF(F30, 30+FPBASE, VFPMASK(30), "f30") -REGDEF(F31, 31+FPBASE, VFPMASK(31), "f31") +REGDEF(F0, 0+FPBASE, VFPMASK(0), "f0", 1) +REGDEF(F1, 1+FPBASE, VFPMASK(1), "f1", 1) +REGDEF(F2, 2+FPBASE, VFPMASK(2), "f2", 1) +REGDEF(F3, 3+FPBASE, VFPMASK(3), "f3", 1) +REGDEF(F4, 4+FPBASE, VFPMASK(4), "f4", 1) +REGDEF(F5, 5+FPBASE, VFPMASK(5), "f5", 1) +REGDEF(F6, 6+FPBASE, VFPMASK(6), "f6", 1) +REGDEF(F7, 7+FPBASE, VFPMASK(7), "f7", 1) +REGDEF(F8, 8+FPBASE, VFPMASK(8), "f8", 1) +REGDEF(F9, 9+FPBASE, VFPMASK(9), "f9", 1) +REGDEF(F10, 10+FPBASE, VFPMASK(10), "f10", 1) +REGDEF(F11, 11+FPBASE, VFPMASK(11), "f11", 1) +REGDEF(F12, 12+FPBASE, VFPMASK(12), "f12", 1) +REGDEF(F13, 13+FPBASE, VFPMASK(13), "f13", 1) +REGDEF(F14, 14+FPBASE, VFPMASK(14), "f14", 1) +REGDEF(F15, 15+FPBASE, VFPMASK(15), "f15", 1) +REGDEF(F16, 16+FPBASE, VFPMASK(16), "f16", 1) +REGDEF(F17, 17+FPBASE, VFPMASK(17), "f17", 1) +REGDEF(F18, 18+FPBASE, VFPMASK(18), "f18", 1) +REGDEF(F19, 19+FPBASE, VFPMASK(19), "f19", 1) +REGDEF(F20, 20+FPBASE, VFPMASK(20), "f20", 1) +REGDEF(F21, 21+FPBASE, VFPMASK(21), "f21", 1) +REGDEF(F22, 22+FPBASE, VFPMASK(22), "f22", 1) +REGDEF(F23, 23+FPBASE, VFPMASK(23), "f23", 1) +REGDEF(F24, 24+FPBASE, VFPMASK(24), "f24", 1) +REGDEF(F25, 25+FPBASE, VFPMASK(25), "f25", 1) +REGDEF(F26, 26+FPBASE, VFPMASK(26), "f26", 1) +REGDEF(F27, 27+FPBASE, VFPMASK(27), "f27", 1) +REGDEF(F28, 28+FPBASE, VFPMASK(28), "f28", 1) +REGDEF(F29, 29+FPBASE, VFPMASK(29), "f29", 1) +REGDEF(F30, 30+FPBASE, VFPMASK(30), "f30", 1) +REGDEF(F31, 31+FPBASE, VFPMASK(31), "f31", 1) // Allow us to call R11/FP, SP, LR and PC by their register number names @@ -75,7 +75,7 @@ REGALIAS(R14, LR) REGALIAS(R15, PC) // This must be last! -REGDEF(STK, 32+FPBASE, 0x0000, "STK") +REGDEF(STK, 32+FPBASE, 0x0000, "STK", -1) /*****************************************************************************/ #undef REGDEF diff --git a/src/coreclr/jit/registerarm64.h b/src/coreclr/jit/registerarm64.h index e8c126fac148cb..d864db3db26c44 100644 --- a/src/coreclr/jit/registerarm64.h +++ b/src/coreclr/jit/registerarm64.h @@ -15,39 +15,39 @@ #define RMASK(x) (1ULL << (x)) /* -REGDEF(name, rnum, mask, xname, wname) */ -REGDEF(R0, 0, 0x0001, "x0" , "w0" ) -REGDEF(R1, 1, 0x0002, "x1" , "w1" ) -REGDEF(R2, 2, 0x0004, "x2" , "w2" ) -REGDEF(R3, 3, 0x0008, "x3" , "w3" ) -REGDEF(R4, 4, 0x0010, "x4" , "w4" ) -REGDEF(R5, 5, 0x0020, "x5" , "w5" ) -REGDEF(R6, 6, 0x0040, "x6" , "w6" ) -REGDEF(R7, 7, 0x0080, "x7" , "w7" ) -REGDEF(R8, 8, 0x0100, "x8" , "w8" ) -REGDEF(R9, 9, 0x0200, "x9" , "w9" ) -REGDEF(R10, 10, 0x0400, "x10", "w10" ) -REGDEF(R11, 11, 0x0800, "x11", "w11" ) -REGDEF(R12, 12, 0x1000, "x12", "w12" ) -REGDEF(R13, 13, 0x2000, "x13", "w13" ) -REGDEF(R14, 14, 0x4000, "x14", "w14" ) -REGDEF(R15, 15, 0x8000, "x15", "w15" ) -REGDEF(IP0, 16, 0x10000, "xip0","wip0" ) -REGDEF(IP1, 17, 0x20000, "xip1","wip1" ) -REGDEF(PR, 18, 0x40000, "xpr", "wpr" ) -REGDEF(R19, 19, 0x80000, "x19", "w19" ) -REGDEF(R20, 20, 0x100000, "x20", "w20" ) -REGDEF(R21, 21, 0x200000, "x21", "w21" ) -REGDEF(R22, 22, 0x400000, "x22", "w22" ) -REGDEF(R23, 23, 0x800000, "x23", "w23" ) -REGDEF(R24, 24, 0x1000000, "x24", "w24" ) -REGDEF(R25, 25, 0x2000000, "x25", "w25" ) -REGDEF(R26, 26, 0x4000000, "x26", "w26" ) -REGDEF(R27, 27, 0x8000000, "x27", "w27" ) -REGDEF(R28, 28, 0x10000000, "x28", "w28" ) -REGDEF(FP, 29, 0x20000000, "fp" , "w29" ) -REGDEF(LR, 30, 0x40000000, "lr" , "w30" ) -REGDEF(ZR, 31, 0x80000000, "xzr", "wzr" ) +REGDEF(name, rnum, mask, xname, wname, regTypeTag) */ +REGDEF(R0, 0, 0x0001, "x0" , "w0", 0) +REGDEF(R1, 1, 0x0002, "x1" , "w1", 0) +REGDEF(R2, 2, 0x0004, "x2" , "w2", 0) +REGDEF(R3, 3, 0x0008, "x3" , "w3", 0) +REGDEF(R4, 4, 0x0010, "x4" , "w4", 0) +REGDEF(R5, 5, 0x0020, "x5" , "w5", 0) +REGDEF(R6, 6, 0x0040, "x6" , "w6", 0) +REGDEF(R7, 7, 0x0080, "x7" , "w7", 0) +REGDEF(R8, 8, 0x0100, "x8" , "w8", 0) +REGDEF(R9, 9, 0x0200, "x9" , "w9", 0) +REGDEF(R10, 10, 0x0400, "x10", "w10", 0) +REGDEF(R11, 11, 0x0800, "x11", "w11", 0) +REGDEF(R12, 12, 0x1000, "x12", "w12", 0) +REGDEF(R13, 13, 0x2000, "x13", "w13", 0) +REGDEF(R14, 14, 0x4000, "x14", "w14", 0) +REGDEF(R15, 15, 0x8000, "x15", "w15", 0) +REGDEF(IP0, 16, 0x10000, "xip0","wip0",0) +REGDEF(IP1, 17, 0x20000, "xip1","wip1",0) +REGDEF(PR, 18, 0x40000, "xpr", "wpr", 0) +REGDEF(R19, 19, 0x80000, "x19", "w19", 0) +REGDEF(R20, 20, 0x100000, "x20", "w20", 0) +REGDEF(R21, 21, 0x200000, "x21", "w21", 0) +REGDEF(R22, 22, 0x400000, "x22", "w22", 0) +REGDEF(R23, 23, 0x800000, "x23", "w23", 0) +REGDEF(R24, 24, 0x1000000, "x24", "w24", 0) +REGDEF(R25, 25, 0x2000000, "x25", "w25", 0) +REGDEF(R26, 26, 0x4000000, "x26", "w26", 0) +REGDEF(R27, 27, 0x8000000, "x27", "w27", 0) +REGDEF(R28, 28, 0x10000000, "x28", "w28", 0) +REGDEF(FP, 29, 0x20000000, "fp" , "w29", 0) +REGDEF(LR, 30, 0x40000000, "lr" , "w30", 0) +REGDEF(ZR, 31, 0x80000000, "xzr", "wzr", 0) // Allow us to call IP0,IP1,PR,FP,LR by their register number names REGALIAS(R16, IP0) @@ -60,39 +60,39 @@ REGALIAS(R30, LR) #define VMASK(x) (1ULL << (VBASE+(x))) /* -REGDEF(name, rnum, mask, xname, wname) */ -REGDEF(V0, 0+VBASE, VMASK(0), "d0", "s0") -REGDEF(V1, 1+VBASE, VMASK(1), "d1", "s1") -REGDEF(V2, 2+VBASE, VMASK(2), "d2", "s2") -REGDEF(V3, 3+VBASE, VMASK(3), "d3", "s3") -REGDEF(V4, 4+VBASE, VMASK(4), "d4", "s4") -REGDEF(V5, 5+VBASE, VMASK(5), "d5", "s5") -REGDEF(V6, 6+VBASE, VMASK(6), "d6", "s6") -REGDEF(V7, 7+VBASE, VMASK(7), "d7", "s7") -REGDEF(V8, 8+VBASE, VMASK(8), "d8", "s8") -REGDEF(V9, 9+VBASE, VMASK(9), "d9", "s9") -REGDEF(V10, 10+VBASE, VMASK(10), "d10", "s10") -REGDEF(V11, 11+VBASE, VMASK(11), "d11", "s11") -REGDEF(V12, 12+VBASE, VMASK(12), "d12", "s12") -REGDEF(V13, 13+VBASE, VMASK(13), "d13", "s13") -REGDEF(V14, 14+VBASE, VMASK(14), "d14", "s14") -REGDEF(V15, 15+VBASE, VMASK(15), "d15", "s15") -REGDEF(V16, 16+VBASE, VMASK(16), "d16", "s16") -REGDEF(V17, 17+VBASE, VMASK(17), "d17", "s17") -REGDEF(V18, 18+VBASE, VMASK(18), "d18", "s18") -REGDEF(V19, 19+VBASE, VMASK(19), "d19", "s19") -REGDEF(V20, 20+VBASE, VMASK(20), "d20", "s20") -REGDEF(V21, 21+VBASE, VMASK(21), "d21", "s21") -REGDEF(V22, 22+VBASE, VMASK(22), "d22", "s22") -REGDEF(V23, 23+VBASE, VMASK(23), "d23", "s23") -REGDEF(V24, 24+VBASE, VMASK(24), "d24", "s24") -REGDEF(V25, 25+VBASE, VMASK(25), "d25", "s25") -REGDEF(V26, 26+VBASE, VMASK(26), "d26", "s26") -REGDEF(V27, 27+VBASE, VMASK(27), "d27", "s27") -REGDEF(V28, 28+VBASE, VMASK(28), "d28", "s28") -REGDEF(V29, 29+VBASE, VMASK(29), "d29", "s29") -REGDEF(V30, 30+VBASE, VMASK(30), "d30", "s30") -REGDEF(V31, 31+VBASE, VMASK(31), "d31", "s31") +REGDEF(name, rnum, mask, xname, wname, regTypeTag) */ +REGDEF(V0, 0+VBASE, VMASK(0), "d0", "s0", 1) +REGDEF(V1, 1+VBASE, VMASK(1), "d1", "s1", 1) +REGDEF(V2, 2+VBASE, VMASK(2), "d2", "s2", 1) +REGDEF(V3, 3+VBASE, VMASK(3), "d3", "s3", 1) +REGDEF(V4, 4+VBASE, VMASK(4), "d4", "s4", 1) +REGDEF(V5, 5+VBASE, VMASK(5), "d5", "s5", 1) +REGDEF(V6, 6+VBASE, VMASK(6), "d6", "s6", 1) +REGDEF(V7, 7+VBASE, VMASK(7), "d7", "s7", 1) +REGDEF(V8, 8+VBASE, VMASK(8), "d8", "s8", 1) +REGDEF(V9, 9+VBASE, VMASK(9), "d9", "s9", 1) +REGDEF(V10, 10+VBASE, VMASK(10), "d10", "s10", 1) +REGDEF(V11, 11+VBASE, VMASK(11), "d11", "s11", 1) +REGDEF(V12, 12+VBASE, VMASK(12), "d12", "s12", 1) +REGDEF(V13, 13+VBASE, VMASK(13), "d13", "s13", 1) +REGDEF(V14, 14+VBASE, VMASK(14), "d14", "s14", 1) +REGDEF(V15, 15+VBASE, VMASK(15), "d15", "s15", 1) +REGDEF(V16, 16+VBASE, VMASK(16), "d16", "s16", 1) +REGDEF(V17, 17+VBASE, VMASK(17), "d17", "s17", 1) +REGDEF(V18, 18+VBASE, VMASK(18), "d18", "s18", 1) +REGDEF(V19, 19+VBASE, VMASK(19), "d19", "s19", 1) +REGDEF(V20, 20+VBASE, VMASK(20), "d20", "s20", 1) +REGDEF(V21, 21+VBASE, VMASK(21), "d21", "s21", 1) +REGDEF(V22, 22+VBASE, VMASK(22), "d22", "s22", 1) +REGDEF(V23, 23+VBASE, VMASK(23), "d23", "s23", 1) +REGDEF(V24, 24+VBASE, VMASK(24), "d24", "s24", 1) +REGDEF(V25, 25+VBASE, VMASK(25), "d25", "s25", 1) +REGDEF(V26, 26+VBASE, VMASK(26), "d26", "s26", 1) +REGDEF(V27, 27+VBASE, VMASK(27), "d27", "s27", 1) +REGDEF(V28, 28+VBASE, VMASK(28), "d28", "s28", 1) +REGDEF(V29, 29+VBASE, VMASK(29), "d29", "s29", 1) +REGDEF(V30, 30+VBASE, VMASK(30), "d30", "s30", 1) +REGDEF(V31, 31+VBASE, VMASK(31), "d31", "s31", 1) // TODO-SVE: Fix once we add predicate registers REGALIAS(P0, V0) @@ -117,9 +117,9 @@ REGALIAS(P15, V15) // The registers with values 64 (NBASE) and above are not real register numbers #define NBASE 64 -REGDEF(SP, 0+NBASE, 0x0000, "sp", "wsp?") +REGDEF(SP, 0+NBASE, 0x0000, "sp", "wsp?", -1) // This must be last! -REGDEF(STK, 1+NBASE, 0x0000, "STK", "STK") +REGDEF(STK, 1+NBASE, 0x0000, "STK", "STK", -1) /*****************************************************************************/ #undef RMASK diff --git a/src/coreclr/jit/registerloongarch64.h b/src/coreclr/jit/registerloongarch64.h index 8f3cd157016bb2..2c063b8a76985f 100644 --- a/src/coreclr/jit/registerloongarch64.h +++ b/src/coreclr/jit/registerloongarch64.h @@ -15,39 +15,39 @@ #define RMASK(x) (1ULL << (x)) /* -REGDEF(name, rnum, mask, sname) */ -REGDEF(R0, 0, 0x0001, "zero") -REGDEF(RA, 1, 0x0002, "ra" ) -REGDEF(TP, 2, 0x0004, "tp" ) -REGDEF(SP, 3, 0x0008, "sp" ) -REGDEF(A0, 4, 0x0010, "a0" ) -REGDEF(A1, 5, 0x0020, "a1" ) -REGDEF(A2, 6, 0x0040, "a2" ) -REGDEF(A3, 7, 0x0080, "a3" ) -REGDEF(A4, 8, 0x0100, "a4" ) -REGDEF(A5, 9, 0x0200, "a5" ) -REGDEF(A6, 10, 0x0400, "a6" ) -REGDEF(A7, 11, 0x0800, "a7" ) -REGDEF(T0, 12, 0x1000, "t0" ) -REGDEF(T1, 13, 0x2000, "t1" ) -REGDEF(T2, 14, 0x4000, "t2" ) -REGDEF(T3, 15, 0x8000, "t3" ) -REGDEF(T4, 16, 0x10000, "t4" ) -REGDEF(T5, 17, 0x20000, "t5" ) -REGDEF(T6, 18, 0x40000, "t6" ) -REGDEF(T7, 19, 0x80000, "t7" ) -REGDEF(T8, 20, 0x100000, "t8" ) -REGDEF(X0, 21, 0x200000, "x0" ) -REGDEF(FP, 22, 0x400000, "fp" ) -REGDEF(S0, 23, 0x800000, "s0" ) -REGDEF(S1, 24, 0x1000000, "s1" ) -REGDEF(S2, 25, 0x2000000, "s2" ) -REGDEF(S3, 26, 0x4000000, "s3" ) -REGDEF(S4, 27, 0x8000000, "s4" ) -REGDEF(S5, 28, 0x10000000, "s5" ) -REGDEF(S6, 29, 0x20000000, "s6" ) -REGDEF(S7, 30, 0x40000000, "s7" ) -REGDEF(S8, 31, 0x80000000, "s8" ) +REGDEF(name, rnum, mask, sname, regTypeTag) */ +REGDEF(R0, 0, 0x0001, "zero", 0) +REGDEF(RA, 1, 0x0002, "ra" , 0) +REGDEF(TP, 2, 0x0004, "tp" , 0) +REGDEF(SP, 3, 0x0008, "sp" , 0) +REGDEF(A0, 4, 0x0010, "a0" , 0) +REGDEF(A1, 5, 0x0020, "a1" , 0) +REGDEF(A2, 6, 0x0040, "a2" , 0) +REGDEF(A3, 7, 0x0080, "a3" , 0) +REGDEF(A4, 8, 0x0100, "a4" , 0) +REGDEF(A5, 9, 0x0200, "a5" , 0) +REGDEF(A6, 10, 0x0400, "a6" , 0) +REGDEF(A7, 11, 0x0800, "a7" , 0) +REGDEF(T0, 12, 0x1000, "t0" , 0) +REGDEF(T1, 13, 0x2000, "t1" , 0) +REGDEF(T2, 14, 0x4000, "t2" , 0) +REGDEF(T3, 15, 0x8000, "t3" , 0) +REGDEF(T4, 16, 0x10000, "t4" , 0) +REGDEF(T5, 17, 0x20000, "t5" , 0) +REGDEF(T6, 18, 0x40000, "t6" , 0) +REGDEF(T7, 19, 0x80000, "t7" , 0) +REGDEF(T8, 20, 0x100000, "t8" , 0) +REGDEF(X0, 21, 0x200000, "x0" , 0) +REGDEF(FP, 22, 0x400000, "fp" , 0) +REGDEF(S0, 23, 0x800000, "s0" , 0) +REGDEF(S1, 24, 0x1000000, "s1" , 0) +REGDEF(S2, 25, 0x2000000, "s2" , 0) +REGDEF(S3, 26, 0x4000000, "s3" , 0) +REGDEF(S4, 27, 0x8000000, "s4" , 0) +REGDEF(S5, 28, 0x10000000, "s5" , 0) +REGDEF(S6, 29, 0x20000000, "s6" , 0) +REGDEF(S7, 30, 0x40000000, "s7" , 0) +REGDEF(S8, 31, 0x80000000, "s8" , 0) //NOTE for LoongArch64: // The `REG_R21` which alias `REG_X0` is specially reserved !!! @@ -64,44 +64,44 @@ REGALIAS(R21, X0) /* REGDEF(name, rnum, mask, sname) */ -REGDEF(F0, 0+FBASE, FMASK(0), "f0") -REGDEF(F1, 1+FBASE, FMASK(1), "f1") -REGDEF(F2, 2+FBASE, FMASK(2), "f2") -REGDEF(F3, 3+FBASE, FMASK(3), "f3") -REGDEF(F4, 4+FBASE, FMASK(4), "f4") -REGDEF(F5, 5+FBASE, FMASK(5), "f5") -REGDEF(F6, 6+FBASE, FMASK(6), "f6") -REGDEF(F7, 7+FBASE, FMASK(7), "f7") -REGDEF(F8, 8+FBASE, FMASK(8), "f8") -REGDEF(F9, 9+FBASE, FMASK(9), "f9") -REGDEF(F10, 10+FBASE, FMASK(10), "f10") -REGDEF(F11, 11+FBASE, FMASK(11), "f11") -REGDEF(F12, 12+FBASE, FMASK(12), "f12") -REGDEF(F13, 13+FBASE, FMASK(13), "f13") -REGDEF(F14, 14+FBASE, FMASK(14), "f14") -REGDEF(F15, 15+FBASE, FMASK(15), "f15") -REGDEF(F16, 16+FBASE, FMASK(16), "f16") -REGDEF(F17, 17+FBASE, FMASK(17), "f17") -REGDEF(F18, 18+FBASE, FMASK(18), "f18") -REGDEF(F19, 19+FBASE, FMASK(19), "f19") -REGDEF(F20, 20+FBASE, FMASK(20), "f20") -REGDEF(F21, 21+FBASE, FMASK(21), "f21") -REGDEF(F22, 22+FBASE, FMASK(22), "f22") -REGDEF(F23, 23+FBASE, FMASK(23), "f23") -REGDEF(F24, 24+FBASE, FMASK(24), "f24") -REGDEF(F25, 25+FBASE, FMASK(25), "f25") -REGDEF(F26, 26+FBASE, FMASK(26), "f26") -REGDEF(F27, 27+FBASE, FMASK(27), "f27") -REGDEF(F28, 28+FBASE, FMASK(28), "f28") -REGDEF(F29, 29+FBASE, FMASK(29), "f29") -REGDEF(F30, 30+FBASE, FMASK(30), "f30") -REGDEF(F31, 31+FBASE, FMASK(31), "f31") +REGDEF(F0, 0+FBASE, FMASK(0), "f0", 1) +REGDEF(F1, 1+FBASE, FMASK(1), "f1", 1) +REGDEF(F2, 2+FBASE, FMASK(2), "f2", 1) +REGDEF(F3, 3+FBASE, FMASK(3), "f3", 1) +REGDEF(F4, 4+FBASE, FMASK(4), "f4", 1) +REGDEF(F5, 5+FBASE, FMASK(5), "f5", 1) +REGDEF(F6, 6+FBASE, FMASK(6), "f6", 1) +REGDEF(F7, 7+FBASE, FMASK(7), "f7", 1) +REGDEF(F8, 8+FBASE, FMASK(8), "f8", 1) +REGDEF(F9, 9+FBASE, FMASK(9), "f9", 1) +REGDEF(F10, 10+FBASE, FMASK(10), "f10", 1) +REGDEF(F11, 11+FBASE, FMASK(11), "f11", 1) +REGDEF(F12, 12+FBASE, FMASK(12), "f12", 1) +REGDEF(F13, 13+FBASE, FMASK(13), "f13", 1) +REGDEF(F14, 14+FBASE, FMASK(14), "f14", 1) +REGDEF(F15, 15+FBASE, FMASK(15), "f15", 1) +REGDEF(F16, 16+FBASE, FMASK(16), "f16", 1) +REGDEF(F17, 17+FBASE, FMASK(17), "f17", 1) +REGDEF(F18, 18+FBASE, FMASK(18), "f18", 1) +REGDEF(F19, 19+FBASE, FMASK(19), "f19", 1) +REGDEF(F20, 20+FBASE, FMASK(20), "f20", 1) +REGDEF(F21, 21+FBASE, FMASK(21), "f21", 1) +REGDEF(F22, 22+FBASE, FMASK(22), "f22", 1) +REGDEF(F23, 23+FBASE, FMASK(23), "f23", 1) +REGDEF(F24, 24+FBASE, FMASK(24), "f24", 1) +REGDEF(F25, 25+FBASE, FMASK(25), "f25", 1) +REGDEF(F26, 26+FBASE, FMASK(26), "f26", 1) +REGDEF(F27, 27+FBASE, FMASK(27), "f27", 1) +REGDEF(F28, 28+FBASE, FMASK(28), "f28", 1) +REGDEF(F29, 29+FBASE, FMASK(29), "f29", 1) +REGDEF(F30, 30+FBASE, FMASK(30), "f30", 1) +REGDEF(F31, 31+FBASE, FMASK(31), "f31", 1) // The registers with values 64 (NBASE) and above are not real register numbers #define NBASE 64 // This must be last! -REGDEF(STK, 0+NBASE, 0x0000, "STK") +REGDEF(STK, 0+NBASE, 0x0000, "STK", -1) /*****************************************************************************/ #undef RMASK diff --git a/src/coreclr/jit/registerriscv64.h b/src/coreclr/jit/registerriscv64.h index fe6d3cf8ece424..96c4a7505424b8 100644 --- a/src/coreclr/jit/registerriscv64.h +++ b/src/coreclr/jit/registerriscv64.h @@ -15,39 +15,39 @@ #define RMASK(x) (1ULL << (x)) /* -REGDEF(name, rnum, mask, sname) */ -REGDEF(R0, 0, 0x0001, "zero") -REGDEF(RA, 1, 0x0002, "ra" ) -REGDEF(SP, 2, 0x0004, "sp" ) -REGDEF(GP, 3, 0x0008, "gp" ) -REGDEF(TP, 4, 0x0010, "tp" ) -REGDEF(T0, 5, 0x0020, "t0" ) -REGDEF(T1, 6, 0x0040, "t1" ) -REGDEF(T2, 7, 0x0080, "t2" ) -REGDEF(FP, 8, 0x0100, "fp" ) -REGDEF(S1, 9, 0x0200, "s1" ) -REGDEF(A0, 10, 0x0400, "a0" ) -REGDEF(A1, 11, 0x0800, "a1" ) -REGDEF(A2, 12, 0x1000, "a2" ) -REGDEF(A3, 13, 0x2000, "a3" ) -REGDEF(A4, 14, 0x4000, "a4" ) -REGDEF(A5, 15, 0x8000, "a5" ) -REGDEF(A6, 16, 0x10000, "a6" ) -REGDEF(A7, 17, 0x20000, "a7" ) -REGDEF(S2, 18, 0x40000, "s2" ) -REGDEF(S3, 19, 0x80000, "s3" ) -REGDEF(S4, 20, 0x100000, "s4" ) -REGDEF(S5, 21, 0x200000, "s5" ) -REGDEF(S6, 22, 0x400000, "s6" ) -REGDEF(S7, 23, 0x800000, "s7" ) -REGDEF(S8, 24, 0x1000000, "s8" ) -REGDEF(S9, 25, 0x2000000, "s9" ) -REGDEF(S10, 26, 0x4000000, "s10" ) -REGDEF(S11, 27, 0x8000000, "s11" ) -REGDEF(T3, 28, 0x10000000, "t3" ) -REGDEF(T4, 29, 0x20000000, "t4" ) -REGDEF(T5, 30, 0x40000000, "t5" ) -REGDEF(T6, 31, 0x80000000, "t6" ) +REGDEF(name, rnum, mask, sname, regTypeTag) */ +REGDEF(R0, 0, 0x0001, "zero", 0) +REGDEF(RA, 1, 0x0002, "ra" , 0) +REGDEF(SP, 2, 0x0004, "sp" , 0) +REGDEF(GP, 3, 0x0008, "gp" , 0) +REGDEF(TP, 4, 0x0010, "tp" , 0) +REGDEF(T0, 5, 0x0020, "t0" , 0) +REGDEF(T1, 6, 0x0040, "t1" , 0) +REGDEF(T2, 7, 0x0080, "t2", 0) +REGDEF(FP, 8, 0x0100, "fp", 0) +REGDEF(S1, 9, 0x0200, "s1", 0) +REGDEF(A0, 10, 0x0400, "a0", 0) +REGDEF(A1, 11, 0x0800, "a1", 0) +REGDEF(A2, 12, 0x1000, "a2", 0) +REGDEF(A3, 13, 0x2000, "a3", 0) +REGDEF(A4, 14, 0x4000, "a4", 0) +REGDEF(A5, 15, 0x8000, "a5", 0) +REGDEF(A6, 16, 0x10000, "a6", 0) +REGDEF(A7, 17, 0x20000, "a7", 0) +REGDEF(S2, 18, 0x40000, "s2", 0) +REGDEF(S3, 19, 0x80000, "s3", 0) +REGDEF(S4, 20, 0x100000, "s4", 0) +REGDEF(S5, 21, 0x200000, "s5", 0) +REGDEF(S6, 22, 0x400000, "s6", 0) +REGDEF(S7, 23, 0x800000, "s7", 0) +REGDEF(S8, 24, 0x1000000, "s8", 0) +REGDEF(S9, 25, 0x2000000, "s9", 0) +REGDEF(S10, 26, 0x4000000, "s10", 0) +REGDEF(S11, 27, 0x8000000, "s11", 0) +REGDEF(T3, 28, 0x10000000, "t3", 0) +REGDEF(T4, 29, 0x20000000, "t4", 0) +REGDEF(T5, 30, 0x40000000, "t5", 0) +REGDEF(T6, 31, 0x80000000, "t6", 0) REGALIAS(R8, FP) REGALIAS(ZERO, R0) @@ -57,43 +57,43 @@ REGALIAS(ZERO, R0) /* REGDEF(name, rnum, mask, sname) */ -REGDEF(F0, 0+FBASE, FMASK(0), "f0") -REGDEF(F1, 1+FBASE, FMASK(1), "f1") -REGDEF(F2, 2+FBASE, FMASK(2), "f2") -REGDEF(F3, 3+FBASE, FMASK(3), "f3") -REGDEF(F4, 4+FBASE, FMASK(4), "f4") -REGDEF(F5, 5+FBASE, FMASK(5), "f5") -REGDEF(F6, 6+FBASE, FMASK(6), "f6") -REGDEF(F7, 7+FBASE, FMASK(7), "f7") -REGDEF(F8, 8+FBASE, FMASK(8), "f8") -REGDEF(F9, 9+FBASE, FMASK(9), "f9") -REGDEF(F10, 10+FBASE, FMASK(10), "f10") -REGDEF(F11, 11+FBASE, FMASK(11), "f11") -REGDEF(F12, 12+FBASE, FMASK(12), "f12") -REGDEF(F13, 13+FBASE, FMASK(13), "f13") -REGDEF(F14, 14+FBASE, FMASK(14), "f14") -REGDEF(F15, 15+FBASE, FMASK(15), "f15") -REGDEF(F16, 16+FBASE, FMASK(16), "f16") -REGDEF(F17, 17+FBASE, FMASK(17), "f17") -REGDEF(F18, 18+FBASE, FMASK(18), "f18") -REGDEF(F19, 19+FBASE, FMASK(19), "f19") -REGDEF(F20, 20+FBASE, FMASK(20), "f20") -REGDEF(F21, 21+FBASE, FMASK(21), "f21") -REGDEF(F22, 22+FBASE, FMASK(22), "f22") -REGDEF(F23, 23+FBASE, FMASK(23), "f23") -REGDEF(F24, 24+FBASE, FMASK(24), "f24") -REGDEF(F25, 25+FBASE, FMASK(25), "f25") -REGDEF(F26, 26+FBASE, FMASK(26), "f26") -REGDEF(F27, 27+FBASE, FMASK(27), "f27") -REGDEF(F28, 28+FBASE, FMASK(28), "f28") -REGDEF(F29, 29+FBASE, FMASK(29), "f29") -REGDEF(F30, 30+FBASE, FMASK(30), "f30") -REGDEF(F31, 31+FBASE, FMASK(31), "f31") +REGDEF(F0, 0+FBASE, FMASK(0), "f0", 1) +REGDEF(F1, 1+FBASE, FMASK(1), "f1", 1) +REGDEF(F2, 2+FBASE, FMASK(2), "f2", 1) +REGDEF(F3, 3+FBASE, FMASK(3), "f3", 1) +REGDEF(F4, 4+FBASE, FMASK(4), "f4", 1) +REGDEF(F5, 5+FBASE, FMASK(5), "f5", 1) +REGDEF(F6, 6+FBASE, FMASK(6), "f6", 1) +REGDEF(F7, 7+FBASE, FMASK(7), "f7", 1) +REGDEF(F8, 8+FBASE, FMASK(8), "f8", 1) +REGDEF(F9, 9+FBASE, FMASK(9), "f9", 1) +REGDEF(F10, 10+FBASE, FMASK(10), "f10", 1) +REGDEF(F11, 11+FBASE, FMASK(11), "f11", 1) +REGDEF(F12, 12+FBASE, FMASK(12), "f12", 1) +REGDEF(F13, 13+FBASE, FMASK(13), "f13", 1) +REGDEF(F14, 14+FBASE, FMASK(14), "f14", 1) +REGDEF(F15, 15+FBASE, FMASK(15), "f15", 1) +REGDEF(F16, 16+FBASE, FMASK(16), "f16", 1) +REGDEF(F17, 17+FBASE, FMASK(17), "f17", 1) +REGDEF(F18, 18+FBASE, FMASK(18), "f18", 1) +REGDEF(F19, 19+FBASE, FMASK(19), "f19", 1) +REGDEF(F20, 20+FBASE, FMASK(20), "f20", 1) +REGDEF(F21, 21+FBASE, FMASK(21), "f21", 1) +REGDEF(F22, 22+FBASE, FMASK(22), "f22", 1) +REGDEF(F23, 23+FBASE, FMASK(23), "f23", 1) +REGDEF(F24, 24+FBASE, FMASK(24), "f24", 1) +REGDEF(F25, 25+FBASE, FMASK(25), "f25", 1) +REGDEF(F26, 26+FBASE, FMASK(26), "f26", 1) +REGDEF(F27, 27+FBASE, FMASK(27), "f27", 1) +REGDEF(F28, 28+FBASE, FMASK(28), "f28", 1) +REGDEF(F29, 29+FBASE, FMASK(29), "f29", 1) +REGDEF(F30, 30+FBASE, FMASK(30), "f30", 1) +REGDEF(F31, 31+FBASE, FMASK(31), "f31", 1) // The registers with values 64 (NBASE) and above are not real register numbers #define NBASE 64 -REGDEF(STK, 0+NBASE, 0x0000, "STK") +REGDEF(STK, 0+NBASE, 0x0000, "STK", -1) /*****************************************************************************/ #undef RMASK diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 4c82a7982c299f..c8f30638735b8d 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -25,12 +25,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #if defined(TARGET_ARM64) const regMaskSmall regMasks[] = { -#define REGDEF(name, rnum, mask, xname, wname) mask, +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) mask, #include "register.h" }; #else // !TARGET_ARM64 const regMaskSmall regMasks[] = { -#define REGDEF(name, rnum, mask, sname) mask, +#define REGDEF(name, rnum, mask, sname, regTypeTag) mask, #include "register.h" }; #endif @@ -221,21 +221,12 @@ void RegSet::verifyRegistersUsed(AllRegsMask regs) // See https://github.com/dotnet/runtime/issues/10411 and // https://github.com/dotnet/coreclr/pull/18230 on why we call // rsSetGprRegsModified() instead of assert(rsRegsModified()) - if (regs.gprRegs != RBM_NONE) - { - rsSetGprRegsModified(regs.gprRegs); - } - if (regs.floatRegs != RBM_NONE) - { - rsSetFloatRegsModified(regs.floatRegs); - } + rsSetGprRegsModified(regs.gprRegs()); + rsSetFloatRegsModified(regs.floatRegs()); #ifdef HAS_PREDICATE_REGS - if (regs.predicateRegs != RBM_NONE) - { - rsSetPredicateRegsModified(regs.predicateRegs); - } + rsSetPredicateRegsModified(regs.predicateRegs()); #endif } @@ -280,7 +271,9 @@ void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) DEBUGARG(regMaskOnlyOne calleeSaveMask)) { - assert(modifiedMask != RBM_NONE); + //TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask + //assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index a2590d3b04ce85..ae6e7ae25bc39e 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -145,26 +145,12 @@ class RegSet public: regMaskOnlyOne GetMaskVars(var_types type) const // 'get' property function for rsMaskVars property { - if (varTypeUsesIntReg(type)) - { - return _rsAllMaskVars.gprRegs; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - return _rsAllMaskVars.predicateRegs; - } -#endif - else - { - assert(varTypeUsesFloatReg(type)); - return _rsAllMaskVars.floatRegs; - } + return _rsAllMaskVars.GetRegTypeMask(type); } regMaskGpr GetGprMaskVars() const // 'get' property function for rsMaskVars property { - return _rsAllMaskVars.gprRegs; + return _rsAllMaskVars.gprRegs(); } void SetMaskVars(AllRegsMask newMaskVars); // 'put' property function for rsMaskVars property @@ -172,43 +158,17 @@ class RegSet void AddMaskVars(var_types type, regMaskOnlyOne addMaskVars) // union 'addMaskVars' with the rsMaskVars set { AllRegsMask newMask = _rsAllMaskVars; - if (varTypeUsesIntReg(type)) - { - newMask.gprRegs |= addMaskVars; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - newMask.predicateRegs |= addMaskVars; - } -#endif - else - { - assert(varTypeUsesFloatReg(type)); - newMask.floatRegs |= addMaskVars; - } + newMask.AddRegTypeMask(addMaskVars, type); SetMaskVars(newMask); } // remove 'removeMaskVars' from the rsMaskVars set (like bitset DiffD) void RemoveMaskVars(var_types type, regMaskOnlyOne removeMaskVars) { + // TODO: Skip assigning to newMask, just update _rsAllMaskVars directly. The only thing remaining + // would be to print the change if (newMask != _rsAllMaskVars). AllRegsMask newMask = _rsAllMaskVars; - if (varTypeUsesIntReg(type)) - { - newMask.gprRegs &= ~removeMaskVars; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - newMask.predicateRegs &= ~removeMaskVars; - } -#endif - else - { - assert(varTypeUsesFloatReg(type)); - newMask.floatRegs &= ~removeMaskVars; - } + newMask.RemoveRegTypeFromMask(removeMaskVars, type); SetMaskVars(newMask); } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6ec02aac026de6..20a28bf17765e6 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -109,7 +109,7 @@ inline bool compUnixX86Abi() #if defined(TARGET_ARM) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) enum _regNumber_enum : unsigned { -#define REGDEF(name, rnum, mask, sname) REG_##name = rnum, +#define REGDEF(name, rnum, mask, sname, regTypeTag) REG_##name = rnum, #define REGALIAS(alias, realname) REG_##alias = REG_##realname, #include "register.h" @@ -121,7 +121,7 @@ enum _regNumber_enum : unsigned enum _regMask_enum : unsigned __int64 { RBM_NONE = 0, -#define REGDEF(name, rnum, mask, sname) RBM_##name = mask, +#define REGDEF(name, rnum, mask, sname, regTypeTag) RBM_##name = mask, #define REGALIAS(alias, realname) RBM_##alias = RBM_##realname, #include "register.h" }; @@ -130,7 +130,7 @@ enum _regMask_enum : unsigned __int64 enum _regNumber_enum : unsigned { -#define REGDEF(name, rnum, mask, xname, wname) REG_##name = rnum, +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) REG_##name = rnum, #define REGALIAS(alias, realname) REG_##alias = REG_##realname, #include "register.h" @@ -142,7 +142,7 @@ enum _regNumber_enum : unsigned enum _regMask_enum : unsigned __int64 { RBM_NONE = 0, -#define REGDEF(name, rnum, mask, xname, wname) RBM_##name = mask, +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) RBM_##name = mask, #define REGALIAS(alias, realname) RBM_##alias = RBM_##realname, #include "register.h" }; @@ -151,7 +151,7 @@ enum _regMask_enum : unsigned __int64 enum _regNumber_enum : unsigned { -#define REGDEF(name, rnum, mask, sname) REG_##name = rnum, +#define REGDEF(name, rnum, mask, sname, regTypeTag) REG_##name = rnum, #define REGALIAS(alias, realname) REG_##alias = REG_##realname, #include "register.h" @@ -164,7 +164,7 @@ enum _regMask_enum : uint64_t { RBM_NONE = 0, -#define REGDEF(name, rnum, mask, sname) RBM_##name = mask, +#define REGDEF(name, rnum, mask, sname, regTypeTag) RBM_##name = mask, #define REGALIAS(alias, realname) RBM_##alias = RBM_##realname, #include "register.h" @@ -174,7 +174,7 @@ enum _regMask_enum : uint64_t enum _regNumber_enum : unsigned { -#define REGDEF(name, rnum, mask, sname) REG_##name = rnum, +#define REGDEF(name, rnum, mask, sname, regTypeTag) REG_##name = rnum, #define REGALIAS(alias, realname) REG_##alias = REG_##realname, #include "register.h" @@ -187,7 +187,7 @@ enum _regMask_enum : unsigned { RBM_NONE = 0, -#define REGDEF(name, rnum, mask, sname) RBM_##name = mask, +#define REGDEF(name, rnum, mask, sname, regTypeTag) RBM_##name = mask, #define REGALIAS(alias, realname) RBM_##alias = RBM_##realname, #include "register.h" }; @@ -269,161 +269,103 @@ typedef unsigned char regNumberSmall; typedef struct _regMaskAll { - regMaskGpr gprRegs; - regMaskFloat floatRegs; -#ifdef HAS_PREDICATE_REGS - regMaskPredicate predicateRegs; -#endif // HAS_PREDICATE_REGS - // - // _regMaskAll(regMaskGpr _gprRegMask) - // : gprRegs(_gprRegMask) - // , floatRegs(RBM_NONE) - // #ifdef HAS_PREDICATE_REGS - // , predicateRegs(RBM_NONE) - // #endif - // { - // } - // - // _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask) : gprRegs(_gprRegMask), - // floatRegs(_floatRegMask) - // #ifdef HAS_PREDICATE_REGS - // , predicateRegs(RBM_NONE) - // #endif - // { - // - // } +private: + regMaskTP registers[REGISTER_TYPE_COUNT]; - _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask, regMaskPredicate _predicateRegMask = RBM_NONE) +public: + inline regMaskGpr gprRegs() const + { + return registers[0]; + } + inline regMaskFloat floatRegs() const { - gprRegs = _gprRegMask; - floatRegs = _floatRegMask; + return registers[1]; + } #ifdef HAS_PREDICATE_REGS - predicateRegs = _predicateRegMask; -#endif // HAS_PREDICATE_REGS + inline regMaskPredicate predicateRegs() const + { + static_assert((REGISTER_TYPE_COUNT == 3), "There should be 3 types of registers"); + return registers[2]; } +#endif - // // Useful to create an entity from `mask` when the caller - // // doesn't know what is the type of mask but knows that it - // // is of "onlyone" type. - // _regMaskAll(regMaskOnlyOne mask) - // { - // gprRegs = mask; - // floatRegs = mask; - // #ifdef HAS_PREDICATE_REGS - // predicateRegs = mask; - // #endif - // } - - _regMaskAll() - { - gprRegs = RBM_NONE; - floatRegs = RBM_NONE; + _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask, regMaskPredicate _predicateRegMask = RBM_NONE) + : registers{_gprRegMask, _floatRegMask #ifdef HAS_PREDICATE_REGS - predicateRegs = RBM_NONE; + , + _predicateRegMask #endif + } + { } - void Clear() - { - gprRegs = RBM_NONE; - floatRegs = RBM_NONE; + + + _regMaskAll() + : registers{RBM_NONE, RBM_NONE #ifdef HAS_PREDICATE_REGS - predicateRegs = RBM_NONE; + , RBM_NONE #endif + } + { } - bool IsEmpty() + _regMaskAll(int (&_registers)[3]) { - return (gprRegs == RBM_NONE) && (floatRegs == RBM_NONE) -#ifdef HAS_PREDICATE_REGS - && (predicateRegs == RBM_NONE) -#endif // HAS_PREDICATE_REGS - ; + registers[0] = _registers[0]; + registers[1] = _registers[1]; + registers[2] = _registers[2]; } - unsigned Count() + _regMaskAll(regNumber reg) : _regMaskAll() { - return genCountBits(gprRegs) + genCountBits(floatRegs) -#ifdef HAS_PREDICATE_REGS - + genCountBits(predicateRegs) -#endif // HAS_PREDICATE_REGS - ; + *this |= reg; } + void Clear(); + bool IsEmpty(); + unsigned Count(); + void Create(regNumber reg); // Rename this to AddRegNum + void AddGprRegInMask(regNumber reg); + void AddRegNumInMask(regNumber reg); void AddRegNumInMask(regNumber reg, var_types type); - void RemoveRegNumInMask(regNumber reg); - void RemoveRegNumInMask(regNumber reg, var_types type); + // Rename this to AddRegMask() + void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type); + void AddGprRegMask(regMaskGpr maskToAdd); + void AddFloatRegMask(regMaskFloat maskToAdd); + void RemoveRegNumFromMask(regNumber reg); + void RemoveRegNumFromMask(regNumber reg, var_types type); + void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); bool IsRegNumInMask(regNumber reg, var_types type); bool IsRegNumInMask(regNumber reg); + bool IsGprMaskPresent(regMaskGpr maskToCheck) const; + bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; bool IsOnlyRegNumInMask(regNumber reg); + regMaskOnlyOne GetRegTypeMask(var_types type) const; + regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; + regMaskOnlyOne OrMask(const _regMaskAll& second, var_types regType) const; + + // TODO: this might go away once we have just `regMaskTP` gpr_float field + bool IsGprOrFloatPresent() const; + regMaskTP GetGprFloatCombinedMask() const; + + regMaskOnlyOne operator[](int index) const; + regMaskOnlyOne& operator[](int index); + void operator|=(const _regMaskAll& other); + void operator&=(const _regMaskAll& other); + void operator|=(const regNumber reg); + void operator^=(const regNumber reg); + _regMaskAll operator~(); + bool operator==(const _regMaskAll& other); + bool operator!=(const _regMaskAll& other); + _regMaskAll operator&(const _regMaskAll& other); + _regMaskAll operator|(const _regMaskAll& other); + _regMaskAll operator&(const regNumber reg); - // Rename this to AddRegMask() - void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) - { - if (varTypeUsesIntReg(type)) - { - gprRegs |= maskToAdd; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - predicateRegs |= maskToAdd; - } -#endif - else - { - assert(varTypeUsesFloatReg(type)); - floatRegs |= maskToAdd; - } - } - - regMaskOnlyOne GetRegTypeMask(var_types type) - { - if (varTypeUsesIntReg(type)) - { - return gprRegs; - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - return predicateRegs; - } -#endif - else - { - assert(varTypeUsesFloatReg(type)); - return floatRegs; - } - } - - regMaskOnlyOne& operator[](int index) - { - if (index == 0) - { - return gprRegs; - } - else if (index == 1) - { - return floatRegs; - } - else - { -#ifdef HAS_PREDICATE_REGS - return predicateRegs; -#else - unreached(); -#endif - } - } - - // void AllRegsMask& operator|=(const AllRegsMask& second); - //{ - // gprRegs - // return *this; - //} } AllRegsMask; + #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) #define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs)3 @@ -439,18 +381,8 @@ typedef unsigned __int64 regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif -bool operator==(const AllRegsMask& first, const AllRegsMask& second); -bool operator!=(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator&(const AllRegsMask& first, const AllRegsMask& second); -regMaskOnlyOne operator&(const AllRegsMask& first, const regNumber reg); -AllRegsMask operator|(const AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator|=(AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator&=(AllRegsMask& first, const AllRegsMask& second); -AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); -AllRegsMask operator^=(AllRegsMask& first, const regNumber reg); -// AllRegsMask operator|=(AllRegsMask& first, const regNumber reg); -AllRegsMask operator~(const AllRegsMask& first); -// inline AllRegsMask createRegMask(regNumber reg) + + /*****************************************************************************/ diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index b0a0939efdae06..e70f59015c83f0 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -111,6 +111,20 @@ extern const BYTE opcodeArgKinds[] = { /*****************************************************************************/ +const int regIndexForRegister(regNumber reg) +{ + static const BYTE _registerTypeIndex[] = { +#ifdef TARGET_ARM64 +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) regTypeTag, +#else +#define REGDEF(name, rnum, mask, sname, regTypeTag) regTypeTag, +#endif + #include "register.h" + }; + + return _registerTypeIndex[reg]; +} + const char* varTypeName(var_types vt) { static const char* const varTypeNames[] = { @@ -139,9 +153,9 @@ const char* getRegName(regNumber reg) static const char* const regNames[] = { #if defined(TARGET_ARM64) -#define REGDEF(name, rnum, mask, xname, wname) xname, +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) xname, #else -#define REGDEF(name, rnum, mask, sname) sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) sname, #endif #include "register.h" }; @@ -227,7 +241,7 @@ const char* getRegNameFloat(regNumber reg, var_types type) #elif defined(TARGET_ARM64) static const char* regNamesFloat[] = { -#define REGDEF(name, rnum, mask, xname, wname) xname, +#define REGDEF(name, rnum, mask, xname, wname, regTypeTag) xname, #include "register.h" }; assert((unsigned)reg < ArrLen(regNamesFloat)); @@ -237,7 +251,7 @@ const char* getRegNameFloat(regNumber reg, var_types type) #elif defined(TARGET_LOONGARCH64) static const char* regNamesFloat[] = { -#define REGDEF(name, rnum, mask, sname) sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) sname, #include "register.h" }; @@ -247,16 +261,16 @@ const char* getRegNameFloat(regNumber reg, var_types type) #else static const char* regNamesFloat[] = { -#define REGDEF(name, rnum, mask, sname) "x" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "x" sname, #include "register.h" }; #ifdef FEATURE_SIMD static const char* regNamesYMM[] = { -#define REGDEF(name, rnum, mask, sname) "y" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "y" sname, #include "register.h" }; static const char* regNamesZMM[] = { -#define REGDEF(name, rnum, mask, sname) "z" sname, +#define REGDEF(name, rnum, mask, sname, regTypeTag) "z" sname, #include "register.h" }; #endif // FEATURE_SIMD @@ -455,11 +469,11 @@ void dspRegMask(AllRegsMask mask, size_t minSiz) printf("["); - sep = dspRegRange(mask.gprRegs, minSiz, sep, REG_INT_FIRST, REG_INT_LAST); - sep = dspRegRange(mask.floatRegs, minSiz, sep, REG_FP_FIRST, REG_FP_LAST); + sep = dspRegRange(mask.gprRegs(), minSiz, sep, REG_INT_FIRST, REG_INT_LAST); + sep = dspRegRange(mask.floatRegs(), minSiz, sep, REG_FP_FIRST, REG_FP_LAST); #ifdef TARGET_XARCH - sep = dspRegRange(mask.predicateRegs, minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); + sep = dspRegRange(mask.predicateRegs(), minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); #endif // TARGET_XARCH printf("]"); From b1bbbbf2e0a1c57e100a1a5f4d02c6cd56f6933b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 15 Mar 2024 23:08:59 -0700 Subject: [PATCH 103/201] Some rename methods of AllRegsMask --- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/lsra.cpp | 32 +++++++++++++++---------------- src/coreclr/jit/lsrabuild.cpp | 4 ++-- src/coreclr/jit/regset.h | 4 ++-- src/coreclr/jit/target.h | 5 ++--- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 6a4b7c5d37face..faa79135d01d4d 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -219,7 +219,7 @@ void CodeGen::genCodeForBBlist() regMaskOnlyOne varRegMask = varDsc->lvRegMask(); assert(compiler->IsOnlyOneRegMask(varRegMask)); - newLiveRegSet.AddRegTypeMask(varRegMask, varDsc->TypeGet()); + newLiveRegSet.AddRegMaskForType(varRegMask, varDsc->TypeGet()); if (varDsc->lvType == TYP_REF) { diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index a4a8db8d17d1f2..baa0e70844fb2f 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -395,24 +395,24 @@ void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, { assert(compiler->IsOnlyOneRegMask(regsBusy)); - regsInUseThisLocation.AddRegTypeMask(regsBusy, regType); + regsInUseThisLocation.AddRegMaskForType(regsBusy, regType); if (refPosition.lastUse) { if (refPosition.delayRegFree) { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED, interval, assignedReg)); - delayRegsToFree->AddRegTypeMask(regsBusy, regType); - regsInUseNextLocation.AddRegTypeMask(regsBusy, regType); + delayRegsToFree->AddRegMaskForType(regsBusy, regType); + regsInUseNextLocation.AddRegMaskForType(regsBusy, regType); } else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE, interval, assignedReg)); - regsToFree->AddRegTypeMask(regsBusy, regType); + regsToFree->AddRegMaskForType(regsBusy, regType); } } else if (refPosition.delayRegFree) { - regsInUseNextLocation.AddRegTypeMask(regsBusy, regType); + regsInUseNextLocation.AddRegMaskForType(regsBusy, regType); } } @@ -4152,7 +4152,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regMaskOnlyOne allRegsMask = allRegs(interval->registerType); RegisterType regType = interval->registerType; - regMaskOnlyOne candidateRegs = availableRegs.GetRegTypeMask(regType); + regMaskOnlyOne candidateRegs = availableRegs.GetRegMaskForType(regType); regNumber firstReg = REG_NA; regNumber newReg = REG_NA; @@ -5216,7 +5216,7 @@ void LinearScan::allocateRegistersMinimal() #ifdef SWIFT_SUPPORT if (currentRefPosition.delayRegFree) { - regsInUseNextLocation.AddRegTypeMask(currentRefPosition.registerAssignment, + regsInUseNextLocation.AddRegMaskForType(currentRefPosition.registerAssignment, regRecord->registerType); } #endif // SWIFT_SUPPORT @@ -5932,13 +5932,13 @@ void LinearScan::allocateRegisters() } #endif // TARGET_ARM } - regsInUseThisLocation.AddRegTypeMask(currentRefPosition.registerAssignment, regRecord->registerType); + regsInUseThisLocation.AddRegMaskForType(currentRefPosition.registerAssignment, regRecord->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); #ifdef SWIFT_SUPPORT if (currentRefPosition.delayRegFree) { - regsInUseNextLocation.AddRegTypeMask(currentRefPosition.registerAssignment, + regsInUseNextLocation.AddRegMaskForType(currentRefPosition.registerAssignment, regRecord->registerType); } #endif // SWIFT_SUPPORT @@ -8142,7 +8142,7 @@ void LinearScan::resolveRegisters() // If the localVar is in a register, it must be in a register that is not trashed by // the current node (otherwise it would have already been spilled). assert((genRegMask(localVarInterval->physReg) & - getKillSetForNode(treeNode).GetRegTypeMask(interval->registerType)) == RBM_NONE); + getKillSetForNode(treeNode).GetRegMaskForType(interval->registerType)) == RBM_NONE); // If we have allocated a register to spill it to, we will use that; otherwise, we will // spill it to the stack. We can use as a temp register any non-arg caller-save register. currentRefPosition->referent->recentRefPosition = currentRefPosition; @@ -9035,7 +9035,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) { var_types varType = getIntervalForLocalVar(liveOutVarIndex)->registerType; regMaskOnlyOne fromRegMask = genRegMask(fromReg, varType); - liveOutRegs.AddRegTypeMask(fromRegMask, varType); + liveOutRegs.AddRegMaskForType(fromRegMask, varType); } } @@ -13610,7 +13610,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates &= ~busyRegs.GetRegTypeMask(regType); + candidates &= ~busyRegs.GetRegMaskForType(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -13938,7 +13938,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // When we allocate for USE, we see that the register is busy at current location // and we end up with that candidate is no longer available. AllRegsMask busyRegs = linearScan->regsBusyUntilKill | linearScan->regsInUseThisLocation; - candidates &= ~busyRegs.GetRegTypeMask(regType); + candidates &= ~busyRegs.GetRegMaskForType(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use an even floating-point register for which the odd half @@ -14158,7 +14158,7 @@ _regMaskAll AllRegsMask::operator&(const regNumber reg) } // TODO: Can have version of AddGprMask(regMaskOnlyOne maskToAdd) and AddFloatMask(regMaskOnlyOne maskToAdd) -void AllRegsMask::AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type) +void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { registers[regIndexForType(type)] |= maskToAdd; } @@ -14189,7 +14189,7 @@ void AllRegsMask::AddRegNumInMask(regNumber reg) void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) { regMaskOnlyOne regMask = genRegMask(reg, type); - AddRegTypeMask(regMask, type); + AddRegMaskForType(regMask, type); } void AllRegsMask::RemoveRegNumFromMask(regNumber reg, var_types type) @@ -14237,7 +14237,7 @@ bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) return (registers[regIndexForRegister(reg)] & regMask) == regMask; } -regMaskOnlyOne AllRegsMask::GetRegTypeMask(var_types type) const +regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { return registers[regIndexForType(type)]; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 606eed1a00297a..a9b9b3aecd56fb 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1189,7 +1189,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo continue; } Interval* interval = getIntervalForLocalVar(varIndex); - regMaskOnlyOne regsKillMask = killMask.GetRegTypeMask(interval->registerType); + regMaskOnlyOne regsKillMask = killMask.GetRegMaskForType(interval->registerType); const bool isCallKill = (killMask.gprRegs() == RBM_INT_CALLEE_TRASH) || (killMask == AllRegsMask_CALLEE_TRASH); @@ -3299,7 +3299,7 @@ void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) } #endif - regMaskOnlyOne unprefRegMask = unpref.GetRegTypeMask(interval->registerType); + regMaskOnlyOne unprefRegMask = unpref.GetRegMaskForType(interval->registerType); interval->registerAversion |= unprefRegMask; regMaskOnlyOne newPreferences = allRegs(interval->registerType) & ~unprefRegMask; interval->updateRegisterPreferences(newPreferences); diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index ae6e7ae25bc39e..7446d7ee8474cc 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -145,7 +145,7 @@ class RegSet public: regMaskOnlyOne GetMaskVars(var_types type) const // 'get' property function for rsMaskVars property { - return _rsAllMaskVars.GetRegTypeMask(type); + return _rsAllMaskVars.GetRegMaskForType(type); } regMaskGpr GetGprMaskVars() const // 'get' property function for rsMaskVars property @@ -158,7 +158,7 @@ class RegSet void AddMaskVars(var_types type, regMaskOnlyOne addMaskVars) // union 'addMaskVars' with the rsMaskVars set { AllRegsMask newMask = _rsAllMaskVars; - newMask.AddRegTypeMask(addMaskVars, type); + newMask.AddRegMaskForType(addMaskVars, type); SetMaskVars(newMask); } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 20a28bf17765e6..5fcfd66a8f8763 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -330,8 +330,7 @@ typedef struct _regMaskAll void AddGprRegInMask(regNumber reg); void AddRegNumInMask(regNumber reg); void AddRegNumInMask(regNumber reg, var_types type); - // Rename this to AddRegMask() - void AddRegTypeMask(regMaskOnlyOne maskToAdd, var_types type); + void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); void AddGprRegMask(regMaskGpr maskToAdd); void AddFloatRegMask(regMaskFloat maskToAdd); void RemoveRegNumFromMask(regNumber reg); @@ -342,7 +341,7 @@ typedef struct _regMaskAll bool IsGprMaskPresent(regMaskGpr maskToCheck) const; bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; bool IsOnlyRegNumInMask(regNumber reg); - regMaskOnlyOne GetRegTypeMask(var_types type) const; + regMaskOnlyOne GetRegMaskForType(var_types type) const; regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; regMaskOnlyOne OrMask(const _regMaskAll& second, var_types regType) const; From 5ca0332995f02f426ecf2b24e577e2a3dfaecfe8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 18 Mar 2024 09:19:11 -0700 Subject: [PATCH 104/201] fix some issues --- src/coreclr/jit/codegenxarch.cpp | 4 ++-- src/coreclr/jit/lsra.cpp | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index e447e1be4c29e6..6cc75def01823f 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1604,13 +1604,13 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) inst_RV_TT(INS_mov, emitTypeSize(select), dstReg, falseVal); assert(!trueVal->isContained() || trueVal->isUsedFromMemory()); - assert((trueVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) == 0); + assert(!trueVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)); inst_RV_TT(JumpKindToCmov(desc.jumpKind1), emitTypeSize(select), dstReg, trueVal); if (desc.oper == GT_AND) { assert(falseVal->isUsedFromReg()); - assert((falseVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) == 0); + assert(!falseVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)); inst_RV_TT(JumpKindToCmov(emitter::emitReverseJumpKind(desc.jumpKind2)), emitTypeSize(select), dstReg, falseVal); } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index baa0e70844fb2f..0537c01276c6b0 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4152,7 +4152,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regMaskOnlyOne allRegsMask = allRegs(interval->registerType); RegisterType regType = interval->registerType; - regMaskOnlyOne candidateRegs = availableRegs.GetRegMaskForType(regType); + regMaskOnlyOne candidateRegs = allRegsMask & availableRegs.GetRegMaskForType(regType); regNumber firstReg = REG_NA; regNumber newReg = REG_NA; @@ -14249,11 +14249,13 @@ regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const regMaskTP AllRegsMask::GetGprFloatCombinedMask() const { -#ifdef TARGET_64BIT - return ((regMaskTP)floatRegs() << 32) | gprRegs(); -#else - return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); -#endif + //TODO: NEed to revert this once we change floatRegs to 32-bits +//#ifdef TARGET_64BIT +// return ((regMaskTP)floatRegs() << 32) | gprRegs(); +//#else +// return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); +//#endif + return floatRegs() | gprRegs(); } bool AllRegsMask::IsGprOrFloatPresent() const From f283c30b7752061b0fb51e4afc7052293d354442 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 18 Mar 2024 09:58:27 -0700 Subject: [PATCH 105/201] fix merge conflict issues --- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 7092b9d60cbef3..253abc54138106 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27365,7 +27365,7 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx, CorInfoCallConvExtension // of return registers and wants to know the set of return registers. // // static -AllRegsMask ReturnTypeDesc::GetABIReturnRegs(CorInfoCallConvExtension callCon) const +AllRegsMask ReturnTypeDesc::GetABIReturnRegs(CorInfoCallConvExtension callConv) const { AllRegsMask resultMask; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index b3e5554bb277ee..0f3e96567c6054 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3090,7 +3090,7 @@ void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandi { // In case of multi-reg call node, we have to query the i'th position return register. // For all other cases of multi-reg definitions, the registers must be in sequential order. - regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i); + regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, compiler->info.compCallConv); assert(dstCandidates.IsRegNumInMask(thisReg)); dstCandidates.RemoveRegNumFromMask(thisReg); From e936cb7beccb153fe4b85d8cf88faf803991c7ab Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 19 Mar 2024 08:10:54 -0700 Subject: [PATCH 106/201] Refactoring - Make [] operator private - Make Arm specific methods explicit that relies on `type` just to create the genRegMask --- src/coreclr/jit/lsra.cpp | 311 ++++++++++++++++++---------------- src/coreclr/jit/lsra.h | 176 ++++--------------- src/coreclr/jit/lsraarm64.cpp | 10 +- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/target.h | 32 ++-- 5 files changed, 224 insertions(+), 307 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 0537c01276c6b0..48115ae81b16fe 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -278,12 +278,12 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo if (nextRefPosition == nullptr) { nextLocation = MaxLocation; - fixedRegs[regIndexForType(regRecord->registerType)] &= ~genRegMask(regRecord->regNum); + fixedRegs.RemoveRegNumFromMask(regRecord->regNum); } else { nextLocation = nextRefPosition->nodeLocation; - fixedRegs[regIndexForType(regRecord->registerType)] |= genRegMask(regRecord->regNum); + fixedRegs.AddRegNumInMask(regRecord->regNum); } nextFixedRef[regRecord->regNum] = nextLocation; } @@ -293,7 +293,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, RefPosition* refPosition) { assert(currentInterval->isConstant && RefTypeIsDef(refPosition->refType)); - regMaskOnlyOne candidates = (mask & m_RegistersWithConstants[regIndexForType(currentInterval->registerType)]); + regMaskOnlyOne candidates = (mask & m_RegistersWithConstants.GetRegMaskForType(currentInterval->registerType)); regMaskOnlyOne result = RBM_NONE; while (candidates != RBM_NONE) { @@ -494,7 +494,7 @@ regMaskOnlyOne LinearScan::getConstrainedRegMask(RefPosition* refPosition, if ((refPosition != nullptr) && !refPosition->RegOptional()) { regMaskOnlyOne busyRegs = RBM_NONE; - busyRegs = regsBusyUntilKill.OrMask(regsInUseThisLocation, regType); + busyRegs = (regsBusyUntilKill | regsInUseThisLocation).GetRegMaskForType(regType); if ((newMask & ~busyRegs) == RBM_NONE) { // Constrained mask does not have at least one free register to allocate. @@ -4398,6 +4398,43 @@ void LinearScan::resetAllRegistersState() } } + +void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap) +{ + while (deadRegMask != RBM_NONE) + { + regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); + RegRecord* physRegRecord = getRegisterRecord(reg); + + makeRegAvailable(reg, physRegRecord->registerType); + Interval* assignedInterval = physRegRecord->assignedInterval; + + if (assignedInterval != nullptr) + { + assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + + if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) + { + assignedInterval->isActive = false; + if (assignedInterval->getNextRefPosition() == nullptr) + { + unassignPhysReg(physRegRecord, nullptr); + } + if (!assignedInterval->IsUpperVector()) + { + inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; + } + } + else + { + // This interval may still be active, but was in another register in an + // intervening block. + clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); + } + } + } +} + //------------------------------------------------------------------------ // processBlockStartLocations: Update var locations on entry to 'currentBlock' and clear constant // registers. @@ -4535,7 +4572,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Case #1 above. assert(getVarReg(predVarToRegMap, varIndex) == targetReg || getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE); - } + }// Keep the register assignment - if another var has it, it will get unassigned. else if (!nextRefPosition->copyReg) { // case #2 above. @@ -4554,7 +4591,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assert(targetReg != REG_STK); assert(interval->assignedReg != nullptr && interval->assignedReg->regNum == targetReg && interval->assignedReg->assignedInterval == interval); - liveRegs.AddRegNumInMask(targetReg, interval->registerType); + liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); continue; } } @@ -4584,7 +4621,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // likely to match other assignments this way. targetReg = interval->physReg; interval->isActive = true; - liveRegs.AddRegNumInMask(targetReg, interval->registerType); + liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); INDEBUG(inactiveRegs |= targetReg); setVarReg(inVarToRegMap, varIndex, targetReg); } @@ -4596,7 +4633,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (targetReg != REG_STK) { RegRecord* targetRegRecord = getRegisterRecord(targetReg); - liveRegs.AddRegNumInMask(targetReg, interval->registerType); + liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); if (!allocationPassComplete) { updateNextIntervalRef(targetReg, interval); @@ -4722,44 +4759,12 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Only focus on actual registers present deadCandidates &= actualRegistersMask; - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) - { - regMaskOnlyOne deadRegMask = deadCandidates[rType]; - - while (deadRegMask != RBM_NONE) - { - regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); - RegRecord* physRegRecord = getRegisterRecord(reg); - - makeRegAvailable(reg, physRegRecord->registerType); - Interval* assignedInterval = physRegRecord->assignedInterval; + regMaskTP deadGprFloatCandidates = deadCandidates.GetGprFloatCombinedMask(); - if (assignedInterval != nullptr) - { - assert(assignedInterval->isLocalVar || assignedInterval->isConstant || - assignedInterval->IsUpperVector()); - - if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) - { - assignedInterval->isActive = false; - if (assignedInterval->getNextRefPosition() == nullptr) - { - unassignPhysReg(physRegRecord, nullptr); - } - if (!assignedInterval->IsUpperVector()) - { - inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; - } - } - else - { - // This interval may still be active, but was in another register in an - // intervening block. - clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); - } - } - } - } + updateDeadCandidatesAtBlockStart(deadGprFloatCandidates, inVarToRegMap); +#ifdef HAS_PREDICATE_REGS + updateDeadCandidatesAtBlockStart(deadCandidates.predicateRegs(), inVarToRegMap); +#endif // HAS_PREDICATE_REGS #endif // TARGET_ARM } @@ -4855,6 +4860,17 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) } } +void LinearScan::inActivateRegisters(regMaskTP inactiveMask) +{ + while (inactiveMask != RBM_NONE) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(inactiveMask); + RegRecord* regRecord = getRegisterRecord(nextReg); + clearSpillCost(regRecord->regNum, regRecord->registerType); + makeRegisterInactive(regRecord); + } +} + //------------------------------------------------------------------------ // LinearScan::freeRegister: Make a register available for use // @@ -4924,25 +4940,27 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FREE_REGS)); makeRegsAvailable(regsToFree); + freeRegisterMask(regsToFree.GetGprFloatCombinedMask()); +#ifdef HAS_PREDICATE_REGS + freeRegisterMask(regsToFree.predicateRegs()); +#endif // HAS_PREDICATE_REGS +} - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) +void LinearScan::freeRegisterMask(regMaskTP freeMask) +{ + while (freeMask != RBM_NONE) { - regMaskOnlyOne freeMask = regsToFree[rType]; + regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); - while (freeMask != RBM_NONE) - { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); - - RegRecord* regRecord = getRegisterRecord(nextReg); + RegRecord* regRecord = getRegisterRecord(nextReg); #ifdef TARGET_ARM - if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) - { - assert(genIsValidDoubleReg(nextReg)); - freeMask ^= genRegMask(regNumber(nextReg + 1)); - } -#endif - freeRegister(regRecord); + if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) + { + assert(genIsValidDoubleReg(nextReg)); + freeMask ^= genRegMask(regNumber(nextReg + 1)); } +#endif + freeRegister(regRecord); } } @@ -5019,17 +5037,11 @@ void LinearScan::allocateRegistersMinimal() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) - { - regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; - while (tempRegsMaskInactive != RBM_NONE) - { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsMaskInactive); - RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); - makeRegisterInactive(regRecord); - } - } + inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); +#ifdef HAS_PREDICATE_REGS + inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); +#endif + if (currentRefPosition.nodeLocation > prevLocation) { makeRegsAvailable(regsToMakeInactive); @@ -5210,7 +5222,7 @@ void LinearScan::allocateRegistersMinimal() } #endif // TARGET_ARM } - regsInUseThisLocation.AddRegNumInMask(regRecord->regNum, regRecord->registerType); + regsInUseThisLocation.AddRegMaskForType(currentRefPosition.registerAssignment, regRecord->registerType); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); #ifdef SWIFT_SUPPORT @@ -5358,7 +5370,7 @@ void LinearScan::allocateRegistersMinimal() DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); + copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); } // For tree temp (non-localVar) interval, we will need an explicit move. @@ -5373,7 +5385,7 @@ void LinearScan::allocateRegistersMinimal() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -5471,17 +5483,17 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskOnlyOne regMask = getRegMask(assignedRegister, currentInterval->registerType); - regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); + AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); + + regsInUseThisLocation |= assignedRegMask; if (currentRefPosition.delayRegFree) { - regsInUseNextLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsInUseNextLocation |= assignedRegMask; } currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumFromMask(assignedRegister, - currentInterval->registerType); // we'll set it again later if it's dead + regsToFree &= ~assignedRegMask; // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -5502,11 +5514,11 @@ void LinearScan::allocateRegistersMinimal() { if (currentRefPosition.delayRegFree) { - delayRegsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); + delayRegsToMakeInactive |= assignedRegMask; } else { - regsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToMakeInactive |= assignedRegMask; } // TODO-Cleanup: this makes things consistent with previous, and will enable preferences // to be propagated, but it seems less than ideal. @@ -5525,13 +5537,13 @@ void LinearScan::allocateRegistersMinimal() { if (currentRefPosition.delayRegFree) { - delayRegsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + delayRegsToFree |= assignedRegMask; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED)); } else { - regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToFree |= assignedRegMask; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE)); } @@ -5660,7 +5672,7 @@ void LinearScan::allocateRegisters() updateNextIntervalRef(reg, interval); updateSpillCost(reg, interval); setRegInUse(reg, interval->registerType); - INDEBUG(registersToDump.AddRegNumInMask(reg, interval->registerType)); + INDEBUG(registersToDump.AddRegNumInMask(reg ARM_ARG(interval->registerType))); } } else @@ -5713,17 +5725,11 @@ void LinearScan::allocateRegisters() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); - for (int rType = 0; rType < REGISTER_TYPE_COUNT; rType++) - { - regMaskOnlyOne tempRegsMaskInactive = tempRegsToMakeInactive[rType]; - while (tempRegsMaskInactive != RBM_NONE) - { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(tempRegsMaskInactive); - RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); - makeRegisterInactive(regRecord); - } - } + inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); +#ifdef HAS_PREDICATE_REGS + inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); +#endif + if (currentRefPosition.nodeLocation > prevLocation) { makeRegsAvailable(regsToMakeInactive); @@ -6087,7 +6093,7 @@ void LinearScan::allocateRegisters() updateSpillCost(assignedRegister, currentInterval); } - regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, nullptr, assignedRegister)); currentRefPosition.registerAssignment = RBM_NONE; @@ -6404,7 +6410,7 @@ void LinearScan::allocateRegisters() currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); + copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6513,7 +6519,7 @@ void LinearScan::allocateRegisters() DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg, currentInterval->registerType); + copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6534,7 +6540,7 @@ void LinearScan::allocateRegisters() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -6702,16 +6708,17 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regsInUseThisLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); + AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); + + regsInUseThisLocation |= assignedRegMask; if (currentRefPosition.delayRegFree) { - regsInUseNextLocation.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsInUseNextLocation |= assignedRegMask; } currentRefPosition.registerAssignment = assignedRegBit; currentInterval->physReg = assignedRegister; - regsToFree.RemoveRegNumFromMask(assignedRegister, - currentInterval->registerType); // we'll set it again later if it's dead + regsToFree &= ~assignedRegMask; // we'll set it again later if it's dead // If this interval is dead, free the register. // The interval could be dead if this is a user variable, or if the @@ -6748,11 +6755,11 @@ void LinearScan::allocateRegisters() { if (currentRefPosition.delayRegFree) { - delayRegsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); + delayRegsToMakeInactive |= assignedRegMask; } else { - regsToMakeInactive.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToMakeInactive |= assignedRegMask; } // TODO-Cleanup: this makes things consistent with previous, and will enable preferences // to be propagated, but it seems less than ideal. @@ -6771,13 +6778,13 @@ void LinearScan::allocateRegisters() { if (currentRefPosition.delayRegFree) { - delayRegsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + delayRegsToFree |= assignedRegMask; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE_DELAYED)); } else { - regsToFree.AddRegNumInMask(assignedRegister, currentInterval->registerType); + regsToFree |= assignedRegMask; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_LAST_USE)); } @@ -9034,8 +9041,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) if (fromReg != REG_STK) { var_types varType = getIntervalForLocalVar(liveOutVarIndex)->registerType; - regMaskOnlyOne fromRegMask = genRegMask(fromReg, varType); - liveOutRegs.AddRegMaskForType(fromRegMask, varType); + liveOutRegs.AddRegNumInMask(fromReg ARM_ARG(varType)); } } @@ -9204,8 +9210,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) // live only at another target), and we can't copy another lclVar into that reg in this block. regMaskOnlyOne sameToRegMask = genRegMask(sameToReg, outVarRegType); - if (maybeSameLivePaths && (liveOutRegs.IsRegNumInMask(sameToReg, outVarRegType) || - sameWriteRegs.IsRegNumInMask(sameToReg, outVarRegType) != RBM_NONE)) + if (maybeSameLivePaths && (liveOutRegs.IsRegNumInMask(sameToReg ARM_ARG(outVarRegType)) || + sameWriteRegs.IsRegNumInMask(sameToReg ARM_ARG(outVarRegType)) != RBM_NONE)) { sameToReg = REG_NA; } @@ -9248,7 +9254,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) VarSetOps::AddElemD(compiler, diffResolutionSet, outResolutionSetVarIndex); if (fromReg != REG_STK) { - diffReadRegs.AddRegNumInMask(fromReg, getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); + diffReadRegs.AddRegNumInMask(fromReg ARM_ARG(getIntervalForLocalVar(outResolutionSetVarIndex)->registerType)); } } else if (sameToReg != fromReg) @@ -9257,8 +9263,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) setVarReg(sameVarToRegMap, outResolutionSetVarIndex, sameToReg); if (sameToReg != REG_STK) { - sameWriteRegs.AddRegNumInMask(sameToReg, - getIntervalForLocalVar(outResolutionSetVarIndex)->registerType); + sameWriteRegs.AddRegNumInMask( + sameToReg ARM_ARG(getIntervalForLocalVar(outResolutionSetVarIndex)->registerType)); } } } @@ -9832,7 +9838,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, // Do we have a free targetReg? if (fromReg == sourceReg) { - if (source[fromReg] != REG_NA && !targetRegsFromStack.IsOnlyRegNumInMask(fromReg)) + if (source[fromReg] != REG_NA && !targetRegsFromStack.IsRegNumInMask(fromReg)) { targetRegsReady |= fromReg; #ifdef TARGET_ARM @@ -9868,7 +9874,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, if ((lowerHalfSrcReg != REG_NA) && (lowerHalfSrcLoc == REG_NA) && (sourceIntervals[lowerHalfSrcReg] != nullptr) && !targetRegsReady.IsRegNumInMask(lowerHalfReg) && - !targetRegsFromStack.IsOnlyRegNumInMask(lowerHalfReg)) + !targetRegsFromStack.IsRegNumInMask(lowerHalfReg)) { // This must be a double interval, otherwise it would be in targetRegsReady, or already // completed. @@ -11118,7 +11124,7 @@ void LinearScan::dumpLsraAllocationEvent( } if ((interval != nullptr) && (reg != REG_NA) && (reg != REG_STK)) { - registersToDump.AddRegNumInMask(reg, interval->registerType); + registersToDump.AddRegNumInMask(reg ARM_ARG(interval->registerType)); dumpRegRecordTitleIfNeeded(); } @@ -13631,7 +13637,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regIndexForType(regType)]; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs.GetRegMaskForType(regType); while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -13710,7 +13716,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // refpositions. assert((refPosition->refType == RefTypeUpperVectorRestore) || (genCountBits(candidates) == 1)); - freeCandidates = candidates & linearScan->m_AvailableRegs[regIndexForType(currentInterval->registerType)]; + freeCandidates = candidates & linearScan->m_AvailableRegs.GetRegMaskForType(currentInterval->registerType); } if ((freeCandidates == RBM_NONE) && (candidates == RBM_NONE)) @@ -13955,7 +13961,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // Also eliminate as busy any register with a conflicting fixed reference at this or // the next location. // Note that this will eliminate the fixedReg, if any, but we'll add it back below. - regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs[regIndexForType(regType)]; + regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs.GetRegMaskForType(regType); while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); @@ -14180,27 +14186,35 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) registers[0] |= -static_cast(!regIndex) & genRegMask(reg); } +#ifdef TARGET_ARM void AllRegsMask::AddRegNumInMask(regNumber reg) { regMaskOnlyOne regMask = genRegMask(reg); registers[regIndexForRegister(reg)] |= regMask; } +#endif -void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) -{ - regMaskOnlyOne regMask = genRegMask(reg, type); - AddRegMaskForType(regMask, type); -} +// +//void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) +//{ +// //TODO: All these methods should be revisited to see if they should be operated on `type` or `reg` +// regMaskOnlyOne regMask = genRegMask(reg, type); +// AddRegMaskForType(regMask, type); +//} -void AllRegsMask::RemoveRegNumFromMask(regNumber reg, var_types type) +// ---------------------------------------------------------- +// AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except +// that it takes `type` as an argument and adds `reg` to the mask for that type. +// +void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMaskToRemove = genRegMask(reg, type); - registers[regIndexForType(type)] &= ~regMaskToRemove; + regMaskOnlyOne regMask = genRegMask(reg ARM_ARG(type)); + registers[regIndexForRegister(reg)] |= regMask; } -void AllRegsMask::RemoveRegNumFromMask(regNumber reg) +void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMaskToRemove = genRegMask(reg); + regMaskOnlyOne regMaskToRemove = genRegMask(reg ARM_ARG(type)); registers[regIndexForRegister(reg)] &= ~regMaskToRemove; } @@ -14209,17 +14223,25 @@ void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_type registers[regIndexForType(type)] &= ~regMaskToRemove; } -bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) +bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMaskToCheck = genRegMask(reg, type); + regMaskOnlyOne regMaskToCheck = genRegMask(reg ARM_ARG(type)); return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; } +#ifdef TARGET_ARM +void AllRegsMask::RemoveRegNumFromMask(regNumber reg) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg); + registers[regIndexForRegister(reg)] &= ~regMaskToRemove; +} + bool AllRegsMask::IsRegNumInMask(regNumber reg) { regMaskOnlyOne regMaskToCheck = genRegMask(reg); return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; } +#endif bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const { @@ -14230,12 +14252,12 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const { return (floatRegs() & maskToCheck) != RBM_NONE; } - -bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) -{ - regMaskOnlyOne regMask = genRegMask(reg); - return (registers[regIndexForRegister(reg)] & regMask) == regMask; -} +// +//bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) +//{ +// regMaskOnlyOne regMask = genRegMask(reg); +// return (registers[regIndexForRegister(reg)] & regMask) == regMask; +//} regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { @@ -14255,16 +14277,15 @@ regMaskTP AllRegsMask::GetGprFloatCombinedMask() const //#else // return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); //#endif - return floatRegs() | gprRegs(); + // TODO: Cannot get RBM_ALLFLOAT because it is not defined for AllRegsMask + // Moving this definition to lsra might work, but this is temporary so + // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in + // the comment above. + regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; + return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); } bool AllRegsMask::IsGprOrFloatPresent() const { - return (gprRegs() | floatRegs()) != RBM_NONE; -} - -regMaskOnlyOne AllRegsMask::OrMask(const _regMaskAll& other, var_types regType) const -{ - int regIndex = regIndexForType(regType); - return registers[regIndex] | other[regIndex]; + return GetGprFloatCombinedMask() != RBM_NONE; } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 71d44abc6edd7e..d628ab774cec6b 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1026,6 +1026,7 @@ class LinearScan : public LinearScanInterface void processBlockStartLocations(BasicBlock* current); void processBlockEndLocations(BasicBlock* current); void resetAllRegistersState(); + FORCEINLINE void updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap); #ifdef TARGET_ARM bool isSecondHalfReg(RegRecord* regRec, Interval* interval); @@ -1116,8 +1117,10 @@ class LinearScan : public LinearScanInterface regMaskFloat internalFloatRegCandidates(); void makeRegisterInactive(RegRecord* physRegRecord); + FORCEINLINE void inActivateRegisters(regMaskTP inactiveMask); void freeRegister(RegRecord* physRegRecord); void freeRegisters(AllRegsMask regsToFree); + FORCEINLINE void freeRegisterMask(regMaskTP freeMask); // Get the type that this tree defines. var_types getDefType(GenTree* tree) @@ -1248,13 +1251,13 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne getFreeCandidates(regMaskOnlyOne candidates, var_types regType) { - regMaskOnlyOne result = candidates & m_AvailableRegs[regIndexForType(regType)]; + regMaskOnlyOne result = candidates & m_AvailableRegs.GetRegMaskForType(regType); #ifdef TARGET_ARM // For TYP_DOUBLE on ARM, we can only use register for which the odd half is // also available. if (regType == TYP_DOUBLE) { - result &= (m_AvailableRegs[1] >> 1); + result &= (m_AvailableRegs.floatRegs() >> 1); } #endif // TARGET_ARM return result; @@ -1749,14 +1752,7 @@ class LinearScan : public LinearScanInterface // gets reset for every refposition we are processing depending on the // register type. That wawy we do not have to query and fetch the appropriate // entry again and agin. -#ifdef HAS_PREDICATE_REGS - // TODO: Change this to m_AvailableGprRegs, m_AvailableFloatRegs, etc. - // TODO: Check why this can't be AllRegsMask - regMaskOnlyOne m_AvailableRegs[3]; - -#else - regMaskOnlyOne m_AvailableRegs[2]; -#endif // HAS_PREDICATE_REGS + AllRegsMask m_AvailableRegs; regNumber getRegForType(regNumber reg, var_types regType) { @@ -1785,145 +1781,42 @@ class LinearScan : public LinearScanInterface void resetAvailableRegs() { - m_AvailableRegs[0] = availableIntRegs; - m_AvailableRegs[1] = availableFloatRegs; -#ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] = availableMaskRegs; -#endif - m_RegistersWithConstants[0] = RBM_NONE; - m_RegistersWithConstants[1] = RBM_NONE; + m_AvailableRegs = AllRegsMask(availableIntRegs, availableFloatRegs #ifdef HAS_PREDICATE_REGS - m_RegistersWithConstants[2] = RBM_NONE; + , availableMaskRegs #endif + ); + m_RegistersWithConstants.Clear(); } bool isRegAvailable(regNumber reg, var_types regType) // only used in asserts { - regMaskOnlyOne regMask = getRegMask(reg, regType); - -#ifdef TARGET_ARM64 - if (emitter::isGeneralRegisterOrZR(reg)) -#else - if (emitter::isGeneralRegister(reg)) -#endif - { - return (m_AvailableRegs[0] & regMask) == regMask; - } - else if (emitter::isFloatReg(reg)) - { - return (m_AvailableRegs[1] & regMask) == regMask; - } - - else - { -#ifdef HAS_PREDICATE_REGS - assert(emitter::isMaskReg(reg)); - return (m_AvailableRegs[2] & regMask) == regMask; -#else - unreached(); -#endif - } + return m_AvailableRegs.IsRegNumInMask(reg ARM_ARG(regType)); } void setRegsInUse(AllRegsMask regMask) { - // TODO: Fix this later. - m_AvailableRegs[0] &= ~regMask.gprRegs(); - m_AvailableRegs[1] &= ~regMask.floatRegs(); -#ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] &= ~regMask.predicateRegs(); -#endif + m_AvailableRegs &= ~regMask; } + void setRegInUse(regNumber reg, var_types regType) { - regMaskOnlyOne regMask = getRegMask(reg, regType); - -#ifdef TARGET_ARM64 - if (emitter::isGeneralRegisterOrZR(reg)) -#else - if (emitter::isGeneralRegister(reg)) -#endif - { - m_AvailableRegs[0] &= ~regMask; - } - else if (emitter::isFloatReg(reg)) - { - m_AvailableRegs[1] &= ~regMask; - } -#ifdef HAS_PREDICATE_REGS - else - { - assert(emitter::isMaskReg(reg)); - m_AvailableRegs[2] &= ~regMask; - } -#endif + m_AvailableRegs.RemoveRegNumFromMask(reg ARM_ARG(regType)); } + void makeRegsAvailable(AllRegsMask regMask) { - // TODO: This will be just `regMask` - m_AvailableRegs[0] |= regMask.gprRegs(); - m_AvailableRegs[1] |= regMask.floatRegs(); + m_AvailableRegs |= regMask; + assert(compiler->IsGprRegMask(m_AvailableRegs.gprRegs())); + assert(compiler->IsFloatRegMask(m_AvailableRegs.floatRegs())); #ifdef HAS_PREDICATE_REGS - m_AvailableRegs[2] |= regMask.predicateRegs(); - assert(compiler->IsPredicateRegMask(m_AvailableRegs[2])); -#endif - assert(compiler->IsGprRegMask(m_AvailableRegs[0])); - assert(compiler->IsFloatRegMask(m_AvailableRegs[1])); - } - void makeRegAvailable(regNumber reg, var_types regType) - { - regMaskOnlyOne regMask = getRegMask(reg, regType); -#ifdef TARGET_ARM64 - // TODO: Convert it in to a lookup array that maps register number - // to '0', '1' or '2'. So something like this: - // rax - 0 - // rbx - 0 - // ... - // xmm0 - 1 - // here it will be, m_AvailableRegs[regIndexForType[reg]] = - if (emitter::isGeneralRegisterOrZR(reg)) -#else - if (emitter::isGeneralRegister(reg)) + assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs())); #endif - { - m_AvailableRegs[0] |= regMask; - } - else if (emitter::isFloatReg(reg)) - { - m_AvailableRegs[1] |= regMask; - } - else - { -#ifdef HAS_PREDICATE_REGS - assert(emitter::isMaskReg(reg)); - m_AvailableRegs[2] |= regMask; -#else - unreached(); -#endif - } } - - void makeRegAvailable(regMaskOnlyOne regMask, var_types regType) + void makeRegAvailable(regNumber reg, var_types regType) { -#ifdef DEBUG - if (varTypeUsesIntReg(regType)) - { - assert(compiler->IsGprRegMask(regMask)); - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(regType)) - { - assert(compiler->IsPredicateRegMask(regMask)); - } -#endif // HAS_PREDICATE_REGS - else - { - assert(varTypeUsesFloatReg(regType)); - assert(compiler->IsFloatRegMask(regMask)); - } -#endif - m_AvailableRegs[regIndexForType(regType)] |= regMask; + m_AvailableRegs.AddRegNumInMask(reg ARM_ARG(regType)); } void clearAllNextIntervalRef(); @@ -1941,14 +1834,8 @@ class LinearScan : public LinearScanInterface RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); -#ifdef HAS_PREDICATE_REGS - regMaskOnlyOne m_RegistersWithConstants[3]; // TODO: Change this to m_GprRegistersWithConstant, - // m_FloatRegistersWithConstant, etc. - regMaskOnlyOne fixedRegs[3]; -#else - regMaskOnlyOne m_RegistersWithConstants[2]; - regMaskOnlyOne fixedRegs[2]; -#endif + AllRegsMask m_RegistersWithConstants; + AllRegsMask fixedRegs; void clearConstantReg(regNumber reg, var_types regType) { @@ -1956,17 +1843,16 @@ class LinearScan : public LinearScanInterface // just operate on curr_RegistersWithConstants and assert // assert(m_RegistersWithConstants[regIndexForType(regType)] == curr_RegistersWithConstants); // but we will have to make sure that we save it back too?? - m_RegistersWithConstants[regIndexForType(regType)] &= ~getRegMask(reg, regType); + m_RegistersWithConstants.RemoveRegNumFromMask(reg ARM_ARG(regType)); } void setConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants[regIndexForType(regType)] |= getRegMask(reg, regType); + m_RegistersWithConstants.AddRegNumInMask(reg ARM_ARG(regType)); } bool isRegConstant(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); - regMaskOnlyOne regMask = getRegMask(reg, regType); - return (m_RegistersWithConstants[regIndexForType(regType)] & regMask) == regMask; + return m_RegistersWithConstants.IsRegNumInMask(reg ARM_ARG(regType)); } regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); @@ -2007,11 +1893,11 @@ class LinearScan : public LinearScanInterface #endif bool isRegBusy(regNumber reg, var_types regType) { - return regsBusyUntilKill.IsRegNumInMask(reg, regType); + return regsBusyUntilKill.IsRegNumInMask(reg ARM_ARG(regType)); } void setRegBusyUntilKill(regNumber reg, var_types regType) { - regsBusyUntilKill.AddRegNumInMask(reg, regType); + regsBusyUntilKill.AddRegNumInMask(reg ARM_ARG(regType)); } void clearRegBusyUntilKill(regNumber reg) { @@ -2021,7 +1907,7 @@ class LinearScan : public LinearScanInterface bool isRegInUse(regNumber reg, var_types regType) { regMaskOnlyOne regMask = getRegMask(reg, regType); - return regsInUseThisLocation.IsRegNumInMask(reg, regType); + return regsInUseThisLocation.IsRegNumInMask(reg ARM_ARG(regType)); } void resetRegState() @@ -2196,7 +2082,7 @@ class LinearScan : public LinearScanInterface #endif // FEATURE_ARG_SPLIT int BuildLclHeap(GenTree* tree); -#if defined(TARGET_AMD64) +//#if defined(TARGET_AMD64) regMaskFloat rbmAllFloat; regMaskFloat rbmFltCalleeTrash; @@ -2208,7 +2094,7 @@ class LinearScan : public LinearScanInterface { return this->rbmFltCalleeTrash; } -#endif // TARGET_AMD64 +//#endif // TARGET_AMD64 #if defined(TARGET_XARCH) regMaskPredicate rbmAllMask; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 25b00a3dd95336..c8c6822796abea 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -364,7 +364,7 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec } maskForCurRange |= (registersNeededMask << regAvailableStartIndex); - maskForCurRange &= m_AvailableRegs[1]; + maskForCurRange &= m_AvailableRegs.floatRegs(); if (maskForCurRange != RBM_NONE) { @@ -420,7 +420,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { assert(compiler->info.compNeedsConsecutiveRegisters); assert(refPosition->isFirstRefPositionOfConsecutiveRegisters()); - regMaskFloat freeCandidates = allCandidates & m_AvailableRegs[1]; + regMaskFloat freeCandidates = allCandidates & m_AvailableRegs.floatRegs(); #ifdef DEBUG if (getStressLimitRegs() != LSRA_LIMIT_NONE) @@ -536,7 +536,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, *busyCandidates = consecutiveResultForBusy; // Check if we can further check better registers amoung consecutiveResultForBusy. - if ((m_AvailableRegs[1] & overallResultForBusy) != RBM_NONE) + if ((m_AvailableRegs.floatRegs() & overallResultForBusy) != RBM_NONE) { // `overallResultForBusy` contains the mask of entire series that can be the consecutive candidates. // If there is an overlap of that with free registers, then try to find a series that will need least @@ -549,13 +549,13 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { *busyCandidates = optimalConsecutiveResultForBusy; } - else if ((m_AvailableRegs[1] & consecutiveResultForBusy) != RBM_NONE) + else if ((m_AvailableRegs.floatRegs() & consecutiveResultForBusy) != RBM_NONE) { // We did not find free consecutive candidates, however we found some registers among the // `allCandidates` that are mix of free and busy. Since `busyCandidates` just has bit set for first // register of such series, return the mask that starts with free register, if possible. The busy // registers will be spilled during assignment of subsequent RefPosition. - *busyCandidates = (m_AvailableRegs[1] & consecutiveResultForBusy); + *busyCandidates = (m_AvailableRegs.floatRegs() & consecutiveResultForBusy); } } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0f3e96567c6054..e6c35f0765ac88 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2365,7 +2365,7 @@ void LinearScan::buildIntervals() assert(inArgReg < REG_COUNT); mask = genRegMask(inArgReg); assignPhysReg(inArgReg, interval); - INDEBUG(registersToDump.AddRegNumInMask(inArgReg, interval->registerType)); + INDEBUG(registersToDump.AddRegNumInMask(inArgReg ARM_ARG(interval->registerType))); } RefPosition* pos = newRefPosition(interval, MinLocation, RefTypeParamDef, nullptr, mask); pos->setRegOptional(true); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 5fcfd66a8f8763..c23be4bf3ed13c 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -271,6 +271,8 @@ typedef struct _regMaskAll { private: regMaskTP registers[REGISTER_TYPE_COUNT]; + regMaskOnlyOne operator[](int index) const; + regMaskOnlyOne& operator[](int index); public: inline regMaskGpr gprRegs() const @@ -317,9 +319,16 @@ typedef struct _regMaskAll registers[2] = _registers[2]; } +#ifdef TARGET_ARM _regMaskAll(regNumber reg) : _regMaskAll() { - *this |= reg; + AddRegNumInMask(reg); + } +#endif + + _regMaskAll(regNumber reg ARM_ARG(var_types type)) : _regMaskAll() + { + AddRegNumInMask(reg ARM_ARG(type)); } void Clear(); @@ -328,29 +337,30 @@ typedef struct _regMaskAll void Create(regNumber reg); // Rename this to AddRegNum void AddGprRegInMask(regNumber reg); - void AddRegNumInMask(regNumber reg); - void AddRegNumInMask(regNumber reg, var_types type); + void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); void AddGprRegMask(regMaskGpr maskToAdd); void AddFloatRegMask(regMaskFloat maskToAdd); - void RemoveRegNumFromMask(regNumber reg); - void RemoveRegNumFromMask(regNumber reg, var_types type); - void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - bool IsRegNumInMask(regNumber reg, var_types type); + +#ifdef TARGET_ARM + void AddRegNumInMask(regNumber reg); + void RemoveRegNumFromMask(regNumber reg); bool IsRegNumInMask(regNumber reg); +#endif + void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); + bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); bool IsGprMaskPresent(regMaskGpr maskToCheck) const; bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; - bool IsOnlyRegNumInMask(regNumber reg); + //bool IsOnlyRegNumInMask(regNumber reg); regMaskOnlyOne GetRegMaskForType(var_types type) const; regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; - regMaskOnlyOne OrMask(const _regMaskAll& second, var_types regType) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field bool IsGprOrFloatPresent() const; regMaskTP GetGprFloatCombinedMask() const; - regMaskOnlyOne operator[](int index) const; - regMaskOnlyOne& operator[](int index); + void operator|=(const _regMaskAll& other); void operator&=(const _regMaskAll& other); void operator|=(const regNumber reg); From 150dd9446b7540cf8af21461dfc810b1ca4ba008 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 19 Mar 2024 08:13:29 -0700 Subject: [PATCH 107/201] jit format --- src/coreclr/jit/codegencommon.cpp | 14 +++--- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/emit.cpp | 2 +- src/coreclr/jit/emit.h | 2 +- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/lsra.cpp | 84 +++++++++++++------------------ src/coreclr/jit/lsra.h | 26 +++++----- src/coreclr/jit/lsrabuild.cpp | 5 +- src/coreclr/jit/regset.cpp | 4 +- src/coreclr/jit/target.h | 48 ++++++++---------- src/coreclr/jit/utils.cpp | 2 +- 13 files changed, 87 insertions(+), 108 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7f533e28c3793a..400fa676cbba29 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8237,13 +8237,13 @@ void CodeGen::genRegCopy(GenTree* treeNode) #ifdef DEBUG - if (targetReg != sourceReg) - { - singleRegMask targetRegMask = genRegMask(targetReg); - assert((busyRegs.GetMaskForRegNum(targetReg) & targetRegMask) == 0); - busyRegs.RemoveRegNumFromMask(sourceReg); - } - busyRegs.AddRegNumInMask(targetReg); + if (targetReg != sourceReg) + { + singleRegMask targetRegMask = genRegMask(targetReg); + assert((busyRegs.GetMaskForRegNum(targetReg) & targetRegMask) == 0); + busyRegs.RemoveRegNumFromMask(sourceReg); + } + busyRegs.AddRegNumInMask(targetReg); #endif // DEBUG } return; diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index e5448c3211eb57..c357ef9c990606 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6035,7 +6035,7 @@ void CodeGen::genCall(GenTreeCall* call) } assert(!killMask.IsGprMaskPresent(gcInfo.gcRegGCrefSetCur)); - assert(!killMask.IsGprMaskPresent(gcInfo.gcRegByrefSetCur)); + assert(!killMask.IsGprMaskPresent(gcInfo.gcRegByrefSetCur)); #endif var_types returnType = call->TypeGet(); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 6a73e69764fa3c..5981b54ecd0f00 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1197,7 +1197,7 @@ inline const char* varTypeGCstring(var_types type) /*****************************************************************************/ const char* varTypeName(var_types); -const int regIndexForRegister(regNumber reg); +const int regIndexForRegister(regNumber reg); /*****************************************************************************/ // Helpers to pull little-endian values out of a byte stream. diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 5801f41e9790b7..bcba1907435101 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -747,7 +747,7 @@ void emitter::emitBegCG(Compiler* comp, COMP_HANDLE cmpHandle) #endif #if defined(TARGET_AMD64) - rbmAllFloat = emitComp->rbmAllFloat; + rbmAllFloat = emitComp->rbmAllFloat; rbmFltCalleeTrash = emitComp->rbmFltCalleeTrash; #endif // TARGET_AMD64 diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 93012cce6c7d7f..634561f3b34747 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2477,7 +2477,7 @@ class emitter private: #if defined(TARGET_AMD64) regMaskFloat rbmAllFloat; - regMaskFloat rbmFltCalleeTrash; + regMaskFloat rbmFltCalleeTrash; FORCEINLINE regMaskFloat get_RBM_ALLFLOAT() const { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 253abc54138106..00f1853b0c7ee2 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1081,7 +1081,7 @@ AllRegsMask GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -//TODO: All the callers of gtGetRegMask() are just interested in gpr +// TODO: All the callers of gtGetRegMask() are just interested in gpr // so, this can just return gpr mask and if none exist, just return RBM_NONE AllRegsMask GenTree::gtGetRegMask() const { diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 3de2317d9244e2..e0b77154156c1c 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -955,7 +955,7 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; AllRegsMask gtGetRegMask() const; - regMaskGpr gtGetGprRegMask() const; + regMaskGpr gtGetGprRegMask() const; AllRegsMask gtGetContainedRegMask(); GenTreeFlags gtFlags; diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 48115ae81b16fe..a98c00d6ae045a 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -541,10 +541,10 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly { // The refPosition could be null, for example when called // by getTempRegForResolution(). - int minRegCount = 1; + int minRegCount = 1; if (refPosition != nullptr) { - minRegCount = refPosition->minRegCandidateCount; + minRegCount = refPosition->minRegCandidateCount; RegisterType currRegType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType : refPosition->getReg()->registerType; assert(regtype == currRegType); @@ -4398,7 +4398,6 @@ void LinearScan::resetAllRegistersState() } } - void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap) { while (deadRegMask != RBM_NONE) @@ -4572,7 +4571,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Case #1 above. assert(getVarReg(predVarToRegMap, varIndex) == targetReg || getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE); - }// Keep the register assignment - if another var has it, it will get unassigned. + } // Keep the register assignment - if another var has it, it will get unassigned. else if (!nextRefPosition->copyReg) { // case #2 above. @@ -5229,7 +5228,7 @@ void LinearScan::allocateRegistersMinimal() if (currentRefPosition.delayRegFree) { regsInUseNextLocation.AddRegMaskForType(currentRefPosition.registerAssignment, - regRecord->registerType); + regRecord->registerType); } #endif // SWIFT_SUPPORT } @@ -5482,7 +5481,7 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); regsInUseThisLocation |= assignedRegMask; @@ -5945,7 +5944,7 @@ void LinearScan::allocateRegisters() if (currentRefPosition.delayRegFree) { regsInUseNextLocation.AddRegMaskForType(currentRefPosition.registerAssignment, - regRecord->registerType); + regRecord->registerType); } #endif // SWIFT_SUPPORT } @@ -6707,7 +6706,7 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); regsInUseThisLocation |= assignedRegMask; @@ -9040,7 +9039,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) regNumber fromReg = getVarReg(outVarToRegMap, liveOutVarIndex); if (fromReg != REG_STK) { - var_types varType = getIntervalForLocalVar(liveOutVarIndex)->registerType; + var_types varType = getIntervalForLocalVar(liveOutVarIndex)->registerType; liveOutRegs.AddRegNumInMask(fromReg ARM_ARG(varType)); } } @@ -9254,7 +9253,8 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) VarSetOps::AddElemD(compiler, diffResolutionSet, outResolutionSetVarIndex); if (fromReg != REG_STK) { - diffReadRegs.AddRegNumInMask(fromReg ARM_ARG(getIntervalForLocalVar(outResolutionSetVarIndex)->registerType)); + diffReadRegs.AddRegNumInMask( + fromReg ARM_ARG(getIntervalForLocalVar(outResolutionSetVarIndex)->registerType)); } } else if (sameToReg != fromReg) @@ -14119,7 +14119,7 @@ _regMaskAll AllRegsMask::operator~() , ~predicateRegs() #endif - ); + ); } bool AllRegsMask::operator==(const AllRegsMask& other) @@ -14128,7 +14128,7 @@ bool AllRegsMask::operator==(const AllRegsMask& other) #ifdef HAS_PREDICATE_REGS && (predicateRegs() == other.predicateRegs()) #endif - ); + ); } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -14140,20 +14140,20 @@ _regMaskAll AllRegsMask::operator&(const AllRegsMask& other) { return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() #ifdef HAS_PREDICATE_REGS - , + , predicateRegs() & other.predicateRegs() #endif - ); + ); } _regMaskAll AllRegsMask::operator|(const _regMaskAll& other) { return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() #ifdef HAS_PREDICATE_REGS - , + , predicateRegs() | other.predicateRegs() #endif - ); + ); } _regMaskAll AllRegsMask::operator&(const regNumber reg) @@ -14192,15 +14192,19 @@ void AllRegsMask::AddRegNumInMask(regNumber reg) regMaskOnlyOne regMask = genRegMask(reg); registers[regIndexForRegister(reg)] |= regMask; } -#endif -// -//void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) -//{ -// //TODO: All these methods should be revisited to see if they should be operated on `type` or `reg` -// regMaskOnlyOne regMask = genRegMask(reg, type); -// AddRegMaskForType(regMask, type); -//} +void AllRegsMask::RemoveRegNumFromMask(regNumber reg) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg); + registers[regIndexForRegister(reg)] &= ~regMaskToRemove; +} + +bool AllRegsMask::IsRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMaskToCheck = genRegMask(reg); + return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; +} +#endif // ---------------------------------------------------------- // AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except @@ -14229,20 +14233,6 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; } -#ifdef TARGET_ARM -void AllRegsMask::RemoveRegNumFromMask(regNumber reg) -{ - regMaskOnlyOne regMaskToRemove = genRegMask(reg); - registers[regIndexForRegister(reg)] &= ~regMaskToRemove; -} - -bool AllRegsMask::IsRegNumInMask(regNumber reg) -{ - regMaskOnlyOne regMaskToCheck = genRegMask(reg); - return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; -} -#endif - bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const { return (gprRegs() & maskToCheck) != RBM_NONE; @@ -14252,12 +14242,6 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const { return (floatRegs() & maskToCheck) != RBM_NONE; } -// -//bool AllRegsMask::IsOnlyRegNumInMask(regNumber reg) -//{ -// regMaskOnlyOne regMask = genRegMask(reg); -// return (registers[regIndexForRegister(reg)] & regMask) == regMask; -//} regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { @@ -14271,12 +14255,12 @@ regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const regMaskTP AllRegsMask::GetGprFloatCombinedMask() const { - //TODO: NEed to revert this once we change floatRegs to 32-bits -//#ifdef TARGET_64BIT -// return ((regMaskTP)floatRegs() << 32) | gprRegs(); -//#else -// return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); -//#endif + // TODO: NEed to revert this once we change floatRegs to 32-bits + //#ifdef TARGET_64BIT + // return ((regMaskTP)floatRegs() << 32) | gprRegs(); + //#else + // return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); + //#endif // TODO: Cannot get RBM_ALLFLOAT because it is not defined for AllRegsMask // Moving this definition to lsra might work, but this is temporary so // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index d628ab774cec6b..c4a1b8c02d438f 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1025,7 +1025,7 @@ class LinearScan : public LinearScanInterface // Record variable locations at start/end of block void processBlockStartLocations(BasicBlock* current); void processBlockEndLocations(BasicBlock* current); - void resetAllRegistersState(); + void resetAllRegistersState(); FORCEINLINE void updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap); #ifdef TARGET_ARM @@ -1744,14 +1744,14 @@ class LinearScan : public LinearScanInterface VARSET_TP largeVectorCalleeSaveCandidateVars; #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE -//----------------------------------------------------------------------- -// Register status -//----------------------------------------------------------------------- + //----------------------------------------------------------------------- + // Register status + //----------------------------------------------------------------------- -// TODO: One option is to also just have another current_AvailableREgs that -// gets reset for every refposition we are processing depending on the -// register type. That wawy we do not have to query and fetch the appropriate -// entry again and agin. + // TODO: One option is to also just have another current_AvailableREgs that + // gets reset for every refposition we are processing depending on the + // register type. That wawy we do not have to query and fetch the appropriate + // entry again and agin. AllRegsMask m_AvailableRegs; regNumber getRegForType(regNumber reg, var_types regType) @@ -1783,9 +1783,10 @@ class LinearScan : public LinearScanInterface { m_AvailableRegs = AllRegsMask(availableIntRegs, availableFloatRegs #ifdef HAS_PREDICATE_REGS - , availableMaskRegs + , + availableMaskRegs #endif - ); + ); m_RegistersWithConstants.Clear(); } @@ -1812,7 +1813,6 @@ class LinearScan : public LinearScanInterface #ifdef HAS_PREDICATE_REGS assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs())); #endif - } void makeRegAvailable(regNumber reg, var_types regType) { @@ -1851,7 +1851,7 @@ class LinearScan : public LinearScanInterface } bool isRegConstant(regNumber reg, var_types regType) { - reg = getRegForType(reg, regType); + reg = getRegForType(reg, regType); return m_RegistersWithConstants.IsRegNumInMask(reg ARM_ARG(regType)); } regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); @@ -2082,7 +2082,7 @@ class LinearScan : public LinearScanInterface #endif // FEATURE_ARG_SPLIT int BuildLclHeap(GenTree* tree); -//#if defined(TARGET_AMD64) + //#if defined(TARGET_AMD64) regMaskFloat rbmAllFloat; regMaskFloat rbmFltCalleeTrash; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index e6c35f0765ac88..ad2089a7516a1d 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -4122,8 +4122,9 @@ int LinearScan::BuildReturn(GenTree* tree) RegisterType dstType = regType(retTypeDesc.GetReturnRegType(i)); if (srcType != dstType) { - hasMismatchedRegTypes = true; - regMaskOnlyOne dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); + hasMismatchedRegTypes = true; + regMaskOnlyOne dstRegMask = + genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); if (varTypeUsesIntReg(dstType)) { diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index c8f30638735b8d..66d71a04192ca9 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -271,9 +271,9 @@ void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) DEBUGARG(regMaskOnlyOne calleeSaveMask)) { - //TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask - //assert(modifiedMask != RBM_NONE); + // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index c23be4bf3ed13c..a747f270862dc3 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -253,7 +253,7 @@ typedef unsigned __int64 singleRegMask; #else // x86 and arm -typedef unsigned regMaskTP; +typedef unsigned regMaskTP; #define regMaskGpr regMaskTP #define regMaskFloat regMaskTP #define regMaskPredicate regMaskTP @@ -271,7 +271,7 @@ typedef struct _regMaskAll { private: regMaskTP registers[REGISTER_TYPE_COUNT]; - regMaskOnlyOne operator[](int index) const; + regMaskOnlyOne operator[](int index) const; regMaskOnlyOne& operator[](int index); public: @@ -301,12 +301,11 @@ typedef struct _regMaskAll { } - - _regMaskAll() : registers{RBM_NONE, RBM_NONE #ifdef HAS_PREDICATE_REGS - , RBM_NONE + , + RBM_NONE #endif } { @@ -331,52 +330,50 @@ typedef struct _regMaskAll AddRegNumInMask(reg ARM_ARG(type)); } - void Clear(); - bool IsEmpty(); + void Clear(); + bool IsEmpty(); unsigned Count(); - void Create(regNumber reg); + void Create(regNumber reg); // Rename this to AddRegNum void AddGprRegInMask(regNumber reg); - void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); + void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); - void AddGprRegMask(regMaskGpr maskToAdd); - void AddFloatRegMask(regMaskFloat maskToAdd); + void AddGprRegMask(regMaskGpr maskToAdd); + void AddFloatRegMask(regMaskFloat maskToAdd); #ifdef TARGET_ARM void AddRegNumInMask(regNumber reg); - void RemoveRegNumFromMask(regNumber reg); + void RemoveRegNumFromMask(regNumber reg); bool IsRegNumInMask(regNumber reg); #endif - void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); + bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); bool IsGprMaskPresent(regMaskGpr maskToCheck) const; - bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; - //bool IsOnlyRegNumInMask(regNumber reg); + bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; + // bool IsOnlyRegNumInMask(regNumber reg); regMaskOnlyOne GetRegMaskForType(var_types type) const; regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field bool IsGprOrFloatPresent() const; - regMaskTP GetGprFloatCombinedMask() const; - + regMaskTP GetGprFloatCombinedMask() const; void operator|=(const _regMaskAll& other); void operator&=(const _regMaskAll& other); void operator|=(const regNumber reg); void operator^=(const regNumber reg); - _regMaskAll operator~(); + _regMaskAll operator~(); bool operator==(const _regMaskAll& other); - bool operator!=(const _regMaskAll& other); - _regMaskAll operator&(const _regMaskAll& other); - _regMaskAll operator|(const _regMaskAll& other); + bool operator!=(const _regMaskAll& other); + _regMaskAll operator&(const _regMaskAll& other); + _regMaskAll operator|(const _regMaskAll& other); _regMaskAll operator&(const regNumber reg); } AllRegsMask; - #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) -#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs)3 +#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) 3 #define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask((gprRegs & ~RBM_ALLFLOAT), (floatRegs & RBM_ALLFLOAT)) @@ -390,9 +387,6 @@ typedef unsigned __int64 regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif - - - /*****************************************************************************/ #ifdef DEBUG diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 10ef8130c2d7ec..d5698faabd6a1b 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -119,7 +119,7 @@ const int regIndexForRegister(regNumber reg) #else #define REGDEF(name, rnum, mask, sname, regTypeTag) regTypeTag, #endif - #include "register.h" +#include "register.h" }; return _registerTypeIndex[reg]; From 679cca8bafa03b6398a83a376c5e65182534ac87 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 19 Mar 2024 09:39:50 -0700 Subject: [PATCH 108/201] Make the IsGprRegMask() and IsFloatRegMask() checks more robust --- src/coreclr/jit/compiler.h | 6 +++--- src/coreclr/jit/lsra.cpp | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 645064d33737d7..99599c1430cd24 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8512,18 +8512,18 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool IsGprRegMask(regMaskTP regMask) { - return (regMask & ~RBM_ALLFLOAT) == regMask; + return (regMask & RBM_ALLFLOAT) == RBM_NONE; } bool IsFloatRegMask(regMaskTP regMask) { - return (regMask & RBM_ALLFLOAT) == regMask; + return (regMask & ~RBM_ALLFLOAT) == RBM_NONE; } #ifdef HAS_PREDICATE_REGS bool IsPredicateRegMask(regMaskTP regMask) { - return (regMask & RBM_ALLMASK) == regMask; + return (regMask & ~RBM_ALLMASK) == RBM_NONE; } #endif // HAS_PREDICATE_REGS diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index a98c00d6ae045a..d70acf83035a91 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -14163,7 +14163,6 @@ _regMaskAll AllRegsMask::operator&(const regNumber reg) return result; } -// TODO: Can have version of AddGprMask(regMaskOnlyOne maskToAdd) and AddFloatMask(regMaskOnlyOne maskToAdd) void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { registers[regIndexForType(type)] |= maskToAdd; From 6e48e2a76225f3784ce4be76d23d5522f76cdf6c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 19 Mar 2024 10:44:31 -0700 Subject: [PATCH 109/201] Mark bunch of methods as FORCEINLINE, added TODO for others --- src/coreclr/jit/target.h | 51 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a747f270862dc3..006cb844e34adf 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -267,6 +267,7 @@ unsigned genCountBits(uint64_t bits); typedef _regNumber_enum regNumber; typedef unsigned char regNumberSmall; +//TODO: Need to make all methods FORCEINLINE typedef struct _regMaskAll { private: @@ -330,51 +331,49 @@ typedef struct _regMaskAll AddRegNumInMask(reg ARM_ARG(type)); } - void Clear(); - bool IsEmpty(); + FORCEINLINE void Clear(); + FORCEINLINE bool IsEmpty(); unsigned Count(); - void Create(regNumber reg); + FORCEINLINE void Create(regNumber reg); // Rename this to AddRegNum - void AddGprRegInMask(regNumber reg); + FORCEINLINE void AddGprRegInMask(regNumber reg); void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); - void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); + FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); void AddGprRegMask(regMaskGpr maskToAdd); - void AddFloatRegMask(regMaskFloat maskToAdd); + FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); #ifdef TARGET_ARM void AddRegNumInMask(regNumber reg); - void RemoveRegNumFromMask(regNumber reg); - bool IsRegNumInMask(regNumber reg); + FORCEINLINE void RemoveRegNumFromMask(regNumber reg); + FORCEINLINE bool IsRegNumInMask(regNumber reg); #endif - void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + FORCEINLINE void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); + FORCEINLINE bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); bool IsGprMaskPresent(regMaskGpr maskToCheck) const; bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; // bool IsOnlyRegNumInMask(regNumber reg); - regMaskOnlyOne GetRegMaskForType(var_types type) const; + FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field - bool IsGprOrFloatPresent() const; - regMaskTP GetGprFloatCombinedMask() const; - - void operator|=(const _regMaskAll& other); - void operator&=(const _regMaskAll& other); - void operator|=(const regNumber reg); - void operator^=(const regNumber reg); - _regMaskAll operator~(); - bool operator==(const _regMaskAll& other); - bool operator!=(const _regMaskAll& other); - _regMaskAll operator&(const _regMaskAll& other); - _regMaskAll operator|(const _regMaskAll& other); - _regMaskAll operator&(const regNumber reg); + FORCEINLINE bool IsGprOrFloatPresent() const; + FORCEINLINE regMaskTP GetGprFloatCombinedMask() const; + + FORCEINLINE void operator|=(const _regMaskAll& other); + FORCEINLINE void operator&=(const _regMaskAll& other); + FORCEINLINE void operator|=(const regNumber reg); + FORCEINLINE void operator^=(const regNumber reg); + FORCEINLINE _regMaskAll operator~(); + bool operator==(const _regMaskAll& other); + FORCEINLINE bool operator!=(const _regMaskAll& other); + FORCEINLINE _regMaskAll operator&(const _regMaskAll& other); + FORCEINLINE _regMaskAll operator|(const _regMaskAll& other); + FORCEINLINE _regMaskAll operator&(const regNumber reg); } AllRegsMask; #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) -#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) 3 - #define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask((gprRegs & ~RBM_ALLFLOAT), (floatRegs & RBM_ALLFLOAT)) #if REGMASK_BITS == 32 From e6e226946f0fdae041191dcc61feb03544e78427 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 10:34:07 -0700 Subject: [PATCH 110/201] Revert "Mark bunch of methods as FORCEINLINE, added TODO for others" This reverts commit 6e48e2a76225f3784ce4be76d23d5522f76cdf6c. --- src/coreclr/jit/target.h | 51 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 006cb844e34adf..a747f270862dc3 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -267,7 +267,6 @@ unsigned genCountBits(uint64_t bits); typedef _regNumber_enum regNumber; typedef unsigned char regNumberSmall; -//TODO: Need to make all methods FORCEINLINE typedef struct _regMaskAll { private: @@ -331,49 +330,51 @@ typedef struct _regMaskAll AddRegNumInMask(reg ARM_ARG(type)); } - FORCEINLINE void Clear(); - FORCEINLINE bool IsEmpty(); + void Clear(); + bool IsEmpty(); unsigned Count(); - FORCEINLINE void Create(regNumber reg); + void Create(regNumber reg); // Rename this to AddRegNum - FORCEINLINE void AddGprRegInMask(regNumber reg); + void AddGprRegInMask(regNumber reg); void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); - FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); + void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); void AddGprRegMask(regMaskGpr maskToAdd); - FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); + void AddFloatRegMask(regMaskFloat maskToAdd); #ifdef TARGET_ARM void AddRegNumInMask(regNumber reg); - FORCEINLINE void RemoveRegNumFromMask(regNumber reg); - FORCEINLINE bool IsRegNumInMask(regNumber reg); + void RemoveRegNumFromMask(regNumber reg); + bool IsRegNumInMask(regNumber reg); #endif - FORCEINLINE void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - FORCEINLINE bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); + bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); bool IsGprMaskPresent(regMaskGpr maskToCheck) const; bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; // bool IsOnlyRegNumInMask(regNumber reg); - FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; + regMaskOnlyOne GetRegMaskForType(var_types type) const; regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field - FORCEINLINE bool IsGprOrFloatPresent() const; - FORCEINLINE regMaskTP GetGprFloatCombinedMask() const; - - FORCEINLINE void operator|=(const _regMaskAll& other); - FORCEINLINE void operator&=(const _regMaskAll& other); - FORCEINLINE void operator|=(const regNumber reg); - FORCEINLINE void operator^=(const regNumber reg); - FORCEINLINE _regMaskAll operator~(); - bool operator==(const _regMaskAll& other); - FORCEINLINE bool operator!=(const _regMaskAll& other); - FORCEINLINE _regMaskAll operator&(const _regMaskAll& other); - FORCEINLINE _regMaskAll operator|(const _regMaskAll& other); - FORCEINLINE _regMaskAll operator&(const regNumber reg); + bool IsGprOrFloatPresent() const; + regMaskTP GetGprFloatCombinedMask() const; + + void operator|=(const _regMaskAll& other); + void operator&=(const _regMaskAll& other); + void operator|=(const regNumber reg); + void operator^=(const regNumber reg); + _regMaskAll operator~(); + bool operator==(const _regMaskAll& other); + bool operator!=(const _regMaskAll& other); + _regMaskAll operator&(const _regMaskAll& other); + _regMaskAll operator|(const _regMaskAll& other); + _regMaskAll operator&(const regNumber reg); } AllRegsMask; #define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) +#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) 3 + #define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask((gprRegs & ~RBM_ALLFLOAT), (floatRegs & RBM_ALLFLOAT)) #if REGMASK_BITS == 32 From 6af17c4b74bb1497cc08682c25f8a1abf5c178b5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 13:24:24 -0700 Subject: [PATCH 111/201] Change RegSet to use AllRegMask branch-free --- src/coreclr/jit/codegencommon.cpp | 13 +- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/lsra.cpp | 31 +--- src/coreclr/jit/lsra.h | 1 - src/coreclr/jit/lsrabuild.cpp | 29 +--- src/coreclr/jit/regset.cpp | 252 +++++++++++++----------------- src/coreclr/jit/regset.h | 61 ++++---- src/coreclr/jit/target.h | 4 +- 8 files changed, 152 insertions(+), 241 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 400fa676cbba29..7dc865d3534c50 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -3798,7 +3798,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, srcRegNum, /* canSkip */ false); assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(srcRegNum)); - regSet.verifyRegUsed(destRegNum, destMemType); + regSet.verifyRegUsed(destRegNum); /* mark 'src' as processed */ noway_assert(srcReg < argMax); @@ -3850,7 +3850,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, xtraReg, /* canSkip */ false); assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(xtraReg)); - regSet.verifyRegUsed(destRegNum, destMemType); + regSet.verifyRegUsed(destRegNum); /* mark the beginning register as processed */ regArgTab[srcReg].processed = true; @@ -4305,7 +4305,7 @@ void CodeGen::genEnregisterIncomingStackArgs() GetEmitter()->emitIns_R_S(ins_Load(regType), emitTypeSize(regType), regNum, varNum, 0); #endif // !TARGET_LOONGARCH64 - regSet.verifyRegUsed(regNum, regType); + regSet.verifyRegUsed(regNum); } } @@ -5342,12 +5342,7 @@ void CodeGen::genFinalizeFrame() if (verbose) { printf("Modified regs: "); - dspRegMask(AllRegsMask(regSet.rsGetModifiedGprRegsMask(), regSet.rsGetModifiedFloatRegsMask() -#ifdef HAS_PREDICATE_REGS - , - regSet.rsGetModifiedPredicateRegsMask() -#endif - )); + dspRegMask(regSet.rsGetModifiedRegsMask()); printf("\n"); } #endif // DEBUG diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index c357ef9c990606..d7c8b6073600fa 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1143,7 +1143,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) { var_types op1Type = op1->TypeGet(); inst_Mov(op1Type, targetReg, op1reg, /* canSkip */ false); - regSet.verifyRegUsed(targetReg, targetType); + regSet.verifyRegUsed(targetReg); gcInfo.gcMarkRegPtrVal(targetReg, op1Type); dst = treeNode; src = op2; diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index d70acf83035a91..dc414103f2bf04 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -3508,27 +3508,12 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - regNumber reg = regRec->regNum; - singleRegMask assignedRegMask = genRegMask(reg); + regNumber reg = regRec->regNum; // TODO: genIsValid* is heavy operation because it does 2 comparisons, but unfortunately // we cannot use `varTypeRegister[interval->registerType] == VTR_INT` because sometimes // we do `vmovd xmm1, rcx` where we assign gpr for interval of type TYP_SIMD8 - if (genIsValidIntReg(reg)) - { - compiler->codeGen->regSet.rsSetGprRegsModified(assignedRegMask DEBUGARG(true)); - } -#ifdef HAS_PREDICATE_REGS - else if (genIsValidMaskReg(reg)) - { - compiler->codeGen->regSet.rsSetPredicateRegsModified(assignedRegMask DEBUGARG(true)); - } -#endif - else - { - assert(genIsValidFloatReg(reg)); - compiler->codeGen->regSet.rsSetFloatRegsModified(assignedRegMask DEBUGARG(true)); - } + compiler->codeGen->regSet.rsSetRegModified(reg DEBUGARG(true)); interval->assignedReg = regRec; checkAndAssignInterval(regRec, interval); @@ -3538,7 +3523,7 @@ void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) if (interval->isLocalVar) { // Prefer this register for future references - interval->updateRegisterPreferences(assignedRegMask); + interval->updateRegisterPreferences(genRegMask(reg)); } } @@ -10019,15 +10004,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, } else { - if (emitter::isFloatReg(tempReg)) - { - compiler->codeGen->regSet.rsSetFloatRegsModified(genRegMask(tempReg) DEBUGARG(true)); - } - else - { - assert(emitter::isGeneralRegister(tempReg)); - compiler->codeGen->regSet.rsSetGprRegsModified(genRegMask(tempReg) DEBUGARG(true)); - } + compiler->codeGen->regSet.rsSetRegModified(tempReg DEBUGARG(true)); #ifdef TARGET_ARM if (sourceIntervals[fromReg]->registerType == TYP_DOUBLE) { diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index c4a1b8c02d438f..bbffbe4bfc1aac 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1050,7 +1050,6 @@ class LinearScan : public LinearScanInterface // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions void addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse); - void addRefsForPhysRegMask(regMaskOnlyOne mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ad2089a7516a1d..a49a69a5545458 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -704,30 +704,7 @@ void LinearScan::addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc { assert(refType == RefTypeKill); - compiler->codeGen->regSet.rsSetGprRegsModified(mask.gprRegs() DEBUGARG(true)); - compiler->codeGen->regSet.rsSetFloatRegsModified(mask.floatRegs() DEBUGARG(true)); - - addRefsForPhysRegMask(mask.gprRegs(), currentLoc, refType, isLastUse); - addRefsForPhysRegMask(mask.floatRegs(), currentLoc, refType, isLastUse); - -#ifdef HAS_PREDICATE_REGS - compiler->codeGen->regSet.rsSetPredicateRegsModified(mask.predicateRegs() DEBUGARG(true)); - addRefsForPhysRegMask(mask.predicateRegs(), currentLoc, refType, isLastUse); -#endif -} - -//------------------------------------------------------------------------ -// addRefsForPhysRegMask: Adds RefPositions of the given type for specific registers in 'mask'. -// -// Arguments: -// mask - the mask (set) of registers - either gpr/float/predicate -// currentLoc - the location at which they should be added -// refType - the type of refposition -// isLastUse - true IFF this is a last use of the register -// -void LinearScan::addRefsForPhysRegMask(regMaskOnlyOne mask, LsraLocation currentLoc, RefType refType, bool isLastUse) -{ - assert(compiler->IsOnlyOneRegMask(mask)); + compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); // The mask identifies a set of registers that will be used during // codegen. Mark these as modified here, so when we do final frame @@ -738,9 +715,9 @@ void LinearScan::addRefsForPhysRegMask(regMaskOnlyOne mask, LsraLocation current // CORINFO_HELP_ASSIGN_BYREF helper, which kills callee-saved RSI and // RDI, if LSRA doesn't assign RSI/RDI, they wouldn't get marked as // modified until codegen, which is too late. - for (regMaskOnlyOne candidates = mask; candidates != RBM_NONE;) + while (!mask.IsEmpty()) { - regNumber reg = genFirstRegNumFromMaskAndToggle(candidates); + regNumber reg = genFirstRegNumFromMaskAndToggle(mask); // This assumes that these are all "special" RefTypes that // don't need to be recorded on the tree (hence treeNode is nullptr) RefPosition* pos = newRefPosition(reg, currentLoc, refType, nullptr, diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 66d71a04192ca9..efdb2e69f8787f 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -68,91 +68,6 @@ void RegSet::verifyGprRegUsed(regNumber reg) rsSetGprRegsModified(genRegMask(reg)); } -//------------------------------------------------------------------------ -// verifyFloatRegUsed: verify that the Float register is marked as used. -// -// Arguments: -// reg - The register to verify. -// -// Return Value: -// None. -// -// Assumptions: -// The caller must have ensured that the register is already marked -// as used. -// -// Notes: -// This method is intended to be called during code generation, and -// should simply validate that the register (or registers) have -// already been added to the modified set. -// -void RegSet::verifyFloatRegUsed(regNumber reg) -{ - assert(emitter::isFloatReg(reg)); - rsSetFloatRegsModified(genRegMask(reg)); -} - -#ifdef HAS_PREDICATE_REGS -//------------------------------------------------------------------------ -// verifyPredicateRegUsed: verify that the mask register is marked as used. -// -// Arguments: -// reg - The register to verify. -// -// Return Value: -// None. -// -// Assumptions: -// The caller must have ensured that the register is already marked -// as used. -// -// Notes: -// This method is intended to be called during code generation, and -// should simply validate that the register (or registers) have -// already been added to the modified set. -void RegSet::verifyPredicateRegUsed(regNumber reg) -{ - assert(emitter::isMaskReg(reg)); - rsSetPredicateRegsModified(genRegMask(reg)); -} -#endif // HAS_PREDICATE_REGS - -//------------------------------------------------------------------------ -// verifyRegUsed: verify that the mask register is marked as used. -// -// Arguments: -// reg - The register to verify. -// -// Return Value: -// None. -// -// Assumptions: -// The caller must have ensured that the register is already marked -// as used. -// -// Notes: -// This method is intended to be called during code generation, and -// should simply validate that the register (or registers) have -// already been added to the modified set. -void RegSet::verifyRegUsed(regNumber reg, var_types type) -{ - if (varTypeUsesIntReg(type)) - { - verifyGprRegUsed(reg); - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - verifyPredicateRegUsed(reg); - } -#endif // HAS_PREDICATE_REGS - else - { - assert(varTypeUsesFloatReg(type)); - verifyFloatRegUsed(reg); - } -} - //------------------------------------------------------------------------ // verifyRegUsed: verify that the mask register is marked as used. // @@ -172,21 +87,7 @@ void RegSet::verifyRegUsed(regNumber reg, var_types type) // already been added to the modified set. void RegSet::verifyRegUsed(regNumber reg) { - if (emitter::isGeneralRegister(reg)) - { - verifyGprRegUsed(reg); - } -#ifdef HAS_PREDICATE_REGS - else if (emitter::isMaskReg(reg)) - { - verifyPredicateRegUsed(reg); - } -#endif // HAS_PREDICATE_REGS - else - { - assert(emitter::isFloatReg(reg)); - verifyFloatRegUsed(reg); - } + rsSetRegModified(reg); } //------------------------------------------------------------------------ @@ -214,6 +115,11 @@ void RegSet::verifyRegistersUsed(AllRegsMask regs) return; } + if (regs.IsEmpty()) + { + return; + } + // TODO-Cleanup: // We need to identify the places where the register // is not marked as used when this is called. @@ -222,12 +128,7 @@ void RegSet::verifyRegistersUsed(AllRegsMask regs) // https://github.com/dotnet/coreclr/pull/18230 on why we call // rsSetGprRegsModified() instead of assert(rsRegsModified()) - rsSetGprRegsModified(regs.gprRegs()); - rsSetFloatRegsModified(regs.floatRegs()); - -#ifdef HAS_PREDICATE_REGS - rsSetPredicateRegsModified(regs.predicateRegs()); -#endif + rsSetRegsModified(regs); } void RegSet::rsClearRegsModified() @@ -242,30 +143,119 @@ void RegSet::rsClearRegsModified() rsModifiedRegsMaskInitialized = true; #endif // DEBUG - rsModifiedGprRegsMask = RBM_NONE; - rsModifiedFloatRegsMask = RBM_NONE; - rsModifiedPredicateRegsMask = RBM_NONE; + rsModifiedRegsMask.Clear(); + // rsModifiedGprRegsMask = RBM_NONE; + // rsModifiedFloatRegsMask = RBM_NONE; + // rsModifiedPredicateRegsMask = RBM_NONE; } +#ifdef DEBUG +void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, + regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) + DEBUGARG(regMaskOnlyOne calleeSaveMask)) const +{ + regMaskOnlyOne newMask = (currentMask | modifiedMask); + + // We can't update the modified registers set after final frame layout (that is, during code + // generation and after). Ignore prolog and epilog generation: they call register tracking to + // modify rbp, for example, even in functions that use rbp as a frame pointer. Make sure normal + // code generation isn't actually adding to set of modified registers. + // Frame layout is only affected by callee-saved registers, so only ensure that callee-saved + // registers aren't modified after final frame layout. + assert((m_rsCompiler->lvaDoneFrameLayout < Compiler::FINAL_FRAME_LAYOUT) || m_rsCompiler->compGeneratingProlog || + m_rsCompiler->compGeneratingEpilog || ((newMask & calleeSaveMask) == (currentMask & calleeSaveMask))); + + if (m_rsCompiler->verbose && !suppressDump) + { + if (currentMask != newMask) + { + printf("Marking regs modified: "); + dspRegMask(modifiedMask); + printf(" ("); + dspRegMask(currentMask); + printf(" => "); + dspRegMask(newMask); + printf(")\n"); + } + } +} +#endif + void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsGprRegMask(mask)); - rsSetRegsModified(rsModifiedGprRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_INT_CALLEE_SAVED)); + // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask + // assert(modifiedMask != RBM_NONE); + assert(rsModifiedRegsMaskInitialized); +#ifdef DEBUG + printModifiedRegsMask(rsModifiedRegsMask.gprRegs(), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); +#endif + + rsModifiedRegsMask.AddGprRegMask(mask); } void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsFloatRegMask(mask)); - rsSetRegsModified(rsModifiedFloatRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_FLT_CALLEE_SAVED)); + // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask + // assert(modifiedMask != RBM_NONE); + assert(rsModifiedRegsMaskInitialized); +#ifdef DEBUG + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); +#endif + + rsModifiedRegsMask.AddFloatRegMask(mask); } -#ifdef HAS_PREDICATE_REGS -void RegSet::rsSetPredicateRegsModified(regMaskPredicate mask DEBUGARG(bool suppressDump)) +void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) { - assert(m_rsCompiler->IsPredicateRegMask(mask)); - rsSetRegsModified(rsModifiedPredicateRegsMask, mask DEBUGARG(suppressDump) DEBUGARG(RBM_MSK_CALLEE_SAVED)); + // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask + // assert(modifiedMask != RBM_NONE); + assert(rsModifiedRegsMaskInitialized); + +#ifdef DEBUG + if (genIsValidIntReg(reg)) + { + printModifiedRegsMask(rsModifiedRegsMask.gprRegs(), + genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); + } + else if (genIsValidFloatReg(reg)) + { + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), + genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); + } + else + { + assert(genIsValidMaskReg(reg)); + printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), + genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); + } +#endif // DEBUG + + rsModifiedRegsMask.AddRegNumInMask(reg); +} + +void RegSet::rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump)) +{ + // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE + // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask + // assert(modifiedMask != RBM_NONE); + assert(rsModifiedRegsMaskInitialized); + +#ifdef DEBUG + printModifiedRegsMask(rsModifiedRegsMask.gprRegs(), + modifiedMask.gprRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), + modifiedMask.floatRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), + modifiedMask.predicateRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); +#endif // DEBUG + + rsModifiedRegsMask |= modifiedMask; } -#endif // HAS_PREDICATE_REGS void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) @@ -277,30 +267,7 @@ void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG - - regMaskOnlyOne newMask = (trackingMask | modifiedMask); - - // We can't update the modified registers set after final frame layout (that is, during code - // generation and after). Ignore prolog and epilog generation: they call register tracking to - // modify rbp, for example, even in functions that use rbp as a frame pointer. Make sure normal - // code generation isn't actually adding to set of modified registers. - // Frame layout is only affected by callee-saved registers, so only ensure that callee-saved - // registers aren't modified after final frame layout. - assert((m_rsCompiler->lvaDoneFrameLayout < Compiler::FINAL_FRAME_LAYOUT) || m_rsCompiler->compGeneratingProlog || - m_rsCompiler->compGeneratingEpilog || ((newMask & calleeSaveMask) == (trackingMask & calleeSaveMask))); - if (m_rsCompiler->verbose && !suppressDump) - { - if (trackingMask != newMask) - { - printf("Marking regs modified: "); - dspRegMask(modifiedMask); - printf(" ("); - dspRegMask(trackingMask); - printf(" => "); - dspRegMask(newMask); - printf(")\n"); - } - } + printModifiedRegsMask(trackingMask, modifiedMask DEBUG_ARG(suppressDump) DEBUG_ARG(calleeSaveMask)); #endif // DEBUG trackingMask |= modifiedMask; @@ -312,12 +279,13 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) assert(rsModifiedRegsMaskInitialized); assert(m_rsCompiler->IsGprRegMask(mask)); +#ifdef DEBUG + regMaskGpr rsModifiedGprRegsMask = rsGetModifiedGprRegsMask(); // See comment in rsSetRegsModified(). assert((m_rsCompiler->lvaDoneFrameLayout < Compiler::FINAL_FRAME_LAYOUT) || m_rsCompiler->compGeneratingProlog || m_rsCompiler->compGeneratingEpilog || (((rsModifiedGprRegsMask & ~mask) & RBM_CALLEE_SAVED) == (rsModifiedGprRegsMask & RBM_CALLEE_SAVED))); -#ifdef DEBUG if (m_rsCompiler->verbose) { printf("Removing modified regs: "); @@ -338,7 +306,7 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) } #endif // DEBUG - rsModifiedGprRegsMask &= ~mask; + rsModifiedRegsMask.RemoveRegTypeFromMask(mask, TYP_INT); } void RegSet::SetMaskVars(AllRegsMask newMaskVars) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 7446d7ee8474cc..0eab0ca558177d 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -67,78 +67,71 @@ class RegSet // private: - bool rsNeededSpillReg; // true if this method needed to spill any registers - regMaskGpr rsModifiedGprRegsMask; // mask of the registers modified by the current function. - regMaskFloat rsModifiedFloatRegsMask; // mask of the registers modified by the current function. - regMaskPredicate rsModifiedPredicateRegsMask; // mask of the registers modified by the current function. + bool rsNeededSpillReg; // true if this method needed to spill any registers + AllRegsMask rsModifiedRegsMask; // mask of the registers modified by the current function. FORCEINLINE void rsSetRegsModified(regMaskOnlyOne& trackingMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)); - #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. -#endif // DEBUG + void printModifiedRegsMask(regMaskOnlyOne currentMask, + regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) + DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; +#endif // DEBUG public: - regMaskGpr rsGetModifiedRegsMask(var_types type) const + FORCEINLINE void rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump = false)); + FORCEINLINE void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); + +#ifdef DEBUG + AllRegsMask rsGetModifiedRegsMask() const { - if (varTypeUsesIntReg(type)) - { - return rsGetModifiedGprRegsMask(); - } -#ifdef HAS_PREDICATE_REGS - else if (varTypeUsesMaskReg(type)) - { - return rsGetModifiedPredicateRegsMask(); - } -#endif // HAS_PREDICATE_REGS - else - { - assert(varTypeUsesFloatReg(type)); - return rsGetModifiedFloatRegsMask(); - } + assert(rsModifiedRegsMaskInitialized); + return rsModifiedRegsMask; } regMaskGpr rsGetModifiedGprRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedGprRegsMask; + return rsModifiedRegsMask.gprRegs(); } regMaskFloat rsGetModifiedFloatRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedFloatRegsMask; + return rsModifiedRegsMask.floatRegs(); } +#ifdef HAS_PREDICATE_REGS regMaskPredicate rsGetModifiedPredicateRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedPredicateRegsMask; + return rsModifiedRegsMask.predicateRegs(); + } +#endif // HAS_PREDICATE_REGS + +#endif + + regMaskGpr rsGetModifiedRegsMask(var_types type) const + { + return rsModifiedRegsMask.GetRegMaskForType(type); } void rsClearRegsModified(); void rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump = false)); void rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump = false)); -#ifdef HAS_PREDICATE_REGS - void rsSetPredicateRegsModified(regMaskPredicate mask DEBUGARG(bool suppressDump = false)); -#endif // HAS_PREDICATE_REGS void rsRemoveRegsModified(regMaskGpr mask); bool rsRegsModified(regMaskGpr mask) const { assert(rsModifiedRegsMaskInitialized); - return (rsModifiedGprRegsMask & mask) != 0; + return (rsModifiedRegsMask.gprRegs() & mask) != 0; + // return (rsModifiedGprRegsMask & mask) != 0; } void verifyRegUsed(regNumber reg); - void verifyRegUsed(regNumber reg, var_types type); void verifyGprRegUsed(regNumber reg); - void verifyFloatRegUsed(regNumber reg); -#ifdef HAS_PREDICATE_REGS - void verifyPredicateRegUsed(regNumber reg); -#endif // HAS_PREDICATE_REGS void verifyRegistersUsed(AllRegsMask mask); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a747f270862dc3..d76f7437be0db2 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -311,11 +311,13 @@ typedef struct _regMaskAll { } - _regMaskAll(int (&_registers)[3]) + _regMaskAll(int (&_registers)[REGISTER_TYPE_COUNT]) { registers[0] = _registers[0]; registers[1] = _registers[1]; +#ifdef HAS_PREDICATE_REGS registers[2] = _registers[2]; +#endif } #ifdef TARGET_ARM From ce78527aff8a2734248351f28eeff82ee614ac4f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 13:47:00 -0700 Subject: [PATCH 112/201] Misc change around rsRegsModified --- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/gcencode.cpp | 4 ++-- src/coreclr/jit/lsra.cpp | 6 +++--- src/coreclr/jit/regset.h | 6 ++++++ src/coreclr/jit/target.h | 6 +++--- src/coreclr/jit/targetamd64.h | 3 ++- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index d7c8b6073600fa..c0eca3cc6280ce 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -10229,7 +10229,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) // lea esp, [ebp - compiler->compCalleeRegsPushed * REGSIZE_BYTES] needLea = true; } - else if (!regSet.rsRegsModified(RBM_CALLEE_SAVED)) + else if (!regSet.rsRegsModified(AllRegsMask_CALLEE_SAVED)) { if (compiler->compLclFrameSize != 0) { diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 57a825694a9dcc..5012045ded1291 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -4479,8 +4479,8 @@ void GCInfo::gcMakeRegPtrTable( assert(call->u1.cdArgMask == 0 && call->cdArgCnt == 0); // Other than that, we just have to deal with the regmasks. - regMaskSmall gcrefRegMask = call->cdGCrefRegs & RBM_CALLEE_SAVED; - regMaskSmall byrefRegMask = call->cdByrefRegs & RBM_CALLEE_SAVED; + regMaskSmall gcrefRegMask = call->cdGCrefRegs & RBM_INT_CALLEE_SAVED; + regMaskSmall byrefRegMask = call->cdByrefRegs & RBM_INT_CALLEE_SAVED; assert((gcrefRegMask & byrefRegMask) == 0); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index dc414103f2bf04..ece05117f4343a 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -14113,7 +14113,7 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) return !(*this == other); } -_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) +_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) const { return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() #ifdef HAS_PREDICATE_REGS @@ -14123,7 +14123,7 @@ _regMaskAll AllRegsMask::operator&(const AllRegsMask& other) ); } -_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) +_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) const { return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() #ifdef HAS_PREDICATE_REGS @@ -14133,7 +14133,7 @@ _regMaskAll AllRegsMask::operator|(const _regMaskAll& other) ); } -_regMaskAll AllRegsMask::operator&(const regNumber reg) +_regMaskAll AllRegsMask::operator&(const regNumber reg) const { _regMaskAll result = *this; result[regIndexForRegister(reg)] &= genRegMask(reg); diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 0eab0ca558177d..5c89d31f6e685d 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -130,6 +130,12 @@ class RegSet // return (rsModifiedGprRegsMask & mask) != 0; } + bool rsRegsModified(const AllRegsMask& mask) const + { + assert(rsModifiedRegsMaskInitialized); + return !((rsModifiedRegsMask & mask).IsEmpty()); + } + void verifyRegUsed(regNumber reg); void verifyGprRegUsed(regNumber reg); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index d76f7437be0db2..4a47fdd457d3e2 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -368,9 +368,9 @@ typedef struct _regMaskAll _regMaskAll operator~(); bool operator==(const _regMaskAll& other); bool operator!=(const _regMaskAll& other); - _regMaskAll operator&(const _regMaskAll& other); - _regMaskAll operator|(const _regMaskAll& other); - _regMaskAll operator&(const regNumber reg); + _regMaskAll operator&(const _regMaskAll& other) const; + _regMaskAll operator|(const _regMaskAll& other) const; + _regMaskAll operator&(const regNumber reg) const; } AllRegsMask; diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index d382d6bfb9d41d..9f75b65b55c01b 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -172,7 +172,8 @@ #define REG_FLT_CALLEE_SAVED_LAST REG_XMM15 #define RBM_CALLEE_TRASH (RBM_INT_CALLEE_TRASH | RBM_FLT_CALLEE_TRASH | RBM_MSK_CALLEE_TRASH) - #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) + #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) // This will need to change once floatRegs are changed to 4-bytes + #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) From 314c4642347b6ea2d325398faa7e1224ae5cf5a1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 13:52:31 -0700 Subject: [PATCH 113/201] Make some methods out of #ifdef DEBUG --- src/coreclr/jit/regset.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 5c89d31f6e685d..580450abe5e7a6 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -89,6 +89,7 @@ class RegSet assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; } +#endif regMaskGpr rsGetModifiedGprRegsMask() const { @@ -110,8 +111,6 @@ class RegSet } #endif // HAS_PREDICATE_REGS -#endif - regMaskGpr rsGetModifiedRegsMask(var_types type) const { return rsModifiedRegsMask.GetRegMaskForType(type); From 973e14e6ed79823e4af8c34274f63c87888774ef Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 14:44:46 -0700 Subject: [PATCH 114/201] Pass AllRegsMask& by reference --- src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/lsrabuild.cpp | 6 +++--- src/coreclr/jit/regset.cpp | 20 ++++---------------- src/coreclr/jit/regset.h | 8 +++----- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index bbffbe4bfc1aac..934601eff165ac 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1049,7 +1049,7 @@ class LinearScan : public LinearScanInterface void insertZeroInitRefPositions(); // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions - void addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(AllRegsMask& mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index a49a69a5545458..ba15af59f25fcd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -700,12 +700,10 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(AllRegsMask& mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); - compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); - // The mask identifies a set of registers that will be used during // codegen. Mark these as modified here, so when we do final frame // layout, we'll know about all these registers. This is especially @@ -715,6 +713,8 @@ void LinearScan::addRefsForPhysRegMask(AllRegsMask mask, LsraLocation currentLoc // CORINFO_HELP_ASSIGN_BYREF helper, which kills callee-saved RSI and // RDI, if LSRA doesn't assign RSI/RDI, they wouldn't get marked as // modified until codegen, which is too late. + compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); + while (!mask.IsEmpty()) { regNumber reg = genFirstRegNumFromMaskAndToggle(mask); diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index efdb2e69f8787f..a5e7ee3f2e67cb 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -227,12 +227,14 @@ void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); } +#ifdef HAS_PREDICATE_REGS else { assert(genIsValidMaskReg(reg)); printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); } +#endif // HAS_PREDICATE_REGS #endif // DEBUG rsModifiedRegsMask.AddRegNumInMask(reg); @@ -250,29 +252,15 @@ void RegSet::rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressD modifiedMask.gprRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), modifiedMask.floatRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); +#ifdef HAS_PREDICATE_REGS printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), modifiedMask.predicateRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); +#endif // HAS_PREDICATE_REGS #endif // DEBUG rsModifiedRegsMask |= modifiedMask; } -void RegSet::rsSetRegsModified(regMaskOnlyOne& trackingMask, - regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) - DEBUGARG(regMaskOnlyOne calleeSaveMask)) -{ - // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE - // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask - // assert(modifiedMask != RBM_NONE); - assert(rsModifiedRegsMaskInitialized); - -#ifdef DEBUG - printModifiedRegsMask(trackingMask, modifiedMask DEBUG_ARG(suppressDump) DEBUG_ARG(calleeSaveMask)); -#endif // DEBUG - - trackingMask |= modifiedMask; -} - void RegSet::rsRemoveRegsModified(regMaskGpr mask) { assert(mask != RBM_NONE); diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 580450abe5e7a6..f43276afbc74bf 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -69,9 +69,7 @@ class RegSet private: bool rsNeededSpillReg; // true if this method needed to spill any registers AllRegsMask rsModifiedRegsMask; // mask of the registers modified by the current function. - FORCEINLINE void rsSetRegsModified(regMaskOnlyOne& trackingMask, - regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) - DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)); + #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. void printModifiedRegsMask(regMaskOnlyOne currentMask, @@ -80,8 +78,8 @@ class RegSet #endif // DEBUG public: - FORCEINLINE void rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump = false)); - FORCEINLINE void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); + void rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump = false)); + void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); #ifdef DEBUG AllRegsMask rsGetModifiedRegsMask() const From 4551999755e559405d789878c437b4105bf9f03b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 20 Mar 2024 18:48:38 -0700 Subject: [PATCH 115/201] Fix the byref problem --- src/coreclr/jit/lsrabuild.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ba15af59f25fcd..2dc35cfb0a45da 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -715,9 +715,10 @@ void LinearScan::addRefsForPhysRegMask(AllRegsMask& mask, LsraLocation currentLo // modified until codegen, which is too late. compiler->codeGen->regSet.rsSetRegsModified(mask DEBUGARG(true)); - while (!mask.IsEmpty()) + AllRegsMask maskForRefPositions = mask; + while (!maskForRefPositions.IsEmpty()) { - regNumber reg = genFirstRegNumFromMaskAndToggle(mask); + regNumber reg = genFirstRegNumFromMaskAndToggle(maskForRefPositions); // This assumes that these are all "special" RefTypes that // don't need to be recorded on the tree (hence treeNode is nullptr) RefPosition* pos = newRefPosition(reg, currentLoc, refType, nullptr, From 522fbd84c129ef6d8e5406b9cb8e79541b53f6a3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 21 Mar 2024 13:13:05 -0700 Subject: [PATCH 116/201] Mark AllRegsMask operators as FORCEINLINE --- src/coreclr/jit/compiler.hpp | 84 ++++++++++++++++++++++++++++++++++++ src/coreclr/jit/lsra.cpp | 83 ----------------------------------- src/coreclr/jit/target.h | 20 ++++----- 3 files changed, 94 insertions(+), 93 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 5981b54ecd0f00..0dc0241d140767 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5053,6 +5053,90 @@ BasicBlockVisit FlowGraphNaturalLoop::VisitRegularExitBlocks(TFunc func) return BasicBlockVisit::Continue; } + +void AllRegsMask::operator|=(const _regMaskAll& other) +{ + // TODO: Can we optimize to reintrepret_cast + // Something like https://godbolt.org/z/1KevT8Edh + registers[0] |= other.gprRegs(); + registers[1] |= other.floatRegs(); +#ifdef HAS_PREDICATE_REGS + registers[2] |= other.predicateRegs(); +#endif +} + +void AllRegsMask::operator&=(const _regMaskAll& other) +{ + // TODO: Can we optimize to reintrepret_cast + // Something like https://godbolt.org/z/1KevT8Edh + registers[0] &= other.gprRegs(); + registers[1] &= other.floatRegs(); +#ifdef HAS_PREDICATE_REGS + registers[2] &= other.predicateRegs(); +#endif +} + +void AllRegsMask::operator|=(const regNumber reg) +{ + registers[regIndexForRegister(reg)] |= genRegMask(reg); +} + +void AllRegsMask::operator^=(const regNumber reg) +{ + registers[regIndexForRegister(reg)] ^= genRegMask(reg); +} + +_regMaskAll AllRegsMask::operator~() +{ + return _regMaskAll(~gprRegs(), ~floatRegs() +#ifdef HAS_PREDICATE_REGS + , + ~predicateRegs() +#endif + ); +} + +bool AllRegsMask::operator==(const AllRegsMask& other) +{ + return ((gprRegs() == other.gprRegs()) && (floatRegs() == other.floatRegs()) +#ifdef HAS_PREDICATE_REGS + && (predicateRegs() == other.predicateRegs()) +#endif + ); +} + +bool AllRegsMask::operator!=(const AllRegsMask& other) +{ + return !(*this == other); +} + +_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) const +{ + return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() +#ifdef HAS_PREDICATE_REGS + , + predicateRegs() & other.predicateRegs() +#endif + ); +} + +_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) const +{ + return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() +#ifdef HAS_PREDICATE_REGS + , + predicateRegs() | other.predicateRegs() +#endif + ); +} + +_regMaskAll AllRegsMask::operator&(const regNumber reg) const +{ + _regMaskAll result = *this; + result[regIndexForRegister(reg)] &= genRegMask(reg); + return result; +} + /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ece05117f4343a..9147b3dcd46d07 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -14057,89 +14057,6 @@ regMaskOnlyOne& AllRegsMask::operator[](int index) return registers[index]; } -void AllRegsMask::operator|=(const _regMaskAll& other) -{ - // TODO: Can we optimize to reintrepret_cast - // Something like https://godbolt.org/z/1KevT8Edh - registers[0] |= other.gprRegs(); - registers[1] |= other.floatRegs(); -#ifdef HAS_PREDICATE_REGS - registers[2] |= other.predicateRegs(); -#endif -} - -void AllRegsMask::operator&=(const _regMaskAll& other) -{ - // TODO: Can we optimize to reintrepret_cast - // Something like https://godbolt.org/z/1KevT8Edh - registers[0] &= other.gprRegs(); - registers[1] &= other.floatRegs(); -#ifdef HAS_PREDICATE_REGS - registers[2] &= other.predicateRegs(); -#endif -} - -void AllRegsMask::operator|=(const regNumber reg) -{ - registers[regIndexForRegister(reg)] |= genRegMask(reg); -} - -void AllRegsMask::operator^=(const regNumber reg) -{ - registers[regIndexForRegister(reg)] ^= genRegMask(reg); -} - -_regMaskAll AllRegsMask::operator~() -{ - return _regMaskAll(~gprRegs(), ~floatRegs() -#ifdef HAS_PREDICATE_REGS - , - ~predicateRegs() -#endif - ); -} - -bool AllRegsMask::operator==(const AllRegsMask& other) -{ - return ((gprRegs() == other.gprRegs()) && (floatRegs() == other.floatRegs()) -#ifdef HAS_PREDICATE_REGS - && (predicateRegs() == other.predicateRegs()) -#endif - ); -} - -bool AllRegsMask::operator!=(const AllRegsMask& other) -{ - return !(*this == other); -} - -_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) const -{ - return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() -#ifdef HAS_PREDICATE_REGS - , - predicateRegs() & other.predicateRegs() -#endif - ); -} - -_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) const -{ - return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() -#ifdef HAS_PREDICATE_REGS - , - predicateRegs() | other.predicateRegs() -#endif - ); -} - -_regMaskAll AllRegsMask::operator&(const regNumber reg) const -{ - _regMaskAll result = *this; - result[regIndexForRegister(reg)] &= genRegMask(reg); - return result; -} - void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { registers[regIndexForType(type)] |= maskToAdd; diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 4a47fdd457d3e2..ce6d27c17ffcb7 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -361,16 +361,16 @@ typedef struct _regMaskAll bool IsGprOrFloatPresent() const; regMaskTP GetGprFloatCombinedMask() const; - void operator|=(const _regMaskAll& other); - void operator&=(const _regMaskAll& other); - void operator|=(const regNumber reg); - void operator^=(const regNumber reg); - _regMaskAll operator~(); - bool operator==(const _regMaskAll& other); - bool operator!=(const _regMaskAll& other); - _regMaskAll operator&(const _regMaskAll& other) const; - _regMaskAll operator|(const _regMaskAll& other) const; - _regMaskAll operator&(const regNumber reg) const; + FORCEINLINE void operator|=(const _regMaskAll& other); + FORCEINLINE void operator&=(const _regMaskAll& other); + FORCEINLINE void operator|=(const regNumber reg); + FORCEINLINE void operator^=(const regNumber reg); + FORCEINLINE _regMaskAll operator~(); + FORCEINLINE bool operator==(const _regMaskAll& other); + FORCEINLINE bool operator!=(const _regMaskAll& other); + FORCEINLINE _regMaskAll operator&(const _regMaskAll& other) const; + FORCEINLINE _regMaskAll operator|(const _regMaskAll& other) const; + FORCEINLINE _regMaskAll operator&(const regNumber reg) const; } AllRegsMask; From 020b7093324d7492be20dbc7026427f6a3ef8495 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 21 Mar 2024 13:24:39 -0700 Subject: [PATCH 117/201] Mark AllRegsMask methods as FORCEINLINE --- src/coreclr/jit/compiler.hpp | 188 ++++++++++++++++++++++++++++++++--- src/coreclr/jit/lsra.cpp | 149 --------------------------- src/coreclr/jit/lsra.h | 12 --- src/coreclr/jit/regset.h | 4 +- src/coreclr/jit/target.h | 60 +++++------ 5 files changed, 206 insertions(+), 207 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 0dc0241d140767..80c9056820677a 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5053,8 +5053,19 @@ BasicBlockVisit FlowGraphNaturalLoop::VisitRegularExitBlocks(TFunc func) return BasicBlockVisit::Continue; } +// TODO: If there are lot of callers of RegisterType, simplify it. +template +FORCEINLINE int regIndexForType(T vt) +{ + int type = varTypeRegister[TypeGet(vt)]; + assert(type <= REGISTER_TYPE_COUNT); +#ifndef HAS_PREDICATE_REGS + assert(type != VTR_MASK); +#endif + return (type - 1); +} -void AllRegsMask::operator|=(const _regMaskAll& other) +void AllRegsMask::operator|=(const AllRegsMask& other) { // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh @@ -5065,7 +5076,7 @@ void AllRegsMask::operator|=(const _regMaskAll& other) #endif } -void AllRegsMask::operator&=(const _regMaskAll& other) +void AllRegsMask::operator&=(const AllRegsMask& other) { // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh @@ -5086,14 +5097,14 @@ void AllRegsMask::operator^=(const regNumber reg) registers[regIndexForRegister(reg)] ^= genRegMask(reg); } -_regMaskAll AllRegsMask::operator~() +AllRegsMask AllRegsMask::operator~() { - return _regMaskAll(~gprRegs(), ~floatRegs() + return AllRegsMask(~gprRegs(), ~floatRegs() #ifdef HAS_PREDICATE_REGS , ~predicateRegs() #endif - ); + ); } bool AllRegsMask::operator==(const AllRegsMask& other) @@ -5102,7 +5113,7 @@ bool AllRegsMask::operator==(const AllRegsMask& other) #ifdef HAS_PREDICATE_REGS && (predicateRegs() == other.predicateRegs()) #endif - ); + ); } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5110,33 +5121,182 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) return !(*this == other); } -_regMaskAll AllRegsMask::operator&(const AllRegsMask& other) const +AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { - return _regMaskAll(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() + return AllRegsMask(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() #ifdef HAS_PREDICATE_REGS , predicateRegs() & other.predicateRegs() #endif - ); + ); } -_regMaskAll AllRegsMask::operator|(const _regMaskAll& other) const +AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { - return _regMaskAll(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() + return AllRegsMask(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() #ifdef HAS_PREDICATE_REGS , predicateRegs() | other.predicateRegs() #endif - ); + ); } -_regMaskAll AllRegsMask::operator&(const regNumber reg) const +AllRegsMask AllRegsMask::operator&(const regNumber reg) const { - _regMaskAll result = *this; + AllRegsMask result = *this; result[regIndexForRegister(reg)] &= genRegMask(reg); return result; } +void AllRegsMask::Clear() +{ + registers[0] = RBM_NONE; + registers[1] = RBM_NONE; +#ifdef HAS_PREDICATE_REGS + registers[2] = RBM_NONE; +#endif +} + +bool AllRegsMask::IsEmpty() +{ + return (gprRegs() | floatRegs() +#ifdef HAS_PREDICATE_REGS + | predicateRegs() +#endif // HAS_PREDICATE_REGS + ) == RBM_NONE; +} + +unsigned AllRegsMask::Count() +{ + return genCountBits(gprRegs()) + genCountBits(floatRegs()) +#ifdef HAS_PREDICATE_REGS + + genCountBits(predicateRegs()) +#endif // HAS_PREDICATE_REGS + ; +} + +regMaskOnlyOne AllRegsMask::operator[](int index) const +{ + assert(index <= REGISTER_TYPE_COUNT); + return registers[index]; +} + +regMaskOnlyOne& AllRegsMask::operator[](int index) +{ + assert(index <= REGISTER_TYPE_COUNT); + return registers[index]; +} + +void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) +{ + registers[regIndexForType(type)] |= maskToAdd; +} + +void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) +{ + registers[0] |= maskToAdd; +} + +void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) +{ + registers[1] |= maskToAdd; +} + +// Adds reg only if it is gpr register +void AllRegsMask::AddGprRegInMask(regNumber reg) +{ + int regIndex = regIndexForRegister(reg); + registers[0] |= -static_cast(!regIndex) & genRegMask(reg); +} + +#ifdef TARGET_ARM +void AllRegsMask::AddRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMask = genRegMask(reg); + registers[regIndexForRegister(reg)] |= regMask; +} + +void AllRegsMask::RemoveRegNumFromMask(regNumber reg) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg); + registers[regIndexForRegister(reg)] &= ~regMaskToRemove; +} + +bool AllRegsMask::IsRegNumInMask(regNumber reg) +{ + regMaskOnlyOne regMaskToCheck = genRegMask(reg); + return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; +} +#endif + +// ---------------------------------------------------------- +// AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except +// that it takes `type` as an argument and adds `reg` to the mask for that type. +// +void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) +{ + regMaskOnlyOne regMask = genRegMask(reg ARM_ARG(type)); + registers[regIndexForRegister(reg)] |= regMask; +} + +void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) +{ + regMaskOnlyOne regMaskToRemove = genRegMask(reg ARM_ARG(type)); + registers[regIndexForRegister(reg)] &= ~regMaskToRemove; +} + +void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) +{ + registers[regIndexForType(type)] &= ~regMaskToRemove; +} + +bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) +{ + regMaskOnlyOne regMaskToCheck = genRegMask(reg ARM_ARG(type)); + return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; +} + +bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const +{ + return (gprRegs() & maskToCheck) != RBM_NONE; +} + +bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const +{ + return (floatRegs() & maskToCheck) != RBM_NONE; +} + +regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const +{ + return registers[regIndexForType(type)]; +} + +regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const +{ + return registers[regIndexForRegister(reg)]; +} + +regMaskTP AllRegsMask::GetGprFloatCombinedMask() const +{ + // TODO: NEed to revert this once we change floatRegs to 32-bits + // #ifdef TARGET_64BIT + // return ((regMaskTP)floatRegs() << 32) | gprRegs(); + // #else + // return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); + // #endif + // TODO: Cannot get RBM_ALLFLOAT because it is not defined for AllRegsMask + // Moving this definition to lsra might work, but this is temporary so + // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in + // the comment above. + regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; + return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); +} + +bool AllRegsMask::IsGprOrFloatPresent() const +{ + return GetGprFloatCombinedMask() != RBM_NONE; +} + /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 9147b3dcd46d07..127deef4f4e2cd 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -14017,152 +14017,3 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( assert(found && isSingleRegister(candidates)); return candidates; } - -void AllRegsMask::Clear() -{ - registers[0] = RBM_NONE; - registers[1] = RBM_NONE; -#ifdef HAS_PREDICATE_REGS - registers[2] = RBM_NONE; -#endif -} - -bool AllRegsMask::IsEmpty() -{ - return (gprRegs() | floatRegs() -#ifdef HAS_PREDICATE_REGS - | predicateRegs() -#endif // HAS_PREDICATE_REGS - ) == RBM_NONE; -} - -unsigned AllRegsMask::Count() -{ - return genCountBits(gprRegs()) + genCountBits(floatRegs()) -#ifdef HAS_PREDICATE_REGS - + genCountBits(predicateRegs()) -#endif // HAS_PREDICATE_REGS - ; -} - -regMaskOnlyOne AllRegsMask::operator[](int index) const -{ - assert(index <= REGISTER_TYPE_COUNT); - return registers[index]; -} - -regMaskOnlyOne& AllRegsMask::operator[](int index) -{ - assert(index <= REGISTER_TYPE_COUNT); - return registers[index]; -} - -void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) -{ - registers[regIndexForType(type)] |= maskToAdd; -} - -void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) -{ - registers[0] |= maskToAdd; -} - -void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) -{ - registers[1] |= maskToAdd; -} - -// Adds reg only if it is gpr register -void AllRegsMask::AddGprRegInMask(regNumber reg) -{ - int regIndex = regIndexForRegister(reg); - registers[0] |= -static_cast(!regIndex) & genRegMask(reg); -} - -#ifdef TARGET_ARM -void AllRegsMask::AddRegNumInMask(regNumber reg) -{ - regMaskOnlyOne regMask = genRegMask(reg); - registers[regIndexForRegister(reg)] |= regMask; -} - -void AllRegsMask::RemoveRegNumFromMask(regNumber reg) -{ - regMaskOnlyOne regMaskToRemove = genRegMask(reg); - registers[regIndexForRegister(reg)] &= ~regMaskToRemove; -} - -bool AllRegsMask::IsRegNumInMask(regNumber reg) -{ - regMaskOnlyOne regMaskToCheck = genRegMask(reg); - return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; -} -#endif - -// ---------------------------------------------------------- -// AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except -// that it takes `type` as an argument and adds `reg` to the mask for that type. -// -void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) -{ - regMaskOnlyOne regMask = genRegMask(reg ARM_ARG(type)); - registers[regIndexForRegister(reg)] |= regMask; -} - -void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) -{ - regMaskOnlyOne regMaskToRemove = genRegMask(reg ARM_ARG(type)); - registers[regIndexForRegister(reg)] &= ~regMaskToRemove; -} - -void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) -{ - registers[regIndexForType(type)] &= ~regMaskToRemove; -} - -bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) -{ - regMaskOnlyOne regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; -} - -bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const -{ - return (gprRegs() & maskToCheck) != RBM_NONE; -} - -bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const -{ - return (floatRegs() & maskToCheck) != RBM_NONE; -} - -regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const -{ - return registers[regIndexForType(type)]; -} - -regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const -{ - return registers[regIndexForRegister(reg)]; -} - -regMaskTP AllRegsMask::GetGprFloatCombinedMask() const -{ - // TODO: NEed to revert this once we change floatRegs to 32-bits - //#ifdef TARGET_64BIT - // return ((regMaskTP)floatRegs() << 32) | gprRegs(); - //#else - // return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); - //#endif - // TODO: Cannot get RBM_ALLFLOAT because it is not defined for AllRegsMask - // Moving this definition to lsra might work, but this is temporary so - // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in - // the comment above. - regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; - return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); -} - -bool AllRegsMask::IsGprOrFloatPresent() const -{ - return GetGprFloatCombinedMask() != RBM_NONE; -} diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 934601eff165ac..07f850cb7de49e 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -64,18 +64,6 @@ RegisterType regType(T type) } } -// TODO: If there are lot of callers of RegisterType, simplify it. -template -FORCEINLINE int regIndexForType(T vt) -{ - int type = varTypeRegister[TypeGet(vt)]; - assert(type <= REGISTER_TYPE_COUNT); -#ifndef HAS_PREDICATE_REGS - assert(type != VTR_MASK); -#endif - return (type - 1); -} - //------------------------------------------------------------------------ // useFloatReg: Check if the given var_type should be allocated to a FloatRegisterType // diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index f43276afbc74bf..18a9871db8e393 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -67,8 +67,8 @@ class RegSet // private: - bool rsNeededSpillReg; // true if this method needed to spill any registers - AllRegsMask rsModifiedRegsMask; // mask of the registers modified by the current function. + bool rsNeededSpillReg; // true if this method needed to spill any registers + AllRegsMask rsModifiedRegsMask; // mask of the registers modified by the current function. #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index ce6d27c17ffcb7..100c721d330aa7 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -332,42 +332,42 @@ typedef struct _regMaskAll AddRegNumInMask(reg ARM_ARG(type)); } - void Clear(); - bool IsEmpty(); - unsigned Count(); - void Create(regNumber reg); + FORCEINLINE void Clear(); + FORCEINLINE bool IsEmpty(); + FORCEINLINE unsigned Count(); + FORCEINLINE void Create(regNumber reg); // Rename this to AddRegNum - void AddGprRegInMask(regNumber reg); - void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); - void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); - void AddGprRegMask(regMaskGpr maskToAdd); - void AddFloatRegMask(regMaskFloat maskToAdd); + FORCEINLINE void AddGprRegInMask(regNumber reg); + FORCEINLINE void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); + FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); + FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); + FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); #ifdef TARGET_ARM - void AddRegNumInMask(regNumber reg); - void RemoveRegNumFromMask(regNumber reg); - bool IsRegNumInMask(regNumber reg); -#endif - void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); - void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); - bool IsGprMaskPresent(regMaskGpr maskToCheck) const; - bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; + FORCEINLINE void AddRegNumInMask(regNumber reg); + FORCEINLINE void RemoveRegNumFromMask(regNumber reg); + FORCEINLINE bool IsRegNumInMask(regNumber reg); +#endif + FORCEINLINE void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + FORCEINLINE void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); + FORCEINLINE bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); + FORCEINLINE bool IsGprMaskPresent(regMaskGpr maskToCheck) const; + FORCEINLINE bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; // bool IsOnlyRegNumInMask(regNumber reg); - regMaskOnlyOne GetRegMaskForType(var_types type) const; - regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; + FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; + FORCEINLINE regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field - bool IsGprOrFloatPresent() const; - regMaskTP GetGprFloatCombinedMask() const; - - FORCEINLINE void operator|=(const _regMaskAll& other); - FORCEINLINE void operator&=(const _regMaskAll& other); - FORCEINLINE void operator|=(const regNumber reg); - FORCEINLINE void operator^=(const regNumber reg); - FORCEINLINE _regMaskAll operator~(); - FORCEINLINE bool operator==(const _regMaskAll& other); - FORCEINLINE bool operator!=(const _regMaskAll& other); + FORCEINLINE bool IsGprOrFloatPresent() const; + FORCEINLINE regMaskTP GetGprFloatCombinedMask() const; + + FORCEINLINE void operator|=(const _regMaskAll& other); + FORCEINLINE void operator&=(const _regMaskAll& other); + FORCEINLINE void operator|=(const regNumber reg); + FORCEINLINE void operator^=(const regNumber reg); + FORCEINLINE _regMaskAll operator~(); + FORCEINLINE bool operator==(const _regMaskAll& other); + FORCEINLINE bool operator!=(const _regMaskAll& other); FORCEINLINE _regMaskAll operator&(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator|(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator&(const regNumber reg) const; From 31a940e79600ac5f57217b421164cb18ce60680d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 24 Mar 2024 07:18:40 -0700 Subject: [PATCH 118/201] Prototype of union using _float_gpr This reduces the MinOpts TP by 0.5%, but increases the TP for fullopts --- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.hpp | 149 ++++++++++++++++++++-------------- src/coreclr/jit/lsra.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 6 +- src/coreclr/jit/lsraxarch.cpp | 4 +- src/coreclr/jit/target.h | 99 ++++++++++++++++------ 6 files changed, 170 insertions(+), 92 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 0c90c5efe2a4fc..dea65fc4103bd9 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3671,9 +3671,9 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) void Compiler::dumpRegMask(regMaskOnlyOne mask) const { - dumpRegMask(AllRegsMask(mask, mask, mask)); } + //------------------------------------------------------------------------ // dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of // a potentially large number of registers. diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 80c9056820677a..a7ff82c0e3315f 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5069,10 +5069,9 @@ void AllRegsMask::operator|=(const AllRegsMask& other) { // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh - registers[0] |= other.gprRegs(); - registers[1] |= other.floatRegs(); + _float_gpr |= other._float_gpr; #ifdef HAS_PREDICATE_REGS - registers[2] |= other.predicateRegs(); + _predicateRegs |= other._predicateRegs; #endif } @@ -5080,40 +5079,53 @@ void AllRegsMask::operator&=(const AllRegsMask& other) { // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh - registers[0] &= other.gprRegs(); - registers[1] &= other.floatRegs(); + _float_gpr &= other._float_gpr; #ifdef HAS_PREDICATE_REGS - registers[2] &= other.predicateRegs(); + _predicateRegs &= other._predicateRegs; #endif } void AllRegsMask::operator|=(const regNumber reg) { - registers[regIndexForRegister(reg)] |= genRegMask(reg); + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] |= encodeForIndex(index, value); + //// Alternatively, we can write this, which is shorter + //// version, but it involves a branch + // if (index == 2) + //{ + // registers[2] = value; + // } + // else + //{ + // float_gpr = value; + // } } void AllRegsMask::operator^=(const regNumber reg) { - registers[regIndexForRegister(reg)] ^= genRegMask(reg); + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] ^= encodeForIndex(index, value); } AllRegsMask AllRegsMask::operator~() { - return AllRegsMask(~gprRegs(), ~floatRegs() + AllRegsMask result; + result._float_gpr = ~_float_gpr; #ifdef HAS_PREDICATE_REGS - , - ~predicateRegs() + result._predicateRegs = ~_predicateRegs; #endif - ); + return result; } bool AllRegsMask::operator==(const AllRegsMask& other) { - return ((gprRegs() == other.gprRegs()) && (floatRegs() == other.floatRegs()) + bool result = (_float_gpr == other._float_gpr); #ifdef HAS_PREDICATE_REGS - && (predicateRegs() == other.predicateRegs()) + result &= (_predicateRegs == other._predicateRegs); #endif - ); + return result; } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5123,54 +5135,55 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { - return AllRegsMask(gprRegs() & other.gprRegs(), floatRegs() & other.floatRegs() + AllRegsMask result; + result._float_gpr = _float_gpr & other._float_gpr; #ifdef HAS_PREDICATE_REGS - , - predicateRegs() & other.predicateRegs() + result._predicateRegs = _predicateRegs & other._predicateRegs; #endif - ); + return result; } AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { - return AllRegsMask(gprRegs() | other.gprRegs(), floatRegs() | other.floatRegs() + AllRegsMask result; + result._float_gpr = _float_gpr | other._float_gpr; #ifdef HAS_PREDICATE_REGS - , - predicateRegs() | other.predicateRegs() + result._predicateRegs = _predicateRegs | other._predicateRegs; #endif - ); + return result; } AllRegsMask AllRegsMask::operator&(const regNumber reg) const { AllRegsMask result = *this; - result[regIndexForRegister(reg)] &= genRegMask(reg); + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + result[index] &= encodeForIndex(index, value); return result; } void AllRegsMask::Clear() { - registers[0] = RBM_NONE; - registers[1] = RBM_NONE; + _float_gpr = RBM_NONE; #ifdef HAS_PREDICATE_REGS - registers[2] = RBM_NONE; + _predicateRegs = RBM_NONE; #endif } bool AllRegsMask::IsEmpty() { - return (gprRegs() | floatRegs() + return (_float_gpr #ifdef HAS_PREDICATE_REGS - | predicateRegs() + | _predicateRegs #endif // HAS_PREDICATE_REGS ) == RBM_NONE; } unsigned AllRegsMask::Count() { - return genCountBits(gprRegs()) + genCountBits(floatRegs()) + return genCountBits(_float_gpr) #ifdef HAS_PREDICATE_REGS - + genCountBits(predicateRegs()) + + genCountBits(_predicateRegs) #endif // HAS_PREDICATE_REGS ; } @@ -5178,54 +5191,59 @@ unsigned AllRegsMask::Count() regMaskOnlyOne AllRegsMask::operator[](int index) const { assert(index <= REGISTER_TYPE_COUNT); - return registers[index]; + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); } -regMaskOnlyOne& AllRegsMask::operator[](int index) -{ - assert(index <= REGISTER_TYPE_COUNT); - return registers[index]; -} +//regMaskOnlyOne& AllRegsMask::operator[](int index) +//{ +// assert(index <= REGISTER_TYPE_COUNT); +// return _registers[index]; +//} void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { - registers[regIndexForType(type)] |= maskToAdd; + int index = regIndexForType(type); + _registers[regIndexForType(type)] |= encodeForIndex(index, maskToAdd); } void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) { - registers[0] |= maskToAdd; + _registers[0] |= maskToAdd; } void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) { - registers[1] |= maskToAdd; + // TODO: Add assert that maskToAdd has low-32 bit set to 0 + _float_gpr |= maskToAdd; } // Adds reg only if it is gpr register void AllRegsMask::AddGprRegInMask(regNumber reg) { - int regIndex = regIndexForRegister(reg); - registers[0] |= -static_cast(!regIndex) & genRegMask(reg); + AddGprRegMask(genRegMask(reg)); } #ifdef TARGET_ARM void AllRegsMask::AddRegNumInMask(regNumber reg) { - regMaskOnlyOne regMask = genRegMask(reg); - registers[regIndexForRegister(reg)] |= regMask; + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] |= encodeForIndex(index, value); } void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { - regMaskOnlyOne regMaskToRemove = genRegMask(reg); - registers[regIndexForRegister(reg)] &= ~regMaskToRemove; + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToRemove = genRegMask(reg); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); } bool AllRegsMask::IsRegNumInMask(regNumber reg) { - regMaskOnlyOne regMaskToCheck = genRegMask(reg); - return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToCheck = genRegMask(reg); + return (_float_gpr & regMaskToCheck) != RBM_NONE; } #endif @@ -5235,25 +5253,33 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) // void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMask = genRegMask(reg ARM_ARG(type)); - registers[regIndexForRegister(reg)] |= regMask; + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg ARM_ARG(type)); + _registers[index] |= encodeForIndex(index, value); } void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMaskToRemove = genRegMask(reg ARM_ARG(type)); - registers[regIndexForRegister(reg)] &= ~regMaskToRemove; + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); } void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { - registers[regIndexForType(type)] &= ~regMaskToRemove; + int index = regIndexForType(type); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); } bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) { - regMaskOnlyOne regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return (registers[regIndexForRegister(reg)] & regMaskToCheck) != RBM_NONE; + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); + return ((_float_gpr & regMaskToCheck) +#ifdef HAS_PREDICATE_REGS + | (_predicateRegs & regMaskToCheck) +#endif // HAS_PREDICATE_REGS + ) != RBM_NONE; } bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const @@ -5268,12 +5294,16 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { - return registers[regIndexForType(type)]; + int index = regIndexForType(type); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); } regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const { - return registers[regIndexForRegister(reg)]; + int index = regIndexForRegister(reg); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const @@ -5288,8 +5318,9 @@ regMaskTP AllRegsMask::GetGprFloatCombinedMask() const // Moving this definition to lsra might work, but this is temporary so // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in // the comment above. - regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; - return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); + //regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; + //return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); + return _float_gpr; } bool AllRegsMask::IsGprOrFloatPresent() const diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 127deef4f4e2cd..fca0323b00861b 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5707,7 +5707,7 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below - // to dumpRegRecords. + // to dumpRegRecords. TODO: Do not do anything if tempRegsToMakeInactive == RBM_NONE AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); #ifdef HAS_PREDICATE_REGS diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 2dc35cfb0a45da..66aff89585842c 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2822,12 +2822,14 @@ void LinearScan::buildIntervals() // availableMaskRegs = RBM_NONE; // Is this also needed? } - actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs + //TODO: Fix this for x86 and arm + actualRegistersMask = AllRegsMask(~RBM_NONE, ~RBM_NONE #ifdef HAS_PREDICATE_REGS , availableMaskRegs #endif // HAS_PREDICATE_REGS - ); + ); + #ifdef DEBUG // Make sure we don't have any blocks that were not visited diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index f86e88dc6d4db7..b0628d5a22dc6a 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1973,7 +1973,7 @@ int LinearScan::BuildModDiv(GenTree* tree) srcCount += BuildDelayFreeUses(op2, op1, availableIntRegs & ~(RBM_RAX | RBM_RDX)); buildInternalRegisterUses(); - AllRegsMask killMask(getKillSetForModDiv(tree->AsOp()), RBM_NONE); + AllRegsMask killMask(getKillSetForModDiv(tree->AsOp())); BuildDefWithKills(tree, 1, dstCandidates, killMask); return srcCount; } @@ -3094,7 +3094,7 @@ int LinearScan::BuildMul(GenTree* tree) assert(compiler->IsGprRegMask(dstCandidates)); - AllRegsMask killMask(getKillSetForMul(tree->AsOp()), RBM_NONE); + AllRegsMask killMask(getKillSetForMul(tree->AsOp())); BuildDefWithKills(tree, dstCount, dstCandidates, killMask); return srcCount; } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 100c721d330aa7..6acb97e873e258 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -216,6 +216,9 @@ enum _regMask_enum : unsigned // In any case, we believe that is OK to freely cast between these types; no information will // be lost. +typedef unsigned __int32 RegBitSet32; +typedef unsigned __int64 RegBitSet64; + #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; typedef unsigned __int64 regMaskGpr; @@ -270,56 +273,100 @@ typedef unsigned char regNumberSmall; typedef struct _regMaskAll { private: - regMaskTP registers[REGISTER_TYPE_COUNT]; + union + { + RegBitSet32 _registers[REGISTER_TYPE_COUNT]; + struct + { + RegBitSet64 _float_gpr; +#ifdef HAS_PREDICATE_REGS + RegBitSet32 _predicateRegs; +#endif + }; + struct + { + RegBitSet32 _gprRegs; + RegBitSet32 _floatRegs; +#ifdef HAS_PREDICATE_REGS + RegBitSet32 _predicateRegs; +#endif + }; + }; + regMaskOnlyOne operator[](int index) const; regMaskOnlyOne& operator[](int index); + // This method shifts the high-32 bits of float to low-32 bits and return. + // For gpr and predicate registers, it returns the same value. + FORCEINLINE static RegBitSet32 encodeForIndex(int index, RegBitSet64 value) + { + int shiftAmount = 32 * (index == 1); + return (RegBitSet32)(value >> shiftAmount); + } + + FORCEINLINE static RegBitSet64 decodeForIndex(int index, RegBitSet32 value) + { + int shiftAmount = 32 * (index == 1); + return ((RegBitSet64)value << shiftAmount); + } public: inline regMaskGpr gprRegs() const { - return registers[0]; + return _gprRegs; } inline regMaskFloat floatRegs() const { - return registers[1]; + return _float_gpr & 0xFFFFFFFF00000000; } #ifdef HAS_PREDICATE_REGS inline regMaskPredicate predicateRegs() const { static_assert((REGISTER_TYPE_COUNT == 3), "There should be 3 types of registers"); - return registers[2]; + return _predicateRegs; } -#endif - _regMaskAll(regMaskGpr _gprRegMask, regMaskFloat _floatRegMask, regMaskPredicate _predicateRegMask = RBM_NONE) - : registers{_gprRegMask, _floatRegMask -#ifdef HAS_PREDICATE_REGS - , - _predicateRegMask -#endif - } + // TODO: See if we can avoid the '|' operation here. + _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegMask) + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegMask) { } - _regMaskAll() - : registers{RBM_NONE, RBM_NONE -#ifdef HAS_PREDICATE_REGS - , - RBM_NONE -#endif - } + _regMaskAll(RegBitSet64 gprRegMask) : _float_gpr(gprRegMask), _predicateRegs(RBM_NONE) + { + } + +#else + _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegMask) + : _float_gpr(floatRegMask | gprRegMask) + { + } + + _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) + : _float_gpr(floatRegMask | gprRegMask) { } - _regMaskAll(int (&_registers)[REGISTER_TYPE_COUNT]) + _regMaskAll(RegBitSet64 gprRegMask) : _float_gpr(gprRegMask) { - registers[0] = _registers[0]; - registers[1] = _registers[1]; + } +#endif + + _regMaskAll() : _float_gpr(RBM_NONE) #ifdef HAS_PREDICATE_REGS - registers[2] = _registers[2]; + , _predicateRegs(RBM_NONE) #endif + { } +// _regMaskAll(int (&_registers)[REGISTER_TYPE_COUNT]) +// { +//// registers[0] = _registers[0]; +//// registers[1] = _registers[1]; +////#ifdef HAS_PREDICATE_REGS +//// registers[2] = _registers[2]; +////#endif +// } + #ifdef TARGET_ARM _regMaskAll(regNumber reg) : _regMaskAll() { @@ -374,10 +421,8 @@ typedef struct _regMaskAll } AllRegsMask; -#define GprRegsMask(gprRegs) AllRegsMask(gprRegs, RBM_NONE) -#define FloatRegsMask(floatRegs) AllRegsMask(RBM_NONE, floatRegs) 3 - -#define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask((gprRegs & ~RBM_ALLFLOAT), (floatRegs & RBM_ALLFLOAT)) +#define GprRegsMask(gprRegs) AllRegsMask(gprRegs) +#define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask(gprRegs | floatRegs) #if REGMASK_BITS == 32 typedef unsigned regMaskSmall; From 864998707f92c1b6ac56f2b4f2f07f2c1865e0e8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 25 Mar 2024 12:18:21 -0700 Subject: [PATCH 119/201] hide predicate logic under #ifdef to see TP impact. It is just 0.5% regression on real --- src/coreclr/jit/compiler.hpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index a7ff82c0e3315f..041a62ae3c282e 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5190,9 +5190,13 @@ unsigned AllRegsMask::Count() regMaskOnlyOne AllRegsMask::operator[](int index) const { +#ifdef HAS_PREDICATE_REGS assert(index <= REGISTER_TYPE_COUNT); RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); +#else + return _float_gpr; +#endif } //regMaskOnlyOne& AllRegsMask::operator[](int index) @@ -5203,13 +5207,21 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { +#ifdef HAS_PREDICATE_REGS int index = regIndexForType(type); _registers[regIndexForType(type)] |= encodeForIndex(index, maskToAdd); +#else + _float_gpr |= maskToAdd; +#endif } void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) { +#ifdef HAS_PREDICATE_REGS _registers[0] |= maskToAdd; +#else + _float_gpr |= maskToAdd; +#endif } void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) @@ -5253,22 +5265,35 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) // void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { - int index = regIndexForRegister(reg); RegBitSet64 value = genRegMask(reg ARM_ARG(type)); +#ifdef HAS_PREDICATE_REGS + int index = regIndexForRegister(reg); _registers[index] |= encodeForIndex(index, value); +#else + _float_gpr |= value; +#endif + } void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { - int index = regIndexForRegister(reg); RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); +#ifdef HAS_PREDICATE_REGS + int index = regIndexForRegister(reg); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#else + _float_gpr &= ~regMaskToRemove; +#endif } void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { +#ifdef HAS_PREDICATE_REGS int index = regIndexForType(type); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#else + _float_gpr &= ~regMaskToRemove; +#endif } bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) @@ -5294,9 +5319,13 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { +#ifdef HAS_PREDICATE_REGS int index = regIndexForType(type); RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); +#else + return _float_gpr; +#endif } regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const From 25bdad9db5818edf77d5b7f6d68fa2df87e8726f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 26 Mar 2024 08:16:00 -0700 Subject: [PATCH 120/201] Remove GetMaskForRegNum() --- src/coreclr/jit/codegencommon.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/compiler.hpp | 7 ------- src/coreclr/jit/target.h | 1 - 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7dc865d3534c50..ec2399d15f162f 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8235,7 +8235,7 @@ void CodeGen::genRegCopy(GenTree* treeNode) if (targetReg != sourceReg) { singleRegMask targetRegMask = genRegMask(targetReg); - assert((busyRegs.GetMaskForRegNum(targetReg) & targetRegMask) == 0); + assert(!busyRegs.IsRegNumInMask(targetReg)); busyRegs.RemoveRegNumFromMask(sourceReg); } busyRegs.AddRegNumInMask(targetReg); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index c0eca3cc6280ce..8686fe1c0e227e 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1595,7 +1595,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // There may also be a conflict with the falseVal in case this is an AND // condition. Once again, after swapping there should be no conflict as // ensured by LSRA. - if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask().GetMaskForRegNum(dstReg) & dstReg) != 0) + if ((desc.oper == GT_AND) && falseVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 041a62ae3c282e..14e2bcb7a24f49 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5328,13 +5328,6 @@ regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const #endif } -regMaskOnlyOne AllRegsMask::GetMaskForRegNum(regNumber reg) const -{ - int index = regIndexForRegister(reg); - RegBitSet32 value = _registers[index]; - return decodeForIndex(index, value); -} - regMaskTP AllRegsMask::GetGprFloatCombinedMask() const { // TODO: NEed to revert this once we change floatRegs to 32-bits diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6acb97e873e258..c42423ee505e80 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -402,7 +402,6 @@ typedef struct _regMaskAll FORCEINLINE bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; // bool IsOnlyRegNumInMask(regNumber reg); FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; - FORCEINLINE regMaskOnlyOne GetMaskForRegNum(regNumber reg) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field FORCEINLINE bool IsGprOrFloatPresent() const; From e5d5e3b1d810d34bca7578ff1cd5d3819846378f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 26 Mar 2024 08:24:59 -0700 Subject: [PATCH 121/201] fix the creation of certain masks --- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/targetamd64.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 66aff89585842c..e80ff093c55287 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2823,7 +2823,7 @@ void LinearScan::buildIntervals() } //TODO: Fix this for x86 and arm - actualRegistersMask = AllRegsMask(~RBM_NONE, ~RBM_NONE + actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs #ifdef HAS_PREDICATE_REGS , availableMaskRegs diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 9f75b65b55c01b..6b9554e2b60114 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -538,9 +538,9 @@ #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #else // See vm\amd64\asmhelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET) + #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_FLOATRET))) #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) - #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~RBM_INTRET, ~RBM_FLOATRET)) + #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_FLOATRET))) #endif // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. From 951b260d91538f7972652acba06f421193a698aa Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 26 Mar 2024 08:25:31 -0700 Subject: [PATCH 122/201] Introduce HAS_MORE_THAN_64_REGISTERS The TP for benchmarks.run for win/x64 is 0.3% and for win/arm64 is 0.18% --- src/coreclr/jit/compiler.hpp | 130 +++++++++++++++++++++++------------ src/coreclr/jit/target.h | 84 +++++++++++++++------- 2 files changed, 145 insertions(+), 69 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 14e2bcb7a24f49..363a46c1935946 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5067,26 +5067,31 @@ FORCEINLINE int regIndexForType(T vt) void AllRegsMask::operator|=(const AllRegsMask& other) { +#ifdef HAS_MORE_THAN_64_REGISTERS // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh _float_gpr |= other._float_gpr; -#ifdef HAS_PREDICATE_REGS _predicateRegs |= other._predicateRegs; +#else + _allRegisters |= other._allRegisters; #endif } void AllRegsMask::operator&=(const AllRegsMask& other) { +#ifdef HAS_MORE_THAN_64_REGISTERS // TODO: Can we optimize to reintrepret_cast // Something like https://godbolt.org/z/1KevT8Edh _float_gpr &= other._float_gpr; -#ifdef HAS_PREDICATE_REGS _predicateRegs &= other._predicateRegs; +#else + _allRegisters &= other._allRegisters; #endif } void AllRegsMask::operator|=(const regNumber reg) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 value = genRegMask(reg); _registers[index] |= encodeForIndex(index, value); @@ -5100,32 +5105,42 @@ void AllRegsMask::operator|=(const regNumber reg) //{ // float_gpr = value; // } +#else + _allRegisters |= genRegMask(reg); +#endif } void AllRegsMask::operator^=(const regNumber reg) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 value = genRegMask(reg); _registers[index] ^= encodeForIndex(index, value); +#else + _allRegisters ^= genRegMask(reg); +#endif } AllRegsMask AllRegsMask::operator~() { +#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; result._float_gpr = ~_float_gpr; -#ifdef HAS_PREDICATE_REGS result._predicateRegs = ~_predicateRegs; -#endif return result; +#else + return ~_allRegisters; +#endif } bool AllRegsMask::operator==(const AllRegsMask& other) { - bool result = (_float_gpr == other._float_gpr); -#ifdef HAS_PREDICATE_REGS - result &= (_predicateRegs == other._predicateRegs); -#endif +#ifdef HAS_MORE_THAN_64_REGISTERS + bool result = (_float_gpr == other._float_gpr) &= (_predicateRegs == other._predicateRegs); return result; +#else + return _allRegisters == other._allRegisters; +#endif } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5135,67 +5150,77 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { +#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; result._float_gpr = _float_gpr & other._float_gpr; -#ifdef HAS_PREDICATE_REGS result._predicateRegs = _predicateRegs & other._predicateRegs; -#endif return result; +#else + return _allRegisters & other._allRegisters; +#endif } AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { +#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; result._float_gpr = _float_gpr | other._float_gpr; -#ifdef HAS_PREDICATE_REGS result._predicateRegs = _predicateRegs | other._predicateRegs; -#endif return result; +#else + return _allRegisters | other._allRegisters; +#endif } AllRegsMask AllRegsMask::operator&(const regNumber reg) const { +#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result = *this; int index = regIndexForRegister(reg); RegBitSet64 value = genRegMask(reg); result[index] &= encodeForIndex(index, value); return result; +#else + return _allRegisters & genRegMask(reg); +#endif } void AllRegsMask::Clear() { +#ifdef HAS_MORE_THAN_64_REGISTERS _float_gpr = RBM_NONE; -#ifdef HAS_PREDICATE_REGS _predicateRegs = RBM_NONE; +#else + _allRegisters = RBM_NONE; #endif } bool AllRegsMask::IsEmpty() { - return (_float_gpr -#ifdef HAS_PREDICATE_REGS - | _predicateRegs -#endif // HAS_PREDICATE_REGS - ) == RBM_NONE; +#if HAS_MORE_THAN_64_REGISTERS + return ((_float_gpr | _predicateRegs) == RBM_NONE); +#else + return _allRegisters == RBM_NONE; +#endif } unsigned AllRegsMask::Count() { - return genCountBits(_float_gpr) -#ifdef HAS_PREDICATE_REGS - + genCountBits(_predicateRegs) -#endif // HAS_PREDICATE_REGS - ; +#ifdef HAS_MORE_THAN_64_REGISTERS + return genCountBits(_float_gpr) + genCountBits(_predicateRegs); +#else + return genCountBits(_allRegisters); +#endif } regMaskOnlyOne AllRegsMask::operator[](int index) const { -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS assert(index <= REGISTER_TYPE_COUNT); RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); #else - return _float_gpr; + return _allRegisters; #endif } @@ -5207,27 +5232,31 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForType(type); - _registers[regIndexForType(type)] |= encodeForIndex(index, maskToAdd); + _registers[index] |= encodeForIndex(index, maskToAdd); #else - _float_gpr |= maskToAdd; + _allRegisters |= maskToAdd; #endif } void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) { -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS _registers[0] |= maskToAdd; #else - _float_gpr |= maskToAdd; + _allRegisters |= maskToAdd; #endif } void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) { +#ifdef HAS_MORE_THAN_64_REGISTERS // TODO: Add assert that maskToAdd has low-32 bit set to 0 _float_gpr |= maskToAdd; +#else + _allRegisters |= maskToAdd; +#endif } // Adds reg only if it is gpr register @@ -5239,23 +5268,35 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) #ifdef TARGET_ARM void AllRegsMask::AddRegNumInMask(regNumber reg) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 value = genRegMask(reg); _registers[index] |= encodeForIndex(index, value); +#else + _allRegisters |= genRegMask(reg); +#endif } void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 regMaskToRemove = genRegMask(reg); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#else + _allRegisters &= ~genRegMask(reg); +#endif } bool AllRegsMask::IsRegNumInMask(regNumber reg) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 regMaskToCheck = genRegMask(reg); return (_float_gpr & regMaskToCheck) != RBM_NONE; +#else + return (_allRegisters & genRegMask(reg)) != RBM_NONE; +#endif } #endif @@ -5265,46 +5306,45 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) // void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { +#ifdef HAS_MORE_THAN_64_REGISTERS RegBitSet64 value = genRegMask(reg ARM_ARG(type)); -#ifdef HAS_PREDICATE_REGS int index = regIndexForRegister(reg); _registers[index] |= encodeForIndex(index, value); #else - _float_gpr |= value; + _allRegisters |= genRegMask(reg ARM_ARG(type)); #endif - } void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { +#ifdef HAS_MORE_THAN_64_REGISTERS RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); -#ifdef HAS_PREDICATE_REGS int index = regIndexForRegister(reg); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); #else - _float_gpr &= ~regMaskToRemove; + _allRegisters &= ~genRegMask(reg ARM_ARG(type)); #endif } void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForType(type); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); #else - _float_gpr &= ~regMaskToRemove; + _allRegisters &= ~regMaskToRemove; #endif } bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) { +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return ((_float_gpr & regMaskToCheck) -#ifdef HAS_PREDICATE_REGS - | (_predicateRegs & regMaskToCheck) + return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; +#else + return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; #endif // HAS_PREDICATE_REGS - ) != RBM_NONE; } bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const @@ -5319,17 +5359,18 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForType(type); RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); #else - return _float_gpr; + return _allRegisters; #endif } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const { +#ifdef HAS_MORE_THAN_64_REGISTERS // TODO: NEed to revert this once we change floatRegs to 32-bits // #ifdef TARGET_64BIT // return ((regMaskTP)floatRegs() << 32) | gprRegs(); @@ -5343,6 +5384,9 @@ regMaskTP AllRegsMask::GetGprFloatCombinedMask() const //regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; //return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); return _float_gpr; +#else + return _allRegisters; +#endif } bool AllRegsMask::IsGprOrFloatPresent() const diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index c42423ee505e80..7f0b3952dbd660 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -273,27 +273,27 @@ typedef unsigned char regNumberSmall; typedef struct _regMaskAll { private: +#ifdef HAS_MORE_THAN_64_REGISTERS union { RegBitSet32 _registers[REGISTER_TYPE_COUNT]; struct { RegBitSet64 _float_gpr; -#ifdef HAS_PREDICATE_REGS RegBitSet32 _predicateRegs; -#endif }; struct { RegBitSet32 _gprRegs; RegBitSet32 _floatRegs; -#ifdef HAS_PREDICATE_REGS RegBitSet32 _predicateRegs; -#endif }; }; - - regMaskOnlyOne operator[](int index) const; +#else + RegBitSet64 _allRegisters; +#endif + + regMaskOnlyOne operator[](int index) const; regMaskOnlyOne& operator[](int index); // This method shifts the high-32 bits of float to low-32 bits and return. // For gpr and predicate registers, it returns the same value. @@ -312,48 +312,80 @@ typedef struct _regMaskAll public: inline regMaskGpr gprRegs() const { - return _gprRegs; +#ifdef TARGET_AMD64 + return _allRegisters & 0xFFFF; +#elif TARGET_ARM64 + return _allRegisters & 0xFFFFFFFF; +#else + // TODO: Fix this for ARM and x86 + return _allRegisters; +#endif } inline regMaskFloat floatRegs() const { +#ifdef HAS_MORE_THAN_64_REGISTERS return _float_gpr & 0xFFFFFFFF00000000; +#else +#ifdef TARGET_AMD64 + return _allRegisters & 0xFFFFFFFF0000; +#else + //TODO: Fix this for ARM and x86 + return _allRegisters; +#endif // TARGET_AMD64 +#endif // HAS_MORE_THAN_64_REGISTERS + } + #ifdef HAS_PREDICATE_REGS inline regMaskPredicate predicateRegs() const { static_assert((REGISTER_TYPE_COUNT == 3), "There should be 3 types of registers"); +#ifdef HAS_MORE_THAN_64_REGISTERS return _predicateRegs; +#else +#ifdef TARGET_AMD64 + return _allRegisters & 0xFF000000000000; +#else + // TODO: Fix this for ARM and x86 + return _allRegisters; +#endif // TARGET_AMD64 +#endif // HAS_MORE_THAN_64_REGISTERS } +#endif - // TODO: See if we can avoid the '|' operation here. - _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegMask) - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegMask) - { - } - - _regMaskAll(RegBitSet64 gprRegMask) : _float_gpr(gprRegMask), _predicateRegs(RBM_NONE) - { - } - + _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) +#ifdef HAS_MORE_THAN_64_REGISTERS + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE) #else - _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegMask) - : _float_gpr(floatRegMask | gprRegMask) + : _allRegisters(floatRegMask | gprRegMask) +#endif { } - _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) - : _float_gpr(floatRegMask | gprRegMask) + // TODO: See if we can avoid the '|' operation here. + _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegs) +#ifdef HAS_MORE_THAN_64_REGISTERS + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(predicateRegs) +#else + : _allRegisters(predicateRegs | floatRegMask | gprRegMask) +#endif { } - _regMaskAll(RegBitSet64 gprRegMask) : _float_gpr(gprRegMask) + _regMaskAll() +#ifdef HAS_MORE_THAN_64_REGISTERS + : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE) +#else + : _allRegisters(RBM_NONE) +#endif { } -#endif - _regMaskAll() : _float_gpr(RBM_NONE) -#ifdef HAS_PREDICATE_REGS - , _predicateRegs(RBM_NONE) + _regMaskAll(RegBitSet64 allRegistersMask) +#ifdef HAS_MORE_THAN_64_REGISTERS + : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE) +#else + : _allRegisters(allRegistersMask) #endif { } From 33063eb5928cbb69f99aa97aa454571e144addda Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 27 Mar 2024 22:48:01 -0700 Subject: [PATCH 123/201] Introduce _hasPredicateRegister --- src/coreclr/jit/compiler.hpp | 257 ++++++++++++++++++++++------------- src/coreclr/jit/target.h | 38 +++++- 2 files changed, 198 insertions(+), 97 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 363a46c1935946..a0da57b476cd4e 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5092,33 +5092,33 @@ void AllRegsMask::operator&=(const AllRegsMask& other) void AllRegsMask::operator|=(const regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] |= encodeForIndex(index, value); - //// Alternatively, we can write this, which is shorter - //// version, but it involves a branch - // if (index == 2) - //{ - // registers[2] = value; - // } - // else - //{ - // float_gpr = value; - // } -#else - _allRegisters |= genRegMask(reg); + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] |= encodeForIndex(index, value); + } + else #endif + { + _allRegisters |= genRegMask(reg); + } } void AllRegsMask::operator^=(const regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] ^= encodeForIndex(index, value); -#else - _allRegisters ^= genRegMask(reg); + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] ^= encodeForIndex(index, value); + } + else #endif + { + _allRegisters ^= genRegMask(reg); + } } AllRegsMask AllRegsMask::operator~() @@ -5136,11 +5136,16 @@ AllRegsMask AllRegsMask::operator~() bool AllRegsMask::operator==(const AllRegsMask& other) { #ifdef HAS_MORE_THAN_64_REGISTERS - bool result = (_float_gpr == other._float_gpr) &= (_predicateRegs == other._predicateRegs); - return result; -#else - return _allRegisters == other._allRegisters; + if (_hasPredicateRegister) + { + bool result = (_float_gpr == other._float_gpr) && (_predicateRegs == other._predicateRegs); + return result; + } + else #endif + { + return _allRegisters == other._allRegisters; + } } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5151,44 +5156,59 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { #ifdef HAS_MORE_THAN_64_REGISTERS - AllRegsMask result; - result._float_gpr = _float_gpr & other._float_gpr; - result._predicateRegs = _predicateRegs & other._predicateRegs; - return result; -#else - return _allRegisters & other._allRegisters; + if (_hasPredicateRegister) + { + AllRegsMask result; + result._float_gpr = _float_gpr & other._float_gpr; + result._predicateRegs = _predicateRegs & other._predicateRegs; + return result; + } + else #endif + { + return _allRegisters & other._allRegisters; + } } AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { #ifdef HAS_MORE_THAN_64_REGISTERS - AllRegsMask result; - result._float_gpr = _float_gpr | other._float_gpr; - result._predicateRegs = _predicateRegs | other._predicateRegs; - return result; -#else - return _allRegisters | other._allRegisters; + if (_hasPredicateRegister) + { + AllRegsMask result; + result._float_gpr = _float_gpr | other._float_gpr; + result._predicateRegs = _predicateRegs | other._predicateRegs; + return result; + } + else #endif + { + return _allRegisters | other._allRegisters; + } } AllRegsMask AllRegsMask::operator&(const regNumber reg) const { #ifdef HAS_MORE_THAN_64_REGISTERS - AllRegsMask result = *this; - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - result[index] &= encodeForIndex(index, value); - return result; -#else - return _allRegisters & genRegMask(reg); + if (_hasPredicateRegister) + { + AllRegsMask result = *this; + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + result[index] &= encodeForIndex(index, value); + return result; + } + else #endif + { + return _allRegisters & genRegMask(reg); + } } void AllRegsMask::Clear() { #ifdef HAS_MORE_THAN_64_REGISTERS - _float_gpr = RBM_NONE; + _float_gpr = RBM_NONE; _predicateRegs = RBM_NONE; #else _allRegisters = RBM_NONE; @@ -5216,12 +5236,17 @@ unsigned AllRegsMask::Count() regMaskOnlyOne AllRegsMask::operator[](int index) const { #ifdef HAS_MORE_THAN_64_REGISTERS - assert(index <= REGISTER_TYPE_COUNT); - RegBitSet32 value = _registers[index]; - return decodeForIndex(index, value); -#else - return _allRegisters; + if (_hasPredicateRegister) + { + assert(index <= REGISTER_TYPE_COUNT); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); + } + else #endif + { + return _allRegisters; + } } //regMaskOnlyOne& AllRegsMask::operator[](int index) @@ -5233,11 +5258,16 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForType(type); - _registers[index] |= encodeForIndex(index, maskToAdd); -#else - _allRegisters |= maskToAdd; + if (_hasPredicateRegister) + { + int index = regIndexForType(type); + _registers[index] |= encodeForIndex(index, maskToAdd); + } + else #endif + { + _allRegisters |= maskToAdd; + } } void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) @@ -5269,34 +5299,49 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) void AllRegsMask::AddRegNumInMask(regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] |= encodeForIndex(index, value); -#else - _allRegisters |= genRegMask(reg); + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] |= encodeForIndex(index, value); + } + else #endif + { + _allRegisters |= genRegMask(reg); + } } void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToRemove = genRegMask(reg); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); -#else - _allRegisters &= ~genRegMask(reg); + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToRemove = genRegMask(reg); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); + } + else #endif + { + _allRegisters &= ~genRegMask(reg); + } } bool AllRegsMask::IsRegNumInMask(regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToCheck = genRegMask(reg); - return (_float_gpr & regMaskToCheck) != RBM_NONE; -#else - return (_allRegisters & genRegMask(reg)) != RBM_NONE; + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToCheck = genRegMask(reg); + return (_float_gpr & regMaskToCheck) != RBM_NONE; + } + else #endif + { + return (_allRegisters & genRegMask(reg)) != RBM_NONE; + } } #endif @@ -5307,44 +5352,64 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - RegBitSet64 value = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] |= encodeForIndex(index, value); -#else - _allRegisters |= genRegMask(reg ARM_ARG(type)); + if (_hasPredicateRegister) + { + RegBitSet64 value = genRegMask(reg ARM_ARG(type)); + int index = regIndexForRegister(reg); + _registers[index] |= encodeForIndex(index, value); + } + else #endif + { + _allRegisters |= genRegMask(reg ARM_ARG(type)); + } } void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); -#else - _allRegisters &= ~genRegMask(reg ARM_ARG(type)); + if (_hasPredicateRegister) + { + RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); + int index = regIndexForRegister(reg); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); + } + else #endif + { + _allRegisters &= ~genRegMask(reg ARM_ARG(type)); + } } void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForType(type); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); -#else - _allRegisters &= ~regMaskToRemove; + if (_hasPredicateRegister) + { + int index = regIndexForType(type); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); + } + else #endif + { + _allRegisters &= ~regMaskToRemove; + } } bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; -#else - return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; -#endif // HAS_PREDICATE_REGS + if (_hasPredicateRegister) + { + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); + return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; + } + else +#endif + { + return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; + } } bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const @@ -5360,12 +5425,18 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForType(type); - RegBitSet32 value = _registers[index]; - return decodeForIndex(index, value); -#else - return _allRegisters; + if (_hasPredicateRegister) + { + int index = regIndexForType(type); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); + } + else #endif + { + return _allRegisters; + } + } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 7f0b3952dbd660..81aea336095d59 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -205,6 +205,10 @@ enum _regMask_enum : unsigned #define REGISTER_TYPE_COUNT 2 #endif +#ifdef TARGET_ARM64 +#define HAS_MORE_THAN_64_REGISTERS 1 +#endif + /*****************************************************************************/ // TODO-Cleanup: The types defined below are mildly confusing: why are there both? @@ -281,14 +285,33 @@ typedef struct _regMaskAll { RegBitSet64 _float_gpr; RegBitSet32 _predicateRegs; + //RegBitSet32 _predicateRegs : 28; + //bool _hasPredicateRegister : 8; + + }; + struct + { + // This one is just to have uniform code when `if(_hasPredicateRegister)` is used + RegBitSet64 _allRegisters; + RegBitSet32 _predicateRegs; + //RegBitSet32 _predicateRegs : 28; + //bool _hasPredicateRegister : 8; }; struct { RegBitSet32 _gprRegs; RegBitSet32 _floatRegs; RegBitSet32 _predicateRegs; + //RegBitSet32 _predicateRegs : 28; + //bool _hasPredicateRegister : 8; }; }; + // TODO: This can be moved inside the union by trimming the size of _predicateRegs: 28 + // and have this as 8 bits. However, there are chances of accidently overwritting it + // when updating the predicateReg mask. Evaluation this option on how we can secure it. + // Find out if the TP cost is more by having this field separately vs. adding extra checks + // when the predicateReg is updated. + bool _hasPredicateRegister; #else RegBitSet64 _allRegisters; #endif @@ -355,7 +378,7 @@ typedef struct _regMaskAll _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE) + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) #else : _allRegisters(floatRegMask | gprRegMask) #endif @@ -365,7 +388,7 @@ typedef struct _regMaskAll // TODO: See if we can avoid the '|' operation here. _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegs) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(predicateRegs) + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegs), _hasPredicateRegister(true) #else : _allRegisters(predicateRegs | floatRegMask | gprRegMask) #endif @@ -374,7 +397,7 @@ typedef struct _regMaskAll _regMaskAll() #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE) + : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) #else : _allRegisters(RBM_NONE) #endif @@ -383,13 +406,20 @@ typedef struct _regMaskAll _regMaskAll(RegBitSet64 allRegistersMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE) + : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) #else : _allRegisters(allRegistersMask) #endif { } + FORCEINLINE void NeedPredicateRegisterTracking() + { +#ifdef HAS_MORE_THAN_64_REGISTERS + _hasPredicateRegister = true; +#endif + } + // _regMaskAll(int (&_registers)[REGISTER_TYPE_COUNT]) // { //// registers[0] = _registers[0]; From ed29d62431a2221cf6d05557c78c72967335ad34 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 29 Mar 2024 22:29:34 -0700 Subject: [PATCH 124/201] fix some bugs --- src/coreclr/jit/target.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 81aea336095d59..61007848b4f176 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -221,13 +221,13 @@ enum _regMask_enum : unsigned // be lost. typedef unsigned __int32 RegBitSet32; -typedef unsigned __int64 RegBitSet64; #if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; typedef unsigned __int64 regMaskGpr; typedef unsigned __int64 regMaskFloat; typedef unsigned __int64 regMaskPredicate; +typedef unsigned __int64 RegBitSet64; // Design: // 1. Reduce regMaskGpr to 32-bit @@ -261,6 +261,8 @@ typedef unsigned __int64 singleRegMask; #else // x86 and arm typedef unsigned regMaskTP; +typedef unsigned RegBitSet64; + #define regMaskGpr regMaskTP #define regMaskFloat regMaskTP #define regMaskPredicate regMaskTP @@ -351,6 +353,8 @@ typedef struct _regMaskAll #else #ifdef TARGET_AMD64 return _allRegisters & 0xFFFFFFFF0000; +#elif defined(TARGET_ARM64) + return _allRegisters & 0xFFFFFFFF00000000; #else //TODO: Fix this for ARM and x86 return _allRegisters; From 8ee87f4cc1f8be8105e378534ae88ff19055c0b1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sat, 30 Mar 2024 09:10:02 -0700 Subject: [PATCH 125/201] fix some merge conflict --- src/coreclr/jit/codegencommon.cpp | 4 +--- src/coreclr/jit/instr.cpp | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index a6950785caa828..1846a062f85b36 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5411,9 +5411,6 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif -#ifdef TARGET_ARM -// TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? -#endif regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; #ifdef HAS_PREDICATE_REGS @@ -5438,6 +5435,7 @@ void CodeGen::genFinalizeFrame() maskPushRegsInt |= RBM_LR; #if defined(TARGET_ARM) + // TODO-ARM64-Bug?: enable some variant of this for FP on ARM64? if ((maskPushRegsFloat != RBM_NONE) || (compiler->opts.MinOpts() && (regSet.rsMaskResvd & maskPushRegsInt & RBM_OPT_RSVD))) { diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 52b5819056ebb5..d8a1901d909fc7 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -2053,7 +2053,6 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) return ins_Copy(dstType); } -#ifdef TARGET_XARCH // mask to int assert(genIsValidIntOrFakeReg(srcReg)); #if defined(TARGET_XARCH) From f1ae83c2cfa454254300d575b07e6831a9c54147 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 31 Mar 2024 12:13:17 -0700 Subject: [PATCH 126/201] Eliminate HAS_PREDICATE_REGS with FEATURE_MASKED_HW_INTRINSICS/HAS_MORE_THAN_64_REGISTERS --- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/codegenarmarch.cpp | 22 +++++++++++---- src/coreclr/jit/codegencommon.cpp | 11 +++++--- src/coreclr/jit/compiler.h | 12 ++++---- src/coreclr/jit/compiler.hpp | 26 ++++++++++++------ src/coreclr/jit/emit.cpp | 2 +- src/coreclr/jit/emitarm64.h | 8 ++++++ src/coreclr/jit/emitxarch.h | 2 ++ src/coreclr/jit/instr.cpp | 15 +++++++--- src/coreclr/jit/lsra.cpp | 33 +++++++++++++--------- src/coreclr/jit/lsra.h | 24 ++++++++-------- src/coreclr/jit/lsrabuild.cpp | 41 ++++++++++++++++------------ src/coreclr/jit/optimizer.cpp | 6 ++-- src/coreclr/jit/regset.cpp | 12 ++++---- src/coreclr/jit/regset.h | 6 ++-- src/coreclr/jit/target.h | 44 ++++++++---------------------- src/coreclr/jit/targetamd64.h | 1 - src/coreclr/jit/targetarm64.h | 2 ++ src/coreclr/jit/unwind.cpp | 4 +-- src/coreclr/jit/utils.cpp | 13 +-------- 20 files changed, 154 insertions(+), 132 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 2f35f511415be1..e4797b1bee617a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -1746,7 +1746,7 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() regMaskGpr rsMaskSaveGprRegs = regSet.rsGprMaskCalleeSaved; regMaskFloat rsMaskSaveFloatRegs = regSet.rsFloatMaskCalleeSaved; regMaskPredicate rsMaskSavePredicateRegs = RBM_NONE; -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS rsMaskSavePredicateRegs = regSet.rsPredicateMaskCalleeSaved; #endif diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 9ee84c2e7aea00..1082219c1abf9a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3529,9 +3529,9 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { regMaskGpr trashedGprByEpilog = RBM_INT_CALLEE_SAVED; regMaskFloat trashedFloatByEpilog = RBM_FLT_CALLEE_SAVED; -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate trashedPredicateByEpilog = RBM_MSK_CALLEE_SAVED; -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS // The epilog may use and trash REG_GSCOOKIE_TMP_0/1. Make sure we have no // non-standard args that may be trash if this is a tailcall. @@ -3560,7 +3560,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) JITDUMP("Float Register used: %s\n", getRegName(reg)); assert(!"Argument to tailcall may be trashed by epilog"); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if ((trashedPredicateByEpilog & genRegMask(reg)) != 0) { JITDUMP("Tail call node:\n"); @@ -3568,7 +3568,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) JITDUMP("Mask Register used: %s\n", getRegName(reg)); assert(!"Argument to tailcall may be trashed by epilog"); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS } } } @@ -4905,6 +4905,9 @@ void CodeGen::genPushCalleeSavedRegisters() regMaskGpr rsPushGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskFloat rsPushFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskPredicate rsPushPredicateRegs = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; +#endif #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -4940,17 +4943,24 @@ void CodeGen::genPushCalleeSavedRegisters() regSet.rsGprMaskCalleeSaved = rsPushGprRegs; regSet.rsFloatMaskCalleeSaved = rsPushFloatRegs; +#ifdef FEATURE_MASKED_HW_INTRINSICS + regSet.rsPredicateMaskCalleeSaved = rsPushPredicateRegs; +#endif #ifdef DEBUG unsigned pushRegsCnt = genCountBits(rsPushGprRegs) + genCountBits(rsPushFloatRegs); +#ifdef FEATURE_MASKED_HW_INTRINSICS + pushRegsCnt += genCountBits(rsPushPredicateRegs); +#endif + if (compiler->compCalleeRegsPushed != pushRegsCnt) { printf("Error: unexpected number of callee-saved registers to push. Expected: %d. Got: %d ", compiler->compCalleeRegsPushed, pushRegsCnt); dspRegMask(AllRegsMask(rsPushGprRegs, rsPushFloatRegs -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS , - RBM_NONE + rsPushPredicateRegs #endif )); printf("\n"); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 1846a062f85b36..6759f428b4990b 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5377,9 +5377,9 @@ void CodeGen::genFinalizeFrame() noway_assert( (regSet.rsGetModifiedGprRegsMask() & ~(RBM_INT_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED)) == 0); noway_assert((regSet.rsGetModifiedFloatRegsMask() & ~RBM_FLT_CALLEE_TRASH) == 0); -#ifdef TARGET_AMD64 +#ifdef FEATURE_MASKED_HW_INTRINSICS noway_assert((regSet.rsGetModifiedPredicateRegsMask() & ~RBM_MSK_CALLEE_TRASH) == 0); -#endif // TARGET_AMD64 +#endif // FEATURE_MASKED_HW_INTRINSICS #else // !TARGET_AMD64 && !TARGET_ARM64 // On x86 we save all callee saved regs so the saved reg area size is consistent regSet.rsSetGprRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); @@ -5413,7 +5413,7 @@ void CodeGen::genFinalizeFrame() regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate maskPushRegsPredicate = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; #endif @@ -5504,13 +5504,16 @@ void CodeGen::genFinalizeFrame() #endif // TARGET_LOONGARCH64 || TARGET_RISCV64 compiler->compCalleeRegsPushed = genCountBits(maskPushRegsInt) + genCountBits(maskPushRegsFloat); +#ifdef FEATURE_MASKED_HW_INTRINSICS + compiler->compCalleeRegsPushed += genCountBits(maskPushRegsPredicate); +#endif #ifdef DEBUG if (verbose) { printf("Callee-saved registers pushed: %d ", compiler->compCalleeRegsPushed); dspRegMask(AllRegsMask(maskPushRegsInt, maskPushRegsFloat -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS , maskPushRegsPredicate #endif diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index d5f3716f505861..ff3da5f9a572cd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3804,9 +3804,9 @@ class Compiler VARSET_TP lvaLongVars; // set of long (64-bit) variables #endif VARSET_TP lvaFloatVars; // set of floating-point (32-bit and 64-bit) or SIMD variables -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS VARSET_TP lvaMaskVars; // set of mask variables -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS unsigned lvaCurEpoch; // VarSets are relative to a specific set of tracked var indices. // It that changes, this changes. VarSets from different epochs @@ -6795,11 +6795,11 @@ class Compiler int m_loopVarFPCount; int m_hoistedFPExprCount; -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS int m_loopVarInOutMskCount; int m_loopVarMskCount; int m_hoistedMskExprCount; -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS // Get the VN cache for current loop VNSet* GetHoistedInCurLoop(Compiler* comp) @@ -8559,12 +8559,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX return (regMask & ~RBM_ALLFLOAT) == RBM_NONE; } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS bool IsPredicateRegMask(regMaskTP regMask) { return (regMask & ~RBM_ALLMASK) == RBM_NONE; } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS bool IsOnlyOneRegMask(regMaskTP regMask) { diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index acbcbe7ca6e172..345ed13b9afacf 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -988,7 +988,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) regMaskTP gprOrFloatMask = mask.GetGprFloatCombinedMask(); -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS // Only check this condition if there are predicate register support // is present. // If not, then gprOrFloatMask should be non-empty, and we will hit the @@ -1038,7 +1038,8 @@ inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) // The number of the first register contained in the mask and updates the `mask` to toggle // the bit. // - +//TODO: We can make certain methods on compiler object and check if predicate is needed +// and if yes, use optimize path inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) { regNumber regNum = genFirstRegNumFromMask(mask); @@ -4512,10 +4513,10 @@ inline void printRegMask(AllRegsMask mask) printf(" "); printf(REG_MASK_ALL_FMT, mask.floatRegs()); -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS printf(" "); printf(REG_MASK_ALL_FMT, mask.predicateRegs()); -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS } inline void printRegMaskInt(regMaskGpr mask) @@ -5058,8 +5059,11 @@ template FORCEINLINE int regIndexForType(T vt) { int type = varTypeRegister[TypeGet(vt)]; - assert(type <= REGISTER_TYPE_COUNT); -#ifndef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS + assert(type <= 2); +#endif + +#ifndef FEATURE_MASKED_HW_INTRINSICS assert(type != VTR_MASK); #endif return (type - 1); @@ -5238,7 +5242,7 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const #ifdef HAS_MORE_THAN_64_REGISTERS if (_hasPredicateRegister) { - assert(index <= REGISTER_TYPE_COUNT); + assert(index <= 2); RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); } @@ -5436,7 +5440,6 @@ regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { return _allRegisters; } - } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const @@ -5465,6 +5468,13 @@ bool AllRegsMask::IsGprOrFloatPresent() const return GetGprFloatCombinedMask() != RBM_NONE; } +#ifndef HAS_MORE_THAN_64_REGISTERS +regMaskTP AllRegsMask::GetAllRegistersMask() const +{ + return _allRegisters; +} +#endif + /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index bcba1907435101..b5fe718c780738 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3518,7 +3518,7 @@ void emitter::emitDispRegSet(AllRegsMask regs) { emitDispGprRegSet(regs.gprRegs()); emitDispGprRegSet(regs.floatRegs()); -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS emitDispGprRegSet(regs.predicateRegs()); #endif } diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 7b0afc6094b47c..4d3aec30b32a2b 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1169,6 +1169,14 @@ inline static bool isGeneralRegisterOrSP(regNumber reg) return isGeneralRegister(reg) || (reg == REG_SP); } // Includes REG_SP, Excludes REG_ZR +#ifdef FEATURE_MASKED_HW_INTRINSICS +inline static bool isMaskReg(regNumber reg) +{ + // TODO: This assert will no longer be true + return (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST); +} +#endif // FEATURE_MASKED_HW_INTRINSICS + inline static bool isVectorRegister(regNumber reg) { return (reg >= REG_FP_FIRST && reg <= REG_FP_LAST); diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index f01ce5be450e39..b01554277abdd8 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -26,11 +26,13 @@ inline static bool isDoubleReg(regNumber reg) return isFloatReg(reg); } +#ifdef FEATURE_MASKED_HW_INTRINSICS inline static bool isMaskReg(regNumber reg) { // TODO: This assert will no longer be true return (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST); } +#endif // FEATURE_MASKED_HW_INTRINSICS inline static bool isHighSimdReg(regNumber reg) { diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index d8a1901d909fc7..4622461a1a27c0 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -2012,13 +2012,20 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) return ins_Copy(dstType); } -#if defined(TARGET_XARCH) && defined(HAS_PREDICATE_REGS) +#if defined(FEATURE_MASKED_HW_INTRINSICS) if (genIsValidMaskReg(srcReg)) { +#if defined(TARGET_XARCH) // mask to int return INS_kmovq_gpr; +#elif defined(TARGET_ARM64) + unreached(); + return INS_mov; // TODO-SVE: needs testing +#else + unreached(); +#endif } -#endif // TARGET_XARCH && HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS // float to int assert(genIsValidFloatReg(srcReg)); @@ -2255,13 +2262,13 @@ instruction CodeGenInterface::ins_StoreFromSrc(regNumber srcReg, var_types dstTy return ins_Store(dstType, aligned); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS if (genIsValidMaskReg(srcReg)) { // mask to int, treat as mask so it works on 32-bit return ins_Store(TYP_MASK, aligned); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS // float to int, treat as float to float assert(genIsValidFloatReg(srcReg)); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index fca0323b00861b..6e9460d84dd733 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -564,7 +564,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly } else { -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS assert(varTypeUsesMaskReg(regtype)); calleeSaved = RBM_MSK_CALLEE_SAVED; calleeTrash = RBM_MSK_CALLEE_TRASH; @@ -4743,12 +4743,13 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Only focus on actual registers present deadCandidates &= actualRegistersMask; - regMaskTP deadGprFloatCandidates = deadCandidates.GetGprFloatCombinedMask(); - - updateDeadCandidatesAtBlockStart(deadGprFloatCandidates, inVarToRegMap); -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS + updateDeadCandidatesAtBlockStart(deadCandidates.GetGprFloatCombinedMask(), inVarToRegMap); updateDeadCandidatesAtBlockStart(deadCandidates.predicateRegs(), inVarToRegMap); -#endif // HAS_PREDICATE_REGS +#else + updateDeadCandidatesAtBlockStart(deadCandidates.GetAllRegistersMask(), inVarToRegMap); +#endif // HAS_MORE_THAN_64_REGISTERS + #endif // TARGET_ARM } @@ -4924,10 +4925,12 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FREE_REGS)); makeRegsAvailable(regsToFree); +#ifdef HAS_MORE_THAN_64_REGISTERS freeRegisterMask(regsToFree.GetGprFloatCombinedMask()); -#ifdef HAS_PREDICATE_REGS freeRegisterMask(regsToFree.predicateRegs()); -#endif // HAS_PREDICATE_REGS +#else + freeRegisterMask(regsToFree.GetAllRegistersMask()); +#endif // HAS_MORE_THAN_64_REGISTERS } void LinearScan::freeRegisterMask(regMaskTP freeMask) @@ -5021,10 +5024,12 @@ void LinearScan::allocateRegistersMinimal() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); +#ifdef HAS_MORE_THAN_64_REGISTERS inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); -#ifdef HAS_PREDICATE_REGS inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); -#endif +#else + inActivateRegisters(tempRegsToMakeInactive.GetAllRegistersMask()); +#endif // HAS_MORE_THAN_64_REGISTERS if (currentRefPosition.nodeLocation > prevLocation) { @@ -5709,10 +5714,12 @@ void LinearScan::allocateRegisters() // mess with the dump, since this was previously being done before the call below // to dumpRegRecords. TODO: Do not do anything if tempRegsToMakeInactive == RBM_NONE AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); +#ifdef HAS_MORE_THAN_64_REGISTERS inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); -#ifdef HAS_PREDICATE_REGS inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); -#endif +#else + inActivateRegisters(tempRegsToMakeInactive.GetAllRegistersMask()); +#endif // HAS_MORE_THAN_64_REGISTERS if (currentRefPosition.nodeLocation > prevLocation) { @@ -11773,7 +11780,7 @@ void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) regsMaskToFree = regsToFree.floatRegs(); availableRegsMask = availableFloatRegs; } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST) { regsMaskToFree = regsToFree.predicateRegs(); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 07f850cb7de49e..3e620b73802444 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -51,12 +51,12 @@ RegisterType regType(T type) { return IntRegisterType; } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (varTypeUsesMaskReg(type)) { return MaskRegisterType; } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS else { assert(varTypeUsesFloatReg(type)); @@ -498,13 +498,13 @@ class RegRecord : public Referenceable { registerType = FloatRegisterType; } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else { assert(emitter::isMaskReg(reg)); registerType = MaskRegisterType; } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS regNum = reg; isCalleeSave = ((RBM_CALLEE_SAVED & genRegMask(reg)) != 0); } @@ -1128,7 +1128,7 @@ class LinearScan : public LinearScanInterface RefPosition* defineNewInternalTemp(GenTree* tree, RegisterType regType, regMaskOnlyOne candidates); RefPosition* buildInternalIntRegisterDefForNode(GenTree* tree, regMaskGpr internalCands = RBM_NONE); RefPosition* buildInternalFloatRegisterDefForNode(GenTree* tree, regMaskFloat internalCands = RBM_NONE); -#if defined(FEATURE_SIMD) +#ifdef FEATURE_MASKED_HW_INTRINSICS RefPosition* buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskPredicate internalCands = RBM_NONE); #endif void buildInternalRegisterUses(); @@ -1768,12 +1768,11 @@ class LinearScan : public LinearScanInterface void resetAvailableRegs() { - m_AvailableRegs = AllRegsMask(availableIntRegs, availableFloatRegs -#ifdef HAS_PREDICATE_REGS - , - availableMaskRegs -#endif - ); +#ifdef HAS_MORE_THAN_64_REGISTERS + m_AvailableRegs = AllRegsMask(availableIntRegs, availableFloatRegs, availableMaskRegs); +#else + m_AvailableRegs = AllRegsMask(allAvailableRegs); +#endif // HAS_MORE_THAN_64_REGISTERS m_RegistersWithConstants.Clear(); } @@ -1795,12 +1794,13 @@ class LinearScan : public LinearScanInterface void makeRegsAvailable(AllRegsMask regMask) { m_AvailableRegs |= regMask; +#ifdef HAS_MORE_THAN_64_REGISTERS assert(compiler->IsGprRegMask(m_AvailableRegs.gprRegs())); assert(compiler->IsFloatRegMask(m_AvailableRegs.floatRegs())); -#ifdef HAS_PREDICATE_REGS assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs())); #endif } + void makeRegAvailable(regNumber reg, var_types regType) { m_AvailableRegs.AddRegNumInMask(reg ARM_ARG(regType)); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index a9b0447a61c6fc..7c8c218be0b10e 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -869,9 +869,9 @@ AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) if (!compiler->compFloatingPointUsed) { killMask.RemoveRegTypeFromMask(RBM_FLT_CALLEE_TRASH, TYP_FLOAT); -#if defined(TARGET_XARCH) || defined(HAS_PREDICATE_REGS) +#ifdef FEATURE_MASKED_HW_INTRINSICS killMask.RemoveRegTypeFromMask(RBM_MSK_CALLEE_TRASH, TYP_MASK); -#endif // TARGET_XARCH || HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS } #ifdef TARGET_ARM if (call->IsVirtualStub()) @@ -1126,9 +1126,9 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo { assert(compiler->IsGprRegMask(killMask.gprRegs())); assert(compiler->IsFloatRegMask(killMask.floatRegs())); -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS assert(compiler->IsPredicateRegMask(killMask.predicateRegs())); -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS bool insertedKills = false; @@ -1383,7 +1383,7 @@ RefPosition* LinearScan::buildInternalFloatRegisterDefForNode(GenTree* tree, reg return defRefPosition; } -#if defined(FEATURE_SIMD) && defined(TARGET_XARCH) +#ifdef FEATURE_MASKED_HW_INTRINSICS RefPosition* LinearScan::buildInternalMaskRegisterDefForNode(GenTree* tree, regMaskPredicate internalCands) { // The candidate set should contain only float registers. @@ -1391,7 +1391,7 @@ RefPosition* LinearScan::buildInternalMaskRegisterDefForNode(GenTree* tree, regM return defineNewInternalTemp(tree, MaskRegisterType, internalCands); } -#endif +#endif // FEATURE_MASKED_HW_INTRINSICS //------------------------------------------------------------------------ // buildInternalRegisterUses - adds use positions for internal @@ -2759,12 +2759,12 @@ void LinearScan::buildIntervals() { calleeSaveCount = CNT_CALLEE_ENREG; } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (varTypeUsesMaskReg(interval->registerType)) { calleeSaveCount = CNT_CALLEE_SAVED_MASK; } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS else { assert(varTypeUsesFloatReg(interval->registerType)); @@ -2822,14 +2822,19 @@ void LinearScan::buildIntervals() // availableMaskRegs = RBM_NONE; // Is this also needed? } - //TODO: Fix this for x86 and arm - actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs -#ifdef HAS_PREDICATE_REGS - , - availableMaskRegs -#endif // HAS_PREDICATE_REGS - ); - +#ifdef HAS_MORE_THAN_64_REGISTERS + actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs, availableMaskRegs); +#else + if (availableRegCount < (sizeof(regMaskTP) * 8)) + { + // Mask out the bits that are between 64 ~ availableRegCount + actualRegistersMask = AllRegsMask((1ULL << availableRegCount) - 1); + } + else + { + actualRegistersMask = AllRegsMask(~RBM_NONE); + } +#endif // HAS_MORE_THAN_64_REGISTERS #ifdef DEBUG // Make sure we don't have any blocks that were not visited @@ -4110,12 +4115,12 @@ int LinearScan::BuildReturn(GenTree* tree) { buildInternalIntRegisterDefForNode(tree, dstRegMask); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (varTypeUsesMaskReg(dstType)) { buildInternalMaskRegisterDefForNode(tree, dstRegMask); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS else { assert(varTypeUsesFloatReg(dstType)); diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 4496619851d661..f1e5ef360f53c2 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -4161,7 +4161,7 @@ bool Compiler::optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* ho hoistCtxt->m_hoistedFPExprCount = 0; } -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS if (!VarSetOps::IsEmpty(this, lvaMaskVars)) { VARSET_TP loopMskVars(VarSetOps::Intersection(this, loopVars, lvaMaskVars)); @@ -4348,7 +4348,7 @@ bool Compiler::optIsProfitableToHoistTree(GenTree* tree, FlowGraphNaturalLoop* l } #endif } -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (varTypeUsesMaskReg(tree)) { hoistedExprCount = hoistCtxt->m_hoistedMskExprCount; @@ -4361,7 +4361,7 @@ bool Compiler::optIsProfitableToHoistTree(GenTree* tree, FlowGraphNaturalLoop* l availRegCount += CNT_CALLEE_TRASH_MASK - 1; } } -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS else { assert(varTypeUsesFloatReg(tree)); diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 92bdb9334fc40d..70488a80da88b3 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -227,14 +227,14 @@ void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else { assert(genIsValidMaskReg(reg)); printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS #endif // DEBUG rsModifiedRegsMask.AddRegNumInMask(reg); @@ -252,10 +252,10 @@ void RegSet::rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressD modifiedMask.gprRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), modifiedMask.floatRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), modifiedMask.predicateRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS #endif // DEBUG rsModifiedRegsMask |= modifiedMask; @@ -364,9 +364,9 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) : m_rsCompiler(compiler), m_r rsMaskCalleeSaved = RBM_NONE; #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_RISCV64 -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS rsPredicateMaskCalleeSaved = RBM_NONE; -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS #ifdef TARGET_ARM rsMaskPreSpillRegArg = RBM_NONE; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 18a9871db8e393..8904462cd1deba 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -101,13 +101,13 @@ class RegSet return rsModifiedRegsMask.floatRegs(); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate rsGetModifiedPredicateRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask.predicateRegs(); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS regMaskGpr rsGetModifiedRegsMask(var_types type) const { @@ -183,7 +183,7 @@ class RegSet regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_LOONGARCH64 -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate rsPredicateMaskCalleeSaved; #endif diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 4af30c82f29d31..3e458cf7cef83b 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -198,13 +198,6 @@ enum _regMask_enum : unsigned #define AVAILABLE_REG_COUNT get_AVAILABLE_REG_COUNT() -#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) -#define HAS_PREDICATE_REGS -#define REGISTER_TYPE_COUNT 3 -#else -#define REGISTER_TYPE_COUNT 2 -#endif - #ifdef TARGET_ARM64 #define HAS_MORE_THAN_64_REGISTERS 1 #endif @@ -282,7 +275,7 @@ typedef struct _regMaskAll #ifdef HAS_MORE_THAN_64_REGISTERS union { - RegBitSet32 _registers[REGISTER_TYPE_COUNT]; + RegBitSet32 _registers[3]; struct { RegBitSet64 _float_gpr; @@ -309,7 +302,7 @@ typedef struct _regMaskAll }; }; // TODO: This can be moved inside the union by trimming the size of _predicateRegs: 28 - // and have this as 8 bits. However, there are chances of accidently overwritting it + // and have this as 8 bits. However, there are chances of accidently overwriting it // when updating the predicateReg mask. Evaluation this option on how we can secure it. // Find out if the TP cost is more by having this field separately vs. adding extra checks // when the predicateReg is updated. @@ -363,22 +356,16 @@ typedef struct _regMaskAll } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS inline regMaskPredicate predicateRegs() const { - static_assert((REGISTER_TYPE_COUNT == 3), "There should be 3 types of registers"); -#ifdef HAS_MORE_THAN_64_REGISTERS +#if defined(HAS_MORE_THAN_64_REGISTERS) return _predicateRegs; #else -#ifdef TARGET_AMD64 - return _allRegisters & 0xFF000000000000; -#else - // TODO: Fix this for ARM and x86 - return _allRegisters; -#endif // TARGET_AMD64 + return _allRegisters & 0xFF000000000000; // TODO: Fix the mask #endif // HAS_MORE_THAN_64_REGISTERS } -#endif +#endif // FEATURE_MASKED_HW_INTRINSICS _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS @@ -424,15 +411,6 @@ typedef struct _regMaskAll #endif } -// _regMaskAll(int (&_registers)[REGISTER_TYPE_COUNT]) -// { -//// registers[0] = _registers[0]; -//// registers[1] = _registers[1]; -////#ifdef HAS_PREDICATE_REGS -//// registers[2] = _registers[2]; -////#endif -// } - #ifdef TARGET_ARM _regMaskAll(regNumber reg) : _regMaskAll() { @@ -472,6 +450,9 @@ typedef struct _regMaskAll // TODO: this might go away once we have just `regMaskTP` gpr_float field FORCEINLINE bool IsGprOrFloatPresent() const; FORCEINLINE regMaskTP GetGprFloatCombinedMask() const; +#ifndef HAS_MORE_THAN_64_REGISTERS + FORCEINLINE regMaskTP GetAllRegistersMask() const; +#endif // !HAS_MORE_THAN_64_REGISTERS FORCEINLINE void operator|=(const _regMaskAll& other); FORCEINLINE void operator&=(const _regMaskAll& other); @@ -532,12 +513,12 @@ typedef unsigned __int64 regMaskSmall; #error Unsupported or unset target architecture #endif -#ifdef HAS_PREDICATE_REGS +#ifdef HAS_MORE_THAN_64_REGISTERS #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) #else - #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED) - #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH) + #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_CALLEE_SAVED) + #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_CALLEE_TRASH) #endif #ifdef TARGET_XARCH @@ -609,7 +590,6 @@ const char* getRegName(regNumber reg); #ifdef DEBUG const char* getRegNameFloat(regNumber reg, var_types type); -extern void dspRegMask(regMaskOnlyOne mask, size_t minSiz = 0); extern void dspRegMask(AllRegsMask mask, size_t minSiz = 0); #endif diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 632d13bb637522..745075b9b24c05 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -173,7 +173,6 @@ #define RBM_CALLEE_TRASH (RBM_INT_CALLEE_TRASH | RBM_FLT_CALLEE_TRASH | RBM_MSK_CALLEE_TRASH) #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) // This will need to change once floatRegs are changed to 4-bytes - #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 0db4d0abbb7d43..b43e47111c44d1 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -113,6 +113,8 @@ #define CNT_CALLEE_SAVED_FLOAT (8) #define CNT_CALLEE_TRASH_FLOAT (24) + #define CNT_CALLEE_SAVED_MASK (0) + #define CNT_CALLEE_TRASH_MASK (0) #define CALLEE_SAVED_REG_MAXSZ (CNT_CALLEE_SAVED * REGSIZE_BYTES) #define CALLEE_SAVED_FLOAT_MAXSZ (CNT_CALLEE_SAVED_FLOAT * FPSAVE_REGSIZE_BYTES) diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 4cf4215bb0a373..51be148e38af94 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -182,12 +182,12 @@ void Compiler::unwindPushPopCFI(regNumber reg) { shouldCreateCfiCode = (RBM_FLT_CALLEE_SAVED & mask); } -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (emitter::isMaskReg(reg) && (RBM_MSK_CALLEE_SAVED & mask)) { shouldCreateCfiCode = (RBM_MSK_CALLEE_SAVED & mask); } -#endif // HAS_PREDICATE_REGS +#endif // FEATURE_MASKED_HW_INTRINSICS if (shouldCreateCfiCode) { diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index d5698faabd6a1b..3324b8031c9d8e 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -298,7 +298,7 @@ const char* getRegNameFloat(regNumber reg, var_types type) */ const char* dspRegRange(regMaskOnlyOne regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) { -#ifdef HAS_PREDICATE_REGS +#ifdef FEATURE_MASKED_HW_INTRINSICS assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || ((regFirst == REG_FP_FIRST) && (regLast == REG_FP_LAST)) || ((regFirst == REG_MASK_FIRST) && (regLast == REG_MASK_LAST))); @@ -444,17 +444,6 @@ const char* dspRegRange(regMaskOnlyOne regMask, size_t& minSiz, const char* sep, return sep; } -void dspRegMask(regMaskOnlyOne mask, size_t minSiz) -{ - dspRegMask(AllRegsMask(mask, mask -#ifdef HAS_PREDICATE_REGS - , - mask -#endif - ), - minSiz); -} - /***************************************************************************** * * Displays a register set. From 6c2eb77eeec45fd1acb2cd453ae61e7b28574e91 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sun, 31 Mar 2024 12:13:49 -0700 Subject: [PATCH 127/201] jit format --- src/coreclr/jit/compiler.hpp | 14 +++++++------- src/coreclr/jit/instr.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/target.h | 25 +++++++++++-------------- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 345ed13b9afacf..6552ac842c0ba6 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1038,7 +1038,7 @@ inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) // The number of the first register contained in the mask and updates the `mask` to toggle // the bit. // -//TODO: We can make certain methods on compiler object and check if predicate is needed +// TODO: We can make certain methods on compiler object and check if predicate is needed // and if yes, use optimize path inline regNumber genFirstRegNumFromMaskAndToggle(AllRegsMask& mask) { @@ -5129,7 +5129,7 @@ AllRegsMask AllRegsMask::operator~() { #ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; - result._float_gpr = ~_float_gpr; + result._float_gpr = ~_float_gpr; result._predicateRegs = ~_predicateRegs; return result; #else @@ -5253,7 +5253,7 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const } } -//regMaskOnlyOne& AllRegsMask::operator[](int index) +// regMaskOnlyOne& AllRegsMask::operator[](int index) //{ // assert(index <= REGISTER_TYPE_COUNT); // return _registers[index]; @@ -5359,7 +5359,7 @@ void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) if (_hasPredicateRegister) { RegBitSet64 value = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); + int index = regIndexForRegister(reg); _registers[index] |= encodeForIndex(index, value); } else @@ -5375,7 +5375,7 @@ void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) if (_hasPredicateRegister) { RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); + int index = regIndexForRegister(reg); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); } else @@ -5455,8 +5455,8 @@ regMaskTP AllRegsMask::GetGprFloatCombinedMask() const // Moving this definition to lsra might work, but this is temporary so // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in // the comment above. - //regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; - //return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); + // regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; + // return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); return _float_gpr; #else return _allRegisters; diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 4622461a1a27c0..06ebe9ef2e2850 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -2065,7 +2065,7 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) #if defined(TARGET_XARCH) return INS_kmovq_gpr; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing + unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 7c8c218be0b10e..e2a7829ae6888a 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2833,7 +2833,7 @@ void LinearScan::buildIntervals() else { actualRegistersMask = AllRegsMask(~RBM_NONE); - } + } #endif // HAS_MORE_THAN_64_REGISTERS #ifdef DEBUG diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 3e458cf7cef83b..89bc4a59d13cfa 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -253,7 +253,7 @@ typedef unsigned __int64 singleRegMask; #else // x86 and arm -typedef unsigned regMaskTP; +typedef unsigned regMaskTP; typedef unsigned RegBitSet64; #define regMaskGpr regMaskTP @@ -273,32 +273,30 @@ typedef struct _regMaskAll { private: #ifdef HAS_MORE_THAN_64_REGISTERS - union - { + union { RegBitSet32 _registers[3]; struct { RegBitSet64 _float_gpr; RegBitSet32 _predicateRegs; - //RegBitSet32 _predicateRegs : 28; - //bool _hasPredicateRegister : 8; - + // RegBitSet32 _predicateRegs : 28; + // bool _hasPredicateRegister : 8; }; struct { // This one is just to have uniform code when `if(_hasPredicateRegister)` is used RegBitSet64 _allRegisters; RegBitSet32 _predicateRegs; - //RegBitSet32 _predicateRegs : 28; - //bool _hasPredicateRegister : 8; + // RegBitSet32 _predicateRegs : 28; + // bool _hasPredicateRegister : 8; }; struct { RegBitSet32 _gprRegs; RegBitSet32 _floatRegs; RegBitSet32 _predicateRegs; - //RegBitSet32 _predicateRegs : 28; - //bool _hasPredicateRegister : 8; + // RegBitSet32 _predicateRegs : 28; + // bool _hasPredicateRegister : 8; }; }; // TODO: This can be moved inside the union by trimming the size of _predicateRegs: 28 @@ -308,10 +306,10 @@ typedef struct _regMaskAll // when the predicateReg is updated. bool _hasPredicateRegister; #else - RegBitSet64 _allRegisters; + RegBitSet64 _allRegisters; #endif - regMaskOnlyOne operator[](int index) const; + regMaskOnlyOne operator[](int index) const; regMaskOnlyOne& operator[](int index); // This method shifts the high-32 bits of float to low-32 bits and return. // For gpr and predicate registers, it returns the same value. @@ -349,11 +347,10 @@ typedef struct _regMaskAll #elif defined(TARGET_ARM64) return _allRegisters & 0xFFFFFFFF00000000; #else - //TODO: Fix this for ARM and x86 + // TODO: Fix this for ARM and x86 return _allRegisters; #endif // TARGET_AMD64 #endif // HAS_MORE_THAN_64_REGISTERS - } #ifdef FEATURE_MASKED_HW_INTRINSICS From f94e6cf15de488564c25ae8ebc9792fe4b4eeaff Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 11:16:59 -0700 Subject: [PATCH 128/201] Arm64: Have predicate registers from 64~79 --- src/coreclr/jit/compiler.hpp | 4 ++-- src/coreclr/jit/registerarm64.h | 41 ++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 6552ac842c0ba6..f7f93b70417155 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -999,7 +999,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) } else { - regNum = (regNumber)BitOperations::BitScanForward(mask.predicateRegs()); + regNum = (regNumber)(64 + BitOperations::BitScanForward(mask.predicateRegs())); } #else regNum = (regNumber)BitOperations::BitScanForward(gprOrFloatMask); @@ -5060,7 +5060,7 @@ FORCEINLINE int regIndexForType(T vt) { int type = varTypeRegister[TypeGet(vt)]; #ifdef HAS_MORE_THAN_64_REGISTERS - assert(type <= 2); + assert(type <= 3); #endif #ifndef FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/registerarm64.h b/src/coreclr/jit/registerarm64.h index d864db3db26c44..b6da3ff3d4548a 100644 --- a/src/coreclr/jit/registerarm64.h +++ b/src/coreclr/jit/registerarm64.h @@ -94,28 +94,31 @@ REGDEF(V29, 29+VBASE, VMASK(29), "d29", "s29", 1) REGDEF(V30, 30+VBASE, VMASK(30), "d30", "s30", 1) REGDEF(V31, 31+VBASE, VMASK(31), "d31", "s31", 1) -// TODO-SVE: Fix once we add predicate registers -REGALIAS(P0, V0) -REGALIAS(P1, V1) -REGALIAS(P2, V2) -REGALIAS(P3, V3) -REGALIAS(P4, V4) -REGALIAS(P5, V5) -REGALIAS(P6, V6) -REGALIAS(P7, V7) -REGALIAS(P8, V8) -REGALIAS(P9, V9) -REGALIAS(P10, V10) -REGALIAS(P11, V11) -REGALIAS(P12, V12) -REGALIAS(P13, V13) -REGALIAS(P14, V14) -REGALIAS(P15, V15) +#define PBASE 64 +#define PMASK(x) (1ULL << x) +/* +REGDEF(name, rnum, mask, xname, wname, regTypeTag) */ +REGDEF(P0, 0+PBASE, PMASK(0), "p0" , "na", 2) +REGDEF(P1, 1+PBASE, PMASK(1), "p1" , "na", 2) +REGDEF(P2, 2+PBASE, PMASK(2), "p2" , "na", 2) +REGDEF(P3, 3+PBASE, PMASK(3), "p3" , "na", 2) +REGDEF(P4, 4+PBASE, PMASK(4), "p4" , "na", 2) +REGDEF(P5, 5+PBASE, PMASK(5), "p5" , "na", 2) +REGDEF(P6, 6+PBASE, PMASK(6), "p6" , "na", 2) +REGDEF(P7, 7+PBASE, PMASK(7), "p7" , "na", 2) +REGDEF(P8, 8+PBASE, PMASK(8), "p8" , "na", 2) +REGDEF(P9, 9+PBASE, PMASK(9), "p9" , "na", 2) +REGDEF(P10, 10+PBASE, PMASK(10), "p10", "na", 2) +REGDEF(P11, 11+PBASE, PMASK(11), "p11", "na", 2) +REGDEF(P12, 12+PBASE, PMASK(12), "p12", "na", 2) +REGDEF(P13, 13+PBASE, PMASK(13), "p13", "na", 2) +REGDEF(P14, 14+PBASE, PMASK(14), "p14", "na", 2) +REGDEF(P15, 15+PBASE, PMASK(15), "p15", "na", 2) -// The registers with values 64 (NBASE) and above are not real register numbers -#define NBASE 64 +// The registers with values 80 (NBASE) and above are not real register numbers +#define NBASE 80 REGDEF(SP, 0+NBASE, 0x0000, "sp", "wsp?", -1) // This must be last! From d56020079ca4618597d96491f2cd3553b86e31a5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 11:18:26 -0700 Subject: [PATCH 129/201] Add FEATURE_MASKED_HW_INSTRINSICS at few more places --- src/coreclr/jit/lsra.cpp | 4 ++-- src/coreclr/jit/lsra.h | 4 ++-- src/coreclr/jit/optimizer.cpp | 4 ++-- src/coreclr/jit/typelist.h | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 6e9460d84dd733..114e8c2cf16f9f 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -834,7 +834,7 @@ LinearScan::LinearScan(Compiler* theCompiler) availableFloatRegs = RBM_ALLFLOAT; availableDoubleRegs = RBM_ALLDOUBLE; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#ifdef FEATURE_MASKED_HW_INTRINSICS availableMaskRegs = RBM_ALLMASK; #endif @@ -846,7 +846,7 @@ LinearScan::LinearScan(Compiler* theCompiler) availableIntRegs &= ~RBM_INT_CALLEE_SAVED | RBM_ENC_CALLEE_SAVED; availableFloatRegs &= ~RBM_FLT_CALLEE_SAVED; availableDoubleRegs &= ~RBM_FLT_CALLEE_SAVED; -#if defined(TARGET_XARCH) +#ifdef FEATURE_MASKED_HW_INTRINSICS availableMaskRegs &= ~RBM_MSK_CALLEE_SAVED; #endif // TARGET_XARCH } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 3e620b73802444..6e6f4624323654 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1670,13 +1670,13 @@ class LinearScan : public LinearScanInterface PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#ifdef FEATURE_MASKED_HW_INTRINSICS PhasedVar availableMaskRegs; #endif PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, // predicate -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#ifdef FEATURE_MASKED_HW_INTRINSICS #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) #else #define allAvailableRegs (availableIntRegs | availableFloatRegs) diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index f1e5ef360f53c2..3e77b92a9497ea 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -5517,7 +5517,7 @@ void Compiler::optComputeInterestingVarSets() #ifndef TARGET_64BIT VarSetOps::AssignNoCopy(this, lvaLongVars, VarSetOps::MakeEmpty(this)); #endif -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS VarSetOps::AssignNoCopy(this, lvaMaskVars, VarSetOps::MakeEmpty(this)); #endif @@ -5536,7 +5536,7 @@ void Compiler::optComputeInterestingVarSets() VarSetOps::AddElemD(this, lvaLongVars, varDsc->lvVarIndex); } #endif -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS else if (varTypeUsesMaskReg(varDsc->lvType)) { VarSetOps::AddElemD(this, lvaMaskVars, varDsc->lvVarIndex); diff --git a/src/coreclr/jit/typelist.h b/src/coreclr/jit/typelist.h index 1a9a8c4072f6bf..2ef837567d670e 100644 --- a/src/coreclr/jit/typelist.h +++ b/src/coreclr/jit/typelist.h @@ -64,9 +64,9 @@ DEF_TP(SIMD16 ,"simd16" , TYP_SIMD16, 16,16, 16, 4,16, VTR_FLOAT, available DEF_TP(SIMD32 ,"simd32" , TYP_SIMD32, 32,32, 32, 8,16, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S|VTF_VEC) DEF_TP(SIMD64 ,"simd64" , TYP_SIMD64, 64,64, 64, 16,16, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S|VTF_VEC) #endif // TARGET_XARCH -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#ifdef FEATURE_MASKED_HW_INTRINSICS DEF_TP(MASK ,"mask" , TYP_MASK, 8, 8, 8, 2, 8, VTR_MASK, availableMaskRegs, RBM_MSK_CALLEE_SAVED, RBM_MSK_CALLEE_TRASH, VTF_S) -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // FEATURE_MASKED_HW_INTRINSICS #endif // FEATURE_SIMD DEF_TP(UNKNOWN ,"unknown" ,TYP_UNKNOWN, 0, 0, 0, 0, 0, VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY) From 5ff6d61dc1d92c659b0f477a938a45e01c912a88 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 11:19:11 -0700 Subject: [PATCH 130/201] Have own version of updateDeadCandidatesAtBlockStart for AllRegsMask --- src/coreclr/jit/lsra.cpp | 39 +++++++++++++++++++++++++++++++++++++-- src/coreclr/jit/lsra.h | 1 + 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 114e8c2cf16f9f..547c9e1bdbbf39 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4383,6 +4383,42 @@ void LinearScan::resetAllRegistersState() } } +void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap) +{ + while (!deadRegMask.IsEmpty()) + { + regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); + RegRecord* physRegRecord = getRegisterRecord(reg); + + makeRegAvailable(reg, physRegRecord->registerType); + Interval* assignedInterval = physRegRecord->assignedInterval; + + if (assignedInterval != nullptr) + { + assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + + if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) + { + assignedInterval->isActive = false; + if (assignedInterval->getNextRefPosition() == nullptr) + { + unassignPhysReg(physRegRecord, nullptr); + } + if (!assignedInterval->IsUpperVector()) + { + inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; + } + } + else + { + // This interval may still be active, but was in another register in an + // intervening block. + clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); + } + } + } +} + void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap) { while (deadRegMask != RBM_NONE) @@ -4744,8 +4780,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) deadCandidates &= actualRegistersMask; #ifdef HAS_MORE_THAN_64_REGISTERS - updateDeadCandidatesAtBlockStart(deadCandidates.GetGprFloatCombinedMask(), inVarToRegMap); - updateDeadCandidatesAtBlockStart(deadCandidates.predicateRegs(), inVarToRegMap); + updateDeadCandidatesAtBlockStart(deadCandidates, inVarToRegMap); #else updateDeadCandidatesAtBlockStart(deadCandidates.GetAllRegistersMask(), inVarToRegMap); #endif // HAS_MORE_THAN_64_REGISTERS diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 6e6f4624323654..a3e8ce77b89eb2 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1014,6 +1014,7 @@ class LinearScan : public LinearScanInterface void processBlockStartLocations(BasicBlock* current); void processBlockEndLocations(BasicBlock* current); void resetAllRegistersState(); + FORCEINLINE void updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap); FORCEINLINE void updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap); #ifdef TARGET_ARM From 8d6a2b6ee354884d36152f1126ffab4cfac105af Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 11:19:27 -0700 Subject: [PATCH 131/201] Fix resetting availableMaskRegs --- src/coreclr/jit/lsrabuild.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index e2a7829ae6888a..745d845cdbffbe 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2813,13 +2813,15 @@ void LinearScan::buildIntervals() RefPosition* pos = newRefPosition((Interval*)nullptr, currentLoc, RefTypeBB, nullptr, RBM_NONE); } - needNonIntegerRegisters |= compiler->compFloatingPointUsed; + needNonIntegerRegisters |= compiler->compFloatingPointUsed; // TODO: Also track about mask registers if (!needNonIntegerRegisters) { availableRegCount = REG_INT_COUNT; availableFloatRegs.OverrideAssign(RBM_NONE); availableDoubleRegs.OverrideAssign(RBM_NONE); - // availableMaskRegs = RBM_NONE; // Is this also needed? +#ifdef FEATURE_MASKED_HW_INTRINSICS + availableMaskRegs.OverrideAssign(RBM_NONE); +#endif } #ifdef HAS_MORE_THAN_64_REGISTERS From 57a0746a5719a50c8584de95eacd6ff55c0eb336 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 12:31:05 -0700 Subject: [PATCH 132/201] Remove some unneeded methods from AllRegsMask --- src/coreclr/jit/compiler.hpp | 24 ------------------------ src/coreclr/jit/target.h | 4 +--- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index f7f93b70417155..214ad6c2871e6d 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5191,24 +5191,6 @@ AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const } } -AllRegsMask AllRegsMask::operator&(const regNumber reg) const -{ -#ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - AllRegsMask result = *this; - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - result[index] &= encodeForIndex(index, value); - return result; - } - else -#endif - { - return _allRegisters & genRegMask(reg); - } -} - void AllRegsMask::Clear() { #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5253,12 +5235,6 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const } } -// regMaskOnlyOne& AllRegsMask::operator[](int index) -//{ -// assert(index <= REGISTER_TYPE_COUNT); -// return _registers[index]; -//} - void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 89bc4a59d13cfa..b316cbe7254b41 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -309,8 +309,6 @@ typedef struct _regMaskAll RegBitSet64 _allRegisters; #endif - regMaskOnlyOne operator[](int index) const; - regMaskOnlyOne& operator[](int index); // This method shifts the high-32 bits of float to low-32 bits and return. // For gpr and predicate registers, it returns the same value. FORCEINLINE static RegBitSet32 encodeForIndex(int index, RegBitSet64 value) @@ -451,6 +449,7 @@ typedef struct _regMaskAll FORCEINLINE regMaskTP GetAllRegistersMask() const; #endif // !HAS_MORE_THAN_64_REGISTERS + regMaskOnlyOne operator[](int index) const; FORCEINLINE void operator|=(const _regMaskAll& other); FORCEINLINE void operator&=(const _regMaskAll& other); FORCEINLINE void operator|=(const regNumber reg); @@ -460,7 +459,6 @@ typedef struct _regMaskAll FORCEINLINE bool operator!=(const _regMaskAll& other); FORCEINLINE _regMaskAll operator&(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator|(const _regMaskAll& other) const; - FORCEINLINE _regMaskAll operator&(const regNumber reg) const; } AllRegsMask; From 1e960b444ef89eb480872655dc066b2f46c519d1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 12:31:47 -0700 Subject: [PATCH 133/201] Arm64: Update predicate callee-saved and callee-trash registers --- src/coreclr/jit/targetarm64.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index b43e47111c44d1..491646d46548d1 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -113,8 +113,8 @@ #define CNT_CALLEE_SAVED_FLOAT (8) #define CNT_CALLEE_TRASH_FLOAT (24) - #define CNT_CALLEE_SAVED_MASK (0) - #define CNT_CALLEE_TRASH_MASK (0) + #define CNT_CALLEE_SAVED_MASK (4) + #define CNT_CALLEE_TRASH_MASK (8) #define CALLEE_SAVED_REG_MAXSZ (CNT_CALLEE_SAVED * REGSIZE_BYTES) #define CALLEE_SAVED_FLOAT_MAXSZ (CNT_CALLEE_SAVED_FLOAT * FPSAVE_REGSIZE_BYTES) @@ -152,8 +152,8 @@ #define RBM_ALLMASK (RBM_LOWMASK | RBM_HIGHMASK) // TODO-SVE: Fix when adding predicate register allocation - #define RBM_MSK_CALLEE_SAVED (0) - #define RBM_MSK_CALLEE_TRASH (0) + #define RBM_MSK_CALLEE_SAVED (RBM_P0 | RBM_P1 | RBM_P2 | RBM_P3) + #define RBM_MSK_CALLEE_TRASH (RBM_P4 | RBM_P5 | RBM_P6 | RBM_P7 | RBM_P8 | RBM_P9 | RBM_P10 | RBM_P11 | RBM_P12 | RBM_P13 | RBM_P14 | RBM_P15) // ARM64 write barrier ABI (see vm\arm64\asmhelpers.asm, vm\arm64\asmhelpers.S): // CORINFO_HELP_ASSIGN_REF (JIT_WriteBarrier), CORINFO_HELP_CHECKED_ASSIGN_REF (JIT_CheckedWriteBarrier): From adc51ac71c207b5251e726578e5e9cd668279348 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 12:32:32 -0700 Subject: [PATCH 134/201] update save/restore of predicate registers on arm64 --- src/coreclr/jit/codegen.h | 4 +- src/coreclr/jit/codegenarm64.cpp | 72 +++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 023be30a61111a..344011375cd1a9 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -404,8 +404,10 @@ class CodeGen final : public CodeGenInterface { regMaskGpr fiSaveGprRegs; // Set of callee-saved GPR registers saved in the funclet prolog (includes LR) regMaskFloat fiSaveFloatRegs; // Set of callee-saved float registers saved in the funclet prolog (includes LR) - regMaskFloat fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskPredicate fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog // (includes LR) +#endif int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index e4797b1bee617a..d7bcc18bf406c5 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -38,6 +38,9 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; regMaskFloat rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskPredicate rsRestorePredicateRegs = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; +#endif if (isFramePointerUsed()) { @@ -202,7 +205,11 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) } JITDUMP(" calleeSaveSpOffset=%d, calleeSaveSpDelta=%d\n", calleeSaveSpOffset, calleeSaveSpDelta); - genRestoreCalleeSavedRegistersHelp(AllRegsMask(rsRestoreGprRegs, rsRestoreFloatRegs), calleeSaveSpOffset, + genRestoreCalleeSavedRegistersHelp(AllRegsMask(rsRestoreGprRegs, rsRestoreFloatRegs +#ifdef FEATURE_MASKED_HW_INTRINSICS + ,rsRestorePredicateRegs +#endif + ), calleeSaveSpOffset, calleeSaveSpDelta); switch (frameType) @@ -916,6 +923,13 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo assert(compiler->IsFloatRegMask(maskSaveRegsFloat)); unsigned regsToSaveCount = genCountBits(maskSaveRegsFloat) + genCountBits(maskSaveRegsInt); + +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskTP maskSaveRegsPredicate = regsToSaveMask.predicateRegs(); + assert(compiler->IsPredicateRegMask(maskSaveRegsPredicate)); + regsToSaveCount += genCountBits(maskSaveRegsPredicate); +#endif + if (regsToSaveCount == 0) { if (spDelta != 0) @@ -932,6 +946,16 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo // We also can save FP and LR, even though they are not in RBM_CALLEE_SAVED. assert(regsToSaveCount <= genCountBits(RBM_CALLEE_SAVED | RBM_FP | RBM_LR)); + +#ifdef FEATURE_MASKED_HW_INTRINSICS + if (maskSaveRegsPredicate != RBM_NONE) + { + genSaveCalleeSavedRegisterGroup(maskSaveRegsPredicate, spDelta, lowestCalleeSavedOffset); + spDelta = 0; + lowestCalleeSavedOffset += genCountBits(maskSaveRegsPredicate) * FPSAVE_REGSIZE_BYTES; + } +#endif // FEATURE_MASKED_HW_INTRINSICS + // Save integer registers at higher addresses than floating-point registers. if (maskSaveRegsFloat != RBM_NONE) { @@ -1034,6 +1058,13 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, assert(compiler->IsFloatRegMask(maskRestoreRegsFloat)); unsigned regsToRestoreCount = genCountBits(maskRestoreRegsInt) + genCountBits(maskRestoreRegsFloat); + +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskTP maskRestoreRegsPredicate = regsToRestoreMask.predicateRegs(); + assert(compiler->IsPredicateRegMask(maskRestoreRegsPredicate)); + regsToRestoreCount += genCountBits(maskRestoreRegsPredicate); +#endif + if (regsToRestoreCount == 0) { if (spDelta != 0) @@ -1068,8 +1099,16 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, { // If there is any spDelta, it must be used here. genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset); - // No need to update spOffset since it's not used after this. + spOffset -= genCountBits(maskRestoreRegsInt) * FPSAVE_REGSIZE_BYTES; + } + +#ifdef FEATURE_MASKED_HW_INTRINSICS + if (maskRestoreRegsPredicate != RBM_NONE) + { + // TODO: Do we need to adjust spDelta? + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsPredicate, spDelta, spOffset); } +#endif } // clang-format off @@ -1380,6 +1419,10 @@ void CodeGen::genFuncletProlog(BasicBlock* block) regMaskFloat maskSaveRegsFloat = genFuncletInfo.fiSaveFloatRegs; regMaskGpr maskSaveRegsInt = genFuncletInfo.fiSaveGprRegs; +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskPredicate maskSaveRegsPredicate = genFuncletInfo.fiSavePredicateRegs; +#endif + // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskSaveRegsInt & RBM_LR) != 0); assert((maskSaveRegsInt & RBM_FP) != 0); @@ -1496,7 +1539,11 @@ void CodeGen::genFuncletProlog(BasicBlock* block) int lowestCalleeSavedOffset = genFuncletInfo.fiSP_to_CalleeSave_delta + genFuncletInfo.fiSpDelta2; // We haven't done the second adjustment of SP yet (if any) - genSaveCalleeSavedRegistersHelp(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat), lowestCalleeSavedOffset, 0); + genSaveCalleeSavedRegistersHelp(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat +#ifdef FEATURE_MASKED_HW_INTRINSICS + , maskSaveRegsPredicate +#endif + ), lowestCalleeSavedOffset, 0); if ((genFuncletInfo.fiFrameType == 3) || (genFuncletInfo.fiFrameType == 5)) { @@ -1585,6 +1632,10 @@ void CodeGen::genFuncletEpilog() regMaskFloat maskRestoreRegsFloat = genFuncletInfo.fiSaveFloatRegs; regMaskGpr maskRestoreRegsInt = genFuncletInfo.fiSaveGprRegs; +#ifdef FEATURE_MASKED_HW_INTRINSICS + regMaskPredicate maskRestoreRegsPredicate = genFuncletInfo.fiSavePredicateRegs; +#endif + // Funclets must always save LR and FP, since when we have funclets we must have an FP frame. assert((maskRestoreRegsInt & RBM_LR) != 0); assert((maskRestoreRegsInt & RBM_FP) != 0); @@ -1611,7 +1662,11 @@ void CodeGen::genFuncletEpilog() maskRestoreRegsInt &= ~(RBM_LR | RBM_FP); // We restore FP/LR at the end } int lowestCalleeSavedOffset = genFuncletInfo.fiSP_to_CalleeSave_delta + genFuncletInfo.fiSpDelta2; - genRestoreCalleeSavedRegistersHelp(AllRegsMask(maskRestoreRegsInt, maskRestoreRegsFloat), lowestCalleeSavedOffset, + genRestoreCalleeSavedRegistersHelp(AllRegsMask(maskRestoreRegsInt, maskRestoreRegsFloat +#ifdef FEATURE_MASKED_HW_INSTRINSICS + , maskRestoreRegsPredicate +#endif + ), lowestCalleeSavedOffset, 0); if (genFuncletInfo.fiFrameType == 1) @@ -1875,7 +1930,9 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; +#ifdef FEATURE_MASKED_HW_INTRINSICS genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; +#endif genFuncletInfo.fiSP_to_FPLR_save_delta = SP_to_FPLR_save_delta; genFuncletInfo.fiSP_to_PSP_slot_delta = SP_to_PSP_slot_delta; genFuncletInfo.fiSP_to_CalleeSave_delta = SP_to_PSP_slot_delta + PSPSize; @@ -1887,8 +1944,11 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf("\n"); printf("Funclet prolog / epilog info\n"); printf(" Save regs: "); - dspRegMask(AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs, - genFuncletInfo.fiSavePredicateRegs)); + dspRegMask(AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs +#ifdef FEATURE_MASKED_HW_INTRINSICS + , genFuncletInfo.fiSavePredicateRegs +#endif + )); printf("\n"); if (compiler->opts.IsOSR()) { From 06e31ecc31d999445a65f51c02dc5ad6602e109f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 16:17:37 -0700 Subject: [PATCH 135/201] Remove _hasPredicateRegister --- src/coreclr/jit/compiler.hpp | 219 ++++++++++------------------------- src/coreclr/jit/target.h | 27 ++--- 2 files changed, 71 insertions(+), 175 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 214ad6c2871e6d..7a3b8e27cd9eb7 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5096,33 +5096,23 @@ void AllRegsMask::operator&=(const AllRegsMask& other) void AllRegsMask::operator|=(const regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] |= encodeForIndex(index, value); - } - else + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] |= encodeForIndex(index, value); +#else + _allRegisters |= genRegMask(reg); #endif - { - _allRegisters |= genRegMask(reg); - } } void AllRegsMask::operator^=(const regNumber reg) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] ^= encodeForIndex(index, value); - } - else + int index = regIndexForRegister(reg); + RegBitSet64 value = genRegMask(reg); + _registers[index] ^= encodeForIndex(index, value); +#else + _allRegisters ^= genRegMask(reg); #endif - { - _allRegisters ^= genRegMask(reg); - } } AllRegsMask AllRegsMask::operator~() @@ -5140,16 +5130,11 @@ AllRegsMask AllRegsMask::operator~() bool AllRegsMask::operator==(const AllRegsMask& other) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - bool result = (_float_gpr == other._float_gpr) && (_predicateRegs == other._predicateRegs); - return result; - } - else + bool result = (_float_gpr == other._float_gpr) && (_predicateRegs == other._predicateRegs); + return result; +#else + return _allRegisters == other._allRegisters; #endif - { - return _allRegisters == other._allRegisters; - } } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5160,35 +5145,25 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - AllRegsMask result; - result._float_gpr = _float_gpr & other._float_gpr; - result._predicateRegs = _predicateRegs & other._predicateRegs; - return result; - } - else + AllRegsMask result; + result._float_gpr = _float_gpr & other._float_gpr; + result._predicateRegs = _predicateRegs & other._predicateRegs; + return result; +#else + return _allRegisters & other._allRegisters; #endif - { - return _allRegisters & other._allRegisters; - } } AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - AllRegsMask result; - result._float_gpr = _float_gpr | other._float_gpr; - result._predicateRegs = _predicateRegs | other._predicateRegs; - return result; - } - else + AllRegsMask result; + result._float_gpr = _float_gpr | other._float_gpr; + result._predicateRegs = _predicateRegs | other._predicateRegs; + return result; +#else + return _allRegisters | other._allRegisters; #endif - { - return _allRegisters | other._allRegisters; - } } void AllRegsMask::Clear() @@ -5203,7 +5178,7 @@ void AllRegsMask::Clear() bool AllRegsMask::IsEmpty() { -#if HAS_MORE_THAN_64_REGISTERS +#ifdef HAS_MORE_THAN_64_REGISTERS return ((_float_gpr | _predicateRegs) == RBM_NONE); #else return _allRegisters == RBM_NONE; @@ -5222,32 +5197,22 @@ unsigned AllRegsMask::Count() regMaskOnlyOne AllRegsMask::operator[](int index) const { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - assert(index <= 2); - RegBitSet32 value = _registers[index]; - return decodeForIndex(index, value); - } - else + assert(index <= 2); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); +#else + return _allRegisters; #endif - { - return _allRegisters; - } } void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForType(type); - _registers[index] |= encodeForIndex(index, maskToAdd); - } - else + int index = regIndexForType(type); + _registers[index] |= encodeForIndex(index, maskToAdd); +#else + _allRegisters |= maskToAdd; #endif - { - _allRegisters |= maskToAdd; - } } void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) @@ -5278,50 +5243,17 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) #ifdef TARGET_ARM void AllRegsMask::AddRegNumInMask(regNumber reg) { -#ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 value = genRegMask(reg); - _registers[index] |= encodeForIndex(index, value); - } - else -#endif - { - _allRegisters |= genRegMask(reg); - } + _allRegisters |= genRegMask(reg); } void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { -#ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToRemove = genRegMask(reg); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); - } - else -#endif - { - _allRegisters &= ~genRegMask(reg); - } + _allRegisters &= ~genRegMask(reg); } bool AllRegsMask::IsRegNumInMask(regNumber reg) { -#ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToCheck = genRegMask(reg); - return (_float_gpr & regMaskToCheck) != RBM_NONE; - } - else -#endif - { - return (_allRegisters & genRegMask(reg)) != RBM_NONE; - } + return (_allRegisters & genRegMask(reg)) != RBM_NONE; } #endif @@ -5332,64 +5264,44 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - RegBitSet64 value = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] |= encodeForIndex(index, value); - } - else + RegBitSet64 value = genRegMask(reg ARM_ARG(type)); + int index = regIndexForRegister(reg); + _registers[index] |= encodeForIndex(index, value); +#else + _allRegisters |= genRegMask(reg ARM_ARG(type)); #endif - { - _allRegisters |= genRegMask(reg ARM_ARG(type)); - } } void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); - } - else + RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); + int index = regIndexForRegister(reg); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#else + _allRegisters &= ~genRegMask(reg ARM_ARG(type)); #endif - { - _allRegisters &= ~genRegMask(reg ARM_ARG(type)); - } } void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForType(type); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); - } - else + int index = regIndexForType(type); + _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#else + _allRegisters &= ~regMaskToRemove; #endif - { - _allRegisters &= ~regMaskToRemove; - } } bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; - } - else + int index = regIndexForRegister(reg); + RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); + return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; +#else + return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; #endif - { - return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; - } } bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const @@ -5405,17 +5317,12 @@ bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const { #ifdef HAS_MORE_THAN_64_REGISTERS - if (_hasPredicateRegister) - { - int index = regIndexForType(type); - RegBitSet32 value = _registers[index]; - return decodeForIndex(index, value); - } - else + int index = regIndexForType(type); + RegBitSet32 value = _registers[index]; + return decodeForIndex(index, value); +#else + return _allRegisters; #endif - { - return _allRegisters; - } } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index b316cbe7254b41..31d14d8065f5ea 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -299,12 +299,6 @@ typedef struct _regMaskAll // bool _hasPredicateRegister : 8; }; }; - // TODO: This can be moved inside the union by trimming the size of _predicateRegs: 28 - // and have this as 8 bits. However, there are chances of accidently overwriting it - // when updating the predicateReg mask. Evaluation this option on how we can secure it. - // Find out if the TP cost is more by having this field separately vs. adding extra checks - // when the predicateReg is updated. - bool _hasPredicateRegister; #else RegBitSet64 _allRegisters; #endif @@ -351,20 +345,22 @@ typedef struct _regMaskAll #endif // HAS_MORE_THAN_64_REGISTERS } +//#ifdef DEBUG #ifdef FEATURE_MASKED_HW_INTRINSICS inline regMaskPredicate predicateRegs() const { -#if defined(HAS_MORE_THAN_64_REGISTERS) +#ifdef HAS_MORE_THAN_64_REGISTERS return _predicateRegs; #else return _allRegisters & 0xFF000000000000; // TODO: Fix the mask #endif // HAS_MORE_THAN_64_REGISTERS } #endif // FEATURE_MASKED_HW_INTRINSICS +//#endif // DEBUG _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE) #else : _allRegisters(floatRegMask | gprRegMask) #endif @@ -374,7 +370,7 @@ typedef struct _regMaskAll // TODO: See if we can avoid the '|' operation here. _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegs) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegs), _hasPredicateRegister(true) + : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegs) #else : _allRegisters(predicateRegs | floatRegMask | gprRegMask) #endif @@ -383,7 +379,7 @@ typedef struct _regMaskAll _regMaskAll() #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) + : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE) #else : _allRegisters(RBM_NONE) #endif @@ -392,20 +388,13 @@ typedef struct _regMaskAll _regMaskAll(RegBitSet64 allRegistersMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE), _hasPredicateRegister(true) + : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE) #else : _allRegisters(allRegistersMask) #endif { } - FORCEINLINE void NeedPredicateRegisterTracking() - { -#ifdef HAS_MORE_THAN_64_REGISTERS - _hasPredicateRegister = true; -#endif - } - #ifdef TARGET_ARM _regMaskAll(regNumber reg) : _regMaskAll() { @@ -449,7 +438,7 @@ typedef struct _regMaskAll FORCEINLINE regMaskTP GetAllRegistersMask() const; #endif // !HAS_MORE_THAN_64_REGISTERS - regMaskOnlyOne operator[](int index) const; + FORCEINLINE regMaskOnlyOne operator[](int index) const; FORCEINLINE void operator|=(const _regMaskAll& other); FORCEINLINE void operator&=(const _regMaskAll& other); FORCEINLINE void operator|=(const regNumber reg); From e4700f83a0e6f35ba64ba939ebceb32ccefb864e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 22:13:20 -0700 Subject: [PATCH 136/201] fix gprRegs() --- src/coreclr/jit/target.h | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 31d14d8065f5ea..5a9cb4b9b59cb0 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -273,7 +273,8 @@ typedef struct _regMaskAll { private: #ifdef HAS_MORE_THAN_64_REGISTERS - union { + union + { RegBitSet32 _registers[3]; struct { @@ -320,15 +321,23 @@ typedef struct _regMaskAll public: inline regMaskGpr gprRegs() const { + RegBitSet64 resultMask; +#ifdef HAS_MORE_THAN_64_REGISTERS + resultMask = _float_gpr; +#else + resultMask = _allRegisters; +#endif // HAS_MORE_THAN_64_REGISTERS + #ifdef TARGET_AMD64 - return _allRegisters & 0xFFFF; + return resultMask & 0xFFFF; #elif TARGET_ARM64 - return _allRegisters & 0xFFFFFFFF; + return resultMask & 0xFFFFFFFF; #else // TODO: Fix this for ARM and x86 - return _allRegisters; -#endif + return resultMask; +#endif // TARGET_AMD64 } + inline regMaskFloat floatRegs() const { #ifdef HAS_MORE_THAN_64_REGISTERS From 6763c70452278a924abbbecaae7c9dcd11bea801 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 22:22:01 -0700 Subject: [PATCH 137/201] Restruture AllRegsMask to remove build errors for linux --- src/coreclr/jit/target.h | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 5a9cb4b9b59cb0..a29bb8ed2c057a 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -278,31 +278,27 @@ typedef struct _regMaskAll RegBitSet32 _registers[3]; struct { - RegBitSet64 _float_gpr; + union + { + RegBitSet64 _float_gpr; + RegBitSet64 _allRegisters; // just so no need to do #ifdef just to access this field + struct + { + RegBitSet32 _gprRegs; + RegBitSet32 _floatRegs; + }; + }; RegBitSet32 _predicateRegs; - // RegBitSet32 _predicateRegs : 28; - // bool _hasPredicateRegister : 8; - }; - struct - { - // This one is just to have uniform code when `if(_hasPredicateRegister)` is used - RegBitSet64 _allRegisters; - RegBitSet32 _predicateRegs; - // RegBitSet32 _predicateRegs : 28; - // bool _hasPredicateRegister : 8; - }; - struct - { - RegBitSet32 _gprRegs; - RegBitSet32 _floatRegs; - RegBitSet32 _predicateRegs; - // RegBitSet32 _predicateRegs : 28; - // bool _hasPredicateRegister : 8; }; }; #else + //TODO: This is also declared above, so revisit MORE_THAN_64_REGISTERS and see where we differentiated just for this RegBitSet64 _allRegisters; #endif +//TODO: Come up with a name of variable such that: +// 1. If HAS_MORE_THAN_64_REGISTERS==1, it represents float_gpr combined +// 2. If HAS_MORE_THAN_64_REGISTERS==0, it represents all registers possible be - gpr/float/predicate in same place +// Once we have that, we can just use and remove some of the #ifdef HAS_MORE_THAN_64_REGISTERS // This method shifts the high-32 bits of float to low-32 bits and return. // For gpr and predicate registers, it returns the same value. From 5a69528dd1a0c67b20239e284a5951361642c697 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 22:38:39 -0700 Subject: [PATCH 138/201] Replace _float_gpr and _allRegisters with _combinedRegisters --- src/coreclr/jit/compiler.hpp | 112 ++++++++++++----------------------- src/coreclr/jit/target.h | 50 +++++++--------- 2 files changed, 57 insertions(+), 105 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 7a3b8e27cd9eb7..c4199e5f9ab247 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5071,25 +5071,17 @@ FORCEINLINE int regIndexForType(T vt) void AllRegsMask::operator|=(const AllRegsMask& other) { + _combinedRegisters |= other._combinedRegisters; #ifdef HAS_MORE_THAN_64_REGISTERS - // TODO: Can we optimize to reintrepret_cast - // Something like https://godbolt.org/z/1KevT8Edh - _float_gpr |= other._float_gpr; _predicateRegs |= other._predicateRegs; -#else - _allRegisters |= other._allRegisters; #endif } void AllRegsMask::operator&=(const AllRegsMask& other) { + _combinedRegisters &= other._combinedRegisters; #ifdef HAS_MORE_THAN_64_REGISTERS - // TODO: Can we optimize to reintrepret_cast - // Something like https://godbolt.org/z/1KevT8Edh - _float_gpr &= other._float_gpr; _predicateRegs &= other._predicateRegs; -#else - _allRegisters &= other._allRegisters; #endif } @@ -5100,7 +5092,7 @@ void AllRegsMask::operator|=(const regNumber reg) RegBitSet64 value = genRegMask(reg); _registers[index] |= encodeForIndex(index, value); #else - _allRegisters |= genRegMask(reg); + _combinedRegisters |= genRegMask(reg); #endif } @@ -5111,30 +5103,27 @@ void AllRegsMask::operator^=(const regNumber reg) RegBitSet64 value = genRegMask(reg); _registers[index] ^= encodeForIndex(index, value); #else - _allRegisters ^= genRegMask(reg); + _combinedRegisters ^= genRegMask(reg); #endif } AllRegsMask AllRegsMask::operator~() { -#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; - result._float_gpr = ~_float_gpr; + result._combinedRegisters = ~_combinedRegisters; +#ifdef HAS_MORE_THAN_64_REGISTERS result._predicateRegs = ~_predicateRegs; - return result; -#else - return ~_allRegisters; #endif + return result; } bool AllRegsMask::operator==(const AllRegsMask& other) { + return (_combinedRegisters == other._combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS - bool result = (_float_gpr == other._float_gpr) && (_predicateRegs == other._predicateRegs); - return result; -#else - return _allRegisters == other._allRegisters; + && (_predicateRegs == other._predicateRegs); #endif + ; } bool AllRegsMask::operator!=(const AllRegsMask& other) @@ -5144,54 +5133,48 @@ bool AllRegsMask::operator!=(const AllRegsMask& other) AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const { -#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; - result._float_gpr = _float_gpr & other._float_gpr; + result._combinedRegisters = _combinedRegisters & other._combinedRegisters; +#ifdef HAS_MORE_THAN_64_REGISTERS result._predicateRegs = _predicateRegs & other._predicateRegs; - return result; -#else - return _allRegisters & other._allRegisters; #endif + return result; } AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const { -#ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask result; - result._float_gpr = _float_gpr | other._float_gpr; + result._combinedRegisters = _combinedRegisters | other._combinedRegisters; +#ifdef HAS_MORE_THAN_64_REGISTERS result._predicateRegs = _predicateRegs | other._predicateRegs; - return result; -#else - return _allRegisters | other._allRegisters; #endif + return result; } void AllRegsMask::Clear() { + _combinedRegisters = RBM_NONE; #ifdef HAS_MORE_THAN_64_REGISTERS - _float_gpr = RBM_NONE; _predicateRegs = RBM_NONE; -#else - _allRegisters = RBM_NONE; #endif } bool AllRegsMask::IsEmpty() { #ifdef HAS_MORE_THAN_64_REGISTERS - return ((_float_gpr | _predicateRegs) == RBM_NONE); + return ((_combinedRegisters | _predicateRegs) == RBM_NONE); #else - return _allRegisters == RBM_NONE; + return _combinedRegisters == RBM_NONE; #endif } unsigned AllRegsMask::Count() { + return genCountBits(_combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS - return genCountBits(_float_gpr) + genCountBits(_predicateRegs); -#else - return genCountBits(_allRegisters); + + genCountBits(_predicateRegs) #endif + ; } regMaskOnlyOne AllRegsMask::operator[](int index) const @@ -5201,7 +5184,7 @@ regMaskOnlyOne AllRegsMask::operator[](int index) const RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); #else - return _allRegisters; + return _combinedRegisters; #endif } @@ -5211,7 +5194,7 @@ void AllRegsMask::AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type) int index = regIndexForType(type); _registers[index] |= encodeForIndex(index, maskToAdd); #else - _allRegisters |= maskToAdd; + _combinedRegisters |= maskToAdd; #endif } @@ -5220,18 +5203,13 @@ void AllRegsMask::AddGprRegMask(regMaskOnlyOne maskToAdd) #ifdef HAS_MORE_THAN_64_REGISTERS _registers[0] |= maskToAdd; #else - _allRegisters |= maskToAdd; + _combinedRegisters |= maskToAdd; #endif } void AllRegsMask::AddFloatRegMask(regMaskOnlyOne maskToAdd) { -#ifdef HAS_MORE_THAN_64_REGISTERS - // TODO: Add assert that maskToAdd has low-32 bit set to 0 - _float_gpr |= maskToAdd; -#else - _allRegisters |= maskToAdd; -#endif + _combinedRegisters |= maskToAdd; } // Adds reg only if it is gpr register @@ -5243,17 +5221,17 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) #ifdef TARGET_ARM void AllRegsMask::AddRegNumInMask(regNumber reg) { - _allRegisters |= genRegMask(reg); + _combinedRegisters |= genRegMask(reg); } void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { - _allRegisters &= ~genRegMask(reg); + _combinedRegisters &= ~genRegMask(reg); } bool AllRegsMask::IsRegNumInMask(regNumber reg) { - return (_allRegisters & genRegMask(reg)) != RBM_NONE; + return (_combinedRegisters & genRegMask(reg)) != RBM_NONE; } #endif @@ -5268,7 +5246,7 @@ void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) int index = regIndexForRegister(reg); _registers[index] |= encodeForIndex(index, value); #else - _allRegisters |= genRegMask(reg ARM_ARG(type)); + _combinedRegisters |= genRegMask(reg ARM_ARG(type)); #endif } @@ -5279,7 +5257,7 @@ void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) int index = regIndexForRegister(reg); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); #else - _allRegisters &= ~genRegMask(reg ARM_ARG(type)); + _combinedRegisters &= ~genRegMask(reg ARM_ARG(type)); #endif } @@ -5289,7 +5267,7 @@ void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_type int index = regIndexForType(type); _registers[index] &= ~encodeForIndex(index, regMaskToRemove); #else - _allRegisters &= ~regMaskToRemove; + _combinedRegisters &= ~regMaskToRemove; #endif } @@ -5300,7 +5278,7 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; #else - return (_allRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; + return (_combinedRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; #endif } @@ -5321,29 +5299,13 @@ regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const RegBitSet32 value = _registers[index]; return decodeForIndex(index, value); #else - return _allRegisters; + return _combinedRegisters; #endif } regMaskTP AllRegsMask::GetGprFloatCombinedMask() const { -#ifdef HAS_MORE_THAN_64_REGISTERS - // TODO: NEed to revert this once we change floatRegs to 32-bits - // #ifdef TARGET_64BIT - // return ((regMaskTP)floatRegs() << 32) | gprRegs(); - // #else - // return ((regMaskTP)floatRegs() << REG_INT_COUNT) | gprRegs(); - // #endif - // TODO: Cannot get RBM_ALLFLOAT because it is not defined for AllRegsMask - // Moving this definition to lsra might work, but this is temporary so - // just create gprMAsk. Eventually, we will use the `<< 32` mentioned in - // the comment above. - // regMaskTP gprMask = (1ULL << REG_INT_COUNT) - 1; - // return (floatRegs() & ~gprMask) | (gprRegs() & gprMask); - return _float_gpr; -#else - return _allRegisters; -#endif + return _combinedRegisters; } bool AllRegsMask::IsGprOrFloatPresent() const @@ -5352,9 +5314,9 @@ bool AllRegsMask::IsGprOrFloatPresent() const } #ifndef HAS_MORE_THAN_64_REGISTERS -regMaskTP AllRegsMask::GetAllRegistersMask() const +RegBitSet64 AllRegsMask::GetAllRegistersMask() const { - return _allRegisters; + return _combinedRegisters; } #endif diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a29bb8ed2c057a..f23e2d36297f55 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -280,8 +280,8 @@ typedef struct _regMaskAll { union { - RegBitSet64 _float_gpr; - RegBitSet64 _allRegisters; // just so no need to do #ifdef just to access this field + // Represents combined registers bitset including gpr/float + RegBitSet64 _combinedRegisters; struct { RegBitSet32 _gprRegs; @@ -292,8 +292,9 @@ typedef struct _regMaskAll }; }; #else - //TODO: This is also declared above, so revisit MORE_THAN_64_REGISTERS and see where we differentiated just for this - RegBitSet64 _allRegisters; + // Represents combined registers bitset including gpr/float and on some platforms + // mask or predicate registers + RegBitSet64 _combinedRegisters; #endif //TODO: Come up with a name of variable such that: // 1. If HAS_MORE_THAN_64_REGISTERS==1, it represents float_gpr combined @@ -317,37 +318,26 @@ typedef struct _regMaskAll public: inline regMaskGpr gprRegs() const { - RegBitSet64 resultMask; -#ifdef HAS_MORE_THAN_64_REGISTERS - resultMask = _float_gpr; -#else - resultMask = _allRegisters; -#endif // HAS_MORE_THAN_64_REGISTERS - #ifdef TARGET_AMD64 - return resultMask & 0xFFFF; + return _combinedRegisters & 0xFFFF; #elif TARGET_ARM64 - return resultMask & 0xFFFFFFFF; + return _combinedRegisters & 0xFFFFFFFF; #else // TODO: Fix this for ARM and x86 - return resultMask; + return _combinedRegisters; #endif // TARGET_AMD64 } inline regMaskFloat floatRegs() const { -#ifdef HAS_MORE_THAN_64_REGISTERS - return _float_gpr & 0xFFFFFFFF00000000; -#else #ifdef TARGET_AMD64 - return _allRegisters & 0xFFFFFFFF0000; + return _combinedRegisters & 0xFFFFFFFF0000; #elif defined(TARGET_ARM64) - return _allRegisters & 0xFFFFFFFF00000000; + return _combinedRegisters & 0xFFFFFFFF00000000; #else // TODO: Fix this for ARM and x86 - return _allRegisters; + return _combinedRegisters; #endif // TARGET_AMD64 -#endif // HAS_MORE_THAN_64_REGISTERS } //#ifdef DEBUG @@ -357,7 +347,7 @@ typedef struct _regMaskAll #ifdef HAS_MORE_THAN_64_REGISTERS return _predicateRegs; #else - return _allRegisters & 0xFF000000000000; // TODO: Fix the mask + return _combinedRegisters & 0xFF000000000000; // TODO: Fix the mask #endif // HAS_MORE_THAN_64_REGISTERS } #endif // FEATURE_MASKED_HW_INTRINSICS @@ -365,9 +355,9 @@ typedef struct _regMaskAll _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE) + : _combinedRegisters(floatRegMask | gprRegMask), _predicateRegs(RBM_NONE) #else - : _allRegisters(floatRegMask | gprRegMask) + : _combinedRegisters(floatRegMask | gprRegMask) #endif { } @@ -375,27 +365,27 @@ typedef struct _regMaskAll // TODO: See if we can avoid the '|' operation here. _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask, RegBitSet64 predicateRegs) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegs) + : _combinedRegisters(floatRegMask | gprRegMask), _predicateRegs((RegBitSet32)predicateRegs) #else - : _allRegisters(predicateRegs | floatRegMask | gprRegMask) + : _combinedRegisters(predicateRegs | floatRegMask | gprRegMask) #endif { } _regMaskAll() #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(RBM_NONE), _predicateRegs(RBM_NONE) + : _combinedRegisters(RBM_NONE), _predicateRegs(RBM_NONE) #else - : _allRegisters(RBM_NONE) + : _combinedRegisters(RBM_NONE) #endif { } _regMaskAll(RegBitSet64 allRegistersMask) #ifdef HAS_MORE_THAN_64_REGISTERS - : _float_gpr(allRegistersMask), _predicateRegs(RBM_NONE) + : _combinedRegisters(allRegistersMask), _predicateRegs(RBM_NONE) #else - : _allRegisters(allRegistersMask) + : _combinedRegisters(allRegistersMask) #endif { } From 704fcb13172a35b18a87687a6ca02fe61be50c63 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 23:38:51 -0700 Subject: [PATCH 139/201] fix assert in arm64 --- src/coreclr/jit/codegenarmarch.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 1082219c1abf9a..dadbc9611cd2ef 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3545,15 +3545,16 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { for (unsigned j = 0; j < arg.AbiInfo.NumRegs; j++) { - regNumber reg = arg.AbiInfo.GetRegNum(j); - if ((trashedGprByEpilog & genRegMask(reg)) != 0) + regNumber reg = arg.AbiInfo.GetRegNum(j); + var_types argType = arg.AbiInfo.ArgType; + if (varTypeUsesIntReg(argType) && ((trashedGprByEpilog & genRegMask(reg)) != 0)) { JITDUMP("Tail call node:\n"); DISPTREE(call); JITDUMP("Gpr Register used: %s\n", getRegName(reg)); assert(!"Argument to tailcall may be trashed by epilog"); } - else if ((trashedFloatByEpilog & genRegMask(reg)) != 0) + else if (varTypeUsesFloatArgReg(argType) && ((trashedFloatByEpilog & genRegMask(reg)) != 0)) { JITDUMP("Tail call node:\n"); DISPTREE(call); @@ -3561,7 +3562,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) assert(!"Argument to tailcall may be trashed by epilog"); } #ifdef FEATURE_MASKED_HW_INTRINSICS - else if ((trashedPredicateByEpilog & genRegMask(reg)) != 0) + else if (varTypeUsesMaskReg(argType) && ((trashedPredicateByEpilog & genRegMask(reg)) != 0)) { JITDUMP("Tail call node:\n"); DISPTREE(call); From dda6d8a46b7a5e6e99561cfaa6b03cf1ecb3f4bf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 1 Apr 2024 23:39:18 -0700 Subject: [PATCH 140/201] fix some #define and remove Create_AllRegsMask() --- src/coreclr/jit/target.h | 1 - src/coreclr/jit/targetamd64.h | 19 ++++++++++++++----- src/coreclr/jit/targetarm.h | 6 +++++- src/coreclr/jit/targetarm64.h | 6 +++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index f23e2d36297f55..00259a15747391 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -447,7 +447,6 @@ typedef struct _regMaskAll } AllRegsMask; #define GprRegsMask(gprRegs) AllRegsMask(gprRegs) -#define Create_AllRegsMask(gprRegs, floatRegs) AllRegsMask(gprRegs | floatRegs) #if REGMASK_BITS == 32 typedef unsigned regMaskSmall; diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 745075b9b24c05..69714ad02603a4 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -532,15 +532,24 @@ // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. // STOP_FOR_GC helper preserves all the 4 possible return registers. - #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) + +#ifdef HAS_MORE_THAN_64_REGISTERS + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH) +#else + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)) | RBM_MSK_CALLEE_TRASH) +#endif // HAS_MORE_THAN_64_REGISTERS #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) - #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_INTRET | RBM_INTRET_1), ~(RBM_FLOATRET | RBM_FLOATRET_1))) #else // See vm\amd64\asmhelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_FLOATRET))) +#ifdef HAS_MORE_THAN_64_REGISTERS + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH) +#else + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET) | RBM_MSK_CALLEE_TRASH) +#endif // HAS_MORE_THAN_64_REGISTERS #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) - #define AllRegsMask_PROFILER_LEAVE_TRASH (AllRegsMask_CALLEE_TRASH & AllRegsMask(~(RBM_INTRET | RBM_FLOATRET))) -#endif +#endif // UNIX_AMD64_ABI + + #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 50c325a027db21..1245163bf2afc5 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -246,7 +246,11 @@ // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). // See vm\arm\amshelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11), ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) +#ifdef HAS_MORE_THAN_64_REGISTERS + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7)), RBM_MSK_CALLEE_TRASH) +#else + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7)) | RBM_MSK_CALLEE_TRASH) +#endif // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 491646d46548d1..79bccdbd3e5cb8 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -253,7 +253,11 @@ #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 // The registers trashed by profiler enter/leave/tailcall hook - #define AllRegsMask_PROFILER_ENTER_TRASH (AllRegsMask_CALLEE_TRASH & Create_AllRegsMask(~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP), ~RBM_FLTARG_REGS)) +#ifdef HAS_MORE_THAN_64_REGISTERS + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) +#else + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS) | RBM_MSK_CALLEE_TRASH) +#endif #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH From 8adc9f7f235dc5b113fec1781a5a8d662b89f6f4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 09:48:41 -0700 Subject: [PATCH 141/201] fix arm build break --- src/coreclr/jit/targetarm.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 1245163bf2afc5..ef663bdcdf5cd4 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -246,11 +246,7 @@ // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). // See vm\arm\amshelpers.asm for more details. -#ifdef HAS_MORE_THAN_64_REGISTERS - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7)), RBM_MSK_CALLEE_TRASH) -#else - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7)) | RBM_MSK_CALLEE_TRASH) -#endif +#define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) From 4461bdd5d40fcce9401ae592064001af84226ada Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 12:06:12 -0700 Subject: [PATCH 142/201] Arm64: Increase REGNUM_BITS 6->7 --- src/coreclr/jit/emit.h | 10 +++++----- src/coreclr/jit/targetarm64.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index d77536d9a98dcc..8af42b8ea1b7f1 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -753,7 +753,7 @@ class emitter // x86: 38 bits // amd64: 38 bits // arm: 32 bits - // arm64: 44 bits + // arm64: 46 bits // loongarch64: 28 bits // risc-v: 28 bits @@ -824,7 +824,7 @@ class emitter // x86: 48 bits // amd64: 48 bits // arm: 48 bits - // arm64: 53 bits + // arm64: 55 bits // loongarch64: 46 bits // risc-v: 46 bits @@ -837,7 +837,7 @@ class emitter #if defined(TARGET_ARM) #define ID_EXTRA_BITFIELD_BITS (16) #elif defined(TARGET_ARM64) -#define ID_EXTRA_BITFIELD_BITS (21) +#define ID_EXTRA_BITFIELD_BITS (23) #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) #define ID_EXTRA_BITFIELD_BITS (14) #elif defined(TARGET_XARCH) @@ -879,7 +879,7 @@ class emitter // x86: 54/50 bits // amd64: 55/50 bits // arm: 54/50 bits - // arm64: 60/55 bits + // arm64: 62/57 bits // loongarch64: 53/48 bits // risc-v: 53/48 bits CLANG_FORMAT_COMMENT_ANCHOR; @@ -896,7 +896,7 @@ class emitter // x86: 10/14 bits // amd64: 9/14 bits // arm: 10/14 bits - // arm64: 4/9 bits + // arm64: 2/7 bits // loongarch64: 11/16 bits // risc-v: 11/16 bits CLANG_FORMAT_COMMENT_ANCHOR; diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 79bccdbd3e5cb8..9f8cde81f40ce1 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -63,7 +63,7 @@ static_assert_no_msg(REG_PREDICATE_HIGH_LAST == REG_PREDICATE_LAST); - #define REGNUM_BITS 6 // number of bits in a REG_* + #define REGNUM_BITS 7 // number of bits in a REG_* #define REGSIZE_BYTES 8 // number of bytes in one general purpose register #define FP_REGSIZE_BYTES 16 // number of bytes in one FP/SIMD register #define FPSAVE_REGSIZE_BYTES 8 // number of bytes in one FP/SIMD register that are saved/restored, for callee-saved registers From 5d4f507d1c2ab7641a5a8d87a0a59544f6e1c79a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 12:25:40 -0700 Subject: [PATCH 143/201] fix sve unit test --- src/coreclr/jit/codegenarm64test.cpp | 6 +++--- src/coreclr/jit/emitarm64sve.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 946e38ff46707d..750daa569613ff 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4748,7 +4748,7 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_CE_2C theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P0, REG_V31, 1, INS_OPTS_SCALABLE_H, INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V1, REG_P1, 0, INS_OPTS_SCALABLE_H, + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P1, REG_V1, 0, INS_OPTS_SCALABLE_H, INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] // IF_SVE_CE_2D @@ -5396,7 +5396,7 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_D); // UXTW .D, /M, .D // IF_SVE_AR_4A - theEmitter->emitIns_R_R_R_R(INS_sve_mla, EA_SCALABLE, REG_V0, REG_P0, REG_P0, REG_V19, + theEmitter->emitIns_R_R_R_R(INS_sve_mla, EA_SCALABLE, REG_V0, REG_P0, REG_V2, REG_V19, INS_OPTS_SCALABLE_B); // MLA ., /M, ., . theEmitter->emitIns_R_R_R_R(INS_sve_mls, EA_SCALABLE, REG_V2, REG_P1, REG_V31, REG_V31, INS_OPTS_SCALABLE_H); // MLS ., /M, ., . @@ -5578,7 +5578,7 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_H); // CPY ., /Z, #{, } theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P0, -32768, INS_OPTS_SCALABLE_S); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_P0, REG_V0, 32512, + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V0, REG_P0, 32512, INS_OPTS_SCALABLE_D); // MOV ., /Z, #{, } // IF_SVE_BV_2A_A diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 8a0894426cfb53..63fb80089c80b1 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -13350,8 +13350,8 @@ void emitter::emitInsSveSanityCheck(instrDesc* id) case IF_SVE_DP_2A: // ........xx...... .......MMMMddddd -- SVE saturating inc/dec vector by predicate count case IF_SVE_DN_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec vector by predicate count assert(insOptsScalableAtLeastHalf(id->idInsOpt())); // xx - assert(isPredicateRegister(id->idReg1())); // MMMM - assert(isVectorRegister(id->idReg2())); // ddddd + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // MMMM assert(isScalableVectorSize(id->idOpSize())); break; From fc1ff9228ef6e8a589457328a37a24098797d8ad Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 14:27:26 -0700 Subject: [PATCH 144/201] fix osx/arm64 replay failures --- src/coreclr/jit/lsrabuild.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 745d845cdbffbe..bf68bf3935abed 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3077,7 +3077,7 @@ void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandi { // In case of multi-reg call node, we have to query the i'th position return register. // For all other cases of multi-reg definitions, the registers must be in sequential order. - regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, compiler->info.compCallConv); + regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, tree->AsCall()->GetUnmanagedCallConv()); assert(dstCandidates.IsRegNumInMask(thisReg)); dstCandidates.RemoveRegNumFromMask(thisReg); From f27b326dc2403d167f79cbc0371f1bdf91aaa13c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 17:44:19 -0700 Subject: [PATCH 145/201] fix dumpRegMask() --- src/coreclr/jit/compiler.cpp | 17 +++++++++++++++-- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/lsra.cpp | 19 ++++++++++++++----- src/coreclr/jit/lsra.h | 5 +++++ src/coreclr/jit/utils.cpp | 4 ++-- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 43c092255714d7..deb9cb11de2adf 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3730,9 +3730,22 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) return rejectThisPromo; } -void Compiler::dumpRegMask(regMaskOnlyOne mask) const +void Compiler::dumpRegMask(regMaskOnlyOne mask, var_types type) const { - dumpRegMask(AllRegsMask(mask, mask, mask)); + if (varTypeUsesIntReg(type)) + { + dumpRegMask(AllRegsMask(mask, RBM_NONE, RBM_NONE)); + } + else if (varTypeUsesFloatReg(type)) + { + dumpRegMask(AllRegsMask(RBM_NONE, mask, RBM_NONE)); + } +#ifdef FEATURE_MASKED_HW_INTRINSICS + else + { + dumpRegMask(AllRegsMask(RBM_NONE, RBM_NONE, mask)); + } +#endif } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 73254387b008cc..2a253a26357dc4 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10927,7 +10927,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool compJitHaltMethod(); - void dumpRegMask(regMaskOnlyOne mask) const; + void dumpRegMask(regMaskOnlyOne mask, var_types type) const; void dumpRegMask(AllRegsMask mask) const; #endif diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 547c9e1bdbbf39..3ad62c64b94a08 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -545,8 +545,7 @@ regMaskOnlyOne LinearScan::stressLimitRegs(RefPosition* refPosition, regMaskOnly if (refPosition != nullptr) { minRegCount = refPosition->minRegCandidateCount; - RegisterType currRegType = refPosition->isIntervalRef() ? refPosition->getInterval()->registerType - : refPosition->getReg()->registerType; + RegisterType currRegType = refPosition->getRegisterType(); assert(regtype == currRegType); } @@ -10449,7 +10448,17 @@ void RefPosition::dump(LinearScan* linearScan) printf(FMT_BB " ", this->bbNum); printf("regmask="); - linearScan->compiler->dumpRegMask(registerAssignment); + var_types type = TYP_UNKNOWN; + if ((refType == RefTypeBB) || (refType == RefTypeKillGCRefs)) + { + // These refTypes do not have intervals + type = TYP_INT; + } + else + { + type = getRegisterType(); + } + linearScan->compiler->dumpRegMask(registerAssignment, type); printf(" minReg=%d", minRegCandidateCount); @@ -10585,10 +10594,10 @@ void Interval::dump(Compiler* compiler) printf(" physReg:%s", getRegName(physReg)); printf(" Preferences="); - compiler->dumpRegMask(this->registerPreferences); + compiler->dumpRegMask(this->registerPreferences, this->registerType); printf(" Aversions="); - compiler->dumpRegMask(this->registerAversion); + compiler->dumpRegMask(this->registerAversion, this->registerType); if (relatedInterval) { printf(" RelatedInterval "); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index a3e8ce77b89eb2..c6baa36a906aef 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2645,6 +2645,11 @@ class RefPosition return genRegNumFromMask(registerAssignment); } + RegisterType getRegisterType() + { + return isIntervalRef() ? getInterval()->registerType : getReg()->registerType; + } + // Returns true if it is a reference on a GenTree node. bool IsActualRef() { diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 3324b8031c9d8e..21a8f24510bd62 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -461,9 +461,9 @@ void dspRegMask(AllRegsMask mask, size_t minSiz) sep = dspRegRange(mask.gprRegs(), minSiz, sep, REG_INT_FIRST, REG_INT_LAST); sep = dspRegRange(mask.floatRegs(), minSiz, sep, REG_FP_FIRST, REG_FP_LAST); -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS sep = dspRegRange(mask.predicateRegs(), minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS printf("]"); From e9f0a46de7fb54655c5536dc4f3e6621516a9421 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:19:42 -0700 Subject: [PATCH 146/201] Introduce MORE_THAN_64_REG_ARG --- src/coreclr/jit/target.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 00259a15747391..5a5d0a5666e51d 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -200,6 +200,9 @@ enum _regMask_enum : unsigned #ifdef TARGET_ARM64 #define HAS_MORE_THAN_64_REGISTERS 1 +#define MORE_THAN_64_REG_ARG(x) , x +#else +#define MORE_THAN_64_REG_ARG(x) #endif /*****************************************************************************/ From f6f1e03dbd5f59200c2130ce308bf28a5cf168b1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:22:23 -0700 Subject: [PATCH 147/201] Update genRegNumFromMask() to take MORE_THAN_64_REG_ARG --- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/codegencommon.cpp | 6 +++--- src/coreclr/jit/compiler.hpp | 6 +++++- src/coreclr/jit/emit.cpp | 2 +- src/coreclr/jit/gcencode.cpp | 2 +- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/lsra.cpp | 14 +++++++------- src/coreclr/jit/lsra.h | 4 +++- src/coreclr/jit/lsrabuild.cpp | 2 +- 9 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index d7bcc18bf406c5..74127c8bb59040 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5713,7 +5713,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, // We pick the next lowest register number for rLimit noway_assert(availMask != RBM_NONE); tempMask = genFindLowestBit(availMask); - rLimit = genRegNumFromMask(tempMask); + rLimit = genRegNumFromMask(tempMask MORE_THAN_64_REG_ARG(TYP_INT)); // Generate: // diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 6759f428b4990b..8c75b9e757a77b 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -3654,7 +3654,7 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere // We pick the lowest avail register number regMaskFloat tempMask = genFindLowestBit(fpAvailMask); - xtraReg = genRegNumFromMask(tempMask); + xtraReg = genRegNumFromMask(tempMask MORE_THAN_64_REG_ARG(TYP_FLOAT)); } #if defined(TARGET_X86) // This case shouldn't occur on x86 since NYI gets converted to an assert @@ -5905,7 +5905,7 @@ void CodeGen::genFnProlog() // We will use one of the registers that we were planning to zero init anyway. // We pick the lowest register number. tempMask = genFindLowestBit(tempMask); - initReg = genRegNumFromMask(tempMask); + initReg = genRegNumFromMask(tempMask MORE_THAN_64_REG_ARG(TYP_INT)); } // Next we prefer to use one of the unused argument registers. // If they aren't available we use one of the caller-saved integer registers. @@ -5916,7 +5916,7 @@ void CodeGen::genFnProlog() { // We pick the lowest register number tempMask = genFindLowestBit(tempMask); - initReg = genRegNumFromMask(tempMask); + initReg = genRegNumFromMask(tempMask MORE_THAN_64_REG_ARG(TYP_INT)); } } diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index c4199e5f9ab247..af7572004a8653 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -931,7 +931,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block) // Assumptions: // The mask contains one and only one register. -inline regNumber genRegNumFromMask(regMaskOnlyOne mask) +inline regNumber genRegNumFromMask(regMaskOnlyOne mask MORE_THAN_64_REG_ARG(var_types type)) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -943,6 +943,10 @@ inline regNumber genRegNumFromMask(regMaskOnlyOne mask) assert(genRegMask(regNum) == mask); +#ifdef HAS_MORE_THAN_64_REGISTERS + // If this is mask type, add `64` to the regNumber + return (regNumber)((varTypeUsesMaskReg(type) << 6) + regNum); +#endif return regNum; } diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index b5fe718c780738..d7e5e2fd3fabfd 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -8966,7 +8966,7 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskGpr regs, BYTE* addr) do { regMaskGpr bit = genFindLowestBit(chg); - regNumber reg = genRegNumFromMask(bit); + regNumber reg = genRegNumFromMask(bit MORE_THAN_64_REG_ARG(TYP_INT)); if (life & bit) { diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 5012045ded1291..f5a7d9b9f00e38 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -4645,7 +4645,7 @@ void GCInfo::gcInfoRecordGCRegStateChange(GcInfoEncoder* gcInfoEncoder, } // Figure out which register the next bit corresponds to. - regNumber regNum = genRegNumFromMask(tmpMask); + regNumber regNum = genRegNumFromMask(tmpMask MORE_THAN_64_REG_ARG(TYP_INT)); /* Reserve SP future use */ assert(regNum != REG_SPBASE); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ca9282dcd418c8..1a09ea1f050cc6 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27436,7 +27436,7 @@ regNumber GenTree::GetSingleTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 { regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) == 1); - regNumber tempReg = genRegNumFromMask(availableSet); + regNumber tempReg = genRegNumFromMask(availableSet MORE_THAN_64_REG_ARG(TypeGet())); INDEBUG(gtRsvdRegs &= ~availableSet;) // Remove the register from the set, so it can't be used again. return tempReg; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3ad62c64b94a08..575f130c89257e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2999,7 +2999,7 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, return REG_NA; } - foundReg = genRegNumFromMask(foundRegBit); + foundReg = genRegNumFromMask(foundRegBit MORE_THAN_64_REG_ARG(currentInterval->registerType)); availablePhysRegRecord = getRegisterRecord(foundReg); Interval* assignedInterval = availablePhysRegRecord->assignedInterval; if ((assignedInterval != currentInterval) && @@ -3059,7 +3059,7 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, return REG_NA; } - regNumber foundReg = genRegNumFromMask(foundRegBit); + regNumber foundReg = genRegNumFromMask(foundRegBit MORE_THAN_64_REG_ARG(currentInterval->registerType)); RegRecord* availablePhysRegRecord = getRegisterRecord(foundReg); Interval* assignedInterval = availablePhysRegRecord->assignedInterval; if ((assignedInterval != currentInterval) && @@ -8433,7 +8433,7 @@ void LinearScan::resolveRegisters() regMaskOnlyOne initialRegMask = interval->firstRefPosition->registerAssignment; regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) ? REG_STK - : genRegNumFromMask(initialRegMask); + : genRegNumFromMask(initialRegMask MORE_THAN_64_REG_ARG(interval->registerType)); #ifdef TARGET_ARM if (varTypeIsMultiReg(varDsc)) @@ -8866,7 +8866,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, freeRegs &= calleeTrashMask; } - regNumber tempReg = genRegNumFromMask(genFindLowestBit(freeRegs)); + regNumber tempReg = genRegNumFromMask(genFindLowestBit(freeRegs) MORE_THAN_64_REG_ARG(type)); return tempReg; } } @@ -13531,8 +13531,8 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // to achieve zero diffs. // bool thisIsSingleReg = isSingleRegister(newRelatedPreferences); - if (!thisIsSingleReg || - linearScan->isFree(linearScan->getRegisterRecord(genRegNumFromMask(newRelatedPreferences)))) + if (!thisIsSingleReg || linearScan->isFree(linearScan->getRegisterRecord( + genRegNumFromMask(newRelatedPreferences MORE_THAN_64_REG_ARG(regType))))) { relatedPreferences = newRelatedPreferences; // If this Interval has a downstream def without a single-register preference, continue to iterate. @@ -13626,7 +13626,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval if (candidates == refPosition->registerAssignment) { found = true; - if (linearScan->nextIntervalRef[genRegNumFromMask(candidates)] > lastLocation) + if (linearScan->nextIntervalRef[genRegNumFromMask(candidates MORE_THAN_64_REG_ARG(regType))] > lastLocation) { unassignedSet = candidates; } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index c6baa36a906aef..f73cea5246f581 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2642,11 +2642,13 @@ class RefPosition return REG_NA; } - return genRegNumFromMask(registerAssignment); + return genRegNumFromMask(registerAssignment MORE_THAN_64_REG_ARG(getRegisterType())); } RegisterType getRegisterType() { + //TODO: The type should be just on `Referenceable` that refposition can extract + // without having to deal with the check. return isIntervalRef() ? getInterval()->registerType : getReg()->registerType; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index bf68bf3935abed..fb30e758a1e8e0 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -603,7 +603,7 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, if (insertFixedRef) { - regNumber physicalReg = genRegNumFromMask(mask); + regNumber physicalReg = genRegNumFromMask(mask MORE_THAN_64_REG_ARG(theInterval->registerType)); RefPosition* pos = newRefPosition(physicalReg, theLocation, RefTypeFixedReg, nullptr, mask); assert(theInterval != nullptr); #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) From e808d0f352d9730c3dca7c26e9bce83f57f779d4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:30:23 -0700 Subject: [PATCH 148/201] Update genFirstRegNumFromMask() to take MORE_THAN_64_REG_ARG --- src/coreclr/jit/codegen.h | 9 ++++++--- src/coreclr/jit/codegenarm64.cpp | 30 +++++++++++++++++------------- src/coreclr/jit/compiler.hpp | 6 +++++- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/lsra.cpp | 20 ++++++++++---------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 344011375cd1a9..acbd27bc7d9d71 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -329,13 +329,16 @@ class CodeGen final : public CodeGenInterface } }; - static void genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack); + static void genBuildRegPairsStack(regMaskOnlyOne regsMask, + ArrayStack* regStack MORE_THAN_64_REG_ARG(var_types type)); static void genSetUseSaveNextPairs(ArrayStack* regStack); static int genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask); - void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); - void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset); + void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)); + void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, + int spDelta, + int spOffset MORE_THAN_64_REG_ARG(var_types type)); void genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); void genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 74127c8bb59040..7324cfe90287c9 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -725,7 +725,7 @@ void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, reg // no return value; the regStack argument is modified. // // static -void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack) +void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack MORE_THAN_64_REG_ARG(var_types type)) { assert(regStack != nullptr); assert(regStack->Height() == 0); @@ -740,7 +740,7 @@ void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack bool isPairSave = false; if (regsCount > 0) { - regNumber reg2 = genFirstRegNumFromMask(regsMask); + regNumber reg2 = genFirstRegNumFromMask(regsMask MORE_THAN_64_REG_ARG(type)); if (reg2 == REG_NEXT(reg1)) { // The JIT doesn't allow saving pair (R28,FP), even though the @@ -842,15 +842,16 @@ int CodeGen::genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask) // regsMask - a mask of registers for prolog generation; // spDelta - if non-zero, the amount to add to SP before the first register save (or together with it); // spOffset - the offset from SP that is the beginning of the callee-saved register area; +// type - The type of `regsMask` we are operating on. // -void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset) +void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)) { assert(compiler->IsOnlyOneRegMask(regsMask)); const int slotSize = genGetSlotSizeForRegsInMask(regsMask); ArrayStack regStack(compiler->getAllocator(CMK_Codegen)); - genBuildRegPairsStack(regsMask, ®Stack); + genBuildRegPairsStack(regsMask, ®Stack MORE_THAN_64_REG_ARG(type)); for (int i = 0; i < regStack.Height(); ++i) { @@ -950,7 +951,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo #ifdef FEATURE_MASKED_HW_INTRINSICS if (maskSaveRegsPredicate != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsPredicate, spDelta, lowestCalleeSavedOffset); + genSaveCalleeSavedRegisterGroup(maskSaveRegsPredicate, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_MASK)); spDelta = 0; lowestCalleeSavedOffset += genCountBits(maskSaveRegsPredicate) * FPSAVE_REGSIZE_BYTES; } @@ -959,14 +960,14 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo // Save integer registers at higher addresses than floating-point registers. if (maskSaveRegsFloat != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset); + genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_FLOAT)); spDelta = 0; lowestCalleeSavedOffset += genCountBits(maskSaveRegsFloat) * FPSAVE_REGSIZE_BYTES; } if (maskSaveRegsInt != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, lowestCalleeSavedOffset); + genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_INT)); // No need to update spDelta, lowestCalleeSavedOffset since they're not used after this. } } @@ -977,14 +978,17 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo // Arguments: // regsMask - a mask of registers for epilog generation; // spDelta - if non-zero, the amount to add to SP after the last register restore (or together with it); -// spOffset - the offset from SP that is the beginning of the callee-saved register area; +// spOffset - the offset from SP that is the beginning of the callee-saved register area;. +// type - The type of `regsMask` we are operating on. // -void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset) +void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, + int spDelta, + int spOffset MORE_THAN_64_REG_ARG(var_types type)) { const int slotSize = genGetSlotSizeForRegsInMask(regsMask); ArrayStack regStack(compiler->getAllocator(CMK_Codegen)); - genBuildRegPairsStack(regsMask, ®Stack); + genBuildRegPairsStack(regsMask, ®Stack MORE_THAN_64_REG_ARG(type)); int stackDelta = 0; for (int i = 0; i < regStack.Height(); ++i) @@ -1091,14 +1095,14 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, if (maskRestoreRegsInt != RBM_NONE) { int spIntDelta = (maskRestoreRegsFloat != RBM_NONE) ? 0 : spDelta; // should we delay the SP adjustment? - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, spIntDelta, spOffset); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, spIntDelta, spOffset MORE_THAN_64_REG_ARG(TYP_INT)); spOffset -= genCountBits(maskRestoreRegsInt) * REGSIZE_BYTES; } if (maskRestoreRegsFloat != RBM_NONE) { // If there is any spDelta, it must be used here. - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset MORE_THAN_64_REG_ARG(TYP_FLOAT)); spOffset -= genCountBits(maskRestoreRegsInt) * FPSAVE_REGSIZE_BYTES; } @@ -1106,7 +1110,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, if (maskRestoreRegsPredicate != RBM_NONE) { // TODO: Do we need to adjust spDelta? - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsPredicate, spDelta, spOffset); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsPredicate, spDelta, spOffset MORE_THAN_64_REG_ARG(TYP_MASK)); } #endif } diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index af7572004a8653..fb974b498a11b8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1021,7 +1021,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) +inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask MORE_THAN_64_REG_ARG(var_types type)) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -1029,6 +1029,10 @@ inline regNumber genFirstRegNumFromMask(regMaskOnlyOne mask) regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); +#ifdef HAS_MORE_THAN_64_REGISTERS + // If this is mask type, add `64` to the regNumber + return (regNumber)((varTypeUsesMaskReg(type) << 6) + regNum); +#endif return regNum; } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1a09ea1f050cc6..d16355014073e0 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27457,7 +27457,7 @@ regNumber GenTree::ExtractTempReg(regMaskOnlyOne mask /* = (regMaskOnlyOne)-1 */ { regMaskOnlyOne availableSet = gtRsvdRegs & mask; assert(genCountBits(availableSet) >= 1); - regNumber tempReg = genFirstRegNumFromMask(availableSet); + regNumber tempReg = genFirstRegNumFromMask(availableSet MORE_THAN_64_REG_ARG(TypeGet())); gtRsvdRegs ^= genRegMask(tempReg); return tempReg; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 575f130c89257e..40b371451249b1 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -297,7 +297,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, regMaskOnlyOne result = RBM_NONE; while (candidates != RBM_NONE) { - regNumber regNum = genFirstRegNumFromMask(candidates); + regNumber regNum = genFirstRegNumFromMask(candidates MORE_THAN_64_REG_ARG(currentInterval->registerType)); singleRegMask candidateBit = genRegMask(regNum); candidates ^= candidateBit; @@ -12929,7 +12929,7 @@ void LinearScan::RegisterSelection::try_BEST_FIT() LsraLocation bestFitLocation = earliestIsBest ? MaxLocation : MinLocation; for (regMaskOnlyOne bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) { - regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); + regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask bestFitCandidateBit = genRegMask(bestFitCandidateRegNum); bestFitCandidates ^= bestFitCandidateBit; @@ -13028,7 +13028,7 @@ void LinearScan::RegisterSelection::try_REG_ORDER() regMaskOnlyOne lowestRegOrderBit = RBM_NONE; for (regMaskOnlyOne regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { - regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); + regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; @@ -13064,7 +13064,7 @@ void LinearScan::RegisterSelection::try_SPILL_COST() for (regMaskOnlyOne spillCandidates = candidates; spillCandidates != RBM_NONE;) { - regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); + regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask spillCandidateBit = genRegMask(spillCandidateRegNum); spillCandidates ^= spillCandidateBit; @@ -13189,7 +13189,7 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() regMaskOnlyOne farthestSet = RBM_NONE; for (regMaskOnlyOne farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { - regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); + regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask farthestCandidateBit = genRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; @@ -13222,7 +13222,7 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() regMaskOnlyOne prevRegOptSet = RBM_NONE; for (regMaskOnlyOne prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { - regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); + regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; @@ -13325,7 +13325,7 @@ void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate regMaskOnlyOne coversCandidates = candidates; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13353,7 +13353,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() regMaskOnlyOne coversCandidates = (preferenceSet == RBM_NONE) ? candidates : preferenceSet; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates MORE_THAN_64_REG_ARG(regType)); singleRegMask coversCandidateBit = genRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13668,7 +13668,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs.GetRegMaskForType(regType); while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask MORE_THAN_64_REG_ARG(regType)); singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; @@ -13992,7 +13992,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( regMaskOnlyOne checkConflictMask = candidates & linearScan->fixedRegs.GetRegMaskForType(regType); while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask MORE_THAN_64_REG_ARG(regType)); singleRegMask checkConflictBit = genRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; From cdabdcfd08642254bf711f3c02b4b9e30872409c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:31:25 -0700 Subject: [PATCH 149/201] Handle TYP_MASK in getTempRegForResolution() --- src/coreclr/jit/lsra.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 40b371451249b1..40004bc80b507e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -8854,6 +8854,13 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, calleeTrashMask = RBM_INT_CALLEE_TRASH; assert(compiler->IsGprRegMask(terminatorConsumedRegs)); } +#ifdef FEATURE_MASKED_HW_INTRINSICS + else if (varTypeUsesMaskReg(type)) + { + calleeTrashMask = RBM_MSK_CALLEE_TRASH; + assert(compiler->IsPredicateRegMask(terminatorConsumedRegs)); + } +#endif else { assert(varTypeUsesFloatReg(type)); From 47c683fd83d408dd88d1b6ada6703f39e516b186 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:31:46 -0700 Subject: [PATCH 150/201] Add assert in genUpdateRegLife() --- src/coreclr/jit/codegencommon.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 8c75b9e757a77b..7d956ba0b8e665 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -561,6 +561,7 @@ regMaskOnlyOne CodeGenInterface::genGetRegMask(GenTree* tree) void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bool isDying DEBUGARG(GenTree* tree)) { regMaskOnlyOne regMask = genGetRegMask(varDsc); + assert(compiler->IsOnlyOneRegMask(regMask)); #ifdef DEBUG if (compiler->verbose) From 611ed21a80c9b1df5687aff84f769fdf5656f396 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 18:51:51 -0700 Subject: [PATCH 151/201] Update genFirstRegNumFromMaskAndToggle() to take MORE_THAN_64_REG_ARG --- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/compiler.hpp | 7 ++++- src/coreclr/jit/lsra.cpp | 54 ++++++++++++++++++++++++++------ src/coreclr/jit/lsra.h | 13 ++++++++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 7324cfe90287c9..a827aa69a13924 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -734,7 +734,7 @@ void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack while (regsMask != RBM_NONE) { - regNumber reg1 = genFirstRegNumFromMaskAndToggle(regsMask); + regNumber reg1 = genFirstRegNumFromMaskAndToggle(regsMask MORE_THAN_64_REG_ARG(type)); regsCount -= 1; bool isPairSave = false; diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index fb974b498a11b8..86f9dcae1ca321 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -955,13 +955,14 @@ inline regNumber genRegNumFromMask(regMaskOnlyOne mask MORE_THAN_64_REG_ARG(var_ // register number and also toggle the bit in the `mask`. // Arguments: // mask - the register mask +// type - the register type, `mask` represents. // // Return Value: // The number of the first register contained in the mask and updates the `mask` to toggle // the bit. // -inline regNumber genFirstRegNumFromMaskAndToggle(regMaskOnlyOne& mask) +inline regNumber genFirstRegNumFromMaskAndToggle(regMaskOnlyOne& mask MORE_THAN_64_REG_ARG(var_types type)) { assert(mask != 0); // Must have one bit set, so can't have a mask of zero @@ -970,6 +971,10 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskOnlyOne& mask) regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); mask ^= genRegMask(regNum); +#ifdef HAS_MORE_THAN_64_REGISTERS + // If this is mask type, add `64` to the regNumber + return (regNumber)((varTypeUsesMaskReg(type) << 6) + regNum); +#endif return regNum; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 40004bc80b507e..da04f34db0f21e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4039,7 +4039,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(TYP_INT)); RegRecord* regRecord = getRegisterRecord(nextReg); Interval* assignedInterval = regRecord->assignedInterval; @@ -4142,7 +4142,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(interval->registerType)); if (nextReg > targetReg) { newReg = nextReg; @@ -4382,6 +4382,8 @@ void LinearScan::resetAllRegistersState() } } +#ifdef HAS_MORE_THAN_64_REGISTERS + void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap) { while (!deadRegMask.IsEmpty()) @@ -4417,6 +4419,7 @@ void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarT } } } +#else void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap) { @@ -4453,6 +4456,7 @@ void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRe } } } +#endif // HAS_MORE_THAN_64_REGISTERS //------------------------------------------------------------------------ // processBlockStartLocations: Update var locations on entry to 'currentBlock' and clear constant @@ -4879,6 +4883,18 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) } } +#ifdef HAS_MORE_THAN_64_REGISTERS +void LinearScan::inActivateRegisters(AllRegsMask& inactiveMask) +{ + while (!inactiveMask.IsEmpty()) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(inactiveMask); + RegRecord* regRecord = getRegisterRecord(nextReg); + clearSpillCost(regRecord->regNum, regRecord->registerType); + makeRegisterInactive(regRecord); + } +} +#else void LinearScan::inActivateRegisters(regMaskTP inactiveMask) { while (inactiveMask != RBM_NONE) @@ -4889,6 +4905,7 @@ void LinearScan::inActivateRegisters(regMaskTP inactiveMask) makeRegisterInactive(regRecord); } } +#endif //------------------------------------------------------------------------ // LinearScan::freeRegister: Make a register available for use @@ -4960,13 +4977,33 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FREE_REGS)); makeRegsAvailable(regsToFree); #ifdef HAS_MORE_THAN_64_REGISTERS - freeRegisterMask(regsToFree.GetGprFloatCombinedMask()); - freeRegisterMask(regsToFree.predicateRegs()); + freeRegisterMask(regsToFree); #else freeRegisterMask(regsToFree.GetAllRegistersMask()); #endif // HAS_MORE_THAN_64_REGISTERS } +#ifdef HAS_MORE_THAN_64_REGISTERS +//TODO: Can we just if-def the method signature and `IsEmpty()`? +void LinearScan::freeRegisterMask(AllRegsMask& freeMask) +{ + while (freeMask.IsEmpty()) + { + regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); + + RegRecord* regRecord = getRegisterRecord(nextReg); +#ifdef TARGET_ARM + if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) + { + assert(genIsValidDoubleReg(nextReg)); + freeMask ^= genRegMask(regNumber(nextReg + 1)); + } +#endif + freeRegister(regRecord); + } +} + +#else void LinearScan::freeRegisterMask(regMaskTP freeMask) { while (freeMask != RBM_NONE) @@ -4984,6 +5021,7 @@ void LinearScan::freeRegisterMask(regMaskTP freeMask) freeRegister(regRecord); } } +#endif // HAS_MORE_THAN_64_REGISTERS //------------------------------------------------------------------------ // LinearScan::allocateRegistersMinimal: Perform the actual register allocation when localVars @@ -5059,8 +5097,7 @@ void LinearScan::allocateRegistersMinimal() // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); #ifdef HAS_MORE_THAN_64_REGISTERS - inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); - inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); + inActivateRegisters(tempRegsToMakeInactive); #else inActivateRegisters(tempRegsToMakeInactive.GetAllRegistersMask()); #endif // HAS_MORE_THAN_64_REGISTERS @@ -5749,8 +5786,7 @@ void LinearScan::allocateRegisters() // to dumpRegRecords. TODO: Do not do anything if tempRegsToMakeInactive == RBM_NONE AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); #ifdef HAS_MORE_THAN_64_REGISTERS - inActivateRegisters(tempRegsToMakeInactive.GetGprFloatCombinedMask()); - inActivateRegisters(tempRegsToMakeInactive.predicateRegs()); + inActivateRegisters(tempRegsToMakeInactive); #else inActivateRegisters(tempRegsToMakeInactive.GetAllRegistersMask()); #endif // HAS_MORE_THAN_64_REGISTERS @@ -12288,7 +12324,7 @@ void LinearScan::verifyFinalAllocation() regMaskOnlyOne candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(TYP_INT)); RegRecord* regRecord = getRegisterRecord(nextReg); Interval* assignedInterval = regRecord->assignedInterval; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index f73cea5246f581..b289b52775cc60 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1014,8 +1014,11 @@ class LinearScan : public LinearScanInterface void processBlockStartLocations(BasicBlock* current); void processBlockEndLocations(BasicBlock* current); void resetAllRegistersState(); +#ifdef HAS_MORE_THAN_64_REGISTERS FORCEINLINE void updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap); +#else FORCEINLINE void updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap); +#endif #ifdef TARGET_ARM bool isSecondHalfReg(RegRecord* regRec, Interval* interval); @@ -1105,10 +1108,18 @@ class LinearScan : public LinearScanInterface regMaskFloat internalFloatRegCandidates(); void makeRegisterInactive(RegRecord* physRegRecord); +#ifdef HAS_MORE_THAN_64_REGISTERS + FORCEINLINE void inActivateRegisters(AllRegsMask& inactiveMask); +#else FORCEINLINE void inActivateRegisters(regMaskTP inactiveMask); +#endif void freeRegister(RegRecord* physRegRecord); void freeRegisters(AllRegsMask regsToFree); +#ifdef HAS_MORE_THAN_64_REGISTERS + FORCEINLINE void freeRegisterMask(AllRegsMask& freeMask); +#else FORCEINLINE void freeRegisterMask(regMaskTP freeMask); +#endif // Get the type that this tree defines. var_types getDefType(GenTree* tree) @@ -2471,6 +2482,8 @@ class RefPosition // Prior to the allocation pass, registerAssignment captures the valid registers // for this RefPosition. // After the allocation pass, this contains the actual assignment + // TODO: This should really be a union, where before allocation-pass it has `mask` and + // after allocation-pass, it has regNumber directly, to avoid calling assignedReg(); regMaskOnlyOne registerAssignment; RefType refType; From b29b417b3e040af6f16303dbc26a69724dc7c0db Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 22:30:54 -0700 Subject: [PATCH 152/201] fix freeRegisterMask() typo --- src/coreclr/jit/lsra.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index da04f34db0f21e..bc580625e5c31e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4987,7 +4987,7 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) //TODO: Can we just if-def the method signature and `IsEmpty()`? void LinearScan::freeRegisterMask(AllRegsMask& freeMask) { - while (freeMask.IsEmpty()) + while (!freeMask.IsEmpty()) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); From 25b9893698cf19e08c9de133599df058219cfacc Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 2 Apr 2024 22:32:26 -0700 Subject: [PATCH 153/201] Display predicate registers in Allocation table --- src/coreclr/jit/lsra.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index bc580625e5c31e..050a15e74148ad 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -11543,7 +11543,11 @@ void LinearScan::dumpRegRecordTitleIfNeeded() if ((lastDumpedRegisters != registersToDump) || (rowCountSinceLastTitle > MAX_ROWS_BETWEEN_TITLES)) { lastUsedRegNumIndex = 0; - int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_FP_LAST : REG_INT_LAST; +#ifdef FEATURE_MASKED_HW_INTRINSICS + int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_MASK_LAST : REG_INT_LAST; +#else + int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_FP_LAST : REG_INT_LAST; +#endif for (int regNumIndex = 0; regNumIndex <= lastRegNumIndex; regNumIndex++) { if (registersToDump.IsRegNumInMask((regNumber)regNumIndex)) From 952af0d120013b41346a9b5d1d999f062a519d1d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 3 Apr 2024 10:43:38 -0700 Subject: [PATCH 154/201] make gprRegs() use RBM_ALLINT --- src/coreclr/jit/compiler.hpp | 4 ++++ src/coreclr/jit/target.h | 12 +----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 86f9dcae1ca321..f8d05a9906b1d8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5333,6 +5333,10 @@ RegBitSet64 AllRegsMask::GetAllRegistersMask() const } #endif +regMaskGpr AllRegsMask::gprRegs() const +{ + return _combinedRegisters & RBM_ALLINT; +} /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 5a5d0a5666e51d..ea04be36191cac 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -319,17 +319,6 @@ typedef struct _regMaskAll } public: - inline regMaskGpr gprRegs() const - { -#ifdef TARGET_AMD64 - return _combinedRegisters & 0xFFFF; -#elif TARGET_ARM64 - return _combinedRegisters & 0xFFFFFFFF; -#else - // TODO: Fix this for ARM and x86 - return _combinedRegisters; -#endif // TARGET_AMD64 - } inline regMaskFloat floatRegs() const { @@ -355,6 +344,7 @@ typedef struct _regMaskAll } #endif // FEATURE_MASKED_HW_INTRINSICS //#endif // DEBUG + FORCEINLINE regMaskGpr gprRegs() const; _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS From ce370e3257eb7525b4ec48cae117bea0d8e4984c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 3 Apr 2024 10:43:58 -0700 Subject: [PATCH 155/201] Fix JITDump for AllMask --- src/coreclr/jit/compiler.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index deb9cb11de2adf..8b3d5866795419 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3773,12 +3773,12 @@ void Compiler::dumpRegMask(AllRegsMask mask) const { printf("[allDouble]"); } -#ifdef TARGET_XARCH - else if (mask.predicateRegs() == RBM_ALLMASK) +#ifdef FEATURE_MASKED_HW_INTRINSICS + else if ((RBM_ALLMASK != RBM_NONE) && (mask.predicateRegs() == RBM_ALLMASK)) { printf("[allMask]"); } -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS else { dspRegMask(mask); From 6f96b5722fff6aecb21583a53fd4617387b520ca Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 3 Apr 2024 11:26:07 -0700 Subject: [PATCH 156/201] make floatRegs() and predicateRegs() use RBM_ALLFLOAT/RBM_ALLMASK --- src/coreclr/jit/codegenarm.cpp | 2 +- src/coreclr/jit/codegenarm64.cpp | 8 ++++---- src/coreclr/jit/compiler.cpp | 6 +++--- src/coreclr/jit/compiler.hpp | 34 +++++++++++++++++++++++++++----- src/coreclr/jit/emit.cpp | 16 +++++++++++++-- src/coreclr/jit/emit.h | 4 ++++ src/coreclr/jit/lsra.cpp | 10 +++++----- src/coreclr/jit/lsra.h | 6 +++--- src/coreclr/jit/lsraarm64.cpp | 10 +++++----- src/coreclr/jit/lsrabuild.cpp | 8 ++++---- src/coreclr/jit/regset.cpp | 15 +++++++------- src/coreclr/jit/regset.h | 4 ++-- src/coreclr/jit/target.h | 30 ++++++---------------------- src/coreclr/jit/utils.cpp | 4 ++-- 14 files changed, 90 insertions(+), 67 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 9dc136e3a0ab33..530055e967b306 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1823,7 +1823,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) GetEmitter()->emitIns_Mov(INS_mov, attr, REG_R0, REG_PROFILER_RET_SCRATCH, /* canSkip */ false); genTransferRegGCState(REG_R0, REG_PROFILER_RET_SCRATCH); assert(compiler->IsGprRegMask(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs())); - assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs() == RBM_NONE); + assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs(compiler) == RBM_NONE); gcInfo.gcMarkRegSetNpt(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs()); } } diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index a827aa69a13924..3f30cec772794f 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -918,7 +918,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); regMaskTP maskSaveRegsInt = regsToSaveMask.gprRegs(); - regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs(); + regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs(compiler); assert(compiler->IsGprRegMask(maskSaveRegsInt)); assert(compiler->IsFloatRegMask(maskSaveRegsFloat)); @@ -926,7 +926,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo unsigned regsToSaveCount = genCountBits(maskSaveRegsFloat) + genCountBits(maskSaveRegsInt); #ifdef FEATURE_MASKED_HW_INTRINSICS - regMaskTP maskSaveRegsPredicate = regsToSaveMask.predicateRegs(); + regMaskTP maskSaveRegsPredicate = regsToSaveMask.predicateRegs(compiler); assert(compiler->IsPredicateRegMask(maskSaveRegsPredicate)); regsToSaveCount += genCountBits(maskSaveRegsPredicate); #endif @@ -1056,7 +1056,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, assert(spDelta >= 0); regMaskGpr maskRestoreRegsInt = regsToRestoreMask.gprRegs(); - regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs(); + regMaskFloat maskRestoreRegsFloat = regsToRestoreMask.floatRegs(compiler); assert(compiler->IsGprRegMask(maskRestoreRegsInt)); assert(compiler->IsFloatRegMask(maskRestoreRegsFloat)); @@ -1064,7 +1064,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, unsigned regsToRestoreCount = genCountBits(maskRestoreRegsInt) + genCountBits(maskRestoreRegsFloat); #ifdef FEATURE_MASKED_HW_INTRINSICS - regMaskTP maskRestoreRegsPredicate = regsToRestoreMask.predicateRegs(); + regMaskTP maskRestoreRegsPredicate = regsToRestoreMask.predicateRegs(compiler); assert(compiler->IsPredicateRegMask(maskRestoreRegsPredicate)); regsToRestoreCount += genCountBits(maskRestoreRegsPredicate); #endif diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 8b3d5866795419..360bcd9d22b61a 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3765,16 +3765,16 @@ void Compiler::dumpRegMask(AllRegsMask mask) const { printf("[allIntButFP]"); } - else if (mask.floatRegs() == RBM_ALLFLOAT) + else if (mask.floatRegs(this) == RBM_ALLFLOAT) { printf("[allFloat]"); } - else if (mask.floatRegs() == RBM_ALLDOUBLE) + else if (mask.floatRegs(this) == RBM_ALLDOUBLE) { printf("[allDouble]"); } #ifdef FEATURE_MASKED_HW_INTRINSICS - else if ((RBM_ALLMASK != RBM_NONE) && (mask.predicateRegs() == RBM_ALLMASK)) + else if ((RBM_ALLMASK != RBM_NONE) && (mask.predicateRegs(this) == RBM_ALLMASK)) { printf("[allMask]"); } diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index f8d05a9906b1d8..398b0286e9cd84 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1008,7 +1008,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) } else { - regNum = (regNumber)(64 + BitOperations::BitScanForward(mask.predicateRegs())); + regNum = (regNumber)(64 + BitOperations::BitScanForward(mask.predicateRegs(nullptr))); } #else regNum = (regNumber)BitOperations::BitScanForward(gprOrFloatMask); @@ -4524,11 +4524,11 @@ inline void printRegMask(AllRegsMask mask) { printf(REG_MASK_ALL_FMT, mask.gprRegs()); printf(" "); - printf(REG_MASK_ALL_FMT, mask.floatRegs()); + printf(REG_MASK_ALL_FMT, mask.floatRegs(nullptr)); #ifdef FEATURE_MASKED_HW_INTRINSICS printf(" "); - printf(REG_MASK_ALL_FMT, mask.predicateRegs()); + printf(REG_MASK_ALL_FMT, mask.predicateRegs(nullptr)); #endif // FEATURE_MASKED_HW_INTRINSICS } @@ -5300,9 +5300,9 @@ bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const return (gprRegs() & maskToCheck) != RBM_NONE; } -bool AllRegsMask::IsFloatMaskPresent(regMaskFloat maskToCheck) const +bool AllRegsMask::IsFloatMaskPresent(Compiler* compiler, regMaskFloat maskToCheck) const { - return (floatRegs() & maskToCheck) != RBM_NONE; + return (floatRegs(compiler) & maskToCheck) != RBM_NONE; } regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const @@ -5337,6 +5337,30 @@ regMaskGpr AllRegsMask::gprRegs() const { return _combinedRegisters & RBM_ALLINT; } + +regMaskFloat AllRegsMask::floatRegs(const Compiler* compiler) const +{ +#ifdef TARGET_AMD64 + regMaskOnlyOne allFloat = compiler != nullptr ? compiler->get_RBM_ALLFLOAT() : (RBM_HIGHFLOAT | RBM_LOWFLOAT); + return _combinedRegisters & allFloat; +#else + return _combinedRegisters & RBM_ALLFLOAT; +#endif // TARGET_AMD64 +} + +#ifdef FEATURE_MASKED_HW_INTRINSICS +regMaskPredicate AllRegsMask::predicateRegs(const Compiler* compiler) const +{ +#ifdef HAS_MORE_THAN_64_REGISTERS + return _predicateRegs; +#else + regMaskOnlyOne allMask = compiler != nullptr ? compiler->get_RBM_ALLMASK() : (RBM_ALLMASK_EVEX); + return _combinedRegisters & compiler->get_RBM_ALLMASK(); +#endif +} +#endif // FEATURE_MASKED_HW_INTRINSICS + + /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index d7e5e2fd3fabfd..39e6b65f9ca975 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3514,12 +3514,24 @@ void emitter::emitDispGprRegSet(regMaskGpr regs) emitDispRegSet(REG_INT_FIRST, REG_INT_LAST, regs); } +void emitter::emitDispFloatRegSet(regMaskFloat regs) +{ + emitDispRegSet(REG_FP_FIRST, REG_FP_LAST, regs); +} + +#ifdef FEATURE_MASKED_HW_INTRINSICS +void emitter::emitDispMaskRegSet(regMaskPredicate regs) +{ + emitDispRegSet(REG_MASK_FIRST, REG_MASK_LAST, regs); +} +#endif // FEATURE_MASKED_HW_INTRINSICS + void emitter::emitDispRegSet(AllRegsMask regs) { emitDispGprRegSet(regs.gprRegs()); - emitDispGprRegSet(regs.floatRegs()); + emitDispFloatRegSet(regs.floatRegs(nullptr)); #ifdef FEATURE_MASKED_HW_INTRINSICS - emitDispGprRegSet(regs.predicateRegs()); + emitDispMaskRegSet(regs.predicateRegs(nullptr)); #endif } diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 8af42b8ea1b7f1..f281e8d27f4eb5 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3211,6 +3211,10 @@ class emitter const char* emitGetFrameReg(); void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); void emitDispGprRegSet(regMaskGpr regs); + void emitDispFloatRegSet(regMaskFloat regs); +#ifdef FEATURE_MASKED_HW_INTRINSICS + void emitDispMaskRegSet(regMaskPredicate regs); +#endif // FEATURE_MASKED_HW_INTRINSICS void emitDispRegSet(AllRegsMask regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 050a15e74148ad..896745f342abc9 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -11868,13 +11868,13 @@ void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) { if (reg >= REG_FP_FIRST && reg <= REG_FP_LAST) { - regsMaskToFree = regsToFree.floatRegs(); + regsMaskToFree = regsToFree.floatRegs(compiler); availableRegsMask = availableFloatRegs; } #ifdef FEATURE_MASKED_HW_INTRINSICS else if (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST) { - regsMaskToFree = regsToFree.predicateRegs(); + regsMaskToFree = regsToFree.predicateRegs(compiler); availableRegsMask = availableMaskRegs; } #endif @@ -13701,7 +13701,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs.floatRegs() & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs(linearScan->compiler) & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM @@ -13807,7 +13807,7 @@ singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval // such range that is consecutive. Next, append that range to the `candidates`. // regMaskFloat limitCandidatesForConsecutive = - refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs(); + refPosition->registerAssignment & ~inUseOrBusyRegsMask.floatRegs(linearScan->compiler); regMaskFloat overallLimitCandidates; regMaskFloat limitConsecutiveResult = linearScan->filterConsecutiveCandidates(limitCandidatesForConsecutive, refPosition->regCount, @@ -14029,7 +14029,7 @@ singleRegMask LinearScan::RegisterSelection::selectMinimal( // clause below creates a mask to do this. if (currentInterval->registerType == TYP_DOUBLE) { - candidates &= ~((busyRegs.floatRegs() & RBM_ALLDOUBLE_HIGH) >> 1); + candidates &= ~((busyRegs.floatRegs(linearScan->compiler) & RBM_ALLDOUBLE_HIGH) >> 1); } #endif // TARGET_ARM diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index b289b52775cc60..a25df2193b11bf 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1256,7 +1256,7 @@ class LinearScan : public LinearScanInterface // also available. if (regType == TYP_DOUBLE) { - result &= (m_AvailableRegs.floatRegs() >> 1); + result &= (m_AvailableRegs.floatRegs(compiler) >> 1); } #endif // TARGET_ARM return result; @@ -1808,8 +1808,8 @@ class LinearScan : public LinearScanInterface m_AvailableRegs |= regMask; #ifdef HAS_MORE_THAN_64_REGISTERS assert(compiler->IsGprRegMask(m_AvailableRegs.gprRegs())); - assert(compiler->IsFloatRegMask(m_AvailableRegs.floatRegs())); - assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs())); + assert(compiler->IsFloatRegMask(m_AvailableRegs.floatRegs(compiler))); + assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs(compiler))); #endif } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index c8c6822796abea..c5bc3b385a4f24 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -364,7 +364,7 @@ regMaskFloat LinearScan::filterConsecutiveCandidatesForSpill(regMaskFloat consec } maskForCurRange |= (registersNeededMask << regAvailableStartIndex); - maskForCurRange &= m_AvailableRegs.floatRegs(); + maskForCurRange &= m_AvailableRegs.floatRegs(compiler); if (maskForCurRange != RBM_NONE) { @@ -420,7 +420,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { assert(compiler->info.compNeedsConsecutiveRegisters); assert(refPosition->isFirstRefPositionOfConsecutiveRegisters()); - regMaskFloat freeCandidates = allCandidates & m_AvailableRegs.floatRegs(); + regMaskFloat freeCandidates = allCandidates & m_AvailableRegs.floatRegs(compiler); #ifdef DEBUG if (getStressLimitRegs() != LSRA_LIMIT_NONE) @@ -536,7 +536,7 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, *busyCandidates = consecutiveResultForBusy; // Check if we can further check better registers amoung consecutiveResultForBusy. - if ((m_AvailableRegs.floatRegs() & overallResultForBusy) != RBM_NONE) + if ((m_AvailableRegs.floatRegs(compiler) & overallResultForBusy) != RBM_NONE) { // `overallResultForBusy` contains the mask of entire series that can be the consecutive candidates. // If there is an overlap of that with free registers, then try to find a series that will need least @@ -549,13 +549,13 @@ regMaskFloat LinearScan::getConsecutiveCandidates(regMaskFloat allCandidates, { *busyCandidates = optimalConsecutiveResultForBusy; } - else if ((m_AvailableRegs.floatRegs() & consecutiveResultForBusy) != RBM_NONE) + else if ((m_AvailableRegs.floatRegs(compiler) & consecutiveResultForBusy) != RBM_NONE) { // We did not find free consecutive candidates, however we found some registers among the // `allCandidates` that are mix of free and busy. Since `busyCandidates` just has bit set for first // register of such series, return the mask that starts with free register, if possible. The busy // registers will be spilled during assignment of subsequent RefPosition. - *busyCandidates = (m_AvailableRegs.floatRegs() & consecutiveResultForBusy); + *busyCandidates = (m_AvailableRegs.floatRegs(compiler) & consecutiveResultForBusy); } } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index fb30e758a1e8e0..dcfd3ff69600d5 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1125,9 +1125,9 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask) { assert(compiler->IsGprRegMask(killMask.gprRegs())); - assert(compiler->IsFloatRegMask(killMask.floatRegs())); + assert(compiler->IsFloatRegMask(killMask.floatRegs(compiler))); #ifdef FEATURE_MASKED_HW_INTRINSICS - assert(compiler->IsPredicateRegMask(killMask.predicateRegs())); + assert(compiler->IsPredicateRegMask(killMask.predicateRegs(compiler))); #endif // FEATURE_MASKED_HW_INTRINSICS bool insertedKills = false; @@ -3150,10 +3150,10 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) // RefPositions in that case. // This must be done after the kills, so that we know which large vectors are still live. // - if ((killMask.IsFloatMaskPresent(RBM_FLT_CALLEE_TRASH))) + if ((killMask.IsFloatMaskPresent(compiler, RBM_FLT_CALLEE_TRASH))) { buildUpperVectorSaveRefPositions(tree, - currentLoc + 1 DEBUG_ARG((killMask.floatRegs() & RBM_FLT_CALLEE_TRASH))); + currentLoc + 1 DEBUG_ARG((killMask.floatRegs(compiler) & RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 70488a80da88b3..39fba1e888d3f9 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -203,7 +203,7 @@ void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG - printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); #endif rsModifiedRegsMask.AddFloatRegMask(mask); @@ -224,14 +224,14 @@ void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) } else if (genIsValidFloatReg(reg)) { - printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); } #ifdef FEATURE_MASKED_HW_INTRINSICS else { assert(genIsValidMaskReg(reg)); - printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), + printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(m_rsCompiler), genRegMask(reg) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); } #endif // FEATURE_MASKED_HW_INTRINSICS @@ -250,11 +250,12 @@ void RegSet::rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressD #ifdef DEBUG printModifiedRegsMask(rsModifiedRegsMask.gprRegs(), modifiedMask.gprRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); - printModifiedRegsMask(rsModifiedRegsMask.floatRegs(), - modifiedMask.floatRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), + modifiedMask.floatRegs(m_rsCompiler) DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); #ifdef FEATURE_MASKED_HW_INTRINSICS - printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(), - modifiedMask.predicateRegs() DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.predicateRegs(m_rsCompiler), modifiedMask.predicateRegs(m_rsCompiler) + DEBUG_ARG(suppressDump) + DEBUG_ARG(RBM_MSK_CALLEE_SAVED)); #endif // FEATURE_MASKED_HW_INTRINSICS #endif // DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 8904462cd1deba..90549b9e333509 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -98,14 +98,14 @@ class RegSet regMaskFloat rsGetModifiedFloatRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedRegsMask.floatRegs(); + return rsModifiedRegsMask.floatRegs(m_rsCompiler); } #ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate rsGetModifiedPredicateRegsMask() const { assert(rsModifiedRegsMaskInitialized); - return rsModifiedRegsMask.predicateRegs(); + return rsModifiedRegsMask.predicateRegs(m_rsCompiler); } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index ea04be36191cac..e7f7f068808535 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -319,32 +319,14 @@ typedef struct _regMaskAll } public: - - inline regMaskFloat floatRegs() const - { -#ifdef TARGET_AMD64 - return _combinedRegisters & 0xFFFFFFFF0000; -#elif defined(TARGET_ARM64) - return _combinedRegisters & 0xFFFFFFFF00000000; -#else - // TODO: Fix this for ARM and x86 - return _combinedRegisters; -#endif // TARGET_AMD64 - } + FORCEINLINE regMaskGpr gprRegs() const; + FORCEINLINE regMaskFloat floatRegs(const Compiler* compiler) const; //#ifdef DEBUG -#ifdef FEATURE_MASKED_HW_INTRINSICS - inline regMaskPredicate predicateRegs() const - { -#ifdef HAS_MORE_THAN_64_REGISTERS - return _predicateRegs; -#else - return _combinedRegisters & 0xFF000000000000; // TODO: Fix the mask -#endif // HAS_MORE_THAN_64_REGISTERS - } + + #ifdef FEATURE_MASKED_HW_INTRINSICS + FORCEINLINE regMaskPredicate predicateRegs(const Compiler* compiler) const; #endif // FEATURE_MASKED_HW_INTRINSICS -//#endif // DEBUG - FORCEINLINE regMaskGpr gprRegs() const; _regMaskAll(RegBitSet64 gprRegMask, RegBitSet64 floatRegMask) #ifdef HAS_MORE_THAN_64_REGISTERS @@ -415,7 +397,7 @@ typedef struct _regMaskAll FORCEINLINE void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); FORCEINLINE bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); FORCEINLINE bool IsGprMaskPresent(regMaskGpr maskToCheck) const; - FORCEINLINE bool IsFloatMaskPresent(regMaskFloat maskToCheck) const; + FORCEINLINE bool IsFloatMaskPresent(Compiler* compiler, regMaskFloat maskToCheck) const; // bool IsOnlyRegNumInMask(regNumber reg); FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 21a8f24510bd62..4337c42bf44b7c 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -459,10 +459,10 @@ void dspRegMask(AllRegsMask mask, size_t minSiz) printf("["); sep = dspRegRange(mask.gprRegs(), minSiz, sep, REG_INT_FIRST, REG_INT_LAST); - sep = dspRegRange(mask.floatRegs(), minSiz, sep, REG_FP_FIRST, REG_FP_LAST); + sep = dspRegRange(mask.floatRegs(nullptr), minSiz, sep, REG_FP_FIRST, REG_FP_LAST); #ifdef FEATURE_MASKED_HW_INTRINSICS - sep = dspRegRange(mask.predicateRegs(), minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); + sep = dspRegRange(mask.predicateRegs(nullptr), minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); #endif // FEATURE_MASKED_HW_INTRINSICS printf("]"); From ff56a4fb8124aeb34b9c8c61f05f14f2403b59dd Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 3 Apr 2024 22:47:04 -0700 Subject: [PATCH 157/201] fix the dumping of gpr/float registers --- src/coreclr/jit/codegenarmarch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index dadbc9611cd2ef..2841d2bf54248a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -5083,7 +5083,7 @@ void CodeGen::genPushCalleeSavedRegisters() #ifdef DEBUG if (verbose) { - dspRegMask(maskSaveRegsInt, maskSaveRegsFloat); + dspRegMask(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat)); printf("\n"); } #endif // DEBUG From bff97e5989df93bbd6eadea8cb2408fe01cd0866 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 3 Apr 2024 22:47:52 -0700 Subject: [PATCH 158/201] Introduce RBM_ALLGPR --- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/targetamd64.h | 1 + src/coreclr/jit/targetarm.h | 1 + src/coreclr/jit/targetarm64.h | 1 + src/coreclr/jit/targetx86.h | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 398b0286e9cd84..2f39e1138dce16 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5335,7 +5335,7 @@ RegBitSet64 AllRegsMask::GetAllRegistersMask() const regMaskGpr AllRegsMask::gprRegs() const { - return _combinedRegisters & RBM_ALLINT; + return _combinedRegisters & RBM_ALLGPR; } regMaskFloat AllRegsMask::floatRegs(const Compiler* compiler) const diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 69714ad02603a4..1dc9db20530bd7 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -175,6 +175,7 @@ #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) // This will need to change once floatRegs are changed to 4-bytes #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) + #define RBM_ALLGPR (RBM_ALLINT) // AMD64 write barrier ABI (see vm\amd64\JitHelpers_Fast.asm, vm\amd64\JitHelpers_Fast.S): // CORINFO_HELP_ASSIGN_REF (JIT_WriteBarrier), CORINFO_HELP_CHECKED_ASSIGN_REF (JIT_CheckedWriteBarrier): diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index ef663bdcdf5cd4..7e301add395bf8 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -91,6 +91,7 @@ #define RBM_LOW_REGS (RBM_R0|RBM_R1|RBM_R2|RBM_R3|RBM_R4|RBM_R5|RBM_R6|RBM_R7) #define RBM_HIGH_REGS (RBM_R8|RBM_R9|RBM_R10|RBM_R11|RBM_R12|RBM_SP|RBM_LR|RBM_PC) + #define RBM_ALLGPR (RBM_LOW_REGS | RBM_HIGH_REGS) #define REG_CALLEE_SAVED_ORDER REG_R4,REG_R5,REG_R6,REG_R7,REG_R8,REG_R9,REG_R10,REG_R11 #define RBM_CALLEE_SAVED_ORDER RBM_R4,RBM_R5,RBM_R6,RBM_R7,RBM_R8,RBM_R9,RBM_R10,RBM_R11 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 9f8cde81f40ce1..b836427cd270bc 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -87,6 +87,7 @@ #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) #define RBM_ALLFLOAT (RBM_FLT_CALLEE_SAVED | RBM_FLT_CALLEE_TRASH) #define RBM_ALLDOUBLE RBM_ALLFLOAT + #define RBM_ALLGPR (RBM_ALLINT | RBM_FP | RBM_LR | RBM_ZR) // REG_VAR_ORDER is: (CALLEE_TRASH & ~CALLEE_TRASH_NOGC), CALLEE_TRASH_NOGC, CALLEE_SAVED #define REG_VAR_ORDER REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, \ diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index a06de34c1472f8..070549023a2c8e 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -147,6 +147,7 @@ #define RBM_CALLEE_TRASH (RBM_INT_CALLEE_TRASH | RBM_FLT_CALLEE_TRASH | RBM_MSK_CALLEE_TRASH) #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) + #define RBM_ALLGPR (RBM_ALLINT) #define REG_VAR_ORDER REG_EAX,REG_EDX,REG_ECX,REG_ESI,REG_EDI,REG_EBX #define MAX_VAR_ORDER_SIZE 6 From 49194e1747ee218b944812b9862337f0d976851e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 Apr 2024 00:44:11 -0700 Subject: [PATCH 159/201] move getRegType from lsra.h -> target.h --- src/coreclr/jit/lsra.h | 25 ------------------------- src/coreclr/jit/target.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index a25df2193b11bf..c3bbef8937f253 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1753,31 +1753,6 @@ class LinearScan : public LinearScanInterface // entry again and agin. AllRegsMask m_AvailableRegs; - regNumber getRegForType(regNumber reg, var_types regType) - { -#ifdef TARGET_ARM - if ((regType == TYP_DOUBLE) && !genIsValidDoubleReg(reg)) - { - reg = REG_PREV(reg); - } -#endif // TARGET_ARM - return reg; - } - - regMaskOnlyOne getRegMask(regNumber reg, var_types regType) - { - reg = getRegForType(reg, regType); - singleRegMask regMask = genRegMask(reg); -#ifdef TARGET_ARM - if (regType == TYP_DOUBLE) - { - assert(genIsValidDoubleReg(reg)); - regMask |= (regMask << 1); - } -#endif // TARGET_ARM - return regMask; - } - void resetAvailableRegs() { #ifdef HAS_MORE_THAN_64_REGISTERS diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index e7f7f068808535..5d4673824d1cc1 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -828,6 +828,36 @@ inline regMaskFloat genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TY #endif } +inline regNumber getRegForType(regNumber reg, var_types regType) +{ +#ifdef TARGET_ARM + if ((regType == TYP_DOUBLE) && !genIsValidDoubleReg(reg)) + { + reg = REG_PREV(reg); + } +#endif // TARGET_ARM + return reg; +} + +// This is similar to genRegMask(reg, regType) for all platforms +// except Arm. For Arm, if regType is DOUBLE and reg is also a +// valid double register, it is again same as genRegMask(reg, regType) +// but if not, it will return the pair of even/odd registers corresponding +// to the `reg`. +inline regMaskOnlyOne getRegMask(regNumber reg, var_types regType) +{ + reg = getRegForType(reg, regType); + singleRegMask regMask = genRegMask(reg); +#ifdef TARGET_ARM + if (regType == TYP_DOUBLE) + { + assert(genIsValidDoubleReg(reg)); + regMask |= (regMask << 1); + } +#endif // TARGET_ARM + return regMask; +} + //------------------------------------------------------------------------ // genRegMask: Given a register, and its type, generate the appropriate regMask // From 9041b496375c98bb64e8f97fd37a5371918b6fd7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 Apr 2024 00:46:21 -0700 Subject: [PATCH 160/201] Rewrite AddRegNum/RemoveRegNum/IsRegNum for arm --- src/coreclr/jit/compiler.hpp | 112 +++++++++++++++++++++++++--------- src/coreclr/jit/lsra.cpp | 30 ++++----- src/coreclr/jit/lsra.h | 16 ++--- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/target.h | 27 +++----- 5 files changed, 118 insertions(+), 69 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 2f39e1138dce16..23fd517cf7c134 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5231,49 +5231,113 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) AddGprRegMask(genRegMask(reg)); } -#ifdef TARGET_ARM +// ---------------------------------------------------------- +// AddRegNumForType: Adds `reg` to the mask. +// void AllRegsMask::AddRegNumInMask(regNumber reg) { - _combinedRegisters |= genRegMask(reg); + RegBitSet64 value = genRegMask(reg); +#ifdef HAS_MORE_THAN_64_REGISTERS + int index = regIndexForRegister(reg); + _registers[index] |= encodeForIndex(index, value); +#else + _combinedRegisters |= value; +#endif } -void AllRegsMask::RemoveRegNumFromMask(regNumber reg) +// This is similar to AddRegNumInMask(reg, regType) for all platforms +// except Arm. For Arm, it calls getRegMask() instead of genRegMask() +// to create a mask that needs to be added. +void AllRegsMask::AddRegNum(regNumber reg, var_types type) { - _combinedRegisters &= ~genRegMask(reg); +#ifdef TARGET_ARM + _combinedRegisters |= getRegMask(reg, type); +#else + AddRegNumInMask(reg); +#endif } -bool AllRegsMask::IsRegNumInMask(regNumber reg) +// ---------------------------------------------------------- +// RemoveRegNumFromMask: Removes `reg` to the mask. +// +void AllRegsMask::RemoveRegNumFromMask(regNumber reg) { - return (_combinedRegisters & genRegMask(reg)) != RBM_NONE; + RegBitSet64 value = genRegMask(reg); +#ifdef HAS_MORE_THAN_64_REGISTERS + int index = regIndexForRegister(reg); + _registers[index] &= ~encodeForIndex(index, value); +#else + _combinedRegisters &= ~value; +#endif } + +// This is similar to RemoveRegNumFromMask(reg, regType) for all platforms +// except Arm. For Arm, it calls getRegMask() instead of genRegMask() +// to create a mask that needs to be added. +void AllRegsMask::RemoveRegNum(regNumber reg, var_types type) +{ +#ifdef TARGET_ARM + _combinedRegisters &= ~getRegMask(reg, type); +#else + RemoveRegNumFromMask(reg); #endif +} // ---------------------------------------------------------- -// AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except -// that it takes `type` as an argument and adds `reg` to the mask for that type. +// IsRegNumInMask: Checks if `reg` is present in the mask. // -void AllRegsMask::AddRegNumInMask(regNumber reg ARM_ARG(var_types type)) +bool AllRegsMask::IsRegNumInMask(regNumber reg) { + RegBitSet64 value = genRegMask(reg); #ifdef HAS_MORE_THAN_64_REGISTERS - RegBitSet64 value = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] |= encodeForIndex(index, value); + int index = regIndexForRegister(reg); + return (_registers[index] & encodeForIndex(index, value)) != RBM_NONE; #else - _combinedRegisters |= genRegMask(reg ARM_ARG(type)); + return (_combinedRegisters & value) != RBM_NONE; #endif } -void AllRegsMask::RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)) +// This is similar to IsRegNumInMask(reg, regType) for all platforms +// except Arm. For Arm, it calls getRegMask() instead of genRegMask() +// to create a mask that needs to be added. +bool AllRegsMask::IsRegNumPresent(regNumber reg, var_types type) { -#ifdef HAS_MORE_THAN_64_REGISTERS - RegBitSet64 regMaskToRemove = genRegMask(reg ARM_ARG(type)); - int index = regIndexForRegister(reg); - _registers[index] &= ~encodeForIndex(index, regMaskToRemove); +#ifdef TARGET_ARM + return (_combinedRegisters & getRegMask(reg, type)) != RBM_NONE; #else - _combinedRegisters &= ~genRegMask(reg ARM_ARG(type)); + return IsRegNumInMask(reg); #endif } +#ifdef TARGET_ARM +// ---------------------------------------------------------- +// AddRegNumForType: Adds `reg` to the mask. It is same as AddRegNumInMask(reg) except +// that it takes `type` as an argument and adds `reg` to the mask for that type. +// +void AllRegsMask::AddRegNumInMask(regNumber reg, var_types type) +{ + _combinedRegisters |= genRegMask(reg, type); +} + +// ---------------------------------------------------------- +// RemoveRegNumFromMask: Removes `reg` from the mask. It is same as RemoveRegNumFromMask(reg) except +// that it takes `type` as an argument and adds `reg` to the mask for that type. +// +void AllRegsMask::RemoveRegNumFromMask(regNumber reg, var_types type) +{ + _combinedRegisters &= ~genRegMask(reg, type); +} + +// ---------------------------------------------------------- +// IsRegNumInMask: Removes `reg` from the mask. It is same as IsRegNumInMask(reg) except +// that it takes `type` as an argument and adds `reg` to the mask for that type. +// +bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) +{ + return (_combinedRegisters & genRegMask(reg, type)) != RBM_NONE; +} +#endif + void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type) { #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5284,16 +5348,6 @@ void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_type #endif } -bool AllRegsMask::IsRegNumInMask(regNumber reg ARM_ARG(var_types type)) -{ -#ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); - RegBitSet64 regMaskToCheck = genRegMask(reg ARM_ARG(type)); - return (_registers[index] & encodeForIndex(index, regMaskToCheck)) != RBM_NONE; -#else - return (_combinedRegisters & genRegMask(reg ARM_ARG(type))) != RBM_NONE; -#endif -} bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const { diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 896745f342abc9..685516c9338f11 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4614,7 +4614,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assert(targetReg != REG_STK); assert(interval->assignedReg != nullptr && interval->assignedReg->regNum == targetReg && interval->assignedReg->assignedInterval == interval); - liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); + liveRegs.AddRegNum(targetReg, interval->registerType); continue; } } @@ -4644,7 +4644,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // likely to match other assignments this way. targetReg = interval->physReg; interval->isActive = true; - liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); + liveRegs.AddRegNum(targetReg, interval->registerType); INDEBUG(inactiveRegs |= targetReg); setVarReg(inVarToRegMap, varIndex, targetReg); } @@ -4656,7 +4656,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (targetReg != REG_STK) { RegRecord* targetRegRecord = getRegisterRecord(targetReg); - liveRegs.AddRegNumInMask(targetReg ARM_ARG(interval->registerType)); + liveRegs.AddRegNum(targetReg, interval->registerType); if (!allocationPassComplete) { updateNextIntervalRef(targetReg, interval); @@ -4693,7 +4693,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) RegRecord* anotherHalfRegRec = findAnotherHalfRegRec(targetRegRecord); // Use TYP_FLOAT to get the regmask of just the half reg. - liveRegs.RemoveRegNumFromMask(anotherHalfRegRec->regNum, TYP_FLOAT); + liveRegs.RemoveRegNum(anotherHalfRegRec->regNum, TYP_FLOAT); } #endif // TARGET_ARM @@ -5430,7 +5430,7 @@ void LinearScan::allocateRegistersMinimal() DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); + copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); } // For tree temp (non-localVar) interval, we will need an explicit move. @@ -5445,7 +5445,7 @@ void LinearScan::allocateRegistersMinimal() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regsToFree.AddRegNum(assignedRegister, currentInterval->registerType); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -5543,7 +5543,8 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); + AllRegsMask assignedRegMask; + assignedRegMask.AddRegNum(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= assignedRegMask; if (currentRefPosition.delayRegFree) @@ -5732,7 +5733,7 @@ void LinearScan::allocateRegisters() updateNextIntervalRef(reg, interval); updateSpillCost(reg, interval); setRegInUse(reg, interval->registerType); - INDEBUG(registersToDump.AddRegNumInMask(reg ARM_ARG(interval->registerType))); + INDEBUG(registersToDump.AddRegNum(reg, interval->registerType)); } } else @@ -6154,7 +6155,7 @@ void LinearScan::allocateRegisters() updateSpillCost(assignedRegister, currentInterval); } - regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regsToFree.AddRegNum(assignedRegister, currentInterval->registerType); } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, nullptr, assignedRegister)); currentRefPosition.registerAssignment = RBM_NONE; @@ -6471,7 +6472,7 @@ void LinearScan::allocateRegisters() currentInterval) DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); + copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6580,7 +6581,7 @@ void LinearScan::allocateRegisters() DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { - copyRegsToFree.AddRegNumInMask(copyReg ARM_ARG(currentInterval->registerType)); + copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); } // If this is a tree temp (non-localVar) interval, we will need an explicit move. @@ -6601,7 +6602,7 @@ void LinearScan::allocateRegisters() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree.AddRegNumInMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regsToFree.AddRegNum(assignedRegister, currentInterval->registerType); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -6769,7 +6770,8 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - AllRegsMask assignedRegMask = AllRegsMask(assignedRegister ARM_ARG(currentInterval->registerType)); + AllRegsMask assignedRegMask; + assignedRegMask.AddRegNum(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= assignedRegMask; if (currentRefPosition.delayRegFree) @@ -11195,7 +11197,7 @@ void LinearScan::dumpLsraAllocationEvent( } if ((interval != nullptr) && (reg != REG_NA) && (reg != REG_STK)) { - registersToDump.AddRegNumInMask(reg ARM_ARG(interval->registerType)); + registersToDump.AddRegNum(reg, interval->registerType); dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index c3bbef8937f253..00bd15a4ad23e4 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1765,7 +1765,7 @@ class LinearScan : public LinearScanInterface bool isRegAvailable(regNumber reg, var_types regType) // only used in asserts { - return m_AvailableRegs.IsRegNumInMask(reg ARM_ARG(regType)); + return m_AvailableRegs.IsRegNumPresent(reg, regType); } void setRegsInUse(AllRegsMask regMask) @@ -1775,7 +1775,7 @@ class LinearScan : public LinearScanInterface void setRegInUse(regNumber reg, var_types regType) { - m_AvailableRegs.RemoveRegNumFromMask(reg ARM_ARG(regType)); + m_AvailableRegs.RemoveRegNum(reg, regType); } void makeRegsAvailable(AllRegsMask regMask) @@ -1790,7 +1790,7 @@ class LinearScan : public LinearScanInterface void makeRegAvailable(regNumber reg, var_types regType) { - m_AvailableRegs.AddRegNumInMask(reg ARM_ARG(regType)); + m_AvailableRegs.AddRegNum(reg, regType); } void clearAllNextIntervalRef(); @@ -1817,16 +1817,16 @@ class LinearScan : public LinearScanInterface // just operate on curr_RegistersWithConstants and assert // assert(m_RegistersWithConstants[regIndexForType(regType)] == curr_RegistersWithConstants); // but we will have to make sure that we save it back too?? - m_RegistersWithConstants.RemoveRegNumFromMask(reg ARM_ARG(regType)); + m_RegistersWithConstants.RemoveRegNum(reg, regType); } void setConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants.AddRegNumInMask(reg ARM_ARG(regType)); + m_RegistersWithConstants.AddRegNum(reg, regType); } bool isRegConstant(regNumber reg, var_types regType) { reg = getRegForType(reg, regType); - return m_RegistersWithConstants.IsRegNumInMask(reg ARM_ARG(regType)); + return m_RegistersWithConstants.IsRegNumPresent(reg, regType); } regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); @@ -1867,11 +1867,11 @@ class LinearScan : public LinearScanInterface #endif bool isRegBusy(regNumber reg, var_types regType) { - return regsBusyUntilKill.IsRegNumInMask(reg ARM_ARG(regType)); + return regsBusyUntilKill.IsRegNumPresent(reg, regType); } void setRegBusyUntilKill(regNumber reg, var_types regType) { - regsBusyUntilKill.AddRegNumInMask(reg ARM_ARG(regType)); + regsBusyUntilKill.AddRegNum(reg, regType); } void clearRegBusyUntilKill(regNumber reg) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index dcfd3ff69600d5..f809b6caf794c9 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2343,7 +2343,7 @@ void LinearScan::buildIntervals() assert(inArgReg < REG_COUNT); mask = genRegMask(inArgReg); assignPhysReg(inArgReg, interval); - INDEBUG(registersToDump.AddRegNumInMask(inArgReg ARM_ARG(interval->registerType))); + INDEBUG(registersToDump.AddRegNum(inArgReg, interval->registerType)); } RefPosition* pos = newRefPosition(interval, MinLocation, RefTypeParamDef, nullptr, mask); pos->setRegOptional(true); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 5d4673824d1cc1..a675790f0b551d 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -365,40 +365,33 @@ typedef struct _regMaskAll { } -#ifdef TARGET_ARM - _regMaskAll(regNumber reg) : _regMaskAll() - { - AddRegNumInMask(reg); - } -#endif - - _regMaskAll(regNumber reg ARM_ARG(var_types type)) : _regMaskAll() - { - AddRegNumInMask(reg ARM_ARG(type)); - } - FORCEINLINE void Clear(); FORCEINLINE bool IsEmpty(); FORCEINLINE unsigned Count(); FORCEINLINE void Create(regNumber reg); // Rename this to AddRegNum FORCEINLINE void AddGprRegInMask(regNumber reg); - FORCEINLINE void AddRegNumInMask(regNumber reg ARM_ARG(var_types type)); FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); -#ifdef TARGET_ARM FORCEINLINE void AddRegNumInMask(regNumber reg); + FORCEINLINE void AddRegNum(regNumber reg, var_types type); FORCEINLINE void RemoveRegNumFromMask(regNumber reg); + FORCEINLINE void RemoveRegNum(regNumber reg, var_types type); FORCEINLINE bool IsRegNumInMask(regNumber reg); + FORCEINLINE bool IsRegNumPresent(regNumber reg, var_types type); + +#ifdef TARGET_ARM + FORCEINLINE void AddRegNumInMask(regNumber reg, var_types type); + FORCEINLINE void RemoveRegNumFromMask(regNumber reg, var_types type); + FORCEINLINE bool IsRegNumInMask(regNumber reg, var_types type); #endif - FORCEINLINE void RemoveRegNumFromMask(regNumber reg ARM_ARG(var_types type)); + + FORCEINLINE void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); - FORCEINLINE bool IsRegNumInMask(regNumber reg ARM_ARG(var_types type)); FORCEINLINE bool IsGprMaskPresent(regMaskGpr maskToCheck) const; FORCEINLINE bool IsFloatMaskPresent(Compiler* compiler, regMaskFloat maskToCheck) const; - // bool IsOnlyRegNumInMask(regNumber reg); FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; // TODO: this might go away once we have just `regMaskTP` gpr_float field From ccba323055523f661b3dc894225bb17b4174ddaf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 Apr 2024 12:07:48 -0700 Subject: [PATCH 161/201] jit format 17.0.6 --- src/coreclr/jit/_typeinfo.h | 19 +- src/coreclr/jit/alloc.cpp | 5 +- src/coreclr/jit/alloc.h | 13 +- src/coreclr/jit/arraystack.h | 3 +- src/coreclr/jit/assertionprop.cpp | 24 +- src/coreclr/jit/bitset.h | 13 +- src/coreclr/jit/bitsetasshortlong.h | 54 +- src/coreclr/jit/bitsetasuint64.h | 4 +- src/coreclr/jit/bitsetasuint64inclass.h | 7 +- src/coreclr/jit/block.cpp | 8 +- src/coreclr/jit/block.h | 163 ++-- src/coreclr/jit/blockset.h | 9 +- src/coreclr/jit/buildstring.cpp | 4 +- src/coreclr/jit/codegen.h | 163 ++-- src/coreclr/jit/codegenarm.cpp | 10 +- src/coreclr/jit/codegenarm64.cpp | 68 +- src/coreclr/jit/codegenarm64test.cpp | 8 +- src/coreclr/jit/codegenarmarch.cpp | 92 +- src/coreclr/jit/codegencommon.cpp | 40 +- src/coreclr/jit/codegeninterface.h | 16 +- src/coreclr/jit/codegenlinear.cpp | 12 +- src/coreclr/jit/codegenloongarch64.cpp | 46 +- src/coreclr/jit/codegenriscv64.cpp | 46 +- src/coreclr/jit/codegenxarch.cpp | 56 +- src/coreclr/jit/compiler.cpp | 228 ++--- src/coreclr/jit/compiler.h | 387 +++++---- src/coreclr/jit/compiler.hpp | 54 +- src/coreclr/jit/compilerbitsettraits.h | 4 +- src/coreclr/jit/copyprop.cpp | 4 +- src/coreclr/jit/debuginfo.h | 18 +- src/coreclr/jit/decomposelongs.h | 5 +- src/coreclr/jit/disasm.cpp | 34 +- src/coreclr/jit/ee_il_dll.cpp | 8 +- src/coreclr/jit/ee_il_dll.hpp | 4 +- src/coreclr/jit/eeinterface.cpp | 43 +- src/coreclr/jit/emit.cpp | 152 ++-- src/coreclr/jit/emit.h | 399 ++++----- src/coreclr/jit/emitarm.cpp | 42 +- src/coreclr/jit/emitarm.h | 4 +- src/coreclr/jit/emitarm64.cpp | 128 +-- src/coreclr/jit/emitarm64.h | 158 ++-- src/coreclr/jit/emitarm64sve.cpp | 358 ++++---- src/coreclr/jit/emitloongarch64.cpp | 34 +- src/coreclr/jit/emitloongarch64.h | 16 +- src/coreclr/jit/emitpub.h | 36 +- src/coreclr/jit/emitriscv64.cpp | 28 +- src/coreclr/jit/emitriscv64.h | 24 +- src/coreclr/jit/emitxarch.cpp | 118 +-- src/coreclr/jit/emitxarch.h | 50 +- src/coreclr/jit/error.cpp | 4 +- src/coreclr/jit/fgbasic.cpp | 27 +- src/coreclr/jit/fgdiagnostic.cpp | 49 +- src/coreclr/jit/fgehopt.cpp | 8 +- src/coreclr/jit/fginline.cpp | 127 +-- src/coreclr/jit/fgopt.cpp | 38 +- src/coreclr/jit/fgprofile.cpp | 76 +- src/coreclr/jit/fgprofilesynthesis.h | 3 +- src/coreclr/jit/flowgraph.cpp | 35 +- src/coreclr/jit/forwardsub.cpp | 8 +- src/coreclr/jit/gcencode.cpp | 41 +- src/coreclr/jit/gcinfo.cpp | 3 +- src/coreclr/jit/gentree.cpp | 165 ++-- src/coreclr/jit/gentree.h | 517 ++++++++---- src/coreclr/jit/gschecks.cpp | 3 +- src/coreclr/jit/hashbv.cpp | 4 +- src/coreclr/jit/hashbv.h | 31 +- src/coreclr/jit/helperexpansion.cpp | 12 +- src/coreclr/jit/host.h | 6 +- src/coreclr/jit/hostallocator.h | 2 +- src/coreclr/jit/hwintrinsic.cpp | 2 +- src/coreclr/jit/hwintrinsic.h | 25 +- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 5 +- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 30 +- src/coreclr/jit/hwintrinsicxarch.cpp | 14 +- src/coreclr/jit/importer.cpp | 85 +- src/coreclr/jit/importercalls.cpp | 886 ++++++++++---------- src/coreclr/jit/importervectorization.cpp | 12 +- src/coreclr/jit/indirectcalltransformer.cpp | 16 +- src/coreclr/jit/inductionvariableopts.cpp | 7 +- src/coreclr/jit/inline.cpp | 2 +- src/coreclr/jit/inline.h | 42 +- src/coreclr/jit/inlinepolicy.cpp | 25 +- src/coreclr/jit/inlinepolicy.h | 39 +- src/coreclr/jit/instr.cpp | 14 +- src/coreclr/jit/instrsarm.h | 2 +- src/coreclr/jit/instrsarm64.h | 2 +- src/coreclr/jit/instrsloongarch64.h | 2 +- src/coreclr/jit/instrsxarch.h | 2 +- src/coreclr/jit/jit.h | 101 +-- src/coreclr/jit/jitconfig.cpp | 4 +- src/coreclr/jit/jitconfig.h | 8 +- src/coreclr/jit/jitconfigvalues.h | 6 +- src/coreclr/jit/jitee.h | 6 +- src/coreclr/jit/jiteh.cpp | 20 +- src/coreclr/jit/jiteh.h | 7 +- src/coreclr/jit/jitexpandarray.h | 9 +- src/coreclr/jit/jitgcinfo.h | 23 +- src/coreclr/jit/jithashtable.h | 40 +- src/coreclr/jit/layout.cpp | 8 +- src/coreclr/jit/layout.h | 5 +- src/coreclr/jit/lclmorph.cpp | 12 +- src/coreclr/jit/lclvars.cpp | 109 +-- src/coreclr/jit/likelyclass.cpp | 8 +- src/coreclr/jit/lir.cpp | 47 +- src/coreclr/jit/lir.h | 22 +- src/coreclr/jit/liveness.cpp | 32 +- src/coreclr/jit/loopcloning.cpp | 4 +- src/coreclr/jit/loopcloning.h | 75 +- src/coreclr/jit/lower.cpp | 28 +- src/coreclr/jit/lower.h | 136 +-- src/coreclr/jit/lowerarmarch.cpp | 2 +- src/coreclr/jit/lowerxarch.cpp | 40 +- src/coreclr/jit/lsra.cpp | 182 ++-- src/coreclr/jit/lsra.h | 376 +++++---- src/coreclr/jit/lsraarmarch.cpp | 34 +- src/coreclr/jit/lsrabuild.cpp | 186 ++-- src/coreclr/jit/lsraxarch.cpp | 38 +- src/coreclr/jit/morph.cpp | 102 +-- src/coreclr/jit/morphblock.cpp | 10 +- src/coreclr/jit/objectalloc.cpp | 10 +- src/coreclr/jit/objectalloc.h | 30 +- src/coreclr/jit/optcse.cpp | 34 +- src/coreclr/jit/optcse.h | 24 +- src/coreclr/jit/optimizebools.cpp | 8 +- src/coreclr/jit/optimizer.cpp | 37 +- src/coreclr/jit/patchpoint.cpp | 4 +- src/coreclr/jit/phase.h | 17 +- src/coreclr/jit/promotion.cpp | 24 +- src/coreclr/jit/promotion.h | 71 +- src/coreclr/jit/promotiondecomposition.cpp | 8 +- src/coreclr/jit/rangecheck.cpp | 11 +- src/coreclr/jit/rangecheck.h | 29 +- src/coreclr/jit/rationalize.cpp | 3 +- src/coreclr/jit/rationalize.h | 3 +- src/coreclr/jit/redundantbranchopts.cpp | 4 +- src/coreclr/jit/regset.cpp | 15 +- src/coreclr/jit/regset.h | 12 +- src/coreclr/jit/scev.cpp | 8 +- src/coreclr/jit/scev.h | 34 +- src/coreclr/jit/scopeinfo.cpp | 14 +- src/coreclr/jit/sideeffects.cpp | 24 +- src/coreclr/jit/sideeffects.h | 3 +- src/coreclr/jit/simd.h | 18 +- src/coreclr/jit/simdashwintrinsic.cpp | 6 +- src/coreclr/jit/sm.cpp | 4 +- src/coreclr/jit/smallhash.h | 26 +- src/coreclr/jit/smcommon.h | 2 +- src/coreclr/jit/smopenum.h | 3 +- src/coreclr/jit/ssabuilder.cpp | 5 +- src/coreclr/jit/ssabuilder.h | 2 +- src/coreclr/jit/ssarenamestate.cpp | 5 +- src/coreclr/jit/ssarenamestate.h | 7 +- src/coreclr/jit/stacklevelsetter.h | 4 +- src/coreclr/jit/switchrecognition.cpp | 2 +- src/coreclr/jit/target.h | 112 +-- src/coreclr/jit/tinyarray.h | 4 +- src/coreclr/jit/treelifeupdater.cpp | 2 +- src/coreclr/jit/typelist.h | 2 +- src/coreclr/jit/unwind.cpp | 2 +- src/coreclr/jit/unwind.h | 70 +- src/coreclr/jit/unwindamd64.cpp | 2 +- src/coreclr/jit/unwindarm64.cpp | 6 +- src/coreclr/jit/unwindarmarch.cpp | 37 +- src/coreclr/jit/unwindloongarch64.cpp | 12 +- src/coreclr/jit/unwindriscv64.cpp | 2 +- src/coreclr/jit/utils.cpp | 19 +- src/coreclr/jit/utils.h | 44 +- src/coreclr/jit/valuenum.cpp | 40 +- src/coreclr/jit/valuenum.h | 111 ++- src/coreclr/jit/valuenumtype.h | 4 +- src/coreclr/jit/varset.h | 2 +- src/coreclr/jit/vartype.h | 12 +- 172 files changed, 4585 insertions(+), 3679 deletions(-) diff --git a/src/coreclr/jit/_typeinfo.h b/src/coreclr/jit/_typeinfo.h index 42526eeb8de4bd..9285535b5531c4 100644 --- a/src/coreclr/jit/_typeinfo.h +++ b/src/coreclr/jit/_typeinfo.h @@ -41,25 +41,34 @@ class typeInfo private: var_types m_type; - union { + union + { CORINFO_CLASS_HANDLE m_cls; // Valid, but not always available, for TYP_REFs. methodPointerInfo* m_methodPointerInfo; // Valid only for function pointers. }; public: - typeInfo() : m_type(TYP_UNDEF), m_cls(NO_CLASS_HANDLE) + typeInfo() + : m_type(TYP_UNDEF) + , m_cls(NO_CLASS_HANDLE) { } - typeInfo(var_types type) : m_type(type), m_cls(NO_CLASS_HANDLE) + typeInfo(var_types type) + : m_type(type) + , m_cls(NO_CLASS_HANDLE) { } - typeInfo(CORINFO_CLASS_HANDLE cls) : m_type(TYP_REF), m_cls(cls) + typeInfo(CORINFO_CLASS_HANDLE cls) + : m_type(TYP_REF) + , m_cls(cls) { } - typeInfo(methodPointerInfo* methodPointerInfo) : m_type(TYP_I_IMPL), m_methodPointerInfo(methodPointerInfo) + typeInfo(methodPointerInfo* methodPointerInfo) + : m_type(TYP_I_IMPL) + , m_methodPointerInfo(methodPointerInfo) { assert(methodPointerInfo != nullptr); assert(methodPointerInfo->m_token.hMethod != nullptr); diff --git a/src/coreclr/jit/alloc.cpp b/src/coreclr/jit/alloc.cpp index 6300376beeb6d4..7178066ab584c5 100644 --- a/src/coreclr/jit/alloc.cpp +++ b/src/coreclr/jit/alloc.cpp @@ -42,7 +42,10 @@ size_t ArenaAllocator::getDefaultPageSize() // ArenaAllocator::ArenaAllocator: // Default-constructs an arena allocator. ArenaAllocator::ArenaAllocator() - : m_firstPage(nullptr), m_lastPage(nullptr), m_nextFreeByte(nullptr), m_lastFreeByte(nullptr) + : m_firstPage(nullptr) + , m_lastPage(nullptr) + , m_nextFreeByte(nullptr) + , m_lastFreeByte(nullptr) { #if MEASURE_MEM_ALLOC memset(&m_stats, 0, sizeof(m_stats)); diff --git a/src/coreclr/jit/alloc.h b/src/coreclr/jit/alloc.h index cb3da79232f8bb..8899b87ad35523 100644 --- a/src/coreclr/jit/alloc.h +++ b/src/coreclr/jit/alloc.h @@ -22,9 +22,9 @@ enum CompMemKind class ArenaAllocator { private: - ArenaAllocator(const ArenaAllocator& other) = delete; + ArenaAllocator(const ArenaAllocator& other) = delete; ArenaAllocator& operator=(const ArenaAllocator& other) = delete; - ArenaAllocator& operator=(ArenaAllocator&& other) = delete; + ArenaAllocator& operator=(ArenaAllocator&& other) = delete; struct PageDescriptor { @@ -52,7 +52,7 @@ class ArenaAllocator void* allocateNewPage(size_t size); static void* allocateHostMemory(size_t size, size_t* pActualSize); - static void freeHostMemory(void* block, size_t size); + static void freeHostMemory(void* block, size_t size); #if MEASURE_MEM_ALLOC struct MemStats @@ -125,8 +125,8 @@ class ArenaAllocator public: MemStatsAllocator* getMemStatsAllocator(CompMemKind kind); - void finishMemStats(); - void dumpMemStats(FILE* file); + void finishMemStats(); + void dumpMemStats(FILE* file); static void dumpMaxMemStats(FILE* file); static void dumpAggregateMemStats(FILE* file); @@ -276,7 +276,8 @@ class CompIAllocator : public IAllocator char m_zeroLenAllocTarg; public: - CompIAllocator(CompAllocator alloc) : m_alloc(alloc) + CompIAllocator(CompAllocator alloc) + : m_alloc(alloc) { } diff --git a/src/coreclr/jit/arraystack.h b/src/coreclr/jit/arraystack.h index 83a43c9432ba0e..5d8a697a3820d3 100644 --- a/src/coreclr/jit/arraystack.h +++ b/src/coreclr/jit/arraystack.h @@ -10,7 +10,8 @@ class ArrayStack static const int builtinSize = 8; public: - explicit ArrayStack(CompAllocator alloc, int initialCapacity = builtinSize) : m_alloc(alloc) + explicit ArrayStack(CompAllocator alloc, int initialCapacity = builtinSize) + : m_alloc(alloc) { if (initialCapacity > builtinSize) { diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 68a01a1ab7fcd1..41f38d1d9d7397 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1517,9 +1517,8 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, assertion.op1.lcl.ssaNum = op1->AsLclVarCommon()->GetSsaNum(); assert((assertion.op1.lcl.ssaNum == SsaConfig::RESERVED_SSA_NUM) || - (assertion.op1.vn == - vnStore->VNConservativeNormalValue( - lvaGetDesc(lclNum)->GetPerSsaData(assertion.op1.lcl.ssaNum)->m_vnPair))); + (assertion.op1.vn == vnStore->VNConservativeNormalValue( + lvaGetDesc(lclNum)->GetPerSsaData(assertion.op1.lcl.ssaNum)->m_vnPair))); ssize_t cnsValue = 0; GenTreeFlags iconFlags = GTF_EMPTY; @@ -2764,7 +2763,7 @@ GenTree* Compiler::optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTre case GT_CALL: return optVNBasedFoldExpr_Call(block, parent, tree->AsCall()); - // We can add more VN-based foldings here. + // We can add more VN-based foldings here. default: break; @@ -3319,7 +3318,7 @@ bool Compiler::optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, // GenTree* Compiler::optConstantAssertionProp(AssertionDsc* curAssertion, GenTreeLclVarCommon* tree, - Statement* stmt DEBUGARG(AssertionIndex index)) + Statement* stmt DEBUGARG(AssertionIndex index)) { const unsigned lclNum = tree->GetLclNum(); @@ -3574,7 +3573,7 @@ bool Compiler::optAssertionProp_LclVarTypeCheck(GenTree* tree, LclVarDsc* lclVar // GenTree* Compiler::optCopyAssertionProp(AssertionDsc* curAssertion, GenTreeLclVarCommon* tree, - Statement* stmt DEBUGARG(AssertionIndex index)) + Statement* stmt DEBUGARG(AssertionIndex index)) { const AssertionDsc::AssertionDscOp1& op1 = curAssertion->op1; const AssertionDsc::AssertionDscOp2& op2 = curAssertion->op2; @@ -4523,8 +4522,9 @@ GenTree* Compiler::optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, Gen { printf("\nVN relop based copy assertion prop in " FMT_BB ":\n", compCurBB->bbNum); printf("Assertion index=#%02u: V%02d.%02d %s V%02d.%02d\n", index, op1->AsLclVar()->GetLclNum(), - op1->AsLclVar()->GetSsaNum(), (curAssertion->assertionKind == OAK_EQUAL) ? "==" : "!=", - op2->AsLclVar()->GetLclNum(), op2->AsLclVar()->GetSsaNum()); + op1->AsLclVar()->GetSsaNum(), + (curAssertion->assertionKind == OAK_EQUAL) ? "==" : "!=", op2->AsLclVar()->GetLclNum(), + op2->AsLclVar()->GetSsaNum()); gtDispTree(tree, nullptr, nullptr, true); } #endif @@ -4818,7 +4818,7 @@ GenTree* Compiler::optAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* tr // If both VN and assertion table yield a matching assertion, "pVnBased" // is only set and the return value is "NO_ASSERTION_INDEX." // -bool Compiler::optAssertionIsNonNull(GenTree* op, +bool Compiler::optAssertionIsNonNull(GenTree* op, ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased) DEBUGARG(AssertionIndex* pIndex)) { @@ -4865,7 +4865,7 @@ bool Compiler::optAssertionIsNonNull(GenTree* op, // Return Value: // index of assertion, or NO_ASSERTION_INDEX // -AssertionIndex Compiler::optAssertionIsNonNullInternal(GenTree* op, +AssertionIndex Compiler::optAssertionIsNonNullInternal(GenTree* op, ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased)) { @@ -6258,7 +6258,9 @@ struct VNAssertionPropVisitorInfo Statement* stmt; BasicBlock* block; VNAssertionPropVisitorInfo(Compiler* pThis, BasicBlock* block, Statement* stmt) - : pThis(pThis), stmt(stmt), block(block) + : pThis(pThis) + , stmt(stmt) + , block(block) { } }; diff --git a/src/coreclr/jit/bitset.h b/src/coreclr/jit/bitset.h index b34d1f04b85f1f..6f1e3d8dcd0db1 100644 --- a/src/coreclr/jit/bitset.h +++ b/src/coreclr/jit/bitset.h @@ -59,7 +59,10 @@ class BitSetSupport FILE* OpOutputFile; public: - BitSetOpCounter(const char* fileName) : TotalOps(0), m_fileName(fileName), OpOutputFile(nullptr) + BitSetOpCounter(const char* fileName) + : TotalOps(0) + , m_fileName(fileName) + , OpOutputFile(nullptr) { for (unsigned i = 0; i < BSOP_NUMOPS; i++) { @@ -435,7 +438,9 @@ class BitSetOpsWithCounter Env m_env; public: - Iter(Env env, BitSetValueArgType bs) : m_iter(env, bs), m_env(env) + Iter(Env env, BitSetValueArgType bs) + : m_iter(env, bs) + , m_env(env) { } @@ -449,8 +454,8 @@ class BitSetOpsWithCounter // We define symbolic names for the various bitset implementations available, to allow choices between them. -#define BSUInt64 0 -#define BSShortLong 1 +#define BSUInt64 0 +#define BSShortLong 1 #define BSUInt64Class 2 /*****************************************************************************/ diff --git a/src/coreclr/jit/bitsetasshortlong.h b/src/coreclr/jit/bitsetasshortlong.h index 2ef293820fd264..006f66fc178dcb 100644 --- a/src/coreclr/jit/bitsetasshortlong.h +++ b/src/coreclr/jit/bitsetasshortlong.h @@ -32,36 +32,36 @@ class BitSetOps m_bs; public: - BitSetUint64ValueRetType(const BitSetUint64& bs) : m_bs(bs) + BitSetUint64ValueRetType(const BitSetUint64& bs) + : m_bs(bs) { } }; @@ -451,7 +452,9 @@ class BitSetOps, unsigned m_bitNum; public: - Iter(Env env, const BitSetUint64& bs) : m_bits(bs.m_bits), m_bitNum(0) + Iter(Env env, const BitSetUint64& bs) + : m_bits(bs.m_bits) + , m_bitNum(0) { } diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 6eea265871c04c..6cde9e0e93d8b1 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -34,7 +34,7 @@ unsigned BasicBlock::s_nMaxTrees; FlowEdge* ShuffleHelper(unsigned hash, FlowEdge* res) { FlowEdge* head = res; - for (FlowEdge *prev = nullptr; res != nullptr; prev = res, res = res->getNextPredEdge()) + for (FlowEdge* prev = nullptr; res != nullptr; prev = res, res = res->getNextPredEdge()) { unsigned blkHash = (hash ^ (res->getSourceBlock()->bbNum << 16) ^ res->getSourceBlock()->bbNum); if (((blkHash % 1879) & 1) && prev != nullptr) @@ -140,7 +140,8 @@ void FlowEdge::addLikelihood(weight_t addedLikelihood) // comp - Compiler instance // block - The block whose successors are to be iterated // -AllSuccessorEnumerator::AllSuccessorEnumerator(Compiler* comp, BasicBlock* block) : m_block(block) +AllSuccessorEnumerator::AllSuccessorEnumerator(Compiler* comp, BasicBlock* block) + : m_block(block) { m_numSuccs = 0; block->VisitAllSuccs(comp, [this](BasicBlock* succ) { @@ -1891,7 +1892,8 @@ BBswtDesc::BBswtDesc(Compiler* comp, const BBswtDesc* other) // comp - compiler instance // other - existing descriptor to copy // -BBehfDesc::BBehfDesc(Compiler* comp, const BBehfDesc* other) : bbeCount(other->bbeCount) +BBehfDesc::BBehfDesc(Compiler* comp, const BBehfDesc* other) + : bbeCount(other->bbeCount) { // Allocate and fill in a new dst tab // diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index dfaace466a6570..5f2ab1f6e81ba7 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -162,7 +162,8 @@ class MemoryKindIterator int value; public: - explicit inline MemoryKindIterator(int val) : value(val) + explicit inline MemoryKindIterator(int val) + : value(val) { } inline MemoryKindIterator& operator++() @@ -244,7 +245,8 @@ class PredEdgeList }; public: - PredEdgeList(FlowEdge* pred) : m_begin(pred) + PredEdgeList(FlowEdge* pred) + : m_begin(pred) { } @@ -297,7 +299,8 @@ class PredBlockList }; public: - PredBlockList(FlowEdge* pred) : m_begin(pred) + PredBlockList(FlowEdge* pred) + : m_begin(pred) { } @@ -322,7 +325,8 @@ class BBArrayIterator FlowEdge* const* m_edgeEntry; public: - BBArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) + BBArrayIterator(FlowEdge* const* edgeEntry) + : m_edgeEntry(edgeEntry) { } @@ -351,7 +355,8 @@ class FlowEdgeArrayIterator FlowEdge* const* m_edgeEntry; public: - FlowEdgeArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) + FlowEdgeArrayIterator(FlowEdge* const* edgeEntry) + : m_edgeEntry(edgeEntry) { } @@ -727,7 +732,8 @@ struct BasicBlock : private LIR::Range BBKinds bbKind; // jump (if any) at the end of this block /* The following union describes the jump target(s) of this block */ - union { + union + { unsigned bbTargetOffs; // PC offset (temporary only) FlowEdge* bbTargetEdge; // successor edge for block kinds with only one successor (BBJ_ALWAYS, etc) FlowEdge* bbTrueEdge; // BBJ_COND successor edge when its condition is true (alias for bbTargetEdge) @@ -1165,11 +1171,11 @@ struct BasicBlock : private LIR::Range } #ifdef DEBUG - void dspFlags() const; // Print the flags - unsigned dspPreds() const; // Print the predecessors (bbPreds) - void dspSuccs(Compiler* compiler); // Print the successors. The 'compiler' argument determines whether EH - // regions are printed: see NumSucc() for details. - void dspKind() const; // Print the block jump kind (e.g., BBJ_ALWAYS, BBJ_COND, etc.). + void dspFlags() const; // Print the flags + unsigned dspPreds() const; // Print the predecessors (bbPreds) + void dspSuccs(Compiler* compiler); // Print the successors. The 'compiler' argument determines whether EH + // regions are printed: see NumSucc() for details. + void dspKind() const; // Print the block jump kind (e.g., BBJ_ALWAYS, BBJ_COND, etc.). // Print a simple basic block header for various output, including a list of predecessors and successors. void dspBlockHeader(Compiler* compiler, bool showKind = true, bool showFlags = false, bool showPreds = true); @@ -1177,11 +1183,11 @@ struct BasicBlock : private LIR::Range const char* dspToString(int blockNumPadding = 0) const; #endif // DEBUG -#define BB_UNITY_WEIGHT 100.0 // how much a normal execute once block weighs -#define BB_UNITY_WEIGHT_UNSIGNED 100 // how much a normal execute once block weighs -#define BB_LOOP_WEIGHT_SCALE 8.0 // synthetic profile scale factor for loops -#define BB_ZERO_WEIGHT 0.0 -#define BB_MAX_WEIGHT FLT_MAX // maximum finite weight -- needs rethinking. +#define BB_UNITY_WEIGHT 100.0 // how much a normal execute once block weighs +#define BB_UNITY_WEIGHT_UNSIGNED 100 // how much a normal execute once block weighs +#define BB_LOOP_WEIGHT_SCALE 8.0 // synthetic profile scale factor for loops +#define BB_ZERO_WEIGHT 0.0 +#define BB_MAX_WEIGHT FLT_MAX // maximum finite weight -- needs rethinking. weight_t bbWeight; // The dynamic execution weight of this block @@ -1402,12 +1408,14 @@ struct BasicBlock : private LIR::Range #define NO_BASE_TMP UINT_MAX // base# to use when we have none - union { + union + { unsigned bbStkTempsIn; // base# for input stack temps int bbCountSchemaIndex; // schema index for count instrumentation }; - union { + union + { unsigned bbStkTempsOut; // base# for output stack temps int bbHistogramSchemaIndex; // schema index for histogram instrumentation }; @@ -1527,15 +1535,16 @@ struct BasicBlock : private LIR::Range bool hasEHBoundaryOut() const; // Some non-zero value that will not collide with real tokens for bbCatchTyp -#define BBCT_NONE 0x00000000 -#define BBCT_FAULT 0xFFFFFFFC -#define BBCT_FINALLY 0xFFFFFFFD -#define BBCT_FILTER 0xFFFFFFFE -#define BBCT_FILTER_HANDLER 0xFFFFFFFF +#define BBCT_NONE 0x00000000 +#define BBCT_FAULT 0xFFFFFFFC +#define BBCT_FINALLY 0xFFFFFFFD +#define BBCT_FILTER 0xFFFFFFFE +#define BBCT_FILTER_HANDLER 0xFFFFFFFF #define handlerGetsXcptnObj(hndTyp) ((hndTyp) != BBCT_NONE && (hndTyp) != BBCT_FAULT && (hndTyp) != BBCT_FINALLY) // TODO-Cleanup: Get rid of bbStkDepth and use bbStackDepthOnEntry() instead - union { + union + { unsigned short bbStkDepth; // stack depth on entry unsigned short bbFPinVars; // number of inner enregistered FP vars }; @@ -1577,7 +1586,8 @@ struct BasicBlock : private LIR::Range void ensurePredListOrder(Compiler* compiler); void reorderPredList(Compiler* compiler); - union { + union + { BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate // Dominator) used to compute the dominance tree. FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred @@ -1626,7 +1636,9 @@ struct BasicBlock : private LIR::Range return m_ssaNum; } - MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) : m_ssaNum(ssaNum), m_nextArg(nextArg) + MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) + : m_ssaNum(ssaNum) + , m_nextArg(nextArg) { } @@ -1652,18 +1664,21 @@ struct BasicBlock : private LIR::Range * thus we can union them since the two operations are completely disjunct. */ - union { + union + { EXPSET_TP bbCseGen; // CSEs computed by block ASSERT_TP bbAssertionGen; // assertions created by block (global prop) ASSERT_TP bbAssertionOutIfTrue; // assertions available on exit along true/jump edge (BBJ_COND, local prop) }; - union { + union + { EXPSET_TP bbCseIn; // CSEs available on entry ASSERT_TP bbAssertionIn; // assertions available on entry (global prop) }; - union { + union + { EXPSET_TP bbCseOut; // CSEs available on exit ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local prop) @@ -1679,7 +1694,7 @@ struct BasicBlock : private LIR::Range verTypeVal* bbTypesOut; // list of variable types on output #endif // VERIFIER -//------------------------------------------------------------------------- + //------------------------------------------------------------------------- #if MEASURE_BLOCK_SIZE static size_t s_Size; @@ -1714,8 +1729,8 @@ struct BasicBlock : private LIR::Range unsigned bbID; #endif // DEBUG - unsigned bbStackDepthOnEntry() const; - void bbSetStack(StackEntry* stack); + unsigned bbStackDepthOnEntry() const; + void bbSetStack(StackEntry* stack); StackEntry* bbStackOnEntry() const; // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding @@ -1765,7 +1780,10 @@ struct BasicBlock : private LIR::Range Statement* FirstNonPhiDef() const; Statement* FirstNonPhiDefOrCatchArgStore() const; - BasicBlock() : bbStmtList(nullptr), bbLiveIn(VarSetOps::UninitVal()), bbLiveOut(VarSetOps::UninitVal()) + BasicBlock() + : bbStmtList(nullptr) + , bbLiveIn(VarSetOps::UninitVal()) + , bbLiveOut(VarSetOps::UninitVal()) { } @@ -1777,7 +1795,9 @@ struct BasicBlock : private LIR::Range BasicBlock* m_block; public: - Successors(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block) + Successors(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { } @@ -1788,11 +1808,15 @@ struct BasicBlock : private LIR::Range TPosition m_pos; public: - iterator(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block), m_pos(comp, block) + iterator(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) + , m_pos(comp, block) { } - iterator() : m_pos() + iterator() + : m_pos() { } @@ -1865,7 +1889,8 @@ struct BasicBlock : private LIR::Range class BBSuccList : private SuccList { public: - BBSuccList(const BasicBlock* block) : SuccList(block) + BBSuccList(const BasicBlock* block) + : SuccList(block) { } @@ -1887,7 +1912,8 @@ struct BasicBlock : private LIR::Range class BBSuccEdgeList : private SuccList { public: - BBSuccEdgeList(const BasicBlock* block) : SuccList(block) + BBSuccEdgeList(const BasicBlock* block) + : SuccList(block) { } @@ -1923,7 +1949,9 @@ struct BasicBlock : private LIR::Range public: iterator(Compiler* comp, BasicBlock* block, unsigned succNum) - : m_comp(comp), m_block(block), m_succNum(succNum) + : m_comp(comp) + , m_block(block) + , m_succNum(succNum) { } @@ -1948,7 +1976,9 @@ struct BasicBlock : private LIR::Range }; public: - BBCompilerSuccList(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block) + BBCompilerSuccList(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { } @@ -1984,7 +2014,9 @@ struct BasicBlock : private LIR::Range public: iterator(Compiler* comp, BasicBlock* block, unsigned succNum) - : m_comp(comp), m_block(block), m_succNum(succNum) + : m_comp(comp) + , m_block(block) + , m_succNum(succNum) { } @@ -2009,7 +2041,9 @@ struct BasicBlock : private LIR::Range }; public: - BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block) + BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { } @@ -2119,7 +2153,8 @@ class BasicBlockIterator BasicBlock* m_block; public: - BasicBlockIterator(BasicBlock* block) : m_block(block) + BasicBlockIterator(BasicBlock* block) + : m_block(block) { } @@ -2155,7 +2190,8 @@ class BasicBlockSimpleList BasicBlock* m_begin; public: - BasicBlockSimpleList(BasicBlock* begin) : m_begin(begin) + BasicBlockSimpleList(BasicBlock* begin) + : m_begin(begin) { } @@ -2185,7 +2221,9 @@ class BasicBlockRangeList BasicBlock* m_end; public: - BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) : m_begin(begin), m_end(end) + BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) + : m_begin(begin) + , m_end(end) { assert(begin != nullptr); assert(end != nullptr); @@ -2225,7 +2263,9 @@ struct BBswtDesc bool bbsHasDefault; // true if last switch case is a default case bool bbsHasDominantCase; // true if switch has a dominant case - BBswtDesc() : bbsHasDefault(true), bbsHasDominantCase(false) + BBswtDesc() + : bbsHasDefault(true) + , bbsHasDominantCase(false) { } @@ -2252,7 +2292,8 @@ struct BBswtDesc // BBSwitchTargetList out-of-class-declaration implementations (here due to C++ ordering requirements). // -inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) : m_bbsDesc(bbsDesc) +inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) + : m_bbsDesc(bbsDesc) { assert(m_bbsDesc != nullptr); assert(m_bbsDesc->bbsDstTab != nullptr); @@ -2275,7 +2316,9 @@ struct BBehfDesc FlowEdge** bbeSuccs; // array of `FlowEdge*` pointing to BBJ_EHFINALLYRET block successors unsigned bbeCount; // size of `bbeSuccs` array - BBehfDesc() : bbeSuccs(nullptr), bbeCount(0) + BBehfDesc() + : bbeSuccs(nullptr) + , bbeCount(0) { } @@ -2285,7 +2328,8 @@ struct BBehfDesc // BBEhfSuccList out-of-class-declaration implementations (here due to C++ ordering requirements). // -inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) : m_bbeDesc(bbeDesc) +inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) + : m_bbeDesc(bbeDesc) { assert(m_bbeDesc != nullptr); assert((m_bbeDesc->bbeSuccs != nullptr) || (m_bbeDesc->bbeCount == 0)); @@ -2384,11 +2428,15 @@ struct BasicBlockList BasicBlockList* next; // The next BasicBlock in the list, nullptr for end of list. BasicBlock* block; // The BasicBlock of interest. - BasicBlockList() : next(nullptr), block(nullptr) + BasicBlockList() + : next(nullptr) + , block(nullptr) { } - BasicBlockList(BasicBlock* blk, BasicBlockList* rest) : next(rest), block(blk) + BasicBlockList(BasicBlock* blk, BasicBlockList* rest) + : next(rest) + , block(blk) { } }; @@ -2414,7 +2462,8 @@ inline BasicBlock* BBArrayIterator::operator*() const // Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and FlowEdge) -inline PredEdgeList::iterator::iterator(FlowEdge* pred) : m_pred(pred) +inline PredEdgeList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) { #ifdef DEBUG m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); @@ -2436,7 +2485,8 @@ inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() } template -inline PredBlockList::iterator::iterator(FlowEdge* pred) : m_pred(pred) +inline PredBlockList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) { bool initNextPointer = allowEdits; INDEBUG(initNextPointer = true); @@ -2446,13 +2496,13 @@ inline PredBlockList::iterator::iterator(FlowEdge* pred) : m_pred(pr } } -template +template inline BasicBlock* PredBlockList::iterator::operator*() const { return m_pred->getSourceBlock(); } -template +template inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() { if (allowEdits) @@ -2491,7 +2541,8 @@ void* emitCodeGetCookie(const BasicBlock* block); class AllSuccessorEnumerator { BasicBlock* m_block; - union { + union + { // We store up to 4 successors inline in the enumerator. For ASP.NET // and libraries.pmi this is enough in 99.7% of cases. BasicBlock* m_successors[4]; diff --git a/src/coreclr/jit/blockset.h b/src/coreclr/jit/blockset.h index 83de7a5dad1e59..f69e1e59ace324 100644 --- a/src/coreclr/jit/blockset.h +++ b/src/coreclr/jit/blockset.h @@ -24,10 +24,11 @@ #include "compilerbitsettraits.h" #include "bitsetasshortlong.h" -class BlockSetOps : public BitSetOps +class BlockSetOps + : public BitSetOps { public: // Specialize BlockSetOps::MakeFull(). Since we number basic blocks from one, we remove bit zero from diff --git a/src/coreclr/jit/buildstring.cpp b/src/coreclr/jit/buildstring.cpp index f432fec47475ff..3f0222ad2649ac 100644 --- a/src/coreclr/jit/buildstring.cpp +++ b/src/coreclr/jit/buildstring.cpp @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#define STRINGIFY(L) #L +#define STRINGIFY(L) #L #define MAKESTRING(M, L) M(L) -#define STRINGIZE(X) MAKESTRING(STRINGIFY, X) +#define STRINGIZE(X) MAKESTRING(STRINGIFY, X) #if defined(__clang__) #define BUILD_COMPILER \ diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index acbd27bc7d9d71..c0e31841f2cc36 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -103,7 +103,7 @@ class CodeGen final : public CodeGenInterface static bool genShouldRoundFP(); - static GenTreeIndir indirForm(var_types type, GenTree* base); + static GenTreeIndir indirForm(var_types type, GenTree* base); static GenTreeStoreInd storeIndirForm(var_types type, GenTree* base, GenTree* data); GenTreeIntCon intForm(var_types type, ssize_t value); @@ -179,8 +179,8 @@ class CodeGen final : public CodeGenInterface #ifdef JIT32_GCENCODER void* genCreateAndStoreGCInfo(unsigned codeSize, unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)); - void* genCreateAndStoreGCInfoJIT32(unsigned codeSize, - unsigned prologSize, + void* genCreateAndStoreGCInfoJIT32(unsigned codeSize, + unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)); #else // !JIT32_GCENCODER void genCreateAndStoreGCInfo(unsigned codeSize, unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)); @@ -208,7 +208,7 @@ class CodeGen final : public CodeGenInterface unsigned genCurDispOffset; static const char* genInsName(instruction ins); - const char* genInsDisplayName(emitter::instrDesc* id); + const char* genInsDisplayName(emitter::instrDesc* id); static const char* genSizeStr(emitAttr size); @@ -319,11 +319,17 @@ class CodeGen final : public CodeGenInterface regNumber reg2; bool useSaveNextPair; - RegPair(regNumber reg1) : reg1(reg1), reg2(REG_NA), useSaveNextPair(false) + RegPair(regNumber reg1) + : reg1(reg1) + , reg2(REG_NA) + , useSaveNextPair(false) { } - RegPair(regNumber reg1, regNumber reg2) : reg1(reg1), reg2(reg2), useSaveNextPair(false) + RegPair(regNumber reg1, regNumber reg2) + : reg1(reg1) + , reg2(reg2) + , useSaveNextPair(false) { assert(reg2 == REG_NEXT(reg1)); } @@ -335,7 +341,9 @@ class CodeGen final : public CodeGenInterface static int genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask); - void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)); + void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, + int spDelta, + int spOffset MORE_THAN_64_REG_ARG(var_types type)); void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)); @@ -369,8 +377,8 @@ class CodeGen final : public CodeGenInterface bool genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg); - void genPushFltRegs(regMaskFloat regMask); - void genPopFltRegs(regMaskFloat regMask); + void genPushFltRegs(regMaskFloat regMask); + void genPopFltRegs(regMaskFloat regMask); regMaskGpr genStackAllocRegisterMask(unsigned frameSize, regMaskFloat maskCalleeSavedFloat); void genFreeLclFrame(unsigned frameSize, @@ -408,8 +416,8 @@ class CodeGen final : public CodeGenInterface regMaskGpr fiSaveGprRegs; // Set of callee-saved GPR registers saved in the funclet prolog (includes LR) regMaskFloat fiSaveFloatRegs; // Set of callee-saved float registers saved in the funclet prolog (includes LR) #ifdef FEATURE_MASKED_HW_INTRINSICS - regMaskPredicate fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog - // (includes LR) + regMaskPredicate fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog + // (includes LR) #endif int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function // (negative) @@ -688,17 +696,17 @@ class CodeGen final : public CodeGenInterface void genSinglePush(); void genSinglePop(); regMaskGpr genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskGpr* noRefRegs); - void genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs); - -/* -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XX XX -XX Debugging Support XX -XX XX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -*/ + void genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs); + + /* + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XX XX + XX Debugging Support XX + XX XX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + */ #ifdef DEBUG void genIPmappingDisp(unsigned mappingNum, const IPmappingDsc* ipMapping); @@ -948,7 +956,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCompareFloat(GenTree* treeNode); void genCompareInt(GenTree* treeNode); #ifdef TARGET_XARCH - bool genCanAvoidEmittingCompareAgainstZero(GenTree* tree, var_types opType); + bool genCanAvoidEmittingCompareAgainstZero(GenTree* tree, var_types opType); GenTree* genTryFindFlagsConsumer(GenTree* flagsProducer, GenCondition** condition); #endif @@ -1121,12 +1129,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum); void genUnspillLocal( unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse); - void genUnspillRegIfNeeded(GenTree* tree); - void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex); + void genUnspillRegIfNeeded(GenTree* tree); + void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex); regNumber genConsumeReg(GenTree* tree); regNumber genConsumeReg(GenTree* tree, unsigned multiRegIndex); - void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); - void genConsumeRegAndCopy(GenTree* tree, regNumber needReg); + void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); + void genConsumeRegAndCopy(GenTree* tree, regNumber needReg); void genConsumeIfReg(GenTree* tree) { @@ -1136,15 +1144,15 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX } } - void genRegCopy(GenTree* tree); + void genRegCopy(GenTree* tree); regNumber genRegCopy(GenTree* tree, unsigned multiRegIndex); - void genTransferRegGCState(regNumber dst, regNumber src); - void genConsumeAddress(GenTree* addr); - void genConsumeAddrMode(GenTreeAddrMode* mode); - void genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg); - void genConsumeBlockSrc(GenTreeBlk* blkNode); - void genSetBlockSrc(GenTreeBlk* blkNode, regNumber srcReg); - void genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber srcReg, regNumber sizeReg); + void genTransferRegGCState(regNumber dst, regNumber src); + void genConsumeAddress(GenTree* addr); + void genConsumeAddrMode(GenTreeAddrMode* mode); + void genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg); + void genConsumeBlockSrc(GenTreeBlk* blkNode); + void genSetBlockSrc(GenTreeBlk* blkNode, regNumber srcReg); + void genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber srcReg, regNumber sizeReg); #ifdef FEATURE_PUT_STRUCT_ARG_STK void genConsumePutStructArgStk(GenTreePutArgStk* putArgStkNode, @@ -1252,10 +1260,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX unsigned genMove4IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset); unsigned genMove2IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset); unsigned genMove1IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset); - void genCodeForLoadOffset(instruction ins, emitAttr size, regNumber dst, GenTree* base, unsigned offset); - void genStoreRegToStackArg(var_types type, regNumber reg, int offset); - void genStructPutArgRepMovs(GenTreePutArgStk* putArgStkNode); - void genStructPutArgUnroll(GenTreePutArgStk* putArgStkNode); + void genCodeForLoadOffset(instruction ins, emitAttr size, regNumber dst, GenTree* base, unsigned offset); + void genStoreRegToStackArg(var_types type, regNumber reg, int offset); + void genStructPutArgRepMovs(GenTreePutArgStk* putArgStkNode); + void genStructPutArgUnroll(GenTreePutArgStk* putArgStkNode); #ifdef TARGET_X86 void genStructPutArgPush(GenTreePutArgStk* putArgStkNode); #else @@ -1263,13 +1271,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif #endif // FEATURE_PUT_STRUCT_ARG_STK - void genCodeForStoreBlk(GenTreeBlk* storeBlkNode); - void genCodeForInitBlkLoop(GenTreeBlk* initBlkNode); - void genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode); - void genCodeForInitBlkUnroll(GenTreeBlk* initBlkNode); + void genCodeForStoreBlk(GenTreeBlk* storeBlkNode); + void genCodeForInitBlkLoop(GenTreeBlk* initBlkNode); + void genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode); + void genCodeForInitBlkUnroll(GenTreeBlk* initBlkNode); unsigned genEmitJumpTable(GenTree* treeNode, bool relativeAddr); - void genJumpTable(GenTree* tree); - void genTableBasedSwitch(GenTree* tree); + void genJumpTable(GenTree* tree); + void genTableBasedSwitch(GenTree* tree); #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) instruction genGetInsForOper(GenTree* treeNode); #else @@ -1279,13 +1287,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX regNumber targetReg, GenTreeIndir* indir, bool* needsBarrier); - bool genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarrierForm, GenTree* addr, GenTree* data); - GenTree* getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd); - regNumber getCallIndirectionCellReg(GenTreeCall* call); - void genCall(GenTreeCall* call); - void genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes)); - void genDefinePendingCallLabel(GenTreeCall* call); - void genJmpMethod(GenTree* jmp); + bool genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarrierForm, GenTree* addr, GenTree* data); + GenTree* getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd); + regNumber getCallIndirectionCellReg(GenTreeCall* call); + void genCall(GenTreeCall* call); + void genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes)); + void genDefinePendingCallLabel(GenTreeCall* call); + void genJmpMethod(GenTree* jmp); BasicBlock* genCallFinally(BasicBlock* block); #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // TODO: refactor for LA. @@ -1327,13 +1335,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genReturn(GenTree* treeNode); #ifdef TARGET_XARCH - void genStackPointerConstantAdjustment(ssize_t spDelta, bool trackSpAdjustments); - void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, bool trackSpAdjustments); + void genStackPointerConstantAdjustment(ssize_t spDelta, bool trackSpAdjustments); + void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, bool trackSpAdjustments); target_ssize_t genStackPointerConstantAdjustmentLoopWithProbe(ssize_t spDelta, bool trackSpAdjustments); - void genStackPointerDynamicAdjustmentWithProbe(regNumber regSpDelta); + void genStackPointerDynamicAdjustmentWithProbe(regNumber regSpDelta); #else // !TARGET_XARCH - void genStackPointerConstantAdjustment(ssize_t spDelta, regNumber regTmp); - void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, regNumber regTmp); + void genStackPointerConstantAdjustment(ssize_t spDelta, regNumber regTmp); + void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, regNumber regTmp); target_ssize_t genStackPointerConstantAdjustmentLoopWithProbe(ssize_t spDelta, regNumber regTmp); #endif // !TARGET_XARCH @@ -1367,8 +1375,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #ifdef DEBUG GenTree* lastConsumedNode; - void genNumberOperandUse(GenTree* const operand, int& useNum) const; - void genCheckConsumeNode(GenTree* const node); + void genNumberOperandUse(GenTree* const operand, int& useNum) const; + void genCheckConsumeNode(GenTree* const node); #else // !DEBUG inline void genCheckConsumeNode(GenTree* treeNode) { @@ -1446,7 +1454,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #if defined(TARGET_XARCH) - enum class OperandKind{ + enum class OperandKind + { ClsVar, // [CLS_VAR_ADDR] - "C" in the emitter. Local, // [Local or spill temp + offset] - "S" in the emitter. Indir, // [base+index*scale+disp] - "A" in the emitter. @@ -1457,7 +1466,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX class OperandDesc { OperandKind m_kind; - union { + union + { struct { CORINFO_FIELD_HANDLE m_fieldHnd; @@ -1485,30 +1495,45 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX }; public: - OperandDesc(CORINFO_FIELD_HANDLE fieldHnd) : m_kind(OperandKind::ClsVar), m_fieldHnd(fieldHnd) + OperandDesc(CORINFO_FIELD_HANDLE fieldHnd) + : m_kind(OperandKind::ClsVar) + , m_fieldHnd(fieldHnd) { } - OperandDesc(int varNum, uint16_t offset) : m_kind(OperandKind::Local), m_varNum(varNum), m_offset(offset) + OperandDesc(int varNum, uint16_t offset) + : m_kind(OperandKind::Local) + , m_varNum(varNum) + , m_offset(offset) { } OperandDesc(GenTreeIndir* indir) - : m_kind(OperandKind::Indir), m_addr(indir->Addr()), m_indir(indir), m_indirType(indir->TypeGet()) + : m_kind(OperandKind::Indir) + , m_addr(indir->Addr()) + , m_indir(indir) + , m_indirType(indir->TypeGet()) { } OperandDesc(var_types indirType, GenTree* addr) - : m_kind(OperandKind::Indir), m_addr(addr), m_indir(nullptr), m_indirType(indirType) + : m_kind(OperandKind::Indir) + , m_addr(addr) + , m_indir(nullptr) + , m_indirType(indirType) { } OperandDesc(ssize_t immediate, bool immediateNeedsReloc) - : m_kind(OperandKind::Imm), m_immediate(immediate), m_immediateNeedsReloc(immediateNeedsReloc) + : m_kind(OperandKind::Imm) + , m_immediate(immediate) + , m_immediateNeedsReloc(immediateNeedsReloc) { } - OperandDesc(regNumber reg) : m_kind(OperandKind::Reg), m_reg(reg) + OperandDesc(regNumber reg) + : m_kind(OperandKind::Reg) + , m_reg(reg) { } @@ -1698,7 +1723,9 @@ class CodeGenPhase final : public Phase { public: CodeGenPhase(CodeGen* _codeGen, Phases _phase, void (CodeGen::*_action)()) - : Phase(_codeGen->GetCompiler(), _phase), codeGen(_codeGen), action(_action) + : Phase(_codeGen->GetCompiler(), _phase) + , codeGen(_codeGen) + , action(_action) { } diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 530055e967b306..1cb5fef4d14ed9 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -173,9 +173,9 @@ void CodeGen::genEHCatchRet(BasicBlock* block) //------------------------------------------------------------------------ // instGen_Set_Reg_To_Imm: Move an immediate value into an integer register. // -void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, - regNumber reg, - ssize_t imm, +void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, + regNumber reg, + ssize_t imm, insFlags flags DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { // reg cannot be a FP register @@ -1651,7 +1651,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTargetReg, // ireg REG_NA, 0, 0, // xreg, xmul, disp false // isJump - ); + ); } else { @@ -1660,7 +1660,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), REG_NA, REG_NA, 0, 0, /* ilOffset, ireg, xreg, xmul, disp */ false /* isJump */ - ); + ); } regSet.verifyRegistersUsed(AllRegsMask_CALLEE_TRASH); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 3f30cec772794f..ce1327d32de9d6 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -207,10 +207,11 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) JITDUMP(" calleeSaveSpOffset=%d, calleeSaveSpDelta=%d\n", calleeSaveSpOffset, calleeSaveSpDelta); genRestoreCalleeSavedRegistersHelp(AllRegsMask(rsRestoreGprRegs, rsRestoreFloatRegs #ifdef FEATURE_MASKED_HW_INTRINSICS - ,rsRestorePredicateRegs + , + rsRestorePredicateRegs #endif - ), calleeSaveSpOffset, - calleeSaveSpDelta); + ), + calleeSaveSpOffset, calleeSaveSpDelta); switch (frameType) { @@ -725,7 +726,8 @@ void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, reg // no return value; the regStack argument is modified. // // static -void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, ArrayStack* regStack MORE_THAN_64_REG_ARG(var_types type)) +void CodeGen::genBuildRegPairsStack(regMaskOnlyOne regsMask, + ArrayStack* regStack MORE_THAN_64_REG_ARG(var_types type)) { assert(regStack != nullptr); assert(regStack->Height() == 0); @@ -844,7 +846,9 @@ int CodeGen::genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask) // spOffset - the offset from SP that is the beginning of the callee-saved register area; // type - The type of `regsMask` we are operating on. // -void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)) +void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, + int spDelta, + int spOffset MORE_THAN_64_REG_ARG(var_types type)) { assert(compiler->IsOnlyOneRegMask(regsMask)); @@ -930,7 +934,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo assert(compiler->IsPredicateRegMask(maskSaveRegsPredicate)); regsToSaveCount += genCountBits(maskSaveRegsPredicate); #endif - + if (regsToSaveCount == 0) { if (spDelta != 0) @@ -947,11 +951,11 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo // We also can save FP and LR, even though they are not in RBM_CALLEE_SAVED. assert(regsToSaveCount <= genCountBits(RBM_CALLEE_SAVED | RBM_FP | RBM_LR)); - #ifdef FEATURE_MASKED_HW_INTRINSICS if (maskSaveRegsPredicate != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsPredicate, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_MASK)); + genSaveCalleeSavedRegisterGroup(maskSaveRegsPredicate, spDelta, + lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_MASK)); spDelta = 0; lowestCalleeSavedOffset += genCountBits(maskSaveRegsPredicate) * FPSAVE_REGSIZE_BYTES; } @@ -960,14 +964,16 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo // Save integer registers at higher addresses than floating-point registers. if (maskSaveRegsFloat != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_FLOAT)); + genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, + lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_FLOAT)); spDelta = 0; lowestCalleeSavedOffset += genCountBits(maskSaveRegsFloat) * FPSAVE_REGSIZE_BYTES; } if (maskSaveRegsInt != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_INT)); + genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, + lowestCalleeSavedOffset MORE_THAN_64_REG_ARG(TYP_INT)); // No need to update spDelta, lowestCalleeSavedOffset since they're not used after this. } } @@ -1110,7 +1116,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, if (maskRestoreRegsPredicate != RBM_NONE) { // TODO: Do we need to adjust spDelta? - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsPredicate, spDelta, spOffset MORE_THAN_64_REG_ARG(TYP_MASK)); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsPredicate, spDelta, spOffset MORE_THAN_64_REG_ARG(TYP_MASK)); } #endif } @@ -1545,9 +1551,11 @@ void CodeGen::genFuncletProlog(BasicBlock* block) genSaveCalleeSavedRegistersHelp(AllRegsMask(maskSaveRegsInt, maskSaveRegsFloat #ifdef FEATURE_MASKED_HW_INTRINSICS - , maskSaveRegsPredicate + , + maskSaveRegsPredicate #endif - ), lowestCalleeSavedOffset, 0); + ), + lowestCalleeSavedOffset, 0); if ((genFuncletInfo.fiFrameType == 3) || (genFuncletInfo.fiFrameType == 5)) { @@ -1668,10 +1676,11 @@ void CodeGen::genFuncletEpilog() int lowestCalleeSavedOffset = genFuncletInfo.fiSP_to_CalleeSave_delta + genFuncletInfo.fiSpDelta2; genRestoreCalleeSavedRegistersHelp(AllRegsMask(maskRestoreRegsInt, maskRestoreRegsFloat #ifdef FEATURE_MASKED_HW_INSTRINSICS - , maskRestoreRegsPredicate + , + maskRestoreRegsPredicate #endif - ), lowestCalleeSavedOffset, - 0); + ), + lowestCalleeSavedOffset, 0); if (genFuncletInfo.fiFrameType == 1) { @@ -1932,10 +1941,10 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() /* Now save it for future use */ - genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; - genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; + genFuncletInfo.fiSaveGprRegs = rsMaskSaveGprRegs; + genFuncletInfo.fiSaveFloatRegs = rsMaskSaveFloatRegs; #ifdef FEATURE_MASKED_HW_INTRINSICS - genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; + genFuncletInfo.fiSavePredicateRegs = rsMaskSavePredicateRegs; #endif genFuncletInfo.fiSP_to_FPLR_save_delta = SP_to_FPLR_save_delta; genFuncletInfo.fiSP_to_PSP_slot_delta = SP_to_PSP_slot_delta; @@ -1950,9 +1959,10 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() printf(" Save regs: "); dspRegMask(AllRegsMask(genFuncletInfo.fiSaveGprRegs, genFuncletInfo.fiSaveFloatRegs #ifdef FEATURE_MASKED_HW_INTRINSICS - , genFuncletInfo.fiSavePredicateRegs + , + genFuncletInfo.fiSavePredicateRegs #endif - )); + )); printf("\n"); if (compiler->opts.IsOSR()) { @@ -1968,8 +1978,8 @@ void CodeGen::genCaptureFuncletPrologEpilogInfo() if (compiler->lvaPSPSym != BAD_VAR_NUM) { - if (CallerSP_to_PSP_slot_delta != - compiler->lvaGetCallerSPRelativeOffset(compiler->lvaPSPSym)) // for debugging + if (CallerSP_to_PSP_slot_delta != compiler->lvaGetCallerSPRelativeOffset(compiler->lvaPSPSym)) // for + // debugging { printf("lvaGetCallerSPRelativeOffset(lvaPSPSym): %d\n", compiler->lvaGetCallerSPRelativeOffset(compiler->lvaPSPSym)); @@ -2300,9 +2310,9 @@ void CodeGen::genEHCatchRet(BasicBlock* block) // move an immediate value into an integer register -void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, - regNumber reg, - ssize_t imm, +void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, + regNumber reg, + ssize_t imm, insFlags flags DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { // reg cannot be a FP register @@ -5218,7 +5228,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, gcInfo.gcRegByrefSetCur, DebugInfo(), callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); @@ -5832,8 +5842,8 @@ void CodeGen::genCodeForBfiz(GenTreeOp* tree) GenTree* castOp = cast->CastOp(); genConsumeRegs(castOp); - unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE - : genTypeSize(castOp) * BITS_PER_BYTE; + unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE + : genTypeSize(castOp) * BITS_PER_BYTE; const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), castOp->GetRegNum(), (int)shiftByImm, (int)srcBits); diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 750daa569613ff..52633ed6733e6a 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4932,16 +4932,16 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_B); /* SEL .B, , .B, .B */ // IF_SVE_CZ_4A_A - theEmitter->emitIns_R_R(INS_sve_movs, EA_SCALABLE, REG_P0, REG_P15, - INS_OPTS_SCALABLE_B); /* MOVS .B, .B */ + theEmitter->emitIns_R_R(INS_sve_movs, EA_SCALABLE, REG_P0, REG_P15, INS_OPTS_SCALABLE_B); /* MOVS .B, .B + */ // IF_SVE_CZ_4A_K theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_P0, REG_P8, REG_P15, INS_OPTS_SCALABLE_B, INS_SCALABLE_OPTS_PREDICATE_MERGE); /* MOV .B, /M, .B */ // IF_SVE_CZ_4A_L - theEmitter->emitIns_R_R(INS_sve_mov, EA_SCALABLE, REG_P0, REG_P15, - INS_OPTS_SCALABLE_B); /* MOV .B, .B */ + theEmitter->emitIns_R_R(INS_sve_mov, EA_SCALABLE, REG_P0, REG_P15, INS_OPTS_SCALABLE_B); /* MOV .B, .B + */ // IF_SVE_DA_4A theEmitter->emitIns_R_R_R_R(INS_sve_brkpa, EA_SCALABLE, REG_P0, REG_P1, REG_P10, REG_P15, diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 2841d2bf54248a..28cddc7889cf49 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -727,8 +727,8 @@ void CodeGen::genIntrinsic(GenTreeIntrinsic* treeNode) break; #if defined(FEATURE_SIMD) - // The handling is a bit more complex so genSimdUpperSave/Restore - // handles genConsumeOperands and genProduceReg + // The handling is a bit more complex so genSimdUpperSave/Restore + // handles genConsumeOperands and genProduceReg case NI_SIMD_UpperRestore: { @@ -861,7 +861,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) emit->emitIns_S_R(storeIns, storeAttr, REG_ZR, varNumOut, argOffsetOut); #else // !TARGET_ARM64 - // There is no zero register on ARM32 + // There is no zero register on ARM32 unreached(); #endif // !TARGET_ARM64 } @@ -1018,9 +1018,9 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) nextIndex += 2; } #else // TARGET_ARM - // For a >= 4 byte sizes we will generate a ldr and str instruction each loop - // ldr r2, [r0] - // str r2, [sp, #16] + // For a >= 4 byte sizes we will generate a ldr and str instruction each loop + // ldr r2, [r0] + // str r2, [sp, #16] while (remainingSize >= TARGET_POINTER_SIZE) { var_types type = layout->GetGCPtrType(nextIndex); @@ -1812,7 +1812,7 @@ instruction CodeGen::genGetVolatileLdStIns(instruction currentIns, assert(!addrIsInReg); switch (currentIns) { - // Loads + // Loads case INS_ldrb: return INS_ldapurb; @@ -1823,7 +1823,7 @@ instruction CodeGen::genGetVolatileLdStIns(instruction currentIns, case INS_ldr: return INS_ldapur; - // Stores + // Stores case INS_strb: return INS_stlurb; @@ -1855,7 +1855,7 @@ instruction CodeGen::genGetVolatileLdStIns(instruction currentIns, const bool hasRcpc1 = compiler->compOpportunisticallyDependsOn(InstructionSet_Rcpc); switch (currentIns) { - // Loads + // Loads case INS_ldrb: return hasRcpc1 ? INS_ldaprb : INS_ldarb; @@ -1866,7 +1866,7 @@ instruction CodeGen::genGetVolatileLdStIns(instruction currentIns, case INS_ldr: return hasRcpc1 ? INS_ldapr : INS_ldar; - // Stores + // Stores case INS_strb: return INS_stlrb; @@ -2060,7 +2060,10 @@ class ProducingStreamBaseInstrs { public: ProducingStreamBaseInstrs(regNumber intReg1, regNumber intReg2, regNumber addrReg, emitter* emitter) - : intReg1(intReg1), intReg2(intReg2), addrReg(addrReg), emitter(emitter) + : intReg1(intReg1) + , intReg2(intReg2) + , addrReg(addrReg) + , emitter(emitter) { } @@ -2121,7 +2124,11 @@ class ProducingStream { public: ProducingStream(regNumber intReg1, regNumber simdReg1, regNumber simdReg2, regNumber addrReg, emitter* emitter) - : intReg1(intReg1), simdReg1(simdReg1), simdReg2(simdReg2), addrReg(addrReg), emitter(emitter) + : intReg1(intReg1) + , simdReg1(simdReg1) + , simdReg2(simdReg2) + , addrReg(addrReg) + , emitter(emitter) { } @@ -2244,7 +2251,9 @@ class BlockUnrollHelper class InitBlockUnrollHelper { public: - InitBlockUnrollHelper(int dstOffset, unsigned byteCount) : dstStartOffset(dstOffset), byteCount(byteCount) + InitBlockUnrollHelper(int dstOffset, unsigned byteCount) + : dstStartOffset(dstOffset) + , byteCount(byteCount) { } @@ -2373,7 +2382,9 @@ class CopyBlockUnrollHelper { public: CopyBlockUnrollHelper(int srcOffset, int dstOffset, unsigned byteCount) - : srcStartOffset(srcOffset), dstStartOffset(dstOffset), byteCount(byteCount) + : srcStartOffset(srcOffset) + , dstStartOffset(dstOffset) + , byteCount(byteCount) { } @@ -3433,13 +3444,13 @@ void CodeGen::genCall(GenTreeCall* call) else #endif // TARGET_ARM if (varTypeUsesFloatArgReg(returnType)) - { - returnReg = REG_FLOATRET; - } - else - { - returnReg = REG_INTRET; - } + { + returnReg = REG_FLOATRET; + } + else + { + returnReg = REG_INTRET; + } if (call->GetRegNum() != returnReg) { @@ -3716,19 +3727,19 @@ void CodeGen::genCallInstruction(GenTreeCall* call) else #endif // FEATURE_READYTORUN if (call->gtCallType == CT_HELPER) - { - CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); - noway_assert(helperNum != CORINFO_HELP_UNDEF); + { + CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); + noway_assert(helperNum != CORINFO_HELP_UNDEF); - void* pAddr = nullptr; - addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); - assert(pAddr == nullptr); - } - else - { - // Direct call to a non-virtual user function. - addr = call->gtDirectCallAddress; - } + void* pAddr = nullptr; + addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); + assert(pAddr == nullptr); + } + else + { + // Direct call to a non-virtual user function. + addr = call->gtDirectCallAddress; + } assert(addr != nullptr); @@ -4394,8 +4405,8 @@ void CodeGen::genFloatToFloatCast(GenTree* treeNode) //------------------------------------------------------------------------ // genCreateAndStoreGCInfo: Create and record GC Info for the function. // -void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, - unsigned prologSize, +void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, + unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)) { IAllocator* allowZeroAlloc = new (compiler, CMK_GC) CompIAllocator(compiler->getAllocatorGC()); @@ -5566,9 +5577,8 @@ void CodeGen::genFnEpilog(BasicBlock* block) compiler->unwindSetFrameReg(REG_SAVED_LOCALLOC_SP, 0); } - if (jmpEpilog || - genStackAllocRegisterMask(compiler->compLclFrameSize, - regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) + if (jmpEpilog || genStackAllocRegisterMask(compiler->compLclFrameSize, + regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) { genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); } @@ -5640,9 +5650,9 @@ void CodeGen::genFnEpilog(BasicBlock* block) #if !FEATURE_FASTTAILCALL noway_assert(jmpNode->gtOper == GT_JMP); #else // FEATURE_FASTTAILCALL - // armarch - // If jmpNode is GT_JMP then gtNext must be null. - // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. + // armarch + // If jmpNode is GT_JMP then gtNext must be null. + // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. noway_assert((jmpNode->gtOper != GT_JMP) || (jmpNode->gtNext == nullptr)); // Could either be a "jmp method" or "fast tail call" implemented as epilog+jmp diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7d956ba0b8e665..fa8b0415a50748 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -65,7 +65,10 @@ CodeGenInterface* getCodeGenerator(Compiler* comp) // CodeGen constructor CodeGenInterface::CodeGenInterface(Compiler* theCompiler) - : gcInfo(theCompiler), regSet(theCompiler, gcInfo), compiler(theCompiler), treeLifeUpdater(nullptr) + : gcInfo(theCompiler) + , regSet(theCompiler, gcInfo) + , compiler(theCompiler) + , treeLifeUpdater(nullptr) { } @@ -84,7 +87,8 @@ void CodeGenInterface::CopyRegisterInfo() /*****************************************************************************/ -CodeGen::CodeGen(Compiler* theCompiler) : CodeGenInterface(theCompiler) +CodeGen::CodeGen(Compiler* theCompiler) + : CodeGenInterface(theCompiler) { #if defined(TARGET_XARCH) negBitmaskFlt = nullptr; @@ -1901,7 +1905,7 @@ void CodeGen::genGenerateMachineCode() (compiler->compCodeOpt() != Compiler::SMALL_CODE) && !compiler->opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT) #endif - ); + ); /* Now generate code for the function */ genCodeForBBlist(); @@ -3229,9 +3233,9 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere #ifdef TARGET_X86 noway_assert(varDsc->lvType == TYP_STRUCT); #else // !TARGET_X86 - // For LSRA, it may not be in regArgMaskLive if it has a zero - // refcnt. This is in contrast with the non-LSRA case in which all - // non-tracked args are assumed live on entry. + // For LSRA, it may not be in regArgMaskLive if it has a zero + // refcnt. This is in contrast with the non-LSRA case in which all + // non-tracked args are assumed live on entry. noway_assert((varDsc->lvRefCnt() == 0) || (varDsc->lvType == TYP_STRUCT) || (varDsc->IsAddressExposed() && compiler->info.compIsVarArgs) || (varDsc->IsAddressExposed() && compiler->opts.compUseSoftFP)); @@ -4161,8 +4165,8 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere int nextArgNum = argNum + i; LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); regNumber nextRegNum = genMapRegArgNumToRegNum(nextArgNum, regArgTab[nextArgNum].type, - compiler->info.compCallConv); - destRegNum = fieldVarDsc->GetRegNum(); + compiler->info.compCallConv); + destRegNum = fieldVarDsc->GetRegNum(); noway_assert(regArgTab[nextArgNum].varNum == varNum); noway_assert(genIsValidFloatReg(nextRegNum)); noway_assert(genIsValidFloatReg(destRegNum)); @@ -4245,7 +4249,7 @@ void CodeGen::genEnregisterIncomingStackArgs() regNumber tmp_reg = REG_NA; #endif - for (LclVarDsc *varDsc = compiler->lvaTable; varNum < compiler->lvaCount; varNum++, varDsc++) + for (LclVarDsc* varDsc = compiler->lvaTable; varNum < compiler->lvaCount; varNum++, varDsc++) { /* Is this variable a parameter? */ @@ -4319,7 +4323,7 @@ void CodeGen::genEnregisterIncomingStackArgs() } } } -#else // !TARGET_LOONGARCH64 +#else // !TARGET_LOONGARCH64 GetEmitter()->emitIns_R_S(ins_Load(regType), emitTypeSize(regType), regNum, varNum, 0); #endif // !TARGET_LOONGARCH64 @@ -5382,7 +5386,7 @@ void CodeGen::genFinalizeFrame() noway_assert((regSet.rsGetModifiedPredicateRegsMask() & ~RBM_MSK_CALLEE_TRASH) == 0); #endif // FEATURE_MASKED_HW_INTRINSICS #else // !TARGET_AMD64 && !TARGET_ARM64 - // On x86 we save all callee saved regs so the saved reg area size is consistent + // On x86 we save all callee saved regs so the saved reg area size is consistent regSet.rsSetGprRegsModified(RBM_INT_CALLEE_SAVED & ~RBM_FPBASE); #endif // !TARGET_AMD64 && !TARGET_ARM64 } @@ -6119,7 +6123,7 @@ void CodeGen::genFnProlog() } #endif // TARGET_AMD64 -//------------------------------------------------------------------------- + //------------------------------------------------------------------------- #ifdef TARGET_ARM if (compiler->compLocallocUsed) @@ -6145,11 +6149,11 @@ void CodeGen::genFnProlog() #endif // TARGET_AMD64 compiler->unwindEndProlog(); -//------------------------------------------------------------------------- -// -// This is the end of the OS-reported prolog for purposes of unwinding -// -//------------------------------------------------------------------------- + //------------------------------------------------------------------------- + // + // This is the end of the OS-reported prolog for purposes of unwinding + // + //------------------------------------------------------------------------- #ifdef TARGET_ARM if (needToEstablishFP) @@ -8546,7 +8550,7 @@ void CodeGen::genPoisonFrame(regMaskGpr regLiveIn) bool fpBased; int addr = compiler->lvaFrameAddress((int)varNum, &fpBased); #else - int addr = 0; + int addr = 0; #endif int end = addr + (int)size; for (int offs = addr; offs < end;) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index 6d28e6a251b5c6..a4a7ec47effe4e 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -170,8 +170,8 @@ class CodeGenInterface TreeLifeUpdater* treeLifeUpdater; public: - bool genUseOptimizedWriteBarriers(GCInfo::WriteBarrierForm wbf); - bool genUseOptimizedWriteBarriers(GenTreeStoreInd* store); + bool genUseOptimizedWriteBarriers(GCInfo::WriteBarrierForm wbf); + bool genUseOptimizedWriteBarriers(GenTreeStoreInd* store); CorInfoHelpFunc genWriteBarrierHelperForWriteBarrierForm(GCInfo::WriteBarrierForm wbf); #ifdef DEBUG @@ -447,7 +447,8 @@ class CodeGenInterface { siVarLocType vlType; - union { + union + { // VLT_REG/VLT_REG_FP -- Any pointer-sized enregistered value (TYP_INT, TYP_REF, etc) // eg. EAX // VLT_REG_BYREF -- the specified register contains the address of the variable @@ -632,7 +633,9 @@ class CodeGenInterface VariableLiveRange(CodeGenInterface::siVarLoc varLocation, emitLocation startEmitLocation, emitLocation endEmitLocation) - : m_StartEmitLocation(startEmitLocation), m_EndEmitLocation(endEmitLocation), m_VarLocation(varLocation) + : m_StartEmitLocation(startEmitLocation) + , m_EndEmitLocation(endEmitLocation) + , m_VarLocation(varLocation) { } @@ -680,7 +683,8 @@ class CodeGenInterface public: LiveRangeDumper(const LiveRangeList* liveRanges) - : m_startingLiveRange(liveRanges->end()), m_hasLiveRangesToDump(false){}; + : m_startingLiveRange(liveRanges->end()) + , m_hasLiveRangesToDump(false){}; // Make the dumper point to the last "VariableLiveRange" opened or nullptr if all are closed void resetDumper(const LiveRangeList* list); @@ -761,7 +765,7 @@ class CodeGenInterface LiveRangeList* getLiveRangesForVarForBody(unsigned int varNum) const; LiveRangeList* getLiveRangesForVarForProlog(unsigned int varNum) const; - size_t getLiveRangesCount() const; + size_t getLiveRangesCount() const; // For parameters locations on prolog void psiStartVariableLiveRange(CodeGenInterface::siVarLoc varLocation, unsigned int varNum); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 9854030d0f67fb..75ae6a4bdcc6da 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -156,8 +156,8 @@ void CodeGen::genCodeForBBlist() genMarkLabelsForCodegen(); - assert(!compiler->fgFirstBBScratch || - compiler->fgFirstBB == compiler->fgFirstBBScratch); // compiler->fgFirstBBScratch has to be first. + assert(!compiler->fgFirstBBScratch || compiler->fgFirstBB == compiler->fgFirstBBScratch); // compiler->fgFirstBBScratch + // has to be first. /* Initialize structures used in the block list iteration */ genInitialize(); @@ -626,7 +626,7 @@ void CodeGen::genCodeForBBlist() case BBJ_THROW: case BBJ_CALLFINALLY: case BBJ_EHCATCHRET: - // We're going to generate more code below anyway, so no need for the NOP. + // We're going to generate more code below anyway, so no need for the NOP. case BBJ_RETURN: case BBJ_EHFINALLYRET: @@ -637,7 +637,7 @@ void CodeGen::genCodeForBBlist() case BBJ_COND: case BBJ_SWITCH: - // These can't have a call as the last instruction! + // These can't have a call as the last instruction! default: noway_assert(!"Unexpected bbKind"); @@ -1918,8 +1918,8 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk, unsigned outArg var_types type = use.GetType(); unsigned thisFieldOffset = argOffset + use.GetOffset(); -// Emit store instructions to store the registers produced by the GT_FIELD_LIST into the outgoing -// argument area. + // Emit store instructions to store the registers produced by the GT_FIELD_LIST into the outgoing + // argument area. #if defined(FEATURE_SIMD) if (type == TYP_SIMD12) diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index a99199aedc634c..ec27d2ff8ab4da 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -1225,9 +1225,9 @@ void CodeGen::genFnEpilog(BasicBlock* block) #if !FEATURE_FASTTAILCALL noway_assert(jmpNode->gtOper == GT_JMP); #else // FEATURE_FASTTAILCALL - // armarch - // If jmpNode is GT_JMP then gtNext must be null. - // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. + // armarch + // If jmpNode is GT_JMP then gtNext must be null. + // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. noway_assert((jmpNode->gtOper != GT_JMP) || (jmpNode->gtNext == nullptr)); // Could either be a "jmp method" or "fast tail call" implemented as epilog+jmp @@ -1573,9 +1573,9 @@ void CodeGen::genEHCatchRet(BasicBlock* block) } // move an immediate value into an integer register -void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, - regNumber reg, - ssize_t imm, +void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, + regNumber reg, + ssize_t imm, insFlags flags DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { emitter* emit = GetEmitter(); @@ -3334,7 +3334,7 @@ void CodeGen::genCodeForReturnTrap(GenTreeOp* tree) callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); regSet.verifyRegistersUsed(killMask); @@ -4398,7 +4398,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); @@ -6648,19 +6648,19 @@ void CodeGen::genCallInstruction(GenTreeCall* call) else #endif // FEATURE_READYTORUN if (call->gtCallType == CT_HELPER) - { - CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); - noway_assert(helperNum != CORINFO_HELP_UNDEF); + { + CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); + noway_assert(helperNum != CORINFO_HELP_UNDEF); - void* pAddr = nullptr; - addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); - assert(pAddr == nullptr); - } - else - { - // Direct call to a non-virtual user function. - addr = call->gtDirectCallAddress; - } + void* pAddr = nullptr; + addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); + assert(pAddr == nullptr); + } + else + { + // Direct call to a non-virtual user function. + addr = call->gtDirectCallAddress; + } assert(addr != nullptr); @@ -7093,8 +7093,8 @@ void CodeGen::genFloatToFloatCast(GenTree* treeNode) //------------------------------------------------------------------------ // genCreateAndStoreGCInfo: Create and record GC Info for the function. // -void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, - unsigned prologSize, +void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, + unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)) { IAllocator* allowZeroAlloc = new (compiler, CMK_GC) CompIAllocator(compiler->getAllocatorGC()); @@ -7615,7 +7615,7 @@ inline void CodeGen::genJumpToThrowHlpBlk_la( callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)(compiler->acdHelper(codeKind))); regSet.verifyRegistersUsed(killMask); diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index 87745fabe3e04b..1d48582c6c316e 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -1212,9 +1212,9 @@ void CodeGen::genFnEpilog(BasicBlock* block) #if !FEATURE_FASTTAILCALL noway_assert(jmpNode->gtOper == GT_JMP); #else // FEATURE_FASTTAILCALL - // armarch - // If jmpNode is GT_JMP then gtNext must be null. - // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. + // armarch + // If jmpNode is GT_JMP then gtNext must be null. + // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. noway_assert((jmpNode->gtOper != GT_JMP) || (jmpNode->gtNext == nullptr)); // Could either be a "jmp method" or "fast tail call" implemented as epilog+jmp @@ -1534,9 +1534,9 @@ void CodeGen::genEHCatchRet(BasicBlock* block) } // move an immediate value into an integer register -void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, - regNumber reg, - ssize_t imm, +void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, + regNumber reg, + ssize_t imm, insFlags flags DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { emitter* emit = GetEmitter(); @@ -3338,7 +3338,7 @@ void CodeGen::genCodeForReturnTrap(GenTreeOp* tree) callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); regSet.verifyRegistersUsed(killMask); @@ -4359,7 +4359,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); @@ -6724,19 +6724,19 @@ void CodeGen::genCallInstruction(GenTreeCall* call) else #endif // FEATURE_READYTORUN if (call->gtCallType == CT_HELPER) - { - CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); - noway_assert(helperNum != CORINFO_HELP_UNDEF); + { + CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); + noway_assert(helperNum != CORINFO_HELP_UNDEF); - void* pAddr = nullptr; - addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); - assert(pAddr == nullptr); - } - else - { - // Direct call to a non-virtual user function. - addr = call->gtDirectCallAddress; - } + void* pAddr = nullptr; + addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); + assert(pAddr == nullptr); + } + else + { + // Direct call to a non-virtual user function. + addr = call->gtDirectCallAddress; + } assert(addr != nullptr); @@ -7163,8 +7163,8 @@ void CodeGen::genFloatToFloatCast(GenTree* treeNode) //------------------------------------------------------------------------ // genCreateAndStoreGCInfo: Create and record GC Info for the function. // -void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, - unsigned prologSize, +void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, + unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)) { IAllocator* allowZeroAlloc = new (compiler, CMK_GC) CompIAllocator(compiler->getAllocatorGC()); @@ -7683,7 +7683,7 @@ void CodeGen::genJumpToThrowHlpBlk_la( callTarget, /* ireg */ REG_NA, 0, 0, /* xreg, xmul, disp */ false /* isJump */ - ); + ); regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)(compiler->acdHelper(codeKind))); regSet.verifyRegistersUsed(killMask); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 3b36810d03b4bf..fccbd60e8c8d7c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -156,9 +156,9 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg) regGSCheck = REG_EAX; regMaskGSCheck = RBM_EAX; #else // !TARGET_X86 - // Jmp calls: specify method handle using which JIT queries VM for its entry point - // address and hence it can neither be a VSD call nor PInvoke calli with cookie - // parameter. Therefore, in case of jmp calls it is safe to use R11. + // Jmp calls: specify method handle using which JIT queries VM for its entry point + // address and hence it can neither be a VSD call nor PInvoke calli with cookie + // parameter. Therefore, in case of jmp calls it is safe to use R11. regGSCheck = REG_R11; #endif // !TARGET_X86 } @@ -387,9 +387,9 @@ void CodeGen::genEHFinallyOrFilterRet(BasicBlock* block) // Move an immediate value into an integer register -void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, - regNumber reg, - ssize_t imm, +void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, + regNumber reg, + ssize_t imm, insFlags flags DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { // reg cannot be a FP register @@ -2158,8 +2158,8 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) // The last slot is reserved for ICodeManager::FixContext(ppEndRegion) unsigned filterEndOffsetSlotOffs; - PREFIX_ASSUME(compiler->lvaLclSize(compiler->lvaShadowSPslotsVar) > - TARGET_POINTER_SIZE); // below doesn't underflow. + PREFIX_ASSUME(compiler->lvaLclSize(compiler->lvaShadowSPslotsVar) > TARGET_POINTER_SIZE); // below doesn't + // underflow. filterEndOffsetSlotOffs = (unsigned)(compiler->lvaLclSize(compiler->lvaShadowSPslotsVar) - TARGET_POINTER_SIZE); @@ -6098,13 +6098,13 @@ void CodeGen::genCall(GenTreeCall* call) else #endif // TARGET_X86 if (varTypeIsFloating(returnType)) - { - returnReg = REG_FLOATRET; - } - else - { - returnReg = REG_INTRET; - } + { + returnReg = REG_FLOATRET; + } + else + { + returnReg = REG_INTRET; + } inst_Mov(returnType, call->GetRegNum(), returnReg, /* canSkip */ true); } @@ -8048,8 +8048,8 @@ void CodeGen::genIntrinsic(GenTreeIntrinsic* treeNode) } #if defined(FEATURE_SIMD) - // The handling is a bit more complex so genSimdUpperSave/Restore - // handles genConsumeOperands and genProduceReg + // The handling is a bit more complex so genSimdUpperSave/Restore + // handles genConsumeOperands and genProduceReg case NI_SIMD_UpperRestore: { @@ -8119,7 +8119,7 @@ unsigned CodeGen::getBaseVarForPutArgStk(GenTree* treeNode) #ifdef UNIX_AMD64_ABI assert(!varDsc->lvIsRegArg && varDsc->GetArgReg() == REG_STK); #else // !UNIX_AMD64_ABI - // On Windows this assert is always true. The first argument will always be in REG_ARG_0 or REG_FLTARG_0. + // On Windows this assert is always true. The first argument will always be in REG_ARG_0 or REG_FLTARG_0. assert(varDsc->lvIsRegArg && (varDsc->GetArgReg() == REG_ARG_0 || varDsc->GetArgReg() == REG_FLTARG_0)); #endif // !UNIX_AMD64_ABI #endif // !DEBUG @@ -8594,7 +8594,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* putArgStk) unsigned argOffset = putArgStk->getArgOffset(); #ifdef DEBUG - CallArg* callArg = putArgStk->gtCall->gtArgs.FindByNode(putArgStk); + CallArg* callArg = putArgStk->gtCall->gtArgs.FindByNode(putArgStk); assert(callArg != nullptr); assert(argOffset == callArg->AbiInfo.ByteOffset); #endif @@ -8847,8 +8847,8 @@ CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, unsigned prologSize, unsigne } #ifdef JIT32_GCENCODER -void* CodeGen::genCreateAndStoreGCInfoJIT32(unsigned codeSize, - unsigned prologSize, +void* CodeGen::genCreateAndStoreGCInfoJIT32(unsigned codeSize, + unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr)) { BYTE headerBuf[64]; @@ -9250,8 +9250,8 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) unsigned saveStackLvl2 = genStackLevel; -// Important note: when you change enter probe layout, you must also update SKIP_ENTER_PROF_CALLBACK() -// for x86 stack unwinding + // Important note: when you change enter probe layout, you must also update SKIP_ENTER_PROF_CALLBACK() + // for x86 stack unwinding #if defined(UNIX_X86_ABI) // Manually align the stack to be 16-byte aligned. This is similar to CodeGen::genAlignStackBeforeCall() @@ -10212,7 +10212,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) // do an LEA to "pop off" the frame allocation. needLea = true; #else // !TARGET_AMD64 - // We will just generate "mov esp, ebp" and be done with it. + // We will just generate "mov esp, ebp" and be done with it. needMovEspEbp = true; #endif // !TARGET_AMD64 } @@ -10958,8 +10958,8 @@ void CodeGen::genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNu assert(i == alignmentLoBlkSize); } #else // !defined(TARGET_AMD64) - // While we aren't aligning the start, we still want to - // zero anything that is not in a 16 byte chunk at end + // While we aren't aligning the start, we still want to + // zero anything that is not in a 16 byte chunk at end int alignmentBlkSize = blkSize & -XMM_REGSIZE_BYTES; int alignmentHiBlkSize = blkSize - alignmentBlkSize; int alignedLclHi = untrLclLo + alignmentBlkSize; @@ -11138,8 +11138,8 @@ void CodeGen::genPreserveCalleeSavedFltRegs(unsigned lclFrameSize) assert((offset % 16) == 0); instruction copyIns = ins_Copy(TYP_FLOAT); #else // !TARGET_AMD64 - unsigned offset = lclFrameSize - XMM_REGSIZE_BYTES; - instruction copyIns = INS_movupd; + unsigned offset = lclFrameSize - XMM_REGSIZE_BYTES; + instruction copyIns = INS_movupd; #endif // !TARGET_AMD64 for (regNumber reg = REG_FLT_CALLEE_SAVED_FIRST; regMask != RBM_NONE; reg = REG_NEXT(reg)) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 360bcd9d22b61a..34a04399a3127f 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -645,11 +645,11 @@ var_types Compiler::getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd, // have a struct that is larger than that. // if (structSize <= MAX_PASS_SINGLEREG_BYTES) - { - // We set the "primitive" useType based upon the structSize - // and also examine the clsHnd to see if it is an HFA of count one - useType = getPrimitiveTypeForStruct(structSize, clsHnd, isVarArg); - } + { + // We set the "primitive" useType based upon the structSize + // and also examine the clsHnd to see if it is an HFA of count one + useType = getPrimitiveTypeForStruct(structSize, clsHnd, isVarArg); + } #else if (isTrivialPointerSizedStruct(clsHnd)) { @@ -1157,11 +1157,15 @@ struct FileLine unsigned m_line; char* m_condStr; - FileLine() : m_file(nullptr), m_line(0), m_condStr(nullptr) + FileLine() + : m_file(nullptr) + , m_line(0) + , m_condStr(nullptr) { } - FileLine(const char* file, unsigned line, const char* condStr) : m_line(line) + FileLine(const char* file, unsigned line, const char* condStr) + : m_line(line) { size_t newSize = (strlen(file) + 1) * sizeof(char); m_file = HostAllocator::getHostAllocator().allocate(newSize); @@ -1200,7 +1204,7 @@ struct FileLine }; typedef JitHashTable FileLineToCountMap; -FileLineToCountMap* NowayAssertMap; +FileLineToCountMap* NowayAssertMap; void Compiler::RecordNowayAssert(const char* filename, unsigned line, const char* condStr) { @@ -1233,7 +1237,8 @@ struct NowayAssertCountMap size_t count; FileLine fl; - NowayAssertCountMap() : count(0) + NowayAssertCountMap() + : count(0) { } @@ -2026,8 +2031,8 @@ void Compiler::compDone() #endif // LATE_DISASM } -void* Compiler::compGetHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ - void** ppIndirection) /* OUT */ +void* Compiler::compGetHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ + void** ppIndirection) /* OUT */ { void* addr; @@ -3406,11 +3411,10 @@ void Compiler::compInitOptions(JitFlags* jitFlags) printf("OPTIONS: OSR variant with entry point 0x%x\n", info.compILEntry); } - printf("OPTIONS: compCodeOpt = %s\n", - (opts.compCodeOpt == BLENDED_CODE) - ? "BLENDED_CODE" - : (opts.compCodeOpt == SMALL_CODE) ? "SMALL_CODE" - : (opts.compCodeOpt == FAST_CODE) ? "FAST_CODE" : "UNKNOWN_CODE"); + printf("OPTIONS: compCodeOpt = %s\n", (opts.compCodeOpt == BLENDED_CODE) ? "BLENDED_CODE" + : (opts.compCodeOpt == SMALL_CODE) ? "SMALL_CODE" + : (opts.compCodeOpt == FAST_CODE) ? "FAST_CODE" + : "UNKNOWN_CODE"); printf("OPTIONS: compDbgCode = %s\n", dspBool(opts.compDbgCode)); printf("OPTIONS: compDbgInfo = %s\n", dspBool(opts.compDbgInfo)); @@ -4029,8 +4033,9 @@ void Compiler::compSetOptimizationLevel() } if (theMinOptsValue == true) { - JITLOG((LL_INFO10000, "IL Code Size,Instr %4d,%4d, Basic Block count %3d, Local Variable Num,Ref count " - "%3d,%3d for method %s\n", + JITLOG((LL_INFO10000, + "IL Code Size,Instr %4d,%4d, Basic Block count %3d, Local Variable Num,Ref count " + "%3d,%3d for method %s\n", info.compILCodeSize, opts.instrCount, fgBBcount, lvaCount, opts.lvRefCount, info.compFullName)); if (JitConfig.JitBreakOnMinOpts() != 0) { @@ -4811,7 +4816,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl { // Tail merge // - DoPhase(this, PHASE_HEAD_TAIL_MERGE, [this]() { return fgHeadTailMerge(true); }); + DoPhase(this, PHASE_HEAD_TAIL_MERGE, [this]() { + return fgHeadTailMerge(true); + }); // Merge common throw blocks // @@ -4882,7 +4889,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_MORPH_GLOBAL, &Compiler::fgMorphBlocks); auto postMorphPhase = [this]() { - // Fix any LclVar annotations on discarded struct promotion temps for implicit by-ref args fgMarkDemotedImplicitByRefArgs(); lvaRefCountState = RCS_INVALID; @@ -4937,7 +4943,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Second pass of tail merge // - DoPhase(this, PHASE_HEAD_TAIL_MERGE2, [this]() { return fgHeadTailMerge(false); }); + DoPhase(this, PHASE_HEAD_TAIL_MERGE2, [this]() { + return fgHeadTailMerge(false); + }); // Canonicalize entry to give a unique dominator tree root // @@ -5296,7 +5304,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Now that lowering is completed we can proceed to perform register allocation // - auto linearScanPhase = [this]() { m_pLinearScan->doLinearScan(); }; + auto linearScanPhase = [this]() { + m_pLinearScan->doLinearScan(); + }; DoPhase(this, PHASE_LINEAR_SCAN, linearScanPhase); // Copied from rpPredictRegUse() @@ -6193,12 +6203,12 @@ int Compiler::compCompile(CORINFO_MODULE_HANDLE classPtr, // We need to assume, by default, that all flags coming from the VM are invalid. instructionSetFlags.Reset(); -// We then add each available instruction set for the target architecture provided -// that the corresponding JitConfig switch hasn't explicitly asked for it to be -// disabled. This allows us to default to "everything" supported for altjit scenarios -// while also still allowing instruction set opt-out providing users with the ability -// to, for example, see and debug ARM64 codegen for any desired CPU configuration without -// needing to have the hardware in question. + // We then add each available instruction set for the target architecture provided + // that the corresponding JitConfig switch hasn't explicitly asked for it to be + // disabled. This allows us to default to "everything" supported for altjit scenarios + // while also still allowing instruction set opt-out providing users with the ability + // to, for example, see and debug ARM64 codegen for any desired CPU configuration without + // needing to have the hardware in question. #if defined(TARGET_ARM64) if (JitConfig.EnableHWIntrinsic() != 0) @@ -7971,112 +7981,105 @@ int jitNativeCode(CORINFO_METHOD_HANDLE methodHnd, #endif param.result = result; - setErrorTrap(compHnd, Param*, pParamOuter, ¶m) - { - setErrorTrap(nullptr, Param*, pParam, pParamOuter) - { - if (pParam->inlineInfo) - { - // Lazily create the inlinee compiler object - if (pParam->inlineInfo->InlinerCompiler->InlineeCompiler == nullptr) - { - pParam->inlineInfo->InlinerCompiler->InlineeCompiler = - (Compiler*)pParam->pAlloc->allocateMemory(roundUp(sizeof(*pParam->pComp))); - } + setErrorTrap(compHnd, Param*, pParamOuter, ¶m){setErrorTrap(nullptr, Param*, pParam, pParamOuter){ + if (pParam->inlineInfo){// Lazily create the inlinee compiler object + if (pParam->inlineInfo->InlinerCompiler->InlineeCompiler == nullptr){ + pParam->inlineInfo->InlinerCompiler->InlineeCompiler = + (Compiler*)pParam->pAlloc->allocateMemory(roundUp(sizeof(*pParam->pComp))); +} - // Use the inlinee compiler object - pParam->pComp = pParam->inlineInfo->InlinerCompiler->InlineeCompiler; +// Use the inlinee compiler object +pParam->pComp = pParam->inlineInfo->InlinerCompiler->InlineeCompiler; #ifdef DEBUG // memset(pParam->pComp, 0xEE, sizeof(Compiler)); #endif - } - else - { - // Allocate create the inliner compiler object - pParam->pComp = (Compiler*)pParam->pAlloc->allocateMemory(roundUp(sizeof(*pParam->pComp))); - } +} +else +{ + // Allocate create the inliner compiler object + pParam->pComp = (Compiler*)pParam->pAlloc->allocateMemory(roundUp(sizeof(*pParam->pComp))); +} #if MEASURE_CLRAPI_CALLS - pParam->wrapCLR = WrapICorJitInfo::makeOne(pParam->pAlloc, pParam->pComp, pParam->compHnd); +pParam->wrapCLR = WrapICorJitInfo::makeOne(pParam->pAlloc, pParam->pComp, pParam->compHnd); #endif - // push this compiler on the stack (TLS) - pParam->pComp->prevCompiler = JitTls::GetCompiler(); - JitTls::SetCompiler(pParam->pComp); +// push this compiler on the stack (TLS) +pParam->pComp->prevCompiler = JitTls::GetCompiler(); +JitTls::SetCompiler(pParam->pComp); // PREFIX_ASSUME gets turned into ASSERT_CHECK and we cannot have it here #if defined(_PREFAST_) || defined(_PREFIX_) - PREFIX_ASSUME(pParam->pComp != NULL); +PREFIX_ASSUME(pParam->pComp != NULL); #else - assert(pParam->pComp != nullptr); +assert(pParam->pComp != nullptr); #endif - pParam->pComp->compInit(pParam->pAlloc, pParam->methodHnd, pParam->compHnd, pParam->methodInfo, - pParam->inlineInfo); +pParam->pComp->compInit(pParam->pAlloc, pParam->methodHnd, pParam->compHnd, pParam->methodInfo, pParam->inlineInfo); #ifdef DEBUG - pParam->pComp->jitFallbackCompile = pParam->jitFallbackCompile; +pParam->pComp->jitFallbackCompile = pParam->jitFallbackCompile; #endif - // Now generate the code - pParam->result = pParam->pComp->compCompile(pParam->classPtr, pParam->methodCodePtr, pParam->methodCodeSize, - pParam->compileFlags); - } - finallyErrorTrap() - { - Compiler* pCompiler = pParamOuter->pComp; +// Now generate the code +pParam->result = + pParam->pComp->compCompile(pParam->classPtr, pParam->methodCodePtr, pParam->methodCodeSize, pParam->compileFlags); +} +finallyErrorTrap() +{ + Compiler* pCompiler = pParamOuter->pComp; - // If OOM is thrown when allocating memory for a pComp, we will end up here. - // For this case, pComp and also pCompiler will be a nullptr - // - if (pCompiler != nullptr) - { - pCompiler->info.compCode = nullptr; + // If OOM is thrown when allocating memory for a pComp, we will end up here. + // For this case, pComp and also pCompiler will be a nullptr + // + if (pCompiler != nullptr) + { + pCompiler->info.compCode = nullptr; - // pop the compiler off the TLS stack only if it was linked above - assert(JitTls::GetCompiler() == pCompiler); - JitTls::SetCompiler(pCompiler->prevCompiler); - } + // pop the compiler off the TLS stack only if it was linked above + assert(JitTls::GetCompiler() == pCompiler); + JitTls::SetCompiler(pCompiler->prevCompiler); + } - if (pParamOuter->inlineInfo == nullptr) - { - // Free up the allocator we were using - pParamOuter->pAlloc->destroy(); - } - } - endErrorTrap() + if (pParamOuter->inlineInfo == nullptr) + { + // Free up the allocator we were using + pParamOuter->pAlloc->destroy(); } - impJitErrorTrap() +} +endErrorTrap() +} +impJitErrorTrap() +{ + // If we were looking at an inlinee.... + if (inlineInfo != nullptr) { - // If we were looking at an inlinee.... - if (inlineInfo != nullptr) - { - // Note that we failed to compile the inlinee, and that - // there's no point trying to inline it again anywhere else. - inlineInfo->inlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); - } - param.result = __errc; + // Note that we failed to compile the inlinee, and that + // there's no point trying to inline it again anywhere else. + inlineInfo->inlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); } - endErrorTrap() + param.result = __errc; +} +endErrorTrap() - result = param.result; + result = param.result; - if (!inlineInfo && - (result == CORJIT_INTERNALERROR || result == CORJIT_RECOVERABLEERROR || result == CORJIT_IMPLLIMITATION) && - !jitFallbackCompile) - { - // If we failed the JIT, reattempt with debuggable code. - jitFallbackCompile = true; +if (!inlineInfo && + (result == CORJIT_INTERNALERROR || result == CORJIT_RECOVERABLEERROR || result == CORJIT_IMPLLIMITATION) && + !jitFallbackCompile) +{ + // If we failed the JIT, reattempt with debuggable code. + jitFallbackCompile = true; - // Update the flags for 'safer' code generation. - compileFlags->Set(JitFlags::JIT_FLAG_MIN_OPT); - compileFlags->Clear(JitFlags::JIT_FLAG_SIZE_OPT); - compileFlags->Clear(JitFlags::JIT_FLAG_SPEED_OPT); + // Update the flags for 'safer' code generation. + compileFlags->Set(JitFlags::JIT_FLAG_MIN_OPT); + compileFlags->Clear(JitFlags::JIT_FLAG_SIZE_OPT); + compileFlags->Clear(JitFlags::JIT_FLAG_SPEED_OPT); - goto START; - } + goto START; +} - return result; +return result; } #if defined(UNIX_AMD64_ABI) @@ -8827,8 +8830,9 @@ void CompTimeSummaryInfo::Print(FILE* f) double pslop_pct = 100.0 * m_total.m_parentPhaseEndSlop * 1000.0 / countsPerSec / totTime_ms; if (pslop_pct >= 1.0) { - fprintf(f, "\n 'End phase slop' should be very small (if not, there's unattributed time): %9.3f Mcycles = " - "%3.1f%% of total.\n\n", + fprintf(f, + "\n 'End phase slop' should be very small (if not, there's unattributed time): %9.3f Mcycles = " + "%3.1f%% of total.\n\n", m_total.m_parentPhaseEndSlop / 1000000.0, pslop_pct); } } @@ -8868,8 +8872,9 @@ void CompTimeSummaryInfo::Print(FILE* f) double fslop_ms = m_filtered.m_parentPhaseEndSlop * 1000.0 / countsPerSec; if (fslop_ms > 1.0) { - fprintf(f, "\n 'End phase slop' should be very small (if not, there's unattributed time): %9.3f Mcycles = " - "%3.1f%% of total.\n\n", + fprintf(f, + "\n 'End phase slop' should be very small (if not, there's unattributed time): %9.3f Mcycles = " + "%3.1f%% of total.\n\n", m_filtered.m_parentPhaseEndSlop / 1000000.0, fslop_ms); } } @@ -8967,7 +8972,8 @@ void CompTimeSummaryInfo::Print(FILE* f) fprintf(f, "\n"); } -JitTimer::JitTimer(unsigned byteCodeSize) : m_info(byteCodeSize) +JitTimer::JitTimer(unsigned byteCodeSize) + : m_info(byteCodeSize) { #if MEASURE_CLRAPI_CALLS m_CLRcallInvokes = 0; @@ -9219,7 +9225,7 @@ void JitTimer::PrintCsvMethodStats(Compiler* comp) // for a DEBUG build (presumably not for the time info), just re-use it. const char* methName = comp->info.compFullName; #else - const char* methName = comp->eeGetMethodFullName(comp->info.compMethodHnd); + const char* methName = comp->eeGetMethodFullName(comp->info.compMethodHnd); #endif // Try and access the SPMI index to report in the data set. diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 2a253a26357dc4..e60f4ae556b2f3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -235,11 +235,13 @@ class LclSsaVarDsc { } - LclSsaVarDsc(BasicBlock* block) : m_block(block) + LclSsaVarDsc(BasicBlock* block) + : m_block(block) { } - LclSsaVarDsc(BasicBlock* block, GenTreeLclVarCommon* defNode) : m_block(block) + LclSsaVarDsc(BasicBlock* block, GenTreeLclVarCommon* defNode) + : m_block(block) { SetDefNode(defNode); } @@ -364,7 +366,10 @@ class SsaDefArray public: // Construct an empty SsaDefArray. - SsaDefArray() : m_array(nullptr), m_arraySize(0), m_count(0) + SsaDefArray() + : m_array(nullptr) + , m_arraySize(0) + , m_count(0) { } @@ -504,11 +509,11 @@ class LclVarDsc // note this only packs because var_types is a typedef of unsigned char var_types lvType : 5; // TYP_INT/LONG/FLOAT/DOUBLE/REF - unsigned char lvIsParam : 1; // is this a parameter? - unsigned char lvIsRegArg : 1; // is this an argument that was passed by register? + unsigned char lvIsParam : 1; // is this a parameter? + unsigned char lvIsRegArg : 1; // is this an argument that was passed by register? unsigned char lvFramePointerBased : 1; // 0 = off of REG_SPBASE (e.g., ESP), 1 = off of REG_FPBASE (e.g., EBP) - unsigned char lvOnFrame : 1; // (part of) the variable lives on the frame + unsigned char lvOnFrame : 1; // (part of) the variable lives on the frame unsigned char lvRegister : 1; // assigned to live in a register? For RyuJIT backend, this is only set if the // variable is in the same register for the entire function. unsigned char lvTracked : 1; // is this a tracked variable? @@ -530,16 +535,16 @@ class LclVarDsc // We cannot reason reliably about the value of the variable. public: unsigned char lvDoNotEnregister : 1; // Do not enregister this variable. - unsigned char lvFieldAccessed : 1; // The var is a struct local, and a field of the variable is accessed. Affects + unsigned char lvFieldAccessed : 1; // The var is a struct local, and a field of the variable is accessed. Affects // struct promotion. unsigned char lvLiveInOutOfHndlr : 1; // The variable is live in or out of an exception handler, and therefore must // be on the stack (at least at those boundaries.) - unsigned char lvInSsa : 1; // The variable is in SSA form (set by SsaBuilder) - unsigned char lvIsCSE : 1; // Indicates if this LclVar is a CSE variable. + unsigned char lvInSsa : 1; // The variable is in SSA form (set by SsaBuilder) + unsigned char lvIsCSE : 1; // Indicates if this LclVar is a CSE variable. unsigned char lvHasLdAddrOp : 1; // has ldloca or ldarga opcode on this local. - unsigned char lvHasILStoreOp : 1; // there is at least one STLOC or STARG on this local + unsigned char lvHasILStoreOp : 1; // there is at least one STLOC or STARG on this local unsigned char lvHasMultipleILStoreOp : 1; // there is more than one STLOC on this local unsigned char lvIsTemp : 1; // Short-lifetime compiler temp @@ -554,13 +559,13 @@ class LclVarDsc #if defined(TARGET_LOONGARCH64) unsigned char lvIs4Field1 : 1; // Set if the 1st field is int or float within struct for LA-ABI64. unsigned char lvIs4Field2 : 1; // Set if the 2nd field is int or float within struct for LA-ABI64. - unsigned char lvIsSplit : 1; // Set if the argument is splited. + unsigned char lvIsSplit : 1; // Set if the argument is splited. #endif // defined(TARGET_LOONGARCH64) #if defined(TARGET_RISCV64) unsigned char lvIs4Field1 : 1; // Set if the 1st field is int or float within struct for RISCV64. unsigned char lvIs4Field2 : 1; // Set if the 2nd field is int or float within struct for RISCV64. - unsigned char lvIsSplit : 1; // Set if the argument is splited. + unsigned char lvIsSplit : 1; // Set if the argument is splited. #endif // defined(TARGET_RISCV64) unsigned char lvSingleDef : 1; // variable has a single def. Used to identify ref type locals that can get type @@ -589,7 +594,7 @@ class LclVarDsc unsigned char lvQuirkToLong : 1; // Quirk to allocate this LclVar as a 64-bit long #endif #ifdef DEBUG - unsigned char lvKeepType : 1; // Don't change the type of this variable + unsigned char lvKeepType : 1; // Don't change the type of this variable unsigned char lvNoLclFldStress : 1; // Can't apply local field stress on this one #endif unsigned char lvIsPtr : 1; // Might this be used in an address computation? (used by buffer overflow security @@ -644,8 +649,8 @@ class LclVarDsc #ifdef DEBUG unsigned char lvClassInfoUpdated : 1; // true if this var has updated class handle or exactness - unsigned char lvIsHoist : 1; // CSE temp for a hoisted tree - unsigned char lvIsMultiDefCSE : 1; // CSE temp for a multi-def CSE + unsigned char lvIsHoist : 1; // CSE temp for a hoisted tree + unsigned char lvIsMultiDefCSE : 1; // CSE temp for a multi-def CSE #endif unsigned char lvImplicitlyReferenced : 1; // true if there are non-IR references to this local (prolog, epilog, gc, @@ -670,7 +675,8 @@ class LclVarDsc unsigned char lvIsSpan : 1; // The local is a Span public: - union { + union + { unsigned lvFieldLclStart; // The index of the local var representing the first field in the promoted struct // local. For implicit byref parameters, this gets hijacked between // fgRetypeImplicitByRefArgs and fgMarkDemotedImplicitByRefArgs to point to the @@ -890,7 +896,7 @@ class LclVarDsc assert(_lvRegNum == reg); } -///////////////////// + ///////////////////// #if defined(TARGET_64BIT) @@ -1076,13 +1082,13 @@ class LclVarDsc public: unsigned short lvRefCnt(RefCountState state = RCS_NORMAL) const; - void incLvRefCnt(unsigned short delta, RefCountState state = RCS_NORMAL); - void setLvRefCnt(unsigned short newValue, RefCountState state = RCS_NORMAL); - void incLvRefCntSaturating(unsigned short delta, RefCountState state = RCS_NORMAL); + void incLvRefCnt(unsigned short delta, RefCountState state = RCS_NORMAL); + void setLvRefCnt(unsigned short newValue, RefCountState state = RCS_NORMAL); + void incLvRefCntSaturating(unsigned short delta, RefCountState state = RCS_NORMAL); weight_t lvRefCntWtd(RefCountState state = RCS_NORMAL) const; - void incLvRefCntWtd(weight_t delta, RefCountState state = RCS_NORMAL); - void setLvRefCntWtd(weight_t newValue, RefCountState state = RCS_NORMAL); + void incLvRefCntWtd(weight_t delta, RefCountState state = RCS_NORMAL); + void setLvRefCntWtd(weight_t newValue, RefCountState state = RCS_NORMAL); private: int lvStkOffs; // stack offset of home in bytes. @@ -1340,7 +1346,8 @@ class IntegralRange IntegralRange() = default; IntegralRange(SymbolicIntegerValue lowerBound, SymbolicIntegerValue upperBound) - : m_lowerBound(lowerBound), m_upperBound(upperBound) + : m_lowerBound(lowerBound) + , m_upperBound(upperBound) { assert(lowerBound <= upperBound); } @@ -1372,7 +1379,7 @@ class IntegralRange return (m_lowerBound == other.m_lowerBound) && (m_upperBound == other.m_upperBound); } - static int64_t SymbolicToRealValue(SymbolicIntegerValue value); + static int64_t SymbolicToRealValue(SymbolicIntegerValue value); static SymbolicIntegerValue LowerBoundForType(var_types type); static SymbolicIntegerValue UpperBoundForType(var_types type); @@ -1428,7 +1435,10 @@ class TempDsc var_types tdType; public: - TempDsc(int _tdNum, unsigned _tdSize, var_types _tdType) : tdNum(_tdNum), tdSize((BYTE)_tdSize), tdType(_tdType) + TempDsc(int _tdNum, unsigned _tdSize, var_types _tdType) + : tdNum(_tdNum) + , tdSize((BYTE)_tdSize) + , tdType(_tdType) { #ifdef DEBUG // temps must have a negative number (so they have a different number from all local variables) @@ -1492,9 +1502,9 @@ enum class PhaseStatus : unsigned class LinearScanInterface { public: - virtual PhaseStatus doLinearScan() = 0; - virtual void recordVarLocationsAtStartOfBB(BasicBlock* bb) = 0; - virtual bool willEnregisterLocalVars() const = 0; + virtual PhaseStatus doLinearScan() = 0; + virtual void recordVarLocationsAtStartOfBB(BasicBlock* bb) = 0; + virtual bool willEnregisterLocalVars() const = 0; #if TRACK_LSRA_STATS virtual void dumpLsraStatsCsv(FILE* file) = 0; virtual void dumpLsraStatsSummary(FILE* file) = 0; @@ -7448,23 +7458,23 @@ class Compiler typedef JitHashTable, GenTree*> LocalNumberToNullCheckTreeMap; - GenTree* getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBlock* block)); - GenTree* optPropGetValueRec(unsigned lclNum, unsigned ssaNum, optPropKind valueKind, int walkDepth); - GenTree* optPropGetValue(unsigned lclNum, unsigned ssaNum, optPropKind valueKind); - GenTree* optEarlyPropRewriteTree(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); - bool optDoEarlyPropForBlock(BasicBlock* block); + GenTree* getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBlock* block)); + GenTree* optPropGetValueRec(unsigned lclNum, unsigned ssaNum, optPropKind valueKind, int walkDepth); + GenTree* optPropGetValue(unsigned lclNum, unsigned ssaNum, optPropKind valueKind); + GenTree* optEarlyPropRewriteTree(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); + bool optDoEarlyPropForBlock(BasicBlock* block); bool optDoEarlyPropForFunc(); PhaseStatus optEarlyProp(); - bool optFoldNullCheck(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); - GenTree* optFindNullCheckToFold(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); - bool optIsNullCheckFoldingLegal(GenTree* tree, - GenTree* nullCheckTree, - GenTree** nullCheckParent, - Statement** nullCheckStmt); - bool optCanMoveNullCheckPastTree(GenTree* tree, - unsigned nullCheckLclNum, - bool isInsideTry, - bool checkSideEffectSummary); + bool optFoldNullCheck(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); + GenTree* optFindNullCheckToFold(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap); + bool optIsNullCheckFoldingLegal(GenTree* tree, + GenTree* nullCheckTree, + GenTree** nullCheckParent, + Statement** nullCheckStmt); + bool optCanMoveNullCheckPastTree(GenTree* tree, + unsigned nullCheckLclNum, + bool isInsideTry, + bool checkSideEffectSummary); #if DEBUG void optCheckFlagsAreSet(unsigned methodFlag, const char* methodFlagStr, @@ -7475,30 +7485,30 @@ class Compiler #endif PhaseStatus optInductionVariables(); - bool optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop); - bool optIsIVWideningProfitable(unsigned lclNum, - BasicBlock* initBlock, - bool initedToConstant, - FlowGraphNaturalLoop* loop, - ArrayStack& ivUses); - void optBestEffortReplaceNarrowIVUses( - unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt); + bool optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop); + bool optIsIVWideningProfitable(unsigned lclNum, + BasicBlock* initBlock, + bool initedToConstant, + FlowGraphNaturalLoop* loop, + ArrayStack& ivUses); + void optBestEffortReplaceNarrowIVUses( + unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt); void optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned newLclNum, Statement* stmt); void optSinkWidenedIV(unsigned lclNum, unsigned newLclNum, FlowGraphNaturalLoop* loop); // Redundant branch opts // - PhaseStatus optRedundantBranches(); - bool optRedundantRelop(BasicBlock* const block); - bool optRedundantBranch(BasicBlock* const block); - bool optJumpThreadDom(BasicBlock* const block, BasicBlock* const domBlock, bool domIsSameRelop); - bool optJumpThreadPhi(BasicBlock* const block, GenTree* tree, ValueNum treeNormVN); - bool optJumpThreadCheck(BasicBlock* const block, BasicBlock* const domBlock); - bool optJumpThreadCore(JumpThreadInfo& jti); - bool optReachable(BasicBlock* const fromBlock, BasicBlock* const toBlock, BasicBlock* const excludedBlock); + PhaseStatus optRedundantBranches(); + bool optRedundantRelop(BasicBlock* const block); + bool optRedundantBranch(BasicBlock* const block); + bool optJumpThreadDom(BasicBlock* const block, BasicBlock* const domBlock, bool domIsSameRelop); + bool optJumpThreadPhi(BasicBlock* const block, GenTree* tree, ValueNum treeNormVN); + bool optJumpThreadCheck(BasicBlock* const block, BasicBlock* const domBlock); + bool optJumpThreadCore(JumpThreadInfo& jti); + bool optReachable(BasicBlock* const fromBlock, BasicBlock* const toBlock, BasicBlock* const excludedBlock); BitVecTraits* optReachableBitVecTraits; BitVec optReachableBitVec; - void optRelopImpliesRelop(RelopImplicationInfo* rii); + void optRelopImpliesRelop(RelopImplicationInfo* rii); /************************************************************************** * Value/Assertion propagation @@ -7565,7 +7575,8 @@ class Compiler { optOp1Kind kind; // a normal LclVar, or Exact-type or Subtype ValueNum vn; - union { + union + { SsaVar lcl; ArrBnd bnd; }; @@ -7585,7 +7596,8 @@ class Compiler #endif FieldSeq* fieldSeq; }; - union { + union + { SsaVar lcl; IntVal u1; __int64 lconVal; @@ -7802,49 +7814,49 @@ class Compiler bool optCanPropSubRange; public: - void optVnNonNullPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree); + void optVnNonNullPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree); fgWalkResult optVNBasedFoldCurStmt(BasicBlock* block, Statement* stmt, GenTree* parent, GenTree* tree); - GenTree* optVNConstantPropOnJTrue(BasicBlock* block, GenTree* test); - GenTree* optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree); - GenTree* optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree); - GenTree* optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call); - GenTree* optExtractSideEffListFromConst(GenTree* tree); + GenTree* optVNConstantPropOnJTrue(BasicBlock* block, GenTree* test); + GenTree* optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree); + GenTree* optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree); + GenTree* optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call); + GenTree* optExtractSideEffListFromConst(GenTree* tree); AssertionIndex GetAssertionCount() { return optAssertionCount; } - ASSERT_TP* bbJtrueAssertionOut; + ASSERT_TP* bbJtrueAssertionOut; typedef JitHashTable, ASSERT_TP> ValueNumToAssertsMap; - ValueNumToAssertsMap* optValueNumToAsserts; + ValueNumToAssertsMap* optValueNumToAsserts; // Assertion prop helpers. - ASSERT_TP& GetAssertionDep(unsigned lclNum); + ASSERT_TP& GetAssertionDep(unsigned lclNum); AssertionDsc* optGetAssertion(AssertionIndex assertIndex); - void optAssertionInit(bool isLocalProp); - void optAssertionTraitsInit(AssertionIndex assertionCount); - void optAssertionReset(AssertionIndex limit); - void optAssertionRemove(AssertionIndex index); + void optAssertionInit(bool isLocalProp); + void optAssertionTraitsInit(AssertionIndex assertionCount); + void optAssertionReset(AssertionIndex limit); + void optAssertionRemove(AssertionIndex index); // Assertion prop data flow functions. PhaseStatus optAssertionPropMain(); - Statement* optVNAssertionPropCurStmt(BasicBlock* block, Statement* stmt); - bool optIsTreeKnownIntValue(bool vnBased, GenTree* tree, ssize_t* pConstant, GenTreeFlags* pIconFlags); - ASSERT_TP* optInitAssertionDataflowFlags(); - ASSERT_TP* optComputeAssertionGen(); + Statement* optVNAssertionPropCurStmt(BasicBlock* block, Statement* stmt); + bool optIsTreeKnownIntValue(bool vnBased, GenTree* tree, ssize_t* pConstant, GenTreeFlags* pIconFlags); + ASSERT_TP* optInitAssertionDataflowFlags(); + ASSERT_TP* optComputeAssertionGen(); // Assertion Gen functions. - void optAssertionGen(GenTree* tree); + void optAssertionGen(GenTree* tree); AssertionIndex optAssertionGenCast(GenTreeCast* cast); AssertionIndex optAssertionGenPhiDefn(GenTree* tree); - AssertionInfo optCreateJTrueBoundsAssertion(GenTree* tree); - AssertionInfo optAssertionGenJtrue(GenTree* tree); + AssertionInfo optCreateJTrueBoundsAssertion(GenTree* tree); + AssertionInfo optAssertionGenJtrue(GenTree* tree); AssertionIndex optCreateJtrueAssertions(GenTree* op1, GenTree* op2, Compiler::optAssertionKind assertionKind, bool helperCallArgs = false); AssertionIndex optFindComplementary(AssertionIndex assertionIndex); - void optMapComplementary(AssertionIndex assertionIndex, AssertionIndex index); + void optMapComplementary(AssertionIndex assertionIndex, AssertionIndex index); ValueNum optConservativeNormalVN(GenTree* tree); @@ -7865,9 +7877,9 @@ class Compiler GenTree* op2, bool helperCallArgs = false); - bool optAssertionVnInvolvesNan(AssertionDsc* assertion); + bool optAssertionVnInvolvesNan(AssertionDsc* assertion); AssertionIndex optAddAssertion(AssertionDsc* assertion); - void optAddVnAssertionMapping(ValueNum vn, AssertionIndex index); + void optAddVnAssertionMapping(ValueNum vn, AssertionIndex index); #ifdef DEBUG void optPrintVnAssertionMapping(); #endif @@ -7877,8 +7889,8 @@ class Compiler AssertionIndex optAssertionIsSubrange(GenTree* tree, IntegralRange range, ASSERT_VALARG_TP assertions); AssertionIndex optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions); AssertionIndex optAssertionIsNonNullInternal(GenTree* op, ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased)); - bool optAssertionIsNonNull(GenTree* op, - ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased) DEBUGARG(AssertionIndex* pIndex)); + bool optAssertionIsNonNull(GenTree* op, + ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased) DEBUGARG(AssertionIndex* pIndex)); AssertionIndex optGlobalAssertionIsEqualOrNotEqual(ASSERT_VALARG_TP assertions, GenTree* op1, GenTree* op2); AssertionIndex optGlobalAssertionIsEqualOrNotEqualZero(ASSERT_VALARG_TP assertions, GenTree* op1); @@ -7886,15 +7898,15 @@ class Compiler optOp1Kind op1Kind, unsigned lclNum, optOp2Kind op2Kind, ssize_t cnsVal, ASSERT_VALARG_TP assertions); // Assertion prop for lcl var functions. - bool optAssertionProp_LclVarTypeCheck(GenTree* tree, LclVarDsc* lclVarDsc, LclVarDsc* copyVarDsc); + bool optAssertionProp_LclVarTypeCheck(GenTree* tree, LclVarDsc* lclVarDsc, LclVarDsc* copyVarDsc); GenTree* optCopyAssertionProp(AssertionDsc* curAssertion, GenTreeLclVarCommon* tree, - Statement* stmt DEBUGARG(AssertionIndex index)); + Statement* stmt DEBUGARG(AssertionIndex index)); GenTree* optConstantAssertionProp(AssertionDsc* curAssertion, GenTreeLclVarCommon* tree, - Statement* stmt DEBUGARG(AssertionIndex index)); - bool optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, GenTree* destParent, GenTree* value); - bool optZeroObjAssertionProp(GenTree* tree, ASSERT_VALARG_TP assertions); + Statement* stmt DEBUGARG(AssertionIndex index)); + bool optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, GenTree* destParent, GenTree* value); + bool optZeroObjAssertionProp(GenTree* tree, ASSERT_VALARG_TP assertions); // Assertion propagation functions. GenTree* optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block); @@ -7914,8 +7926,8 @@ class Compiler GenTree* optAssertionPropLocal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt); GenTree* optAssertionProp_Update(GenTree* newTree, GenTree* tree, Statement* stmt); GenTree* optNonNullAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call); - bool optNonNullAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* indir); - bool optWriteBarrierAssertionProp_StoreInd(ASSERT_VALARG_TP assertions, GenTreeStoreInd* indir); + bool optNonNullAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* indir); + bool optWriteBarrierAssertionProp_StoreInd(ASSERT_VALARG_TP assertions, GenTreeStoreInd* indir); void optAssertionProp_RangeProperties(ASSERT_VALARG_TP assertions, GenTree* tree, @@ -7971,11 +7983,11 @@ class Compiler bool optReconstructArrIndex(GenTree* tree, ArrIndex* result); bool optIdentifyLoopOptInfo(FlowGraphNaturalLoop* loop, LoopCloneContext* context); static fgWalkPreFn optCanOptimizeByLoopCloningVisitor; - fgWalkResult optCanOptimizeByLoopCloning(GenTree* tree, LoopCloneVisitorInfo* info); - bool optObtainLoopCloningOpts(LoopCloneContext* context); - bool optIsLoopClonable(FlowGraphNaturalLoop* loop, LoopCloneContext* context); - bool optCheckLoopCloningGDVTestProfitable(GenTreeOp* guard, LoopCloneVisitorInfo* info); - bool optIsHandleOrIndirOfHandle(GenTree* tree, GenTreeFlags handleType); + fgWalkResult optCanOptimizeByLoopCloning(GenTree* tree, LoopCloneVisitorInfo* info); + bool optObtainLoopCloningOpts(LoopCloneContext* context); + bool optIsLoopClonable(FlowGraphNaturalLoop* loop, LoopCloneContext* context); + bool optCheckLoopCloningGDVTestProfitable(GenTreeOp* guard, LoopCloneVisitorInfo* info); + bool optIsHandleOrIndirOfHandle(GenTree* tree, GenTreeFlags handleType); static bool optLoopCloningEnabled(); @@ -8137,7 +8149,7 @@ class Compiler const char* eeGetClassName(CORINFO_CLASS_HANDLE clsHnd, char* buffer = nullptr, size_t bufferSize = 0); - void eePrintObjectDescription(const char* prefix, CORINFO_OBJECT_HANDLE handle); + void eePrintObjectDescription(const char* prefix, CORINFO_OBJECT_HANDLE handle); const char* eeGetShortClassName(CORINFO_CLASS_HANDLE clsHnd); #if defined(DEBUG) @@ -8146,12 +8158,12 @@ class Compiler unsigned compMethodHash(CORINFO_METHOD_HANDLE methodHandle); - var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig); - var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig, bool* isPinned); + var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig); + var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig, bool* isPinned); CORINFO_CLASS_HANDLE eeGetArgClass(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE list); CORINFO_CLASS_HANDLE eeGetClassFromContext(CORINFO_CONTEXT_HANDLE context); - unsigned eeGetArgSize(CorInfoType corInfoType, CORINFO_CLASS_HANDLE typeHnd); - static unsigned eeGetArgSizeAlignment(var_types type, bool isFloatHfa); + unsigned eeGetArgSize(CorInfoType corInfoType, CORINFO_CLASS_HANDLE typeHnd); + static unsigned eeGetArgSizeAlignment(var_types type, bool isFloatHfa); // VOM info, method sigs @@ -8279,7 +8291,7 @@ class Compiler unsigned eeBoundariesCount; ICorDebugInfo::OffsetMapping* eeBoundaries; // Boundaries to report to the EE - void eeSetLIcount(unsigned count); + void eeSetLIcount(unsigned count); void eeSetLIinfo(unsigned which, UNATIVE_OFFSET offs, IPmappingDscKind kind, const ILLocation& loc); void eeSetLIdone(); @@ -8287,7 +8299,7 @@ class Compiler static void eeDispILOffs(IL_OFFSET offs); static void eeDispSourceMappingOffs(uint32_t offs); static void eeDispLineInfo(const ICorDebugInfo::OffsetMapping* line); - void eeDispLineInfos(); + void eeDispLineInfos(); #endif // DEBUG // Debugging support - Local var info @@ -8302,7 +8314,7 @@ class Compiler UNATIVE_OFFSET endOffset; DWORD varNumber; CodeGenInterface::siVarLoc loc; - } * eeVars; + }* eeVars; void eeSetLVcount(unsigned count); void eeSetLVinfo(unsigned which, UNATIVE_OFFSET startOffs, @@ -8336,7 +8348,7 @@ class Compiler WORD eeGetRelocTypeHint(void* target); -// ICorStaticInfo wrapper functions + // ICorStaticInfo wrapper functions #if defined(UNIX_AMD64_ABI) #ifdef DEBUG @@ -8365,7 +8377,11 @@ class Compiler template bool eeRunFunctorWithSPMIErrorTrap(Functor f) { - return eeRunWithSPMIErrorTrap([](Functor* pf) { (*pf)(); }, &f); + return eeRunWithSPMIErrorTrap( + [](Functor* pf) { + (*pf)(); + }, + &f); } bool eeRunWithSPMIErrorTrapImp(void (*function)(void*), void* param); @@ -8373,7 +8389,7 @@ class Compiler // Utility functions static CORINFO_METHOD_HANDLE eeFindHelper(unsigned helper); - static CorInfoHelpFunc eeGetHelperNum(CORINFO_METHOD_HANDLE method); + static CorInfoHelpFunc eeGetHelperNum(CORINFO_METHOD_HANDLE method); enum StaticHelperReturnValue { @@ -8424,7 +8440,7 @@ class Compiler // structure and IL offset is needed only when generating debuggable code. Therefore // it is desirable to avoid memory size penalty in retail scenarios. typedef JitHashTable, DebugInfo> CallSiteDebugInfoTable; - CallSiteDebugInfoTable* genCallSite2DebugInfoMap; + CallSiteDebugInfoTable* genCallSite2DebugInfoMap; unsigned genReturnLocal; // Local number for the return value when applicable. BasicBlock* genReturnBB; // jumped to when not optimizing for speed. @@ -8458,11 +8474,11 @@ class Compiler return codeGen->doDoubleAlign(); } DWORD getCanDoubleAlign(); - bool shouldDoubleAlign(unsigned refCntStk, - unsigned refCntReg, - weight_t refCntWtdReg, - unsigned refCntStkParam, - weight_t refCntWtdStkDbl); + bool shouldDoubleAlign(unsigned refCntStk, + unsigned refCntReg, + weight_t refCntWtdReg, + unsigned refCntStkParam, + weight_t refCntWtdStkDbl); #endif // DOUBLE_ALIGN bool IsFullPtrRegMapRequired() @@ -8474,7 +8490,7 @@ class Compiler codeGen->SetFullPtrRegMapRequired(value); } -// Things that MAY belong either in CodeGen or CodeGenContext + // Things that MAY belong either in CodeGen or CodeGenContext #if defined(FEATURE_EH_FUNCLETS) FuncInfoDsc* compFuncInfos; @@ -8507,7 +8523,7 @@ class Compiler #endif // !FEATURE_EH_FUNCLETS FuncInfoDsc* funCurrentFunc(); - void funSetCurrentFunc(unsigned funcIdx); + void funSetCurrentFunc(unsigned funcIdx); FuncInfoDsc* funGetFunc(unsigned funcIdx); unsigned int funGetFuncIdx(BasicBlock* block); @@ -8530,15 +8546,15 @@ class Compiler // not all JIT Helper calls follow the standard ABI on the target architecture. AllRegsMask compHelperCallKillSet(CorInfoHelpFunc helper); -/* -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XX XX -XX UnwindInfo XX -XX XX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -*/ + /* + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XX XX + XX UnwindInfo XX + XX XX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + */ #if !defined(__GNUC__) #pragma region Unwind information @@ -8674,13 +8690,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #if defined(FEATURE_CFI_SUPPORT) short mapRegNumToDwarfReg(regNumber reg); - void createCfiCode(FuncInfoDsc* func, UNATIVE_OFFSET codeOffset, UCHAR opcode, short dwarfReg, INT offset = 0); - void unwindPushPopCFI(regNumber reg); - void unwindBegPrologCFI(); - void unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat); - void unwindAllocStackCFI(unsigned size); - void unwindSetFrameRegCFI(regNumber reg, unsigned offset); - void unwindEmitFuncCFI(FuncInfoDsc* func, void* pHotCode, void* pColdCode); + void createCfiCode(FuncInfoDsc* func, UNATIVE_OFFSET codeOffset, UCHAR opcode, short dwarfReg, INT offset = 0); + void unwindPushPopCFI(regNumber reg); + void unwindBegPrologCFI(); + void unwindPushPopMaskCFI(regMaskOnlyOne regMask, bool isFloat); + void unwindAllocStackCFI(unsigned size); + void unwindSetFrameRegCFI(regNumber reg, unsigned offset); + void unwindEmitFuncCFI(FuncInfoDsc* func, void* pHotCode, void* pColdCode); #ifdef DEBUG void DumpCfiInfo(bool isHotCode, UNATIVE_OFFSET startOffset, @@ -8929,11 +8945,11 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX GenTree* impSIMDPopStack(); - void setLclRelatedToSIMDIntrinsic(GenTree* tree); - bool areFieldsContiguous(GenTreeIndir* op1, GenTreeIndir* op2); - bool areLocalFieldsContiguous(GenTreeLclFld* first, GenTreeLclFld* second); - bool areArrayElementsContiguous(GenTree* op1, GenTree* op2); - bool areArgumentsContiguous(GenTree* op1, GenTree* op2); + void setLclRelatedToSIMDIntrinsic(GenTree* tree); + bool areFieldsContiguous(GenTreeIndir* op1, GenTreeIndir* op2); + bool areLocalFieldsContiguous(GenTreeLclFld* first, GenTreeLclFld* second); + bool areArrayElementsContiguous(GenTree* op1, GenTree* op2); + bool areArgumentsContiguous(GenTree* op1, GenTree* op2); GenTree* CreateAddressNodeForSimdHWIntrinsicCreate(GenTree* tree, var_types simdBaseType, unsigned simdSize); // Get the size of the SIMD type in bytes @@ -9587,8 +9603,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bool compSwitchedToMinOpts; // Codegen initially was Tier1/FullOpts but jit switched to MinOpts bool compSuppressedZeroInit; // There are vars with lvSuppressedZeroInit set -// NOTE: These values are only reliable after -// the importing is completely finished. + // NOTE: These values are only reliable after + // the importing is completely finished. #ifdef DEBUG // State information - which phases have completed? @@ -9676,11 +9692,11 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX uint32_t preferredVectorByteLength; #endif // TARGET_XARCH -// optimize maximally and/or favor speed over size? + // optimize maximally and/or favor speed over size? -#define DEFAULT_MIN_OPTS_CODE_SIZE 60000 -#define DEFAULT_MIN_OPTS_INSTR_COUNT 20000 -#define DEFAULT_MIN_OPTS_BB_COUNT 2000 +#define DEFAULT_MIN_OPTS_CODE_SIZE 60000 +#define DEFAULT_MIN_OPTS_INSTR_COUNT 20000 +#define DEFAULT_MIN_OPTS_BB_COUNT 2000 #define DEFAULT_MIN_OPTS_LV_NUM_COUNT 2000 #define DEFAULT_MIN_OPTS_LV_REF_COUNT 8000 @@ -10104,7 +10120,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // DEBUG -// clang-format off + // clang-format off #define STRESS_MODES \ \ STRESS_MODE(NONE) \ @@ -10174,7 +10190,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX STRESS_MODES #undef STRESS_MODE }; -// clang-format on + // clang-format on #ifdef DEBUG static const LPCWSTR s_compStressModeNamesW[STRESS_COUNT + 1]; @@ -10184,8 +10200,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #define MAX_STRESS_WEIGHT 100 - bool compStressCompile(compStressArea stressArea, unsigned weightPercentage); - bool compStressCompileHelper(compStressArea stressArea, unsigned weightPercentage); + bool compStressCompile(compStressArea stressArea, unsigned weightPercentage); + bool compStressCompileHelper(compStressArea stressArea, unsigned weightPercentage); static unsigned compStressAreaHash(compStressArea area); #ifdef DEBUG @@ -10289,11 +10305,11 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // (2) the code is hot/cold split, and we issued less code than we expected // in the cold section (the hot section will always be padded out to compTotalHotCodeSize). - bool compIsStatic : 1; // Is the method static (no 'this' pointer)? - bool compIsVarArgs : 1; // Does the method have varargs parameters? - bool compInitMem : 1; // Is the CORINFO_OPT_INIT_LOCALS bit set in the method info options? - bool compProfilerCallback : 1; // JIT inserted a profiler Enter callback - bool compPublishStubParam : 1; // EAX captured in prolog will be available through an intrinsic + bool compIsStatic : 1; // Is the method static (no 'this' pointer)? + bool compIsVarArgs : 1; // Does the method have varargs parameters? + bool compInitMem : 1; // Is the CORINFO_OPT_INIT_LOCALS bit set in the method info options? + bool compProfilerCallback : 1; // JIT inserted a profiler Enter callback + bool compPublishStubParam : 1; // EAX captured in prolog will be available through an intrinsic bool compHasNextCallRetAddr : 1; // The NextCallReturnAddress intrinsic is used. var_types compRetType; // Return type of the method as declared in IL (including SIMD normalization) @@ -10413,7 +10429,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX } } #endif // TARGET_ARM64 - // 4. x86 unmanaged calling conventions require the address of RetBuff to be returned in eax. + // 4. x86 unmanaged calling conventions require the address of RetBuff to be returned in eax. CLANG_FORMAT_COMMENT_ANCHOR; #if defined(TARGET_X86) if (info.compCallConv != CorInfoCallConvExtension::Managed) @@ -10502,7 +10518,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX var_types TypeHandleToVarType(CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr); var_types TypeHandleToVarType(CorInfoType jitType, CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr); -//-------------------------- Global Compiler Data ------------------------------------ + //-------------------------- Global Compiler Data ------------------------------------ #ifdef DEBUG private: @@ -10610,8 +10626,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX //------------ Some utility functions -------------- - void* compGetHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ - void** ppIndirection); /* OUT */ + void* compGetHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ + void** ppIndirection); /* OUT */ // Several JIT/EE interface functions return a CorInfoType, and also return a // class handle as an out parameter if the type is a value class. Returns the @@ -10626,17 +10642,17 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void compDoComponentUnitTestsOnce(); #endif // DEBUG - int compCompile(CORINFO_MODULE_HANDLE classPtr, - void** methodCodePtr, - uint32_t* methodCodeSize, - JitFlags* compileFlags); + int compCompile(CORINFO_MODULE_HANDLE classPtr, + void** methodCodePtr, + uint32_t* methodCodeSize, + JitFlags* compileFlags); void compCompileFinish(); - int compCompileHelper(CORINFO_MODULE_HANDLE classPtr, - COMP_HANDLE compHnd, - CORINFO_METHOD_INFO* methodInfo, - void** methodCodePtr, - uint32_t* methodCodeSize, - JitFlags* compileFlag); + int compCompileHelper(CORINFO_MODULE_HANDLE classPtr, + COMP_HANDLE compHnd, + CORINFO_METHOD_INFO* methodInfo, + void** methodCodePtr, + uint32_t* methodCodeSize, + JitFlags* compileFlag); ArenaAllocator* compGetArenaAllocator(); @@ -10727,10 +10743,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX static unsigned char compGetJitDefaultFill(Compiler* comp); const char* compLocalVarName(unsigned varNum, unsigned offs); - VarName compVarName(regNumber reg, bool isFloatReg = false); + VarName compVarName(regNumber reg, bool isFloatReg = false); const char* compFPregVarName(unsigned fpReg, bool displayVar = false); - void compDspSrcLinesByNativeIP(UNATIVE_OFFSET curIP); - void compDspSrcLinesByLineNum(unsigned line, bool seek = false); + void compDspSrcLinesByNativeIP(UNATIVE_OFFSET curIP); + void compDspSrcLinesByLineNum(unsigned line, bool seek = false); #endif // DEBUG const char* compRegNameForSize(regNumber reg, size_t size); const char* compRegVarName(regNumber reg, bool displayVar = false, bool isFloatReg = false); @@ -10902,8 +10918,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void verInitCurrentState(); void verResetCurrentState(BasicBlock* block, EntryState* currentState); - void verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)); - void verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)); + void verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)); + void verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)); typeInfo verMakeTypeInfoForLocal(unsigned lclNum); typeInfo verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd); // converts from jit type representation to typeInfo typeInfo verMakeTypeInfo(CorInfoType ciType, @@ -11001,8 +11017,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX static fgWalkPreFn gsMarkPtrsAndAssignGroups; // Shadow param analysis tree-walk static fgWalkPreFn gsReplaceShadowParams; // Shadow param replacement tree-walk -#define DEFAULT_MAX_INLINE_SIZE 100 // Methods with > DEFAULT_MAX_INLINE_SIZE IL bytes will never be inlined. - // This can be overwritten by setting DOTNET_JITInlineSize env variable. +#define DEFAULT_MAX_INLINE_SIZE \ + 100 // Methods with > DEFAULT_MAX_INLINE_SIZE IL bytes will never be inlined. + // This can be overwritten by setting DOTNET_JITInlineSize env variable. #define DEFAULT_MAX_INLINE_DEPTH 20 // Methods at more than this level deep will not be inlined @@ -11186,7 +11203,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // defined(UNIX_AMD64_ABI) - void fgMorphMultiregStructArgs(GenTreeCall* call); + void fgMorphMultiregStructArgs(GenTreeCall* call); GenTree* fgMorphMultiregStructArg(CallArg* arg); bool killGCRefs(GenTree* tree); @@ -11341,7 +11358,9 @@ class GenTreeVisitor Compiler* m_compiler; ArrayStack m_ancestors; - GenTreeVisitor(Compiler* compiler) : m_compiler(compiler), m_ancestors(compiler->getAllocator(CMK_ArrayStack)) + GenTreeVisitor(Compiler* compiler) + : m_compiler(compiler) + , m_ancestors(compiler->getAllocator(CMK_ArrayStack)) { assert(compiler != nullptr); @@ -11764,7 +11783,8 @@ class DomTreeVisitor protected: Compiler* m_compiler; - DomTreeVisitor(Compiler* compiler) : m_compiler(compiler) + DomTreeVisitor(Compiler* compiler) + : m_compiler(compiler) { } @@ -11853,7 +11873,8 @@ class EHClauses EHblkDsc* m_ehDsc; public: - iterator(EHblkDsc* ehDsc) : m_ehDsc(ehDsc) + iterator(EHblkDsc* ehDsc) + : m_ehDsc(ehDsc) { } @@ -11875,7 +11896,9 @@ class EHClauses }; public: - EHClauses(Compiler* comp) : m_begin(comp->compHndBBtab), m_end(comp->compHndBBtab + comp->compHndBBtabCount) + EHClauses(Compiler* comp) + : m_begin(comp->compHndBBtab) + , m_end(comp->compHndBBtab + comp->compHndBBtabCount) { assert((m_begin != nullptr) || (m_begin == m_end)); } @@ -11912,7 +11935,9 @@ class StringPrinter public: StringPrinter(CompAllocator alloc, char* buffer = nullptr, size_t bufferMax = 0) - : m_alloc(alloc), m_buffer(buffer), m_bufferMax(bufferMax) + : m_alloc(alloc) + , m_buffer(buffer) + , m_bufferMax(bufferMax) { if ((m_buffer == nullptr) || (m_bufferMax == 0)) { diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 23fd517cf7c134..771ea9aaa90c03 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -98,9 +98,9 @@ inline T genFindLowestBit(T value) } /***************************************************************************** -* -* Return true if the given value has exactly zero or one bits set. -*/ + * + * Return true if the given value has exactly zero or one bits set. + */ template inline bool genMaxOneBit(T value) @@ -109,9 +109,9 @@ inline bool genMaxOneBit(T value) } /***************************************************************************** -* -* Return true if the given value has exactly one bit set. -*/ + * + * Return true if the given value has exactly one bit set. + */ template inline bool genExactlyOneBit(T value) @@ -300,7 +300,8 @@ class Counter : public Dumpable public: int64_t Value; - Counter(int64_t initialValue = 0) : Value(initialValue) + Counter(int64_t initialValue = 0) + : Value(initialValue) { } @@ -352,7 +353,8 @@ class Histogram : public Dumpable class NodeCounts : public Dumpable { public: - NodeCounts() : m_counts() + NodeCounts() + : m_counts() { } @@ -564,7 +566,7 @@ BasicBlockVisit BasicBlock::VisitEHEnclosedHandlerSecondPassSuccs(Compiler* comp // 3. As part of two pass EH, control may bypass filters and flow directly to // filter-handlers // -template +template static BasicBlockVisit VisitEHSuccs(Compiler* comp, BasicBlock* block, TFunc func) { if (!block->HasPotentialEHSuccs(comp)) @@ -1211,7 +1213,7 @@ inline const char* varTypeGCstring(var_types type) /*****************************************************************************/ const char* varTypeName(var_types); -const int regIndexForRegister(regNumber reg); +const int regIndexForRegister(regNumber reg); /*****************************************************************************/ // Helpers to pull little-endian values out of a byte stream. @@ -1364,8 +1366,8 @@ inline Statement* Compiler::gtNewStmt(GenTree* expr, const DebugInfo& di) inline GenTree* Compiler::gtNewOperNode(genTreeOps oper, var_types type, GenTree* op1) { assert((GenTree::OperKind(oper) & (GTK_UNOP | GTK_BINOP)) != 0); - assert((GenTree::OperKind(oper) & GTK_EXOP) == - 0); // Can't use this to construct any types that extend unary/binary operator. + assert((GenTree::OperKind(oper) & GTK_EXOP) == 0); // Can't use this to construct any types that extend unary/binary + // operator. assert(op1 != nullptr || oper == GT_RETFILT || (oper == GT_RETURN && type == TYP_VOID)); GenTree* node = new (this, oper) GenTreeOp(oper, type, op1, nullptr); @@ -1411,7 +1413,7 @@ inline GenTreeIntCon* Compiler::gtNewIconHandleNode(size_t value, GenTreeFlags f node = new (this, LargeOpOpcode()) GenTreeIntCon(gtGetTypeForIconFlags(flags), value, fields DEBUGARG(/*largeNode*/ true)); #else - node = new (this, GT_CNS_INT) GenTreeIntCon(gtGetTypeForIconFlags(flags), value, fields); + node = new (this, GT_CNS_INT) GenTreeIntCon(gtGetTypeForIconFlags(flags), value, fields); #endif node->gtFlags |= flags; return node; @@ -2611,8 +2613,8 @@ inline assert(varDsc->lvIsParam); #endif // UNIX_AMD64_ABI #else // !TARGET_AMD64 - // For other targets, a stack parameter that is enregistered or prespilled - // for profiling on ARM will have a stack location. + // For other targets, a stack parameter that is enregistered or prespilled + // for profiling on ARM will have a stack location. assert((varDsc->lvIsParam && !varDsc->lvIsRegArg) || isPrespilledArg); #endif // !TARGET_AMD64 } @@ -2699,7 +2701,7 @@ inline #ifdef TARGET_ARM varOffset = codeGen->genCallerSPtoInitialSPdelta() - codeGen->genCallerSPtoFPdelta(); #else - varOffset = -(codeGen->genTotalFrameSize()); + varOffset = -(codeGen->genTotalFrameSize()); #endif } } @@ -2753,7 +2755,7 @@ inline *pBaseReg = REG_SPBASE; } #else - *pFPbased = FPbased; + *pFPbased = FPbased; #endif return varOffset; @@ -4836,7 +4838,6 @@ unsigned Compiler::fgRunDfs(VisitPreorder visitPreorder, VisitPostorder visitPos ArrayStack blocks(getAllocator(CMK_DepthFirstSearch)); auto dfsFrom = [&](BasicBlock* firstBB) { - BitVecOps::AddElemD(&traits, visited, firstBB->bbNum); blocks.Emplace(this, firstBB); visitPreorder(firstBB, preOrderIndex++); @@ -4862,7 +4863,6 @@ unsigned Compiler::fgRunDfs(VisitPreorder visitPreorder, VisitPostorder visitPos visitPostorder(block, postOrderIndex++); } } - }; dfsFrom(fgFirstBB); @@ -4907,7 +4907,7 @@ template BasicBlockVisit FlowGraphNaturalLoop::VisitLoopBlocksReversePostOrder(TFunc func) { BitVecTraits traits(m_blocksSize, m_dfsTree->GetCompiler()); - bool result = BitVecOps::VisitBits(&traits, m_blocks, [=](unsigned index) { + bool result = BitVecOps::VisitBits(&traits, m_blocks, [=](unsigned index) { // head block rpo index = PostOrderCount - 1 - headPreOrderIndex // loop block rpo index = head block rpoIndex + index // loop block po index = PostOrderCount - 1 - loop block rpo index @@ -4939,7 +4939,7 @@ template BasicBlockVisit FlowGraphNaturalLoop::VisitLoopBlocksPostOrder(TFunc func) { BitVecTraits traits(m_blocksSize, m_dfsTree->GetCompiler()); - bool result = BitVecOps::VisitBitsReverse(&traits, m_blocks, [=](unsigned index) { + bool result = BitVecOps::VisitBitsReverse(&traits, m_blocks, [=](unsigned index) { unsigned poIndex = m_header->bbPostorderNum - index; assert(poIndex < m_dfsTree->GetPostOrderCount()); return func(m_dfsTree->GetPostOrder(poIndex)) == BasicBlockVisit::Continue; @@ -5134,7 +5134,7 @@ bool AllRegsMask::operator==(const AllRegsMask& other) { return (_combinedRegisters == other._combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS - && (_predicateRegs == other._predicateRegs); + && (_predicateRegs == other._predicateRegs); #endif ; } @@ -5185,9 +5185,9 @@ unsigned AllRegsMask::Count() { return genCountBits(_combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS - + genCountBits(_predicateRegs) + + genCountBits(_predicateRegs) #endif - ; + ; } regMaskOnlyOne AllRegsMask::operator[](int index) const @@ -5237,7 +5237,7 @@ void AllRegsMask::AddGprRegInMask(regNumber reg) void AllRegsMask::AddRegNumInMask(regNumber reg) { RegBitSet64 value = genRegMask(reg); -#ifdef HAS_MORE_THAN_64_REGISTERS +#ifdef HAS_MORE_THAN_64_REGISTERS int index = regIndexForRegister(reg); _registers[index] |= encodeForIndex(index, value); #else @@ -5290,7 +5290,7 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) { RegBitSet64 value = genRegMask(reg); #ifdef HAS_MORE_THAN_64_REGISTERS - int index = regIndexForRegister(reg); + int index = regIndexForRegister(reg); return (_registers[index] & encodeForIndex(index, value)) != RBM_NONE; #else return (_combinedRegisters & value) != RBM_NONE; @@ -5348,7 +5348,6 @@ void AllRegsMask::RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_type #endif } - bool AllRegsMask::IsGprMaskPresent(regMaskGpr maskToCheck) const { return (gprRegs() & maskToCheck) != RBM_NONE; @@ -5414,7 +5413,6 @@ regMaskPredicate AllRegsMask::predicateRegs(const Compiler* compiler) const } #endif // FEATURE_MASKED_HW_INTRINSICS - /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/coreclr/jit/compilerbitsettraits.h b/src/coreclr/jit/compilerbitsettraits.h index 02223b1ecedfce..965ffac55465e1 100644 --- a/src/coreclr/jit/compilerbitsettraits.h +++ b/src/coreclr/jit/compilerbitsettraits.h @@ -107,7 +107,9 @@ struct BitVecTraits Compiler* comp; public: - BitVecTraits(unsigned size, Compiler* comp) : size(size), comp(comp) + BitVecTraits(unsigned size, Compiler* comp) + : size(size) + , comp(comp) { const unsigned elemBits = 8 * sizeof(size_t); arraySize = roundUp(size, elemBits) / elemBits; diff --git a/src/coreclr/jit/copyprop.cpp b/src/coreclr/jit/copyprop.cpp index 90a593ef65b2fe..142c745fc7c317 100644 --- a/src/coreclr/jit/copyprop.cpp +++ b/src/coreclr/jit/copyprop.cpp @@ -462,7 +462,9 @@ PhaseStatus Compiler::optVnCopyProp() public: CopyPropDomTreeVisitor(Compiler* compiler) - : DomTreeVisitor(compiler), m_curSsaName(compiler->getAllocator(CMK_CopyProp)), m_madeChanges(false) + : DomTreeVisitor(compiler) + , m_curSsaName(compiler->getAllocator(CMK_CopyProp)) + , m_madeChanges(false) { } diff --git a/src/coreclr/jit/debuginfo.h b/src/coreclr/jit/debuginfo.h index 3f628840765dc7..72119b905c948a 100644 --- a/src/coreclr/jit/debuginfo.h +++ b/src/coreclr/jit/debuginfo.h @@ -12,12 +12,17 @@ class InlineContext; class ILLocation { public: - ILLocation() : m_offset(BAD_IL_OFFSET), m_isStackEmpty(false), m_isCall(false) + ILLocation() + : m_offset(BAD_IL_OFFSET) + , m_isStackEmpty(false) + , m_isCall(false) { } ILLocation(IL_OFFSET offset, bool isStackEmpty, bool isCall) - : m_offset(offset), m_isStackEmpty(isStackEmpty), m_isCall(isCall) + : m_offset(offset) + , m_isStackEmpty(isStackEmpty) + , m_isCall(isCall) { } @@ -65,18 +70,21 @@ class ILLocation private: IL_OFFSET m_offset; bool m_isStackEmpty : 1; - bool m_isCall : 1; + bool m_isCall : 1; }; // Represents debug information about a statement. class DebugInfo { public: - DebugInfo() : m_inlineContext(nullptr) + DebugInfo() + : m_inlineContext(nullptr) { } - DebugInfo(InlineContext* inlineContext, ILLocation loc) : m_inlineContext(inlineContext), m_location(loc) + DebugInfo(InlineContext* inlineContext, ILLocation loc) + : m_inlineContext(inlineContext) + , m_location(loc) { } diff --git a/src/coreclr/jit/decomposelongs.h b/src/coreclr/jit/decomposelongs.h index b8ddc621079925..744061091e42be 100644 --- a/src/coreclr/jit/decomposelongs.h +++ b/src/coreclr/jit/decomposelongs.h @@ -18,7 +18,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX class DecomposeLongs { public: - DecomposeLongs(Compiler* compiler) : m_compiler(compiler) + DecomposeLongs(Compiler* compiler) + : m_compiler(compiler) { } @@ -72,7 +73,7 @@ class DecomposeLongs GenTree* RepresentOpAsLocalVar(GenTree* op, GenTree* user, GenTree** edge); GenTree* EnsureIntSized(GenTree* node, bool signExtend); - GenTree* StoreNodeToVar(LIR::Use& use); + GenTree* StoreNodeToVar(LIR::Use& use); static genTreeOps GetHiOper(genTreeOps oper); static genTreeOps GetLoOper(genTreeOps oper); diff --git a/src/coreclr/jit/disasm.cpp b/src/coreclr/jit/disasm.cpp index 2a49f9d8cb55c5..bff93c85150a68 100644 --- a/src/coreclr/jit/disasm.cpp +++ b/src/coreclr/jit/disasm.cpp @@ -1,12 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. /*********************************************************************** -* -* File: disasm.cpp -* -* This file handles disassembly for the "late disassembler". -* -***********************************************************************/ + * + * File: disasm.cpp + * + * This file handles disassembly for the "late disassembler". + * + ***********************************************************************/ #include "jitpch.h" #ifdef _MSC_VER @@ -23,7 +23,7 @@ FILE* g_disAsmFileCorDisTools; #endif // USE_COREDISTOOLS // Define DISASM_DEBUG to get verbose output of late disassembler inner workings. -//#define DISASM_DEBUG +// #define DISASM_DEBUG #ifdef DISASM_DEBUG #ifdef DEBUG #define DISASM_DUMP(...) \ @@ -96,12 +96,12 @@ typedef struct codeFix { codeFix* cfNext; unsigned cfFixup; -} * codeFixPtr; +}* codeFixPtr; typedef struct codeBlk { codeFix* cbFixupLst; -} * codeBlkPtr; +}* codeBlkPtr; #ifdef USE_MSVCDIS @@ -139,7 +139,7 @@ size_t DisAssembler::disCchAddrMember( switch (terminationType) { - // int disCallSize; + // int disCallSize; case DISX86::trmtaJmpShort: case DISX86::trmtaJmpCcShort: @@ -228,7 +228,7 @@ size_t DisAssembler::disCchAddrMember( switch (terminationType) { - // int disCallSize; + // int disCallSize; case DISARM64::TRMTA::trmtaBra: case DISARM64::TRMTA::trmtaBraCase: @@ -620,7 +620,7 @@ size_t DisAssembler::disCchRegRelMember( case DISX86::trmtaFallThrough: - /* some instructions like division have a TRAP termination type - ignore it */ + /* some instructions like division have a TRAP termination type - ignore it */ case DISX86::trmtaTrap: case DISX86::trmtaTrapCc: @@ -715,7 +715,7 @@ size_t DisAssembler::disCchRegRelMember( case DISARM64::TRMTA::trmtaFallThrough: - /* some instructions like division have a TRAP termination type - ignore it */ + /* some instructions like division have a TRAP termination type - ignore it */ case DISARM64::TRMTA::trmtaTrap: case DISARM64::TRMTA::trmtaTrapCc: @@ -1261,7 +1261,7 @@ void DisAssembler::DisasmBuffer(FILE* pfile, bool printit) #elif defined(TARGET_AMD64) pdis = DIS::PdisNew(DIS::distX8664); #elif defined(TARGET_ARM64) - pdis = DIS::PdisNew(DIS::distArm64); + pdis = DIS::PdisNew(DIS::distArm64); #else // TARGET* #error Unsupported or unset target architecture #endif @@ -1340,7 +1340,7 @@ void DisAssembler::DisasmBuffer(FILE* pfile, bool printit) #else false // Display code bytes? #endif - ); + ); ibCur += (unsigned)cb; } @@ -1680,7 +1680,7 @@ bool DisAssembler::InitCoredistoolsLibrary() s_disCoreDisToolsLibraryLoadSuccessful = true; // We made it! -// done initializing + // done initializing FinishedInitializing: InterlockedExchange(&s_disCoreDisToolsLibraryInitializing, 0); // unlock initialization @@ -1703,7 +1703,7 @@ bool DisAssembler::InitCoredistoolsDisasm() #if defined(TARGET_ARM64) coreDisTargetArchitecture = Target_Arm64; #elif defined(TARGET_ARM) - coreDisTargetArchitecture = Target_Thumb; + coreDisTargetArchitecture = Target_Thumb; #elif defined(TARGET_X86) coreDisTargetArchitecture = Target_X86; #elif defined(TARGET_AMD64) diff --git a/src/coreclr/jit/ee_il_dll.cpp b/src/coreclr/jit/ee_il_dll.cpp index cfe50b492bb321..b33e6eed17bbc3 100644 --- a/src/coreclr/jit/ee_il_dll.cpp +++ b/src/coreclr/jit/ee_il_dll.cpp @@ -211,7 +211,9 @@ void SetJitTls(void* value) #if defined(DEBUG) -JitTls::JitTls(ICorJitInfo* jitInfo) : m_compiler(nullptr), m_logEnv(jitInfo) +JitTls::JitTls(ICorJitInfo* jitInfo) + : m_compiler(nullptr) + , m_logEnv(jitInfo) { m_next = reinterpret_cast(GetJitTls()); SetJitTls(this); @@ -1407,7 +1409,9 @@ bool Compiler::eeRunWithSPMIErrorTrapImp(void (*function)(void*), void* param) unsigned Compiler::eeTryGetClassSize(CORINFO_CLASS_HANDLE clsHnd) { unsigned classSize = UINT_MAX; - eeRunFunctorWithSPMIErrorTrap([&]() { classSize = info.compCompHnd->getClassSize(clsHnd); }); + eeRunFunctorWithSPMIErrorTrap([&]() { + classSize = info.compCompHnd->getClassSize(clsHnd); + }); return classSize; } diff --git a/src/coreclr/jit/ee_il_dll.hpp b/src/coreclr/jit/ee_il_dll.hpp index c3801d88292f5f..d676ba8caa479a 100644 --- a/src/coreclr/jit/ee_il_dll.hpp +++ b/src/coreclr/jit/ee_il_dll.hpp @@ -10,12 +10,12 @@ class CILJit : public ICorJitCompiler unsigned flags, /* IN */ uint8_t** nativeEntry, /* OUT */ uint32_t* nativeSizeOfCode /* OUT */ - ); + ); void ProcessShutdownWork(ICorStaticInfo* statInfo); void getVersionIdentifier(GUID* versionIdentifier /* OUT */ - ); + ); void setTargetOS(CORINFO_OS os); }; diff --git a/src/coreclr/jit/eeinterface.cpp b/src/coreclr/jit/eeinterface.cpp index 0578dee4109ef1..a6552c2194294c 100644 --- a/src/coreclr/jit/eeinterface.cpp +++ b/src/coreclr/jit/eeinterface.cpp @@ -210,7 +210,7 @@ void Compiler::eePrintTypeOrJitAlias(StringPrinter* printer, CORINFO_CLASS_HANDL } static const char* s_jitHelperNames[CORINFO_HELP_COUNT] = { -#define JITHELPER(code, pfnHelper, sig) #code, +#define JITHELPER(code, pfnHelper, sig) #code, #define DYNAMICJITHELPER(code, pfnHelper, sig) #code, #include "jithelpers.h" }; @@ -403,10 +403,9 @@ const char* Compiler::eeGetMethodFullName( CORINFO_SIG_INFO sig; eeGetMethodSig(hnd, &sig); eePrintMethod(&p, clsHnd, hnd, &sig, - /* includeClassInstantiation */ true, - /* includeMethodInstantiation */ true, - /* includeSignature */ true, includeReturnType, includeThisSpecifier); - + /* includeClassInstantiation */ true, + /* includeMethodInstantiation */ true, + /* includeSignature */ true, includeReturnType, includeThisSpecifier); }); if (success) @@ -475,13 +474,12 @@ const char* Compiler::eeGetMethodName(CORINFO_METHOD_HANDLE methHnd, char* buffe StringPrinter p(getAllocator(CMK_DebugOnly), buffer, bufferSize); bool success = eeRunFunctorWithSPMIErrorTrap([&]() { eePrintMethod(&p, NO_CLASS_HANDLE, methHnd, - /* sig */ nullptr, - /* includeClassInstantiation */ false, - /* includeMethodInstantiation */ false, - /* includeSignature */ false, - /* includeReturnType */ false, - /* includeThisSpecifier */ false); - + /* sig */ nullptr, + /* includeClassInstantiation */ false, + /* includeMethodInstantiation */ false, + /* includeSignature */ false, + /* includeReturnType */ false, + /* includeThisSpecifier */ false); }); if (!success) @@ -512,7 +510,9 @@ const char* Compiler::eeGetMethodName(CORINFO_METHOD_HANDLE methHnd, char* buffe const char* Compiler::eeGetFieldName(CORINFO_FIELD_HANDLE fldHnd, bool includeType, char* buffer, size_t bufferSize) { StringPrinter p(getAllocator(CMK_DebugOnly), buffer, bufferSize); - bool success = eeRunFunctorWithSPMIErrorTrap([&]() { eePrintField(&p, fldHnd, includeType); }); + bool success = eeRunFunctorWithSPMIErrorTrap([&]() { + eePrintField(&p, fldHnd, includeType); + }); if (success) { @@ -525,7 +525,9 @@ const char* Compiler::eeGetFieldName(CORINFO_FIELD_HANDLE fldHnd, bool includeTy { p.Append(":"); - success = eeRunFunctorWithSPMIErrorTrap([&]() { eePrintField(&p, fldHnd, false); }); + success = eeRunFunctorWithSPMIErrorTrap([&]() { + eePrintField(&p, fldHnd, false); + }); if (success) { @@ -560,7 +562,9 @@ const char* Compiler::eeGetFieldName(CORINFO_FIELD_HANDLE fldHnd, bool includeTy const char* Compiler::eeGetClassName(CORINFO_CLASS_HANDLE clsHnd, char* buffer, size_t bufferSize) { StringPrinter printer(getAllocator(CMK_DebugOnly), buffer, bufferSize); - if (!eeRunFunctorWithSPMIErrorTrap([&]() { eePrintType(&printer, clsHnd, true); })) + if (!eeRunFunctorWithSPMIErrorTrap([&]() { + eePrintType(&printer, clsHnd, true); + })) { printer.Truncate(0); printer.Append(""); @@ -581,7 +585,9 @@ const char* Compiler::eeGetClassName(CORINFO_CLASS_HANDLE clsHnd, char* buffer, const char* Compiler::eeGetShortClassName(CORINFO_CLASS_HANDLE clsHnd) { StringPrinter printer(getAllocator(CMK_DebugOnly)); - if (!eeRunFunctorWithSPMIErrorTrap([&]() { eePrintType(&printer, clsHnd, false); })) + if (!eeRunFunctorWithSPMIErrorTrap([&]() { + eePrintType(&printer, clsHnd, false); + })) { printer.Truncate(0); printer.Append(""); @@ -597,8 +603,9 @@ void Compiler::eePrintObjectDescription(const char* prefix, CORINFO_OBJECT_HANDL size_t actualLen = 0; // Ignore potential SPMI failures - bool success = eeRunFunctorWithSPMIErrorTrap( - [&]() { actualLen = this->info.compCompHnd->printObjectDescription(handle, str, maxStrSize); }); + bool success = eeRunFunctorWithSPMIErrorTrap([&]() { + actualLen = this->info.compCompHnd->printObjectDescription(handle, str, maxStrSize); + }); if (!success) { diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 39e6b65f9ca975..f28a241b7842d1 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -784,7 +784,7 @@ void emitter::emitGenIG(insGroup* ig) IMPL_LIMITATION("Too many arguments pushed on stack"); } -// printf("Start IG #%02u [stk=%02u]\n", ig->igNum, emitCurStackLvl); + // printf("Start IG #%02u [stk=%02u]\n", ig->igNum, emitCurStackLvl); #endif @@ -1202,7 +1202,7 @@ void emitter::emitBegFN(bool hasFramePtr , bool chkAlign #endif - ) +) { insGroup* ig; @@ -1609,7 +1609,7 @@ void* emitter::emitAllocAnyInstr(size_t sz, emitAttr opsz) #if defined(FEATURE_EH_FUNCLETS) && !emitIGisInFuncletProlog(emitCurIG) && !emitIGisInFuncletEpilog(emitCurIG) #endif // FEATURE_EH_FUNCLETS - ) + ) { emitNxtIG(true); } @@ -1624,7 +1624,7 @@ void* emitter::emitAllocAnyInstr(size_t sz, emitAttr opsz) !emitIGisInProlog(emitCurIG) && // don't do this in prolog or epilog !emitIGisInEpilog(emitCurIG) && emitRandomNops // sometimes we turn off where exact codegen is needed (pinvoke inline) - ) + ) { if (emitNextNop == 0) { @@ -1758,7 +1758,7 @@ void* emitter::emitAllocAnyInstr(size_t sz, emitAttr opsz) #ifndef TARGET_AMD64 && emitComp->opts.compReloc #endif // TARGET_AMD64 - ) + ) { /* Mark idInfo()->idDspReloc to remember that the */ /* address mode has a displacement that is relocatable */ @@ -2071,7 +2071,7 @@ void emitter::emitCreatePlaceholderIG(insGroupPlaceholderType igType, #if defined(FEATURE_EH_FUNCLETS) || igType == IGPT_FUNCLET_EPILOG #endif // FEATURE_EH_FUNCLETS - ) + ) { #ifdef TARGET_AMD64 emitOutputPreEpilogNOP(); @@ -2199,7 +2199,7 @@ void emitter::emitCreatePlaceholderIG(insGroupPlaceholderType igType, #if defined(FEATURE_EH_FUNCLETS) || igType == IGPT_FUNCLET_EPILOG #endif // FEATURE_EH_FUNCLETS - ) + ) { // If this was an epilog, then assume this is the end of any currently in progress // no-GC region. If a block after the epilog needs to be no-GC, it needs to call @@ -2506,11 +2506,11 @@ void emitter::emitEndFnEpilog() // because the only instruction is the last one and thus a slight // underestimation of the epilog size is harmless (since the EIP // can not be between instructions). - assert(emitEpilogCnt == 1 || - (emitExitSeqSize - newSize) <= 5 // delta between size of various forms of jmp (size is either 6 or 5), - // and various forms of ret (size is either 1 or 3). The combination can - // be anything between 1 and 5. - ); + assert(emitEpilogCnt == 1 || (emitExitSeqSize - newSize) <= 5 // delta between size of various forms of jmp + // (size is either 6 or 5), and various forms of + // ret (size is either 1 or 3). The combination + // can be anything between 1 and 5. + ); emitExitSeqSize = newSize; } #endif // JIT32_GCENCODER @@ -2822,11 +2822,11 @@ bool emitter::emitNoGChelper(CorInfoHelpFunc helpFunc) case CORINFO_HELP_LRSH: case CORINFO_HELP_LRSZ: -// case CORINFO_HELP_LMUL: -// case CORINFO_HELP_LDIV: -// case CORINFO_HELP_LMOD: -// case CORINFO_HELP_ULDIV: -// case CORINFO_HELP_ULMOD: + // case CORINFO_HELP_LMUL: + // case CORINFO_HELP_LDIV: + // case CORINFO_HELP_LMOD: + // case CORINFO_HELP_ULDIV: + // case CORINFO_HELP_ULMOD: #ifdef TARGET_X86 case CORINFO_HELP_ASSIGN_REF_EAX: @@ -2887,8 +2887,8 @@ bool emitter::emitNoGChelper(CORINFO_METHOD_HANDLE methHnd) * Mark the current spot as having a label. */ -void* emitter::emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskGpr gcrefRegs, +void* emitter::emitAddLabel(VARSET_VALARG_TP GCvars, + regMaskGpr gcrefRegs, regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block)) { /* Create a new IG if the current one is non-empty */ @@ -3085,7 +3085,7 @@ void emitter::emitSplit(emitLocation* startLoc, return; } -// Report it! + // Report it! #ifdef DEBUG if (EMITVERBOSE) @@ -3627,7 +3627,7 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt, VARSET_VALARG_TP GCvars, regMaskGpr gcrefRegs, regMaskGpr byrefRegs, - emitAttr retSizeIn + emitAttr retSizeIn MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)) { assert(emitComp->IsGprRegMask(gcrefRegs)); @@ -3712,7 +3712,7 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int argCnt, VARSET_VALARG_TP GCvars, regMaskGpr gcrefRegs, regMaskGpr byrefRegs, - emitAttr retSizeIn + emitAttr retSizeIn MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)) { assert(emitComp->IsGprRegMask(gcrefRegs)); @@ -3942,8 +3942,8 @@ void emitter::emitDispRegPtrListDelta() // Dump any deltas in regPtrDsc's for outgoing args; these aren't captured in the other sets. if (debugPrevRegPtrDsc != codeGen->gcInfo.gcRegPtrLast) { - for (regPtrDsc* dsc = (debugPrevRegPtrDsc == nullptr) ? codeGen->gcInfo.gcRegPtrList - : debugPrevRegPtrDsc->rpdNext; + for (regPtrDsc* dsc = (debugPrevRegPtrDsc == nullptr) ? codeGen->gcInfo.gcRegPtrList + : debugPrevRegPtrDsc->rpdNext; dsc != nullptr; dsc = dsc->rpdNext) { // The non-arg regPtrDscs are reflected in the register sets debugPrevGCrefRegs/emitThisGCrefRegs @@ -4427,7 +4427,7 @@ size_t emitter::emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp) ig->igPerfScore += insPerfScore; #endif // defined(DEBUG) || defined(LATE_DISASM) -// printf("[S=%02u]\n", emitCurStackLvl); + // printf("[S=%02u]\n", emitCurStackLvl); #if EMIT_TRACK_STACK_DEPTH @@ -4589,7 +4589,7 @@ void emitter::emitDispCommentForHandle(size_t handle, size_t cookie, GenTreeFlag #ifdef DEBUG emitComp->eePrintObjectDescription(commentPrefix, (CORINFO_OBJECT_HANDLE)handle); #else - str = "frozen object handle"; + str = "frozen object handle"; #endif } else if (flag == GTF_ICON_CLASS_HDL) @@ -4900,9 +4900,9 @@ void emitter::emitJumpDistBind() int jmp_iteration = 1; -/*****************************************************************************/ -/* If we iterate to look for more jumps to shorten, we start again here. */ -/*****************************************************************************/ + /*****************************************************************************/ + /* If we iterate to look for more jumps to shorten, we start again here. */ + /*****************************************************************************/ AGAIN: @@ -4910,10 +4910,10 @@ void emitter::emitJumpDistBind() emitCheckIGList(); #endif -/* - In the following loop we convert all jump targets from "BasicBlock *" - to "insGroup *" values. We also estimate which jumps will be short. - */ + /* + In the following loop we convert all jump targets from "BasicBlock *" + to "insGroup *" values. We also estimate which jumps will be short. + */ #ifdef DEBUG insGroup* lastIG = nullptr; @@ -5053,7 +5053,7 @@ void emitter::emitJumpDistBind() } #endif // TARGET_ARM64 -/* Make sure the jumps are properly ordered */ + /* Make sure the jumps are properly ordered */ #ifdef DEBUG assert(lastLJ == nullptr || lastIG != jmp->idjIG || lastLJ->idjOffs < jmp->idjOffs); @@ -5457,9 +5457,9 @@ void emitter::emitJumpDistBind() continue; - /*****************************************************************************/ - /* Handle conversion to short jump */ - /*****************************************************************************/ + /*****************************************************************************/ + /* Handle conversion to short jump */ + /*****************************************************************************/ SHORT_JMP: @@ -5499,9 +5499,9 @@ void emitter::emitJumpDistBind() #if defined(TARGET_ARM) - /*****************************************************************************/ - /* Handle conversion to medium jump */ - /*****************************************************************************/ + /*****************************************************************************/ + /* Handle conversion to medium jump */ + /*****************************************************************************/ MEDIUM_JMP: @@ -5526,7 +5526,7 @@ void emitter::emitJumpDistBind() #endif // TARGET_ARM - /*****************************************************************************/ + /*****************************************************************************/ NEXT_JMP: @@ -5602,7 +5602,7 @@ void emitter::emitJumpDistBind() #if defined(TARGET_ARM) || (minMediumExtra <= adjIG) #endif // TARGET_ARM - ) + ) { jmp_iteration++; @@ -5857,8 +5857,8 @@ bool emitter::emitEndsWithAlignInstr() // Returns: size of a loop in bytes. // unsigned emitter::getLoopSize(insGroup* igLoopHeader, - unsigned maxLoopSize // - DEBUG_ARG(bool isAlignAdjusted) // + unsigned maxLoopSize // + DEBUG_ARG(bool isAlignAdjusted) // DEBUG_ARG(UNATIVE_OFFSET containingIGNum) // DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum)) { @@ -6266,7 +6266,7 @@ void emitter::emitLoopAlignAdjustments() } #endif // TARGET_XARCH & TARGET_ARM64 #endif // DEBUG - // Adjust the padding amount in all align instructions in this IG + // Adjust the padding amount in all align instructions in this IG instrDescAlign *alignInstrToAdj = alignInstr, *prevAlignInstr = nullptr; for (; alignInstrToAdj != nullptr && alignInstrToAdj->idaIG == alignInstr->idaIG; alignInstrToAdj = alignInstrToAdj->idaNext) @@ -6362,7 +6362,7 @@ void emitter::emitLoopAlignAdjustments() // 3b. If the loop already fits in minimum alignmentBoundary blocks, then return 0. // already best aligned // 3c. return paddingNeeded. // -unsigned emitter::emitCalculatePaddingForLoopAlignment(insGroup* loopHeadIG, +unsigned emitter::emitCalculatePaddingForLoopAlignment(insGroup* loopHeadIG, size_t offset DEBUG_ARG(bool isAlignAdjusted) DEBUG_ARG(UNATIVE_OFFSET containingIGNum) DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum)) @@ -6703,18 +6703,18 @@ void emitter::emitComputeCodeSizes() // Returns: // size of the method code, in bytes // -unsigned emitter::emitEndCodeGen(Compiler* comp, - bool contTrkPtrLcls, - bool fullyInt, - bool fullPtrMap, - unsigned xcptnsCount, - unsigned* prologSize, - unsigned* epilogSize, - void** codeAddr, - void** codeAddrRW, - void** coldCodeAddr, - void** coldCodeAddrRW, - void** consAddr, +unsigned emitter::emitEndCodeGen(Compiler* comp, + bool contTrkPtrLcls, + bool fullyInt, + bool fullPtrMap, + unsigned xcptnsCount, + unsigned* prologSize, + unsigned* epilogSize, + void** codeAddr, + void** codeAddrRW, + void** coldCodeAddr, + void** coldCodeAddrRW, + void** consAddr, void** consAddrRW DEBUGARG(unsigned* instrCount)) { #ifdef DEBUG @@ -7146,7 +7146,7 @@ unsigned emitter::emitEndCodeGen(Compiler* comp, assert(indx < emitComp->lvaTrackedCount); -// printf("Variable #%2u/%2u is at stack offset %d\n", num, indx, offs); + // printf("Variable #%2u/%2u is at stack offset %d\n", num, indx, offs); #ifdef JIT32_GCENCODER #ifndef FEATURE_EH_FUNCLETS @@ -8475,7 +8475,7 @@ void emitter::emitDispDataSec(dataSecDsc* section, BYTE* dst) printf("\tdd\t%08Xh", (uint32_t)(size_t)emitOffsetToPtr(ig->igOffs)); } #else // TARGET_64BIT - // We have a 64-BIT target + // We have a 64-BIT target if (emitComp->opts.disDiffable) { printf("\tdq\t%s\n", blockLabel); @@ -9074,7 +9074,7 @@ void emitter::emitGCregDeadSet(GCtype gcType, regMaskGpr regMask, BYTE* addr) unsigned char emitter::emitOutputByte(BYTE* dst, ssize_t val) { - BYTE* dstRW = dst + writeableOffset; + BYTE* dstRW = dst + writeableOffset; *castto(dstRW, unsigned char*) = (unsigned char)val; #ifdef DEBUG @@ -9840,13 +9840,13 @@ cnsval_ssize_t emitter::emitGetInsSC(const instrDesc* id) const else #endif // TARGET_ARM if (id->idIsLargeCns()) - { - return ((instrDescCns*)id)->idcCnsVal; - } - else - { - return id->idSmallCns(); - } + { + return ((instrDescCns*)id)->idcCnsVal; + } + else + { + return id->idSmallCns(); + } } #ifdef TARGET_ARM @@ -9957,7 +9957,7 @@ void emitter::emitStackPop(BYTE* addr, bool isCall, unsigned char callInstrSize, #ifndef JIT32_GCENCODER || (emitComp->IsFullPtrRegMapRequired() && !emitComp->GetInterruptible() && isCall) #endif // JIT32_GCENCODER - ) + ) { emitStackPopLargeStk(addr, isCall, callInstrSize, 0); } @@ -10234,17 +10234,17 @@ void emitter::emitStackKillArgs(BYTE* addr, unsigned count, unsigned char callIn #ifdef DEBUG -void emitter::emitRecordRelocationHelp(void* location, /* IN */ - void* target, /* IN */ - uint16_t fRelocType, /* IN */ - const char* relocTypeName, /* IN */ +void emitter::emitRecordRelocationHelp(void* location, /* IN */ + void* target, /* IN */ + uint16_t fRelocType, /* IN */ + const char* relocTypeName, /* IN */ int32_t addlDelta /* = 0 */) /* IN */ #else // !DEBUG -void emitter::emitRecordRelocation(void* location, /* IN */ - void* target, /* IN */ - uint16_t fRelocType, /* IN */ +void emitter::emitRecordRelocation(void* location, /* IN */ + void* target, /* IN */ + uint16_t fRelocType, /* IN */ int32_t addlDelta /* = 0 */) /* IN */ #endif // !DEBUG diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index f281e8d27f4eb5..37755723ebf88a 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -118,7 +118,7 @@ inline const char* GCtypeStr(GCtype gcType) #if DEBUG_EMIT #define INTERESTING_JUMP_NUM -1 // set to 0 to see all jump info -//#define INTERESTING_JUMP_NUM 0 +// #define INTERESTING_JUMP_NUM 0 #endif /***************************************************************************** @@ -129,11 +129,15 @@ inline const char* GCtypeStr(GCtype gcType) class emitLocation { public: - emitLocation() : ig(nullptr), codePos(0) + emitLocation() + : ig(nullptr) + , codePos(0) { } - emitLocation(insGroup* _ig) : ig(_ig), codePos(0) + emitLocation(insGroup* _ig) + : ig(_ig) + , codePos(0) { } @@ -147,7 +151,9 @@ class emitLocation CaptureLocation(emit); } - emitLocation(void* emitCookie) : ig((insGroup*)emitCookie), codePos(0) + emitLocation(void* emitCookie) + : ig((insGroup*)emitCookie) + , codePos(0) { } @@ -286,20 +292,23 @@ struct insGroup insGroup* igLoopBackEdge; // "last" back-edge that branches back to an aligned loop head. #endif -#define IGF_GC_VARS 0x0001 // new set of live GC ref variables -#define IGF_BYREF_REGS 0x0002 // new set of live by-ref registers +#define IGF_GC_VARS 0x0001 // new set of live GC ref variables +#define IGF_BYREF_REGS 0x0002 // new set of live by-ref registers #define IGF_FUNCLET_PROLOG 0x0004 // this group belongs to a funclet prolog #define IGF_FUNCLET_EPILOG 0x0008 // this group belongs to a funclet epilog. -#define IGF_EPILOG 0x0010 // this group belongs to a main function epilog -#define IGF_NOGCINTERRUPT 0x0020 // this IG is in a no-interrupt region (prolog, epilog, etc.) -#define IGF_UPD_ISZ 0x0040 // some instruction sizes updated -#define IGF_PLACEHOLDER 0x0080 // this is a placeholder group, to be filled in later -#define IGF_EXTEND 0x0100 // this block is conceptually an extension of the previous block - // and the emitter should continue to track GC info as if there was no new block. -#define IGF_HAS_ALIGN 0x0200 // this group contains an alignment instruction(s) at the end to align either the next - // IG, or, if this IG contains with an unconditional branch, some subsequent IG. -#define IGF_REMOVED_ALIGN 0x0400 // IG was marked as having an alignment instruction(s), but was later unmarked - // without updating the IG's size/offsets. +#define IGF_EPILOG 0x0010 // this group belongs to a main function epilog +#define IGF_NOGCINTERRUPT 0x0020 // this IG is in a no-interrupt region (prolog, epilog, etc.) +#define IGF_UPD_ISZ 0x0040 // some instruction sizes updated +#define IGF_PLACEHOLDER 0x0080 // this is a placeholder group, to be filled in later +#define IGF_EXTEND \ + 0x0100 // this block is conceptually an extension of the previous block + // and the emitter should continue to track GC info as if there was no new block. +#define IGF_HAS_ALIGN \ + 0x0200 // this group contains an alignment instruction(s) at the end to align either the next + // IG, or, if this IG contains with an unconditional branch, some subsequent IG. +#define IGF_REMOVED_ALIGN \ + 0x0400 // IG was marked as having an alignment instruction(s), but was later unmarked + // without updating the IG's size/offsets. #define IGF_HAS_REMOVABLE_JMP 0x0800 // this group ends with an unconditional jump which is a candidate for removal #ifdef TARGET_ARM64 #define IGF_HAS_REMOVED_INSTR 0x1000 // this group has an instruction that was removed. @@ -323,7 +332,8 @@ struct insGroup regMaskSmall igGCregs; // set of registers with live GC refs - union { + union + { BYTE* igData; // addr of instruction descriptors insPlaceholderGroupData* igPhData; // when igFlags & IGF_PLACEHOLDER }; @@ -422,8 +432,8 @@ struct emitLclVarAddr // protected: unsigned _lvaVarNum : 15; // Usually the lvaVarNum - unsigned _lvaExtra : 15; // Usually the lvaOffset - unsigned _lvaTag : 2; // tag field to support larger varnums + unsigned _lvaExtra : 15; // Usually the lvaOffset + unsigned _lvaTag : 2; // tag field to support larger varnums }; enum idAddrUnionTag @@ -507,7 +517,7 @@ class emitter #ifdef TARGET_AMD64 OPSZP = OPSZ8, #else - OPSZP = OPSZ4, + OPSZP = OPSZ4, #endif }; @@ -516,7 +526,7 @@ class emitter static const emitAttr emitSizeDecode[]; static emitter::opSize emitEncodeSize(emitAttr size); - static emitAttr emitDecodeSize(emitter::opSize ensz); + static emitAttr emitDecodeSize(emitter::opSize ensz); // Currently, we only allow one IG for the prolog bool emitIGisInProlog(const insGroup* ig) @@ -564,10 +574,10 @@ class emitter #ifdef TARGET_XARCH -#define AM_DISP_BITS ((sizeof(unsigned) * 8) - 2 * (REGNUM_BITS + 1) - 2) +#define AM_DISP_BITS ((sizeof(unsigned) * 8) - 2 * (REGNUM_BITS + 1) - 2) #define AM_DISP_BIG_VAL (-(1 << (AM_DISP_BITS - 1))) -#define AM_DISP_MIN (-((1 << (AM_DISP_BITS - 1)) - 1)) -#define AM_DISP_MAX (+((1 << (AM_DISP_BITS - 1)) - 1)) +#define AM_DISP_MIN (-((1 << (AM_DISP_BITS - 1)) - 1)) +#define AM_DISP_MAX (+((1 << (AM_DISP_BITS - 1)) - 1)) struct emitAddrMode { @@ -637,9 +647,9 @@ class emitter static_assert_no_msg(IF_COUNT <= 128); insFormat _idInsFmt : 7; #elif defined(TARGET_LOONGARCH64) - unsigned _idCodeSize : 5; // the instruction(s) size of this instrDesc described. + unsigned _idCodeSize : 5; // the instruction(s) size of this instrDesc described. #elif defined(TARGET_RISCV64) - unsigned _idCodeSize : 6; // the instruction(s) size of this instrDesc described. + unsigned _idCodeSize : 6; // the instruction(s) size of this instrDesc described. #elif defined(TARGET_ARM64) static_assert_no_msg(IF_COUNT <= 1024); insFormat _idInsFmt : 10; @@ -679,7 +689,7 @@ class emitter { } #elif defined(TARGET_RISCV64) - insFormat idInsFmt() const + insFormat idInsFmt() const { NYI_RISCV64("idInsFmt-----unimplemented on RISCV64 yet----"); return (insFormat)0; @@ -689,7 +699,7 @@ class emitter NYI_RISCV64("idInsFmt-----unimplemented on RISCV64 yet----"); } #else - insFormat idInsFmt() const + insFormat idInsFmt() const { return _idInsFmt; } @@ -715,7 +725,7 @@ class emitter private: #if defined(TARGET_XARCH) unsigned _idCodeSize : 4; // size of instruction in bytes. Max size of an Intel instruction is 15 bytes. - opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16, 5=32 + opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16, 5=32 // At this point we have fully consumed first DWORD so that next field // doesn't cross a byte boundary. #elif defined(TARGET_ARM64) @@ -724,7 +734,7 @@ class emitter #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) /* _idOpSize defined below. */ #else - opSize _idOpSize : 2; // operand size: 0=1 , 1=2 , 2=4 , 3=8 + opSize _idOpSize : 2; // operand size: 0=1 , 1=2 , 2=4 , 3=8 #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 // On Amd64, this is where the second DWORD begins @@ -757,9 +767,9 @@ class emitter // loongarch64: 28 bits // risc-v: 28 bits - unsigned _idSmallDsc : 1; // is this a "small" descriptor? - unsigned _idLargeCns : 1; // does a large constant follow? - unsigned _idLargeDsp : 1; // does a large displacement follow? + unsigned _idSmallDsc : 1; // is this a "small" descriptor? + unsigned _idLargeCns : 1; // does a large constant follow? + unsigned _idLargeDsp : 1; // does a large displacement follow? unsigned _idLargeCall : 1; // large call descriptor used // We have several pieces of information we need to encode but which are only applicable @@ -770,15 +780,15 @@ class emitter unsigned _idCustom2 : 1; unsigned _idCustom3 : 1; -#define _idBound _idCustom1 /* jump target / frame offset bound */ -#define _idTlsGD _idCustom2 /* Used to store information related to TLS GD access on linux */ -#define _idNoGC _idCustom3 /* Some helpers don't get recorded in GC tables */ +#define _idBound _idCustom1 /* jump target / frame offset bound */ +#define _idTlsGD _idCustom2 /* Used to store information related to TLS GD access on linux */ +#define _idNoGC _idCustom3 /* Some helpers don't get recorded in GC tables */ #define _idEvexAaaContext (_idCustom3 << 2) | (_idCustom2 << 1) | _idCustom1 /* bits used for the EVEX.aaa context */ #if !defined(TARGET_ARMARCH) unsigned _idCustom4 : 1; -#define _idCallRegPtr _idCustom4 /* IL indirect calls : addr in reg */ +#define _idCallRegPtr _idCustom4 /* IL indirect calls : addr in reg */ #define _idEvexZContext _idCustom4 /* bits used for the EVEX.z context */ #endif // !TARGET_ARMARCH @@ -792,12 +802,12 @@ class emitter #ifdef TARGET_ARM64 - unsigned _idLclVar : 1; // access a local on stack - unsigned _idLclVarPair : 1 // carries information for 2 GC lcl vars. + unsigned _idLclVar : 1; // access a local on stack + unsigned _idLclVarPair : 1 // carries information for 2 GC lcl vars. #endif #ifdef TARGET_LOONGARCH64 - // TODO-LoongArch64: maybe delete on future. + // TODO-LoongArch64: maybe delete on future. opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16 insOpts _idInsOpt : 6; // loongarch options for special: placeholders. e.g emitIns_R_C, also identifying the // accessing a local on stack. @@ -812,11 +822,11 @@ class emitter #endif #ifdef TARGET_ARM - insSize _idInsSize : 2; // size of instruction: 16, 32 or 48 bits - insFlags _idInsFlags : 1; // will this instruction set the flags - unsigned _idLclVar : 1; // access a local on stack + insSize _idInsSize : 2; // size of instruction: 16, 32 or 48 bits + insFlags _idInsFlags : 1; // will this instruction set the flags + unsigned _idLclVar : 1; // access a local on stack unsigned _idLclFPBase : 1; // access a local on stack - SP based offset - insOpts _idInsOpt : 3; // options for Load/Store instructions + insOpts _idInsOpt : 3; // options for Load/Store instructions #endif //////////////////////////////////////////////////////////////////////// @@ -886,7 +896,7 @@ class emitter #define ID_EXTRA_BITS (ID_EXTRA_RELOC_BITS + ID_EXTRA_BITFIELD_BITS + ID_EXTRA_PREV_OFFSET_BITS) -/* Use whatever bits are left over for small constants */ + /* Use whatever bits are left over for small constants */ #define ID_BIT_SMALL_CNS (32 - ID_EXTRA_BITS) C_ASSERT(ID_BIT_SMALL_CNS > 0); @@ -945,9 +955,10 @@ class emitter void checkSizes(); - union idAddrUnion { -// TODO-Cleanup: We should really add a DEBUG-only tag to this union so we can add asserts -// about reading what we think is here, to avoid unexpected corruption issues. + union idAddrUnion + { + // TODO-Cleanup: We should really add a DEBUG-only tag to this union so we can add asserts + // about reading what we think is here, to avoid unexpected corruption issues. #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) emitLclVarAddr iiaLclVar; @@ -1851,137 +1862,137 @@ class emitter #define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue -#define PERFSCORE_THROUGHPUT_2C 2.0f // slower - 2 cycles -#define PERFSCORE_THROUGHPUT_3C 3.0f // slower - 3 cycles -#define PERFSCORE_THROUGHPUT_4C 4.0f // slower - 4 cycles -#define PERFSCORE_THROUGHPUT_5C 5.0f // slower - 5 cycles -#define PERFSCORE_THROUGHPUT_6C 6.0f // slower - 6 cycles -#define PERFSCORE_THROUGHPUT_7C 7.0f // slower - 7 cycles -#define PERFSCORE_THROUGHPUT_8C 8.0f // slower - 8 cycles -#define PERFSCORE_THROUGHPUT_9C 9.0f // slower - 9 cycles -#define PERFSCORE_THROUGHPUT_10C 10.0f // slower - 10 cycles -#define PERFSCORE_THROUGHPUT_11C 10.0f // slower - 10 cycles -#define PERFSCORE_THROUGHPUT_13C 13.0f // slower - 13 cycles -#define PERFSCORE_THROUGHPUT_14C 14.0f // slower - 13 cycles -#define PERFSCORE_THROUGHPUT_16C 16.0f // slower - 13 cycles -#define PERFSCORE_THROUGHPUT_19C 19.0f // slower - 19 cycles -#define PERFSCORE_THROUGHPUT_25C 25.0f // slower - 25 cycles -#define PERFSCORE_THROUGHPUT_33C 33.0f // slower - 33 cycles -#define PERFSCORE_THROUGHPUT_50C 50.0f // slower - 50 cycles -#define PERFSCORE_THROUGHPUT_52C 52.0f // slower - 52 cycles -#define PERFSCORE_THROUGHPUT_57C 57.0f // slower - 57 cycles +#define PERFSCORE_THROUGHPUT_2C 2.0f // slower - 2 cycles +#define PERFSCORE_THROUGHPUT_3C 3.0f // slower - 3 cycles +#define PERFSCORE_THROUGHPUT_4C 4.0f // slower - 4 cycles +#define PERFSCORE_THROUGHPUT_5C 5.0f // slower - 5 cycles +#define PERFSCORE_THROUGHPUT_6C 6.0f // slower - 6 cycles +#define PERFSCORE_THROUGHPUT_7C 7.0f // slower - 7 cycles +#define PERFSCORE_THROUGHPUT_8C 8.0f // slower - 8 cycles +#define PERFSCORE_THROUGHPUT_9C 9.0f // slower - 9 cycles +#define PERFSCORE_THROUGHPUT_10C 10.0f // slower - 10 cycles +#define PERFSCORE_THROUGHPUT_11C 10.0f // slower - 10 cycles +#define PERFSCORE_THROUGHPUT_13C 13.0f // slower - 13 cycles +#define PERFSCORE_THROUGHPUT_14C 14.0f // slower - 13 cycles +#define PERFSCORE_THROUGHPUT_16C 16.0f // slower - 13 cycles +#define PERFSCORE_THROUGHPUT_19C 19.0f // slower - 19 cycles +#define PERFSCORE_THROUGHPUT_25C 25.0f // slower - 25 cycles +#define PERFSCORE_THROUGHPUT_33C 33.0f // slower - 33 cycles +#define PERFSCORE_THROUGHPUT_50C 50.0f // slower - 50 cycles +#define PERFSCORE_THROUGHPUT_52C 52.0f // slower - 52 cycles +#define PERFSCORE_THROUGHPUT_57C 57.0f // slower - 57 cycles #define PERFSCORE_THROUGHPUT_140C 140.0f // slower - 140 cycles #define PERFSCORE_LATENCY_ILLEGAL -1024.0f #define PERFSCORE_LATENCY_ZERO 0.0f -#define PERFSCORE_LATENCY_1C 1.0f -#define PERFSCORE_LATENCY_2C 2.0f -#define PERFSCORE_LATENCY_3C 3.0f -#define PERFSCORE_LATENCY_4C 4.0f -#define PERFSCORE_LATENCY_5C 5.0f -#define PERFSCORE_LATENCY_6C 6.0f -#define PERFSCORE_LATENCY_7C 7.0f -#define PERFSCORE_LATENCY_8C 8.0f -#define PERFSCORE_LATENCY_9C 9.0f -#define PERFSCORE_LATENCY_10C 10.0f -#define PERFSCORE_LATENCY_11C 11.0f -#define PERFSCORE_LATENCY_12C 12.0f -#define PERFSCORE_LATENCY_13C 13.0f -#define PERFSCORE_LATENCY_14C 14.0f -#define PERFSCORE_LATENCY_15C 15.0f -#define PERFSCORE_LATENCY_16C 16.0f -#define PERFSCORE_LATENCY_18C 18.0f -#define PERFSCORE_LATENCY_20C 20.0f -#define PERFSCORE_LATENCY_22C 22.0f -#define PERFSCORE_LATENCY_23C 23.0f -#define PERFSCORE_LATENCY_26C 26.0f -#define PERFSCORE_LATENCY_62C 62.0f -#define PERFSCORE_LATENCY_69C 69.0f +#define PERFSCORE_LATENCY_1C 1.0f +#define PERFSCORE_LATENCY_2C 2.0f +#define PERFSCORE_LATENCY_3C 3.0f +#define PERFSCORE_LATENCY_4C 4.0f +#define PERFSCORE_LATENCY_5C 5.0f +#define PERFSCORE_LATENCY_6C 6.0f +#define PERFSCORE_LATENCY_7C 7.0f +#define PERFSCORE_LATENCY_8C 8.0f +#define PERFSCORE_LATENCY_9C 9.0f +#define PERFSCORE_LATENCY_10C 10.0f +#define PERFSCORE_LATENCY_11C 11.0f +#define PERFSCORE_LATENCY_12C 12.0f +#define PERFSCORE_LATENCY_13C 13.0f +#define PERFSCORE_LATENCY_14C 14.0f +#define PERFSCORE_LATENCY_15C 15.0f +#define PERFSCORE_LATENCY_16C 16.0f +#define PERFSCORE_LATENCY_18C 18.0f +#define PERFSCORE_LATENCY_20C 20.0f +#define PERFSCORE_LATENCY_22C 22.0f +#define PERFSCORE_LATENCY_23C 23.0f +#define PERFSCORE_LATENCY_26C 26.0f +#define PERFSCORE_LATENCY_62C 62.0f +#define PERFSCORE_LATENCY_69C 69.0f #define PERFSCORE_LATENCY_140C 140.0f #define PERFSCORE_LATENCY_400C 400.0f // Intel microcode issue with these instructions -#define PERFSCORE_LATENCY_BRANCH_DIRECT 1.0f // cost of an unconditional branch -#define PERFSCORE_LATENCY_BRANCH_COND 2.0f // includes cost of a possible misprediction +#define PERFSCORE_LATENCY_BRANCH_DIRECT 1.0f // cost of an unconditional branch +#define PERFSCORE_LATENCY_BRANCH_COND 2.0f // includes cost of a possible misprediction #define PERFSCORE_LATENCY_BRANCH_INDIRECT 2.0f // includes cost of a possible misprediction #if defined(TARGET_XARCH) // a read,write or modify from stack location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_2C -#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_2C +#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_2C +#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_2C #define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_5C // a read, write or modify from constant location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_2C -#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_2C +#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_2C +#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_2C #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_5C // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache // plus an extra cost (of 1.0) for a increased chance of a cache miss -#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_3C #define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_6C #elif defined(TARGET_ARM64) || defined(TARGET_ARM) // a read,write or modify from stack location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C // a read, write or modify from constant location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache // plus an extra cost (of 1.0) for a increased chance of a cache miss -#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C -#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C #elif defined(TARGET_LOONGARCH64) // a read,write or modify from stack location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C // a read, write or modify from constant location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache // plus an extra cost (of 1.0) for a increased chance of a cache miss -#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C -#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C #elif defined(TARGET_RISCV64) // a read,write or modify from stack location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C // a read, write or modify from constant location, possible def to use latency from L0 cache -#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C -#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C +#define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache // plus an extra cost (of 1.0) for a increased chance of a cache miss -#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C -#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C -#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C +#define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C +#define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C #endif // TARGET_XXX // Make this an enum: // -#define PERFSCORE_MEMORY_NONE 0 -#define PERFSCORE_MEMORY_READ 1 -#define PERFSCORE_MEMORY_WRITE 2 +#define PERFSCORE_MEMORY_NONE 0 +#define PERFSCORE_MEMORY_READ 1 +#define PERFSCORE_MEMORY_WRITE 2 #define PERFSCORE_MEMORY_READ_WRITE 3 struct insExecutionCharacteristics @@ -2014,7 +2025,8 @@ class emitter instrDescJmp* idjNext; // next jump in the group/method insGroup* idjIG; // containing group - union { + union + { BYTE* idjAddr; // address of jump ins (for patching) } idjTemp; @@ -2037,7 +2049,7 @@ class emitter #else 30; #endif - unsigned idjShort : 1; // is the jump known to be a short one? + unsigned idjShort : 1; // is the jump known to be a short one? unsigned idjKeepLong : 1; // should the jump be kept long? (used for hot to cold and cold to hot jumps) }; @@ -2178,7 +2190,9 @@ class emitter alignas(alignof(T)) char idStorage[sizeof(T)]; public: - inlineInstrDesc() : idDebugInfo(nullptr), idStorage() + inlineInstrDesc() + : idDebugInfo(nullptr) + , idStorage() { static_assert_no_msg((offsetof(inlineInstrDesc, idStorage) - sizeof(instrDescDebugInfo*)) == offsetof(inlineInstrDesc, idDebugInfo)); @@ -2204,7 +2218,7 @@ class emitter #endif // TARGET_ARM insUpdateModes emitInsUpdateMode(instruction ins); - insFormat emitInsModeFormat(instruction ins, insFormat base); + insFormat emitInsModeFormat(instruction ins, insFormat base); static const BYTE emitInsModeFmtTab[]; #ifdef DEBUG @@ -2219,7 +2233,7 @@ class emitter ssize_t emitGetInsDsp(instrDesc* id); ssize_t emitGetInsAmd(instrDesc* id); - ssize_t emitGetInsCIdisp(instrDesc* id); + ssize_t emitGetInsCIdisp(instrDesc* id); unsigned emitGetInsCIargs(instrDesc* id); inline emitAttr emitGetMemOpSize(instrDesc* id) const; @@ -2232,7 +2246,7 @@ class emitter #endif // TARGET_XARCH cnsval_ssize_t emitGetInsSC(const instrDesc* id) const; - unsigned emitInsCount; + unsigned emitInsCount; /************************************************************************/ /* A few routines used for debug display purposes */ @@ -2258,11 +2272,11 @@ class emitter regMaskGpr debugPrevGCrefRegs; regMaskGpr debugPrevByrefRegs; void emitDispInsIndent(); - void emitDispGCDeltaTitle(const char* title); - void emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMaskGpr curRegs); - void emitDispGCVarDelta(); - void emitDispRegPtrListDelta(); - void emitDispGCInfoDelta(); + void emitDispGCDeltaTitle(const char* title); + void emitDispGCRegDelta(const char* title, regMaskGpr prevRegs, regMaskGpr curRegs); + void emitDispGCVarDelta(); + void emitDispRegPtrListDelta(); + void emitDispGCInfoDelta(); void emitDispIGflags(unsigned flags); void emitDispIG(insGroup* ig, @@ -2319,7 +2333,9 @@ class emitter EpilogList* elNext; emitLocation elLoc; - EpilogList() : elNext(nullptr), elLoc() + EpilogList() + : elNext(nullptr) + , elLoc() { } }; @@ -2356,12 +2372,12 @@ class emitter /* Methods to record a code position and later convert to offset */ /************************************************************************/ - unsigned emitFindInsNum(const insGroup* ig, const instrDesc* id) const; + unsigned emitFindInsNum(const insGroup* ig, const instrDesc* id) const; UNATIVE_OFFSET emitFindOffset(const insGroup* ig, unsigned insNum) const; -/************************************************************************/ -/* Members and methods used to issue (encode) instructions. */ -/************************************************************************/ + /************************************************************************/ + /* Members and methods used to issue (encode) instructions. */ + /************************************************************************/ #ifdef DEBUG // If we have started issuing instructions from the list of instrDesc, this is set @@ -2452,9 +2468,9 @@ class emitter #endif // TARGET_LOONGARCH64 || TARGET_RISCV64 instrDesc* emitFirstInstrDesc(BYTE* idData) const; - void emitAdvanceInstrDesc(instrDesc** id, size_t idSize) const; - size_t emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp); - size_t emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp); + void emitAdvanceInstrDesc(instrDesc** id, size_t idSize) const; + size_t emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp); + size_t emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp); bool emitHasFramePtr; @@ -2511,13 +2527,13 @@ class emitter #endif // FEATURE_SIMD regNumber emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, GenTree* src); regNumber emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, GenTree* src1, GenTree* src2); - void emitInsLoadInd(instruction ins, emitAttr attr, regNumber dstReg, GenTreeIndir* mem); - void emitInsStoreInd(instruction ins, emitAttr attr, GenTreeStoreInd* mem); - void emitInsStoreLcl(instruction ins, emitAttr attr, GenTreeLclVarCommon* varNode); + void emitInsLoadInd(instruction ins, emitAttr attr, regNumber dstReg, GenTreeIndir* mem); + void emitInsStoreInd(instruction ins, emitAttr attr, GenTreeStoreInd* mem); + void emitInsStoreLcl(instruction ins, emitAttr attr, GenTreeLclVarCommon* varNode); insFormat emitMapFmtForIns(insFormat fmt, instruction ins); insFormat emitMapFmtAtoM(insFormat fmt); - void emitHandleMemOp(GenTreeIndir* indir, instrDesc* id, insFormat fmt, instruction ins); - void spillIntArgRegsToShadowSlots(); + void emitHandleMemOp(GenTreeIndir* indir, instrDesc* id, insFormat fmt, instruction ins); + void spillIntArgRegsToShadowSlots(); #ifdef TARGET_XARCH bool emitIsInstrWritingToReg(instrDesc* id, regNumber reg); @@ -2604,22 +2620,22 @@ class emitter // non-adaptive alignment on xarch, this points to the first align instruction of the series of align instructions. instrDescAlign* emitAlignLastGroup; - unsigned getLoopSize(insGroup* igLoopHeader, + unsigned getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG_ARG(bool isAlignAdjusted) DEBUG_ARG(UNATIVE_OFFSET containingIGNum) DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum)); // Get the smallest loop size - void emitLoopAlignment(DEBUG_ARG1(bool isPlacedBehindJmp)); - bool emitEndsWithAlignInstr(); // Validate if newLabel is appropriate - bool emitSetLoopBackEdge(const BasicBlock* loopTopBlock); + void emitLoopAlignment(DEBUG_ARG1(bool isPlacedBehindJmp)); + bool emitEndsWithAlignInstr(); // Validate if newLabel is appropriate + bool emitSetLoopBackEdge(const BasicBlock* loopTopBlock); void emitLoopAlignAdjustments(); // Predict if loop alignment is needed and make appropriate adjustments - unsigned emitCalculatePaddingForLoopAlignment(insGroup* ig, + unsigned emitCalculatePaddingForLoopAlignment(insGroup* ig, size_t offset DEBUG_ARG(bool isAlignAdjusted) DEBUG_ARG(UNATIVE_OFFSET containingIGNum) DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum)); - void emitLoopAlign(unsigned paddingBytes, bool isFirstAlign DEBUG_ARG(bool isPlacedBehindJmp)); - void emitLongLoopAlign(unsigned alignmentBoundary DEBUG_ARG(bool isPlacedBehindJmp)); + void emitLoopAlign(unsigned paddingBytes, bool isFirstAlign DEBUG_ARG(bool isPlacedBehindJmp)); + void emitLongLoopAlign(unsigned alignmentBoundary DEBUG_ARG(bool isPlacedBehindJmp)); instrDescAlign* emitAlignInNextIG(instrDescAlign* alignInstr); - void emitConnectAlignInstrWithCurIG(); + void emitConnectAlignInstrWithCurIG(); #endif @@ -2692,7 +2708,7 @@ class emitter void emitSetSecondRetRegGCType(instrDescCGCA* id, emitAttr secondRetSize); #endif // MULTIREG_HAS_SECOND_GC_RET - static void emitEncodeCallGCregs(regMaskGpr regs, instrDesc* id); + static void emitEncodeCallGCregs(regMaskGpr regs, instrDesc* id); static unsigned emitDecodeCallGCregs(instrDesc* id); unsigned emitNxtIGnum; @@ -2716,8 +2732,8 @@ class emitter insGroup* emitAllocAndLinkIG(); insGroup* emitAllocIG(); - void emitInitIG(insGroup* ig); - void emitInsertIGAfter(insGroup* insertAfterIG, insGroup* ig); + void emitInitIG(insGroup* ig); + void emitInsertIGAfter(insGroup* insertAfterIG, insGroup* ig); void emitNewIG(); @@ -2732,9 +2748,9 @@ class emitter static bool emitJmpInstHasNoCode(instrDesc* id); #endif - void emitGenIG(insGroup* ig); + void emitGenIG(insGroup* ig); insGroup* emitSavIG(bool emitAdd = false); - void emitNxtIG(bool extend = false); + void emitNxtIG(bool extend = false); #ifdef TARGET_ARM64 void emitRemoveLastInstruction(); @@ -2864,8 +2880,8 @@ class emitter // Mark this instruction group as having a label; return the new instruction group. // Sets the emitter's record of the currently live GC variables // and registers. - void* emitAddLabel(VARSET_VALARG_TP GCvars, - regMaskGpr gcrefRegs, + void* emitAddLabel(VARSET_VALARG_TP GCvars, + regMaskGpr gcrefRegs, regMaskGpr byrefRegs DEBUG_ARG(BasicBlock* block = nullptr)); // Same as above, except the label is added and is conceptually "inline" in @@ -2873,7 +2889,7 @@ class emitter // continues to track GC info as if there was no label. void* emitAddInlineLabel(); - void emitPrintLabel(const insGroup* ig) const; + void emitPrintLabel(const insGroup* ig) const; const char* emitLabelString(const insGroup* ig) const; #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) @@ -3096,7 +3112,7 @@ class emitter return (offs >= emitGCrFrameOffsMin) && (offs < emitGCrFrameOffsMax); } - static instruction emitJumpKindToIns(emitJumpKind jumpKind); + static instruction emitJumpKindToIns(emitJumpKind jumpKind); static emitJumpKind emitInsToJumpKind(instruction ins); static emitJumpKind emitReverseJumpKind(emitJumpKind jumpKind); @@ -3161,7 +3177,8 @@ class emitter bool emitSimpleStkUsed; // using the "simple" stack table? - union { + union + { struct // if emitSimpleStkUsed==true { @@ -3209,8 +3226,8 @@ class emitter #ifdef DEBUG const char* emitGetFrameReg(); - void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); - void emitDispGprRegSet(regMaskGpr regs); + void emitDispRegSet(regNumber firstReg, regNumber lastReg, regMaskOnlyOne regs); + void emitDispGprRegSet(regMaskGpr regs); void emitDispFloatRegSet(regMaskFloat regs); #ifdef FEATURE_MASKED_HW_INTRINSICS void emitDispMaskRegSet(regMaskPredicate regs); @@ -3281,7 +3298,11 @@ class emitter UNATIVE_OFFSET dsdOffs; UNATIVE_OFFSET alignment; // in bytes, defaults to 4 - dataSecDsc() : dsdList(nullptr), dsdLast(nullptr), dsdOffs(0), alignment(4) + dataSecDsc() + : dsdList(nullptr) + , dsdLast(nullptr) + , dsdOffs(0) + , alignment(4) { } }; @@ -3299,9 +3320,9 @@ class emitter COMP_HANDLE emitCmpHandle; -/************************************************************************/ -/* Helpers for interface to EE */ -/************************************************************************/ + /************************************************************************/ + /* Helpers for interface to EE */ + /************************************************************************/ #ifdef DEBUG @@ -3311,25 +3332,25 @@ class emitter #define emitRecordRelocationWithAddlDelta(location, target, fRelocType, addlDelta) \ emitRecordRelocationHelp(location, target, fRelocType, #fRelocType, addlDelta) - void emitRecordRelocationHelp(void* location, /* IN */ - void* target, /* IN */ - uint16_t fRelocType, /* IN */ - const char* relocTypeName, /* IN */ - int32_t addlDelta = 0); /* IN */ + void emitRecordRelocationHelp(void* location, /* IN */ + void* target, /* IN */ + uint16_t fRelocType, /* IN */ + const char* relocTypeName, /* IN */ + int32_t addlDelta = 0); /* IN */ #else // !DEBUG void emitRecordRelocationWithAddlDelta(void* location, /* IN */ void* target, /* IN */ uint16_t fRelocType, /* IN */ - int32_t addlDelta) /* IN */ + int32_t addlDelta) /* IN */ { emitRecordRelocation(location, target, fRelocType, addlDelta); } - void emitRecordRelocation(void* location, /* IN */ - void* target, /* IN */ - uint16_t fRelocType, /* IN */ + void emitRecordRelocation(void* location, /* IN */ + void* target, /* IN */ + uint16_t fRelocType, /* IN */ int32_t addlDelta = 0); /* IN */ #endif // !DEBUG @@ -3349,9 +3370,9 @@ class emitter CORINFO_SIG_INFO* emitScratchSigInfo; #endif // DEBUG -/************************************************************************/ -/* Logic to collect and display statistics */ -/************************************************************************/ + /************************************************************************/ + /* Logic to collect and display statistics */ + /************************************************************************/ #if EMITTER_STATS @@ -3488,10 +3509,10 @@ class emitter } #endif // EMITTER_STATS -/************************************************************************* - * - * Define any target-dependent emitter members. - */ + /************************************************************************* + * + * Define any target-dependent emitter members. + */ #include "emitdef.h" diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 0221ca80681cc3..de1b46f9d52d80 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -700,8 +700,8 @@ emitter::insFormat emitter::emitInsFormat(instruction ins) } // INST_FP is 1 -#define LD 2 -#define ST 4 +#define LD 2 +#define ST 4 #define CMP 8 // clang-format off @@ -1708,10 +1708,10 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg) * Add an instruction referencing a register and a constant. */ -void emitter::emitIns_R_I(instruction ins, - emitAttr attr, - regNumber reg, - target_ssize_t imm, +void emitter::emitIns_R_I(instruction ins, + emitAttr attr, + regNumber reg, + target_ssize_t imm, insFlags flags /* = INS_FLAGS_DONT_CARE */ DEBUGARG(GenTreeFlags gtFlags)) { @@ -1738,7 +1738,7 @@ void emitter::emitIns_R_I(instruction ins, ins = INS_sub; else // ins == INS_sub ins = INS_add; - imm = -imm; + imm = -imm; } fmt = IF_T1_J0; sf = INS_FLAGS_SET; @@ -2607,7 +2607,7 @@ void emitter::emitIns_R_R_I(instruction ins, ins = INS_sub; else ins = INS_add; - imm = -imm; + imm = -imm; } fmt = IF_T1_G; sf = INS_FLAGS_SET; @@ -2621,7 +2621,7 @@ void emitter::emitIns_R_R_I(instruction ins, ins = INS_sub; else ins = INS_add; - imm = -imm; + imm = -imm; } // Use Thumb-1 encoding emitIns_R_I(ins, attr, reg1, imm, flags); @@ -2982,9 +2982,9 @@ void emitter::emitIns_R_R_I(instruction ins, } } } - // - // If we did not find a thumb-1 encoding above - // + // + // If we did not find a thumb-1 encoding above + // COMMON_THUMB2_LDST: assert(fmt == IF_NONE); @@ -3185,8 +3185,8 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_mul: if (insMustSetFlags(flags)) { - assert(reg1 != - REG_PC); // VM debugging single stepper doesn't support PC register with this instruction. + assert(reg1 != REG_PC); // VM debugging single stepper doesn't support PC register with this + // instruction. assert(reg2 != REG_PC); assert(reg3 != REG_PC); @@ -4837,7 +4837,7 @@ void emitter::emitIns_Call(EmitCallType callType, if (m_debugInfoSize > 0) { INDEBUG(id->idDebugOnlyInfo()->idCallSig = sigInfo); - id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token + id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token } #ifdef LATE_DISASM @@ -5237,7 +5237,7 @@ unsigned emitter::emitOutput_Thumb1Instr(BYTE* dst, code_t code) unsigned emitter::emitOutput_Thumb2Instr(BYTE* dst, code_t code) { unsigned short word1 = (code >> 16) & 0xffff; - unsigned short word2 = (code)&0xffff; + unsigned short word2 = (code) & 0xffff; assert((code_t)((word1 << 16) | word2) == code); #ifdef DEBUG @@ -5343,7 +5343,7 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) if (dstOffs <= srcOffs) { -/* This is a backward jump - distance is known at this point */ + /* This is a backward jump - distance is known at this point */ #if DEBUG_EMIT if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) @@ -5732,7 +5732,7 @@ BYTE* emitter::emitOutputIT(BYTE* dst, instruction ins, insFormat fmt, code_t co #endif // FEATURE_ITINSTRUCTION /***************************************************************************** -* + * * Append the machine code corresponding to the given instruction descriptor * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' * is the instruction group that contains the instruction. Updates '*dp' to @@ -6562,9 +6562,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; - /********************************************************************/ - /* oops */ - /********************************************************************/ + /********************************************************************/ + /* oops */ + /********************************************************************/ default: diff --git a/src/coreclr/jit/emitarm.h b/src/coreclr/jit/emitarm.h index cf4754fc9d32d5..83541a36427516 100644 --- a/src/coreclr/jit/emitarm.h +++ b/src/coreclr/jit/emitarm.h @@ -81,7 +81,7 @@ bool emitInsIsStore(instruction ins); bool emitInsIsLoadOrStore(instruction ins); emitter::insFormat emitInsFormat(instruction ins); -emitter::code_t emitInsCode(instruction ins, insFormat fmt); +emitter::code_t emitInsCode(instruction ins, insFormat fmt); // Generate code for a load or store operation and handle the case // of contained GT_LEA op1 with [base + index<idInsOpt())) { - assert((emitGetInsSC(id) > 0) || - (id->idReg2() == REG_ZR)); // REG_ZR encodes SP and we allow a shift of zero + assert((emitGetInsSC(id) > 0) || (id->idReg2() == REG_ZR)); // REG_ZR encodes SP and we allow a shift of + // zero } break; @@ -967,7 +967,7 @@ bool emitter::emitInsMayWriteToGCReg(instrDesc* id) switch (fmt) { - // These are the formats with "destination" registers: + // These are the formats with "destination" registers: case IF_DI_1B: // DI_1B X........hwiiiii iiiiiiiiiiiddddd Rd imm(i16,hw) case IF_DI_1D: // DI_1D X........Nrrrrrr ssssss.....ddddd Rd imm(N,r,s) @@ -1031,7 +1031,7 @@ bool emitter::emitInsMayWriteToGCReg(instrDesc* id) // Tracked GC pointers cannot be placed into the SIMD registers. return false; - // These are the load/store formats with "target" registers: + // These are the load/store formats with "target" registers: case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn @@ -1471,8 +1471,8 @@ emitter::insFormat emitter::emitInsFormat(instruction ins) return insFormats[ins]; } -#define LD 1 -#define ST 2 +#define LD 1 +#define ST 2 #define CMP 4 #define RSH 8 #define WID 16 @@ -1733,8 +1733,8 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) }; // clang-format on - const static insFormat formatEncode9[9] = {IF_DR_2E, IF_DR_2G, IF_DI_1B, IF_DI_1D, IF_DV_3C, - IF_DV_2B, IF_DV_2C, IF_DV_2E, IF_DV_2F}; + const static insFormat formatEncode9[9] = {IF_DR_2E, IF_DR_2G, IF_DI_1B, IF_DI_1D, IF_DV_3C, + IF_DV_2B, IF_DV_2C, IF_DV_2E, IF_DV_2F}; const static insFormat formatEncode6A[6] = {IF_DR_3A, IF_DR_3B, IF_DR_3C, IF_DI_2A, IF_DV_3A, IF_DV_3E}; const static insFormat formatEncode6B[6] = {IF_LS_2D, IF_LS_3F, IF_LS_2E, IF_LS_2F, IF_LS_3G, IF_LS_2G}; const static insFormat formatEncode5A[5] = {IF_LS_2A, IF_LS_2B, IF_LS_2C, IF_LS_3A, IF_LS_1A}; @@ -3748,13 +3748,13 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts o * Add an instruction referencing a register and a constant. */ -void emitter::emitIns_R_I(instruction ins, - emitAttr attr, - regNumber reg, - ssize_t imm, - insOpts opt, /* = INS_OPTS_NONE */ +void emitter::emitIns_R_I(instruction ins, + emitAttr attr, + regNumber reg, + ssize_t imm, + insOpts opt, /* = INS_OPTS_NONE */ insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */ - DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = GTF_EMPTY */)) + DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = GTF_EMPTY */)) { emitAttr size = EA_SIZE(attr); emitAttr elemsize = EA_UNKNOWN; @@ -3940,8 +3940,8 @@ void emitter::emitIns_R_I(instruction ins, // First try the standard 'byteShifted immediate' imm(i8,bySh) bsi.immBSVal = 0; canEncode = canEncodeByteShiftedImm(imm, elemsize, - (ins == INS_mvni), // mvni supports the ones shifting variant (aka MSL) - &bsi); + (ins == INS_mvni), // mvni supports the ones shifting variant (aka MSL) + &bsi); if (canEncode) { imm = bsi.immBSVal; @@ -4955,8 +4955,8 @@ void emitter::emitIns_R_I_I(instruction ins, regNumber reg, ssize_t imm1, ssize_t imm2, - insOpts opt /* = INS_OPTS_NONE */ - DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = 0 */)) + insOpts opt /* = INS_OPTS_NONE */ + DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = 0 */)) { emitAttr size = EA_SIZE(attr); insFormat fmt = IF_NONE; @@ -5792,15 +5792,15 @@ void emitter::emitIns_R_R_F( } /***************************************************************************** -* -* Add an instruction referencing two registers and a constant. -* Also checks for a large immediate that needs a second instruction -* and will load it in reg1 -* -* - Supports instructions: add, adds, sub, subs, and, ands, eor and orr -* - Requires that reg1 is a general register and not SP or ZR -* - Requires that reg1 != reg2 -*/ + * + * Add an instruction referencing two registers and a constant. + * Also checks for a large immediate that needs a second instruction + * and will load it in reg1 + * + * - Supports instructions: add, adds, sub, subs, and, ands, eor and orr + * - Requires that reg1 is a general register and not SP or ZR + * - Requires that reg1 != reg2 + */ void emitter::emitIns_R_R_Imm(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, ssize_t imm) { assert(isGeneralRegister(reg1)); @@ -6554,7 +6554,7 @@ void emitter::emitIns_R_R_R_I_LdStPair(instruction ins, int varx1, int varx2, int offs1, - int offs2 DEBUG_ARG(unsigned var1RefsOffs) DEBUG_ARG(unsigned var2RefsOffs)) + int offs2 DEBUG_ARG(unsigned var1RefsOffs) DEBUG_ARG(unsigned var2RefsOffs)) { assert((ins == INS_stp) || (ins == INS_ldp)); emitAttr size = EA_SIZE(attr); @@ -7147,7 +7147,7 @@ void emitter::emitIns_R_R_R_Ext(instruction ins, regNumber reg1, regNumber reg2, regNumber reg3, - insOpts opt, /* = INS_OPTS_NONE */ + insOpts opt, /* = INS_OPTS_NONE */ int shiftAmount) /* = -1 -- unset */ { emitAttr size = EA_SIZE(attr); @@ -8482,9 +8482,9 @@ void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNu } // This generates code to populate the access for TLS on linux -void emitter::emitIns_Adrp_Ldr_Add(emitAttr attr, - regNumber reg1, - regNumber reg2, +void emitter::emitIns_Adrp_Ldr_Add(emitAttr attr, + regNumber reg1, + regNumber reg2, ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { assert(emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)); @@ -8537,9 +8537,9 @@ void emitter::emitIns_Adrp_Ldr_Add(emitAttr attr, } // This computes address from the immediate which is relocatable. -void emitter::emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber ireg, +void emitter::emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber ireg, ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { assert(EA_IS_RELOC(attr)); @@ -9109,7 +9109,7 @@ void emitter::emitIns_Call(EmitCallType callType, if (m_debugInfoSize > 0) { INDEBUG(id->idDebugOnlyInfo()->idCallSig = sigInfo); - id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token + id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token } #ifdef LATE_DISASM @@ -10439,9 +10439,9 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) } /***************************************************************************** -* -* Output a short branch instruction. -*/ + * + * Output a short branch instruction. + */ BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) { code_t code = emitInsCode(ins, fmt); @@ -10504,9 +10504,9 @@ BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, } /***************************************************************************** -* -* Output a short address instruction. -*/ + * + * Output a short address instruction. + */ BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) { ssize_t loBits = (distVal & 3); @@ -10534,9 +10534,9 @@ BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, } /***************************************************************************** -* -* Output a short constant instruction. -*/ + * + * Output a short constant instruction. + */ BYTE* emitter::emitOutputShortConstant( BYTE* dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) { @@ -10769,7 +10769,7 @@ unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) } /***************************************************************************** -* + * * Append the machine code corresponding to the given instruction descriptor * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' * is the instruction group that contains the instruction. Updates '*dp' to @@ -11165,8 +11165,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) code = emitInsCode(ins, fmt); code |= insEncodeReg_Rd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 - : IMAGE_REL_ARM64_PAGEBASE_REL21); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, + id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 + : IMAGE_REL_ARM64_PAGEBASE_REL21); } else { @@ -11209,8 +11210,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { assert(sz == sizeof(instrDesc)); assert(id->idAddr()->iiaAddr != nullptr); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 - : IMAGE_REL_ARM64_PAGEOFFSET_12A); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, + id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 + : IMAGE_REL_ARM64_PAGEOFFSET_12A); } break; @@ -12357,7 +12359,7 @@ void emitter::emitDispCond(insCond cond) { const static char* armCond[16] = {"eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", "hi", "ls", "ge", "lt", "gt", "le", "AL", "NV"}; // The last two are invalid - unsigned imm = (unsigned)cond; + unsigned imm = (unsigned)cond; assert((0 <= imm) && (imm < ArrLen(armCond))); printf(armCond[imm]); } @@ -12370,7 +12372,7 @@ void emitter::emitDispFlags(insCflags flags) { const static char* armFlags[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; - unsigned imm = (unsigned)flags; + unsigned imm = (unsigned)flags; assert((0 <= imm) && (imm < ArrLen(armFlags))); printf(armFlags[imm]); } @@ -12383,7 +12385,7 @@ void emitter::emitDispBarrier(insBarrier barrier) { const static char* armBarriers[16] = {"#0", "oshld", "oshst", "osh", "#4", "nshld", "nshst", "nsh", "#8", "ishld", "ishst", "ish", "#12", "ld", "st", "sy"}; - unsigned imm = (unsigned)barrier; + unsigned imm = (unsigned)barrier; assert((0 <= imm) && (imm < ArrLen(armBarriers))); printf(armBarriers[imm]); } @@ -14682,9 +14684,9 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins switch (insFmt) { - // - // Branch Instructions - // + // + // Branch Instructions + // case IF_BI_0A: // b, bl_local case IF_BI_0C: // bl, b_tail @@ -14937,9 +14939,9 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; } - // - // Load/Store Instructions - // + // + // Load/Store Instructions + // case IF_LS_1A: // ldr, ldrsw (literal, pc relative immediate) result.insThroughput = PERFSCORE_THROUGHPUT_1C; @@ -16618,7 +16620,7 @@ bool emitter::OptimizeLdrStr(instruction ins, insFormat fmt, bool localVar, int varx, - int offs DEBUG_ARG(bool useRsvdReg)) + int offs DEBUG_ARG(bool useRsvdReg)) { assert(ins == INS_ldr || ins == INS_str); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 4d3aec30b32a2b..eda020eb002ce5 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -124,21 +124,21 @@ enum RegisterOrder /************************************************************************/ private: -bool emitInsIsCompare(instruction ins); -bool emitInsIsLoad(instruction ins); -bool emitInsIsStore(instruction ins); -bool emitInsIsLoadOrStore(instruction ins); -bool emitInsIsVectorRightShift(instruction ins); -bool emitInsIsVectorLong(instruction ins); -bool emitInsIsVectorNarrow(instruction ins); -bool emitInsIsVectorWide(instruction ins); -bool emitInsDestIsOp2(instruction ins); +bool emitInsIsCompare(instruction ins); +bool emitInsIsLoad(instruction ins); +bool emitInsIsStore(instruction ins); +bool emitInsIsLoadOrStore(instruction ins); +bool emitInsIsVectorRightShift(instruction ins); +bool emitInsIsVectorLong(instruction ins); +bool emitInsIsVectorNarrow(instruction ins); +bool emitInsIsVectorWide(instruction ins); +bool emitInsDestIsOp2(instruction ins); emitAttr emitInsTargetRegSize(instrDesc* id); emitAttr emitInsLoadStoreSize(instrDesc* id); emitter::insFormat emitInsFormat(instruction ins); -emitter::code_t emitInsCode(instruction ins, insFormat fmt); -emitter::code_t emitInsCodeSve(instruction ins, insFormat fmt); +emitter::code_t emitInsCode(instruction ins, insFormat fmt); +emitter::code_t emitInsCodeSve(instruction ins, insFormat fmt); // Generate code for a load or store operation and handle the case of contained GT_LEA op1 with [base + index<(id->idReg1()); // ddddd - code |= insEncodeSimm<9, 5>(imm1); // iiiii - code |= insEncodeSimm<20, 16>(imm2); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } + { + ssize_t imm1; + ssize_t imm2; + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V<4, 0>(id->idReg1()); // ddddd + code |= insEncodeSimm<9, 5>(imm1); // iiiii + code |= insEncodeSimm<20, 16>(imm2); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register // increment) @@ -11451,16 +11451,16 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) case IF_SVE_HM_2A: // ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate // (predicated) - { - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V<4, 0>(id->idReg1()); // ddddd - code |= insEncodeReg_P<12, 10>(id->idReg2()); // ggg - code |= insEncodeSveSmallFloatImm(imm); // i - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - } - break; + { + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V<4, 0>(id->idReg1()); // ddddd + code |= insEncodeReg_P<12, 10>(id->idReg2()); // ggg + code |= insEncodeSveSmallFloatImm(imm); // i + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + } + break; case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient imm = emitGetInsSC(id); @@ -13443,17 +13443,17 @@ void emitter::emitInsSveSanityCheck(instrDesc* id) case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate // increment) - { - ssize_t imm1; - ssize_t imm2; - insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<5>(imm1)); // iiiii - assert(isValidSimm<5>(imm2)); // iiiii - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - break; - } + { + ssize_t imm1; + ssize_t imm2; + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isValidSimm<5>(imm1)); // iiiii + assert(isValidSimm<5>(imm2)); // iiiii + assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx + break; + } case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register // increment) @@ -14579,37 +14579,37 @@ void emitter::emitDispInsSveHelp(instrDesc* id) // ., #, # case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate // increment) - { - ssize_t imm1; - ssize_t imm2; - insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImm(imm1, true); // iiiii - emitDispImm(imm2, false); // iiiii - break; - } + { + ssize_t imm1; + ssize_t imm2; + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispImm(imm1, true); // iiiii + emitDispImm(imm2, false); // iiiii + break; + } // ., #, case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register // increment) - { - const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImm(emitGetInsSC(id), true); // iiiii - emitDispReg(id->idReg2(), intRegSize, false); // mmmmm - break; - } + { + const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispImm(emitGetInsSC(id), true); // iiiii + emitDispReg(id->idReg2(), intRegSize, false); // mmmmm + break; + } // ., , # case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate // increment) - { - const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispReg(id->idReg2(), intRegSize, true); // mmmmm - emitDispImm(emitGetInsSC(id), false); // iiiii - break; - } + { + const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispReg(id->idReg2(), intRegSize, true); // mmmmm + emitDispImm(emitGetInsSC(id), false); // iiiii + break; + } // .H, .B, .B case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 8686aa924134aa..c389efcd5caa04 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -51,9 +51,9 @@ const emitJumpKind emitReverseJumpKinds[] = { } /***************************************************************************** -* Look up the jump kind for an instruction. It better be a conditional -* branch instruction with a jump kind! -*/ + * Look up the jump kind for an instruction. It better be a conditional + * branch instruction with a jump kind! + */ /*static*/ emitJumpKind emitter::emitInsToJumpKind(instruction ins) { @@ -2047,9 +2047,9 @@ void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNu } // This computes address from the immediate which is relocatable. -void emitter::emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber reg, +void emitter::emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber reg, ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { assert(EA_IS_RELOC(attr)); // EA_PTR_DSP_RELOC @@ -2381,8 +2381,8 @@ void emitter::emitIns_I_la(emitAttr size, regNumber reg, ssize_t imm) void emitter::emitIns_Call(EmitCallType callType, CORINFO_METHOD_HANDLE methHnd, INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE - void* addr, - ssize_t argSize, + void* addr, + ssize_t argSize, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, regMaskTP gcrefRegs, @@ -2786,9 +2786,9 @@ void emitter::emitJumpDistBind() B_DIST_SMALL_MAX_POS - emitCounts_INS_OPTS_J * (3 << 2); // the max placeholder sizeof(INS_OPTS_JIRL) - sizeof(INS_OPTS_J). -/*****************************************************************************/ -/* If the default small encoding is not enough, we start again here. */ -/*****************************************************************************/ + /*****************************************************************************/ + /* If the default small encoding is not enough, we start again here. */ + /*****************************************************************************/ AGAIN: @@ -2819,7 +2819,7 @@ void emitter::emitJumpDistBind() UNATIVE_OFFSET dstOffs; NATIVE_OFFSET jmpDist; // the relative jump distance, as it will be encoded -/* Make sure the jumps are properly ordered */ + /* Make sure the jumps are properly ordered */ #ifdef DEBUG assert(lastSJ == nullptr || lastIG != jmp->idjIG || lastSJ->idjOffs < (jmp->idjOffs + adjSJ)); @@ -2997,8 +2997,8 @@ void emitter::emitJumpDistBind() instruction ins = jmp->idIns(); assert((INS_bceqz <= ins) && (ins <= INS_bl)); - if (ins < - INS_beqz) // bceqz/bcnez/beq/bne/blt/bltu/bge/bgeu < beqz < bnez // See instrsloongarch64.h. + if (ins < INS_beqz) // bceqz/bcnez/beq/bne/blt/bltu/bge/bgeu < beqz < bnez // See + // instrsloongarch64.h. { if ((jmpDist + emitCounts_INS_OPTS_J * 4) < 0x8000000) { @@ -3085,8 +3085,8 @@ void emitter::emitJumpDistBind() instruction ins = jmp->idIns(); assert((INS_bceqz <= ins) && (ins <= INS_bl)); - if (ins < - INS_beqz) // bceqz/bcnez/beq/bne/blt/bltu/bge/bgeu < beqz < bnez // See instrsloongarch64.h. + if (ins < INS_beqz) // bceqz/bcnez/beq/bne/blt/bltu/bge/bgeu < beqz < bnez // See + // instrsloongarch64.h. { if ((jmpDist + emitCounts_INS_OPTS_J * 4) < 0x8000000) { @@ -3181,7 +3181,7 @@ void emitter::emitJumpDistBind() } /***************************************************************************** -* + * * Append the machine code corresponding to the given instruction descriptor * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' * is the instruction group that contains the instruction. Updates '*dp' to diff --git a/src/coreclr/jit/emitloongarch64.h b/src/coreclr/jit/emitloongarch64.h index 11a2f9ee90710e..135f9cf4006735 100644 --- a/src/coreclr/jit/emitloongarch64.h +++ b/src/coreclr/jit/emitloongarch64.h @@ -104,10 +104,10 @@ enum insDisasmFmt #endif }; -code_t emitGetInsMask(int ins); +code_t emitGetInsMask(int ins); insDisasmFmt emitGetInsFmt(instruction ins); -void emitDispInst(instruction ins); -void emitDisInsName(code_t code, const BYTE* addr, instrDesc* id); +void emitDispInst(instruction ins); +void emitDisInsName(code_t code, const BYTE* addr, instrDesc* id); #endif // DEBUG void emitIns_J_cond_la(instruction ins, BasicBlock* dst, regNumber reg1 = REG_R0, regNumber reg2 = REG_R0); @@ -316,9 +316,9 @@ void emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) void emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs); -void emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber reg, +void emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber reg, ssize_t disp DEBUGARG(size_t targetHandle = 0) DEBUGARG(GenTreeFlags gtFlags = GTF_EMPTY)); enum EmitCallType @@ -343,8 +343,8 @@ enum EmitCallType void emitIns_Call(EmitCallType callType, CORINFO_METHOD_HANDLE methHnd, INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE - void* addr, - ssize_t argSize, + void* addr, + ssize_t argSize, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, regMaskTP gcrefRegs, diff --git a/src/coreclr/jit/emitpub.h b/src/coreclr/jit/emitpub.h index 9a6db100265975..5c524812a9755b 100644 --- a/src/coreclr/jit/emitpub.h +++ b/src/coreclr/jit/emitpub.h @@ -16,24 +16,24 @@ void emitBegFN(bool hasFramePtr , bool checkAlign #endif - ); +); void emitEndFN(); void emitComputeCodeSizes(); -unsigned emitEndCodeGen(Compiler* comp, - bool contTrkPtrLcls, - bool fullyInt, - bool fullPtrMap, - unsigned xcptnsCount, - unsigned* prologSize, - unsigned* epilogSize, - void** codeAddr, - void** codeAddrRW, - void** coldCodeAddr, - void** coldCodeAddrRW, - void** consAddr, +unsigned emitEndCodeGen(Compiler* comp, + bool contTrkPtrLcls, + bool fullyInt, + bool fullPtrMap, + unsigned xcptnsCount, + unsigned* prologSize, + unsigned* epilogSize, + void** codeAddr, + void** codeAddrRW, + void** coldCodeAddr, + void** coldCodeAddrRW, + void** consAddr, void** consAddrRW DEBUGARG(unsigned* instrCount)); /************************************************************************/ @@ -102,11 +102,11 @@ UNATIVE_OFFSET emitDataSize(); /************************************************************************/ #ifdef TARGET_XARCH -static bool instrIs3opImul(instruction ins); -static bool instrIsExtendedReg3opImul(instruction ins); -static bool instrHasImplicitRegPairDest(instruction ins); -static void check3opImulValues(); -static regNumber inst3opImulReg(instruction ins); +static bool instrIs3opImul(instruction ins); +static bool instrIsExtendedReg3opImul(instruction ins); +static bool instrHasImplicitRegPairDest(instruction ins); +static void check3opImulValues(); +static regNumber inst3opImulReg(instruction ins); static instruction inst3opImulForReg(regNumber reg); #endif diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index a3c2a482e4ebcc..fa7f73a881d921 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -988,9 +988,9 @@ void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNu } // This computes address from the immediate which is relocatable. -void emitter::emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber reg, +void emitter::emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber reg, ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { assert(EA_IS_RELOC(attr)); // EA_PTR_DSP_RELOC @@ -1290,8 +1290,8 @@ void emitter::emitLoadImmediate(emitAttr size, regNumber reg, ssize_t imm) void emitter::emitIns_Call(EmitCallType callType, CORINFO_METHOD_HANDLE methHnd, INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE - void* addr, - ssize_t argSize, + void* addr, + ssize_t argSize, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, regMaskTP gcrefRegs, @@ -1760,9 +1760,9 @@ void emitter::emitJumpDistBind() emitCounts_INS_OPTS_J * (6 << 2); // the max placeholder sizeof(INS_OPTS_JALR) - sizeof(INS_OPTS_J) NATIVE_OFFSET psd = B_DIST_SMALL_MAX_POS - maxPlaceholderSize; -/*****************************************************************************/ -/* If the default small encoding is not enough, we start again here. */ -/*****************************************************************************/ + /*****************************************************************************/ + /* If the default small encoding is not enough, we start again here. */ + /*****************************************************************************/ AGAIN: @@ -1793,7 +1793,7 @@ void emitter::emitJumpDistBind() UNATIVE_OFFSET dstOffs; NATIVE_OFFSET jmpDist; // the relative jump distance, as it will be encoded -/* Make sure the jumps are properly ordered */ + /* Make sure the jumps are properly ordered */ #ifdef DEBUG assert(lastSJ == nullptr || lastIG != jmp->idjIG || lastSJ->idjOffs < (jmp->idjOffs + adjSJ)); @@ -1948,8 +1948,8 @@ void emitter::emitJumpDistBind() instruction ins = jmp->idIns(); assert((INS_jal <= ins) && (ins <= INS_bgeu)); - if (ins > INS_jalr || - (ins < INS_jalr && ins > INS_j)) // jal < beqz < bnez < jalr < beq/bne/blt/bltu/bge/bgeu + if (ins > INS_jalr || (ins < INS_jalr && ins > INS_j)) // jal < beqz < bnez < jalr < + // beq/bne/blt/bltu/bge/bgeu { if (isValidSimm13(jmpDist + maxPlaceholderSize)) { @@ -2022,8 +2022,8 @@ void emitter::emitJumpDistBind() instruction ins = jmp->idIns(); assert((INS_jal <= ins) && (ins <= INS_bgeu)); - if (ins > INS_jalr || - (ins < INS_jalr && ins > INS_j)) // jal < beqz < bnez < jalr < beq/bne/blt/bltu/bge/bgeu + if (ins > INS_jalr || (ins < INS_jalr && ins > INS_j)) // jal < beqz < bnez < jalr < + // beq/bne/blt/bltu/bge/bgeu { if (isValidSimm13(jmpDist + maxPlaceholderSize)) { @@ -2966,7 +2966,7 @@ BYTE* emitter::emitOutputInstr_OptsRcNoReloc(BYTE* dst, instruction* ins, unsign const regNumber rsvdReg = codeGen->rsGetRsvdReg(); const instruction lastIns = (*ins == INS_jal) ? (*ins = INS_addi) : *ins; - const ssize_t high = immediate >> 11; + const ssize_t high = immediate >> 11; dst += emitOutput_UTypeInstr(dst, INS_lui, rsvdReg, UpperNBitsOfWordSignExtend<20>(high)); dst += emitOutput_ITypeInstr(dst, INS_addi, rsvdReg, rsvdReg, LowerNBitsOfWord<12>(high)); diff --git a/src/coreclr/jit/emitriscv64.h b/src/coreclr/jit/emitriscv64.h index aef61a029cb576..07e603a70afb7c 100644 --- a/src/coreclr/jit/emitriscv64.h +++ b/src/coreclr/jit/emitriscv64.h @@ -82,17 +82,17 @@ void emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataReg, GenTr unsigned emitOutput_Instr(BYTE* dst, code_t code) const; ssize_t emitOutputInstrJumpDistance(const BYTE* src, const insGroup* ig, instrDescJmp* jmp); -void emitOutputInstrJumpDistanceHelper(const insGroup* ig, - instrDescJmp* jmp, - UNATIVE_OFFSET& dstOffs, - const BYTE*& dstAddr) const; +void emitOutputInstrJumpDistanceHelper(const insGroup* ig, + instrDescJmp* jmp, + UNATIVE_OFFSET& dstOffs, + const BYTE*& dstAddr) const; // Method to do check if mov is redundant with respect to the last instruction. // If yes, the caller of this method can choose to omit current mov instruction. static bool IsMovInstruction(instruction ins); -bool IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regNumber src, bool canSkip); -bool IsRedundantLdStr( - instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt); // New functions end. +bool IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regNumber src, bool canSkip); +bool IsRedundantLdStr( + instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt); // New functions end. static code_t insEncodeRTypeInstr( unsigned opcode, unsigned rd, unsigned funct3, unsigned rs1, unsigned rs2, unsigned funct7); @@ -293,9 +293,9 @@ void emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) void emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs); -void emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber reg, +void emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber reg, ssize_t disp DEBUGARG(size_t targetHandle = 0) DEBUGARG(GenTreeFlags gtFlags = GTF_EMPTY)); enum EmitCallType @@ -324,8 +324,8 @@ enum EmitCallType void emitIns_Call(EmitCallType callType, CORINFO_METHOD_HANDLE methHnd, INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE - void* addr, - ssize_t argSize, + void* addr, + ssize_t argSize, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), VARSET_VALARG_TP ptrVars, regMaskTP gcrefRegs, diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index b3a518a9017430..06cec6c0056e79 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -1287,10 +1287,10 @@ bool emitter::TakesEvexPrefix(const instrDesc* id) const #define DEFAULT_BYTE_EVEX_PREFIX 0x62F07C0800000000ULL #define DEFAULT_BYTE_EVEX_PREFIX_MASK 0xFFFFFFFF00000000ULL -#define BBIT_IN_BYTE_EVEX_PREFIX 0x0000001000000000ULL -#define LBIT_IN_BYTE_EVEX_PREFIX 0x0000002000000000ULL +#define BBIT_IN_BYTE_EVEX_PREFIX 0x0000001000000000ULL +#define LBIT_IN_BYTE_EVEX_PREFIX 0x0000002000000000ULL #define LPRIMEBIT_IN_BYTE_EVEX_PREFIX 0x0000004000000000ULL -#define ZBIT_IN_BYTE_EVEX_PREFIX 0x0000008000000000ULL +#define ZBIT_IN_BYTE_EVEX_PREFIX 0x0000008000000000ULL //------------------------------------------------------------------------ // AddEvexPrefix: Add default EVEX prefix with only LL' bits set. @@ -1460,9 +1460,9 @@ bool emitter::TakesVexPrefix(instruction ins) const // 01 - 66 (66 0F - packed double) // 10 - F3 (F3 0F - scalar float // 11 - F2 (F2 0F - scalar double) -#define DEFAULT_3BYTE_VEX_PREFIX 0xC4E07800000000ULL +#define DEFAULT_3BYTE_VEX_PREFIX 0xC4E07800000000ULL #define DEFAULT_3BYTE_VEX_PREFIX_MASK 0xFFFFFF00000000ULL -#define LBIT_IN_3BYTE_VEX_PREFIX 0x00000400000000ULL +#define LBIT_IN_3BYTE_VEX_PREFIX 0x00000400000000ULL emitter::code_t emitter::AddVexPrefix(instruction ins, code_t code, emitAttr attr) { // The 2-byte VEX encoding is preferred when possible, but actually emitting @@ -3597,7 +3597,7 @@ bool emitter::emitVerifyEncodable(instruction ins, emitAttr size, regNumber reg1 #ifdef FEATURE_HW_INTRINSICS && (ins != INS_crc32) #endif - ) + ) { // reg1 must be a byte-able register if ((genRegMask(reg1) & RBM_BYTE_REGS) == 0) @@ -4108,7 +4108,8 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) assert((attrSize == EA_4BYTE) || (attrSize == EA_PTRSIZE) // Only for x64 || (attrSize == EA_16BYTE) || (attrSize == EA_32BYTE) || (attrSize == EA_64BYTE) // only for x64 - || (ins == INS_movzx) || (ins == INS_movsx) || (ins == INS_cmpxchg) + || (ins == INS_movzx) || (ins == INS_movsx) || + (ins == INS_cmpxchg) // The prefetch instructions are always 3 bytes and have part of their modr/m byte hardcoded || isPrefetch(ins)); @@ -4489,9 +4490,9 @@ emitter::instrDesc* emitter::emitNewInstrAmdCns(emitAttr size, ssize_t dsp, int } /***************************************************************************** -* -* Add a data16 instruction of the 1 byte. -*/ + * + * Add a data16 instruction of the 1 byte. + */ void emitter::emitIns_Data16() { @@ -4539,7 +4540,8 @@ void emitter::emitIns(instruction ins) (ins == INS_cdq || ins == INS_int3 || ins == INS_lock || ins == INS_leave || ins == INS_movsb || ins == INS_movsd || ins == INS_movsp || ins == INS_nop || ins == INS_r_movsb || ins == INS_r_movsd || ins == INS_r_movsp || ins == INS_r_stosb || ins == INS_r_stosd || ins == INS_r_stosp || ins == INS_ret || - ins == INS_sahf || ins == INS_stosb || ins == INS_stosd || ins == INS_stosp + ins == INS_sahf || ins == INS_stosb || ins == INS_stosd || + ins == INS_stosp // These instructions take zero operands || ins == INS_vzeroupper || ins == INS_lfence || ins == INS_mfence || ins == INS_sfence || ins == INS_pause || ins == INS_serialize); @@ -6969,9 +6971,9 @@ void emitter::emitIns_R_R_C(instruction ins, } /***************************************************************************** -* -* Add an instruction with three register operands. -*/ + * + * Add an instruction with three register operands. + */ void emitter::emitIns_R_R_R( instruction ins, emitAttr attr, regNumber targetReg, regNumber reg1, regNumber reg2, insOpts instOptions) @@ -7102,16 +7104,16 @@ void emitter::emitIns_R_R_C_I( } /********************************************************************************** -* emitIns_R_R_R_I: Add an instruction with three register operands and an immediate. -* -* Arguments: -* ins - the instruction to add -* attr - the emitter attribute for instruction -* targetReg - the target (destination) register -* reg1 - the first source register -* reg2 - the second source register -* ival - the immediate value -*/ + * emitIns_R_R_R_I: Add an instruction with three register operands and an immediate. + * + * Arguments: + * ins - the instruction to add + * attr - the emitter attribute for instruction + * targetReg - the target (destination) register + * reg1 - the first source register + * reg2 - the second source register + * ival - the immediate value + */ void emitter::emitIns_R_R_R_I( instruction ins, emitAttr attr, regNumber targetReg, regNumber reg1, regNumber reg2, int ival) @@ -7745,9 +7747,9 @@ void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber reg, regNum emitIns_R_ARX(ins, attr, reg, base, REG_NA, 1, disp); } -void emitter::emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber ireg, +void emitter::emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber ireg, ssize_t disp DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) { assert((CodeGen::instIsFP(ins) == false) && (EA_SIZE(attr) <= EA_8BYTE) && (ireg != REG_NA)); @@ -9710,7 +9712,7 @@ void emitter::emitIns_Call(EmitCallType callType, if (m_debugInfoSize > 0) { INDEBUG(id->idDebugOnlyInfo()->idCallSig = sigInfo); - id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token + id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token } #ifdef LATE_DISASM @@ -11702,7 +11704,7 @@ void emitter::emitDispIns( #ifdef TARGET_AMD64 || ins == INS_shrx || ins == INS_shlx || ins == INS_sarx #endif - ) + ) { // BMI bextr,bzhi, shrx, shlx and sarx encode the reg2 in VEX.vvvv and reg3 in modRM, // which is different from most of other instructions @@ -13003,9 +13005,9 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputWord(dst, code | 0x0500); } #else // TARGET_AMD64 - // Amd64: addr fits within 32-bits and can be encoded as a displacement relative to zero. - // This addr mode should never be used while generating relocatable ngen code nor if - // the addr can be encoded as pc-relative address. + // Amd64: addr fits within 32-bits and can be encoded as a displacement relative to zero. + // This addr mode should never be used while generating relocatable ngen code nor if + // the addr can be encoded as pc-relative address. noway_assert(!emitComp->opts.compReloc); noway_assert(codeGen->genAddrRelocTypeHint((size_t)dsp) != IMAGE_REL_BASED_REL32); noway_assert((int)dsp == dsp); @@ -13929,7 +13931,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case IF_SRW_CNS: case IF_SRW_RRD: case IF_SRW_RRW: - // += -= of a byref, no change + // += -= of a byref, no change case IF_SRW: break; @@ -16442,9 +16444,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - /********************************************************************/ - /* Simple constant, local label, method */ - /********************************************************************/ + /********************************************************************/ + /* Simple constant, local label, method */ + /********************************************************************/ case IF_CNS: { @@ -16562,9 +16564,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) #ifdef TARGET_X86 dst += emitOutputWord(dst, code | 0x0500); #else // TARGET_AMD64 - // Amd64: addr fits within 32-bits and can be encoded as a displacement relative to zero. - // This addr mode should never be used while generating relocatable ngen code nor if - // the addr can be encoded as pc-relative address. + // Amd64: addr fits within 32-bits and can be encoded as a displacement relative to zero. + // This addr mode should never be used while generating relocatable ngen code nor if + // the addr can be encoded as pc-relative address. noway_assert(!emitComp->opts.compReloc); noway_assert(codeGen->genAddrRelocTypeHint((size_t)addr) != IMAGE_REL_BASED_REL32); noway_assert(static_cast(reinterpret_cast(addr)) == (ssize_t)addr); @@ -16717,9 +16719,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - /********************************************************************/ - /* One register operand */ - /********************************************************************/ + /********************************************************************/ + /* One register operand */ + /********************************************************************/ case IF_RRD: case IF_RWR: @@ -16730,9 +16732,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - /********************************************************************/ - /* Register and register/constant */ - /********************************************************************/ + /********************************************************************/ + /* Register and register/constant */ + /********************************************************************/ case IF_RRW_SHF: { @@ -16957,9 +16959,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - /********************************************************************/ - /* Address mode operand */ - /********************************************************************/ + /********************************************************************/ + /* Address mode operand */ + /********************************************************************/ case IF_ARD: case IF_AWR: @@ -17196,9 +17198,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - /********************************************************************/ - /* Stack-based operand */ - /********************************************************************/ + /********************************************************************/ + /* Stack-based operand */ + /********************************************************************/ case IF_SRD: case IF_SWR: @@ -17460,9 +17462,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) unreached(); } - /********************************************************************/ - /* Direct memory address */ - /********************************************************************/ + /********************************************************************/ + /* Direct memory address */ + /********************************************************************/ case IF_MRD: case IF_MRW: @@ -17762,9 +17764,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) unreached(); } - /********************************************************************/ - /* oops */ - /********************************************************************/ + /********************************************************************/ + /* oops */ + /********************************************************************/ default: @@ -18229,7 +18231,7 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins #ifdef TARGET_AMD64 || ins == INS_movsxd #endif - ) + ) { result.insLatency += PERFSCORE_LATENCY_2C; } diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index b01554277abdd8..820e211ff25b89 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -99,7 +99,7 @@ code_t emitExtractEvexPrefix(instruction ins, code_t& code) const; unsigned insEncodeReg012(const instrDesc* id, regNumber reg, emitAttr size, code_t* code); unsigned insEncodeReg345(const instrDesc* id, regNumber reg, emitAttr size, code_t* code); -code_t insEncodeReg3456(const instrDesc* id, regNumber reg, emitAttr size, code_t code); +code_t insEncodeReg3456(const instrDesc* id, regNumber reg, emitAttr size, code_t code); unsigned insEncodeRegSIB(const instrDesc* id, regNumber reg, code_t* code); code_t insEncodeMRreg(const instrDesc* id, code_t code); @@ -122,11 +122,11 @@ static bool IsKInstruction(instruction ins); static regNumber getBmiRegNumber(instruction ins); static regNumber getSseShiftRegNumber(instruction ins); -bool HasVexEncoding(instruction ins) const; -bool HasEvexEncoding(instruction ins) const; -bool IsVexEncodableInstruction(instruction ins) const; -bool IsEvexEncodableInstruction(instruction ins) const; -bool IsVexOrEvexEncodableInstruction(instruction ins) const; +bool HasVexEncoding(instruction ins) const; +bool HasEvexEncoding(instruction ins) const; +bool IsVexEncodableInstruction(instruction ins) const; +bool IsEvexEncodableInstruction(instruction ins) const; +bool IsVexOrEvexEncodableInstruction(instruction ins) const; code_t insEncodeMIreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code); @@ -136,15 +136,15 @@ code_t AddRexXPrefix(const instrDesc* id, code_t code); code_t AddRexBPrefix(const instrDesc* id, code_t code); code_t AddRexPrefix(instruction ins, code_t code); -bool EncodedBySSE38orSSE3A(instruction ins) const; -bool Is4ByteSSEInstruction(instruction ins) const; +bool EncodedBySSE38orSSE3A(instruction ins) const; +bool Is4ByteSSEInstruction(instruction ins) const; code_t AddEvexVPrimePrefix(code_t code); code_t AddEvexRPrimePrefix(code_t code); static bool IsMovInstruction(instruction ins); -bool HasSideEffect(instruction ins, emitAttr size); -bool IsRedundantMov( - instruction ins, insFormat fmt, emitAttr size, regNumber dst, regNumber src, bool canIgnoreSideEffects); +bool HasSideEffect(instruction ins, emitAttr size); +bool IsRedundantMov( + instruction ins, insFormat fmt, emitAttr size, regNumber dst, regNumber src, bool canIgnoreSideEffects); bool EmitMovsxAsCwde(instruction ins, emitAttr size, regNumber dst, regNumber src); bool IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size, regNumber ireg, int varx, int offs); @@ -484,15 +484,15 @@ void SetContainsCallNeedingVzeroupper(bool value) containsCallNeedingVzeroupper = value; } -bool IsDstDstSrcAVXInstruction(instruction ins) const; -bool IsDstSrcSrcAVXInstruction(instruction ins) const; -bool IsThreeOperandAVXInstruction(instruction ins) const; +bool IsDstDstSrcAVXInstruction(instruction ins) const; +bool IsDstSrcSrcAVXInstruction(instruction ins) const; +bool IsThreeOperandAVXInstruction(instruction ins) const; static bool HasRegularWideForm(instruction ins); static bool HasRegularWideImmediateForm(instruction ins); static bool DoesWriteZeroFlag(instruction ins); static bool DoesWriteSignFlag(instruction ins); static bool DoesResetOverflowAndCarryFlags(instruction ins); -bool IsFlagsAlwaysModified(instrDesc* id); +bool IsFlagsAlwaysModified(instrDesc* id); static bool IsRexW0Instruction(instruction ins); static bool IsRexW1Instruction(instruction ins); static bool IsRexWXInstruction(instruction ins); @@ -534,7 +534,7 @@ const char* emitZMMregName(unsigned reg) const; /************************************************************************/ private: -void emitSetAmdDisp(instrDescAmd* id, ssize_t dsp); +void emitSetAmdDisp(instrDescAmd* id, ssize_t dsp); instrDesc* emitNewInstrAmd(emitAttr attr, ssize_t dsp); instrDesc* emitNewInstrAmdCns(emitAttr attr, ssize_t dsp, int cns); @@ -551,9 +551,9 @@ instrDesc* emitNewInstrCallInd(int argCnt, regMaskGpr byrefRegs, emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize)); -void emitGetInsCns(const instrDesc* id, CnsVal* cv) const; +void emitGetInsCns(const instrDesc* id, CnsVal* cv) const; ssize_t emitGetInsAmdCns(const instrDesc* id, CnsVal* cv) const; -void emitGetInsDcmCns(const instrDesc* id, CnsVal* cv) const; +void emitGetInsDcmCns(const instrDesc* id, CnsVal* cv) const; ssize_t emitGetInsAmdAny(const instrDesc* id) const; /************************************************************************/ @@ -586,10 +586,10 @@ size_t emitSizeOfInsDsc_NONE(instrDesc* id) const; size_t emitSizeOfInsDsc_SPEC(instrDesc* id) const; /***************************************************************************** -* -* Convert between an index scale in bytes to a smaller encoding used for -* storage in instruction descriptors. -*/ + * + * Convert between an index scale in bytes to a smaller encoding used for + * storage in instruction descriptors. + */ inline emitter::opSize emitEncodeScale(size_t scale) { @@ -758,9 +758,9 @@ void emitIns_I_AI(instruction ins, emitAttr attr, int val, ssize_t disp); void emitIns_R_AR(instruction ins, emitAttr attr, regNumber reg, regNumber base, int disp); -void emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber ireg, +void emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber ireg, ssize_t disp DEBUGARG(size_t targetHandle = 0) DEBUGARG(GenTreeFlags gtFlags = GTF_EMPTY)); void emitIns_AR_R(instruction ins, emitAttr attr, regNumber reg, regNumber base, cnsval_ssize_t disp); diff --git a/src/coreclr/jit/error.cpp b/src/coreclr/jit/error.cpp index a45ad7c7df0ef0..5ae6cea056efeb 100644 --- a/src/coreclr/jit/error.cpp +++ b/src/coreclr/jit/error.cpp @@ -250,7 +250,9 @@ void debugError(const char* msg, const char* file, unsigned line) } /*****************************************************************************/ -LogEnv::LogEnv(ICorJitInfo* aCompHnd) : compHnd(aCompHnd), compiler(nullptr) +LogEnv::LogEnv(ICorJitInfo* aCompHnd) + : compHnd(aCompHnd) + , compiler(nullptr) { } diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 5b4fcd33f8e214..a650ae437fccc6 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -899,7 +899,10 @@ BasicBlock* Compiler::fgLookupBB(unsigned addr) class FgStack { public: - FgStack() : slot0(SLOT_INVALID), slot1(SLOT_INVALID), depth(0) + FgStack() + : slot0(SLOT_INVALID) + , slot1(SLOT_INVALID) + , depth(0) { // Empty } @@ -3122,7 +3125,7 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F codeAddr += sizeof(__int8); goto DECODE_OPCODE; - /* Check to see if we have a jump/return opcode */ + /* Check to see if we have a jump/return opcode */ case CEE_BRFALSE: case CEE_BRFALSE_S: @@ -3305,7 +3308,7 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F // statement in the block. // Otherwise, we will assert at the following line in fgMorphCall() // noway_assert(fgMorphStmt->GetNextStmt() == NULL); - ) + ) { // Neither .tailcall prefix, no tailcall stress. So move on. break; @@ -4030,11 +4033,11 @@ void Compiler::fgFindBasicBlocks() #endif } -/* Init ebdHandlerNestingLevel of current clause, and bump up value for all - * enclosed clauses (which have to be before it in the table). - * Innermost try-finally blocks must precede outermost - * try-finally blocks. - */ + /* Init ebdHandlerNestingLevel of current clause, and bump up value for all + * enclosed clauses (which have to be before it in the table). + * Innermost try-finally blocks must precede outermost + * try-finally blocks. + */ #if !defined(FEATURE_EH_FUNCLETS) HBtab->ebdHandlerNestingLevel = 0; @@ -5917,8 +5920,8 @@ BasicBlock* Compiler::fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE r } else { - assert(fgFirstFuncletBB != - insertAfterBlk->Next()); // We insert at the end, not at the beginning, of the funclet region. + assert(fgFirstFuncletBB != insertAfterBlk->Next()); // We insert at the end, not at the beginning, of the + // funclet region. } #ifdef DEBUG @@ -6244,8 +6247,8 @@ BasicBlock* Compiler::fgFindInsertPoint(unsigned regionIndex, noway_assert(startBlk != nullptr); noway_assert(startBlk != endBlk); noway_assert((regionIndex == 0 && putInTryRegion) || // Search in the main method - (putInTryRegion && regionIndex > 0 && - startBlk->bbTryIndex == regionIndex) || // Search in the specified try region + (putInTryRegion && regionIndex > 0 && startBlk->bbTryIndex == regionIndex) || // Search in the + // specified try region (!putInTryRegion && regionIndex > 0 && startBlk->bbHndIndex == regionIndex)); // Search in the specified handler region diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index f49f863b16dc57..e2af55f45ac820 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -389,7 +389,7 @@ const char* ConvertToUtf8(LPCWSTR wideString, CompAllocator& allocator) return alloc; } -} +} // namespace #endif //------------------------------------------------------------------------ @@ -546,7 +546,7 @@ FILE* Compiler::fgOpenFlowGraphFile(bool* wbDontClose, Phases phase, PhasePositi ONE_FILE_PER_METHOD:; -#define FILENAME_PATTERN "%s-%s-%s-%s.%s" +#define FILENAME_PATTERN "%s-%s-%s-%s.%s" #define FILENAME_PATTERN_WITH_NUMBER "%s-%s-%s-%s~%d.%s" const size_t MaxFileNameLength = MAX_PATH_FNAME - 20 /* give us some extra buffer */; @@ -1249,7 +1249,10 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) public: RegionGraph(Compiler* comp, unsigned* blkMap, unsigned blkMapSize) - : m_comp(comp), m_rgnRoot(nullptr), m_blkMap(blkMap), m_blkMapSize(blkMapSize) + : m_comp(comp) + , m_rgnRoot(nullptr) + , m_blkMap(blkMap) + , m_blkMapSize(blkMapSize) { // Create a root region that encompasses the whole function. m_rgnRoot = @@ -2642,7 +2645,8 @@ void Compiler::fgStress64RsltMul() class BBPredsChecker { public: - BBPredsChecker(Compiler* compiler) : comp(compiler) + BBPredsChecker(Compiler* compiler) + : comp(compiler) { } @@ -3240,7 +3244,7 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef #ifndef JIT32_GCENCODER copiedForGenericsCtxt = ((info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_THIS) != 0); #else // JIT32_GCENCODER - copiedForGenericsCtxt = false; + copiedForGenericsCtxt = false; #endif // JIT32_GCENCODER // This if only in support of the noway_asserts it contains. @@ -3284,7 +3288,8 @@ void Compiler::fgDebugCheckTypes(GenTree* tree) DoPostOrder = true, }; - NodeTypeValidator(Compiler* comp) : GenTreeVisitor(comp) + NodeTypeValidator(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -3733,7 +3738,9 @@ void Compiler::fgDebugCheckLinkedLocals() UseExecutionOrder = true, }; - DebugLocalSequencer(Compiler* comp) : GenTreeVisitor(comp), m_locals(comp->getAllocator(CMK_DebugOnly)) + DebugLocalSequencer(Compiler* comp) + : GenTreeVisitor(comp) + , m_locals(comp->getAllocator(CMK_DebugOnly)) { } @@ -4014,7 +4021,9 @@ class UniquenessCheckWalker { public: UniquenessCheckWalker(Compiler* comp) - : comp(comp), nodesVecTraits(comp->compGenTreeID, comp), uniqueNodes(BitVecOps::MakeEmpty(&nodesVecTraits)) + : comp(comp) + , nodesVecTraits(comp->compGenTreeID, comp) + , uniqueNodes(BitVecOps::MakeEmpty(&nodesVecTraits)) { } @@ -4132,11 +4141,15 @@ class SsaCheckVisitor : public GenTreeVisitor unsigned m_ssaNum; public: - SsaKey() : m_lclNum(BAD_VAR_NUM), m_ssaNum(SsaConfig::RESERVED_SSA_NUM) + SsaKey() + : m_lclNum(BAD_VAR_NUM) + , m_ssaNum(SsaConfig::RESERVED_SSA_NUM) { } - SsaKey(unsigned lclNum, unsigned ssaNum) : m_lclNum(lclNum), m_ssaNum(ssaNum) + SsaKey(unsigned lclNum, unsigned ssaNum) + : m_lclNum(lclNum) + , m_ssaNum(ssaNum) { } @@ -4773,13 +4786,15 @@ void Compiler::fgDebugCheckFlowGraphAnnotations() return; } - unsigned count = - fgRunDfs([](BasicBlock* block, unsigned preorderNum) { assert(block->bbPreorderNum == preorderNum); }, - [=](BasicBlock* block, unsigned postorderNum) { - assert(block->bbPostorderNum == postorderNum); - assert(m_dfsTree->GetPostOrder(postorderNum) == block); - }, - [](BasicBlock* block, BasicBlock* succ) {}); + unsigned count = fgRunDfs( + [](BasicBlock* block, unsigned preorderNum) { + assert(block->bbPreorderNum == preorderNum); + }, + [=](BasicBlock* block, unsigned postorderNum) { + assert(block->bbPostorderNum == postorderNum); + assert(m_dfsTree->GetPostOrder(postorderNum) == block); + }, + [](BasicBlock* block, BasicBlock* succ) {}); assert(m_dfsTree->GetPostOrderCount() == count); diff --git a/src/coreclr/jit/fgehopt.cpp b/src/coreclr/jit/fgehopt.cpp index a437a3da128d4c..0e1ce24c39ed87 100644 --- a/src/coreclr/jit/fgehopt.cpp +++ b/src/coreclr/jit/fgehopt.cpp @@ -1867,11 +1867,15 @@ PhaseStatus Compiler::fgTailMergeThrows() BasicBlock* m_block; GenTreeCall* m_call; - ThrowHelper() : m_block(nullptr), m_call(nullptr) + ThrowHelper() + : m_block(nullptr) + , m_call(nullptr) { } - ThrowHelper(BasicBlock* block, GenTreeCall* call) : m_block(block), m_call(call) + ThrowHelper(BasicBlock* block, GenTreeCall* call) + : m_block(block) + , m_call(call) { } diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 8ea17f2bff05a0..ba5ed96610dd32 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -214,7 +214,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor( [](Param* pParam) { - // Init the local var info of the inlinee - pParam->pThis->impInlineInitVars(pParam->inlineInfo); + // Init the local var info of the inlinee + pParam->pThis->impInlineInitVars(pParam->inlineInfo); - if (pParam->inlineInfo->inlineResult->IsCandidate()) - { - /* Clear the temp table */ - memset(pParam->inlineInfo->lclTmpNum, -1, sizeof(pParam->inlineInfo->lclTmpNum)); + if (pParam->inlineInfo->inlineResult->IsCandidate()) + { + /* Clear the temp table */ + memset(pParam->inlineInfo->lclTmpNum, -1, sizeof(pParam->inlineInfo->lclTmpNum)); - // - // Prepare the call to jitNativeCode - // + // + // Prepare the call to jitNativeCode + // - pParam->inlineInfo->InlinerCompiler = pParam->pThis; - if (pParam->pThis->impInlineInfo == nullptr) - { - pParam->inlineInfo->InlineRoot = pParam->pThis; - } - else - { - pParam->inlineInfo->InlineRoot = pParam->pThis->impInlineInfo->InlineRoot; - } + pParam->inlineInfo->InlinerCompiler = pParam->pThis; + if (pParam->pThis->impInlineInfo == nullptr) + { + pParam->inlineInfo->InlineRoot = pParam->pThis; + } + else + { + pParam->inlineInfo->InlineRoot = pParam->pThis->impInlineInfo->InlineRoot; + } - // The inline context is part of debug info and must be created - // before we start creating statements; we lazily create it as - // late as possible, which is here. - pParam->inlineInfo->inlineContext = - pParam->inlineInfo->InlineRoot->m_inlineStrategy - ->NewContext(pParam->inlineInfo->inlineCandidateInfo->inlinersContext, - pParam->inlineInfo->iciStmt, pParam->inlineInfo->iciCall); - pParam->inlineInfo->argCnt = pParam->inlineCandidateInfo->methInfo.args.totalILArgs(); - pParam->inlineInfo->tokenLookupContextHandle = pParam->inlineCandidateInfo->exactContextHnd; - - JITLOG_THIS(pParam->pThis, - (LL_INFO100000, "INLINER: inlineInfo.tokenLookupContextHandle for %s set to 0x%p:\n", - pParam->pThis->eeGetMethodFullName(pParam->fncHandle), - pParam->pThis->dspPtr(pParam->inlineInfo->tokenLookupContextHandle))); - - JitFlags compileFlagsForInlinee = *pParam->pThis->opts.jitFlags; - - // The following flags are lost when inlining. - // (This is checked in Compiler::compInitOptions().) - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_BBINSTR); - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_BBINSTR_IF_LOOPS); - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_PROF_ENTERLEAVE); - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_DEBUG_EnC); - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_REVERSE_PINVOKE); - compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_TRACK_TRANSITIONS); + // The inline context is part of debug info and must be created + // before we start creating statements; we lazily create it as + // late as possible, which is here. + pParam->inlineInfo->inlineContext = + pParam->inlineInfo->InlineRoot->m_inlineStrategy + ->NewContext(pParam->inlineInfo->inlineCandidateInfo->inlinersContext, pParam->inlineInfo->iciStmt, + pParam->inlineInfo->iciCall); + pParam->inlineInfo->argCnt = pParam->inlineCandidateInfo->methInfo.args.totalILArgs(); + pParam->inlineInfo->tokenLookupContextHandle = pParam->inlineCandidateInfo->exactContextHnd; + + JITLOG_THIS(pParam->pThis, + (LL_INFO100000, "INLINER: inlineInfo.tokenLookupContextHandle for %s set to 0x%p:\n", + pParam->pThis->eeGetMethodFullName(pParam->fncHandle), + pParam->pThis->dspPtr(pParam->inlineInfo->tokenLookupContextHandle))); + + JitFlags compileFlagsForInlinee = *pParam->pThis->opts.jitFlags; + + // The following flags are lost when inlining. + // (This is checked in Compiler::compInitOptions().) + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_BBINSTR); + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_BBINSTR_IF_LOOPS); + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_PROF_ENTERLEAVE); + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_DEBUG_EnC); + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_REVERSE_PINVOKE); + compileFlagsForInlinee.Clear(JitFlags::JIT_FLAG_TRACK_TRANSITIONS); #ifdef DEBUG - if (pParam->pThis->verbose) - { - printf("\nInvoking compiler for the inlinee method %s :\n", - pParam->pThis->eeGetMethodFullName(pParam->fncHandle)); - } + if (pParam->pThis->verbose) + { + printf("\nInvoking compiler for the inlinee method %s :\n", + pParam->pThis->eeGetMethodFullName(pParam->fncHandle)); + } #endif // DEBUG - int result = - jitNativeCode(pParam->fncHandle, pParam->inlineCandidateInfo->methInfo.scope, - pParam->pThis->info.compCompHnd, &pParam->inlineCandidateInfo->methInfo, - (void**)pParam->inlineInfo, nullptr, &compileFlagsForInlinee, pParam->inlineInfo); + int result = + jitNativeCode(pParam->fncHandle, pParam->inlineCandidateInfo->methInfo.scope, + pParam->pThis->info.compCompHnd, &pParam->inlineCandidateInfo->methInfo, + (void**)pParam->inlineInfo, nullptr, &compileFlagsForInlinee, pParam->inlineInfo); - if (result != CORJIT_OK) - { - // If we haven't yet determined why this inline fails, use - // a catch-all something bad happened observation. - InlineResult* innerInlineResult = pParam->inlineInfo->inlineResult; + if (result != CORJIT_OK) + { + // If we haven't yet determined why this inline fails, use + // a catch-all something bad happened observation. + InlineResult* innerInlineResult = pParam->inlineInfo->inlineResult; - if (!innerInlineResult->IsFailure()) - { - innerInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_FAILURE); - } + if (!innerInlineResult->IsFailure()) + { + innerInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_FAILURE); } } - }, + } + }, ¶m); if (!success) { diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 0f20dccd2fef35..433a512469816a 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -731,7 +731,6 @@ PhaseStatus Compiler::fgPostImportationCleanup() // auto addConditionalFlow = [this, entryStateVar, &entryJumpTarget, &addedBlocks](BasicBlock* fromBlock, BasicBlock* toBlock) { - // We may have previously though this try entry was unreachable, but now we're going to // step through it on the way to the OSR entry. So ensure it has plausible profile weight. // @@ -2600,7 +2599,7 @@ void Compiler::fgRemoveConditionalJump(BasicBlock* block) assert(block->TargetIs(target)); /* Update bbRefs and bbNum - Conditional predecessors to the same - * block are counted twice so we have to remove one of them */ + * block are counted twice so we have to remove one of them */ noway_assert(target->countOfInEdges() > 1); fgRemoveRefPred(block->GetTargetEdge()); @@ -3969,8 +3968,8 @@ bool Compiler::fgReorderBlocks(bool useProfile) bNext = bEnd->Next(); bool connected_bDest = false; - if ((backwardBranch && !isRare) || - block->HasFlag(BBF_DONT_REMOVE)) // Don't choose option #1 when block is the start of a try region + if ((backwardBranch && !isRare) || block->HasFlag(BBF_DONT_REMOVE)) // Don't choose option #1 when block is the + // start of a try region { bStart = nullptr; bEnd = nullptr; @@ -4779,11 +4778,11 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh continue; } - /* We jump to the REPEAT label if we performed a change involving the current block - * This is in case there are other optimizations that can show up - * (e.g. - compact 3 blocks in a row) - * If nothing happens, we then finish the iteration and move to the next block - */ + /* We jump to the REPEAT label if we performed a change involving the current block + * This is in case there are other optimizations that can show up + * (e.g. - compact 3 blocks in a row) + * If nothing happens, we then finish the iteration and move to the next block + */ REPEAT:; @@ -5364,12 +5363,13 @@ unsigned Compiler::fgMeasureIR() { for (Statement* const stmt : block->Statements()) { - fgWalkTreePre(stmt->GetRootNodePointer(), - [](GenTree** slot, fgWalkData* data) -> Compiler::fgWalkResult { - (*reinterpret_cast(data->pCallbackData))++; - return Compiler::WALK_CONTINUE; - }, - &nodeCount); + fgWalkTreePre( + stmt->GetRootNodePointer(), + [](GenTree** slot, fgWalkData* data) -> Compiler::fgWalkResult { + (*reinterpret_cast(data->pCallbackData))++; + return Compiler::WALK_CONTINUE; + }, + &nodeCount); } } else @@ -5444,7 +5444,9 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early) struct PredInfo { - PredInfo(BasicBlock* block, Statement* stmt) : m_block(block), m_stmt(stmt) + PredInfo(BasicBlock* block, Statement* stmt) + : m_block(block) + , m_stmt(stmt) { } BasicBlock* m_block; @@ -5750,7 +5752,6 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early) }; auto iterateTailMerge = [&](BasicBlock* block) -> void { - int numOpts = 0; while (tailMerge(block)) @@ -5968,7 +5969,8 @@ bool Compiler::gtTreeContainsTailCall(GenTree* tree) DoPreOrder = true }; - HasTailCallCandidateVisitor(Compiler* comp) : GenTreeVisitor(comp) + HasTailCallCandidateVisitor(Compiler* comp) + : GenTreeVisitor(comp) { } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index d53abf356150c3..9fa4e7273862a2 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -309,7 +309,11 @@ class Instrumentor bool m_modifiedFlow; protected: - Instrumentor(Compiler* comp) : m_comp(comp), m_schemaCount(0), m_instrCount(0), m_modifiedFlow(false) + Instrumentor(Compiler* comp) + : m_comp(comp) + , m_schemaCount(0) + , m_instrCount(0) + , m_modifiedFlow(false) { } @@ -360,7 +364,8 @@ class Instrumentor class NonInstrumentor : public Instrumentor { public: - NonInstrumentor(Compiler* comp) : Instrumentor(comp) + NonInstrumentor(Compiler* comp) + : Instrumentor(comp) { } }; @@ -376,7 +381,9 @@ class BlockCountInstrumentor : public Instrumentor BasicBlock* m_entryBlock; public: - BlockCountInstrumentor(Compiler* comp) : Instrumentor(comp), m_entryBlock(nullptr) + BlockCountInstrumentor(Compiler* comp) + : Instrumentor(comp) + , m_entryBlock(nullptr) { } bool ShouldProcess(BasicBlock* block) override @@ -566,8 +573,8 @@ void BlockCountInstrumentor::BuildSchemaElements(BasicBlock* block, Schema& sche schemaElem.InstrumentationKind = m_comp->opts.compCollect64BitCounts ? ICorJitInfo::PgoInstrumentationKind::BasicBlockLongCount : ICorJitInfo::PgoInstrumentationKind::BasicBlockIntCount; - schemaElem.ILOffset = offset; - schemaElem.Offset = 0; + schemaElem.ILOffset = offset; + schemaElem.Offset = 0; schema.push_back(schemaElem); @@ -841,9 +848,9 @@ class SpanningTreeVisitor Duplicate }; - virtual void Badcode() = 0; - virtual void VisitBlock(BasicBlock* block) = 0; - virtual void VisitTreeEdge(BasicBlock* source, BasicBlock* target) = 0; + virtual void Badcode() = 0; + virtual void VisitBlock(BasicBlock* block) = 0; + virtual void VisitTreeEdge(BasicBlock* source, BasicBlock* target) = 0; virtual void VisitNonTreeEdge(BasicBlock* source, BasicBlock* target, EdgeKind kind) = 0; }; @@ -1239,7 +1246,9 @@ static int32_t EfficientEdgeCountBlockToKey(BasicBlock* block) // Based on "Optimally Profiling and Tracing Programs," // Ball and Larus PLDI '92. // -class EfficientEdgeCountInstrumentor : public Instrumentor, public SpanningTreeVisitor +class EfficientEdgeCountInstrumentor + : public Instrumentor + , public SpanningTreeVisitor { private: // A particular edge probe. These are linked @@ -1753,8 +1762,8 @@ void EfficientEdgeCountInstrumentor::BuildSchemaElements(BasicBlock* block, Sche schemaElem.InstrumentationKind = m_comp->opts.compCollect64BitCounts ? ICorJitInfo::PgoInstrumentationKind::EdgeLongCount : ICorJitInfo::PgoInstrumentationKind::EdgeIntCount; - schemaElem.ILOffset = sourceKey; - schemaElem.Offset = 0; + schemaElem.ILOffset = sourceKey; + schemaElem.Offset = 0; schema.push_back(schemaElem); @@ -1903,7 +1912,9 @@ class HandleHistogramProbeVisitor final : public GenTreeVisitor(compiler), m_functor(functor), m_compiler(compiler) + : GenTreeVisitor(compiler) + , m_functor(functor) + , m_compiler(compiler) { } Compiler::fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) @@ -1935,7 +1946,9 @@ class ValueHistogramProbeVisitor final : public GenTreeVisitor(compiler), m_functor(functor), m_compiler(compiler) + : GenTreeVisitor(compiler) + , m_functor(functor) + , m_compiler(compiler) { } @@ -1965,7 +1978,8 @@ class BuildHandleHistogramProbeSchemaGen public: BuildHandleHistogramProbeSchemaGen(Schema& schema, unsigned& schemaCount) - : m_schema(schema), m_schemaCount(schemaCount) + : m_schema(schema) + , m_schemaCount(schemaCount) { } @@ -2003,8 +2017,8 @@ class BuildHandleHistogramProbeSchemaGen schemaElem.InstrumentationKind = compiler->opts.compCollect64BitCounts ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount : ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; - schemaElem.ILOffset = (int32_t)call->gtHandleHistogramProfileCandidateInfo->ilOffset; - schemaElem.Offset = 0; + schemaElem.ILOffset = (int32_t)call->gtHandleHistogramProfileCandidateInfo->ilOffset; + schemaElem.Offset = 0; m_schema.push_back(schemaElem); @@ -2013,7 +2027,7 @@ class BuildHandleHistogramProbeSchemaGen // Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount schemaElem.InstrumentationKind = isTypeHistogram ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes : ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods; - schemaElem.Count = ICorJitInfo::HandleHistogram32::SIZE; + schemaElem.Count = ICorJitInfo::HandleHistogram32::SIZE; m_schema.push_back(schemaElem); m_schemaCount++; @@ -2027,7 +2041,8 @@ class BuildValueHistogramProbeSchemaGen public: BuildValueHistogramProbeSchemaGen(Schema& schema, unsigned& schemaCount) - : m_schema(schema), m_schemaCount(schemaCount) + : m_schema(schema) + , m_schemaCount(schemaCount) { } @@ -2036,8 +2051,8 @@ class BuildValueHistogramProbeSchemaGen ICorJitInfo::PgoInstrumentationSchema schemaElem = {}; schemaElem.Count = 1; schemaElem.InstrumentationKind = compiler->opts.compCollect64BitCounts - ? ICorJitInfo::PgoInstrumentationKind::ValueHistogramLongCount - : ICorJitInfo::PgoInstrumentationKind::ValueHistogramIntCount; + ? ICorJitInfo::PgoInstrumentationKind::ValueHistogramLongCount + : ICorJitInfo::PgoInstrumentationKind::ValueHistogramIntCount; schemaElem.ILOffset = (int32_t)call->AsCall()->gtHandleHistogramProfileCandidateInfo->ilOffset; m_schema.push_back(schemaElem); m_schemaCount++; @@ -2332,7 +2347,8 @@ class ValueHistogramProbeInserter class HandleHistogramProbeInstrumentor : public Instrumentor { public: - HandleHistogramProbeInstrumentor(Compiler* comp) : Instrumentor(comp) + HandleHistogramProbeInstrumentor(Compiler* comp) + : Instrumentor(comp) { } bool ShouldProcess(BasicBlock* block) override @@ -2350,7 +2366,8 @@ class HandleHistogramProbeInstrumentor : public Instrumentor class ValueInstrumentor : public Instrumentor { public: - ValueInstrumentor(Compiler* comp) : Instrumentor(comp) + ValueInstrumentor(Compiler* comp) + : Instrumentor(comp) { } bool ShouldProcess(BasicBlock* block) override @@ -2727,7 +2744,7 @@ PhaseStatus Compiler::fgInstrumentMethod() // uint8_t* profileMemory; HRESULT res = info.compCompHnd->allocPgoInstrumentationBySchema(info.compMethodHnd, schema.data(), - (UINT32)schema.size(), &profileMemory); + (UINT32)schema.size(), &profileMemory); // Deal with allocation failures. // @@ -3102,7 +3119,7 @@ class EfficientEdgeCountReconstructor : public SpanningTreeVisitor // Map correlating block keys to blocks. // typedef JitHashTable, BasicBlock*> KeyToBlockMap; - KeyToBlockMap m_keyToBlockMap; + KeyToBlockMap m_keyToBlockMap; // Key for finding an edge based on schema info. // @@ -3111,7 +3128,9 @@ class EfficientEdgeCountReconstructor : public SpanningTreeVisitor int32_t const m_sourceKey; int32_t const m_targetKey; - EdgeKey(int32_t sourceKey, int32_t targetKey) : m_sourceKey(sourceKey), m_targetKey(targetKey) + EdgeKey(int32_t sourceKey, int32_t targetKey) + : m_sourceKey(sourceKey) + , m_targetKey(targetKey) { } @@ -3159,7 +3178,7 @@ class EfficientEdgeCountReconstructor : public SpanningTreeVisitor // Map for correlating EdgeIntCount schema entries with edges // typedef JitHashTable EdgeKeyToEdgeMap; - EdgeKeyToEdgeMap m_edgeKeyToEdgeMap; + EdgeKeyToEdgeMap m_edgeKeyToEdgeMap; // Per block data // @@ -3519,8 +3538,9 @@ void EfficientEdgeCountReconstructor::Solve() // if (m_badcode || m_mismatch || m_allWeightsZero) { - JITDUMP("... not solving because of the %s\n", - m_badcode ? "badcode" : m_allWeightsZero ? "zero counts" : "mismatch"); + JITDUMP("... not solving because of the %s\n", m_badcode ? "badcode" + : m_allWeightsZero ? "zero counts" + : "mismatch"); return; } diff --git a/src/coreclr/jit/fgprofilesynthesis.h b/src/coreclr/jit/fgprofilesynthesis.h index 216bd58297286a..e2e7c58cbac4f2 100644 --- a/src/coreclr/jit/fgprofilesynthesis.h +++ b/src/coreclr/jit/fgprofilesynthesis.h @@ -40,7 +40,8 @@ class ProfileSynthesis static constexpr weight_t epsilon = 0.001; private: - ProfileSynthesis(Compiler* compiler) : m_comp(compiler) + ProfileSynthesis(Compiler* compiler) + : m_comp(compiler) { } diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index e2f450a7cb194d..a3ee97d731519d 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -1063,7 +1063,7 @@ GenTree* Compiler::fgOptimizeDelegateConstructor(GenTreeCall* call, &genericLookup); GenTree* ctxTree = getRuntimeContextTree(pLookup.lookupKind.runtimeLookupKind); call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_DELEGATE_CTOR, TYP_VOID, thisPointer, - targetObjPointers, ctxTree); + targetObjPointers, ctxTree); call->setEntryPoint(genericLookup); } } @@ -1647,8 +1647,8 @@ void Compiler::fgConvertSyncReturnToLeave(BasicBlock* block) // try/finally, which must be the last EH region. EHblkDsc* ehDsc = ehGetDsc(tryIndex); - assert(ehDsc->ebdEnclosingTryIndex == - EHblkDsc::NO_ENCLOSING_INDEX); // There are no enclosing regions of the BBJ_RETURN block + assert(ehDsc->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX); // There are no enclosing regions of the + // BBJ_RETURN block assert(ehDsc->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX); // Convert the BBJ_RETURN to BBJ_ALWAYS, jumping to genReturnBB. @@ -1823,7 +1823,8 @@ class MergedReturns bool mergingReturns = false; public: - MergedReturns(Compiler* comp) : comp(comp) + MergedReturns(Compiler* comp) + : comp(comp) { comp->fgReturnCount = 0; } @@ -2266,7 +2267,7 @@ class MergedReturns return nullptr; } }; -} +} // namespace //------------------------------------------------------------------------ // fgAddInternal: add blocks and trees to express special method semantics @@ -2325,7 +2326,7 @@ PhaseStatus Compiler::fgAddInternal() #ifndef JIT32_GCENCODER lva0CopiedForGenericsCtxt = ((info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_THIS) != 0); #else // JIT32_GCENCODER - lva0CopiedForGenericsCtxt = false; + lva0CopiedForGenericsCtxt = false; #endif // JIT32_GCENCODER noway_assert(lva0CopiedForGenericsCtxt || !lvaTable[info.compThisArg].IsAddressExposed()); noway_assert(!lvaTable[info.compThisArg].lvHasILStoreOp); @@ -3589,7 +3590,9 @@ GenTree* Compiler::fgSetTreeSeq(GenTree* tree, bool isLIR) }; SetTreeSeqVisitor(Compiler* compiler, GenTree* tree, bool isLIR) - : GenTreeVisitor(compiler), m_prevNode(tree), m_isLIR(isLIR) + : GenTreeVisitor(compiler) + , m_prevNode(tree) + , m_isLIR(isLIR) { INDEBUG(tree->gtSeqNum = 0); } @@ -3677,7 +3680,8 @@ PhaseStatus Compiler::fgSetBlockOrder() class GCSafePointSuccessorEnumerator { BasicBlock* m_block; - union { + union + { BasicBlock* m_successors[2]; BasicBlock** m_pSuccessors; }; @@ -3688,7 +3692,8 @@ class GCSafePointSuccessorEnumerator public: // Constructs an enumerator of successors to be used for checking for GC // safe point cycles. - GCSafePointSuccessorEnumerator(Compiler* comp, BasicBlock* block) : m_block(block) + GCSafePointSuccessorEnumerator(Compiler* comp, BasicBlock* block) + : m_block(block) { m_numSuccs = 0; block->VisitRegularSuccs(comp, [this](BasicBlock* succ) { @@ -4212,7 +4217,9 @@ unsigned FlowGraphNaturalLoop::NumLoopBlocks() // dfs - A DFS tree. // FlowGraphNaturalLoops::FlowGraphNaturalLoops(const FlowGraphDfsTree* dfsTree) - : m_dfsTree(dfsTree), m_loops(m_dfsTree->GetCompiler()->getAllocator(CMK_Loops)), m_improperLoopHeaders(0) + : m_dfsTree(dfsTree) + , m_loops(m_dfsTree->GetCompiler()->getAllocator(CMK_Loops)) + , m_improperLoopHeaders(0) { } @@ -4847,7 +4854,9 @@ bool FlowGraphNaturalLoop::VisitDefs(TFunc func) DoPreOrder = true, }; - VisitDefsVisitor(Compiler* comp, TFunc& func) : GenTreeVisitor(comp), m_func(func) + VisitDefsVisitor(Compiler* comp, TFunc& func) + : GenTreeVisitor(comp) + , m_func(func) { } @@ -6098,7 +6107,9 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df public: NumberDomTreeVisitor(Compiler* comp, unsigned* preorderNums, unsigned* postorderNums) - : DomTreeVisitor(comp), m_preorderNums(preorderNums), m_postorderNums(postorderNums) + : DomTreeVisitor(comp) + , m_preorderNums(preorderNums) + , m_postorderNums(postorderNums) { } diff --git a/src/coreclr/jit/forwardsub.cpp b/src/coreclr/jit/forwardsub.cpp index 8e450d7cbb35e1..de4ac5fe8a4758 100644 --- a/src/coreclr/jit/forwardsub.cpp +++ b/src/coreclr/jit/forwardsub.cpp @@ -191,7 +191,9 @@ class ForwardSubVisitor final : public GenTreeVisitor UseExecutionOrder = true }; - ForwardSubVisitor(Compiler* compiler, unsigned lclNum) : GenTreeVisitor(compiler), m_lclNum(lclNum) + ForwardSubVisitor(Compiler* compiler, unsigned lclNum) + : GenTreeVisitor(compiler) + , m_lclNum(lclNum) { LclVarDsc* dsc = compiler->lvaGetDesc(m_lclNum); if (dsc->lvIsStructField) @@ -399,7 +401,9 @@ class EffectsVisitor final : public GenTreeVisitor UseExecutionOrder = true }; - EffectsVisitor(Compiler* compiler) : GenTreeVisitor(compiler), m_flags(GTF_EMPTY) + EffectsVisitor(Compiler* compiler) + : GenTreeVisitor(compiler) + , m_flags(GTF_EMPTY) { } diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index f5a7d9b9f00e38..836d2bc33631ba 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -433,12 +433,13 @@ static void regenLog(unsigned encoding, InfoHdr* header, InfoHdr* state) EnterCriticalSection(&logFileLock); - fprintf(logFile, "InfoHdr( %2d, %2d, %1d, %1d, %1d," - " %1d, %1d, %1d, %1d, %1d," - " %1d, %1d, %1d, %1d, %1d, %1d," - " %1d, %1d, %1d," - " %1d, %2d, %2d," - " %2d, %2d, %2d, %2d, %2d, %2d), \n", + fprintf(logFile, + "InfoHdr( %2d, %2d, %1d, %1d, %1d," + " %1d, %1d, %1d, %1d, %1d," + " %1d, %1d, %1d, %1d, %1d, %1d," + " %1d, %1d, %1d," + " %1d, %2d, %2d," + " %2d, %2d, %2d, %2d, %2d, %2d), \n", state->prologSize, state->epilogSize, state->epilogCount, state->epilogAtEnd, state->ediSaved, state->esiSaved, state->ebxSaved, state->ebpSaved, state->ebpFrame, state->interruptible, state->doubleAlign, state->security, state->handlers, state->localloc, state->editNcontinue, state->varargs, @@ -1817,7 +1818,7 @@ static int (*zeroFunc)() = zeroFN; */ typedef unsigned pasMaskType; -#define BITS_IN_pasMask (BITS_PER_BYTE * sizeof(pasMaskType)) +#define BITS_IN_pasMask (BITS_PER_BYTE * sizeof(pasMaskType)) #define HIGHEST_pasMask_BIT (((pasMaskType)0x1) << (BITS_IN_pasMask - 1)) //----------------------------------------------------------------------------- @@ -1850,8 +1851,8 @@ class PendingArgsStack // Use these in the case where there actually are more ptrs than pasArgMask unsigned pasEnumGCoffsCount(); #define pasENUM_START ((unsigned)-1) -#define pasENUM_LAST ((unsigned)-2) -#define pasENUM_END ((unsigned)-3) +#define pasENUM_LAST ((unsigned)-2) +#define pasENUM_END ((unsigned)-3) unsigned pasEnumGCoffs(unsigned iter, unsigned* offs); protected: @@ -2331,7 +2332,7 @@ size_t GCInfo::gcMakeRegPtrTable(BYTE* dest, int mask, const InfoHdr& header, un unsigned varOffs = compiler->lvaTable[compiler->info.compThisArg].GetStackOffset(); /* For negative stack offsets we must reset the low bits, - * take abs and then set them back */ + * take abs and then set them back */ varOffs = abs(static_cast(varOffs)); varOffs |= this_OFFSET_FLAG; @@ -3285,7 +3286,7 @@ size_t GCInfo::gcMakeRegPtrTable(BYTE* dest, int mask, const InfoHdr& header, un assert(regMask || argMask || callArgCnt || pasStk.pasCurDepth()); -// Emit IPtrMask if needed + // Emit IPtrMask if needed #define CHK_NON_INTRPT_ESP_IPtrMask \ \ @@ -3571,7 +3572,7 @@ size_t GCInfo::gcInfoBlockHdrDump(const BYTE* table, InfoHdr* header, unsigned* #ifdef DEBUG gcDump.gcPrintf = gcDump_logf; // use my printf (which logs to VM) #else - gcDump.gcPrintf = printf; + gcDump.gcPrintf = printf; #endif printf("Method info block:\n"); @@ -3590,7 +3591,7 @@ size_t GCInfo::gcDumpPtrTable(const BYTE* table, const InfoHdr& header, unsigned #ifdef DEBUG gcDump.gcPrintf = gcDump_logf; // use my printf (which logs to VM) #else - gcDump.gcPrintf = printf; + gcDump.gcPrintf = printf; #endif return gcDump.DumpGCTable(table, header, methodSize, verifyGCTables); @@ -3608,7 +3609,7 @@ void GCInfo::gcFindPtrsInFrame(const void* infoBlock, const void* codeBlock, uns #ifdef DEBUG gcDump.gcPrintf = gcDump_logf; // use my printf (which logs to VM) #else - gcDump.gcPrintf = printf; + gcDump.gcPrintf = printf; #endif gcDump.DumpPtrsInFrame((PTR_CBYTE)infoBlock, (const BYTE*)codeBlock, offs, verifyGCTables); @@ -3646,7 +3647,8 @@ class GcInfoEncoderWithLogging public: GcInfoEncoderWithLogging(GcInfoEncoder* gcInfoEncoder, bool verbose) - : m_gcInfoEncoder(gcInfoEncoder), m_doLogging(verbose INDEBUG(|| JitConfig.JitGCInfoLogging() != 0)) + : m_gcInfoEncoder(gcInfoEncoder) + , m_doLogging(verbose INDEBUG(|| JitConfig.JitGCInfoLogging() != 0)) { } @@ -4024,7 +4026,8 @@ struct InterruptibleRangeReporter Encoder* gcInfoEncoderWithLog; InterruptibleRangeReporter(unsigned _prevStart, Encoder* _gcInfo) - : prevStart(_prevStart), gcInfoEncoderWithLog(_gcInfo) + : prevStart(_prevStart) + , gcInfoEncoderWithLog(_gcInfo) { } @@ -4793,7 +4796,7 @@ void GCInfo::gcInfoRecordGCStackArgLive(GcInfoEncoder* gcInfoEncoder, MakeRegPtr StackSlotIdKey sskey(genStackPtr->rpdPtrArg, false, GcSlotFlags(genStackPtr->rpdGCtypeGet() == GCT_BYREF ? GC_SLOT_INTERIOR : GC_SLOT_BASE)); - GcSlotId varSlotId; + GcSlotId varSlotId; if (mode == MAKE_REG_PTR_MODE_ASSIGN_SLOTS) { if (!m_stackSlotMap->Lookup(sskey, &varSlotId)) @@ -4841,8 +4844,8 @@ void GCInfo::gcInfoRecordGCStackArgsDead(GcInfoEncoder* gcInfoEncoder, StackSlotIdKey sskey(genRegPtrTemp->rpdPtrArg, false, genRegPtrTemp->rpdGCtypeGet() == GCT_BYREF ? GC_SLOT_INTERIOR : GC_SLOT_BASE); - GcSlotId varSlotId; - bool b = m_stackSlotMap->Lookup(sskey, &varSlotId); + GcSlotId varSlotId; + bool b = m_stackSlotMap->Lookup(sskey, &varSlotId); assert(b); // Should have been added in the first pass. // Live until the call. gcInfoEncoderWithLog->SetSlotState(instrOffset, varSlotId, GC_SLOT_DEAD); diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 0cdbd5e94a15ce..9706ebd3e5376d 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -46,7 +46,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ -GCInfo::GCInfo(Compiler* theCompiler) : compiler(theCompiler) +GCInfo::GCInfo(Compiler* theCompiler) + : compiler(theCompiler) { regSet = nullptr; gcVarPtrList = nullptr; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index d16355014073e0..0f911a50a94bad 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -21,13 +21,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX /*****************************************************************************/ const unsigned char GenTree::gtOperKindTable[] = { -#define GTNODE(en, st, cm, ivn, ok) ((ok)>K_MASK) + GTK_COMMUTE *cm, +#define GTNODE(en, st, cm, ivn, ok) ((ok) & GTK_MASK) + GTK_COMMUTE *cm, #include "gtlist.h" }; #ifdef DEBUG const GenTreeDebugOperKind GenTree::gtDebugOperKindTable[] = { -#define GTNODE(en, st, cm, ivn, ok) static_cast((ok)&DBK_MASK), +#define GTNODE(en, st, cm, ivn, ok) static_cast((ok) & DBK_MASK), #include "gtlist.h" }; #endif // DEBUG @@ -78,7 +78,8 @@ struct IndentStack const char** indents; // Constructor for IndentStack. Uses 'compiler' to determine the mode of printing. - IndentStack(Compiler* compiler) : stack(compiler->getAllocator(CMK_DebugOnly)) + IndentStack(Compiler* compiler) + : stack(compiler->getAllocator(CMK_DebugOnly)) { if (compiler->asciiTrees) { @@ -3309,7 +3310,8 @@ bool Compiler::gtHasLocalsWithAddrOp(GenTree* tree) DoLclVarsOnly = true, }; - LocalsWithAddrOpVisitor(Compiler* comp) : GenTreeVisitor(comp) + LocalsWithAddrOpVisitor(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -3350,7 +3352,8 @@ bool Compiler::gtHasAddressExposedLocals(GenTree* tree) DoLclVarsOnly = true, }; - Visitor(Compiler* comp) : GenTreeVisitor(comp) + Visitor(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -3439,7 +3442,7 @@ unsigned Compiler::gtHashValue(GenTree* tree) #ifdef HOST_64BIT add = bits; #else // 32-bit host - add = genTreeHashAdd(uhi32(bits), ulo32(bits)); + add = genTreeHashAdd(uhi32(bits), ulo32(bits)); #endif break; case GT_CNS_DBL: @@ -3449,7 +3452,7 @@ unsigned Compiler::gtHashValue(GenTree* tree) #ifdef HOST_64BIT add = bits; #else // 32-bit host - add = genTreeHashAdd(uhi32(bits), ulo32(bits)); + add = genTreeHashAdd(uhi32(bits), ulo32(bits)); #endif break; } @@ -6512,7 +6515,9 @@ bool Compiler::gtMayHaveStoreInterference(GenTree* treeWithStores, GenTree* tree DoPreOrder = true, }; - Visitor(Compiler* compiler, GenTree* readTree) : GenTreeVisitor(compiler), m_readTree(readTree) + Visitor(Compiler* compiler, GenTree* readTree) + : GenTreeVisitor(compiler) + , m_readTree(readTree) { } @@ -6573,7 +6578,9 @@ bool Compiler::gtTreeHasLocalRead(GenTree* tree, unsigned lclNum) unsigned m_lclNum; LclVarDsc* m_lclDsc; - Visitor(Compiler* compiler, unsigned lclNum) : GenTreeVisitor(compiler), m_lclNum(lclNum) + Visitor(Compiler* compiler, unsigned lclNum) + : GenTreeVisitor(compiler) + , m_lclNum(lclNum) { m_lclDsc = compiler->lvaGetDesc(lclNum); } @@ -7556,7 +7563,7 @@ GenTree::VtablePtr GenTree::GetVtableForOper(genTreeOps oper) switch (oper) { -// clang-format off + // clang-format off #define GTSTRUCT_0(nm, tag) /*handle explicitly*/ #define GTSTRUCT_1(nm, tag) \ @@ -7618,8 +7625,8 @@ GenTree::VtablePtr GenTree::GetVtableForOper(genTreeOps oper) } break; - // We don't need to handle GTSTRUCT_N for LclVarCommon, since all those allowed opers are specified - // in their proper subtype. Similarly for GenTreeIndir. + // We don't need to handle GTSTRUCT_N for LclVarCommon, since all those allowed opers are specified + // in their proper subtype. Similarly for GenTreeIndir. default: { @@ -9102,7 +9109,7 @@ GenTree* Compiler::gtNewPutArgReg(var_types type, GenTree* arg, regNumber argReg node->AsMultiRegOp()->gtOtherReg = REG_NEXT(argReg); } #else - node = gtNewOperNode(GT_PUTARG_REG, type, arg); + node = gtNewOperNode(GT_PUTARG_REG, type, arg); #endif node->SetRegNum(argReg); @@ -9133,7 +9140,7 @@ GenTree* Compiler::gtNewBitCastNode(var_types type, GenTree* arg) // A BITCAST could be a MultiRegOp on arm since we could move a double register to two int registers. node = new (this, GT_BITCAST) GenTreeMultiRegOp(GT_BITCAST, type, arg, nullptr); #else - node = gtNewOperNode(GT_BITCAST, type, arg); + node = gtNewOperNode(GT_BITCAST, type, arg); #endif return node; @@ -9538,7 +9545,7 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) tree->AsLclFld()->Data(), tree->AsLclFld()->GetLayout()); break; - /* These nodes sometimes get bashed to "fat" ones */ + /* These nodes sometimes get bashed to "fat" ones */ case GT_MUL: case GT_DIV: @@ -9920,7 +9927,9 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree) copy->gtCallMoreFlags = tree->gtCallMoreFlags; INDEBUG(copy->gtCallDebugFlags = tree->gtCallDebugFlags); - copy->gtArgs.InternalCopyFrom(this, &tree->gtArgs, [=](GenTree* node) { return gtCloneExpr(node); }); + copy->gtArgs.InternalCopyFrom(this, &tree->gtArgs, [=](GenTree* node) { + return gtCloneExpr(node); + }); // The call sig comes from the EE and doesn't change throughout the compilation process, meaning // we only really need one physical copy of it. Therefore a shallow pointer copy will suffice. @@ -10070,7 +10079,8 @@ void Compiler::gtUpdateStmtSideEffects(Statement* stmt) DoPostOrder = true, }; - UpdateSideEffectsWalker(Compiler* comp) : GenTreeVisitor(comp) + UpdateSideEffectsWalker(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -10267,12 +10277,20 @@ bool GenTree::gtRequestSetFlags() } GenTreeUseEdgeIterator::GenTreeUseEdgeIterator() - : m_advance(nullptr), m_node(nullptr), m_edge(nullptr), m_statePtr(nullptr), m_state(-1) + : m_advance(nullptr) + , m_node(nullptr) + , m_edge(nullptr) + , m_statePtr(nullptr) + , m_state(-1) { } GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) - : m_advance(nullptr), m_node(node), m_edge(nullptr), m_statePtr(nullptr), m_state(0) + : m_advance(nullptr) + , m_node(node) + , m_edge(nullptr) + , m_statePtr(nullptr) + , m_state(0) { assert(m_node != nullptr); @@ -10556,7 +10574,7 @@ void GenTreeUseEdgeIterator::AdvanceConditional() // `GTF_REVERSE_OPS` flag. // template -void GenTreeUseEdgeIterator::AdvanceBinOp() +void GenTreeUseEdgeIterator::AdvanceBinOp() { assert(ReverseOperands == ((m_node->gtFlags & GTF_REVERSE_OPS) != 0)); @@ -10679,7 +10697,7 @@ void GenTreeUseEdgeIterator::SetEntryStateForMultiOp() // component operands. // template -void GenTreeUseEdgeIterator::AdvanceCall() +void GenTreeUseEdgeIterator::AdvanceCall() { GenTreeCall* const call = m_node->AsCall(); @@ -10892,10 +10910,12 @@ bool GenTree::HandleKindDataIsInvariant(GenTreeFlags flags) printf("%c", (flags & GTF_EXCEPT) ? 'X' : '-'); printf("%c", (flags & GTF_GLOB_REF) ? 'G' : '-'); printf("%c", (debugFlags & GTF_DEBUG_NODE_MORPHED) ? '+' : // First print '+' if GTF_DEBUG_NODE_MORPHED is set - (flags & GTF_ORDER_SIDEEFF) ? 'O' : '-'); // otherwise print 'O' or '-' + (flags & GTF_ORDER_SIDEEFF) ? 'O' + : '-'); // otherwise print 'O' or '-' printf("%c", (flags & GTF_COLON_COND) ? '?' : '-'); - printf("%c", (flags & GTF_DONT_CSE) ? 'N' : // N is for No cse - (flags & GTF_MAKE_CSE) ? 'H' : '-'); // H is for Hoist this expr + printf("%c", (flags & GTF_DONT_CSE) ? 'N' : // N is for No cse + (flags & GTF_MAKE_CSE) ? 'H' + : '-'); // H is for Hoist this expr printf("%c", (flags & GTF_REVERSE_OPS) ? 'R' : '-'); printf("%c", (flags & GTF_UNSIGNED) ? 'U' : (flags & GTF_BOOLEAN) ? 'B' : '-'); #if FEATURE_SET_FLAGS @@ -11744,8 +11764,8 @@ void Compiler::gtDispRegVal(GenTree* tree) { switch (tree->GetRegTag()) { - // Don't display anything for the GT_REGTAG_NONE case; - // the absence of printed register values will imply this state. + // Don't display anything for the GT_REGTAG_NONE case; + // the absence of printed register values will imply this state. case GenTree::GT_REGTAG_REG: printf(" REG %s", compRegVarName(tree->GetRegNum())); @@ -11773,9 +11793,9 @@ void Compiler::gtDispRegVal(GenTree* tree) } // We usually/commonly don't expect to print anything longer than this string, -#define LONGEST_COMMON_LCL_VAR_DISPLAY "V99 PInvokeFrame" +#define LONGEST_COMMON_LCL_VAR_DISPLAY "V99 PInvokeFrame" #define LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH (sizeof(LONGEST_COMMON_LCL_VAR_DISPLAY)) -#define BUF_SIZE (LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH * 2) +#define BUF_SIZE (LONGEST_COMMON_LCL_VAR_DISPLAY_LENGTH * 2) void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, const char** ilNameOut, unsigned* ilNumOut) { @@ -12010,7 +12030,7 @@ static const char* InsCflagsToString(insCflags flags) { const static char* s_table[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; - unsigned index = (unsigned)flags; + unsigned index = (unsigned)flags; assert((0 <= index) && (index < ArrLen(s_table))); return s_table[index]; } @@ -12421,7 +12441,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) break; #endif // !FEATURE_EH_FUNCLETS - // Vanilla leaves. No qualifying information available. So do nothing + // Vanilla leaves. No qualifying information available. So do nothing case GT_NOP: case GT_NO_OP: @@ -12573,8 +12593,8 @@ void Compiler::gtDispLocal(GenTreeLclVarCommon* tree, IndentStack* indentStack) void Compiler::gtDispChild(GenTree* child, IndentStack* indentStack, IndentInfo arcType, - _In_opt_ const char* msg, /* = nullptr */ - bool topOnly) /* = false */ + _In_opt_ const char* msg, /* = nullptr */ + bool topOnly) /* = false */ { indentStack->Push(arcType); gtDispTree(child, indentStack, msg, topOnly); @@ -12583,11 +12603,11 @@ void Compiler::gtDispChild(GenTree* child, /*****************************************************************************/ -void Compiler::gtDispTree(GenTree* tree, - IndentStack* indentStack, /* = nullptr */ - _In_ _In_opt_z_ const char* msg, /* = nullptr */ - bool topOnly, /* = false */ - bool isLIR) /* = false */ +void Compiler::gtDispTree(GenTree* tree, + IndentStack* indentStack, /* = nullptr */ + _In_ _In_opt_z_ const char* msg, /* = nullptr */ + bool topOnly, /* = false */ + bool isLIR) /* = false */ { if (tree == nullptr) { @@ -13084,9 +13104,10 @@ void Compiler::gtDispTree(GenTree* tree, case GT_HWINTRINSIC: if (tree->OperIs(GT_HWINTRINSIC)) { - printf(" %s %s", tree->AsHWIntrinsic()->GetSimdBaseType() == TYP_UNKNOWN - ? "" - : varTypeName(tree->AsHWIntrinsic()->GetSimdBaseType()), + printf(" %s %s", + tree->AsHWIntrinsic()->GetSimdBaseType() == TYP_UNKNOWN + ? "" + : varTypeName(tree->AsHWIntrinsic()->GetSimdBaseType()), HWIntrinsicInfo::lookupName(tree->AsHWIntrinsic()->GetHWIntrinsicId())); } @@ -15640,8 +15661,8 @@ GenTree* Compiler::gtFoldExprConst(GenTree* tree) switch (switchType) { - // Fold constant REF of BYREF binary operator. - // These can only be comparisons or null pointers. + // Fold constant REF of BYREF binary operator. + // These can only be comparisons or null pointers. case TYP_REF: @@ -15710,7 +15731,7 @@ GenTree* Compiler::gtFoldExprConst(GenTree* tree) return tree; - // Fold constant INT binary operator. + // Fold constant INT binary operator. case TYP_INT: @@ -15837,8 +15858,8 @@ GenTree* Compiler::gtFoldExprConst(GenTree* tree) i1 = (i1 << ((32 - i2) & 0x1f)) | (UINT32(i1) >> (i2 & 0x1f)); break; - // DIV and MOD can throw an exception - if the division is by 0 - // or there is overflow - when dividing MIN by -1. + // DIV and MOD can throw an exception - if the division is by 0 + // or there is overflow - when dividing MIN by -1. case GT_DIV: case GT_MOD: @@ -15907,7 +15928,7 @@ GenTree* Compiler::gtFoldExprConst(GenTree* tree) goto DONE; - // Fold constant LONG binary operator. + // Fold constant LONG binary operator. case TYP_LONG: @@ -16130,7 +16151,7 @@ GenTree* Compiler::gtFoldExprConst(GenTree* tree) goto DONE; - // Fold constant FLOAT or DOUBLE binary operator + // Fold constant FLOAT or DOUBLE binary operator case TYP_FLOAT: case TYP_DOUBLE: @@ -17168,7 +17189,9 @@ void Compiler::gtExtractSideEffList(GenTree* expr, return m_result; } - SideEffectExtractor(Compiler* compiler, GenTreeFlags flags) : GenTreeVisitor(compiler), m_flags(flags) + SideEffectExtractor(Compiler* compiler, GenTreeFlags flags) + : GenTreeVisitor(compiler) + , m_flags(flags) { } @@ -17449,7 +17472,9 @@ Compiler::FindLinkData Compiler::gtFindLink(Statement* stmt, GenTree* node) DoPreOrder = true, }; - FindLinkWalker(Compiler* comp, GenTree* node) : GenTreeVisitor(comp), m_node(node) + FindLinkWalker(Compiler* comp, GenTree* node) + : GenTreeVisitor(comp) + , m_node(node) { } @@ -17635,7 +17660,9 @@ bool Compiler::gtTreeContainsOper(GenTree* tree, genTreeOps oper) genTreeOps m_oper; public: - Visitor(Compiler* comp, genTreeOps oper) : GenTreeVisitor(comp), m_oper(oper) + Visitor(Compiler* comp, genTreeOps oper) + : GenTreeVisitor(comp) + , m_oper(oper) { } @@ -17676,7 +17703,8 @@ ExceptionSetFlags Compiler::gtCollectExceptions(GenTree* tree) ExceptionSetFlags m_preciseExceptions = ExceptionSetFlags::None; public: - ExceptionsWalker(Compiler* comp) : GenTreeVisitor(comp) + ExceptionsWalker(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -17734,7 +17762,9 @@ bool Compiler::gtComplexityExceeds(GenTree* tree, unsigned limit) DoPreOrder = true, }; - ComplexityVisitor(Compiler* comp, unsigned limit) : GenTreeVisitor(comp), m_limit(limit) + ComplexityVisitor(Compiler* comp, unsigned limit) + : GenTreeVisitor(comp) + , m_limit(limit) { } @@ -19503,7 +19533,8 @@ FieldSeq* FieldSeqStore::Append(FieldSeq* a, FieldSeq* b) return nullptr; } -FieldSeq::FieldSeq(CORINFO_FIELD_HANDLE fieldHnd, ssize_t offset, FieldKind fieldKind) : m_offset(offset) +FieldSeq::FieldSeq(CORINFO_FIELD_HANDLE fieldHnd, ssize_t offset, FieldKind fieldKind) + : m_offset(offset) { assert(fieldHnd != NO_FIELD_HANDLE); @@ -20719,8 +20750,8 @@ GenTree* Compiler::gtNewSimdBinOpNode( assert(!compIsaSupportedDebugOnly(InstructionSet_AVX512F_VL)); // Vector256 maskedProduct = Avx2.And(widenedProduct, vecCon1).AsInt16() - GenTree* maskedProduct = gtNewSimdBinOpNode(GT_AND, widenedType, widenedProduct, vecCon1, - widenedSimdBaseJitType, widenedSimdSize); + GenTree* maskedProduct = gtNewSimdBinOpNode(GT_AND, widenedType, widenedProduct, vecCon1, + widenedSimdBaseJitType, widenedSimdSize); GenTree* maskedProductDup = fgMakeMultiUse(&maskedProduct); // Vector256 packedProduct = Avx2.PackUnsignedSaturate(maskedProduct, @@ -21717,10 +21748,10 @@ GenTree* Compiler::gtNewSimdCmpOpNode( op1 = gtNewSimdHWIntrinsicNode(type, t, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, CORINFO_TYPE_INT, simdSize); - u = gtNewSimdHWIntrinsicNode(type, u, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); - v = gtNewSimdHWIntrinsicNode(type, v, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); + u = gtNewSimdHWIntrinsicNode(type, u, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, + CORINFO_TYPE_INT, simdSize); + v = gtNewSimdHWIntrinsicNode(type, v, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), NI_SSE2_Shuffle, + CORINFO_TYPE_INT, simdSize); // Validate we can't use AVX512F_VL_TernaryLogic here assert(!compIsaSupportedDebugOnly(InstructionSet_AVX512F_VL)); @@ -21972,10 +22003,10 @@ GenTree* Compiler::gtNewSimdCmpOpNode( op1 = gtNewSimdHWIntrinsicNode(type, t, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, CORINFO_TYPE_INT, simdSize); - u = gtNewSimdHWIntrinsicNode(type, u, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); - v = gtNewSimdHWIntrinsicNode(type, v, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), NI_SSE2_Shuffle, - CORINFO_TYPE_INT, simdSize); + u = gtNewSimdHWIntrinsicNode(type, u, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), NI_SSE2_Shuffle, + CORINFO_TYPE_INT, simdSize); + v = gtNewSimdHWIntrinsicNode(type, v, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), NI_SSE2_Shuffle, + CORINFO_TYPE_INT, simdSize); // Validate we can't use AVX512F_VL_TernaryLogic here assert(!compIsaSupportedDebugOnly(InstructionSet_AVX512F_VL)); @@ -25121,8 +25152,8 @@ GenTree* Compiler::gtNewSimdSumNode(var_types type, GenTree* op1, CorInfoType si tmp = fgMakeMultiUse(&op1); opShifted = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(shiftVal, TYP_INT), NI_SSE2_ShiftRightLogical128BitLane, simdBaseJitType, simdSize); - op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, opShifted, tmp, simdBaseJitType, simdSize); - shiftVal = shiftVal / 2; + op1 = gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, opShifted, tmp, simdBaseJitType, simdSize); + shiftVal = shiftVal / 2; } return gtNewSimdToScalarNode(type, op1, simdBaseJitType, simdSize); @@ -26875,7 +26906,7 @@ genTreeOps GenTreeHWIntrinsic::HWOperGet() const return GT_AND_NOT; } #endif - // TODO: Handle other cases + // TODO: Handle other cases default: { @@ -27023,7 +27054,7 @@ void ReturnTypeDesc::InitializeStructReturnType(Compiler* comp, #else uint32_t floatFieldFlags = comp->info.compCompHnd->getRISCV64PassStructInRegisterFlags(retClsHnd); #endif - BYTE gcPtrs[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; + BYTE gcPtrs[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; comp->info.compCompHnd->getClassGClayout(retClsHnd, &gcPtrs[0]); if (floatFieldFlags & STRUCT_FLOAT_FIELD_ONLY_TWO) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index ebc73e51e8c53e..21ec294ac2d592 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -156,10 +156,10 @@ inline ExceptionSetFlags& operator&=(ExceptionSetFlags& a, ExceptionSetFlags b) #ifdef DEBUG /***************************************************************************** -* -* TargetHandleTypes are used to determine the type of handle present inside GenTreeIntCon node. -* The values are such that they don't overlap with helper's or user function's handle. -*/ + * + * TargetHandleTypes are used to determine the type of handle present inside GenTreeIntCon node. + * The values are such that they don't overlap with helper's or user function's handle. + */ enum TargetHandleType : BYTE { THT_Unknown = 2, @@ -200,17 +200,20 @@ class AssertionInfo unsigned short m_assertionIndex : 15; AssertionInfo(bool assertionHoldsOnFalseEdge, AssertionIndex assertionIndex) - : m_assertionHoldsOnFalseEdge(assertionHoldsOnFalseEdge), m_assertionIndex(assertionIndex) + : m_assertionHoldsOnFalseEdge(assertionHoldsOnFalseEdge) + , m_assertionIndex(assertionIndex) { assert(m_assertionIndex == assertionIndex); } public: - AssertionInfo() : AssertionInfo(false, 0) + AssertionInfo() + : AssertionInfo(false, 0) { } - AssertionInfo(AssertionIndex assertionIndex) : AssertionInfo(false, assertionIndex) + AssertionInfo(AssertionIndex assertionIndex) + : AssertionInfo(false, assertionIndex) { } @@ -314,7 +317,8 @@ class FieldSeqStore JitHashTable, FieldSeq> m_map; public: - FieldSeqStore(CompAllocator alloc) : m_map(alloc) + FieldSeqStore(CompAllocator alloc) + : m_map(alloc) { } @@ -331,13 +335,13 @@ struct Statement; /*****************************************************************************/ // Forward declarations of the subtypes -#define GTSTRUCT_0(fn, en) struct GenTree##fn; -#define GTSTRUCT_1(fn, en) struct GenTree##fn; -#define GTSTRUCT_2(fn, en, en2) struct GenTree##fn; -#define GTSTRUCT_3(fn, en, en2, en3) struct GenTree##fn; -#define GTSTRUCT_4(fn, en, en2, en3, en4) struct GenTree##fn; -#define GTSTRUCT_N(fn, ...) struct GenTree##fn; -#define GTSTRUCT_2_SPECIAL(fn, en, en2) GTSTRUCT_2(fn, en, en2) +#define GTSTRUCT_0(fn, en) struct GenTree##fn; +#define GTSTRUCT_1(fn, en) struct GenTree##fn; +#define GTSTRUCT_2(fn, en, en2) struct GenTree##fn; +#define GTSTRUCT_3(fn, en, en2, en3) struct GenTree##fn; +#define GTSTRUCT_4(fn, en, en2, en3, en4) struct GenTree##fn; +#define GTSTRUCT_N(fn, ...) struct GenTree##fn; +#define GTSTRUCT_2_SPECIAL(fn, en, en2) GTSTRUCT_2(fn, en, en2) #define GTSTRUCT_3_SPECIAL(fn, en, en2, en3) GTSTRUCT_3(fn, en, en2, en3) #include "gtstructs.h" @@ -690,11 +694,11 @@ struct GenTree return *As##fn(); \ } -#define GTSTRUCT_1(fn, en) GTSTRUCT_N(fn, en) -#define GTSTRUCT_2(fn, en, en2) GTSTRUCT_N(fn, en, en2) -#define GTSTRUCT_3(fn, en, en2, en3) GTSTRUCT_N(fn, en, en2, en3) -#define GTSTRUCT_4(fn, en, en2, en3, en4) GTSTRUCT_N(fn, en, en2, en3, en4) -#define GTSTRUCT_2_SPECIAL(fn, en, en2) GTSTRUCT_2(fn, en, en2) +#define GTSTRUCT_1(fn, en) GTSTRUCT_N(fn, en) +#define GTSTRUCT_2(fn, en, en2) GTSTRUCT_N(fn, en, en2) +#define GTSTRUCT_3(fn, en, en2, en3) GTSTRUCT_N(fn, en, en2, en3) +#define GTSTRUCT_4(fn, en, en2, en3, en4) GTSTRUCT_N(fn, en, en2, en3, en4) +#define GTSTRUCT_2_SPECIAL(fn, en, en2) GTSTRUCT_2(fn, en, en2) #define GTSTRUCT_3_SPECIAL(fn, en, en2, en3) GTSTRUCT_3(fn, en, en2, en3) #include "gtstructs.h" @@ -719,11 +723,11 @@ struct GenTree #define NO_CSE (0) -#define IS_CSE_INDEX(x) ((x) != 0) -#define IS_CSE_USE(x) ((x) > 0) -#define IS_CSE_DEF(x) ((x) < 0) +#define IS_CSE_INDEX(x) ((x) != 0) +#define IS_CSE_USE(x) ((x) > 0) +#define IS_CSE_DEF(x) ((x) < 0) #define GET_CSE_INDEX(x) (((x) > 0) ? x : -(x)) -#define TO_CSE_DEF(x) (-(x)) +#define TO_CSE_DEF(x) (-(x)) signed char gtCSEnum; // 0 or the CSE index (negated if def) // valid only for CSE expressions @@ -766,7 +770,7 @@ struct GenTree bool gtCostsInitialized; #endif // DEBUG -#define MAX_COST UCHAR_MAX +#define MAX_COST UCHAR_MAX #define IND_COST_EX 3 // execution cost for an indirection unsigned char GetCostEx() const @@ -968,7 +972,7 @@ struct GenTree regMaskOnlyOne gtRsvdRegs; // set of fixed trashed registers - unsigned AvailableTempRegCount(regMaskOnlyOne mask = (regMaskOnlyOne)-1) const; + unsigned AvailableTempRegCount(regMaskOnlyOne mask = (regMaskOnlyOne)-1) const; regNumber GetSingleTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); regNumber ExtractTempReg(regMaskOnlyOne mask = (regMaskOnlyOne)-1); @@ -1532,7 +1536,7 @@ struct GenTree #if !defined(TARGET_64BIT) || (gtOper == GT_ADD_HI) || (gtOper == GT_SUB_HI) #endif - ); + ); } bool OperMayOverflow() const @@ -1777,9 +1781,9 @@ struct GenTree return (DebugOperKind() & DBK_NOTLIR) == 0; } - bool OperSupportsReverseOpEvalOrder(Compiler* comp) const; + bool OperSupportsReverseOpEvalOrder(Compiler* comp) const; static bool RequiresNonNullOp2(genTreeOps oper); - bool IsValidCallArgument(); + bool IsValidCallArgument(); #endif // DEBUG inline bool IsIntegralConst(ssize_t constVal) const; @@ -1880,7 +1884,7 @@ struct GenTree bool OperRequiresCallFlag(Compiler* comp) const; ExceptionSetFlags OperExceptions(Compiler* comp); - bool OperMayThrow(Compiler* comp); + bool OperMayThrow(Compiler* comp); bool OperRequiresGlobRefFlag(Compiler* comp) const; @@ -1917,7 +1921,7 @@ struct GenTree static bool Compare(GenTree* op1, GenTree* op2, bool swapOK = false); -//--------------------------------------------------------------------- + //--------------------------------------------------------------------- #if defined(DEBUG) || CALL_ARG_STATS || COUNT_BASIC_BLOCKS || COUNT_LOOPS || EMITTER_STATS || MEASURE_MEM_ALLOC || \ NODEBASH_STATS || MEASURE_NODE_SIZE || COUNT_AST_OPERS || DUMP_FLOWGRAPHS @@ -1961,8 +1965,8 @@ struct GenTree } template - void BashToConst(T value, var_types type = TYP_UNDEF); - void BashToZeroConst(var_types type); + void BashToConst(T value, var_types type = TYP_UNDEF); + void BashToZeroConst(var_types type); GenTreeLclVar* BashToLclVar(Compiler* comp, unsigned lclNum); #if NODEBASH_STATS @@ -2000,7 +2004,7 @@ struct GenTree unsigned* pSize = nullptr); GenTreeLclVarCommon* IsImplicitByrefParameterValuePreMorph(Compiler* compiler); - GenTreeLclVar* IsImplicitByrefParameterValuePostMorph(Compiler* compiler, GenTree** addr); + GenTreeLclVar* IsImplicitByrefParameterValuePostMorph(Compiler* compiler, GenTree** addr); // Determine whether this is an assignment tree of the form X = X (op) Y, // where Y is an arbitrary tree, and X is a lclVar. @@ -2279,7 +2283,7 @@ struct GenTree bool gtRequestSetFlags(); #ifdef DEBUG - static int gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags); + static int gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags); static const char* gtGetHandleKindString(GenTreeFlags flags); #endif @@ -2398,7 +2402,7 @@ struct GenTree typedef void* VtablePtr; VtablePtr GetVtableForOper(genTreeOps oper); - void SetVtableForOper(genTreeOps oper); + void SetVtableForOper(genTreeOps oper); static VtablePtr s_vtablesForOpers[GT_COUNT]; static VtablePtr s_vtableForOp; @@ -2432,7 +2436,9 @@ struct GenTreePhi final : public GenTree Use* m_next; public: - Use(GenTree* node, Use* next = nullptr) : m_node(node), m_next(next) + Use(GenTree* node, Use* next = nullptr) + : m_node(node) + , m_next(next) { assert(node->OperIs(GT_PHI_ARG)); } @@ -2470,7 +2476,8 @@ struct GenTreePhi final : public GenTree Use* m_use; public: - UseIterator(Use* use) : m_use(use) + UseIterator(Use* use) + : m_use(use) { } @@ -2506,7 +2513,8 @@ struct GenTreePhi final : public GenTree Use* m_uses; public: - UseList(Use* uses) : m_uses(uses) + UseList(Use* uses) + : m_uses(uses) { } @@ -2523,7 +2531,9 @@ struct GenTreePhi final : public GenTree Use* gtUses; - GenTreePhi(var_types type) : GenTree(GT_PHI, type), gtUses(nullptr) + GenTreePhi(var_types type) + : GenTree(GT_PHI, type) + , gtUses(nullptr) { } @@ -2572,7 +2582,8 @@ struct GenTreePhi final : public GenTree } #if DEBUGGABLE_GENTREE - GenTreePhi() : GenTree() + GenTreePhi() + : GenTree() { } #endif @@ -2591,7 +2602,10 @@ struct GenTreeFieldList : public GenTree public: Use(GenTree* node, unsigned offset, var_types type) - : m_node(node), m_next(nullptr), m_offset(static_cast(offset)), m_type(type) + : m_node(node) + , m_next(nullptr) + , m_offset(static_cast(offset)) + , m_type(type) { // We can save space on 32 bit hosts by storing the offset as uint16_t. Struct promotion // only accepts structs which are much smaller than that - 128 bytes = max 4 fields * max @@ -2651,7 +2665,8 @@ struct GenTreeFieldList : public GenTree Use* use; public: - UseIterator(Use* use) : use(use) + UseIterator(Use* use) + : use(use) { } @@ -2687,7 +2702,9 @@ struct GenTreeFieldList : public GenTree Use* m_tail; public: - UseList() : m_head(nullptr), m_tail(nullptr) + UseList() + : m_head(nullptr) + , m_tail(nullptr) { } @@ -2767,7 +2784,8 @@ struct GenTreeFieldList : public GenTree UseList m_uses; public: - GenTreeFieldList() : GenTree(GT_FIELD_LIST, TYP_STRUCT) + GenTreeFieldList() + : GenTree(GT_FIELD_LIST, TYP_STRUCT) { SetContained(); } @@ -2871,12 +2889,12 @@ class GenTreeUseEdgeIterator final void AdvanceConditional(); template - void AdvanceBinOp(); - void SetEntryStateForBinOp(); + void AdvanceBinOp(); + void SetEntryStateForBinOp(); // The advance function for call nodes template - void AdvanceCall(); + void AdvanceCall(); #if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS) void AdvanceMultiOp(); @@ -2935,12 +2953,14 @@ class GenTreeOperandIterator final GenTreeUseEdgeIterator m_useEdges; - GenTreeOperandIterator(GenTree* node) : m_useEdges(node) + GenTreeOperandIterator(GenTree* node) + : m_useEdges(node) { } public: - GenTreeOperandIterator() : m_useEdges() + GenTreeOperandIterator() + : m_useEdges() { } @@ -2983,12 +3003,14 @@ struct GenTreeUnOp : public GenTree protected: GenTreeUnOp(genTreeOps oper, var_types type DEBUGARG(bool largeNode = false)) - : GenTree(oper, type DEBUGARG(largeNode)), gtOp1(nullptr) + : GenTree(oper, type DEBUGARG(largeNode)) + , gtOp1(nullptr) { } GenTreeUnOp(genTreeOps oper, var_types type, GenTree* op1 DEBUGARG(bool largeNode = false)) - : GenTree(oper, type DEBUGARG(largeNode)), gtOp1(op1) + : GenTree(oper, type DEBUGARG(largeNode)) + , gtOp1(op1) { assert(op1 != nullptr || NullOp1Legal()); if (op1 != nullptr) @@ -2998,7 +3020,9 @@ struct GenTreeUnOp : public GenTree } #if DEBUGGABLE_GENTREE - GenTreeUnOp() : GenTree(), gtOp1(nullptr) + GenTreeUnOp() + : GenTree() + , gtOp1(nullptr) { } #endif @@ -3009,7 +3033,8 @@ struct GenTreeOp : public GenTreeUnOp GenTree* gtOp2; GenTreeOp(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2 DEBUGARG(bool largeNode = false)) - : GenTreeUnOp(oper, type, op1 DEBUGARG(largeNode)), gtOp2(op2) + : GenTreeUnOp(oper, type, op1 DEBUGARG(largeNode)) + , gtOp2(op2) { // comparisons are always integral types assert(!GenTree::OperIsCompare(oper) || varTypeIsIntegral(type)); @@ -3028,7 +3053,8 @@ struct GenTreeOp : public GenTreeUnOp // A small set of types are unary operators with optional arguments. We use // this constructor to build those. GenTreeOp(genTreeOps oper, var_types type DEBUGARG(bool largeNode = false)) - : GenTreeUnOp(oper, type DEBUGARG(largeNode)), gtOp2(nullptr) + : GenTreeUnOp(oper, type DEBUGARG(largeNode)) + , gtOp2(nullptr) { // Unary operators with optional arguments: assert(oper == GT_RETURN || oper == GT_RETFILT || OperIsBlk(oper)); @@ -3050,7 +3076,9 @@ struct GenTreeOp : public GenTreeUnOp #endif #if DEBUGGABLE_GENTREE - GenTreeOp() : GenTreeUnOp(), gtOp2(nullptr) + GenTreeOp() + : GenTreeUnOp() + , gtOp2(nullptr) { } #endif @@ -3060,11 +3088,14 @@ struct GenTreeVal : public GenTree { size_t gtVal1; - GenTreeVal(genTreeOps oper, var_types type, ssize_t val) : GenTree(oper, type), gtVal1(val) + GenTreeVal(genTreeOps oper, var_types type, ssize_t val) + : GenTree(oper, type) + , gtVal1(val) { } #if DEBUGGABLE_GENTREE - GenTreeVal() : GenTree() + GenTreeVal() + : GenTree() { } #endif @@ -3072,12 +3103,12 @@ struct GenTreeVal : public GenTree struct GenTreeIntConCommon : public GenTree { - inline INT64 LngValue() const; - inline void SetLngValue(INT64 val); + inline INT64 LngValue() const; + inline void SetLngValue(INT64 val); inline ssize_t IconValue() const; - inline void SetIconValue(ssize_t val); - inline INT64 IntegralValue() const; - inline void SetIntegralValue(int64_t value); + inline void SetIconValue(ssize_t val); + inline INT64 IntegralValue() const; + inline void SetIntegralValue(int64_t value); template inline void SetValueTruncating(T value); @@ -3120,7 +3151,8 @@ struct GenTreeIntConCommon : public GenTree #endif #if DEBUGGABLE_GENTREE - GenTreeIntConCommon() : GenTree() + GenTreeIntConCommon() + : GenTree() { } #endif @@ -3133,11 +3165,14 @@ struct GenTreePhysReg : public GenTree // GetRegNum() indicates the destination (and can be changed) // whereas reg indicates the source regNumber gtSrcReg; - GenTreePhysReg(regNumber r, var_types type = TYP_I_IMPL) : GenTree(GT_PHYSREG, type), gtSrcReg(r) + GenTreePhysReg(regNumber r, var_types type = TYP_I_IMPL) + : GenTree(GT_PHYSREG, type) + , gtSrcReg(r) { } #if DEBUGGABLE_GENTREE - GenTreePhysReg() : GenTree() + GenTreePhysReg() + : GenTree() { } #endif @@ -3196,7 +3231,8 @@ struct GenTreeIntCon : public GenTreeIntConCommon void FixupInitBlkValue(var_types type); #if DEBUGGABLE_GENTREE - GenTreeIntCon() : GenTreeIntConCommon() + GenTreeIntCon() + : GenTreeIntConCommon() { } #endif @@ -3217,12 +3253,14 @@ struct GenTreeLngCon : public GenTreeIntConCommon return (INT32)(gtLconVal >> 32); } - GenTreeLngCon(INT64 val) : GenTreeIntConCommon(GT_CNS_NATIVELONG, TYP_LONG) + GenTreeLngCon(INT64 val) + : GenTreeIntConCommon(GT_CNS_NATIVELONG, TYP_LONG) { SetLngValue(val); } #if DEBUGGABLE_GENTREE - GenTreeLngCon() : GenTreeIntConCommon() + GenTreeLngCon() + : GenTreeIntConCommon() { } #endif @@ -3353,13 +3391,15 @@ struct GenTreeDblCon : public GenTree return (bits == otherBits); } - GenTreeDblCon(double val, var_types type = TYP_DOUBLE) : GenTree(GT_CNS_DBL, type) + GenTreeDblCon(double val, var_types type = TYP_DOUBLE) + : GenTree(GT_CNS_DBL, type) { assert(varTypeIsFloating(type)); SetDconValue(val); } #if DEBUGGABLE_GENTREE - GenTreeDblCon() : GenTree() + GenTreeDblCon() + : GenTree() { } #endif @@ -3383,11 +3423,14 @@ struct GenTreeStrCon : public GenTree // Because this node can come from an inlined method we need to // have the scope handle, since it will become a helper call. GenTreeStrCon(unsigned sconCPX, CORINFO_MODULE_HANDLE mod DEBUGARG(bool largeNode = false)) - : GenTree(GT_CNS_STR, TYP_REF DEBUGARG(largeNode)), gtSconCPX(sconCPX), gtScpHnd(mod) + : GenTree(GT_CNS_STR, TYP_REF DEBUGARG(largeNode)) + , gtSconCPX(sconCPX) + , gtScpHnd(mod) { } #if DEBUGGABLE_GENTREE - GenTreeStrCon() : GenTree() + GenTreeStrCon() + : GenTree() { } #endif @@ -3430,12 +3473,14 @@ class SsaNumInfo final int m_value; - SsaNumInfo(int value) : m_value(value) + SsaNumInfo(int value) + : m_value(value) { } public: - SsaNumInfo() : m_value(SsaConfig::RESERVED_SSA_NUM) + SsaNumInfo() + : m_value(SsaConfig::RESERVED_SSA_NUM) { } @@ -3568,7 +3613,8 @@ struct GenTreeLclVarCommon : public GenTreeUnOp } #if DEBUGGABLE_GENTREE - GenTreeLclVarCommon() : GenTreeUnOp() + GenTreeLclVarCommon() + : GenTreeUnOp() { } #endif @@ -3707,7 +3753,7 @@ struct GenTreeLclVar : public GenTreeLclVarCommon } unsigned int GetFieldCount(Compiler* compiler) const; - var_types GetFieldTypeByIndex(Compiler* compiler, unsigned idx); + var_types GetFieldTypeByIndex(Compiler* compiler, unsigned idx); bool IsNeverNegative(Compiler* comp) const; @@ -3746,8 +3792,8 @@ struct GenTreeLclVar : public GenTreeLclVarCommon } #endif - GenTreeLclVar(genTreeOps oper, - var_types type, + GenTreeLclVar(genTreeOps oper, + var_types type, unsigned lclNum DEBUGARG(IL_OFFSET ilOffs = BAD_IL_OFFSET) DEBUGARG(bool largeNode = false)) : GenTreeLclVarCommon(oper, type, lclNum DEBUGARG(largeNode)) DEBUGARG(gtLclILoffs(ilOffs)) { @@ -3760,7 +3806,8 @@ struct GenTreeLclVar : public GenTreeLclVarCommon } #if DEBUGGABLE_GENTREE - GenTreeLclVar() : GenTreeLclVarCommon() + GenTreeLclVar() + : GenTreeLclVarCommon() { } #endif @@ -3776,14 +3823,16 @@ struct GenTreeLclFld : public GenTreeLclVarCommon public: GenTreeLclFld(genTreeOps oper, var_types type, unsigned lclNum, unsigned lclOffs, ClassLayout* layout = nullptr) - : GenTreeLclVarCommon(oper, type, lclNum), m_lclOffs(static_cast(lclOffs)) + : GenTreeLclVarCommon(oper, type, lclNum) + , m_lclOffs(static_cast(lclOffs)) { assert(lclOffs <= UINT16_MAX); SetLayout(layout); } GenTreeLclFld(var_types type, unsigned lclNum, unsigned lclOffs, GenTree* data, ClassLayout* layout) - : GenTreeLclVarCommon(GT_STORE_LCL_FLD, type, lclNum, data), m_lclOffs(static_cast(lclOffs)) + : GenTreeLclVarCommon(GT_STORE_LCL_FLD, type, lclNum, data) + , m_lclOffs(static_cast(lclOffs)) { assert(lclOffs <= UINT16_MAX); SetLayout(layout); @@ -3818,7 +3867,8 @@ struct GenTreeLclFld : public GenTreeLclVarCommon #endif // TARGET_ARM #if DEBUGGABLE_GENTREE - GenTreeLclFld() : GenTreeLclVarCommon() + GenTreeLclFld() + : GenTreeLclVarCommon() { } #endif @@ -3860,7 +3910,8 @@ struct GenTreeCast : public GenTreeOp var_types gtCastType; GenTreeCast(var_types type, GenTree* op, bool fromUnsigned, var_types castType DEBUGARG(bool largeNode = false)) - : GenTreeOp(GT_CAST, type, op, nullptr DEBUGARG(largeNode)), gtCastType(castType) + : GenTreeOp(GT_CAST, type, op, nullptr DEBUGARG(largeNode)) + , gtCastType(castType) { // We do not allow casts from floating point types to be treated as from // unsigned to avoid bugs related to wrong GTF_UNSIGNED in case the @@ -3870,7 +3921,8 @@ struct GenTreeCast : public GenTreeOp gtFlags |= fromUnsigned ? GTF_UNSIGNED : GTF_EMPTY; } #if DEBUGGABLE_GENTREE - GenTreeCast() : GenTreeOp() + GenTreeCast() + : GenTreeOp() { } #endif @@ -3919,7 +3971,8 @@ struct GenTreeBox : public GenTreeUnOp { } #if DEBUGGABLE_GENTREE - GenTreeBox() : GenTreeUnOp() + GenTreeBox() + : GenTreeUnOp() { } #endif @@ -3963,7 +4016,8 @@ struct GenTreeFieldAddr : public GenTreeUnOp } #if DEBUGGABLE_GENTREE - GenTreeFieldAddr() : GenTreeUnOp() + GenTreeFieldAddr() + : GenTreeUnOp() { } #endif @@ -4033,12 +4087,14 @@ struct GenTreeColon : public GenTreeOp } #if DEBUGGABLE_GENTREE - GenTreeColon() : GenTreeOp() + GenTreeColon() + : GenTreeOp() { } #endif - GenTreeColon(var_types typ, GenTree* thenNode, GenTree* elseNode) : GenTreeOp(GT_COLON, typ, elseNode, thenNode) + GenTreeColon(var_types typ, GenTree* thenNode, GenTree* elseNode) + : GenTreeOp(GT_COLON, typ, elseNode, thenNode) { } }; @@ -4051,13 +4107,15 @@ struct GenTreeConditional : public GenTreeOp GenTreeConditional( genTreeOps oper, var_types type, GenTree* cond, GenTree* op1, GenTree* op2 DEBUGARG(bool largeNode = false)) - : GenTreeOp(oper, type, op1, op2 DEBUGARG(largeNode)), gtCond(cond) + : GenTreeOp(oper, type, op1, op2 DEBUGARG(largeNode)) + , gtCond(cond) { assert(cond != nullptr); } #if DEBUGGABLE_GENTREE - GenTreeConditional() : GenTreeOp() + GenTreeConditional() + : GenTreeOp() { } #endif @@ -4338,7 +4396,7 @@ struct ReturnTypeDesc class TailCallSiteInfo { bool m_isCallvirt : 1; - bool m_isCalli : 1; + bool m_isCalli : 1; CORINFO_SIG_INFO m_sig; CORINFO_RESOLVED_TOKEN m_token; @@ -4524,7 +4582,7 @@ struct CallArgABIInformation bool IsHfaArg() const; bool IsHfaRegArg() const; var_types GetHfaType() const; - void SetHfaType(var_types type, unsigned hfaSlots); + void SetHfaType(var_types type, unsigned hfaSlots); regNumber GetRegNum() const { @@ -4699,7 +4757,8 @@ class CallArg public: CallArgABIInformation AbiInfo; - CallArg(const NewCallArg& arg) : CallArg() + CallArg(const NewCallArg& arg) + : CallArg() { m_earlyNode = arg.Node; m_wellKnownArg = arg.WellKnownArg; @@ -4707,7 +4766,7 @@ class CallArg m_signatureClsHnd = arg.SignatureClsHnd; } - CallArg(const CallArg&) = delete; + CallArg(const CallArg&) = delete; CallArg& operator=(CallArg&) = delete; // clang-format off @@ -4769,9 +4828,9 @@ class CallArgs // made for this call. unsigned m_padStkAlign; #endif - bool m_hasThisPointer : 1; - bool m_hasRetBuffer : 1; - bool m_isVarArgs : 1; + bool m_hasThisPointer : 1; + bool m_hasRetBuffer : 1; + bool m_isVarArgs : 1; bool m_abiInformationDetermined : 1; // True if we have one or more register arguments. bool m_hasRegArgs : 1; @@ -4785,15 +4844,15 @@ class CallArgs bool m_alignmentDone : 1; #endif - void AddedWellKnownArg(WellKnownArg arg); - void RemovedWellKnownArg(WellKnownArg arg); + void AddedWellKnownArg(WellKnownArg arg); + void RemovedWellKnownArg(WellKnownArg arg); regNumber GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, WellKnownArg arg); - void SplitArg(CallArg* arg, unsigned numRegs, unsigned numSlots); - void SortArgs(Compiler* comp, GenTreeCall* call, CallArg** sortedArgs); + void SplitArg(CallArg* arg, unsigned numRegs, unsigned numSlots); + void SortArgs(Compiler* comp, GenTreeCall* call, CallArg** sortedArgs); public: CallArgs(); - CallArgs(const CallArgs&) = delete; + CallArgs(const CallArgs&) = delete; CallArgs& operator=(CallArgs&) = delete; CallArg* FindByNode(GenTree* node); @@ -4818,8 +4877,8 @@ class CallArgs CallArg* InsertAfterUnchecked(Compiler* comp, CallArg* after, const NewCallArg& arg); CallArg* InsertInstParam(Compiler* comp, GenTree* node); CallArg* InsertAfterThisOrFirst(Compiler* comp, const NewCallArg& arg); - void PushLateBack(CallArg* arg); - void Remove(CallArg* arg); + void PushLateBack(CallArg* arg); + void Remove(CallArg* arg); template void InternalCopyFrom(Compiler* comp, CallArgs* other, CopyNodeFunc copyFunc); @@ -4840,7 +4899,7 @@ class CallArgs bool IsNonStandard(Compiler* comp, GenTreeCall* call, CallArg* arg); GenTree* MakeTmpArgNode(Compiler* comp, CallArg* arg); - void SetTemp(CallArg* arg, unsigned tmpNum); + void SetTemp(CallArg* arg, unsigned tmpNum); // clang-format off bool HasThisPointer() const { return m_hasThisPointer; } @@ -4878,7 +4937,8 @@ class CallArgs CallArg* m_arg; public: - explicit CallArgIterator(CallArg* arg) : m_arg(arg) + explicit CallArgIterator(CallArg* arg) + : m_arg(arg) { } @@ -4922,7 +4982,8 @@ class CallArgs } public: - explicit EarlyArgIterator(CallArg* arg) : m_arg(arg) + explicit EarlyArgIterator(CallArg* arg) + : m_arg(arg) { } @@ -4978,7 +5039,8 @@ struct GenTreeCall final : public GenTree CORINFO_SIG_INFO* callSig; #endif - union { + union + { TailCallSiteInfo* tailCallInfo; // Only used for unmanaged calls, which cannot be tail-called CorInfoCallConvExtension unmgdCallConv; @@ -5239,7 +5301,7 @@ struct GenTreeCall final : public GenTree } bool HasNonStandardAddedArgs(Compiler* compiler) const; - int GetNonStandardAddedArgCount(Compiler* compiler) const; + int GetNonStandardAddedArgCount(Compiler* compiler) const; // Returns true if the ABI dictates that this call should get a ret buf // arg. This may be out of sync with gtArgs.HasRetBuffer during import @@ -5606,19 +5668,21 @@ struct GenTreeCall final : public GenTree } GenTreeCallFlags gtCallMoreFlags; // in addition to gtFlags - gtCallTypes gtCallType : 3; // value from the gtCallTypes enumeration + gtCallTypes gtCallType : 3; // value from the gtCallTypes enumeration var_types gtReturnType : 5; // exact return type uint8_t gtInlineInfoCount; // number of inline candidates for the given call CORINFO_CLASS_HANDLE gtRetClsHnd; // The return type handle of the call if it is a struct; always available - union { + union + { void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined CORINFO_CLASS_HANDLE gtInitClsHnd; // Used by static init helpers, represents a class they init IL_OFFSET gtCastHelperILOffset; // Used by cast helpers to save corresponding IL offset }; - union { + union + { // only used for CALLI unmanaged calls (CT_INDIRECT) GenTree* gtCallCookie; @@ -5636,7 +5700,8 @@ struct GenTreeCall final : public GenTree // expression evaluated after args are placed which determines the control target GenTree* gtControlExpr; - union { + union + { CORINFO_METHOD_HANDLE gtCallMethHnd; // CT_USER_FUNC or CT_HELPER GenTree* gtCallAddr; // CT_INDIRECT }; @@ -5689,11 +5754,13 @@ struct GenTreeCall final : public GenTree static bool Equals(GenTreeCall* c1, GenTreeCall* c2); - GenTreeCall(var_types type) : GenTree(GT_CALL, type) + GenTreeCall(var_types type) + : GenTree(GT_CALL, type) { } #if DEBUGGABLE_GENTREE - GenTreeCall() : GenTree() + GenTreeCall() + : GenTree() { } #endif @@ -5712,7 +5779,8 @@ struct GenTreeMultiRegOp : public GenTreeOp MultiRegSpillFlags gtSpillFlags; GenTreeMultiRegOp(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2) - : GenTreeOp(oper, type, op1, op2), gtOtherReg(REG_NA) + : GenTreeOp(oper, type, op1, op2) + , gtOtherReg(REG_NA) { ClearOtherRegFlags(); } @@ -5795,7 +5863,8 @@ struct GenTreeMultiRegOp : public GenTreeOp } #if DEBUGGABLE_GENTREE - GenTreeMultiRegOp() : GenTreeOp() + GenTreeMultiRegOp() + : GenTreeOp() { } #endif @@ -5813,7 +5882,9 @@ struct GenTreeFptrVal : public GenTree #endif GenTreeFptrVal(var_types type, CORINFO_METHOD_HANDLE meth) - : GenTree(GT_FTN_ADDR, type), gtFptrMethod(meth), gtFptrDelegateTarget(false) + : GenTree(GT_FTN_ADDR, type) + , gtFptrMethod(meth) + , gtFptrDelegateTarget(false) { #ifdef FEATURE_READYTORUN gtEntryPoint.addr = nullptr; @@ -5821,7 +5892,8 @@ struct GenTreeFptrVal : public GenTree #endif } #if DEBUGGABLE_GENTREE - GenTreeFptrVal() : GenTree() + GenTreeFptrVal() + : GenTree() { } #endif @@ -5833,7 +5905,8 @@ struct GenTreeQmark : public GenTreeOp unsigned gtThenLikelihood; GenTreeQmark(var_types type, GenTree* cond, GenTreeColon* colon, unsigned thenLikelihood = 50) - : GenTreeOp(GT_QMARK, type, cond, colon), gtThenLikelihood(thenLikelihood) + : GenTreeOp(GT_QMARK, type, cond, colon) + , gtThenLikelihood(thenLikelihood) { // These must follow a specific form. assert((cond != nullptr) && cond->TypeIs(TYP_INT)); @@ -5869,7 +5942,8 @@ struct GenTreeQmark : public GenTreeOp } #if DEBUGGABLE_GENTREE - GenTreeQmark() : GenTreeOp() + GenTreeQmark() + : GenTreeOp() { } #endif @@ -5888,20 +5962,25 @@ struct GenTreeIntrinsic : public GenTreeOp #endif GenTreeIntrinsic(var_types type, GenTree* op1, NamedIntrinsic intrinsicName, CORINFO_METHOD_HANDLE methodHandle) - : GenTreeOp(GT_INTRINSIC, type, op1, nullptr), gtIntrinsicName(intrinsicName), gtMethodHandle(methodHandle) + : GenTreeOp(GT_INTRINSIC, type, op1, nullptr) + , gtIntrinsicName(intrinsicName) + , gtMethodHandle(methodHandle) { assert(intrinsicName != NI_Illegal); } GenTreeIntrinsic( var_types type, GenTree* op1, GenTree* op2, NamedIntrinsic intrinsicName, CORINFO_METHOD_HANDLE methodHandle) - : GenTreeOp(GT_INTRINSIC, type, op1, op2), gtIntrinsicName(intrinsicName), gtMethodHandle(methodHandle) + : GenTreeOp(GT_INTRINSIC, type, op1, op2) + , gtIntrinsicName(intrinsicName) + , gtMethodHandle(methodHandle) { assert(intrinsicName != NI_Illegal); } #if DEBUGGABLE_GENTREE - GenTreeIntrinsic() : GenTreeOp() + GenTreeIntrinsic() + : GenTreeOp() { } #endif @@ -5921,7 +6000,8 @@ struct GenTreeMultiOp : public GenTree protected: GenTree** m_use; - Iterator(GenTree** use) : m_use(use) + Iterator(GenTree** use) + : m_use(use) { } @@ -5946,7 +6026,8 @@ struct GenTreeMultiOp : public GenTree class OperandsIterator final : public Iterator { public: - OperandsIterator(GenTree** use) : Iterator(use) + OperandsIterator(GenTree** use) + : Iterator(use) { } @@ -5959,7 +6040,8 @@ struct GenTreeMultiOp : public GenTree class UseEdgesIterator final : public Iterator { public: - UseEdgesIterator(GenTree** use) : Iterator(use) + UseEdgesIterator(GenTree** use) + : Iterator(use) { } @@ -6006,7 +6088,8 @@ struct GenTreeMultiOp : public GenTree public: #if DEBUGGABLE_GENTREE - GenTreeMultiOp() : GenTree() + GenTreeMultiOp() + : GenTree() { } #endif @@ -6079,7 +6162,8 @@ class IntrinsicNodeBuilder final GenTree* m_inlineOperands[2]; public: - IntrinsicNodeBuilder(CompAllocator allocator, size_t operandCount) : m_operandCount(operandCount) + IntrinsicNodeBuilder(CompAllocator allocator, size_t operandCount) + : m_operandCount(operandCount) { m_operands = (operandCount <= ArrLen(m_inlineOperands)) ? m_inlineOperands : allocator.allocate(operandCount); @@ -6091,7 +6175,8 @@ class IntrinsicNodeBuilder final #endif // DEBUG } - IntrinsicNodeBuilder(CompAllocator allocator, GenTreeMultiOp* source) : m_operandCount(source->GetOperandCount()) + IntrinsicNodeBuilder(CompAllocator allocator, GenTreeMultiOp* source) + : m_operandCount(source->GetOperandCount()) { m_operands = (m_operandCount <= ArrLen(m_inlineOperands)) ? m_inlineOperands : allocator.allocate(m_operandCount); @@ -6303,7 +6388,8 @@ struct GenTreeJitIntrinsic : public GenTreeMultiOp } #if DEBUGGABLE_GENTREE - GenTreeJitIntrinsic() : GenTreeMultiOp() + GenTreeJitIntrinsic() + : GenTreeMultiOp() { } #endif @@ -6364,7 +6450,8 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic } #if DEBUGGABLE_GENTREE - GenTreeHWIntrinsic() : GenTreeJitIntrinsic() + GenTreeHWIntrinsic() + : GenTreeJitIntrinsic() { } #endif @@ -6406,7 +6493,7 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic bool OperRequiresGlobRefFlag() const; unsigned GetResultOpNumForRmwIntrinsic(GenTree* use, GenTree* op1, GenTree* op2, GenTree* op3); - uint8_t GetTernaryControlByte(GenTreeHWIntrinsic* second) const; + uint8_t GetTernaryControlByte(GenTreeHWIntrinsic* second) const; ClassLayout* GetLayout(Compiler* compiler) const; @@ -6509,7 +6596,8 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic // struct GenTreeVecCon : public GenTree { - union { + union + { simd8_t gtSimd8Val; simd12_t gtSimd12Val; simd16_t gtSimd16Val; @@ -6561,7 +6649,7 @@ struct GenTreeVecCon : public GenTree // These intrinsics are meant to set the same value to every element. if ((argCnt == 1) && HandleArgForHWIntrinsicCreate(node->Op(1), 0, simdVal, simdBaseType)) { -// CreateScalar leaves the upper bits as zero + // CreateScalar leaves the upper bits as zero #if defined(TARGET_XARCH) if ((intrinsic != NI_Vector128_CreateScalar) && (intrinsic != NI_Vector256_CreateScalar) && @@ -6881,7 +6969,8 @@ struct GenTreeVecCon : public GenTree } } - GenTreeVecCon(var_types type) : GenTree(GT_CNS_VEC, type) + GenTreeVecCon(var_types type) + : GenTree(GT_CNS_VEC, type) { assert(varTypeIsSIMD(type)); @@ -6897,7 +6986,8 @@ struct GenTreeVecCon : public GenTree } #if DEBUGGABLE_GENTREE - GenTreeVecCon() : GenTree() + GenTreeVecCon() + : GenTree() { } #endif @@ -6954,7 +7044,8 @@ struct GenTreeIndexAddr : public GenTreeOp } #if DEBUGGABLE_GENTREE - GenTreeIndexAddr() : GenTreeOp() + GenTreeIndexAddr() + : GenTreeOp() { } #endif @@ -6994,7 +7085,8 @@ struct GenTreeArrAddr : GenTreeUnOp } #if DEBUGGABLE_GENTREE - GenTreeArrAddr() : GenTreeUnOp() + GenTreeArrAddr() + : GenTreeUnOp() { } #endif @@ -7041,12 +7133,14 @@ struct GenTreeArrCommon : public GenTreeUnOp return gtOp1; } - GenTreeArrCommon(genTreeOps oper, var_types type, GenTree* arrRef) : GenTreeUnOp(oper, type, arrRef) + GenTreeArrCommon(genTreeOps oper, var_types type, GenTree* arrRef) + : GenTreeUnOp(oper, type, arrRef) { } #if DEBUGGABLE_GENTREE - GenTreeArrCommon() : GenTreeUnOp() + GenTreeArrCommon() + : GenTreeUnOp() { } #endif @@ -7071,12 +7165,14 @@ struct GenTreeArrLen : public GenTreeArrCommon } GenTreeArrLen(var_types type, GenTree* arrRef, int lenOffset) - : GenTreeArrCommon(GT_ARR_LENGTH, type, arrRef), gtArrLenOffset(lenOffset) + : GenTreeArrCommon(GT_ARR_LENGTH, type, arrRef) + , gtArrLenOffset(lenOffset) { } #if DEBUGGABLE_GENTREE - GenTreeArrLen() : GenTreeArrCommon() + GenTreeArrLen() + : GenTreeArrCommon() { } #endif @@ -7103,13 +7199,16 @@ struct GenTreeMDArr : public GenTreeArrCommon } GenTreeMDArr(genTreeOps oper, GenTree* arrRef, unsigned dim, unsigned rank) - : GenTreeArrCommon(oper, TYP_INT, arrRef), gtDim(dim), gtRank(rank) + : GenTreeArrCommon(oper, TYP_INT, arrRef) + , gtDim(dim) + , gtRank(rank) { assert(OperIs(GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND)); } #if DEBUGGABLE_GENTREE - GenTreeMDArr() : GenTreeArrCommon() + GenTreeMDArr() + : GenTreeArrCommon() { } #endif @@ -7141,7 +7240,8 @@ struct GenTreeBoundsChk : public GenTreeOp gtFlags |= GTF_EXCEPT; } #if DEBUGGABLE_GENTREE - GenTreeBoundsChk() : GenTreeOp() + GenTreeBoundsChk() + : GenTreeOp() { } #endif @@ -7191,7 +7291,10 @@ struct GenTreeArrElem : public GenTree // Requires that "inds" is a pointer to an array of "rank" nodes for the indices. GenTreeArrElem(var_types type, GenTree* arr, unsigned char rank, unsigned char elemSize, GenTree** inds) - : GenTree(GT_ARR_ELEM, type), gtArrObj(arr), gtArrRank(rank), gtArrElemSize(elemSize) + : GenTree(GT_ARR_ELEM, type) + , gtArrObj(arr) + , gtArrRank(rank) + , gtArrElemSize(elemSize) { assert(rank <= ArrLen(gtArrInds)); gtFlags |= (arr->gtFlags & GTF_ALL_EFFECT); @@ -7203,7 +7306,8 @@ struct GenTreeArrElem : public GenTree gtFlags |= GTF_EXCEPT; } #if DEBUGGABLE_GENTREE - GenTreeArrElem() : GenTree() + GenTreeArrElem() + : GenTree() { } #endif @@ -7297,7 +7401,8 @@ struct GenTreeAddrMode : public GenTreeOp protected: friend GenTree; // Used only for GenTree::GetVtableForOper() - GenTreeAddrMode() : GenTreeOp() + GenTreeAddrMode() + : GenTreeOp() { } #endif @@ -7333,7 +7438,8 @@ struct GenTreeIndir : public GenTreeOp unsigned Size() const; - GenTreeIndir(genTreeOps oper, var_types type, GenTree* addr, GenTree* data) : GenTreeOp(oper, type, addr, data) + GenTreeIndir(genTreeOps oper, var_types type, GenTree* addr, GenTree* data) + : GenTreeOp(oper, type, addr, data) { } @@ -7359,12 +7465,14 @@ struct GenTreeIndir : public GenTreeOp #if DEBUGGABLE_GENTREE // Used only for GenTree::GetVtableForOper() - GenTreeIndir() : GenTreeOp() + GenTreeIndir() + : GenTreeOp() { } #else // Used by XARCH codegen to construct temporary trees to pass to the emitter. - GenTreeIndir() : GenTreeOp(GT_NOP, TYP_UNDEF) + GenTreeIndir() + : GenTreeOp(GT_NOP, TYP_UNDEF) { } #endif @@ -7472,7 +7580,8 @@ struct GenTreeBlk : public GenTreeIndir #if DEBUGGABLE_GENTREE protected: friend GenTree; - GenTreeBlk() : GenTreeIndir() + GenTreeBlk() + : GenTreeIndir() { } #endif // DEBUGGABLE_GENTREE @@ -7578,7 +7687,8 @@ struct GenTreeStoreInd : public GenTreeIndir return gtOp2; } - GenTreeStoreInd(var_types type, GenTree* destPtr, GenTree* data) : GenTreeIndir(GT_STOREIND, type, destPtr, data) + GenTreeStoreInd(var_types type, GenTree* destPtr, GenTree* data) + : GenTreeIndir(GT_STOREIND, type, destPtr, data) { SetRMWStatusDefault(); } @@ -7587,7 +7697,8 @@ struct GenTreeStoreInd : public GenTreeIndir protected: friend GenTree; // Used only for GenTree::GetVtableForOper() - GenTreeStoreInd() : GenTreeIndir() + GenTreeStoreInd() + : GenTreeIndir() { SetRMWStatusDefault(); } @@ -7601,13 +7712,15 @@ struct GenTreeCmpXchg : public GenTreeIndir public: GenTreeCmpXchg(var_types type, GenTree* loc, GenTree* val, GenTree* comparand) - : GenTreeIndir(GT_CMPXCHG, type, loc, val), m_comparand(comparand) + : GenTreeIndir(GT_CMPXCHG, type, loc, val) + , m_comparand(comparand) { gtFlags |= comparand->gtFlags & GTF_ALL_EFFECT; } #if DEBUGGABLE_GENTREE - GenTreeCmpXchg() : GenTreeIndir() + GenTreeCmpXchg() + : GenTreeIndir() { } #endif @@ -7635,11 +7748,13 @@ struct GenTreeRetExpr : public GenTree // nullptr for cases where gtSubstExpr is not a tree from the inlinee. BasicBlock* gtSubstBB; - GenTreeRetExpr(var_types type) : GenTree(GT_RET_EXPR, type) + GenTreeRetExpr(var_types type) + : GenTree(GT_RET_EXPR, type) { } #if DEBUGGABLE_GENTREE - GenTreeRetExpr() : GenTree() + GenTreeRetExpr() + : GenTree() { } #endif @@ -7663,7 +7778,8 @@ struct GenTreeILOffset : public GenTree } #if DEBUGGABLE_GENTREE - GenTreeILOffset() : GenTree(GT_IL_OFFSET, TYP_VOID) + GenTreeILOffset() + : GenTree(GT_IL_OFFSET, TYP_VOID) { } #endif @@ -7685,7 +7801,8 @@ class GenTreeList GenTree* m_tree; public: - explicit iterator(GenTree* tree) : m_tree(tree) + explicit iterator(GenTree* tree) + : m_tree(tree) { } @@ -7706,7 +7823,8 @@ class GenTreeList } }; - explicit GenTreeList(GenTree* trees) : m_trees(trees) + explicit GenTreeList(GenTree* trees) + : m_trees(trees) { } @@ -7731,7 +7849,8 @@ class LocalsGenTreeList GenTreeLclVarCommon* m_tree; public: - explicit iterator(GenTreeLclVarCommon* tree) : m_tree(tree) + explicit iterator(GenTreeLclVarCommon* tree) + : m_tree(tree) { } @@ -7760,7 +7879,8 @@ class LocalsGenTreeList } }; - explicit LocalsGenTreeList(Statement* stmt) : m_stmt(stmt) + explicit LocalsGenTreeList(Statement* stmt) + : m_stmt(stmt) { } @@ -7960,7 +8080,8 @@ class StatementList Statement* m_stmt; public: - iterator(Statement* stmt) : m_stmt(stmt) + iterator(Statement* stmt) + : m_stmt(stmt) { } @@ -7982,7 +8103,8 @@ class StatementList }; public: - StatementList(Statement* stmts) : m_stmts(stmts) + StatementList(Statement* stmts) + : m_stmts(stmts) { } @@ -8007,13 +8129,15 @@ struct GenTreePhiArg : public GenTreeLclVarCommon BasicBlock* gtPredBB; GenTreePhiArg(var_types type, unsigned lclNum, unsigned ssaNum, BasicBlock* block) - : GenTreeLclVarCommon(GT_PHI_ARG, type, lclNum), gtPredBB(block) + : GenTreeLclVarCommon(GT_PHI_ARG, type, lclNum) + , gtPredBB(block) { SetSsaNum(ssaNum); } #if DEBUGGABLE_GENTREE - GenTreePhiArg() : GenTreeLclVarCommon() + GenTreePhiArg() + : GenTreeLclVarCommon() { } #endif @@ -8051,8 +8175,13 @@ struct GenTreePutArgStk : public GenTreeUnOp // TODO-Throughput: The following information should be obtained from the child // block node. - enum class Kind : int8_t{ - Invalid, RepInstr, PartialRepInstr, Unroll, Push, + enum class Kind : int8_t + { + Invalid, + RepInstr, + PartialRepInstr, + Unroll, + Push, }; Kind gtPutArgStkKind; @@ -8184,7 +8313,8 @@ struct GenTreePutArgStk : public GenTreeUnOp #endif // !FEATURE_PUT_STRUCT_ARG_STK #if DEBUGGABLE_GENTREE - GenTreePutArgStk() : GenTreeUnOp() + GenTreePutArgStk() + : GenTreeUnOp() { } #endif @@ -8334,7 +8464,8 @@ struct GenTreePutArgSplit : public GenTreePutArgStk } #if DEBUGGABLE_GENTREE - GenTreePutArgSplit() : GenTreePutArgStk() + GenTreePutArgSplit() + : GenTreePutArgStk() { } #endif @@ -8461,7 +8592,8 @@ struct GenTreeCopyOrReload : public GenTreeUnOp return 1; } - GenTreeCopyOrReload(genTreeOps oper, var_types type, GenTree* op1) : GenTreeUnOp(oper, type, op1) + GenTreeCopyOrReload(genTreeOps oper, var_types type, GenTree* op1) + : GenTreeUnOp(oper, type, op1) { assert(type != TYP_STRUCT || op1->IsMultiRegNode()); SetRegNum(REG_NA); @@ -8469,7 +8601,8 @@ struct GenTreeCopyOrReload : public GenTreeUnOp } #if DEBUGGABLE_GENTREE - GenTreeCopyOrReload() : GenTreeUnOp() + GenTreeCopyOrReload() + : GenTreeUnOp() { } #endif @@ -8499,7 +8632,8 @@ struct GenTreeAllocObj final : public GenTreeUnOp #endif } #if DEBUGGABLE_GENTREE - GenTreeAllocObj() : GenTreeUnOp() + GenTreeAllocObj() + : GenTreeUnOp() { } #endif @@ -8513,12 +8647,15 @@ struct GenTreeRuntimeLookup final : public GenTreeUnOp CorInfoGenericHandleType gtHndType; GenTreeRuntimeLookup(CORINFO_GENERIC_HANDLE hnd, CorInfoGenericHandleType hndTyp, GenTree* tree) - : GenTreeUnOp(GT_RUNTIMELOOKUP, tree->gtType, tree DEBUGARG(/*largeNode*/ FALSE)), gtHnd(hnd), gtHndType(hndTyp) + : GenTreeUnOp(GT_RUNTIMELOOKUP, tree->gtType, tree DEBUGARG(/*largeNode*/ FALSE)) + , gtHnd(hnd) + , gtHndType(hndTyp) { assert(hnd != nullptr); } #if DEBUGGABLE_GENTREE - GenTreeRuntimeLookup() : GenTreeUnOp() + GenTreeRuntimeLookup() + : GenTreeUnOp() { } #endif @@ -8682,11 +8819,13 @@ struct GenCondition return names[m_code]; } - GenCondition() : m_code() + GenCondition() + : m_code() { } - GenCondition(Code cond) : m_code(cond) + GenCondition(Code cond) + : m_code(cond) { } @@ -8810,13 +8949,15 @@ struct GenTreeCC final : public GenTree GenCondition gtCondition; GenTreeCC(genTreeOps oper, var_types type, GenCondition condition) - : GenTree(oper, type DEBUGARG(/*largeNode*/ FALSE)), gtCondition(condition) + : GenTree(oper, type DEBUGARG(/*largeNode*/ FALSE)) + , gtCondition(condition) { assert(OperIs(GT_JCC, GT_SETCC)); } #if DEBUGGABLE_GENTREE - GenTreeCC() : GenTree() + GenTreeCC() + : GenTree() { } #endif // DEBUGGABLE_GENTREE @@ -8828,7 +8969,8 @@ struct GenTreeOpCC : public GenTreeOp GenCondition gtCondition; GenTreeOpCC(genTreeOps oper, var_types type, GenCondition condition, GenTree* op1 = nullptr, GenTree* op2 = nullptr) - : GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ FALSE)), gtCondition(condition) + : GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ FALSE)) + , gtCondition(condition) { #ifdef TARGET_ARM64 assert(OperIs(GT_SELECTCC, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)); @@ -8838,7 +8980,8 @@ struct GenTreeOpCC : public GenTreeOp } #if DEBUGGABLE_GENTREE - GenTreeOpCC() : GenTreeOp() + GenTreeOpCC() + : GenTreeOp() { } #endif // DEBUGGABLE_GENTREE @@ -8873,12 +9016,14 @@ struct GenTreeCCMP final : public GenTreeOpCC insCflags gtFlagsVal; GenTreeCCMP(var_types type, GenCondition condition, GenTree* op1, GenTree* op2, insCflags flagsVal) - : GenTreeOpCC(GT_CCMP, type, condition, op1, op2), gtFlagsVal(flagsVal) + : GenTreeOpCC(GT_CCMP, type, condition, op1, op2) + , gtFlagsVal(flagsVal) { } #if DEBUGGABLE_GENTREE - GenTreeCCMP() : GenTreeOpCC() + GenTreeCCMP() + : GenTreeOpCC() { } #endif // DEBUGGABLE_GENTREE diff --git a/src/coreclr/jit/gschecks.cpp b/src/coreclr/jit/gschecks.cpp index 12c610ceaefa45..7b448b8ca3d24e 100644 --- a/src/coreclr/jit/gschecks.cpp +++ b/src/coreclr/jit/gschecks.cpp @@ -455,7 +455,8 @@ void Compiler::gsParamsToShadows() DoPostOrder = true }; - ReplaceShadowParamsVisitor(Compiler* compiler) : GenTreeVisitor(compiler) + ReplaceShadowParamsVisitor(Compiler* compiler) + : GenTreeVisitor(compiler) { } diff --git a/src/coreclr/jit/hashbv.cpp b/src/coreclr/jit/hashbv.cpp index 854215235261db..203219a7ec20ca 100644 --- a/src/coreclr/jit/hashbv.cpp +++ b/src/coreclr/jit/hashbv.cpp @@ -1948,7 +1948,7 @@ indexType hashBvIterator::nextBit() current_element++; // printf("current element is %d\n", current_element); // reached the end of this node - if (current_element == (indexType) this->currNode->numElements()) + if (current_element == (indexType)this->currNode->numElements()) { // printf("going to next node\n"); this->nextNode(); @@ -1956,7 +1956,7 @@ indexType hashBvIterator::nextBit() } else { - assert(current_element < (indexType) this->currNode->numElements()); + assert(current_element < (indexType)this->currNode->numElements()); // printf("getting more data\n"); current_data = this->currNode->elements[current_element]; current_base = this->currNode->baseIndex + current_element * BITS_PER_ELEMENT; diff --git a/src/coreclr/jit/hashbv.h b/src/coreclr/jit/hashbv.h index 7ad95998add8e4..561a1c5641e491 100644 --- a/src/coreclr/jit/hashbv.h +++ b/src/coreclr/jit/hashbv.h @@ -13,15 +13,15 @@ #include #include -//#define TESTING 1 +// #define TESTING 1 -#define LOG2_BITS_PER_ELEMENT 5 +#define LOG2_BITS_PER_ELEMENT 5 #define LOG2_ELEMENTS_PER_NODE 2 -#define LOG2_BITS_PER_NODE (LOG2_BITS_PER_ELEMENT + LOG2_ELEMENTS_PER_NODE) +#define LOG2_BITS_PER_NODE (LOG2_BITS_PER_ELEMENT + LOG2_ELEMENTS_PER_NODE) -#define BITS_PER_ELEMENT (1 << LOG2_BITS_PER_ELEMENT) +#define BITS_PER_ELEMENT (1 << LOG2_BITS_PER_ELEMENT) #define ELEMENTS_PER_NODE (1 << LOG2_ELEMENTS_PER_NODE) -#define BITS_PER_NODE (1 << LOG2_BITS_PER_NODE) +#define BITS_PER_NODE (1 << LOG2_BITS_PER_NODE) #ifdef TARGET_AMD64 typedef unsigned __int64 elemType; @@ -128,8 +128,8 @@ class hashBvNode { } static hashBvNode* Create(indexType base, Compiler* comp); - void Reconstruct(indexType base); - int numElements() + void Reconstruct(indexType base); + int numElements() { return ELEMENTS_PER_NODE; } @@ -172,7 +172,8 @@ class hashBv hashBvNode** nodeArr; hashBvNode* initialVector[1]; - union { + union + { Compiler* compiler; // for freelist hashBv* next; @@ -186,9 +187,9 @@ class hashBv public: hashBv(Compiler* comp); static hashBv* Create(Compiler* comp); - static void Init(Compiler* comp); + static void Init(Compiler* comp); static hashBv* CreateFrom(hashBv* other, Compiler* comp); - void hbvFree(); + void hbvFree(); #ifdef DEBUG void dump(); void dumpFancy(); @@ -201,18 +202,18 @@ class hashBv hashBvGlobalData* globalData(); static hashBvNode*& nodeFreeList(hashBvGlobalData* globalData); - static hashBv*& hbvFreeList(hashBvGlobalData* data); + static hashBv*& hbvFreeList(hashBvGlobalData* data); hashBvNode** getInsertionPointForIndex(indexType index); private: hashBvNode* getNodeForIndexHelper(indexType index, bool canAdd); - int getHashForIndex(indexType index, int table_size); - int getRehashForIndex(indexType thisIndex, int thisTableSize, int newTableSize); + int getHashForIndex(indexType index, int table_size); + int getRehashForIndex(indexType thisIndex, int thisTableSize, int newTableSize); // maintain free lists for vectors hashBvNode** getNewVector(int vectorLength); - int getNodeCount(); + int getNodeCount(); public: inline hashBvNode* getOrAddNodeForIndex(indexType index) @@ -221,7 +222,7 @@ class hashBv return temp; } hashBvNode* getNodeForIndex(indexType index); - void removeNodeAtBase(indexType index); + void removeNodeAtBase(indexType index); public: void setBit(indexType index); diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index 3bf37b93798e93..6c8251eee257f5 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -1945,13 +1945,13 @@ static int PickCandidatesForTypeCheck(Compiler* comp, isCastClass = false; break; - // These are never expanded: - // CORINFO_HELP_ISINSTANCEOF_EXCEPTION - // CORINFO_HELP_CHKCASTCLASS_SPECIAL - // CORINFO_HELP_READYTORUN_ISINSTANCEOF, - // CORINFO_HELP_READYTORUN_CHKCAST, + // These are never expanded: + // CORINFO_HELP_ISINSTANCEOF_EXCEPTION + // CORINFO_HELP_CHKCASTCLASS_SPECIAL + // CORINFO_HELP_READYTORUN_ISINSTANCEOF, + // CORINFO_HELP_READYTORUN_CHKCAST, - // Other helper calls are not cast helpers + // Other helper calls are not cast helpers default: return 0; diff --git a/src/coreclr/jit/host.h b/src/coreclr/jit/host.h index 6667fbb3994a76..d10eb93ca9a122 100644 --- a/src/coreclr/jit/host.h +++ b/src/coreclr/jit/host.h @@ -28,10 +28,10 @@ class LogEnv }; bool vlogf(unsigned level, const char* fmt, va_list args); -int vflogf(FILE* file, const char* fmt, va_list args); +int vflogf(FILE* file, const char* fmt, va_list args); -int logf(const char* fmt, ...); -int flogf(FILE* file, const char* fmt, ...); +int logf(const char* fmt, ...); +int flogf(FILE* file, const char* fmt, ...); void gcDump_logf(const char* fmt, ...); void logf(unsigned level, const char* fmt, ...); diff --git a/src/coreclr/jit/hostallocator.h b/src/coreclr/jit/hostallocator.h index a91f7f1fb4ab9b..0e8f192063fb06 100644 --- a/src/coreclr/jit/hostallocator.h +++ b/src/coreclr/jit/hostallocator.h @@ -37,7 +37,7 @@ class HostAllocator final private: void* allocateHostMemory(size_t size); - void freeHostMemory(void* p); + void freeHostMemory(void* p); }; // Global operator new overloads that work with HostAllocator diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index e8b60b07909d95..53970ef4a7460b 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -832,7 +832,7 @@ GenTree* Compiler::addRangeCheckIfNeeded( #ifdef TARGET_XARCH && !HWIntrinsicInfo::isAVX2GatherIntrinsic(intrinsic) && !HWIntrinsicInfo::HasFullRangeImm(intrinsic) #endif - ) + ) { assert(!immOp->IsCnsIntOrI()); assert(varTypeIsUnsigned(immOp)); diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index cac041eb83ea6d..5ca302e126f32a 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -451,13 +451,13 @@ struct TernaryLogicInfo // We have 256 entries, so we compress as much as possible // This gives us 3-bytes per entry (21-bits) - TernaryLogicOperKind oper1 : 4; + TernaryLogicOperKind oper1 : 4; TernaryLogicUseFlags oper1Use : 3; - TernaryLogicOperKind oper2 : 4; + TernaryLogicOperKind oper2 : 4; TernaryLogicUseFlags oper2Use : 3; - TernaryLogicOperKind oper3 : 4; + TernaryLogicOperKind oper3 : 4; TernaryLogicUseFlags oper3Use : 3; static const TernaryLogicInfo& lookup(uint8_t control); @@ -491,11 +491,11 @@ struct HWIntrinsicInfo static const HWIntrinsicInfo& lookup(NamedIntrinsic id); - static NamedIntrinsic lookupId(Compiler* comp, - CORINFO_SIG_INFO* sig, - const char* className, - const char* methodName, - const char* enclosingClassName); + static NamedIntrinsic lookupId(Compiler* comp, + CORINFO_SIG_INFO* sig, + const char* className, + const char* methodName, + const char* enclosingClassName); static CORINFO_InstructionSet lookupIsa(const char* className, const char* enclosingClassName); static unsigned lookupSimdSize(Compiler* comp, NamedIntrinsic id, CORINFO_SIG_INFO* sig); @@ -514,7 +514,7 @@ struct HWIntrinsicInfo static bool isScalarIsa(CORINFO_InstructionSet isa); #ifdef TARGET_XARCH - static bool isAVX2GatherIntrinsic(NamedIntrinsic id); + static bool isAVX2GatherIntrinsic(NamedIntrinsic id); static FloatComparisonMode lookupFloatComparisonModeForSwappedArgs(FloatComparisonMode comparison); #endif @@ -927,7 +927,12 @@ struct HWIntrinsicInfo struct HWIntrinsic final { HWIntrinsic(const GenTreeHWIntrinsic* node) - : op1(nullptr), op2(nullptr), op3(nullptr), op4(nullptr), numOperands(0), baseType(TYP_UNDEF) + : op1(nullptr) + , op2(nullptr) + , op3(nullptr) + , op4(nullptr) + , numOperands(0) + , baseType(TYP_UNDEF) { assert(node != nullptr); diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6418b72a8f3075..9a3a98e087a274 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -36,7 +36,10 @@ // of a for-loop. // CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTree* immOp, GenTreeHWIntrinsic* intrin) - : codeGen(codeGen), endLabel(nullptr), nonZeroLabel(nullptr), branchTargetReg(REG_NA) + : codeGen(codeGen) + , endLabel(nullptr) + , nonZeroLabel(nullptr) + , branchTargetReg(REG_NA) { assert(codeGen != nullptr); assert(varTypeIsIntegral(immOp)); diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 5e44772e7115ac..79e6b497c368a9 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -317,9 +317,9 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { case 1: { - regNumber targetReg = node->GetRegNum(); - GenTree* rmOp = node->Op(1); - auto emitSwCase = [&](int8_t i) { + regNumber targetReg = node->GetRegNum(); + GenTree* rmOp = node->Op(1); + auto emitSwCase = [&](int8_t i) { insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_RM(node, ins, simdSize, targetReg, rmOp, newInstOptions); }; @@ -559,7 +559,9 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) if (HWIntrinsicInfo::isImmOp(intrinsicId, op3)) { - auto emitSwCase = [&](int8_t i) { genHWIntrinsic_R_R_RM_I(node, ins, simdSize, i); }; + auto emitSwCase = [&](int8_t i) { + genHWIntrinsic_R_R_RM_I(node, ins, simdSize, i); + }; if (op3->IsCnsIntOrI()) { @@ -653,7 +655,9 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) if (HWIntrinsicInfo::isImmOp(intrinsicId, op4)) { - auto emitSwCase = [&](int8_t i) { genHWIntrinsic_R_R_R_RM_I(node, ins, simdSize, i); }; + auto emitSwCase = [&](int8_t i) { + genHWIntrinsic_R_R_R_RM_I(node, ins, simdSize, i); + }; if (op4->IsCnsIntOrI()) { @@ -1208,10 +1212,10 @@ void CodeGen::genHWIntrinsic_R_R_R_RM_I(GenTreeHWIntrinsic* node, instruction in if (op2->isContained()) { -// op2 is never selected by the table so -// we can contain and ignore any register -// allocated to it resulting in better -// non-RMW based codegen. + // op2 is never selected by the table so + // we can contain and ignore any register + // allocated to it resulting in better + // non-RMW based codegen. #if defined(DEBUG) NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); @@ -1364,8 +1368,8 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* { // This intrinsic has several overloads, only the ones with floating number inputs should reach this part. assert(varTypeIsFloating(baseType)); - GenTree* rmOp = node->Op(1); - auto emitSwCase = [&](int8_t i) { + GenTree* rmOp = node->Op(1); + auto emitSwCase = [&](int8_t i) { insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_RM(node, ins, attr, targetReg, rmOp, newInstOptions); }; @@ -2055,7 +2059,9 @@ void CodeGen::genSSE41Intrinsic(GenTreeHWIntrinsic* node) instruction ins = HWIntrinsicInfo::lookupIns(intrinsicId, baseType); emitAttr attr = emitActualTypeSize(node->TypeGet()); - auto emitSwCase = [&](int8_t i) { inst_RV_TT_IV(ins, attr, targetReg, op1, i); }; + auto emitSwCase = [&](int8_t i) { + inst_RV_TT_IV(ins, attr, targetReg, op1, i); + }; if (op2->IsCnsIntOrI()) { diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index f88cf6ec99ec3c..87332c07f0113f 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -371,7 +371,7 @@ FloatComparisonMode HWIntrinsicInfo::lookupFloatComparisonModeForSwappedArgs(Flo { switch (comparison) { - // These comparison modes are the same even if the operands are swapped + // These comparison modes are the same even if the operands are swapped case FloatComparisonMode::OrderedEqualNonSignaling: return FloatComparisonMode::OrderedEqualNonSignaling; @@ -406,7 +406,7 @@ FloatComparisonMode HWIntrinsicInfo::lookupFloatComparisonModeForSwappedArgs(Flo case FloatComparisonMode::UnorderedTrueSignaling: return FloatComparisonMode::UnorderedTrueSignaling; - // These comparison modes need a different mode if the operands are swapped + // These comparison modes need a different mode if the operands are swapped case FloatComparisonMode::OrderedLessThanSignaling: return FloatComparisonMode::OrderedGreaterThanSignaling; @@ -2498,7 +2498,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, // TODO-XARCH-CQ: We should support long/ulong multiplication break; } -// else if simdSize == 64 then above assert would check if baseline isa supported + // else if simdSize == 64 then above assert would check if baseline isa supported #if defined(TARGET_X86) // TODO-XARCH-CQ: We need to support 64-bit CreateBroadcast @@ -3274,13 +3274,13 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, int ival = HWIntrinsicInfo::lookupIval(this, intrinsic, simdBaseType); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, gtNewIconNode(ival), NI_AVX_CompareScalar, - simdBaseJitType, simdSize); + simdBaseJitType, simdSize); } else { GenTree* clonedOp1 = nullptr; op1 = impCloneExpr(op1, &clonedOp1, CHECK_SPILL_ALL, - nullptr DEBUGARG("Clone op1 for Sse.CompareScalarGreaterThan")); + nullptr DEBUGARG("Clone op1 for Sse.CompareScalarGreaterThan")); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, intrinsic, simdBaseJitType, simdSize); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, clonedOp1, retNode, NI_SSE_MoveScalar, simdBaseJitType, @@ -3333,13 +3333,13 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, int ival = HWIntrinsicInfo::lookupIval(this, intrinsic, simdBaseType); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, gtNewIconNode(ival), NI_AVX_CompareScalar, - simdBaseJitType, simdSize); + simdBaseJitType, simdSize); } else { GenTree* clonedOp1 = nullptr; op1 = impCloneExpr(op1, &clonedOp1, CHECK_SPILL_ALL, - nullptr DEBUGARG("Clone op1 for Sse2.CompareScalarGreaterThan")); + nullptr DEBUGARG("Clone op1 for Sse2.CompareScalarGreaterThan")); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, intrinsic, simdBaseJitType, simdSize); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, clonedOp1, retNode, NI_SSE2_MoveScalar, simdBaseJitType, diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 2b649302d5e1c7..6e493148088c3e 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -70,16 +70,16 @@ bool Compiler::impILConsumesAddr(const BYTE* codeAddr) switch (opcode) { - // case CEE_LDFLDA: We're taking this one out as if you have a sequence - // like - // - // ldloca.0 - // ldflda whatever - // - // of a primitivelike struct, you end up after morphing with addr of a local - // that's not marked as addrtaken, which is wrong. Also ldflda is usually used - // for structs that contain other structs, which isnt a case we handle very - // well now for other reasons. + // case CEE_LDFLDA: We're taking this one out as if you have a sequence + // like + // + // ldloca.0 + // ldflda whatever + // + // of a primitivelike struct, you end up after morphing with addr of a local + // that's not marked as addrtaken, which is wrong. Also ldflda is usually used + // for structs that contain other structs, which isnt a case we handle very + // well now for other reasons. case CEE_LDFLD: { @@ -670,7 +670,7 @@ void Compiler::impStoreTemp(unsigned lclNum, Statement** pAfterStmt, /* = NULL */ const DebugInfo& di, /* = DebugInfo() */ BasicBlock* block /* = NULL */ - ) +) { GenTree* store = gtNewTempStore(lclNum, val, curLevel, pAfterStmt, di, block); @@ -815,7 +815,7 @@ GenTree* Compiler::impStoreStruct(GenTree* store, Statement** pAfterStmt, /* = nullptr */ const DebugInfo& di, /* = DebugInfo() */ BasicBlock* block /* = nullptr */ - ) +) { assert(varTypeIsStruct(store) && store->OperIsStore()); @@ -1718,7 +1718,7 @@ bool Compiler::impSpillStackEntry(unsigned level, bool bAssertOnRecursion, const char* reason #endif - ) +) { #ifdef DEBUG @@ -2067,9 +2067,9 @@ BasicBlock* Compiler::impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_H * If the tree has side-effects, it will be spilled to a temp. */ -GenTree* Compiler::impCloneExpr(GenTree* tree, - GenTree** pClone, - unsigned curLevel, +GenTree* Compiler::impCloneExpr(GenTree* tree, + GenTree** pClone, + unsigned curLevel, Statement** pAfterStmt DEBUGARG(const char* reason)) { if (!(tree->gtFlags & GTF_GLOB_EFFECT)) @@ -4260,12 +4260,13 @@ GenTree* Compiler::impFixupStructReturnType(GenTree* op) // In contrast, we can only use multi-reg calls directly if they have the exact same ABI. // Calling convention equality is a conservative approximation for that check. - if (op->IsCall() && (op->AsCall()->GetUnmanagedCallConv() == info.compCallConv) + if (op->IsCall() && + (op->AsCall()->GetUnmanagedCallConv() == info.compCallConv) #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // TODO-Review: this seems unnecessary. Return ABI doesn't change under varargs. && !op->AsCall()->IsVarargs() #endif // defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - ) + ) { return op; } @@ -6172,7 +6173,8 @@ void Compiler::impImportBlockCode(BasicBlock* block) bool ovfl, unordered, callNode; CORINFO_CLASS_HANDLE tokenType; - union { + union + { int intVal; float fltVal; __int64 lngVal; @@ -6919,7 +6921,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) // Create the store node and append it. ClassLayout* layout = (lclTyp == TYP_STRUCT) ? typGetObjLayout(stelemClsHnd) : nullptr; op1 = (lclTyp == TYP_STRUCT) ? gtNewStoreBlkNode(layout, op1, op2)->AsIndir() - : gtNewStoreIndNode(lclTyp, op1, op2); + : gtNewStoreIndNode(lclTyp, op1, op2); if (varTypeIsStruct(op1)) { op1 = impStoreStruct(op1, CHECK_SPILL_ALL); @@ -6977,7 +6979,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) oper = GT_MUL; goto MATH_MAYBE_CALL_OVF; - // Other binary math operations + // Other binary math operations case CEE_DIV: oper = GT_DIV; @@ -7266,7 +7268,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) op1 = gtNewOperNode(oper, TYP_INT, op1, op2); } - // fall through + // fall through COND_JUMP: @@ -7595,7 +7597,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) goto SPILL_APPEND; - /************************** Casting OPCODES ***************************/ + /************************** Casting OPCODES ***************************/ case CEE_CONV_OVF_I1: lclTyp = TYP_BYTE; @@ -7737,12 +7739,13 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (varTypeIsFloating(lclTyp)) { - callNode = varTypeIsLong(impStackTop().val) || uns // uint->dbl gets turned into uint->long->dbl + callNode = varTypeIsLong(impStackTop().val) || + uns // uint->dbl gets turned into uint->long->dbl #ifdef TARGET_64BIT - // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? - // TYP_BYREF could be used as TYP_I_IMPL which is long. - // TODO-CQ: remove this when we lower casts long/ulong --> float/double - // and generate SSE2 code instead of going through helper calls. + // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? + // TYP_BYREF could be used as TYP_I_IMPL which is long. + // TODO-CQ: remove this when we lower casts long/ulong --> float/double + // and generate SSE2 code instead of going through helper calls. || (impStackTop().val->TypeGet() == TYP_BYREF) #endif ; @@ -8922,7 +8925,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) #if BIGENDIAN op1 = gtNewIconNode(0, lclTyp); #else - op1 = gtNewIconNode(1, lclTyp); + op1 = gtNewIconNode(1, lclTyp); #endif goto FIELD_DONE; } @@ -8937,7 +8940,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) ClassLayout* layout; lclTyp = TypeHandleToVarType(fieldInfo.fieldType, clsHnd, &layout); op1 = (lclTyp == TYP_STRUCT) ? gtNewBlkIndir(layout, op1, indirFlags) - : gtNewIndir(lclTyp, op1, indirFlags); + : gtNewIndir(lclTyp, op1, indirFlags); if ((indirFlags & GTF_IND_INVARIANT) != 0) { // TODO-ASG: delete this zero-diff quirk. @@ -9798,10 +9801,10 @@ void Compiler::impImportBlockCode(BasicBlock* block) } } - assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. - (helper == CORINFO_HELP_UNBOX_NULLABLE && - varTypeIsStruct(op1)) // UnboxNullable helper returns a struct. - ); + assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. + (helper == CORINFO_HELP_UNBOX_NULLABLE && varTypeIsStruct(op1)) // UnboxNullable helper returns a + // struct. + ); /* ---------------------------------------------------------------------- @@ -10065,7 +10068,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) // Pop the exception object and create the 'throw' helper call op1 = gtNewHelperCallNode(CORINFO_HELP_THROW, TYP_VOID, impPopStack().val); - // Fall through to clear out the eval stack. + // Fall through to clear out the eval stack. EVAL_APPEND: if (verCurrentState.esStackDepth > 0) @@ -10352,7 +10355,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) } break; - /******************************** NYI *******************************/ + /******************************** NYI *******************************/ case 0xCC: OutputDebugStringA("CLR: Invalid x86 breakpoint in IL stream\n"); @@ -10505,7 +10508,7 @@ void Compiler::impLoadLoc(unsigned ilLclNum, IL_OFFSET offset) // Returns: // Tree with reference to struct local to use as call return value. -GenTree* Compiler::impStoreMultiRegValueToVar(GenTree* op, +GenTree* Compiler::impStoreMultiRegValueToVar(GenTree* op, CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv)) { unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg return")); @@ -11793,7 +11796,7 @@ unsigned Compiler::impGetSpillTmpBase(BasicBlock* block) // Otherwise, choose one, and propagate to all members of the spill clique. // Grab enough temps for the whole stack. - unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); + unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); SetSpillTempsBase callback(baseTmp); // We do *NOT* need to reset the SpillClique*Members because a block can only be the predecessor @@ -12096,7 +12099,7 @@ void Compiler::impFixPredLists() unsigned XTnum = 0; bool added = false; - for (EHblkDsc *HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + for (EHblkDsc* HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) { if (HBtab->HasFinallyHandler()) { @@ -12245,7 +12248,7 @@ void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, I { assert((pInlineInfo != nullptr && compIsForInlining()) || // Perform the actual inlining. (pInlineInfo == nullptr && !compIsForInlining()) // Calculate the static inlining hint for ngen. - ); + ); // If we're really inlining, we should just have one result in play. assert((pInlineInfo == nullptr) || (inlineResult == pInlineInfo->inlineResult)); @@ -13290,7 +13293,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l assert(!argInfo.argIsUsed); /* Reserve a temp for the expression. - * Use a large size node as we may change it later */ + * Use a large size node as we may change it later */ const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Inlining Arg")); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 53f33c45a98c76..52fdf5ab3cd476 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2203,10 +2203,10 @@ void Compiler::impPopArgsForSwiftCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, } else { - unsigned relOffset = 0; - auto addSegment = [=, &loweredNode, &relOffset](var_types type) { + unsigned relOffset = 0; + auto addSegment = [=, &loweredNode, &relOffset](var_types type) { GenTree* val = gtNewLclFldNode(structVal->GetLclNum(), type, - structVal->GetLclOffs() + offset + relOffset); + structVal->GetLclOffs() + offset + relOffset); if (loweredType == TYP_LONG) { @@ -2216,7 +2216,7 @@ void Compiler::impPopArgsForSwiftCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, if (relOffset > 0) { val = gtNewOperNode(GT_LSH, genActualType(loweredType), val, - gtNewIconNode(relOffset * 8)); + gtNewIconNode(relOffset * 8)); } if (loweredNode == nullptr) @@ -3286,7 +3286,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, GenTree* op1 = impPopStack().val; GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars, OFFSETOF__CORINFO_String__stringLen); - retNode = gtNewIndexIndir(addr->AsIndexAddr()); + retNode = gtNewIndexIndir(addr->AsIndexAddr()); break; } @@ -3633,8 +3633,8 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL; } assert(op1->AsCall()->gtArgs.CountArgs() == 1); - op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, - op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, + op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); op1->gtType = TYP_REF; retNode = op1; } @@ -6115,7 +6115,8 @@ void Compiler::impCheckForPInvokeCall( class SpillRetExprHelper { public: - SpillRetExprHelper(Compiler* comp) : comp(comp) + SpillRetExprHelper(Compiler* comp) + : comp(comp) { } @@ -6783,7 +6784,9 @@ void Compiler::considerGuardedDevirtualization(GenTreeCall* call, #ifdef DEBUG char buffer[256]; JITDUMP("%s call would invoke method %s\n", - isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", + isInterface ? "interface" + : call->IsDelegateInvoke() ? "delegate" + : "virtual", eeGetMethodFullName(likelyMethod, true, true, buffer, sizeof(buffer))); #endif @@ -7278,8 +7281,8 @@ bool Compiler::IsTargetIntrinsic(NamedIntrinsic intrinsicName) #if defined(TARGET_XARCH) switch (intrinsicName) { - // AMD64/x86 has SSE2 instructions to directly compute sqrt/abs and SSE4.1 - // instructions to directly compute round/ceiling/floor/truncate. + // AMD64/x86 has SSE2 instructions to directly compute sqrt/abs and SSE4.1 + // instructions to directly compute round/ceiling/floor/truncate. case NI_System_Math_Abs: case NI_System_Math_Sqrt: @@ -7482,8 +7485,8 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // Optionally, print info on devirtualization Compiler* const rootCompiler = impInlineRoot(); const bool doPrint = JitConfig.JitPrintDevirtualizedMethods().contains(rootCompiler->info.compMethodHnd, - rootCompiler->info.compClassHnd, - &rootCompiler->info.compMethodInfo->args); + rootCompiler->info.compClassHnd, + &rootCompiler->info.compMethodInfo->args); #endif // DEBUG // Fetch information about the virtual method we're calling. @@ -8432,161 +8435,160 @@ void Compiler::impCheckCanInline(GenTreeCall* call, bool success = eeRunWithErrorTrap( [](Param* pParam) { - - // Cache some frequently accessed state. - // - Compiler* const compiler = pParam->pThis; - COMP_HANDLE compCompHnd = compiler->info.compCompHnd; - CORINFO_METHOD_HANDLE ftn = pParam->fncHandle; - InlineResult* const inlineResult = pParam->result; + // Cache some frequently accessed state. + // + Compiler* const compiler = pParam->pThis; + COMP_HANDLE compCompHnd = compiler->info.compCompHnd; + CORINFO_METHOD_HANDLE ftn = pParam->fncHandle; + InlineResult* const inlineResult = pParam->result; #ifdef DEBUG - if (JitConfig.JitNoInline()) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_IS_JIT_NOINLINE); - return; - } + if (JitConfig.JitNoInline()) + { + inlineResult->NoteFatal(InlineObservation::CALLEE_IS_JIT_NOINLINE); + return; + } #endif - JITDUMP("\nCheckCanInline: fetching method info for inline candidate %s -- context %p\n", - compiler->eeGetMethodName(ftn), compiler->dspPtr(pParam->exactContextHnd)); + JITDUMP("\nCheckCanInline: fetching method info for inline candidate %s -- context %p\n", + compiler->eeGetMethodName(ftn), compiler->dspPtr(pParam->exactContextHnd)); - if (pParam->exactContextHnd == METHOD_BEING_COMPILED_CONTEXT()) - { - JITDUMP("Current method context\n"); - } - else if ((((size_t)pParam->exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_METHOD)) - { - JITDUMP("Method context: %s\n", - compiler->eeGetMethodFullName((CORINFO_METHOD_HANDLE)pParam->exactContextHnd)); - } - else - { - JITDUMP("Class context: %s\n", compiler->eeGetClassName((CORINFO_CLASS_HANDLE)( - (size_t)pParam->exactContextHnd & ~CORINFO_CONTEXTFLAGS_MASK))); - } - - // Fetch method info. This may fail, if the method doesn't have IL. - // - CORINFO_METHOD_INFO methInfo; - if (!compCompHnd->getMethodInfo(ftn, &methInfo, pParam->exactContextHnd)) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_NO_METHOD_INFO); - return; - } + if (pParam->exactContextHnd == METHOD_BEING_COMPILED_CONTEXT()) + { + JITDUMP("Current method context\n"); + } + else if ((((size_t)pParam->exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_METHOD)) + { + JITDUMP("Method context: %s\n", + compiler->eeGetMethodFullName((CORINFO_METHOD_HANDLE)pParam->exactContextHnd)); + } + else + { + JITDUMP("Class context: %s\n", + compiler->eeGetClassName( + (CORINFO_CLASS_HANDLE)((size_t)pParam->exactContextHnd & ~CORINFO_CONTEXTFLAGS_MASK))); + } - // Profile data allows us to avoid early "too many IL bytes" outs. - // - inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE_WEIGHTS, - compiler->fgHaveSufficientProfileWeights()); - inlineResult->NoteBool(InlineObservation::CALLSITE_INSIDE_THROW_BLOCK, - compiler->compCurBB->KindIs(BBJ_THROW)); + // Fetch method info. This may fail, if the method doesn't have IL. + // + CORINFO_METHOD_INFO methInfo; + if (!compCompHnd->getMethodInfo(ftn, &methInfo, pParam->exactContextHnd)) + { + inlineResult->NoteFatal(InlineObservation::CALLEE_NO_METHOD_INFO); + return; + } - bool const forceInline = (pParam->methAttr & CORINFO_FLG_FORCEINLINE) != 0; + // Profile data allows us to avoid early "too many IL bytes" outs. + // + inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE_WEIGHTS, + compiler->fgHaveSufficientProfileWeights()); + inlineResult->NoteBool(InlineObservation::CALLSITE_INSIDE_THROW_BLOCK, compiler->compCurBB->KindIs(BBJ_THROW)); - compiler->impCanInlineIL(ftn, &methInfo, forceInline, inlineResult); + bool const forceInline = (pParam->methAttr & CORINFO_FLG_FORCEINLINE) != 0; - if (inlineResult->IsFailure()) - { - assert(inlineResult->IsNever()); - return; - } + compiler->impCanInlineIL(ftn, &methInfo, forceInline, inlineResult); - // Speculatively check if initClass() can be done. - // If it can be done, we will try to inline the method. - CorInfoInitClassResult const initClassResult = - compCompHnd->initClass(nullptr /* field */, ftn /* method */, pParam->exactContextHnd /* context */); + if (inlineResult->IsFailure()) + { + assert(inlineResult->IsNever()); + return; + } - if (initClassResult & CORINFO_INITCLASS_DONT_INLINE) - { - inlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_CLASS_INIT); - return; - } + // Speculatively check if initClass() can be done. + // If it can be done, we will try to inline the method. + CorInfoInitClassResult const initClassResult = + compCompHnd->initClass(nullptr /* field */, ftn /* method */, pParam->exactContextHnd /* context */); - // Given the VM the final say in whether to inline or not. - // This should be last since for verifiable code, this can be expensive - // - CorInfoInline const vmResult = compCompHnd->canInline(compiler->info.compMethodHnd, ftn); + if (initClassResult & CORINFO_INITCLASS_DONT_INLINE) + { + inlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_CLASS_INIT); + return; + } - if (vmResult == INLINE_FAIL) - { - inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_VM_NOINLINE); - } - else if (vmResult == INLINE_NEVER) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_IS_VM_NOINLINE); - } + // Given the VM the final say in whether to inline or not. + // This should be last since for verifiable code, this can be expensive + // + CorInfoInline const vmResult = compCompHnd->canInline(compiler->info.compMethodHnd, ftn); - if (inlineResult->IsFailure()) - { - // The VM already self-reported this failure, so mark it specially - // so the JIT doesn't also try reporting it. - // - inlineResult->SetVMFailure(); - return; - } + if (vmResult == INLINE_FAIL) + { + inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_VM_NOINLINE); + } + else if (vmResult == INLINE_NEVER) + { + inlineResult->NoteFatal(InlineObservation::CALLEE_IS_VM_NOINLINE); + } - // Get the method's class properties + if (inlineResult->IsFailure()) + { + // The VM already self-reported this failure, so mark it specially + // so the JIT doesn't also try reporting it. // - CORINFO_CLASS_HANDLE clsHandle = compCompHnd->getMethodClass(ftn); - unsigned const clsAttr = compCompHnd->getClassAttribs(clsHandle); + inlineResult->SetVMFailure(); + return; + } - // Return type - // - var_types const fncRetType = pParam->call->TypeGet(); + // Get the method's class properties + // + CORINFO_CLASS_HANDLE clsHandle = compCompHnd->getMethodClass(ftn); + unsigned const clsAttr = compCompHnd->getClassAttribs(clsHandle); + + // Return type + // + var_types const fncRetType = pParam->call->TypeGet(); #ifdef DEBUG - var_types fncRealRetType = JITtype2varType(methInfo.args.retType); + var_types fncRealRetType = JITtype2varType(methInfo.args.retType); - assert((genActualType(fncRealRetType) == genActualType(fncRetType)) || - // VSW 288602 - // In case of IJW, we allow to assign a native pointer to a BYREF. - (fncRetType == TYP_BYREF && methInfo.args.retType == CORINFO_TYPE_PTR) || - (varTypeIsStruct(fncRetType) && (fncRealRetType == TYP_STRUCT))); + assert((genActualType(fncRealRetType) == genActualType(fncRetType)) || + // VSW 288602 + // In case of IJW, we allow to assign a native pointer to a BYREF. + (fncRetType == TYP_BYREF && methInfo.args.retType == CORINFO_TYPE_PTR) || + (varTypeIsStruct(fncRetType) && (fncRealRetType == TYP_STRUCT))); #endif - // Allocate an InlineCandidateInfo structure, - // - // Or, reuse the existing GuardedDevirtualizationCandidateInfo, - // which was pre-allocated to have extra room. - // - InlineCandidateInfo* pInfo; + // Allocate an InlineCandidateInfo structure, + // + // Or, reuse the existing GuardedDevirtualizationCandidateInfo, + // which was pre-allocated to have extra room. + // + InlineCandidateInfo* pInfo; - if (pParam->call->IsGuardedDevirtualizationCandidate()) - { - pInfo = pParam->call->GetGDVCandidateInfo(pParam->candidateIndex); - } - else - { - pInfo = new (pParam->pThis, CMK_Inlining) InlineCandidateInfo; + if (pParam->call->IsGuardedDevirtualizationCandidate()) + { + pInfo = pParam->call->GetGDVCandidateInfo(pParam->candidateIndex); + } + else + { + pInfo = new (pParam->pThis, CMK_Inlining) InlineCandidateInfo; - // Null out bits we don't use when we're just inlining - // - pInfo->guardedClassHandle = nullptr; - pInfo->guardedMethodHandle = nullptr; - pInfo->guardedMethodUnboxedEntryHandle = nullptr; - pInfo->likelihood = 0; - pInfo->requiresInstMethodTableArg = false; - } - - pInfo->methInfo = methInfo; - pInfo->ilCallerHandle = pParam->pThis->info.compMethodHnd; - pInfo->clsHandle = clsHandle; - pInfo->exactContextHnd = pParam->exactContextHnd; - pInfo->retExpr = nullptr; - pInfo->preexistingSpillTemp = BAD_VAR_NUM; - pInfo->clsAttr = clsAttr; - pInfo->methAttr = pParam->methAttr; - pInfo->initClassResult = initClassResult; - pInfo->fncRetType = fncRetType; - pInfo->exactContextNeedsRuntimeLookup = false; - pInfo->inlinersContext = pParam->pThis->compInlineContext; - - // Note exactContextNeedsRuntimeLookup is reset later on, - // over in impMarkInlineCandidate. + // Null out bits we don't use when we're just inlining // - *(pParam->ppInlineCandidateInfo) = pInfo; - }, + pInfo->guardedClassHandle = nullptr; + pInfo->guardedMethodHandle = nullptr; + pInfo->guardedMethodUnboxedEntryHandle = nullptr; + pInfo->likelihood = 0; + pInfo->requiresInstMethodTableArg = false; + } + + pInfo->methInfo = methInfo; + pInfo->ilCallerHandle = pParam->pThis->info.compMethodHnd; + pInfo->clsHandle = clsHandle; + pInfo->exactContextHnd = pParam->exactContextHnd; + pInfo->retExpr = nullptr; + pInfo->preexistingSpillTemp = BAD_VAR_NUM; + pInfo->clsAttr = clsAttr; + pInfo->methAttr = pParam->methAttr; + pInfo->initClassResult = initClassResult; + pInfo->fncRetType = fncRetType; + pInfo->exactContextNeedsRuntimeLookup = false; + pInfo->inlinersContext = pParam->pThis->compInlineContext; + + // Note exactContextNeedsRuntimeLookup is reset later on, + // over in impMarkInlineCandidate. + // + *(pParam->ppInlineCandidateInfo) = pInfo; + }, ¶m); if (!success) @@ -9560,372 +9562,374 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) else #endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) if (strcmp(namespaceName, "Collections.Generic") == 0) - { - if (strcmp(className, "Comparer`1") == 0) { - if (strcmp(methodName, "get_Default") == 0) + if (strcmp(className, "Comparer`1") == 0) + { + if (strcmp(methodName, "get_Default") == 0) + { + result = NI_System_Collections_Generic_Comparer_get_Default; + } + } + else if (strcmp(className, "EqualityComparer`1") == 0) { - result = NI_System_Collections_Generic_Comparer_get_Default; + if (strcmp(methodName, "get_Default") == 0) + { + result = NI_System_Collections_Generic_EqualityComparer_get_Default; + } } } - else if (strcmp(className, "EqualityComparer`1") == 0) + else if (strcmp(namespaceName, "Numerics") == 0) { - if (strcmp(methodName, "get_Default") == 0) + if (strcmp(className, "BitOperations") == 0) { - result = NI_System_Collections_Generic_EqualityComparer_get_Default; + result = lookupPrimitiveIntNamedIntrinsic(method, methodName); } - } - } - else if (strcmp(namespaceName, "Numerics") == 0) - { - if (strcmp(className, "BitOperations") == 0) - { - result = lookupPrimitiveIntNamedIntrinsic(method, methodName); - } - else - { + else + { #ifdef FEATURE_HW_INTRINSICS - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(method, &sig); + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(method, &sig); - result = SimdAsHWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); + result = SimdAsHWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); #endif // FEATURE_HW_INTRINSICS - if (result == NI_Illegal) - { - // This allows the relevant code paths to be dropped as dead code even - // on platforms where FEATURE_HW_INTRINSICS is not supported. - - if (strcmp(methodName, "get_IsSupported") == 0) - { - assert(strcmp(className, "Vector`1") == 0); - result = NI_IsSupported_Type; - } - else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) - { - result = NI_IsSupported_False; - } - else if (strcmp(methodName, "get_Count") == 0) - { - assert(strcmp(className, "Vector`1") == 0); - result = NI_Vector_GetCount; - } - else if (gtIsRecursiveCall(method)) + if (result == NI_Illegal) { - // For the framework itself, any recursive intrinsics will either be - // only supported on a single platform or will be guarded by a relevant - // IsSupported check so the throw PNSE will be valid or dropped. + // This allows the relevant code paths to be dropped as dead code even + // on platforms where FEATURE_HW_INTRINSICS is not supported. - result = NI_Throw_PlatformNotSupportedException; + if (strcmp(methodName, "get_IsSupported") == 0) + { + assert(strcmp(className, "Vector`1") == 0); + result = NI_IsSupported_Type; + } + else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) + { + result = NI_IsSupported_False; + } + else if (strcmp(methodName, "get_Count") == 0) + { + assert(strcmp(className, "Vector`1") == 0); + result = NI_Vector_GetCount; + } + else if (gtIsRecursiveCall(method)) + { + // For the framework itself, any recursive intrinsics will either be + // only supported on a single platform or will be guarded by a relevant + // IsSupported check so the throw PNSE will be valid or dropped. + + result = NI_Throw_PlatformNotSupportedException; + } } } } - } - else if (strncmp(namespaceName, "Runtime.", 8) == 0) - { - namespaceName += 8; - - if (strcmp(namespaceName, "CompilerServices") == 0) + else if (strncmp(namespaceName, "Runtime.", 8) == 0) { - if (strcmp(className, "RuntimeHelpers") == 0) + namespaceName += 8; + + if (strcmp(namespaceName, "CompilerServices") == 0) { - if (strcmp(methodName, "CreateSpan") == 0) + if (strcmp(className, "RuntimeHelpers") == 0) { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan; - } - else if (strcmp(methodName, "InitializeArray") == 0) - { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray; + if (strcmp(methodName, "CreateSpan") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan; + } + else if (strcmp(methodName, "InitializeArray") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray; + } + else if (strcmp(methodName, "IsKnownConstant") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant; + } } - else if (strcmp(methodName, "IsKnownConstant") == 0) + else if (strcmp(className, "Unsafe") == 0) { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant; + if (strcmp(methodName, "Add") == 0) + { + result = NI_SRCS_UNSAFE_Add; + } + else if (strcmp(methodName, "AddByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_AddByteOffset; + } + else if (strcmp(methodName, "AreSame") == 0) + { + result = NI_SRCS_UNSAFE_AreSame; + } + else if (strcmp(methodName, "As") == 0) + { + result = NI_SRCS_UNSAFE_As; + } + else if (strcmp(methodName, "AsPointer") == 0) + { + result = NI_SRCS_UNSAFE_AsPointer; + } + else if (strcmp(methodName, "AsRef") == 0) + { + result = NI_SRCS_UNSAFE_AsRef; + } + else if (strcmp(methodName, "BitCast") == 0) + { + result = NI_SRCS_UNSAFE_BitCast; + } + else if (strcmp(methodName, "ByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_ByteOffset; + } + else if (strcmp(methodName, "Copy") == 0) + { + result = NI_SRCS_UNSAFE_Copy; + } + else if (strcmp(methodName, "CopyBlock") == 0) + { + result = NI_SRCS_UNSAFE_CopyBlock; + } + else if (strcmp(methodName, "CopyBlockUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_CopyBlockUnaligned; + } + else if (strcmp(methodName, "InitBlock") == 0) + { + result = NI_SRCS_UNSAFE_InitBlock; + } + else if (strcmp(methodName, "InitBlockUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_InitBlockUnaligned; + } + else if (strcmp(methodName, "IsAddressGreaterThan") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressGreaterThan; + } + else if (strcmp(methodName, "IsAddressLessThan") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressLessThan; + } + else if (strcmp(methodName, "IsNullRef") == 0) + { + result = NI_SRCS_UNSAFE_IsNullRef; + } + else if (strcmp(methodName, "NullRef") == 0) + { + result = NI_SRCS_UNSAFE_NullRef; + } + else if (strcmp(methodName, "Read") == 0) + { + result = NI_SRCS_UNSAFE_Read; + } + else if (strcmp(methodName, "ReadUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_ReadUnaligned; + } + else if (strcmp(methodName, "SizeOf") == 0) + { + result = NI_SRCS_UNSAFE_SizeOf; + } + else if (strcmp(methodName, "SkipInit") == 0) + { + result = NI_SRCS_UNSAFE_SkipInit; + } + else if (strcmp(methodName, "Subtract") == 0) + { + result = NI_SRCS_UNSAFE_Subtract; + } + else if (strcmp(methodName, "SubtractByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_SubtractByteOffset; + } + else if (strcmp(methodName, "Unbox") == 0) + { + result = NI_SRCS_UNSAFE_Unbox; + } + else if (strcmp(methodName, "Write") == 0) + { + result = NI_SRCS_UNSAFE_Write; + } + else if (strcmp(methodName, "WriteUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_WriteUnaligned; + } } } - else if (strcmp(className, "Unsafe") == 0) + else if (strcmp(namespaceName, "InteropServices") == 0) { - if (strcmp(methodName, "Add") == 0) - { - result = NI_SRCS_UNSAFE_Add; - } - else if (strcmp(methodName, "AddByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_AddByteOffset; - } - else if (strcmp(methodName, "AreSame") == 0) - { - result = NI_SRCS_UNSAFE_AreSame; - } - else if (strcmp(methodName, "As") == 0) - { - result = NI_SRCS_UNSAFE_As; - } - else if (strcmp(methodName, "AsPointer") == 0) - { - result = NI_SRCS_UNSAFE_AsPointer; - } - else if (strcmp(methodName, "AsRef") == 0) - { - result = NI_SRCS_UNSAFE_AsRef; - } - else if (strcmp(methodName, "BitCast") == 0) - { - result = NI_SRCS_UNSAFE_BitCast; - } - else if (strcmp(methodName, "ByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_ByteOffset; - } - else if (strcmp(methodName, "Copy") == 0) - { - result = NI_SRCS_UNSAFE_Copy; - } - else if (strcmp(methodName, "CopyBlock") == 0) - { - result = NI_SRCS_UNSAFE_CopyBlock; - } - else if (strcmp(methodName, "CopyBlockUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_CopyBlockUnaligned; - } - else if (strcmp(methodName, "InitBlock") == 0) - { - result = NI_SRCS_UNSAFE_InitBlock; - } - else if (strcmp(methodName, "InitBlockUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_InitBlockUnaligned; - } - else if (strcmp(methodName, "IsAddressGreaterThan") == 0) - { - result = NI_SRCS_UNSAFE_IsAddressGreaterThan; - } - else if (strcmp(methodName, "IsAddressLessThan") == 0) - { - result = NI_SRCS_UNSAFE_IsAddressLessThan; - } - else if (strcmp(methodName, "IsNullRef") == 0) - { - result = NI_SRCS_UNSAFE_IsNullRef; - } - else if (strcmp(methodName, "NullRef") == 0) - { - result = NI_SRCS_UNSAFE_NullRef; - } - else if (strcmp(methodName, "Read") == 0) - { - result = NI_SRCS_UNSAFE_Read; - } - else if (strcmp(methodName, "ReadUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_ReadUnaligned; - } - else if (strcmp(methodName, "SizeOf") == 0) - { - result = NI_SRCS_UNSAFE_SizeOf; - } - else if (strcmp(methodName, "SkipInit") == 0) - { - result = NI_SRCS_UNSAFE_SkipInit; - } - else if (strcmp(methodName, "Subtract") == 0) - { - result = NI_SRCS_UNSAFE_Subtract; - } - else if (strcmp(methodName, "SubtractByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_SubtractByteOffset; - } - else if (strcmp(methodName, "Unbox") == 0) - { - result = NI_SRCS_UNSAFE_Unbox; - } - else if (strcmp(methodName, "Write") == 0) - { - result = NI_SRCS_UNSAFE_Write; - } - else if (strcmp(methodName, "WriteUnaligned") == 0) + if (strcmp(className, "MemoryMarshal") == 0) { - result = NI_SRCS_UNSAFE_WriteUnaligned; + if (strcmp(methodName, "GetArrayDataReference") == 0) + { + result = NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference; + } } } - } - else if (strcmp(namespaceName, "InteropServices") == 0) - { - if (strcmp(className, "MemoryMarshal") == 0) + else if (strncmp(namespaceName, "Intrinsics", 10) == 0) { - if (strcmp(methodName, "GetArrayDataReference") == 0) - { - result = NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference; - } - } - } - else if (strncmp(namespaceName, "Intrinsics", 10) == 0) - { - // We go down this path even when FEATURE_HW_INTRINSICS isn't enabled - // so we can specially handle IsSupported and recursive calls. - - // This is required to appropriately handle the intrinsics on platforms - // which don't support them. On such a platform methods like Vector64.Create - // will be seen as `Intrinsic` and `mustExpand` due to having a code path - // which is recursive. When such a path is hit we expect it to be handled by - // the importer and we fire an assert if it wasn't and in previous versions - // of the JIT would fail fast. This was changed to throw a PNSE instead but - // we still assert as most intrinsics should have been recognized/handled. - - // In order to avoid the assert, we specially handle the IsSupported checks - // (to better allow dead-code optimizations) and we explicitly throw a PNSE - // as we know that is the desired behavior for the HWIntrinsics when not - // supported. For cases like Vector64.Create, this is fine because it will - // be behind a relevant IsSupported check and will never be hit and the - // software fallback will be executed instead. - - CLANG_FORMAT_COMMENT_ANCHOR; + // We go down this path even when FEATURE_HW_INTRINSICS isn't enabled + // so we can specially handle IsSupported and recursive calls. + + // This is required to appropriately handle the intrinsics on platforms + // which don't support them. On such a platform methods like Vector64.Create + // will be seen as `Intrinsic` and `mustExpand` due to having a code path + // which is recursive. When such a path is hit we expect it to be handled by + // the importer and we fire an assert if it wasn't and in previous versions + // of the JIT would fail fast. This was changed to throw a PNSE instead but + // we still assert as most intrinsics should have been recognized/handled. + + // In order to avoid the assert, we specially handle the IsSupported checks + // (to better allow dead-code optimizations) and we explicitly throw a PNSE + // as we know that is the desired behavior for the HWIntrinsics when not + // supported. For cases like Vector64.Create, this is fine because it will + // be behind a relevant IsSupported check and will never be hit and the + // software fallback will be executed instead. + + CLANG_FORMAT_COMMENT_ANCHOR; #ifdef FEATURE_HW_INTRINSICS - namespaceName += 10; - const char* platformNamespaceName; + namespaceName += 10; + const char* platformNamespaceName; #if defined(TARGET_XARCH) - platformNamespaceName = ".X86"; + platformNamespaceName = ".X86"; #elif defined(TARGET_ARM64) - platformNamespaceName = ".Arm"; + platformNamespaceName = ".Arm"; #else #error Unsupported platform #endif - if ((namespaceName[0] == '\0') || (strcmp(namespaceName, platformNamespaceName) == 0)) - { - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(method, &sig); + if ((namespaceName[0] == '\0') || (strcmp(namespaceName, platformNamespaceName) == 0)) + { + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(method, &sig); - result = HWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); - } + result = HWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); + } #endif // FEATURE_HW_INTRINSICS - if (result == NI_Illegal) - { - // This allows the relevant code paths to be dropped as dead code even - // on platforms where FEATURE_HW_INTRINSICS is not supported. - - if (strcmp(methodName, "get_IsSupported") == 0) + if (result == NI_Illegal) { - if (strncmp(className, "Vector", 6) == 0) + // This allows the relevant code paths to be dropped as dead code even + // on platforms where FEATURE_HW_INTRINSICS is not supported. + + if (strcmp(methodName, "get_IsSupported") == 0) { - assert( - (strcmp(className, "Vector64`1") == 0) || (strcmp(className, "Vector128`1") == 0) || - (strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector512`1") == 0)); + if (strncmp(className, "Vector", 6) == 0) + { + assert((strcmp(className, "Vector64`1") == 0) || + (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || + (strcmp(className, "Vector512`1") == 0)); - result = NI_IsSupported_Type; + result = NI_IsSupported_Type; + } + else + { + result = NI_IsSupported_False; + } } - else + else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) { result = NI_IsSupported_False; } - } - else if (strcmp(methodName, "get_IsHardwareAccelerated") == 0) - { - result = NI_IsSupported_False; - } - else if (strcmp(methodName, "get_Count") == 0) - { - assert((strcmp(className, "Vector64`1") == 0) || (strcmp(className, "Vector128`1") == 0) || - (strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector512`1") == 0)); + else if (strcmp(methodName, "get_Count") == 0) + { + assert( + (strcmp(className, "Vector64`1") == 0) || (strcmp(className, "Vector128`1") == 0) || + (strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector512`1") == 0)); - result = NI_Vector_GetCount; - } - else if (gtIsRecursiveCall(method)) - { - // For the framework itself, any recursive intrinsics will either be - // only supported on a single platform or will be guarded by a relevant - // IsSupported check so the throw PNSE will be valid or dropped. + result = NI_Vector_GetCount; + } + else if (gtIsRecursiveCall(method)) + { + // For the framework itself, any recursive intrinsics will either be + // only supported on a single platform or will be guarded by a relevant + // IsSupported check so the throw PNSE will be valid or dropped. - result = NI_Throw_PlatformNotSupportedException; + result = NI_Throw_PlatformNotSupportedException; + } } } } - } - else if (strcmp(namespaceName, "StubHelpers") == 0) - { - if (strcmp(className, "StubHelpers") == 0) + else if (strcmp(namespaceName, "StubHelpers") == 0) { - if (strcmp(methodName, "GetStubContext") == 0) + if (strcmp(className, "StubHelpers") == 0) { - result = NI_System_StubHelpers_GetStubContext; - } - else if (strcmp(methodName, "NextCallReturnAddress") == 0) - { - result = NI_System_StubHelpers_NextCallReturnAddress; - } - } - } - else if (strcmp(namespaceName, "Text") == 0) - { - if (strcmp(className, "UTF8EncodingSealed") == 0) - { - if (strcmp(methodName, "ReadUtf8") == 0) - { - assert(strcmp(enclosingClassName, "UTF8Encoding") == 0); - result = NI_System_Text_UTF8Encoding_UTF8EncodingSealed_ReadUtf8; + if (strcmp(methodName, "GetStubContext") == 0) + { + result = NI_System_StubHelpers_GetStubContext; + } + else if (strcmp(methodName, "NextCallReturnAddress") == 0) + { + result = NI_System_StubHelpers_NextCallReturnAddress; + } } } - } - else if (strcmp(namespaceName, "Threading") == 0) - { - if (strcmp(className, "Interlocked") == 0) + else if (strcmp(namespaceName, "Text") == 0) { - if (strcmp(methodName, "And") == 0) + if (strcmp(className, "UTF8EncodingSealed") == 0) { - result = NI_System_Threading_Interlocked_And; - } - else if (strcmp(methodName, "Or") == 0) - { - result = NI_System_Threading_Interlocked_Or; - } - else if (strcmp(methodName, "CompareExchange") == 0) - { - result = NI_System_Threading_Interlocked_CompareExchange; - } - else if (strcmp(methodName, "Exchange") == 0) - { - result = NI_System_Threading_Interlocked_Exchange; - } - else if (strcmp(methodName, "ExchangeAdd") == 0) - { - result = NI_System_Threading_Interlocked_ExchangeAdd; - } - else if (strcmp(methodName, "MemoryBarrier") == 0) - { - result = NI_System_Threading_Interlocked_MemoryBarrier; - } - else if (strcmp(methodName, "ReadMemoryBarrier") == 0) - { - result = NI_System_Threading_Interlocked_ReadMemoryBarrier; + if (strcmp(methodName, "ReadUtf8") == 0) + { + assert(strcmp(enclosingClassName, "UTF8Encoding") == 0); + result = NI_System_Text_UTF8Encoding_UTF8EncodingSealed_ReadUtf8; + } } } - else if (strcmp(className, "Thread") == 0) + else if (strcmp(namespaceName, "Threading") == 0) { - if (strcmp(methodName, "get_CurrentThread") == 0) + if (strcmp(className, "Interlocked") == 0) { - result = NI_System_Threading_Thread_get_CurrentThread; - } - else if (strcmp(methodName, "get_ManagedThreadId") == 0) - { - result = NI_System_Threading_Thread_get_ManagedThreadId; + if (strcmp(methodName, "And") == 0) + { + result = NI_System_Threading_Interlocked_And; + } + else if (strcmp(methodName, "Or") == 0) + { + result = NI_System_Threading_Interlocked_Or; + } + else if (strcmp(methodName, "CompareExchange") == 0) + { + result = NI_System_Threading_Interlocked_CompareExchange; + } + else if (strcmp(methodName, "Exchange") == 0) + { + result = NI_System_Threading_Interlocked_Exchange; + } + else if (strcmp(methodName, "ExchangeAdd") == 0) + { + result = NI_System_Threading_Interlocked_ExchangeAdd; + } + else if (strcmp(methodName, "MemoryBarrier") == 0) + { + result = NI_System_Threading_Interlocked_MemoryBarrier; + } + else if (strcmp(methodName, "ReadMemoryBarrier") == 0) + { + result = NI_System_Threading_Interlocked_ReadMemoryBarrier; + } } - } - else if (strcmp(className, "Volatile") == 0) - { - if (strcmp(methodName, "Read") == 0) + else if (strcmp(className, "Thread") == 0) { - result = NI_System_Threading_Volatile_Read; + if (strcmp(methodName, "get_CurrentThread") == 0) + { + result = NI_System_Threading_Thread_get_CurrentThread; + } + else if (strcmp(methodName, "get_ManagedThreadId") == 0) + { + result = NI_System_Threading_Thread_get_ManagedThreadId; + } } - else if (strcmp(methodName, "Write") == 0) + else if (strcmp(className, "Volatile") == 0) { - result = NI_System_Threading_Volatile_Write; + if (strcmp(methodName, "Read") == 0) + { + result = NI_System_Threading_Volatile_Read; + } + else if (strcmp(methodName, "Write") == 0) + { + result = NI_System_Threading_Volatile_Write; + } } } - } } } else if (strcmp(namespaceName, "Internal.Runtime") == 0) diff --git a/src/coreclr/jit/importervectorization.cpp b/src/coreclr/jit/importervectorization.cpp index af7a2f2791d169..26ae9225cbd7ed 100644 --- a/src/coreclr/jit/importervectorization.cpp +++ b/src/coreclr/jit/importervectorization.cpp @@ -182,7 +182,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( xor1 = gtNewSimdBinOpNode(GT_XOR, simdType, vec1, cnsVec1, baseType, simdSize); } -// ((v1 ^ cns1) | (v2 ^ cns2)) == zero + // ((v1 ^ cns1) | (v2 ^ cns2)) == zero #if defined(TARGET_XARCH) if (compOpportunisticallyDependsOn(InstructionSet_AVX512F_VL)) @@ -317,7 +317,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( assert(len >= 1 && len <= 8); // Compose Int32 or Int64 values from ushort components -#define MAKEINT32(c1, c2) ((UINT64)c2 << 16) | ((UINT64)c1 << 0) +#define MAKEINT32(c1, c2) ((UINT64)c2 << 16) | ((UINT64)c1 << 0) #define MAKEINT64(c1, c2, c3, c4) ((UINT64)c4 << 48) | ((UINT64)c3 << 32) | ((UINT64)c2 << 16) | ((UINT64)c1 << 0) if (len == 1) @@ -516,10 +516,10 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVarCommon* data, GenTree* castedLen = gtNewCastNode(TYP_I_IMPL, gtCloneExpr(lengthFld), false, TYP_I_IMPL); GenTree* byteLen = gtNewOperNode(GT_MUL, TYP_I_IMPL, castedLen, gtNewIconNode(2, TYP_I_IMPL)); GenTreeOp* cmpStart = gtNewOperNode(GT_ADD, TYP_BYREF, gtClone(data), - gtNewOperNode(GT_SUB, TYP_I_IMPL, byteLen, - gtNewIconNode((ssize_t)(len * 2), TYP_I_IMPL))); - GenTree* storeTmp = gtNewTempStore(dataAddr->GetLclNum(), cmpStart); - indirCmp = gtNewOperNode(GT_COMMA, indirCmp->TypeGet(), storeTmp, indirCmp); + gtNewOperNode(GT_SUB, TYP_I_IMPL, byteLen, + gtNewIconNode((ssize_t)(len * 2), TYP_I_IMPL))); + GenTree* storeTmp = gtNewTempStore(dataAddr->GetLclNum(), cmpStart); + indirCmp = gtNewOperNode(GT_COMMA, indirCmp->TypeGet(), storeTmp, indirCmp); } GenTreeColon* lenCheckColon = gtNewColonNode(TYP_INT, indirCmp, gtNewFalse()); diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 0839f9fc2a045e..a85ba05596b519 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -67,7 +67,8 @@ class IndirectCallTransformer { public: - IndirectCallTransformer(Compiler* compiler) : compiler(compiler) + IndirectCallTransformer(Compiler* compiler) + : compiler(compiler) { } @@ -157,7 +158,9 @@ class IndirectCallTransformer { public: Transformer(Compiler* compiler, BasicBlock* block, Statement* stmt) - : compiler(compiler), currBlock(block), stmt(stmt) + : compiler(compiler) + , currBlock(block) + , stmt(stmt) { remainderBlock = nullptr; checkBlock = nullptr; @@ -197,7 +200,7 @@ class IndirectCallTransformer virtual const char* Name() = 0; virtual void ClearFlag() = 0; virtual GenTreeCall* GetCall(Statement* callStmt) = 0; - virtual void FixupRetExpr() = 0; + virtual void FixupRetExpr() = 0; //------------------------------------------------------------------------ // CreateRemainder: split current block at the call stmt and @@ -473,7 +476,8 @@ class IndirectCallTransformer { public: GuardedDevirtualizationTransformer(Compiler* compiler, BasicBlock* block, Statement* stmt) - : Transformer(compiler, block, stmt), returnTemp(BAD_VAR_NUM) + : Transformer(compiler, block, stmt) + , returnTemp(BAD_VAR_NUM) { } @@ -1259,7 +1263,9 @@ class IndirectCallTransformer unsigned m_nodeCount; ClonabilityVisitor(Compiler* compiler) - : GenTreeVisitor(compiler), m_unclonableNode(nullptr), m_nodeCount(0) + : GenTreeVisitor(compiler) + , m_unclonableNode(nullptr) + , m_nodeCount(0) { } diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp index 59e5b6a0d497de..19755c312de350 100644 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -66,7 +66,6 @@ bool Compiler::optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop) LclVarDsc* dsc = lvaGetDesc(lclNum); BasicBlockVisit result = loop->VisitRegularExitBlocks([=](BasicBlock* exit) { - if (!VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) { JITDUMP(" Exit " FMT_BB " does not need a sink; V%02u is not live-in\n", exit->bbNum, lclNum); @@ -94,7 +93,6 @@ bool Compiler::optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop) // unprofitable. If this ever changes we need some more expansive handling // here. loop->VisitLoopBlocks([=](BasicBlock* block) { - block->VisitAllSuccs(this, [=](BasicBlock* succ) { if (!loop->ContainsBlock(succ) && bbIsHandlerBeg(succ)) { @@ -334,7 +332,10 @@ void Compiler::optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned ne }; ReplaceVisitor(Compiler* comp, unsigned lclNum, unsigned ssaNum, unsigned newLclNum) - : GenTreeVisitor(comp), m_lclNum(lclNum), m_ssaNum(ssaNum), m_newLclNum(newLclNum) + : GenTreeVisitor(comp) + , m_lclNum(lclNum) + , m_ssaNum(ssaNum) + , m_newLclNum(newLclNum) { } diff --git a/src/coreclr/jit/inline.cpp b/src/coreclr/jit/inline.cpp index 06ca71126f855c..c8831a75b39bc5 100644 --- a/src/coreclr/jit/inline.cpp +++ b/src/coreclr/jit/inline.cpp @@ -383,7 +383,7 @@ void InlineContext::Dump(bool verbose, unsigned indent) #if defined(DEBUG) calleeName = compiler->eeGetMethodFullName(m_Callee); #else - calleeName = "callee"; + calleeName = "callee"; #endif // defined(DEBUG) } diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 342dc3fca5d238..a87b2de79c058e 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -222,9 +222,9 @@ class InlinePolicy } // Policy observations - virtual void NoteSuccess() = 0; - virtual void NoteBool(InlineObservation obs, bool value) = 0; - virtual void NoteFatal(InlineObservation obs) = 0; + virtual void NoteSuccess() = 0; + virtual void NoteBool(InlineObservation obs, bool value) = 0; + virtual void NoteFatal(InlineObservation obs) = 0; virtual void NoteInt(InlineObservation obs, int value) = 0; virtual void NoteDouble(InlineObservation obs, double value) = 0; @@ -321,7 +321,7 @@ class InlinePolicy private: // No copying or assignment supported - InlinePolicy(const InlinePolicy&) = delete; + InlinePolicy(const InlinePolicy&) = delete; InlinePolicy& operator=(const InlinePolicy&) = delete; protected: @@ -558,7 +558,7 @@ class InlineResult private: // No copying or assignment allowed. - InlineResult(const InlineResult&) = delete; + InlineResult(const InlineResult&) = delete; InlineResult& operator=(const InlineResult&) = delete; // Report/log/dump decision as appropriate @@ -637,16 +637,16 @@ struct InlArgInfo CallArg* arg; // the caller argument GenTree* argBashTmpNode; // tmp node created, if it may be replaced with actual arg unsigned argTmpNum; // the argument tmp number - unsigned argIsUsed : 1; // is this arg used at all? - unsigned argIsInvariant : 1; // the argument is a constant or a local variable address - unsigned argIsLclVar : 1; // the argument is a local variable - unsigned argIsThis : 1; // the argument is the 'this' pointer - unsigned argHasSideEff : 1; // the argument has side effects - unsigned argHasGlobRef : 1; // the argument has a global ref - unsigned argHasCallerLocalRef : 1; // the argument value depends on an aliased caller local - unsigned argHasTmp : 1; // the argument will be evaluated to a temp - unsigned argHasLdargaOp : 1; // Is there LDARGA(s) operation on this argument? - unsigned argHasStargOp : 1; // Is there STARG(s) operation on this argument? + unsigned argIsUsed : 1; // is this arg used at all? + unsigned argIsInvariant : 1; // the argument is a constant or a local variable address + unsigned argIsLclVar : 1; // the argument is a local variable + unsigned argIsThis : 1; // the argument is the 'this' pointer + unsigned argHasSideEff : 1; // the argument has side effects + unsigned argHasGlobRef : 1; // the argument has a global ref + unsigned argHasCallerLocalRef : 1; // the argument value depends on an aliased caller local + unsigned argHasTmp : 1; // the argument will be evaluated to a temp + unsigned argHasLdargaOp : 1; // Is there LDARGA(s) operation on this argument? + unsigned argHasStargOp : 1; // Is there STARG(s) operation on this argument? unsigned argIsByRefToStructLocal : 1; // Is this arg an address of a struct local or a normed struct local or a // field in them? unsigned argIsExact : 1; // Is this arg of an exact class? @@ -658,10 +658,10 @@ struct InlLclVarInfo { CORINFO_CLASS_HANDLE lclTypeHandle; // Type handle from the signature. Available for structs and REFs. var_types lclTypeInfo; // Type from the signature. - unsigned char lclHasLdlocaOp : 1; // Is there LDLOCA(s) operation on this local? - unsigned char lclHasStlocOp : 1; // Is there a STLOC on this local? + unsigned char lclHasLdlocaOp : 1; // Is there LDLOCA(s) operation on this local? + unsigned char lclHasStlocOp : 1; // Is there a STLOC on this local? unsigned char lclHasMultipleStlocOp : 1; // Is there more than one STLOC on this local - unsigned char lclIsPinned : 1; + unsigned char lclIsPinned : 1; }; // InlineInfo provides detailed information about a particular inline candidate. @@ -887,8 +887,8 @@ class InlineContext InlinePolicy* m_Policy; // policy that evaluated this inline unsigned m_TreeID; // ID of the GenTreeCall in the parent bool m_Devirtualized : 1; // true if this was a devirtualized call - bool m_Guarded : 1; // true if this was a guarded call - bool m_Unboxed : 1; // true if this call now invokes the unboxed entry + bool m_Guarded : 1; // true if this was a guarded call + bool m_Unboxed : 1; // true if this call now invokes the unboxed entry #endif // defined(DEBUG) @@ -1026,7 +1026,7 @@ class InlineStrategy void DumpDataContents(FILE* file); // Dump xml-formatted description of inlines - void DumpXml(FILE* file = stderr, unsigned indent = 0); + void DumpXml(FILE* file = stderr, unsigned indent = 0); static void FinalizeXml(FILE* file = stderr); // Cache for file position of this method in the inline xml diff --git a/src/coreclr/jit/inlinepolicy.cpp b/src/coreclr/jit/inlinepolicy.cpp index d057ccd09ed0d9..3b771f291607bb 100644 --- a/src/coreclr/jit/inlinepolicy.cpp +++ b/src/coreclr/jit/inlinepolicy.cpp @@ -945,8 +945,9 @@ void DefaultPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo) { // Inline appears to be unprofitable JITLOG_THIS(m_RootCompiler, - (LL_INFO100000, "Native estimate for function size exceeds threshold" - " for inlining %g > %g (multiplier = %g)\n", + (LL_INFO100000, + "Native estimate for function size exceeds threshold" + " for inlining %g > %g (multiplier = %g)\n", (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier)); // Fail the inline @@ -963,8 +964,9 @@ void DefaultPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo) { // Inline appears to be profitable JITLOG_THIS(m_RootCompiler, - (LL_INFO100000, "Native estimate for function size is within threshold" - " for inlining %g <= %g (multiplier = %g)\n", + (LL_INFO100000, + "Native estimate for function size is within threshold" + " for inlining %g <= %g (multiplier = %g)\n", (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier)); // Update candidacy @@ -1072,7 +1074,8 @@ bool DefaultPolicy::PropagateNeverToRuntime() const // compiler -- compiler instance doing the inlining (root compiler) // isPrejitRoot -- true if this compiler is prejitting the root method -RandomPolicy::RandomPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot) +RandomPolicy::RandomPolicy(Compiler* compiler, bool isPrejitRoot) + : DiscretionaryPolicy(compiler, isPrejitRoot) { m_Random = compiler->m_inlineStrategy->GetRandom(); } @@ -2768,7 +2771,8 @@ void DiscretionaryPolicy::DumpData(FILE* file) const // compiler -- compiler instance doing the inlining (root compiler) // isPrejitRoot -- true if this compiler is prejitting the root method -ModelPolicy::ModelPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot) +ModelPolicy::ModelPolicy(Compiler* compiler, bool isPrejitRoot) + : DiscretionaryPolicy(compiler, isPrejitRoot) { // Empty } @@ -2969,7 +2973,8 @@ void ModelPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo) // compiler -- compiler instance doing the inlining (root compiler) // isPrejitRoot -- true if this compiler is prejitting the root method -ProfilePolicy::ProfilePolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot) +ProfilePolicy::ProfilePolicy(Compiler* compiler, bool isPrejitRoot) + : DiscretionaryPolicy(compiler, isPrejitRoot) { // Empty } @@ -3169,7 +3174,8 @@ void ProfilePolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo) // compiler -- compiler instance doing the inlining (root compiler) // isPrejitRoot -- true if this compiler is prejitting the root method -FullPolicy::FullPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot) +FullPolicy::FullPolicy(Compiler* compiler, bool isPrejitRoot) + : DiscretionaryPolicy(compiler, isPrejitRoot) { // Empty } @@ -3236,7 +3242,8 @@ void FullPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo) // compiler -- compiler instance doing the inlining (root compiler) // isPrejitRoot -- true if this compiler is prejitting the root method -SizePolicy::SizePolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot) +SizePolicy::SizePolicy(Compiler* compiler, bool isPrejitRoot) + : DiscretionaryPolicy(compiler, isPrejitRoot) { // Empty } diff --git a/src/coreclr/jit/inlinepolicy.h b/src/coreclr/jit/inlinepolicy.h index 52333d5aacac3c..a8d8e67f1db3cb 100644 --- a/src/coreclr/jit/inlinepolicy.h +++ b/src/coreclr/jit/inlinepolicy.h @@ -48,7 +48,8 @@ class LegalPolicy : public InlinePolicy public: // Constructor - LegalPolicy(bool isPrejitRoot) : InlinePolicy(isPrejitRoot) + LegalPolicy(bool isPrejitRoot) + : InlinePolicy(isPrejitRoot) { // empty } @@ -157,7 +158,7 @@ class DefaultPolicy : public LegalPolicy // Helper methods virtual double DetermineMultiplier(); int DetermineNativeSizeEstimate(); - int DetermineCallsiteNativeSizeEstimate(CORINFO_METHOD_INFO* methodInfo); + int DetermineCallsiteNativeSizeEstimate(CORINFO_METHOD_INFO* methodInfo); // Data members Compiler* m_RootCompiler; // root compiler instance @@ -174,20 +175,20 @@ class DefaultPolicy : public LegalPolicy unsigned m_ConstantArgFeedsConstantTest; int m_CalleeNativeSizeEstimate; int m_CallsiteNativeSizeEstimate; - bool m_IsForceInline : 1; - bool m_IsForceInlineKnown : 1; - bool m_IsInstanceCtor : 1; + bool m_IsForceInline : 1; + bool m_IsForceInlineKnown : 1; + bool m_IsInstanceCtor : 1; bool m_IsFromPromotableValueClass : 1; - bool m_HasSimd : 1; - bool m_LooksLikeWrapperMethod : 1; - bool m_MethodIsMostlyLoadStore : 1; - bool m_CallsiteIsInTryRegion : 1; - bool m_CallsiteIsInLoop : 1; - bool m_IsNoReturn : 1; - bool m_IsNoReturnKnown : 1; - bool m_ConstArgFeedsIsKnownConst : 1; - bool m_ArgFeedsIsKnownConst : 1; - bool m_InsideThrowBlock : 1; + bool m_HasSimd : 1; + bool m_LooksLikeWrapperMethod : 1; + bool m_MethodIsMostlyLoadStore : 1; + bool m_CallsiteIsInTryRegion : 1; + bool m_CallsiteIsInLoop : 1; + bool m_IsNoReturn : 1; + bool m_IsNoReturnKnown : 1; + bool m_ConstArgFeedsIsKnownConst : 1; + bool m_ArgFeedsIsKnownConst : 1; + bool m_InsideThrowBlock : 1; }; // ExtendedDefaultPolicy is a slightly more aggressive variant of @@ -271,11 +272,11 @@ class ExtendedDefaultPolicy : public DefaultPolicy unsigned m_UnrollableMemop; unsigned m_Switch; unsigned m_DivByCns; - bool m_ReturnsStructByValue : 1; - bool m_IsFromValueClass : 1; - bool m_NonGenericCallsGeneric : 1; + bool m_ReturnsStructByValue : 1; + bool m_IsFromValueClass : 1; + bool m_NonGenericCallsGeneric : 1; bool m_IsCallsiteInNoReturnRegion : 1; - bool m_HasProfileWeights : 1; + bool m_HasProfileWeights : 1; }; // DiscretionaryPolicy is a variant of the default policy. It diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 06ebe9ef2e2850..87e8e18c842fc8 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -876,7 +876,7 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(GenTree* op) // broadcast -> LCL_VAR(TYP_(U)INT) ssize_t scalarValue = hwintrinsicChild->AsIntCon()->IconValue(); UNATIVE_OFFSET cnum = emit->emitDataConst(&scalarValue, genTypeSize(simdBaseType), - genTypeSize(simdBaseType), simdBaseType); + genTypeSize(simdBaseType), simdBaseType); return OperandDesc(compiler->eeFindJitDataOffs(cnum)); } else @@ -1124,9 +1124,9 @@ void CodeGen::inst_RV_TT(instruction ins, emitAttr size, regNumber op1Reg, GenTr } /***************************************************************************** -* -* Generate an instruction of the form "op reg1, reg2, icon". -*/ + * + * Generate an instruction of the form "op reg1, reg2, icon". + */ void CodeGen::inst_RV_RV_IV(instruction ins, emitAttr size, regNumber reg1, regNumber reg2, unsigned ival) { @@ -1256,8 +1256,8 @@ void CodeGen::inst_RV_RV_TT(instruction ins, emitter* emit = GetEmitter(); noway_assert(emit->emitVerifyEncodable(ins, EA_SIZE(size), targetReg)); -// TODO-XArch-CQ: Commutative operations can have op1 be contained -// TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained + // TODO-XArch-CQ: Commutative operations can have op1 be contained + // TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained #if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) if (CodeGenInterface::IsEmbeddedBroadcastEnabled(ins, op2)) @@ -2065,7 +2065,7 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) #if defined(TARGET_XARCH) return INS_kmovq_gpr; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing + unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif } diff --git a/src/coreclr/jit/instrsarm.h b/src/coreclr/jit/instrsarm.h index 9356150d4b2e83..3a1c871d316f6a 100644 --- a/src/coreclr/jit/instrsarm.h +++ b/src/coreclr/jit/instrsarm.h @@ -19,7 +19,7 @@ * e8 -- encoding 8 * e9 -- encoding 9 * -******************************************************************************/ + ******************************************************************************/ #if !defined(TARGET_ARM) #error Unexpected target type diff --git a/src/coreclr/jit/instrsarm64.h b/src/coreclr/jit/instrsarm64.h index c07976f1eca0b6..c6ac7404c569d0 100644 --- a/src/coreclr/jit/instrsarm64.h +++ b/src/coreclr/jit/instrsarm64.h @@ -18,7 +18,7 @@ * e8 -- encoding 8 * e9 -- encoding 9 * -******************************************************************************/ + ******************************************************************************/ #if !defined(TARGET_ARM64) #error Unexpected target type diff --git a/src/coreclr/jit/instrsloongarch64.h b/src/coreclr/jit/instrsloongarch64.h index 4f94516c5fb91d..3794d91e02e388 100644 --- a/src/coreclr/jit/instrsloongarch64.h +++ b/src/coreclr/jit/instrsloongarch64.h @@ -11,7 +11,7 @@ * mask -- instruction's mask * fmt -- disasmbly format * -******************************************************************************/ + ******************************************************************************/ #if !defined(TARGET_LOONGARCH64) #error Unexpected target type diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 17443cb9784927..440cc0033c82f9 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -18,7 +18,7 @@ * tt -- the tupletype for the instruction * flags -- flags, see INS_FLAGS_* enum * -******************************************************************************/ + ******************************************************************************/ // clang-format off #if !defined(TARGET_XARCH) diff --git a/src/coreclr/jit/jit.h b/src/coreclr/jit/jit.h index 1df8c034d0c1df..754e6b27f44416 100644 --- a/src/coreclr/jit/jit.h +++ b/src/coreclr/jit/jit.h @@ -26,8 +26,9 @@ #define ZERO 0 #ifdef _MSC_VER -#define CHECK_STRUCT_PADDING 0 // Set this to '1' to enable warning C4820 "'bytes' bytes padding added after - // construct 'member_name'" on interesting structs/classes +#define CHECK_STRUCT_PADDING \ + 0 // Set this to '1' to enable warning C4820 "'bytes' bytes padding added after + // construct 'member_name'" on interesting structs/classes #else #define CHECK_STRUCT_PADDING 0 // Never enable it for non-MSFT compilers #endif @@ -295,9 +296,9 @@ typedef ptrdiff_t ssize_t; #include "corjit.h" #include "jitee.h" -#define __OPERATOR_NEW_INLINE 1 // indicate that I will define these -#define __PLACEMENT_NEW_INLINE // don't bring in the global placement new, it is easy to make a mistake - // with our new(compiler*) pattern. +#define __OPERATOR_NEW_INLINE 1 // indicate that I will define these +#define __PLACEMENT_NEW_INLINE // don't bring in the global placement new, it is easy to make a mistake + // with our new(compiler*) pattern. #include "utilcode.h" // this defines assert as _ASSERTE #include "host.h" // this redefines assert for the JIT to use assertAbort @@ -319,7 +320,7 @@ typedef ptrdiff_t ssize_t; #endif #ifdef DEBUG -#define INDEBUG(x) x +#define INDEBUG(x) x #define DEBUGARG(x) , x #else #define INDEBUG(x) @@ -334,7 +335,7 @@ typedef ptrdiff_t ssize_t; #if defined(UNIX_AMD64_ABI) #define UNIX_AMD64_ABI_ONLY_ARG(x) , x -#define UNIX_AMD64_ABI_ONLY(x) x +#define UNIX_AMD64_ABI_ONLY(x) x #else // !defined(UNIX_AMD64_ABI) #define UNIX_AMD64_ABI_ONLY_ARG(x) #define UNIX_AMD64_ABI_ONLY(x) @@ -342,7 +343,7 @@ typedef ptrdiff_t ssize_t; #if defined(TARGET_LOONGARCH64) #define UNIX_LOONGARCH64_ONLY_ARG(x) , x -#define UNIX_LOONGARCH64_ONLY(x) x +#define UNIX_LOONGARCH64_ONLY(x) x #else // !TARGET_LOONGARCH64 #define UNIX_LOONGARCH64_ONLY_ARG(x) #define UNIX_LOONGARCH64_ONLY(x) @@ -355,16 +356,16 @@ typedef ptrdiff_t ssize_t; #if defined(UNIX_AMD64_ABI) #define UNIX_AMD64_ABI_ONLY_ARG(x) , x -#define UNIX_AMD64_ABI_ONLY(x) x +#define UNIX_AMD64_ABI_ONLY(x) x #else // !defined(UNIX_AMD64_ABI) #define UNIX_AMD64_ABI_ONLY_ARG(x) #define UNIX_AMD64_ABI_ONLY(x) #endif // defined(UNIX_AMD64_ABI) #if defined(UNIX_AMD64_ABI) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) -#define MULTIREG_HAS_SECOND_GC_RET 1 +#define MULTIREG_HAS_SECOND_GC_RET 1 #define MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(x) , x -#define MULTIREG_HAS_SECOND_GC_RET_ONLY(x) x +#define MULTIREG_HAS_SECOND_GC_RET_ONLY(x) x #else // !defined(UNIX_AMD64_ABI) #define MULTIREG_HAS_SECOND_GC_RET 0 #define MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(x) @@ -385,7 +386,7 @@ typedef ptrdiff_t ssize_t; #define DUMMY_INIT(x) (x) #define REGEN_SHORTCUTS 0 -#define REGEN_CALLPAT 0 +#define REGEN_CALLPAT 0 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -473,9 +474,9 @@ class GlobalJitOptions /*****************************************************************************/ -#define CSE_INTO_HANDLERS 0 -#define DUMP_FLOWGRAPHS DEBUG // Support for creating Xml Flowgraph reports in *.fgx files -#define HANDLER_ENTRY_MUST_BE_IN_HOT_SECTION 0 // if 1 we must have all handler entry points in the Hot code section +#define CSE_INTO_HANDLERS 0 +#define DUMP_FLOWGRAPHS DEBUG // Support for creating Xml Flowgraph reports in *.fgx files +#define HANDLER_ENTRY_MUST_BE_IN_HOT_SECTION 0 // if 1 we must have all handler entry points in the Hot code section /*****************************************************************************/ @@ -483,40 +484,43 @@ class GlobalJitOptions /*****************************************************************************/ -#define DUMP_GC_TABLES DEBUG +#define DUMP_GC_TABLES DEBUG #define VERIFY_GC_TABLES 0 -#define REARRANGE_ADDS 1 +#define REARRANGE_ADDS 1 -#define FUNC_INFO_LOGGING 1 // Support dumping function info to a file. In retail, only NYIs, with no function name, - // are dumped. +#define FUNC_INFO_LOGGING \ + 1 // Support dumping function info to a file. In retail, only NYIs, with no function name, + // are dumped. /*****************************************************************************/ /*****************************************************************************/ /* Set these to 1 to collect and output various statistics about the JIT */ -#define CALL_ARG_STATS 0 // Collect stats about calls and call arguments. -#define COUNT_BASIC_BLOCKS 0 // Create a histogram of basic block sizes, and a histogram of IL sizes in the simple - // case of single block methods. -#define COUNT_LOOPS 0 // Collect stats about loops, such as the total number of natural loops, a histogram of +#define CALL_ARG_STATS 0 // Collect stats about calls and call arguments. +#define COUNT_BASIC_BLOCKS \ + 0 // Create a histogram of basic block sizes, and a histogram of IL sizes in the simple + // case of single block methods. +#define COUNT_LOOPS \ + 0 // Collect stats about loops, such as the total number of natural loops, a histogram of // the number of loop exits, etc. -#define DISPLAY_SIZES 0 // Display generated code, data, and GC information sizes. -#define MEASURE_BLOCK_SIZE 0 // Collect stats about basic block and FlowEdge node sizes and memory allocations. -#define MEASURE_FATAL 0 // Count the number of calls to fatal(), including NYIs and noway_asserts. -#define MEASURE_NODE_SIZE 0 // Collect stats about GenTree node allocations. +#define DISPLAY_SIZES 0 // Display generated code, data, and GC information sizes. +#define MEASURE_BLOCK_SIZE 0 // Collect stats about basic block and FlowEdge node sizes and memory allocations. +#define MEASURE_FATAL 0 // Count the number of calls to fatal(), including NYIs and noway_asserts. +#define MEASURE_NODE_SIZE 0 // Collect stats about GenTree node allocations. #define MEASURE_PTRTAB_SIZE 0 // Collect stats about GC pointer table allocations. -#define EMITTER_STATS 0 // Collect stats on the emitter. -#define NODEBASH_STATS 0 // Collect stats on changed gtOper values in GenTree's. -#define COUNT_AST_OPERS 0 // Display use counts for GenTree operators. +#define EMITTER_STATS 0 // Collect stats on the emitter. +#define NODEBASH_STATS 0 // Collect stats on changed gtOper values in GenTree's. +#define COUNT_AST_OPERS 0 // Display use counts for GenTree operators. #ifdef DEBUG #define MEASURE_MEM_ALLOC 1 // Collect memory allocation stats. -#define LOOP_HOIST_STATS 1 // Collect loop hoisting stats. -#define TRACK_LSRA_STATS 1 // Collect LSRA stats +#define LOOP_HOIST_STATS 1 // Collect loop hoisting stats. +#define TRACK_LSRA_STATS 1 // Collect LSRA stats #define TRACK_ENREG_STATS 1 // Collect enregistration stats #else #define MEASURE_MEM_ALLOC 0 // You can set this to 1 to get memory stats in retail, as well -#define LOOP_HOIST_STATS 0 // You can set this to 1 to get loop hoist stats in retail, as well -#define TRACK_LSRA_STATS 0 // You can set this to 1 to get LSRA stats in retail, as well +#define LOOP_HOIST_STATS 0 // You can set this to 1 to get loop hoist stats in retail, as well +#define TRACK_LSRA_STATS 0 // You can set this to 1 to get LSRA stats in retail, as well #define TRACK_ENREG_STATS 0 #endif @@ -602,7 +606,7 @@ const bool dspGCtbls = true; JitTls::GetCompiler()->fgTableDispBasicBlock(b); #define VERBOSE JitTls::GetCompiler()->verbose // Development-time only macros, simplify guards for specified IL methods one wants to debug/add log messages for -#define ISMETHOD(name) (strcmp(JitTls::GetCompiler()->impInlineRoot()->info.compMethodName, name) == 0) +#define ISMETHOD(name) (strcmp(JitTls::GetCompiler()->impInlineRoot()->info.compMethodName, name) == 0) #define ISMETHODHASH(hash) (JitTls::GetCompiler()->impInlineRoot()->info.compMethodHash() == hash) #else // !DEBUG #define JITDUMP(...) @@ -628,8 +632,9 @@ const bool dspGCtbls = true; */ #ifdef TARGET_X86 -#define DOUBLE_ALIGN 1 // permit the double alignment of ESP in prolog, - // and permit the double alignment of local offsets +#define DOUBLE_ALIGN \ + 1 // permit the double alignment of ESP in prolog, + // and permit the double alignment of local offsets #else #define DOUBLE_ALIGN 0 // no special handling for double alignment #endif @@ -673,7 +678,7 @@ inline bool IsUninitialized(T data); #define MISALIGNED_RD_U2(src) (*castto(src, unsigned short*)) #define MISALIGNED_WR_I2(dst, val) *castto(dst, short*) = val; -#define MISALIGNED_WR_I4(dst, val) *castto(dst, int*) = val; +#define MISALIGNED_WR_I4(dst, val) *castto(dst, int*) = val; #define MISALIGNED_WR_ST(dst, val) *castto(dst, ssize_t*) = val; @@ -740,16 +745,16 @@ inline size_t unsigned_abs(__int64 x) #define FEATURE_TAILCALL_OPT_SHARED_RETURN 0 #endif // !FEATURE_TAILCALL_OPT -#define CLFLG_CODESIZE 0x00001 -#define CLFLG_CODESPEED 0x00002 -#define CLFLG_CSE 0x00004 -#define CLFLG_REGVAR 0x00008 -#define CLFLG_RNGCHKOPT 0x00010 -#define CLFLG_DEADSTORE 0x00020 +#define CLFLG_CODESIZE 0x00001 +#define CLFLG_CODESPEED 0x00002 +#define CLFLG_CSE 0x00004 +#define CLFLG_REGVAR 0x00008 +#define CLFLG_RNGCHKOPT 0x00010 +#define CLFLG_DEADSTORE 0x00020 #define CLFLG_CODEMOTION 0x00040 -#define CLFLG_QMARK 0x00080 -#define CLFLG_TREETRANS 0x00100 -#define CLFLG_INLINING 0x00200 +#define CLFLG_QMARK 0x00080 +#define CLFLG_TREETRANS 0x00100 +#define CLFLG_INLINING 0x00200 #if FEATURE_STRUCTPROMOTE #define CLFLG_STRUCTPROMOTE 0x00400 @@ -813,7 +818,7 @@ class JitTls #endif static Compiler* GetCompiler(); - static void SetCompiler(Compiler* compiler); + static void SetCompiler(Compiler* compiler); }; #if defined(DEBUG) diff --git a/src/coreclr/jit/jitconfig.cpp b/src/coreclr/jit/jitconfig.cpp index 3c85031cee6cda..19730be75c2cdc 100644 --- a/src/coreclr/jit/jitconfig.cpp +++ b/src/coreclr/jit/jitconfig.cpp @@ -193,7 +193,7 @@ void JitConfigValues::initialize(ICorJitHost* host) assert(!m_isInitialized); #define CONFIG_INTEGER(name, key, defaultValue) m_##name = host->getIntConfigValue(key, defaultValue); -#define CONFIG_STRING(name, key) m_##name = host->getStringConfigValue(key); +#define CONFIG_STRING(name, key) m_##name = host->getStringConfigValue(key); #define CONFIG_METHODSET(name, key) \ const WCHAR* name##value = host->getStringConfigValue(key); \ m_##name.initialize(name##value, host); \ @@ -212,7 +212,7 @@ void JitConfigValues::destroy(ICorJitHost* host) } #define CONFIG_INTEGER(name, key, defaultValue) -#define CONFIG_STRING(name, key) host->freeStringConfigValue(m_##name); +#define CONFIG_STRING(name, key) host->freeStringConfigValue(m_##name); #define CONFIG_METHODSET(name, key) m_##name.destroy(host); #include "jitconfigvalues.h" diff --git a/src/coreclr/jit/jitconfig.h b/src/coreclr/jit/jitconfig.h index e19021cd52f22b..bd1c552f59438a 100644 --- a/src/coreclr/jit/jitconfig.h +++ b/src/coreclr/jit/jitconfig.h @@ -31,7 +31,7 @@ class JitConfigValues char* m_list; MethodName* m_names; - MethodSet(const MethodSet& other) = delete; + MethodSet(const MethodSet& other) = delete; MethodSet& operator=(const MethodSet& other) = delete; public: @@ -56,8 +56,8 @@ class JitConfigValues private: #define CONFIG_INTEGER(name, key, defaultValue) int m_##name; -#define CONFIG_STRING(name, key) const WCHAR* m_##name; -#define CONFIG_METHODSET(name, key) MethodSet m_##name; +#define CONFIG_STRING(name, key) const WCHAR* m_##name; +#define CONFIG_METHODSET(name, key) MethodSet m_##name; #include "jitconfigvalues.h" public: @@ -81,7 +81,7 @@ class JitConfigValues private: bool m_isInitialized; - JitConfigValues(const JitConfigValues& other) = delete; + JitConfigValues(const JitConfigValues& other) = delete; JitConfigValues& operator=(const JitConfigValues& other) = delete; public: diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index e8bd8737699aa6..3aed33af413606 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -387,10 +387,10 @@ CONFIG_INTEGER(JitDisableSimdVN, W("JitDisableSimdVN"), 0) // Default 0, ValueNu // CONFIG_INTEGER(JitConstCSE, W("JitConstCSE"), 0) -#define CONST_CSE_ENABLE_ARM 0 -#define CONST_CSE_DISABLE_ALL 1 +#define CONST_CSE_ENABLE_ARM 0 +#define CONST_CSE_DISABLE_ALL 1 #define CONST_CSE_ENABLE_ARM_NO_SHARING 2 -#define CONST_CSE_ENABLE_ALL 3 +#define CONST_CSE_ENABLE_ALL 3 #define CONST_CSE_ENABLE_ALL_NO_SHARING 4 // If nonzero, use the greedy RL policy. diff --git a/src/coreclr/jit/jitee.h b/src/coreclr/jit/jitee.h index 27963ac356efb5..71f53b4e10d7d6 100644 --- a/src/coreclr/jit/jitee.h +++ b/src/coreclr/jit/jitee.h @@ -54,13 +54,15 @@ class JitFlags }; // clang-format on - JitFlags() : m_jitFlags(0) + JitFlags() + : m_jitFlags(0) { // empty } // Convenience constructor to set exactly one flags. - JitFlags(JitFlag flag) : m_jitFlags(0) + JitFlags(JitFlag flag) + : m_jitFlags(0) { Set(flag); } diff --git a/src/coreclr/jit/jiteh.cpp b/src/coreclr/jit/jiteh.cpp index 75749e50fbd3bc..fc12d55c35a468 100644 --- a/src/coreclr/jit/jiteh.cpp +++ b/src/coreclr/jit/jiteh.cpp @@ -1718,7 +1718,7 @@ void Compiler::fgSortEHTable() (hndBegOff >= xtab1->ebdHndBegOffset && hndEndOff <= xtab1->ebdHndEndOffset) || (xtab1->HasFilter() && (hndBegOff >= xtab1->ebdFilterBegOffset && hndEndOff <= xtab1->ebdHndBegOffset)) // Note that end of filter is beginning of handler - ) + ) { #ifdef DEBUG if (verbose) @@ -2082,7 +2082,7 @@ bool Compiler::fgNormalizeEHCase2() if (ehOuter->ebdIsSameTry(mutualTryBeg, mutualTryLast)) { -// clang-format off + // clang-format off // Don't touch mutually-protect regions: their 'try' regions must remain identical! // We want to continue the looping outwards, in case we have something like this: // @@ -2131,7 +2131,7 @@ bool Compiler::fgNormalizeEHCase2() // // In this case, all the 'try' start at the same block! Note that there are two sets of mutually-protect regions, // separated by some nesting. -// clang-format on + // clang-format on #ifdef DEBUG if (verbose) @@ -2361,7 +2361,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() { GenTree* ctxTree = getRuntimeContextTree(embedInfo.lookup.lookupKind.runtimeLookupKind); runtimeLookup = impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_GENERIC_HANDLE, - TYP_I_IMPL, &embedInfo.lookup.lookupKind, ctxTree); + TYP_I_IMPL, &embedInfo.lookup.lookupKind, ctxTree); } else { @@ -3026,8 +3026,8 @@ void Compiler::fgVerifyHandlerTab() assert(blockNumMap[block->bbNum] == 0); // If this fails, we have two blocks with the same block number. blockNumMap[block->bbNum] = newBBnum++; } -// Note that there may be some blockNumMap[x] == 0, for a block number 'x' that has been deleted, if the blocks -// haven't been renumbered since the deletion. + // Note that there may be some blockNumMap[x] == 0, for a block number 'x' that has been deleted, if the blocks + // haven't been renumbered since the deletion. #if 0 // Useful for debugging, but don't want to put this in the dump all the time if (verbose) @@ -3274,9 +3274,9 @@ void Compiler::fgVerifyHandlerTab() assert(bbNumOuterHndLast != 0); assert(bbNumOuterHndBeg <= bbNumOuterHndLast); -// The outer handler must completely contain all the blocks in the EH region nested within it. However, if -// funclets have been created, it's harder to make any relationship asserts about the order of nested -// handlers, which also have been made into funclets. + // The outer handler must completely contain all the blocks in the EH region nested within it. However, if + // funclets have been created, it's harder to make any relationship asserts about the order of nested + // handlers, which also have been made into funclets. #if defined(FEATURE_EH_FUNCLETS) if (fgFuncletsCreated) @@ -4339,7 +4339,7 @@ void Compiler::fgExtendEHRegionBefore(BasicBlock* block) bFilterLast->bbNum, bPrev->bbNum); } #endif // DEBUG - // Change the target for bFilterLast from the old first 'block' to the new first 'bPrev' + // Change the target for bFilterLast from the old first 'block' to the new first 'bPrev' fgRedirectTargetEdge(bFilterLast, bPrev); } } diff --git a/src/coreclr/jit/jiteh.h b/src/coreclr/jit/jiteh.h index 95ae62527897ba..55b56ac9833c42 100644 --- a/src/coreclr/jit/jiteh.h +++ b/src/coreclr/jit/jiteh.h @@ -83,7 +83,8 @@ struct EHblkDsc BasicBlock* ebdTryLast; // Last block of the try BasicBlock* ebdHndBeg; // First block of the handler BasicBlock* ebdHndLast; // Last block of the handler - union { + union + { BasicBlock* ebdFilter; // First block of filter, if HasFilter() unsigned ebdTyp; // Exception type (a class token), otherwise }; @@ -165,8 +166,8 @@ struct EHblkDsc unsigned ebdGetEnclosingRegionIndex(bool* inTryRegion); static bool ebdIsSameTry(EHblkDsc* h1, EHblkDsc* h2); // Same 'try' region? Compare begin/last blocks. - bool ebdIsSameTry(Compiler* comp, unsigned t2); - bool ebdIsSameTry(BasicBlock* ebdTryBeg, BasicBlock* ebdTryLast); + bool ebdIsSameTry(Compiler* comp, unsigned t2); + bool ebdIsSameTry(BasicBlock* ebdTryBeg, BasicBlock* ebdTryLast); #ifdef DEBUG void DispEntry(unsigned num); // Display this table entry diff --git a/src/coreclr/jit/jitexpandarray.h b/src/coreclr/jit/jitexpandarray.h index 646f9e6747a3be..8eaf52705986ad 100644 --- a/src/coreclr/jit/jitexpandarray.h +++ b/src/coreclr/jit/jitexpandarray.h @@ -54,7 +54,10 @@ class JitExpandArray // of size max(`minSize`, `idx`) is allocated. // JitExpandArray(CompAllocator alloc, unsigned minSize = 1) - : m_alloc(alloc), m_members(nullptr), m_size(0), m_minSize(minSize) + : m_alloc(alloc) + , m_members(nullptr) + , m_size(0) + , m_minSize(minSize) { assert(minSize > 0); } @@ -219,7 +222,9 @@ class JitExpandArrayStack : public JitExpandArray // Notes: // See JitExpandArray constructor notes. // - JitExpandArrayStack(CompAllocator alloc, unsigned minSize = 1) : JitExpandArray(alloc, minSize), m_used(0) + JitExpandArrayStack(CompAllocator alloc, unsigned minSize = 1) + : JitExpandArray(alloc, minSize) + , m_used(0) { } diff --git a/src/coreclr/jit/jitgcinfo.h b/src/coreclr/jit/jitgcinfo.h index ed4572f3776a82..c783a2de3cc4b7 100644 --- a/src/coreclr/jit/jitgcinfo.h +++ b/src/coreclr/jit/jitgcinfo.h @@ -27,7 +27,9 @@ struct RegSlotIdKey { } - RegSlotIdKey(unsigned short regNum, unsigned flags) : m_regNum(regNum), m_flags((unsigned short)flags) + RegSlotIdKey(unsigned short regNum, unsigned flags) + : m_regNum(regNum) + , m_flags((unsigned short)flags) { assert(m_flags == flags); } @@ -54,7 +56,9 @@ struct StackSlotIdKey } StackSlotIdKey(int offset, bool fpRel, unsigned flags) - : m_offset(offset), m_fpRel(fpRel), m_flags((unsigned short)flags) + : m_offset(offset) + , m_fpRel(fpRel) + , m_flags((unsigned short)flags) { assert(flags == m_flags); } @@ -167,7 +171,7 @@ class GCInfo unsigned char rpdCallInstrSize; // Length of the call instruction. #endif - unsigned short rpdArg : 1; // is this an argument descriptor? + unsigned short rpdArg : 1; // is this an argument descriptor? unsigned short rpdArgType : 2; // is this an argument push,pop, or kill? rpdArgType_t rpdArgTypeGet() { @@ -181,8 +185,8 @@ class GCInfo } unsigned short rpdIsThis : 1; // is it the 'this' pointer - unsigned short rpdCall : 1; // is this a true call site? - unsigned short : 1; // Padding bit, so next two start on a byte boundary + unsigned short rpdCall : 1; // is this a true call site? + unsigned short : 1; // Padding bit, so next two start on a byte boundary unsigned short rpdCallGCrefRegs : CNT_CALLEE_SAVED; // Callee-saved registers containing GC pointers. unsigned short rpdCallByrefRegs : CNT_CALLEE_SAVED; // Callee-saved registers containing byrefs. @@ -263,7 +267,8 @@ class GCInfo unsigned short cdArgCnt; - union { + union + { struct // used if cdArgCnt == 0 { unsigned cdArgMask; // ptr arg bitfield @@ -280,7 +285,7 @@ class GCInfo CallDsc* gcCallDescList; CallDsc* gcCallDescLast; -//------------------------------------------------------------------------- + //------------------------------------------------------------------------- #ifdef JIT32_GCENCODER void gcCountForHeader(UNALIGNED unsigned int* pUntrackedCount, UNALIGNED unsigned int* pVarPtrTableSize); @@ -305,7 +310,7 @@ class GCInfo #ifdef JIT32_GCENCODER size_t gcPtrTableSize(const InfoHdr& header, unsigned codeSize, size_t* pArgTabOffset); - BYTE* gcPtrTableSave(BYTE* destPtr, const InfoHdr& header, unsigned codeSize, size_t* pArgTabOffset); + BYTE* gcPtrTableSave(BYTE* destPtr, const InfoHdr& header, unsigned codeSize, size_t* pArgTabOffset); #endif void gcRegPtrSetInit(); /*****************************************************************************/ @@ -384,7 +389,7 @@ class GCInfo #ifdef JIT32_GCENCODER size_t gcInfoBlockHdrDump(const BYTE* table, - InfoHdr* header, /* OUT */ + InfoHdr* header, /* OUT */ unsigned* methodSize); /* OUT */ size_t gcDumpPtrTable(const BYTE* table, const InfoHdr& header, unsigned methodSize); diff --git a/src/coreclr/jit/jithashtable.h b/src/coreclr/jit/jithashtable.h index 9ad73dbf2f7d51..f699c3eee19d24 100644 --- a/src/coreclr/jit/jithashtable.h +++ b/src/coreclr/jit/jithashtable.h @@ -57,10 +57,16 @@ class JitHashTableBehavior class JitPrimeInfo { public: - constexpr JitPrimeInfo() : prime(0), magic(0), shift(0) + constexpr JitPrimeInfo() + : prime(0) + , magic(0) + , shift(0) { } - constexpr JitPrimeInfo(unsigned p, unsigned m, unsigned s) : prime(p), magic(m), shift(s) + constexpr JitPrimeInfo(unsigned p, unsigned m, unsigned s) + : prime(p) + , magic(m) + , shift(s) { } unsigned prime; @@ -130,7 +136,10 @@ class JitHashTable Value m_val; template - Node(Node* next, Key k, Args&&... args) : m_next(next), m_key(k), m_val(std::forward(args)...) + Node(Node* next, Key k, Args&&... args) + : m_next(next) + , m_key(k) + , m_val(std::forward(args)...) { } @@ -166,7 +175,12 @@ class JitHashTable // JitHashTable always starts out empty, with no allocation overhead. // Call Reallocate to prime with an initial size if desired. // - JitHashTable(Allocator alloc) : m_alloc(alloc), m_table(nullptr), m_tableSizeInfo(), m_tableCount(0), m_tableMax(0) + JitHashTable(Allocator alloc) + : m_alloc(alloc) + , m_table(nullptr) + , m_tableSizeInfo() + , m_tableCount(0) + , m_tableMax(0) { #ifndef __GNUC__ // these crash GCC static_assert_no_msg(Behavior::s_growth_factor_numerator > Behavior::s_growth_factor_denominator); @@ -492,7 +506,8 @@ class JitHashTable class KeyIterator : public NodeIterator { public: - KeyIterator(const JitHashTable* hash, bool begin) : NodeIterator(hash, begin) + KeyIterator(const JitHashTable* hash, bool begin) + : NodeIterator(hash, begin) { } @@ -506,7 +521,8 @@ class JitHashTable class ValueIterator : public NodeIterator { public: - ValueIterator(const JitHashTable* hash, bool begin) : NodeIterator(hash, begin) + ValueIterator(const JitHashTable* hash, bool begin) + : NodeIterator(hash, begin) { } @@ -521,7 +537,8 @@ class JitHashTable class KeyValueIterator : public NodeIterator { public: - KeyValueIterator(const JitHashTable* hash, bool begin) : NodeIterator(hash, begin) + KeyValueIterator(const JitHashTable* hash, bool begin) + : NodeIterator(hash, begin) { } @@ -538,7 +555,8 @@ class JitHashTable const JitHashTable* const m_hash; public: - KeyIteration(const JitHashTable* hash) : m_hash(hash) + KeyIteration(const JitHashTable* hash) + : m_hash(hash) { } @@ -559,7 +577,8 @@ class JitHashTable const JitHashTable* const m_hash; public: - ValueIteration(const JitHashTable* hash) : m_hash(hash) + ValueIteration(const JitHashTable* hash) + : m_hash(hash) { } @@ -580,7 +599,8 @@ class JitHashTable const JitHashTable* const m_hash; public: - KeyValueIteration(const JitHashTable* hash) : m_hash(hash) + KeyValueIteration(const JitHashTable* hash) + : m_hash(hash) { } diff --git a/src/coreclr/jit/layout.cpp b/src/coreclr/jit/layout.cpp index 918fd4ab6521d4..ad4c0077c22bd3 100644 --- a/src/coreclr/jit/layout.cpp +++ b/src/coreclr/jit/layout.cpp @@ -21,7 +21,8 @@ class ClassLayoutTable typedef JitHashTable, unsigned> BlkLayoutIndexMap; typedef JitHashTable, unsigned> ObjLayoutIndexMap; - union { + union + { // Up to 3 layouts can be stored "inline" and finding a layout by handle/size can be done using linear search. // Most methods need no more than 2 layouts. ClassLayout* m_layoutArray[3]; @@ -43,7 +44,10 @@ class ClassLayoutTable ClassLayout m_zeroSizedBlockLayout; public: - ClassLayoutTable() : m_layoutCount(0), m_layoutLargeCapacity(0), m_zeroSizedBlockLayout(0) + ClassLayoutTable() + : m_layoutCount(0) + , m_layoutLargeCapacity(0) + , m_zeroSizedBlockLayout(0) { } diff --git a/src/coreclr/jit/layout.h b/src/coreclr/jit/layout.h index 59ecaa9405485d..3c6487e516b91c 100644 --- a/src/coreclr/jit/layout.h +++ b/src/coreclr/jit/layout.h @@ -30,7 +30,8 @@ class ClassLayout // Array of CorInfoGCType (as BYTE) that describes the GC layout of the class. // For small classes the array is stored inline, avoiding an extra allocation // and the pointer size overhead. - union { + union + { BYTE* m_gcPtrs; BYTE m_gcPtrsArray[sizeof(BYTE*)]; }; @@ -69,7 +70,7 @@ class ClassLayout ClassLayout(CORINFO_CLASS_HANDLE classHandle, bool isValueClass, unsigned size, - var_types type DEBUGARG(const char* className) DEBUGARG(const char* shortClassName)) + var_types type DEBUGARG(const char* className) DEBUGARG(const char* shortClassName)) : m_classHandle(classHandle) , m_size(size) , m_isValueClass(isValueClass) diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index 44b0afe1caf927..6b4c6cc693f9b2 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -14,7 +14,9 @@ class LocalSequencer final : public GenTreeVisitor UseExecutionOrder = true, }; - LocalSequencer(Compiler* comp) : GenTreeVisitor(comp), m_prevNode(nullptr) + LocalSequencer(Compiler* comp) + : GenTreeVisitor(comp) + , m_prevNode(nullptr) { } @@ -918,9 +920,9 @@ class LocalAddressVisitor final : public GenTreeVisitor break; #ifdef FEATURE_HW_INTRINSICS - // We have two cases we want to handle: - // 1. Vector2/3/4 and Quaternion where we have 4x float fields - // 2. Plane where we have 1x Vector3 and 1x float field + // We have two cases we want to handle: + // 1. Vector2/3/4 and Quaternion where we have 4x float fields + // 2. Plane where we have 1x Vector3 and 1x float field case IndirTransform::GetElement: { @@ -934,7 +936,7 @@ class LocalAddressVisitor final : public GenTreeVisitor { GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType)); hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode, - CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); + CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); break; } case TYP_SIMD12: diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index aa65e789d92cc7..f69fb6f8549060 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -332,9 +332,9 @@ void Compiler::lvaInitTypeRef() } if ( // If there already exist unsafe buffers, don't mark more structs as unsafe - // as that will cause them to be placed along with the real unsafe buffers, - // unnecessarily exposing them to overruns. This can affect GS tests which - // intentionally do buffer-overruns. + // as that will cause them to be placed along with the real unsafe buffers, + // unnecessarily exposing them to overruns. This can affect GS tests which + // intentionally do buffer-overruns. !getNeedsGSSecurityCookie() && // GS checks require the stack to be re-ordered, which can't be done with EnC !opts.compDbgEnC && compStressCompile(STRESS_UNSAFE_BUFFER_CHECKS, 25)) @@ -440,7 +440,7 @@ void Compiler::lvaInitArgs(InitVarDscInfo* varDscInfo) lvaInitRetBuffArg(varDscInfo, true); } -//====================================================================== + //====================================================================== #if USER_ARGS_COME_LAST //@GENERICS: final instantiation-info argument for shared generic methods @@ -611,9 +611,9 @@ void Compiler::lvaInitRetBuffArg(InitVarDscInfo* varDscInfo, bool useFixedRetBuf // void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, unsigned takeArgs) { -//------------------------------------------------------------------------- -// Walk the function signature for the explicit arguments -//------------------------------------------------------------------------- + //------------------------------------------------------------------------- + // Walk the function signature for the explicit arguments + //------------------------------------------------------------------------- #if defined(TARGET_X86) // Only (some of) the implicit args are enregistered for varargs @@ -1412,8 +1412,8 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un #if defined(TARGET_X86) varDsc->SetStackOffset(compArgSize); #else // !TARGET_X86 - // TODO-CQ: We shouldn't have to go as far as to declare these - // address-exposed -- DoNotEnregister should suffice. + // TODO-CQ: We shouldn't have to go as far as to declare these + // address-exposed -- DoNotEnregister should suffice. lvaSetVarAddrExposed(varDscInfo->varNum DEBUGARG(AddressExposedReason::TOO_CONSERVATIVE)); #endif // !TARGET_X86 @@ -1886,7 +1886,9 @@ void Compiler::lvSetMinOptsDoNotEnreg() // Arguments: // compiler - pointer to a compiler to get access to an allocator, compHandle etc. // -Compiler::StructPromotionHelper::StructPromotionHelper(Compiler* compiler) : compiler(compiler), structPromotionInfo() +Compiler::StructPromotionHelper::StructPromotionHelper(Compiler* compiler) + : compiler(compiler) + , structPromotionInfo() { } @@ -2465,12 +2467,12 @@ bool Compiler::StructPromotionHelper::ShouldPromoteStructVar(unsigned lclNum) // with something else occupying the same 4-byte slot, it will // overwrite other fields. if (structPromotionInfo.fieldCnt != 1) - { - JITDUMP("Not promoting promotable struct local V%02u, because lvIsParam is true and #fields = " - "%d.\n", - lclNum, structPromotionInfo.fieldCnt); - shouldPromote = false; - } + { + JITDUMP("Not promoting promotable struct local V%02u, because lvIsParam is true and #fields = " + "%d.\n", + lclNum, structPromotionInfo.fieldCnt); + shouldPromote = false; + } } else if ((lclNum == compiler->genReturnLocal) && (structPromotionInfo.fieldCnt > 1)) { @@ -2509,8 +2511,8 @@ void Compiler::StructPromotionHelper::SortStructFields() { jitstd::sort(structPromotionInfo.fields, structPromotionInfo.fields + structPromotionInfo.fieldCnt, [](const lvaStructFieldInfo& lhs, const lvaStructFieldInfo& rhs) { - return lhs.fldOffset < rhs.fldOffset; - }); + return lhs.fldOffset < rhs.fldOffset; + }); structPromotionInfo.fieldsSorted = true; } } @@ -2565,7 +2567,7 @@ void Compiler::StructPromotionHelper::PromoteStructVar(unsigned lclNum) compiler->compFloatingPointUsed = true; } -// Now grab the temp for the field local. + // Now grab the temp for the field local. #ifdef DEBUG char fieldNameBuffer[128]; @@ -3769,8 +3771,8 @@ void Compiler::lvaSortByRefCount() if (varDsc->IsAddressExposed()) { varDsc->lvTracked = 0; - assert(varDsc->lvType != TYP_STRUCT || - varDsc->lvDoNotEnregister); // For structs, should have set this when we set m_addrExposed. + assert(varDsc->lvType != TYP_STRUCT || varDsc->lvDoNotEnregister); // For structs, should have set this when + // we set m_addrExposed. } if (varTypeIsStruct(varDsc)) { @@ -4002,8 +4004,8 @@ unsigned LclVarDsc::lvSize() const // Size needed for storage representation. On } /********************************************************************************** -* Get stack size of the varDsc. -*/ + * Get stack size of the varDsc. + */ size_t LclVarDsc::lvArgStackSize() const { // Make sure this will have a stack size @@ -4423,7 +4425,10 @@ void Compiler::lvaMarkLocalVars(BasicBlock* block, bool isRecompute) }; MarkLocalVarsVisitor(Compiler* compiler, BasicBlock* block, Statement* stmt, bool isRecompute) - : GenTreeVisitor(compiler), m_block(block), m_stmt(stmt), m_isRecompute(isRecompute) + : GenTreeVisitor(compiler) + , m_block(block) + , m_stmt(stmt) + , m_isRecompute(isRecompute) { } @@ -4848,11 +4853,11 @@ inline void Compiler::lvaIncrementFrameSize(unsigned size) } /**************************************************************************** -* -* Return true if absolute offsets of temps are larger than vars, or in other -* words, did we allocate temps before of after vars. The /GS buffer overrun -* checks want temps to be at low stack addresses than buffers -*/ + * + * Return true if absolute offsets of temps are larger than vars, or in other + * words, did we allocate temps before of after vars. The /GS buffer overrun + * checks want temps to be at low stack addresses than buffers + */ bool Compiler::lvaTempsHaveLargerOffsetThanVars() { #ifdef TARGET_ARM @@ -4871,10 +4876,10 @@ bool Compiler::lvaTempsHaveLargerOffsetThanVars() } /**************************************************************************** -* -* Return an upper bound estimate for the size of the compiler spill temps -* -*/ + * + * Return an upper bound estimate for the size of the compiler spill temps + * + */ unsigned Compiler::lvaGetMaxSpillTempSize() { unsigned result = 0; @@ -5491,7 +5496,7 @@ void Compiler::lvaFixVirtualFrameOffsets() #endif ) #endif // !defined(TARGET_AMD64) - ) + ) { doAssignStkOffs = false; // Not on frame or an incoming stack arg } @@ -5512,8 +5517,8 @@ void Compiler::lvaFixVirtualFrameOffsets() // We need to re-adjust the offsets of the parameters so they are EBP // relative rather than stack/frame pointer relative - varDsc->SetStackOffset(varDsc->GetStackOffset() + - (2 * TARGET_POINTER_SIZE)); // return address and pushed EBP + varDsc->SetStackOffset(varDsc->GetStackOffset() + (2 * TARGET_POINTER_SIZE)); // return address and + // pushed EBP noway_assert(varDsc->GetStackOffset() >= FIRST_ARG_STACK_OFFS); } @@ -5692,7 +5697,7 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() argOffs = lvaAssignVirtualFrameOffsetToArg(lclNum, REGSIZE_BYTES, argOffs); } #elif !defined(UNIX_AMD64_ABI) - argOffs = lvaAssignVirtualFrameOffsetToArg(lclNum, REGSIZE_BYTES, argOffs); + argOffs = lvaAssignVirtualFrameOffsetToArg(lclNum, REGSIZE_BYTES, argOffs); #endif // TARGET_X86 lclNum++; userArgsToSkip++; @@ -5853,8 +5858,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() // ret address slot, stack frame padding, alloca instructions, etc. // Note: This is the implementation for UNIX_AMD64 System V platforms. // -int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, - unsigned argSize, +int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, + unsigned argSize, int argOffs UNIX_AMD64_ABI_ONLY_ARG(int* callerArgOffset)) { noway_assert(lclNum < info.compArgsCount); @@ -5945,8 +5950,8 @@ int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, // The final offset is calculated in lvaFixVirtualFrameOffsets method. It accounts for FP existence, // ret address slot, stack frame padding, alloca instructions, etc. // Note: This implementation for all the platforms but UNIX_AMD64 OSs (System V 64 bit.) -int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, - unsigned argSize, +int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, + unsigned argSize, int argOffs UNIX_AMD64_ABI_ONLY_ARG(int* callerArgOffset)) { noway_assert(lclNum < info.compArgsCount); @@ -6174,8 +6179,8 @@ int Compiler::lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, (codeGen->regSet.rsMaskPreSpillAlign & genRegMask(REG_ARG_LAST)); noway_assert(cond); - noway_assert(sizeofPreSpillRegArgs <= - argOffs + TARGET_POINTER_SIZE); // at most one register of alignment + noway_assert(sizeofPreSpillRegArgs <= argOffs + TARGET_POINTER_SIZE); // at most one register of + // alignment } argOffs = sizeofPreSpillRegArgs; } @@ -6346,8 +6351,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToLocals() stkOffs -= initialStkOffs; } - if (codeGen->IsSaveFpLrWithAllCalleeSavedRegisters() || - !isFramePointerUsed()) // Note that currently we always have a frame pointer + if (codeGen->IsSaveFpLrWithAllCalleeSavedRegisters() || !isFramePointerUsed()) // Note that currently we always have + // a frame pointer { stkOffs -= compCalleeRegsPushed * REGSIZE_BYTES; } @@ -7087,8 +7092,8 @@ void Compiler::lvaAssignVirtualFrameOffsetsToLocals() #endif // FEATURE_EH_FUNCLETS && defined(TARGET_AMD64) #ifdef TARGET_ARM64 - if (!codeGen->IsSaveFpLrWithAllCalleeSavedRegisters() && - isFramePointerUsed()) // Note that currently we always have a frame pointer + if (!codeGen->IsSaveFpLrWithAllCalleeSavedRegisters() && isFramePointerUsed()) // Note that currently we always have + // a frame pointer { // Create space for saving FP and LR. stkOffs -= 2 * REGSIZE_BYTES; @@ -7373,9 +7378,9 @@ void Compiler::lvaAlignFrame() } // Align the stack with STACK_ALIGN value. - int adjustFrameSize = compLclFrameSize; + int adjustFrameSize = compLclFrameSize; #if defined(UNIX_X86_ABI) - bool isEbpPushed = codeGen->isFramePointerUsed(); + bool isEbpPushed = codeGen->isFramePointerUsed(); #if DOUBLE_ALIGN isEbpPushed |= genDoubleAlign(); #endif @@ -7853,9 +7858,9 @@ void Compiler::lvaDumpEntry(unsigned lclNum, FrameLayoutState curState, size_t r } /***************************************************************************** -* -* dump the lvaTable -*/ + * + * dump the lvaTable + */ void Compiler::lvaTableDump(FrameLayoutState curState) { diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index fa0839725c9fb7..e181a2e9a135ab 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -255,8 +255,8 @@ static unsigned getLikelyClassesOrMethods(LikelyClassMethodRecord* jitstd::sort(sortedEntries, sortedEntries + knownHandles, [](const LikelyClassMethodHistogramEntry& h1, const LikelyClassMethodHistogramEntry& h2) -> bool { - return h1.m_count > h2.m_count; - }); + return h1.m_count > h2.m_count; + }); const UINT32 numberOfClasses = min(knownHandles, maxLikelyClasses); @@ -410,7 +410,9 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyValues(LikelyValueRecord* // sort by m_count (descending) jitstd::sort(sortedEntries, sortedEntries + h.countHistogramElements, [](const LikelyClassMethodHistogramEntry& h1, - const LikelyClassMethodHistogramEntry& h2) -> bool { return h1.m_count > h2.m_count; }); + const LikelyClassMethodHistogramEntry& h2) -> bool { + return h1.m_count > h2.m_count; + }); const UINT32 numberOfLikelyConst = min(h.countHistogramElements, maxLikelyValues); diff --git a/src/coreclr/jit/lir.cpp b/src/coreclr/jit/lir.cpp index b10bd98ff6221c..d172cea22d369a 100644 --- a/src/coreclr/jit/lir.cpp +++ b/src/coreclr/jit/lir.cpp @@ -9,7 +9,10 @@ #pragma hdrstop #endif -LIR::Use::Use() : m_range(nullptr), m_edge(nullptr), m_user(nullptr) +LIR::Use::Use() + : m_range(nullptr) + , m_edge(nullptr) + , m_user(nullptr) { } @@ -30,7 +33,10 @@ LIR::Use::Use(const Use& other) // // Return Value: // -LIR::Use::Use(Range& range, GenTree** edge, GenTree* user) : m_range(&range), m_edge(edge), m_user(user) +LIR::Use::Use(Range& range, GenTree** edge, GenTree* user) + : m_range(&range) + , m_edge(edge) + , m_user(user) { AssertIsValid(); } @@ -280,11 +286,15 @@ unsigned LIR::Use::ReplaceWithLclVar(Compiler* compiler, unsigned lclNum, GenTre return lclNum; } -LIR::ReadOnlyRange::ReadOnlyRange() : m_firstNode(nullptr), m_lastNode(nullptr) +LIR::ReadOnlyRange::ReadOnlyRange() + : m_firstNode(nullptr) + , m_lastNode(nullptr) { } -LIR::ReadOnlyRange::ReadOnlyRange(ReadOnlyRange&& other) : m_firstNode(other.m_firstNode), m_lastNode(other.m_lastNode) +LIR::ReadOnlyRange::ReadOnlyRange(ReadOnlyRange&& other) + : m_firstNode(other.m_firstNode) + , m_lastNode(other.m_lastNode) { #ifdef DEBUG other.m_firstNode = nullptr; @@ -301,7 +311,9 @@ LIR::ReadOnlyRange::ReadOnlyRange(ReadOnlyRange&& other) : m_firstNode(other.m_f // firstNode - The first node in the range. // lastNode - The last node in the range. // -LIR::ReadOnlyRange::ReadOnlyRange(GenTree* firstNode, GenTree* lastNode) : m_firstNode(firstNode), m_lastNode(lastNode) +LIR::ReadOnlyRange::ReadOnlyRange(GenTree* firstNode, GenTree* lastNode) + : m_firstNode(firstNode) + , m_lastNode(lastNode) { assert((m_firstNode == nullptr) == (m_lastNode == nullptr)); assert((m_firstNode == m_lastNode) || (Contains(m_lastNode))); @@ -426,11 +438,13 @@ bool LIR::ReadOnlyRange::Contains(GenTree* node) const #endif -LIR::Range::Range() : ReadOnlyRange() +LIR::Range::Range() + : ReadOnlyRange() { } -LIR::Range::Range(Range&& other) : ReadOnlyRange(std::move(other)) +LIR::Range::Range(Range&& other) + : ReadOnlyRange(std::move(other)) { } @@ -442,7 +456,8 @@ LIR::Range::Range(Range&& other) : ReadOnlyRange(std::move(other)) // firstNode - The first node in the range. // lastNode - The last node in the range. // -LIR::Range::Range(GenTree* firstNode, GenTree* lastNode) : ReadOnlyRange(firstNode, lastNode) +LIR::Range::Range(GenTree* firstNode, GenTree* lastNode) + : ReadOnlyRange(firstNode, lastNode) { } @@ -1186,7 +1201,7 @@ bool LIR::Range::TryGetUse(GenTree* node, Use* use) // Returns: // The computed subrange. // -template +template LIR::ReadOnlyRange LIR::Range::GetMarkedRange(unsigned markCount, GenTree* start, bool* isClosed, @@ -1406,8 +1421,8 @@ class CheckLclVarSemanticsHelper // range - a range to do the check. // unusedDefs - map of defs that do no have users. // - CheckLclVarSemanticsHelper(Compiler* compiler, - const LIR::Range* range, + CheckLclVarSemanticsHelper(Compiler* compiler, + const LIR::Range* range, SmallHashTable& unusedDefs) : compiler(compiler) , range(range) @@ -1554,7 +1569,7 @@ class CheckLclVarSemanticsHelper void PopLclVarRead(const AliasSet::NodeInfo& defInfo) { SmallHashTable* reads; - const bool foundReads = unusedLclVarReads.TryGetValue(defInfo.LclNum(), &reads); + const bool foundReads = unusedLclVarReads.TryGetValue(defInfo.LclNum(), &reads); assert(foundReads); bool found = reads->TryRemove(defInfo.Node()); @@ -1569,11 +1584,11 @@ class CheckLclVarSemanticsHelper } private: - Compiler* compiler; - const LIR::Range* range; - SmallHashTable& unusedDefs; + Compiler* compiler; + const LIR::Range* range; + SmallHashTable& unusedDefs; SmallHashTable*, 16U> unusedLclVarReads; - ArrayStack*> lclVarReadsMapsCache; + ArrayStack*> lclVarReadsMapsCache; }; //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/lir.h b/src/coreclr/jit/lir.h index 9b4f940bc0ae38..8a3a9a507a38bb 100644 --- a/src/coreclr/jit/lir.h +++ b/src/coreclr/jit/lir.h @@ -73,7 +73,7 @@ class LIR final void AssertIsValid() const; bool IsDummyUse() const; - void ReplaceWith(GenTree* replacement); + void ReplaceWith(GenTree* replacement); unsigned ReplaceWithLclVar(Compiler* compiler, unsigned lclNum = BAD_VAR_NUM, GenTree** pStore = nullptr); }; @@ -113,7 +113,7 @@ class LIR final GenTree* m_firstNode; GenTree* m_lastNode; - ReadOnlyRange(const ReadOnlyRange& other) = delete; + ReadOnlyRange(const ReadOnlyRange& other) = delete; ReadOnlyRange& operator=(const ReadOnlyRange& other) = delete; public: @@ -125,12 +125,14 @@ class LIR final GenTree* m_node; - Iterator(GenTree* begin) : m_node(begin) + Iterator(GenTree* begin) + : m_node(begin) { } public: - Iterator() : m_node(nullptr) + Iterator() + : m_node(nullptr) { } @@ -167,12 +169,14 @@ class LIR final GenTree* m_node; - ReverseIterator(GenTree* begin) : m_node(begin) + ReverseIterator(GenTree* begin) + : m_node(begin) { } public: - ReverseIterator() : m_node(nullptr) + ReverseIterator() + : m_node(nullptr) { } @@ -245,7 +249,7 @@ class LIR final private: Range(GenTree* firstNode, GenTree* lastNode); - Range(const Range& other) = delete; + Range(const Range& other) = delete; Range& operator=(const Range& other) = delete; template @@ -280,7 +284,7 @@ class LIR final void InsertAtBeginning(Range&& range); void InsertAtEnd(Range&& range); - void Remove(GenTree* node, bool markOperandsUnused = false); + void Remove(GenTree* node, bool markOperandsUnused = false); Range Remove(GenTree* firstNode, GenTree* lastNode); Range Remove(ReadOnlyRange&& range); @@ -303,7 +307,7 @@ class LIR final }; public: - static Range& AsRange(BasicBlock* block); + static Range& AsRange(BasicBlock* block); static const Range& AsRange(const BasicBlock* block); static Range EmptyRange(); diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 78fb96fe3d77d2..7f413b75d6649e 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -811,10 +811,10 @@ void Compiler::fgExtendDbgLifetimes() fgExtendDbgScopes(); -/*------------------------------------------------------------------------- - * Partly update liveness info so that we handle any funky BBF_INTERNAL - * blocks inserted out of sequence. - */ + /*------------------------------------------------------------------------- + * Partly update liveness info so that we handle any funky BBF_INTERNAL + * blocks inserted out of sequence. + */ #ifdef DEBUG if (verbose && 0) @@ -1005,7 +1005,7 @@ void Compiler::fgExtendDbgLifetimes() // So just ensure that they don't have a 0 ref cnt unsigned lclNum = 0; - for (LclVarDsc *varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++) + for (LclVarDsc* varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++) { if (lclNum >= info.compArgsCount) { @@ -1676,10 +1676,10 @@ GenTree* Compiler::fgTryRemoveDeadStoreEarly(Statement* stmt, GenTreeLclVarCommo * or subtree of a statement moving backward from startNode to endNode */ -void Compiler::fgComputeLife(VARSET_TP& life, - GenTree* startNode, - GenTree* endNode, - VARSET_VALARG_TP volatileVars, +void Compiler::fgComputeLife(VARSET_TP& life, + GenTree* startNode, + GenTree* endNode, + VARSET_VALARG_TP volatileVars, bool* pStmtInfoDirty DEBUGARG(bool* treeModf)) { // Don't kill vars in scope @@ -2116,11 +2116,11 @@ bool Compiler::fgTryRemoveDeadStoreLIR(GenTree* store, GenTreeLclVarCommon* lclN // Return Value: // true if we should skip the rest of the statement, false if we should continue // -bool Compiler::fgRemoveDeadStore(GenTree** pTree, - LclVarDsc* varDsc, - VARSET_VALARG_TP life, - bool* doAgain, - bool* pStmtInfoDirty, +bool Compiler::fgRemoveDeadStore(GenTree** pTree, + LclVarDsc* varDsc, + VARSET_VALARG_TP life, + bool* doAgain, + bool* pStmtInfoDirty, bool* pStoreRemoved DEBUGARG(bool* treeModf)) { assert(!compRationalIRForm); @@ -2186,7 +2186,7 @@ bool Compiler::fgRemoveDeadStore(GenTree** pTree, #ifdef DEBUG *treeModf = true; #endif // DEBUG - // Update ordering, costs, FP levels, etc. + // Update ordering, costs, FP levels, etc. gtSetStmtInfo(compCurStmt); // Re-link the nodes for this statement @@ -2278,7 +2278,7 @@ bool Compiler::fgRemoveDeadStore(GenTree** pTree, printf("\n"); } #endif // DEBUG - // No side effects - Change the store to a GT_NOP node + // No side effects - Change the store to a GT_NOP node store->gtBashToNOP(); #ifdef DEBUG diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index 7283cc0d1d88c2..4c5ffe247634a4 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -1371,7 +1371,7 @@ bool Compiler::optDeriveLoopCloningConditions(FlowGraphNaturalLoop* loop, LoopCl LcMdArrayOptInfo* mdArrInfo = optInfo->AsLcMdArrayOptInfo(); LC_Array arrLen(LC_Array(LC_Array::MdArray, mdArrInfo->GetArrIndexForDim(getAllocator(CMK_LoopClone)), mdArrInfo->dim, LC_Array::None)); - LC_Ident arrLenIdent = LC_Ident::CreateArrAccess(arrLen); + LC_Ident arrLenIdent = LC_Ident::CreateArrAccess(arrLen); LC_Condition cond(opLimitCondition, LC_Expr(ident), LC_Expr(arrLenIdent)); context->EnsureConditions(loop->GetIndex())->Push(cond); @@ -1666,7 +1666,7 @@ void Compiler::optDebugLogLoopCloning(BasicBlock* block, Statement* insertBefore // performs the optimizations assuming that the path in which the candidates // were collected is the fast path in which the optimizations will be performed. // -void Compiler::optPerformStaticOptimizations(FlowGraphNaturalLoop* loop, +void Compiler::optPerformStaticOptimizations(FlowGraphNaturalLoop* loop, LoopCloneContext* context DEBUGARG(bool dynamicPath)) { JitExpandArrayStack* optInfos = context->GetLoopOptInfo(loop->GetIndex()); diff --git a/src/coreclr/jit/loopcloning.h b/src/coreclr/jit/loopcloning.h index 64e810be6ff424..20f041eab40a52 100644 --- a/src/coreclr/jit/loopcloning.h +++ b/src/coreclr/jit/loopcloning.h @@ -196,7 +196,12 @@ struct ArrIndex unsigned rank; // Rank of the array BasicBlock* useBlock; // Block where the [] occurs - ArrIndex(CompAllocator alloc) : arrLcl(BAD_VAR_NUM), indLcls(alloc), bndsChks(alloc), rank(0), useBlock(nullptr) + ArrIndex(CompAllocator alloc) + : arrLcl(BAD_VAR_NUM) + , indLcls(alloc) + , bndsChks(alloc) + , rank(0) + , useBlock(nullptr) { } @@ -236,7 +241,8 @@ struct LcOptInfo }; OptType optType; - LcOptInfo(OptType optType) : optType(optType) + LcOptInfo(OptType optType) + : optType(optType) { } @@ -267,7 +273,10 @@ struct LcMdArrayOptInfo : public LcOptInfo ArrIndex* index; // "index" cached computation in the form of an ArrIndex representation. LcMdArrayOptInfo(GenTreeArrElem* arrElem, unsigned dim) - : LcOptInfo(LcMdArray), arrElem(arrElem), dim(dim), index(nullptr) + : LcOptInfo(LcMdArray) + , arrElem(arrElem) + , dim(dim) + , index(nullptr) { } @@ -300,7 +309,10 @@ struct LcJaggedArrayOptInfo : public LcOptInfo Statement* stmt; // "stmt" where the optimization opportunity occurs. LcJaggedArrayOptInfo(ArrIndex& arrIndex, unsigned dim, Statement* stmt) - : LcOptInfo(LcJaggedArray), dim(dim), arrIndex(arrIndex), stmt(stmt) + : LcOptInfo(LcJaggedArray) + , dim(dim) + , arrIndex(arrIndex) + , stmt(stmt) { } }; @@ -319,7 +331,11 @@ struct LcTypeTestOptInfo : public LcOptInfo CORINFO_CLASS_HANDLE clsHnd; LcTypeTestOptInfo(Statement* stmt, GenTreeIndir* methodTableIndir, unsigned lclNum, CORINFO_CLASS_HANDLE clsHnd) - : LcOptInfo(LcTypeTest), stmt(stmt), methodTableIndir(methodTableIndir), lclNum(lclNum), clsHnd(clsHnd) + : LcOptInfo(LcTypeTest) + , stmt(stmt) + , methodTableIndir(methodTableIndir) + , lclNum(lclNum) + , clsHnd(clsHnd) { } }; @@ -343,7 +359,7 @@ struct LcMethodAddrTestOptInfo : public LcOptInfo GenTreeIndir* delegateAddressIndir, unsigned delegateLclNum, void* methAddr, - bool isSlot DEBUG_ARG(CORINFO_METHOD_HANDLE targetMethHnd)) + bool isSlot DEBUG_ARG(CORINFO_METHOD_HANDLE targetMethHnd)) : LcOptInfo(LcMethodAddrTest) , stmt(stmt) , delegateAddressIndir(delegateAddressIndir) @@ -393,15 +409,24 @@ struct LC_Array int dim; // "dim" = which index to invoke arrLen on, if -1 invoke on the whole array // Example 1: a[0][1][2] and dim = 2 implies a[0][1].length // Example 2: a[0][1][2] and dim = -1 implies a[0][1][2].length - LC_Array() : type(Invalid), dim(-1) + LC_Array() + : type(Invalid) + , dim(-1) { } LC_Array(ArrType type, ArrIndex* arrIndex, int dim, OperType oper) - : type(type), arrIndex(arrIndex), oper(oper), dim(dim) + : type(type) + , arrIndex(arrIndex) + , oper(oper) + , dim(dim) { } - LC_Array(ArrType type, ArrIndex* arrIndex, OperType oper) : type(type), arrIndex(arrIndex), oper(oper), dim(-1) + LC_Array(ArrType type, ArrIndex* arrIndex, OperType oper) + : type(type) + , arrIndex(arrIndex) + , oper(oper) + , dim(-1) { } @@ -464,7 +489,8 @@ struct LC_Ident }; private: - union { + union + { unsigned constant; struct { @@ -482,7 +508,8 @@ struct LC_Ident }; }; - LC_Ident(IdentType type) : type(type) + LC_Ident(IdentType type) + : type(type) { } @@ -490,7 +517,8 @@ struct LC_Ident // The type of this object IdentType type; - LC_Ident() : type(Invalid) + LC_Ident() + : type(Invalid) { } @@ -680,10 +708,13 @@ struct LC_Expr } #endif - LC_Expr() : type(Invalid) + LC_Expr() + : type(Invalid) { } - explicit LC_Expr(const LC_Ident& ident) : ident(ident), type(Ident) + explicit LC_Expr(const LC_Ident& ident) + : ident(ident) + , type(Ident) { } @@ -724,7 +755,10 @@ struct LC_Condition { } LC_Condition(genTreeOps oper, const LC_Expr& op1, const LC_Expr& op2, bool asUnsigned = false) - : op1(op1), op2(op2), oper(oper), compareUnsigned(asUnsigned) + : op1(op1) + , op2(op2) + , oper(oper) + , compareUnsigned(asUnsigned) { } @@ -756,7 +790,10 @@ struct LC_ArrayDeref unsigned level; - LC_ArrayDeref(const LC_Array& array, unsigned level) : array(array), children(nullptr), level(level) + LC_ArrayDeref(const LC_Array& array, unsigned level) + : array(array) + , children(nullptr) + , level(level) { } @@ -764,8 +801,8 @@ struct LC_ArrayDeref unsigned Lcl(); - bool HasChildren(); - void EnsureChildren(CompAllocator alloc); + bool HasChildren(); + void EnsureChildren(CompAllocator alloc); static LC_ArrayDeref* Find(JitExpandArrayStack* children, unsigned lcl); void DeriveLevelConditions(JitExpandArrayStack*>* len); @@ -859,7 +896,7 @@ struct LoopCloneContext } NaturalLoopIterInfo* GetLoopIterInfo(unsigned loopNum); - void SetLoopIterInfo(unsigned loopNum, NaturalLoopIterInfo* info); + void SetLoopIterInfo(unsigned loopNum, NaturalLoopIterInfo* info); // Evaluate conditions into a JTRUE stmt and put it in a new block after `insertAfter`. BasicBlock* CondToStmtInBlock(Compiler* comp, diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 935479beff01b3..e5e7aa9dbd301a 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1163,8 +1163,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // |____ (ICon) (The actual case constant) GenTree* gtCaseCond = comp->gtNewOperNode(GT_EQ, TYP_INT, comp->gtNewLclvNode(tempLclNum, tempLclType), comp->gtNewIconNode(i, genActualType(tempLclType))); - GenTree* gtCaseBranch = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, gtCaseCond); - LIR::Range caseRange = LIR::SeqTree(comp, gtCaseBranch); + GenTree* gtCaseBranch = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, gtCaseCond); + LIR::Range caseRange = LIR::SeqTree(comp, gtCaseBranch); currentBBRange->InsertAtEnd(std::move(caseRange)); } } @@ -3887,7 +3887,7 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) #ifdef TARGET_XARCH || IsContainableMemoryOp(castOp) #endif - ); + ); if (removeCast) { @@ -4771,10 +4771,10 @@ void Lowering::LowerStoreLocCommon(GenTreeLclVarCommon* lclStore) } convertToStoreObj = false; #else // TARGET_ARM64 - // This optimization on arm64 allows more SIMD16 vars to be enregistered but it could cause - // regressions when there are many calls and before/after each one we have to store/save the upper - // half of these registers. So enable this for arm64 only when LSRA is taught not to allocate registers when - // it would have to spilled too many times. + // This optimization on arm64 allows more SIMD16 vars to be enregistered but it could cause + // regressions when there are many calls and before/after each one we have to store/save the upper + // half of these registers. So enable this for arm64 only when LSRA is taught not to allocate registers when + // it would have to spilled too many times. convertToStoreObj = true; #endif // TARGET_ARM64 } @@ -5091,8 +5091,8 @@ void Lowering::LowerCallStruct(GenTreeCall* call) break; } #endif // FEATURE_SIMD - // importer has a separate mechanism to retype calls to helpers, - // keep it for now. + // importer has a separate mechanism to retype calls to helpers, + // keep it for now. assert(user->TypeIs(TYP_REF) || (user->TypeIs(TYP_I_IMPL) && comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI))); assert(call->IsHelperCall()); assert(returnType == user->TypeGet()); @@ -8086,7 +8086,7 @@ void Lowering::ContainCheckNode(GenTree* node) #if FEATURE_ARG_SPLIT case GT_PUTARG_SPLIT: #endif // FEATURE_ARG_SPLIT - // The regNum must have been set by the lowering of the call. + // The regNum must have been set by the lowering of the call. assert(node->GetRegNum() != REG_NA); break; #ifdef TARGET_XARCH @@ -8799,7 +8799,7 @@ void Lowering::LowerStoreIndirCommon(GenTreeStoreInd* ind) // const bool isContainable = IsInvariantInRange(ind->Addr(), ind); #else - const bool isContainable = true; + const bool isContainable = true; #endif TryCreateAddrMode(ind->Addr(), isContainable, ind); @@ -8863,7 +8863,7 @@ GenTree* Lowering::LowerIndir(GenTreeIndir* ind) // const bool isContainable = IsInvariantInRange(ind->Addr(), ind); #else - const bool isContainable = true; + const bool isContainable = true; #endif TryCreateAddrMode(ind->Addr(), isContainable, ind); @@ -9294,7 +9294,7 @@ void Lowering::TransformUnusedIndirection(GenTreeIndir* ind, Compiler* comp, Bas #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) bool useNullCheck = true; #elif defined(TARGET_ARM) - bool useNullCheck = false; + bool useNullCheck = false; #else // TARGET_XARCH bool useNullCheck = !ind->Addr()->isContained(); ind->ClearDontExtend(); @@ -9533,7 +9533,7 @@ void Lowering::TryRetypingFloatingPointStoreToIntegerStore(GenTree* store) #if defined(TARGET_XARCH) || defined(TARGET_ARM) bool shouldSwitchToInteger = true; #else // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 - bool shouldSwitchToInteger = FloatingPointUtils::isPositiveZero(dblCns); + bool shouldSwitchToInteger = FloatingPointUtils::isPositiveZero(dblCns); #endif if (shouldSwitchToInteger) diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 76124820944f3c..318a148ee9c1c9 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -88,14 +88,14 @@ class Lowering final : public Phase void ContainCheckLclHeap(GenTreeOp* node); void ContainCheckRet(GenTreeUnOp* ret); #ifdef TARGET_ARM64 - bool TryLowerAndOrToCCMP(GenTreeOp* tree, GenTree** next); + bool TryLowerAndOrToCCMP(GenTreeOp* tree, GenTree** next); insCflags TruthifyingFlags(GenCondition cond); - void ContainCheckConditionalCompare(GenTreeCCMP* ccmp); - void ContainCheckNeg(GenTreeOp* neg); - void TryLowerCnsIntCselToCinc(GenTreeOp* select, GenTree* cond); - void TryLowerCselToCSOp(GenTreeOp* select, GenTree* cond); - bool TryLowerAddSubToMulLongOp(GenTreeOp* op, GenTree** next); - bool TryLowerNegToMulLongOp(GenTreeOp* op, GenTree** next); + void ContainCheckConditionalCompare(GenTreeCCMP* ccmp); + void ContainCheckNeg(GenTreeOp* neg); + void TryLowerCnsIntCselToCinc(GenTreeOp* select, GenTree* cond); + void TryLowerCselToCSOp(GenTreeOp* select, GenTree* cond); + bool TryLowerAddSubToMulLongOp(GenTreeOp* op, GenTree** next); + bool TryLowerNegToMulLongOp(GenTreeOp* op, GenTree** next); #endif void ContainCheckSelect(GenTreeOp* select); void ContainCheckBitCast(GenTree* node); @@ -129,7 +129,7 @@ class Lowering final : public Phase static bool CheckBlock(Compiler* compiler, BasicBlock* block); #endif // DEBUG - void LowerBlock(BasicBlock* block); + void LowerBlock(BasicBlock* block); GenTree* LowerNode(GenTree* node); bool IsCFGCallArgInvariantInRange(GenTree* node, GenTree* endExclusive); @@ -138,28 +138,28 @@ class Lowering final : public Phase // Call Lowering // ------------------------------ GenTree* LowerCall(GenTree* call); - bool LowerCallMemmove(GenTreeCall* call, GenTree** next); - bool LowerCallMemcmp(GenTreeCall* call, GenTree** next); - bool LowerCallMemset(GenTreeCall* call, GenTree** next); - void LowerCFGCall(GenTreeCall* call); - void MoveCFGCallArgs(GenTreeCall* call); - void MoveCFGCallArg(GenTreeCall* call, GenTree* node); + bool LowerCallMemmove(GenTreeCall* call, GenTree** next); + bool LowerCallMemcmp(GenTreeCall* call, GenTree** next); + bool LowerCallMemset(GenTreeCall* call, GenTree** next); + void LowerCFGCall(GenTreeCall* call); + void MoveCFGCallArgs(GenTreeCall* call); + void MoveCFGCallArg(GenTreeCall* call, GenTree* node); #ifndef TARGET_64BIT GenTree* DecomposeLongCompare(GenTree* cmp); #endif - GenTree* OptimizeConstCompare(GenTree* cmp); - GenTree* LowerCompare(GenTree* cmp); - GenTree* LowerJTrue(GenTreeOp* jtrue); - GenTree* LowerSelect(GenTreeConditional* cond); - bool TryLowerConditionToFlagsNode(GenTree* parent, GenTree* condition, GenCondition* code); + GenTree* OptimizeConstCompare(GenTree* cmp); + GenTree* LowerCompare(GenTree* cmp); + GenTree* LowerJTrue(GenTreeOp* jtrue); + GenTree* LowerSelect(GenTreeConditional* cond); + bool TryLowerConditionToFlagsNode(GenTree* parent, GenTree* condition, GenCondition* code); GenTreeCC* LowerNodeCC(GenTree* node, GenCondition condition); - void LowerJmpMethod(GenTree* jmp); - void LowerRet(GenTreeUnOp* ret); - void LowerStoreLocCommon(GenTreeLclVarCommon* lclVar); - void LowerRetStruct(GenTreeUnOp* ret); - void LowerRetSingleRegStructLclVar(GenTreeUnOp* ret); - void LowerCallStruct(GenTreeCall* call); - void LowerStoreSingleRegCallStruct(GenTreeBlk* store); + void LowerJmpMethod(GenTree* jmp); + void LowerRet(GenTreeUnOp* ret); + void LowerStoreLocCommon(GenTreeLclVarCommon* lclVar); + void LowerRetStruct(GenTreeUnOp* ret); + void LowerRetSingleRegStructLclVar(GenTreeUnOp* ret); + void LowerCallStruct(GenTreeCall* call); + void LowerStoreSingleRegCallStruct(GenTreeBlk* store); #if !defined(WINDOWS_AMD64_ABI) GenTreeLclVar* SpillStructCallResult(GenTreeCall* call) const; #endif // WINDOWS_AMD64_ABI @@ -168,29 +168,29 @@ class Lowering final : public Phase GenTree* LowerDirectCall(GenTreeCall* call); GenTree* LowerNonvirtPinvokeCall(GenTreeCall* call); GenTree* LowerTailCallViaJitHelper(GenTreeCall* callNode, GenTree* callTarget); - void LowerFastTailCall(GenTreeCall* callNode); - void RehomeArgForFastTailCall(unsigned int lclNum, - GenTree* insertTempBefore, - GenTree* lookForUsesStart, - GenTreeCall* callNode); - void InsertProfTailCallHook(GenTreeCall* callNode, GenTree* insertionPoint); + void LowerFastTailCall(GenTreeCall* callNode); + void RehomeArgForFastTailCall(unsigned int lclNum, + GenTree* insertTempBefore, + GenTree* lookForUsesStart, + GenTreeCall* callNode); + void InsertProfTailCallHook(GenTreeCall* callNode, GenTree* insertionPoint); GenTree* FindEarliestPutArg(GenTreeCall* call); - size_t MarkPutArgNodes(GenTree* node); + size_t MarkPutArgNodes(GenTree* node); GenTree* LowerVirtualVtableCall(GenTreeCall* call); GenTree* LowerVirtualStubCall(GenTreeCall* call); - void LowerArgsForCall(GenTreeCall* call); - void ReplaceArgWithPutArgOrBitcast(GenTree** ppChild, GenTree* newNode); + void LowerArgsForCall(GenTreeCall* call); + void ReplaceArgWithPutArgOrBitcast(GenTree** ppChild, GenTree* newNode); GenTree* NewPutArg(GenTreeCall* call, GenTree* arg, CallArg* callArg, var_types type); - void LowerArg(GenTreeCall* call, CallArg* callArg, bool late); + void LowerArg(GenTreeCall* call, CallArg* callArg, bool late); #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) GenTree* LowerFloatArg(GenTree** pArg, CallArg* callArg); GenTree* LowerFloatArgReg(GenTree* arg, regNumber regNum); #endif - void InsertPInvokeCallProlog(GenTreeCall* call); - void InsertPInvokeCallEpilog(GenTreeCall* call); - void InsertPInvokeMethodProlog(); - void InsertPInvokeMethodEpilog(BasicBlock* returnBB DEBUGARG(GenTree* lastExpr)); + void InsertPInvokeCallProlog(GenTreeCall* call); + void InsertPInvokeCallEpilog(GenTreeCall* call); + void InsertPInvokeMethodProlog(); + void InsertPInvokeMethodEpilog(BasicBlock* returnBB DEBUGARG(GenTree* lastExpr)); GenTree* SetGCState(int cns); GenTree* CreateReturnTrapSeq(); enum FrameLinkAction @@ -316,31 +316,31 @@ class Lowering final : public Phase #endif // defined(TARGET_XARCH) // Per tree node member functions - void LowerStoreIndirCommon(GenTreeStoreInd* ind); + void LowerStoreIndirCommon(GenTreeStoreInd* ind); GenTree* LowerIndir(GenTreeIndir* ind); - bool OptimizeForLdp(GenTreeIndir* ind); - bool TryMakeIndirsAdjacent(GenTreeIndir* prevIndir, GenTreeIndir* indir); - void MarkTree(GenTree* root); - void UnmarkTree(GenTree* root); - void LowerStoreIndir(GenTreeStoreInd* node); - void LowerStoreIndirCoalescing(GenTreeStoreInd* node); + bool OptimizeForLdp(GenTreeIndir* ind); + bool TryMakeIndirsAdjacent(GenTreeIndir* prevIndir, GenTreeIndir* indir); + void MarkTree(GenTree* root); + void UnmarkTree(GenTree* root); + void LowerStoreIndir(GenTreeStoreInd* node); + void LowerStoreIndirCoalescing(GenTreeStoreInd* node); GenTree* LowerAdd(GenTreeOp* node); GenTree* LowerMul(GenTreeOp* mul); - bool TryLowerAndNegativeOne(GenTreeOp* node, GenTree** nextNode); + bool TryLowerAndNegativeOne(GenTreeOp* node, GenTree** nextNode); GenTree* LowerBinaryArithmetic(GenTreeOp* binOp); - bool LowerUnsignedDivOrMod(GenTreeOp* divMod); - bool TryLowerConstIntDivOrMod(GenTree* node, GenTree** nextNode); + bool LowerUnsignedDivOrMod(GenTreeOp* divMod); + bool TryLowerConstIntDivOrMod(GenTree* node, GenTree** nextNode); GenTree* LowerSignedDivOrMod(GenTree* node); - void LowerBlockStore(GenTreeBlk* blkNode); - void LowerBlockStoreCommon(GenTreeBlk* blkNode); - void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); - void LowerLclHeap(GenTree* node); - void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent); - void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode); + void LowerBlockStore(GenTreeBlk* blkNode); + void LowerBlockStoreCommon(GenTreeBlk* blkNode); + void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); + void LowerLclHeap(GenTree* node); + void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent); + void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode); GenTree* LowerArrLength(GenTreeArrCommon* node); #ifdef TARGET_XARCH - void LowerPutArgStk(GenTreePutArgStk* putArgStk); + void LowerPutArgStk(GenTreePutArgStk* putArgStk); GenTree* TryLowerMulWithConstant(GenTreeOp* node); #endif // TARGET_XARCH @@ -351,12 +351,12 @@ class Lowering final : public Phase void TryRetypingFloatingPointStoreToIntegerStore(GenTree* store); GenTree* LowerSwitch(GenTree* node); - bool TryLowerSwitchToBitTest(FlowEdge* jumpTable[], - unsigned jumpCount, - unsigned targetCount, - BasicBlock* bbSwitch, - GenTree* switchValue, - weight_t defaultLikelihood); + bool TryLowerSwitchToBitTest(FlowEdge* jumpTable[], + unsigned jumpCount, + unsigned targetCount, + BasicBlock* bbSwitch, + GenTree* switchValue, + weight_t defaultLikelihood); void LowerCast(GenTree* node); @@ -374,12 +374,12 @@ class Lowering final : public Phase void LowerShift(GenTreeOp* shift); #ifdef FEATURE_HW_INTRINSICS GenTree* LowerHWIntrinsic(GenTreeHWIntrinsic* node); - void LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIntrinsicId, GenCondition condition); + void LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIntrinsicId, GenCondition condition); GenTree* LowerHWIntrinsicCmpOp(GenTreeHWIntrinsic* node, genTreeOps cmpOp); GenTree* LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicDot(GenTreeHWIntrinsic* node); #if defined(TARGET_XARCH) - void LowerFusedMultiplyAdd(GenTreeHWIntrinsic* node); + void LowerFusedMultiplyAdd(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicToScalar(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* node); @@ -389,7 +389,7 @@ class Lowering final : public Phase GenTree* TryLowerAndOpToExtractLowestSetBit(GenTreeOp* andNode); GenTree* TryLowerAndOpToAndNot(GenTreeOp* andNode); GenTree* TryLowerXorOpToGetMaskUpToLowestSetBit(GenTreeOp* xorNode); - void LowerBswapOp(GenTreeOp* node); + void LowerBswapOp(GenTreeOp* node); #elif defined(TARGET_ARM64) bool IsValidConstForMovImm(GenTreeHWIntrinsic* node); void LowerHWIntrinsicFusedMultiplyAddScalar(GenTreeHWIntrinsic* node); @@ -589,7 +589,9 @@ class Lowering final : public Phase target_ssize_t Offset; SavedIndir(GenTreeIndir* indir, GenTreeLclVar* addrBase, target_ssize_t offset) - : Indir(indir), AddrBase(addrBase), Offset(offset) + : Indir(indir) + , AddrBase(addrBase) + , Offset(offset) { } }; diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 9d28135c92a1a0..498093ae6fc52d 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -726,7 +726,7 @@ void Lowering::ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenT { return; } -#else // !TARGET_ARM +#else // !TARGET_ARM if ((ClrSafeInt(offset) + ClrSafeInt(size)).IsOverflow()) { return; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 5c4f05a04ad57a..999a3fc6d338ca 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -690,13 +690,13 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) else #endif // TARGET_X86 if (loadSize <= comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy)) - { - putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::Unroll; - } - else - { - putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::RepInstr; - } + { + putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::Unroll; + } + else + { + putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::RepInstr; + } } else // There are GC pointers. { @@ -767,7 +767,7 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) #if defined(TARGET_AMD64) && !src->IsIntegralConst(0) #endif // TARGET_AMD64 - ) + ) { MakeSrcContained(putArgStk, src); } @@ -1767,8 +1767,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // currently ANDNOT logic cannot be optimized by the ternary node. break; } - GenTree* op3 = second->AsHWIntrinsic()->Op(1) == node ? second->AsHWIntrinsic()->Op(2) - : second->AsHWIntrinsic()->Op(1); + GenTree* op3 = second->AsHWIntrinsic()->Op(1) == node ? second->AsHWIntrinsic()->Op(2) + : second->AsHWIntrinsic()->Op(1); GenTree* control = comp->gtNewIconNode(node->GetTernaryControlByte(second->AsHWIntrinsic())); CorInfoType simdBaseJitType = node->GetSimdBaseJitType(); unsigned simdSize = node->GetSimdSize(); @@ -6650,12 +6650,12 @@ void Lowering::ContainCheckCallOperands(GenTreeCall* call) else #endif // TARGET_X86 if (ctrlExpr->isIndir()) - { - // We may have cases where we have set a register target on the ctrlExpr, but if it - // contained we must clear it. - ctrlExpr->SetRegNum(REG_NA); - MakeSrcContained(call, ctrlExpr); - } + { + // We may have cases where we have set a register target on the ctrlExpr, but if it + // contained we must clear it. + ctrlExpr->SetRegNum(REG_NA); + MakeSrcContained(call, ctrlExpr); + } } } @@ -10027,8 +10027,8 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) if (op1->IsVectorZero()) { -// When op1 is zero, we can contain it and we expect that -// ival is already in the correct state to account for it + // When op1 is zero, we can contain it and we expect that + // ival is already in the correct state to account for it #if DEBUG ssize_t ival = lastOp->AsIntConCommon()->IconValue(); @@ -10048,8 +10048,8 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else if (op2->IsVectorZero()) { -// When op2 is zero, we can contain it and we expect that -// zmask is already in the correct state to account for it + // When op2 is zero, we can contain it and we expect that + // zmask is already in the correct state to account for it #if DEBUG ssize_t ival = lastOp->AsIntConCommon()->IconValue(); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 685516c9338f11..aeffaa4b370292 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -297,7 +297,7 @@ regMaskOnlyOne LinearScan::getMatchingConstants(regMaskOnlyOne mask, regMaskOnlyOne result = RBM_NONE; while (candidates != RBM_NONE) { - regNumber regNum = genFirstRegNumFromMask(candidates MORE_THAN_64_REG_ARG(currentInterval->registerType)); + regNumber regNum = genFirstRegNumFromMask(candidates MORE_THAN_64_REG_ARG(currentInterval->registerType)); singleRegMask candidateBit = genRegMask(regNum); candidates ^= candidateBit; @@ -386,10 +386,10 @@ void LinearScan::updateSpillCost(regNumber reg, Interval* interval) // interval - Interval of Refposition. // assignedReg - Assigned register for this refposition. // -void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - AllRegsMask* regsToFree, - AllRegsMask* delayRegsToFree, +void LinearScan::updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + AllRegsMask* regsToFree, + AllRegsMask* delayRegsToFree, RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)) { @@ -1482,7 +1482,7 @@ PhaseStatus LinearScan::doLinearScan() #ifdef DEBUG || VERBOSE #endif - ) + ) { dumpLsraStats(jitstdout()); } @@ -1816,7 +1816,7 @@ template void LinearScan::identifyCandidates(); // TODO-Cleanup: This was cloned from Compiler::lvaSortByRefCount() in lclvars.cpp in order // to avoid perturbation, but should be merged. template -void LinearScan::identifyCandidates() +void LinearScan::identifyCandidates() { if (localVarsEnregistered) { @@ -2067,24 +2067,24 @@ void LinearScan::identifyCandidates() else #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE if (regType(type) == FloatRegisterType) - { - floatVarCount++; - weight_t refCntWtd = varDsc->lvRefCntWtd(); - if (varDsc->lvIsRegArg) { - // Don't count the initial reference for register params. In those cases, - // using a callee-save causes an extra copy. - refCntWtd -= BB_UNITY_WEIGHT; - } - if (refCntWtd >= thresholdFPRefCntWtd) - { - VarSetOps::AddElemD(compiler, fpCalleeSaveCandidateVars, varDsc->lvVarIndex); - } - else if (refCntWtd >= maybeFPRefCntWtd) - { - VarSetOps::AddElemD(compiler, fpMaybeCandidateVars, varDsc->lvVarIndex); + floatVarCount++; + weight_t refCntWtd = varDsc->lvRefCntWtd(); + if (varDsc->lvIsRegArg) + { + // Don't count the initial reference for register params. In those cases, + // using a callee-save causes an extra copy. + refCntWtd -= BB_UNITY_WEIGHT; + } + if (refCntWtd >= thresholdFPRefCntWtd) + { + VarSetOps::AddElemD(compiler, fpCalleeSaveCandidateVars, varDsc->lvVarIndex); + } + else if (refCntWtd >= maybeFPRefCntWtd) + { + VarSetOps::AddElemD(compiler, fpMaybeCandidateVars, varDsc->lvVarIndex); + } } - } JITDUMP(" "); DBEXEC(VERBOSE, newInt->dump(compiler)); } @@ -2543,7 +2543,7 @@ void LinearScan::checkLastUses(BasicBlock* block) // the register locations will be "rotated" to stress the resolution and allocation // code. // -BasicBlock* LinearScan::findPredBlockForLiveIn(BasicBlock* block, +BasicBlock* LinearScan::findPredBlockForLiveIn(BasicBlock* block, BasicBlock* prevBlock DEBUGARG(bool* pPredBlockIsAllocated)) { BasicBlock* predBlock = nullptr; @@ -2740,33 +2740,33 @@ void LinearScan::setFrameType() else #endif // DOUBLE_ALIGN if (compiler->codeGen->isFramePointerRequired()) - { - frameType = FT_EBP_FRAME; - } - else - { - if (compiler->rpMustCreateEBPCalled == false) - { -#ifdef DEBUG - const char* reason; -#endif // DEBUG - compiler->rpMustCreateEBPCalled = true; - if (compiler->rpMustCreateEBPFrame(INDEBUG(&reason))) - { - JITDUMP("; Decided to create an EBP based frame for ETW stackwalking (%s)\n", reason); - compiler->codeGen->setFrameRequired(true); - } - } - - if (compiler->codeGen->isFrameRequired()) { frameType = FT_EBP_FRAME; } else { - frameType = FT_ESP_FRAME; + if (compiler->rpMustCreateEBPCalled == false) + { +#ifdef DEBUG + const char* reason; +#endif // DEBUG + compiler->rpMustCreateEBPCalled = true; + if (compiler->rpMustCreateEBPFrame(INDEBUG(&reason))) + { + JITDUMP("; Decided to create an EBP based frame for ETW stackwalking (%s)\n", reason); + compiler->codeGen->setFrameRequired(true); + } + } + + if (compiler->codeGen->isFrameRequired()) + { + frameType = FT_EBP_FRAME; + } + else + { + frameType = FT_ESP_FRAME; + } } - } switch (frameType) { @@ -2986,7 +2986,7 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo // for enregistration. It simply finds the register to be assigned, if it was assigned to something // else, then will unassign it and then assign to the currentInterval // -regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, +regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { assert(!enregisterLocalVars); @@ -3049,7 +3049,7 @@ regNumber LinearScan::allocateRegMinimal(Interval* currentInterval, // no such ref position, no register will be allocated. // template -regNumber LinearScan::allocateReg(Interval* currentInterval, +regNumber LinearScan::allocateReg(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { singleRegMask foundRegBit = @@ -3059,7 +3059,7 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, return REG_NA; } - regNumber foundReg = genRegNumFromMask(foundRegBit MORE_THAN_64_REG_ARG(currentInterval->registerType)); + regNumber foundReg = genRegNumFromMask(foundRegBit MORE_THAN_64_REG_ARG(currentInterval->registerType)); RegRecord* availablePhysRegRecord = getRegisterRecord(foundReg); Interval* assignedInterval = availablePhysRegRecord->assignedInterval; if ((assignedInterval != currentInterval) && @@ -4142,7 +4142,8 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(interval->registerType)); + regNumber nextReg = + genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(interval->registerType)); if (nextReg > targetReg) { newReg = nextReg; @@ -4984,7 +4985,7 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) } #ifdef HAS_MORE_THAN_64_REGISTERS -//TODO: Can we just if-def the method signature and `IsEmpty()`? +// TODO: Can we just if-def the method signature and `IsEmpty()`? void LinearScan::freeRegisterMask(AllRegsMask& freeMask) { while (!freeMask.IsEmpty()) @@ -5426,8 +5427,9 @@ void LinearScan::allocateRegistersMinimal() // happened to be restored in assignedReg, we would need assignedReg to stay alive because // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree, currentInterval->registerType DEBUG_ARG(currentInterval) - DEBUG_ARG(assignedRegister)); + &delayRegsToFree, + currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); @@ -5542,7 +5544,7 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); AllRegsMask assignedRegMask; assignedRegMask.AddRegNum(assignedRegister, currentInterval->registerType); @@ -6468,8 +6470,9 @@ void LinearScan::allocateRegisters() // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree, currentInterval->registerType DEBUG_ARG( - currentInterval) DEBUG_ARG(assignedRegister)); + &delayRegsToFree, + currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); @@ -6577,8 +6580,9 @@ void LinearScan::allocateRegisters() // happened to be restored in assignedReg, we would need assignedReg to stay alive because // we will copy the entire vector value from it to the `copyReg`. updateRegsFreeBusyState(currentRefPosition, assignedRegMask | copyRegMask, ®sToFree, - &delayRegsToFree, currentInterval->registerType DEBUG_ARG(currentInterval) - DEBUG_ARG(assignedRegister)); + &delayRegsToFree, + currentInterval->registerType DEBUG_ARG(currentInterval) + DEBUG_ARG(assignedRegister)); if (!currentRefPosition.lastUse) { copyRegsToFree.AddRegNum(copyReg, currentInterval->registerType); @@ -6769,7 +6773,7 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genRegMask(assignedRegister); AllRegsMask assignedRegMask; assignedRegMask.AddRegNum(assignedRegister, currentInterval->registerType); @@ -7996,7 +8000,7 @@ void LinearScan::updateMaxSpill(RefPosition* refPosition) // the tree, and performs resolution across joins and back edges. // template -void LinearScan::resolveRegisters() +void LinearScan::resolveRegisters() { // Iterate over the tree and the RefPositions in lockstep // - annotate the tree with register assignments by setting GetRegNum() or gtRegPair (for longs) @@ -8469,9 +8473,10 @@ void LinearScan::resolveRegisters() if (varDsc->lvIsParam) { regMaskOnlyOne initialRegMask = interval->firstRefPosition->registerAssignment; - regNumber initialReg = (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) - ? REG_STK - : genRegNumFromMask(initialRegMask MORE_THAN_64_REG_ARG(interval->registerType)); + regNumber initialReg = + (initialRegMask == RBM_NONE || interval->firstRefPosition->spillAfter) + ? REG_STK + : genRegNumFromMask(initialRegMask MORE_THAN_64_REG_ARG(interval->registerType)); #ifdef TARGET_ARM if (varTypeIsMultiReg(varDsc)) @@ -8938,12 +8943,12 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, // Notes: // It inserts at least one move and updates incoming parameter 'location'. // -void LinearScan::addResolutionForDouble(BasicBlock* block, - GenTree* insertionPoint, - Interval** sourceIntervals, - regNumberSmall* location, - regNumber toReg, - regNumber fromReg, +void LinearScan::addResolutionForDouble(BasicBlock* block, + GenTree* insertionPoint, + Interval** sourceIntervals, + regNumberSmall* location, + regNumber toReg, + regNumber fromReg, ResolveType resolveType DEBUG_ARG(BasicBlock* fromBlock) DEBUG_ARG(BasicBlock* toBlock)) { @@ -9013,10 +9018,10 @@ void LinearScan::addResolutionForDouble(BasicBlock* block, // The next time, we want to move from the stack to the destination (toReg), // in which case fromReg will be REG_STK, and we insert at the top. // -void LinearScan::addResolution(BasicBlock* block, - GenTree* insertionPoint, - Interval* interval, - regNumber toReg, +void LinearScan::addResolution(BasicBlock* block, + GenTree* insertionPoint, + Interval* interval, + regNumber toReg, regNumber fromReg DEBUG_ARG(BasicBlock* fromBlock) DEBUG_ARG(BasicBlock* toBlock) DEBUG_ARG(const char* reason)) { @@ -10141,7 +10146,7 @@ const char* LinearScan::getStatName(unsigned stat) #define LSRA_STAT_DEF(stat, name) name, #include "lsra_stats.h" #undef LSRA_STAT_DEF -#define REG_SEL_DEF(stat, value, shortname, orderSeqId) #stat, +#define REG_SEL_DEF(stat, value, shortname, orderSeqId) #stat, #define BUSY_REG_SEL_DEF(stat, value, shortname, orderSeqId) REG_SEL_DEF(stat, value, shortname, orderSeqId) #include "lsra_score.h" }; @@ -11471,9 +11476,8 @@ void LinearScan::dumpRegRecordHeader() // l is either '*' (if a last use) or ' ' (otherwise) // d is either 'D' (if a delayed use) or ' ' (otherwise) - maxNodeLocation = (maxNodeLocation == 0) - ? 1 - : maxNodeLocation; // corner case of a method with an infinite loop without any GenTree nodes + maxNodeLocation = (maxNodeLocation == 0) ? 1 : maxNodeLocation; // corner case of a method with an infinite loop + // without any GenTree nodes assert(maxNodeLocation >= 1); assert(refPositions.size() >= 1); int treeIdWidth = 9; /* '[XXXXX] '*/ @@ -11546,9 +11550,9 @@ void LinearScan::dumpRegRecordTitleIfNeeded() { lastUsedRegNumIndex = 0; #ifdef FEATURE_MASKED_HW_INTRINSICS - int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_MASK_LAST : REG_INT_LAST; + int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_MASK_LAST : REG_INT_LAST; #else - int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_FP_LAST : REG_INT_LAST; + int lastRegNumIndex = compiler->compFloatingPointUsed ? REG_FP_LAST : REG_INT_LAST; #endif for (int regNumIndex = 0; regNumIndex <= lastRegNumIndex; regNumIndex++) { @@ -12330,7 +12334,8 @@ void LinearScan::verifyFinalAllocation() regMaskOnlyOne candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(TYP_INT)); + regNumber nextReg = + genFirstRegNumFromMaskAndToggle(candidateRegs MORE_THAN_64_REG_ARG(TYP_INT)); RegRecord* regRecord = getRegisterRecord(nextReg); Interval* assignedInterval = regRecord->assignedInterval; @@ -12621,7 +12626,7 @@ LinearScan::RegisterSelection::RegisterSelection(LinearScan* linearScan) #ifdef TARGET_ARM64 && !linearScan->compiler->info.compNeedsConsecutiveRegisters #endif - ) + ) { ordering = W("MQQQQQQQQQQQQQQQQ"); } @@ -13077,8 +13082,8 @@ void LinearScan::RegisterSelection::try_REG_ORDER() regMaskOnlyOne lowestRegOrderBit = RBM_NONE; for (regMaskOnlyOne regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { - regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates MORE_THAN_64_REG_ARG(regType)); - singleRegMask regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); + regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates MORE_THAN_64_REG_ARG(regType)); + singleRegMask regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; unsigned thisRegOrder = linearScan->getRegisterRecord(regOrderCandidateRegNum)->regOrder; @@ -13238,8 +13243,8 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() regMaskOnlyOne farthestSet = RBM_NONE; for (regMaskOnlyOne farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { - regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates MORE_THAN_64_REG_ARG(regType)); - singleRegMask farthestCandidateBit = genRegMask(farthestCandidateRegNum); + regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates MORE_THAN_64_REG_ARG(regType)); + singleRegMask farthestCandidateBit = genRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; // Find the next RefPosition of the register. @@ -13271,8 +13276,9 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() regMaskOnlyOne prevRegOptSet = RBM_NONE; for (regMaskOnlyOne prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { - regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates MORE_THAN_64_REG_ARG(regType)); - singleRegMask prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); + regNumber prevRegOptCandidateRegNum = + genFirstRegNumFromMask(prevRegOptCandidates MORE_THAN_64_REG_ARG(regType)); + singleRegMask prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; bool foundPrevRegOptReg = true; @@ -13342,7 +13348,7 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() && !refPosition->needsConsecutive #endif - ) + ) { assert(!"Spill candidate has no assignedInterval recentRefPosition"); } @@ -13474,7 +13480,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() // Register bit selected (a single register) and REG_NA if no register was selected. // template -singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval, +singleRegMask LinearScan::RegisterSelection::select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)) { #ifdef DEBUG diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 00bd15a4ad23e4..ad1d7447475ef7 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -30,13 +30,13 @@ const unsigned int MaxInternalRegisters = 8; const unsigned int RegisterTypeCount = 2; /***************************************************************************** -* Register types -*****************************************************************************/ + * Register types + *****************************************************************************/ typedef var_types RegisterType; -#define IntRegisterType TYP_INT +#define IntRegisterType TYP_INT #define FloatRegisterType TYP_FLOAT -#define MaskRegisterType TYP_MASK +#define MaskRegisterType TYP_MASK //------------------------------------------------------------------------ // regType: Return the RegisterType to use for a given type @@ -83,7 +83,9 @@ struct RefInfo RefPosition* ref; GenTree* treeNode; - RefInfo(RefPosition* r, GenTree* t) : ref(r), treeNode(t) + RefInfo(RefPosition* r, GenTree* t) + : ref(r) + , treeNode(t) { } @@ -107,7 +109,8 @@ class RefInfoListNode final : public RefInfo RefInfoListNode* m_next; // The next node in the list public: - RefInfoListNode(RefPosition* r, GenTree* t) : RefInfo(r, t) + RefInfoListNode(RefPosition* r, GenTree* t) + : RefInfo(r, t) { } @@ -134,11 +137,15 @@ class RefInfoList final RefInfoListNode* m_tail; // The tail of the list public: - RefInfoList() : m_head(nullptr), m_tail(nullptr) + RefInfoList() + : m_head(nullptr) + , m_tail(nullptr) { } - RefInfoList(RefInfoListNode* node) : m_head(node), m_tail(node) + RefInfoList(RefInfoListNode* node) + : m_head(node) + , m_tail(node) { assert(m_head->m_next == nullptr); } @@ -365,7 +372,7 @@ class RefInfoListNodePool final public: RefInfoListNodePool(Compiler* compiler, unsigned preallocate = defaultPreallocation); RefInfoListNode* GetNode(RefPosition* r, GenTree* t); - void ReturnNode(RefInfoListNode* listNode); + void ReturnNode(RefInfoListNode* listNode); }; #if TRACK_LSRA_STATS @@ -374,7 +381,7 @@ enum LsraStat #define LSRA_STAT_DEF(enum_name, enum_str) enum_name, #include "lsra_stats.h" #undef LSRA_STAT_DEF -#define REG_SEL_DEF(enum_name, value, short_str, orderSeqId) STAT_##enum_name, +#define REG_SEL_DEF(enum_name, value, short_str, orderSeqId) STAT_##enum_name, #define BUSY_REG_SEL_DEF(enum_name, value, short_str, orderSeqId) REG_SEL_DEF(enum_name, value, short_str, orderSeqId) #include "lsra_score.h" COUNT @@ -387,11 +394,11 @@ struct LsraBlockInfo // 0 for fgFirstBB. unsigned int predBBNum; weight_t weight; - bool hasCriticalInEdge : 1; + bool hasCriticalInEdge : 1; bool hasCriticalOutEdge : 1; - bool hasEHBoundaryIn : 1; - bool hasEHBoundaryOut : 1; - bool hasEHPred : 1; + bool hasEHBoundaryIn : 1; + bool hasEHBoundaryOut : 1; + bool hasEHPred : 1; #if TRACK_LSRA_STATS // Per block maintained LSRA statistics. @@ -401,7 +408,7 @@ struct LsraBlockInfo enum RegisterScore { -#define REG_SEL_DEF(enum_name, value, short_str, orderSeqId) enum_name = value, +#define REG_SEL_DEF(enum_name, value, short_str, orderSeqId) enum_name = value, #define BUSY_REG_SEL_DEF(enum_name, value, short_str, orderSeqId) REG_SEL_DEF(enum_name, value, short_str, orderSeqId) #include "lsra_score.h" NONE = 0 @@ -635,7 +642,7 @@ class LinearScan : public LinearScanInterface // This does the dataflow analysis and builds the intervals template - void buildIntervals(); + void buildIntervals(); // This is where the actual assignment is done for scenarios where // no local var enregistration is done. @@ -648,7 +655,7 @@ class LinearScan : public LinearScanInterface void allocateRegisters(); // This is the resolution phase, where cross-block mismatches are fixed up template - void resolveRegisters(); + void resolveRegisters(); void writeRegisters(RefPosition* currentRefPosition, GenTree* tree); @@ -658,7 +665,7 @@ class LinearScan : public LinearScanInterface void insertCopyOrReload(BasicBlock* block, GenTree* tree, unsigned multiRegIdx, RefPosition* refPosition); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - void makeUpperVectorInterval(unsigned varIndex); + void makeUpperVectorInterval(unsigned varIndex); Interval* getUpperVectorInterval(unsigned varIndex); // Save the upper half of a vector that lives in a callee-save register at the point of a call. @@ -693,20 +700,20 @@ class LinearScan : public LinearScanInterface }; #ifdef TARGET_ARM - void addResolutionForDouble(BasicBlock* block, - GenTree* insertionPoint, - Interval** sourceIntervals, - regNumberSmall* location, - regNumber toReg, - regNumber fromReg, + void addResolutionForDouble(BasicBlock* block, + GenTree* insertionPoint, + Interval** sourceIntervals, + regNumberSmall* location, + regNumber toReg, + regNumber fromReg, ResolveType resolveType DEBUG_ARG(BasicBlock* fromBlock) DEBUG_ARG(BasicBlock* toBlock)); #endif - void addResolution(BasicBlock* block, - GenTree* insertionPoint, - Interval* interval, - regNumber outReg, + void addResolution(BasicBlock* block, + GenTree* insertionPoint, + Interval* interval, + regNumber outReg, regNumber inReg DEBUG_ARG(BasicBlock* fromBlock) DEBUG_ARG(BasicBlock* toBlock) DEBUG_ARG(const char* reason)); @@ -818,8 +825,14 @@ class LinearScan : public LinearScanInterface // This controls the heuristics used to select registers // These can be combined. - enum LsraSelect{LSRA_SELECT_DEFAULT = 0, LSRA_SELECT_REVERSE_HEURISTICS = 0x04, - LSRA_SELECT_REVERSE_CALLER_CALLEE = 0x08, LSRA_SELECT_NEAREST = 0x10, LSRA_SELECT_MASK = 0x1c}; + enum LsraSelect + { + LSRA_SELECT_DEFAULT = 0, + LSRA_SELECT_REVERSE_HEURISTICS = 0x04, + LSRA_SELECT_REVERSE_CALLER_CALLEE = 0x08, + LSRA_SELECT_NEAREST = 0x10, + LSRA_SELECT_MASK = 0x1c + }; LsraSelect getSelectionHeuristics() { return (LsraSelect)(lsraStressMask & LSRA_SELECT_MASK); @@ -838,9 +851,14 @@ class LinearScan : public LinearScanInterface } // This controls the order in which basic blocks are visited during allocation - enum LsraTraversalOrder{LSRA_TRAVERSE_LAYOUT = 0x20, LSRA_TRAVERSE_PRED_FIRST = 0x40, - LSRA_TRAVERSE_RANDOM = 0x60, // NYI - LSRA_TRAVERSE_DEFAULT = LSRA_TRAVERSE_PRED_FIRST, LSRA_TRAVERSE_MASK = 0x60}; + enum LsraTraversalOrder + { + LSRA_TRAVERSE_LAYOUT = 0x20, + LSRA_TRAVERSE_PRED_FIRST = 0x40, + LSRA_TRAVERSE_RANDOM = 0x60, // NYI + LSRA_TRAVERSE_DEFAULT = LSRA_TRAVERSE_PRED_FIRST, + LSRA_TRAVERSE_MASK = 0x60 + }; LsraTraversalOrder getLsraTraversalOrder() { if ((lsraStressMask & LSRA_TRAVERSE_MASK) == 0) @@ -860,7 +878,12 @@ class LinearScan : public LinearScanInterface // This controls whether lifetimes should be extended to the entire method. // Note that this has no effect under MinOpts - enum LsraExtendLifetimes{LSRA_DONT_EXTEND = 0, LSRA_EXTEND_LIFETIMES = 0x80, LSRA_EXTEND_LIFETIMES_MASK = 0x80}; + enum LsraExtendLifetimes + { + LSRA_DONT_EXTEND = 0, + LSRA_EXTEND_LIFETIMES = 0x80, + LSRA_EXTEND_LIFETIMES_MASK = 0x80 + }; LsraExtendLifetimes getLsraExtendLifeTimes() { return (LsraExtendLifetimes)(lsraStressMask & LSRA_EXTEND_LIFETIMES_MASK); @@ -873,8 +896,13 @@ class LinearScan : public LinearScanInterface // This controls whether variables locations should be set to the previous block in layout order // (LSRA_BLOCK_BOUNDARY_LAYOUT), or to that of the highest-weight predecessor (LSRA_BLOCK_BOUNDARY_PRED - // the default), or rotated (LSRA_BLOCK_BOUNDARY_ROTATE). - enum LsraBlockBoundaryLocations{LSRA_BLOCK_BOUNDARY_PRED = 0, LSRA_BLOCK_BOUNDARY_LAYOUT = 0x100, - LSRA_BLOCK_BOUNDARY_ROTATE = 0x200, LSRA_BLOCK_BOUNDARY_MASK = 0x300}; + enum LsraBlockBoundaryLocations + { + LSRA_BLOCK_BOUNDARY_PRED = 0, + LSRA_BLOCK_BOUNDARY_LAYOUT = 0x100, + LSRA_BLOCK_BOUNDARY_ROTATE = 0x200, + LSRA_BLOCK_BOUNDARY_MASK = 0x300 + }; LsraBlockBoundaryLocations getLsraBlockBoundaryLocations() { return (LsraBlockBoundaryLocations)(lsraStressMask & LSRA_BLOCK_BOUNDARY_MASK); @@ -883,7 +911,12 @@ class LinearScan : public LinearScanInterface // This controls whether we always insert a GT_RELOAD instruction after a spill // Note that this can be combined with LSRA_SPILL_ALWAYS (or not) - enum LsraReload{LSRA_NO_RELOAD_IF_SAME = 0, LSRA_ALWAYS_INSERT_RELOAD = 0x400, LSRA_RELOAD_MASK = 0x400}; + enum LsraReload + { + LSRA_NO_RELOAD_IF_SAME = 0, + LSRA_ALWAYS_INSERT_RELOAD = 0x400, + LSRA_RELOAD_MASK = 0x400 + }; LsraReload getLsraReload() { return (LsraReload)(lsraStressMask & LSRA_RELOAD_MASK); @@ -894,7 +927,12 @@ class LinearScan : public LinearScanInterface } // This controls whether we spill everywhere - enum LsraSpill{LSRA_DONT_SPILL_ALWAYS = 0, LSRA_SPILL_ALWAYS = 0x800, LSRA_SPILL_MASK = 0x800}; + enum LsraSpill + { + LSRA_DONT_SPILL_ALWAYS = 0, + LSRA_SPILL_ALWAYS = 0x800, + LSRA_SPILL_MASK = 0x800 + }; LsraSpill getLsraSpill() { return (LsraSpill)(lsraStressMask & LSRA_SPILL_MASK); @@ -906,8 +944,12 @@ class LinearScan : public LinearScanInterface // This controls whether RefPositions that lower/codegen indicated as reg optional be // allocated a reg at all. - enum LsraRegOptionalControl{LSRA_REG_OPTIONAL_DEFAULT = 0, LSRA_REG_OPTIONAL_NO_ALLOC = 0x1000, - LSRA_REG_OPTIONAL_MASK = 0x1000}; + enum LsraRegOptionalControl + { + LSRA_REG_OPTIONAL_DEFAULT = 0, + LSRA_REG_OPTIONAL_NO_ALLOC = 0x1000, + LSRA_REG_OPTIONAL_MASK = 0x1000 + }; LsraRegOptionalControl getLsraRegOptionalControl() { @@ -990,7 +1032,7 @@ class LinearScan : public LinearScanInterface private: // Determine which locals are candidates for allocation template - void identifyCandidates(); + void identifyCandidates(); // determine which locals are used in EH constructs we don't want to deal with void identifyCandidatesExceptionDataflow(); @@ -999,8 +1041,8 @@ class LinearScan : public LinearScanInterface #ifdef DEBUG void checkLastUses(BasicBlock* block); - int ComputeOperandDstCount(GenTree* operand); - int ComputeAvailableSrcCount(GenTree* node); + int ComputeOperandDstCount(GenTree* operand); + int ComputeAvailableSrcCount(GenTree* node); #endif // DEBUG void setFrameType(); @@ -1013,7 +1055,7 @@ class LinearScan : public LinearScanInterface // Record variable locations at start/end of block void processBlockStartLocations(BasicBlock* current); void processBlockEndLocations(BasicBlock* current); - void resetAllRegistersState(); + void resetAllRegistersState(); #ifdef HAS_MORE_THAN_64_REGISTERS FORCEINLINE void updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap); #else @@ -1021,20 +1063,20 @@ class LinearScan : public LinearScanInterface #endif #ifdef TARGET_ARM - bool isSecondHalfReg(RegRecord* regRec, Interval* interval); + bool isSecondHalfReg(RegRecord* regRec, Interval* interval); RegRecord* getSecondHalfRegRec(RegRecord* regRec); RegRecord* findAnotherHalfRegRec(RegRecord* regRec); - regNumber findAnotherHalfRegNum(regNumber regNum); - bool canSpillDoubleReg(RegRecord* physRegRecord, LsraLocation refLocation); - void unassignDoublePhysReg(RegRecord* doubleRegRecord); + regNumber findAnotherHalfRegNum(regNumber regNum); + bool canSpillDoubleReg(RegRecord* physRegRecord, LsraLocation refLocation); + void unassignDoublePhysReg(RegRecord* doubleRegRecord); #endif - void clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)); - void updateAssignedInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); - void updatePreviousInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); - bool canRestorePreviousInterval(RegRecord* regRec, Interval* assignedInterval); - bool isAssignedToInterval(Interval* interval, RegRecord* regRec); - bool isRefPositionActive(RefPosition* refPosition, LsraLocation refLocation); - bool canSpillReg(RegRecord* physRegRecord, LsraLocation refLocation); + void clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)); + void updateAssignedInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); + void updatePreviousInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); + bool canRestorePreviousInterval(RegRecord* regRec, Interval* assignedInterval); + bool isAssignedToInterval(Interval* interval, RegRecord* regRec); + bool isRefPositionActive(RefPosition* refPosition, LsraLocation refLocation); + bool canSpillReg(RegRecord* physRegRecord, LsraLocation refLocation); weight_t getSpillWeight(RegRecord* physRegRecord); // insert refpositions representing prolog zero-inits which will be added later @@ -1048,7 +1090,7 @@ class LinearScan : public LinearScanInterface void buildRefPositionsForNode(GenTree* tree, LsraLocation loc); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - void buildUpperVectorSaveRefPositions(GenTree* tree, + void buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc DEBUG_ARG(regMaskFloat fpCalleeKillSet)); void buildUpperVectorRestoreRefPosition( Interval* lclVarInterval, LsraLocation currentLoc, GenTree* node, bool isUse, unsigned multiRegIdx); @@ -1080,10 +1122,10 @@ class LinearScan : public LinearScanInterface // Helpers for getKillSetForNode(). AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); - regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); - regMaskGpr getKillSetForMul(GenTreeOp* tree); + regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); + regMaskGpr getKillSetForMul(GenTreeOp* tree); AllRegsMask getKillSetForCall(GenTreeCall* call); - regMaskGpr getKillSetForModDiv(GenTreeOp* tree); + regMaskGpr getKillSetForModDiv(GenTreeOp* tree); AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); AllRegsMask getKillSetForReturn(); AllRegsMask getKillSetForProfilerHook(); @@ -1102,10 +1144,10 @@ class LinearScan : public LinearScanInterface bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask); regMaskOnlyOne allRegs(RegisterType rt); - regMaskGpr allByteRegs(); - regMaskFloat allSIMDRegs(); - regMaskFloat lowSIMDRegs(); - regMaskFloat internalFloatRegCandidates(); + regMaskGpr allByteRegs(); + regMaskFloat allSIMDRegs(); + regMaskFloat lowSIMDRegs(); + regMaskFloat internalFloatRegCandidates(); void makeRegisterInactive(RegRecord* physRegRecord); #ifdef HAS_MORE_THAN_64_REGISTERS @@ -1232,13 +1274,13 @@ class LinearScan : public LinearScanInterface void spillGCRefs(RefPosition* killRefPosition); -/***************************************************************************** -* Register selection -****************************************************************************/ + /***************************************************************************** + * Register selection + ****************************************************************************/ #if defined(TARGET_ARM64) - bool canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); - void assignConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); + bool canAssignNextConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); + void assignConsecutiveRegisters(RefPosition* firstRefPosition, regNumber firstRegAssigned); regMaskFloat getConsecutiveCandidates(regMaskFloat candidates, RefPosition* refPosition, regMaskFloat* busyCandidates); @@ -1278,10 +1320,10 @@ class LinearScan : public LinearScanInterface // Perform register selection and update currentInterval or refPosition template - FORCEINLINE singleRegMask select(Interval* currentInterval, + FORCEINLINE singleRegMask select(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); - FORCEINLINE singleRegMask selectMinimal(Interval* currentInterval, + FORCEINLINE singleRegMask selectMinimal(Interval* currentInterval, RefPosition* refPosition DEBUG_ARG(RegisterScore* registerScore)); // If the register is from unassigned set such that it was not already @@ -1364,14 +1406,14 @@ class LinearScan : public LinearScanInterface return (prevRegBit & preferences) == foundRegBit; } - bool applySelection(int selectionScore, regMaskOnlyOne selectionCandidates); - bool applySingleRegSelection(int selectionScore, regMaskOnlyOne selectionCandidate); + bool applySelection(int selectionScore, regMaskOnlyOne selectionCandidates); + bool applySingleRegSelection(int selectionScore, regMaskOnlyOne selectionCandidate); FORCEINLINE void calculateCoversSets(); FORCEINLINE void calculateUnassignedSets(); FORCEINLINE void reset(Interval* interval, RefPosition* refPosition); FORCEINLINE void resetMinimal(Interval* interval, RefPosition* refPosition); -#define REG_SEL_DEF(stat, value, shortname, orderSeqId) FORCEINLINE void try_##stat(); +#define REG_SEL_DEF(stat, value, shortname, orderSeqId) FORCEINLINE void try_##stat(); #define BUSY_REG_SEL_DEF(stat, value, shortname, orderSeqId) REG_SEL_DEF(stat, value, shortname, orderSeqId) #include "lsra_score.h" }; @@ -1399,8 +1441,8 @@ class LinearScan : public LinearScanInterface unsigned toBBNum; }; typedef JitHashTable, SplitEdgeInfo> SplitBBNumToTargetBBNumMap; - SplitBBNumToTargetBBNumMap* splitBBNumToTargetBBNumMap; - SplitBBNumToTargetBBNumMap* getSplitBBNumToTargetBBNumMap() + SplitBBNumToTargetBBNumMap* splitBBNumToTargetBBNumMap; + SplitBBNumToTargetBBNumMap* getSplitBBNumToTargetBBNumMap() { if (splitBBNumToTargetBBNumMap == nullptr) { @@ -1411,13 +1453,13 @@ class LinearScan : public LinearScanInterface } SplitEdgeInfo getSplitEdgeInfo(unsigned int bbNum); - void initVarRegMaps(); - void setInVarRegForBB(unsigned int bbNum, unsigned int varNum, regNumber reg); - void setOutVarRegForBB(unsigned int bbNum, unsigned int varNum, regNumber reg); + void initVarRegMaps(); + void setInVarRegForBB(unsigned int bbNum, unsigned int varNum, regNumber reg); + void setOutVarRegForBB(unsigned int bbNum, unsigned int varNum, regNumber reg); VarToRegMap getInVarToRegMap(unsigned int bbNum); VarToRegMap getOutVarToRegMap(unsigned int bbNum); - void setVarReg(VarToRegMap map, unsigned int trackedVarIndex, regNumber reg); - regNumber getVarReg(VarToRegMap map, unsigned int trackedVarIndex); + void setVarReg(VarToRegMap map, unsigned int trackedVarIndex, regNumber reg); + regNumber getVarReg(VarToRegMap map, unsigned int trackedVarIndex); // Initialize the incoming VarToRegMap to the given map values (generally a predecessor of // the block) VarToRegMap setInVarToRegMap(unsigned int bbNum, VarToRegMap srcVarToRegMap); @@ -1430,8 +1472,8 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARM64 typedef JitHashTable, RefPosition*> NextConsecutiveRefPositionsMap; - NextConsecutiveRefPositionsMap* nextConsecutiveRefPositionMap; - NextConsecutiveRefPositionsMap* getNextConsecutiveRefPositionsMap() + NextConsecutiveRefPositionsMap* nextConsecutiveRefPositionMap; + NextConsecutiveRefPositionsMap* getNextConsecutiveRefPositionsMap() { if (nextConsecutiveRefPositionMap == nullptr) { @@ -1459,7 +1501,12 @@ class LinearScan : public LinearScanInterface // - In LSRA_DUMP_POST, which is after register allocation, the registers are // shown. - enum LsraTupleDumpMode{LSRA_DUMP_PRE, LSRA_DUMP_REFPOS, LSRA_DUMP_POST}; + enum LsraTupleDumpMode + { + LSRA_DUMP_PRE, + LSRA_DUMP_REFPOS, + LSRA_DUMP_POST + }; void lsraGetOperandString(GenTree* tree, LsraTupleDumpMode mode, char* operandString, unsigned operandStringLength); void lsraDispNode(GenTree* tree, LsraTupleDumpMode mode, bool hasDest); void DumpOperandDefs( @@ -1497,7 +1544,7 @@ class LinearScan : public LinearScanInterface AllRegsMask lastDumpedRegisters; AllRegsMask registersToDump; int lastUsedRegNumIndex; - bool shouldDumpReg(regNumber regNum) + bool shouldDumpReg(regNumber regNum) { return registersToDump.IsRegNumInMask(regNum); } @@ -1518,29 +1565,54 @@ class LinearScan : public LinearScanInterface void dumpIntervalName(Interval* interval); // Events during the allocation phase that cause some dump output - enum LsraDumpEvent{ + enum LsraDumpEvent + { // Conflicting def/use - LSRA_EVENT_DEFUSE_CONFLICT, LSRA_EVENT_DEFUSE_FIXED_DELAY_USE, LSRA_EVENT_DEFUSE_CASE1, LSRA_EVENT_DEFUSE_CASE2, - LSRA_EVENT_DEFUSE_CASE3, LSRA_EVENT_DEFUSE_CASE4, LSRA_EVENT_DEFUSE_CASE5, LSRA_EVENT_DEFUSE_CASE6, + LSRA_EVENT_DEFUSE_CONFLICT, + LSRA_EVENT_DEFUSE_FIXED_DELAY_USE, + LSRA_EVENT_DEFUSE_CASE1, + LSRA_EVENT_DEFUSE_CASE2, + LSRA_EVENT_DEFUSE_CASE3, + LSRA_EVENT_DEFUSE_CASE4, + LSRA_EVENT_DEFUSE_CASE5, + LSRA_EVENT_DEFUSE_CASE6, // Spilling - LSRA_EVENT_SPILL, LSRA_EVENT_SPILL_EXTENDED_LIFETIME, LSRA_EVENT_RESTORE_PREVIOUS_INTERVAL, - LSRA_EVENT_RESTORE_PREVIOUS_INTERVAL_AFTER_SPILL, LSRA_EVENT_DONE_KILL_GC_REFS, LSRA_EVENT_NO_GC_KILLS, + LSRA_EVENT_SPILL, + LSRA_EVENT_SPILL_EXTENDED_LIFETIME, + LSRA_EVENT_RESTORE_PREVIOUS_INTERVAL, + LSRA_EVENT_RESTORE_PREVIOUS_INTERVAL_AFTER_SPILL, + LSRA_EVENT_DONE_KILL_GC_REFS, + LSRA_EVENT_NO_GC_KILLS, // Block boundaries - LSRA_EVENT_START_BB, LSRA_EVENT_END_BB, + LSRA_EVENT_START_BB, + LSRA_EVENT_END_BB, // Miscellaneous - LSRA_EVENT_FREE_REGS, LSRA_EVENT_UPPER_VECTOR_SAVE, LSRA_EVENT_UPPER_VECTOR_RESTORE, + LSRA_EVENT_FREE_REGS, + LSRA_EVENT_UPPER_VECTOR_SAVE, + LSRA_EVENT_UPPER_VECTOR_RESTORE, // Characteristics of the current RefPosition LSRA_EVENT_INCREMENT_RANGE_END, // ??? - LSRA_EVENT_LAST_USE, LSRA_EVENT_LAST_USE_DELAYED, LSRA_EVENT_NEEDS_NEW_REG, + LSRA_EVENT_LAST_USE, + LSRA_EVENT_LAST_USE_DELAYED, + LSRA_EVENT_NEEDS_NEW_REG, // Allocation decisions - LSRA_EVENT_FIXED_REG, LSRA_EVENT_EXP_USE, LSRA_EVENT_ZERO_REF, LSRA_EVENT_NO_ENTRY_REG_ALLOCATED, - LSRA_EVENT_KEPT_ALLOCATION, LSRA_EVENT_COPY_REG, LSRA_EVENT_MOVE_REG, LSRA_EVENT_ALLOC_REG, - LSRA_EVENT_NO_REG_ALLOCATED, LSRA_EVENT_RELOAD, LSRA_EVENT_SPECIAL_PUTARG, LSRA_EVENT_REUSE_REG, + LSRA_EVENT_FIXED_REG, + LSRA_EVENT_EXP_USE, + LSRA_EVENT_ZERO_REF, + LSRA_EVENT_NO_ENTRY_REG_ALLOCATED, + LSRA_EVENT_KEPT_ALLOCATION, + LSRA_EVENT_COPY_REG, + LSRA_EVENT_MOVE_REG, + LSRA_EVENT_ALLOC_REG, + LSRA_EVENT_NO_REG_ALLOCATED, + LSRA_EVENT_RELOAD, + LSRA_EVENT_SPECIAL_PUTARG, + LSRA_EVENT_REUSE_REG, }; void dumpLsraAllocationEvent(LsraDumpEvent event, Interval* interval = nullptr, @@ -1553,14 +1625,14 @@ class LinearScan : public LinearScanInterface #if TRACK_LSRA_STATS unsigned regCandidateVarCount; - void updateLsraStat(LsraStat stat, unsigned currentBBNum); - void dumpLsraStats(FILE* file); + void updateLsraStat(LsraStat stat, unsigned currentBBNum); + void dumpLsraStats(FILE* file); LsraStat getLsraStatFromScore(RegisterScore registerScore); LsraStat firstRegSelStat = STAT_FREE; public: - virtual void dumpLsraStatsCsv(FILE* file); - virtual void dumpLsraStatsSummary(FILE* file); + virtual void dumpLsraStatsCsv(FILE* file); + virtual void dumpLsraStatsSummary(FILE* file); static const char* getStatName(unsigned stat); #define INTRACK_STATS(x) x @@ -1596,7 +1668,7 @@ class LinearScan : public LinearScanInterface // Set of blocks that have been visited. BlockSet bbVisitedSet; - void markBlockVisited(BasicBlock* block) + void markBlockVisited(BasicBlock* block) { BlockSetOps::AddElemD(compiler, bbVisitedSet, block->bbNum); } @@ -1623,17 +1695,17 @@ class LinearScan : public LinearScanInterface BasicBlock** blockSequence; // The verifiedAllBBs flag indicates whether we have verified that all BBs have been // included in the blockSeuqence above, during setBlockSequence(). - bool verifiedAllBBs; - void setBlockSequence(); - int compareBlocksForSequencing(BasicBlock* block1, BasicBlock* block2, bool useBlockWeights); + bool verifiedAllBBs; + void setBlockSequence(); + int compareBlocksForSequencing(BasicBlock* block1, BasicBlock* block2, bool useBlockWeights); BasicBlockList* blockSequenceWorkList; bool blockSequencingDone; #ifdef DEBUG // LSRA must not change number of blocks and blockEpoch that it initializes at start. unsigned blockEpoch; #endif // DEBUG - void addToBlockSequenceWorkList(BlockSet sequencedBlockSet, BasicBlock* block, BlockSet& predSet); - void removeFromBlockSequenceWorkList(BasicBlockList* listNode, BasicBlockList* prevNode); + void addToBlockSequenceWorkList(BlockSet sequencedBlockSet, BasicBlock* block, BlockSet& predSet); + void removeFromBlockSequenceWorkList(BasicBlockList* listNode, BasicBlockList* prevNode); BasicBlock* getNextCandidateFromWorkList(); // Indicates whether the allocation pass has been completed. @@ -1735,7 +1807,7 @@ class LinearScan : public LinearScanInterface #if defined(TARGET_AMD64) static const var_types LargeVectorSaveType = TYP_SIMD16; #elif defined(TARGET_ARM64) - static const var_types LargeVectorSaveType = TYP_DOUBLE; + static const var_types LargeVectorSaveType = TYP_DOUBLE; #endif // !defined(TARGET_AMD64) && !defined(TARGET_ARM64) // Set of large vector (TYP_SIMD32 on AVX) variables. VARSET_TP largeVectorVars; @@ -1801,10 +1873,10 @@ class LinearScan : public LinearScanInterface void clearSpillCost(regNumber reg, var_types regType); void updateSpillCost(regNumber reg, Interval* interval); - FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, - regMaskOnlyOne regsBusy, - AllRegsMask* regsToFree, - AllRegsMask* delayRegsToFree, + FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, + regMaskOnlyOne regsBusy, + AllRegsMask* regsToFree, + AllRegsMask* delayRegsToFree, RegisterType regType DEBUG_ARG(Interval* interval) DEBUG_ARG(regNumber assignedReg)); @@ -1831,7 +1903,7 @@ class LinearScan : public LinearScanInterface regMaskOnlyOne getMatchingConstants(regMaskOnlyOne mask, Interval* currentInterval, RefPosition* refPosition); LsraLocation nextFixedRef[REG_COUNT]; - void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); + void updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPosition); LsraLocation getNextFixedRef(regNumber regNum, var_types regType) { LsraLocation loc = nextFixedRef[regNum]; @@ -1949,11 +2021,11 @@ class LinearScan : public LinearScanInterface bool checkContainedOrCandidateLclVar(GenTreeLclVar* lclNode); RefPosition* BuildUse(GenTree* operand, regMaskOnlyOne candidates = RBM_NONE, int multiRegIdx = 0); - void setDelayFree(RefPosition* use); - int BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates = RBM_NONE); - int BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates); + void setDelayFree(RefPosition* use); + int BuildBinaryUses(GenTreeOp* node, regMaskOnlyOne candidates = RBM_NONE); + int BuildCastUses(GenTreeCast* cast, regMaskOnlyOne candidates); #ifdef TARGET_XARCH - int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates = RBM_NONE); + int BuildRMWUses(GenTree* node, GenTree* op1, GenTree* op2, regMaskOnlyOne candidates = RBM_NONE); inline regMaskFloat BuildEvexIncompatibleMask(GenTree* tree); #endif // !TARGET_XARCH int BuildSelect(GenTreeOp* select); @@ -1965,20 +2037,20 @@ class LinearScan : public LinearScanInterface void getTgtPrefOperands(GenTree* tree, GenTree* op1, GenTree* op2, bool* prefOp1, bool* prefOp2); bool supportsSpecialPutArg(); - int BuildSimple(GenTree* tree); - int BuildOperandUses(GenTree* node, regMaskOnlyOne candidates = RBM_NONE); - void AddDelayFreeUses(RefPosition* refPosition, GenTree* rmwNode); - int BuildDelayFreeUses(GenTree* node, - GenTree* rmwNode = nullptr, - regMaskOnlyOne candidates = RBM_NONE, - RefPosition** useRefPosition = nullptr); - int BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates = RBM_NONE); - int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); - void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); + int BuildSimple(GenTree* tree); + int BuildOperandUses(GenTree* node, regMaskOnlyOne candidates = RBM_NONE); + void AddDelayFreeUses(RefPosition* refPosition, GenTree* rmwNode); + int BuildDelayFreeUses(GenTree* node, + GenTree* rmwNode = nullptr, + regMaskOnlyOne candidates = RBM_NONE, + RefPosition** useRefPosition = nullptr); + int BuildIndirUses(GenTreeIndir* indirTree, regMaskOnlyOne candidates = RBM_NONE); + int BuildAddrUses(GenTree* addr, regMaskOnlyOne candidates = RBM_NONE); + void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); - void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); - void BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates); - void BuildKills(GenTree* tree, AllRegsMask killMask); + void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); + void BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates); + void BuildKills(GenTree* tree, AllRegsMask killMask); #ifdef TARGET_ARMARCH void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask); #else @@ -1995,24 +2067,24 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARM int BuildShiftLongCarry(GenTree* tree); #endif - int BuildPutArgReg(GenTreeUnOp* node); - int BuildCall(GenTreeCall* call); - int BuildCmp(GenTree* tree); - int BuildCmpOperands(GenTree* tree); - int BuildBlockStore(GenTreeBlk* blkNode); - int BuildModDiv(GenTree* tree); - int BuildIntrinsic(GenTree* tree); + int BuildPutArgReg(GenTreeUnOp* node); + int BuildCall(GenTreeCall* call); + int BuildCmp(GenTree* tree); + int BuildCmpOperands(GenTree* tree); + int BuildBlockStore(GenTreeBlk* blkNode); + int BuildModDiv(GenTree* tree); + int BuildIntrinsic(GenTree* tree); void BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, LclVarDsc* varDsc, RefPosition* singleUseRef, int index); - int BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc); - int BuildStoreLoc(GenTreeLclVarCommon* tree); - int BuildIndir(GenTreeIndir* indirTree); - int BuildGCWriteBarrier(GenTree* tree); - int BuildCast(GenTreeCast* cast); + int BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc); + int BuildStoreLoc(GenTreeLclVarCommon* tree); + int BuildIndir(GenTreeIndir* indirTree); + int BuildGCWriteBarrier(GenTree* tree); + int BuildCast(GenTreeCast* cast); #if defined(TARGET_XARCH) // returns true if the tree can use the read-modify-write memory instruction form bool isRMWRegOper(GenTree* tree); - int BuildMul(GenTree* tree); + int BuildMul(GenTree* tree); void SetContainsAVXFlags(unsigned sizeOfSIMDVector = 0); #endif // defined(TARGET_XARCH) @@ -2041,7 +2113,7 @@ class LinearScan : public LinearScanInterface #ifdef FEATURE_HW_INTRINSICS int BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCount); #ifdef TARGET_ARM64 - int BuildConsecutiveRegistersForUse(GenTree* treeNode, GenTree* rmwNode = nullptr); + int BuildConsecutiveRegistersForUse(GenTree* treeNode, GenTree* rmwNode = nullptr); void BuildConsecutiveRegistersForDef(GenTree* treeNode, int fieldCount); #endif // TARGET_ARM64 #endif // FEATURE_HW_INTRINSICS @@ -2056,7 +2128,7 @@ class LinearScan : public LinearScanInterface #endif // FEATURE_ARG_SPLIT int BuildLclHeap(GenTree* tree); - //#if defined(TARGET_AMD64) + // #if defined(TARGET_AMD64) regMaskFloat rbmAllFloat; regMaskFloat rbmFltCalleeTrash; @@ -2068,7 +2140,7 @@ class LinearScan : public LinearScanInterface { return this->rbmFltCalleeTrash; } -//#endif // TARGET_AMD64 + // #endif // TARGET_AMD64 #if defined(TARGET_XARCH) regMaskPredicate rbmAllMask; @@ -2513,8 +2585,8 @@ class RefPosition // we need an explicit move. // - copyReg and moveReg must not exist with each other. - unsigned char reload : 1; - unsigned char spillAfter : 1; + unsigned char reload : 1; + unsigned char spillAfter : 1; unsigned char singleDefSpill : 1; unsigned char writeThru : 1; // true if this var is defined in a register and also spilled. spillAfter must NOT be // set. @@ -2522,7 +2594,7 @@ class RefPosition unsigned char copyReg : 1; unsigned char moveReg : 1; // true if this var is moved to a new register - unsigned char isPhysRegRef : 1; // true if 'referent' points of a RegRecord, false if it points to an Interval + unsigned char isPhysRegRef : 1; // true if 'referent' points of a RegRecord, false if it points to an Interval unsigned char isFixedRegRef : 1; unsigned char isLocalDefUse : 1; @@ -2564,9 +2636,9 @@ class RefPosition GenTree* buildNode; #endif // DEBUG - RefPosition(unsigned int bbNum, - LsraLocation nodeLocation, - GenTree* treeNode, + RefPosition(unsigned int bbNum, + LsraLocation nodeLocation, + GenTree* treeNode, RefType refType DEBUG_ARG(GenTree* buildNode)) : referent(nullptr) , nextRefPosition(nullptr) @@ -2635,8 +2707,8 @@ class RefPosition RegisterType getRegisterType() { - //TODO: The type should be just on `Referenceable` that refposition can extract - // without having to deal with the check. + // TODO: The type should be just on `Referenceable` that refposition can extract + // without having to deal with the check. return isIntervalRef() ? getInterval()->registerType : getReg()->registerType; } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 2b5a0f579dbaba..b451f6062ceb03 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -224,23 +224,23 @@ int LinearScan::BuildCall(GenTreeCall* call) else #endif // TARGET_ARM if (hasMultiRegRetVal) - { - assert(retTypeDesc != nullptr); - dstReturnCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); - assert((int)dstReturnCandidates.Count() == dstCount); - } - else if (varTypeUsesFloatArgReg(registerType)) - { - dstCandidates = RBM_FLOATRET; - } - else if (registerType == TYP_LONG) - { - dstCandidates = RBM_LNGRET; - } - else - { - dstCandidates = RBM_INTRET; - } + { + assert(retTypeDesc != nullptr); + dstReturnCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + assert((int)dstReturnCandidates.Count() == dstCount); + } + else if (varTypeUsesFloatArgReg(registerType)) + { + dstCandidates = RBM_FLOATRET; + } + else if (registerType == TYP_LONG) + { + dstCandidates = RBM_LNGRET; + } + else + { + dstCandidates = RBM_INTRET; + } // First, count reg args // Each register argument corresponds to one source. diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index f809b6caf794c9..2e718dc261f972 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -78,7 +78,8 @@ RefInfoListNode* RefInfoList::removeListNode(GenTree* node, unsigned multiRegIdx // compiler - The compiler context. // preallocate - The number of nodes to preallocate. // -RefInfoListNodePool::RefInfoListNodePool(Compiler* compiler, unsigned preallocate) : m_compiler(compiler) +RefInfoListNodePool::RefInfoListNodePool(Compiler* compiler, unsigned preallocate) + : m_compiler(compiler) { if (preallocate > 0) { @@ -1163,9 +1164,9 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE if (varTypeIsFloating(varDsc) && !VarSetOps::IsMember(compiler, fpCalleeSaveCandidateVars, varIndex)) - { - continue; - } + { + continue; + } Interval* interval = getIntervalForLocalVar(varIndex); regMaskOnlyOne regsKillMask = killMask.GetRegMaskForType(interval->registerType); const bool isCallKill = @@ -1483,7 +1484,7 @@ Interval* LinearScan::getUpperVectorInterval(unsigned varIndex) // But we will use as a proxy any node that kills floating point registers. // (Note that some calls are masquerading as other nodes at this point so we can't just check for calls.) // -void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, +void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation currentLoc DEBUG_ARG(regMaskFloat fpCalleeKillSet)) { assert(compiler->IsFloatRegMask(fpCalleeKillSet)); @@ -2233,7 +2234,7 @@ template void LinearScan::buildIntervals(); // which we will do register allocation. // template -void LinearScan::buildIntervals() +void LinearScan::buildIntervals() { BasicBlock* block; @@ -2489,7 +2490,7 @@ void LinearScan::buildIntervals() assert(isCandidateVar(varDsc)); Interval* interval = getIntervalForLocalVar(varIndex); RefPosition* pos = newRefPosition(interval, currentLoc, RefTypeDummyDef, nullptr, - allRegs(interval->registerType)); + allRegs(interval->registerType)); pos->setRegOptional(true); } JITDUMP("Finished creating dummy definitions\n\n"); @@ -3077,7 +3078,8 @@ void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandi { // In case of multi-reg call node, we have to query the i'th position return register. // For all other cases of multi-reg definitions, the registers must be in sequential order. - regNumber thisReg = tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, tree->AsCall()->GetUnmanagedCallConv()); + regNumber thisReg = + tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, tree->AsCall()->GetUnmanagedCallConv()); assert(dstCandidates.IsRegNumInMask(thisReg)); dstCandidates.RemoveRegNumFromMask(thisReg); @@ -3152,8 +3154,8 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) // if ((killMask.IsFloatMaskPresent(compiler, RBM_FLT_CALLEE_TRASH))) { - buildUpperVectorSaveRefPositions(tree, - currentLoc + 1 DEBUG_ARG((killMask.floatRegs(compiler) & RBM_FLT_CALLEE_TRASH))); + buildUpperVectorSaveRefPositions(tree, currentLoc + 1 DEBUG_ARG((killMask.floatRegs(compiler) & + RBM_FLT_CALLEE_TRASH))); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } @@ -4071,114 +4073,114 @@ int LinearScan::BuildReturn(GenTree* tree) else #endif // !defined(TARGET_64BIT) if ((tree->TypeGet() != TYP_VOID) && !op1->isContained()) - { - regMaskOnlyOne useCandidates = RBM_NONE; + { + regMaskOnlyOne useCandidates = RBM_NONE; #if FEATURE_MULTIREG_RET #ifdef TARGET_ARM64 - if (varTypeIsSIMD(tree) && !op1->IsMultiRegLclVar()) - { - BuildUse(op1, RBM_DOUBLERET); - return 1; - } -#endif // TARGET_ARM64 - - if (varTypeIsStruct(tree)) - { - // op1 has to be either a lclvar or a multi-reg returning call - if ((op1->OperGet() == GT_LCL_VAR) && !op1->IsMultiRegLclVar()) + if (varTypeIsSIMD(tree) && !op1->IsMultiRegLclVar()) { - BuildUse(op1, RBM_NONE); + BuildUse(op1, RBM_DOUBLERET); + return 1; } - else +#endif // TARGET_ARM64 + + if (varTypeIsStruct(tree)) { - noway_assert(op1->IsMultiRegCall() || (op1->IsMultiRegLclVar() && compiler->lvaEnregMultiRegVars)); + // op1 has to be either a lclvar or a multi-reg returning call + if ((op1->OperGet() == GT_LCL_VAR) && !op1->IsMultiRegLclVar()) + { + BuildUse(op1, RBM_NONE); + } + else + { + noway_assert(op1->IsMultiRegCall() || (op1->IsMultiRegLclVar() && compiler->lvaEnregMultiRegVars)); - ReturnTypeDesc retTypeDesc = compiler->compRetTypeDesc; - const int srcCount = retTypeDesc.GetReturnRegCount(); - assert(op1->GetMultiRegCount(compiler) == static_cast(srcCount)); + ReturnTypeDesc retTypeDesc = compiler->compRetTypeDesc; + const int srcCount = retTypeDesc.GetReturnRegCount(); + assert(op1->GetMultiRegCount(compiler) == static_cast(srcCount)); - // For any source that's coming from a different register file, we need to ensure that - // we reserve the specific ABI register we need. - bool hasMismatchedRegTypes = false; - if (op1->IsMultiRegLclVar()) - { - for (int i = 0; i < srcCount; i++) + // For any source that's coming from a different register file, we need to ensure that + // we reserve the specific ABI register we need. + bool hasMismatchedRegTypes = false; + if (op1->IsMultiRegLclVar()) { - RegisterType srcType = regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)); - RegisterType dstType = regType(retTypeDesc.GetReturnRegType(i)); - if (srcType != dstType) + for (int i = 0; i < srcCount; i++) { - hasMismatchedRegTypes = true; - regMaskOnlyOne dstRegMask = - genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); - - if (varTypeUsesIntReg(dstType)) + RegisterType srcType = regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)); + RegisterType dstType = regType(retTypeDesc.GetReturnRegType(i)); + if (srcType != dstType) { - buildInternalIntRegisterDefForNode(tree, dstRegMask); - } + hasMismatchedRegTypes = true; + regMaskOnlyOne dstRegMask = + genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); + + if (varTypeUsesIntReg(dstType)) + { + buildInternalIntRegisterDefForNode(tree, dstRegMask); + } #ifdef FEATURE_MASKED_HW_INTRINSICS - else if (varTypeUsesMaskReg(dstType)) - { - buildInternalMaskRegisterDefForNode(tree, dstRegMask); - } + else if (varTypeUsesMaskReg(dstType)) + { + buildInternalMaskRegisterDefForNode(tree, dstRegMask); + } #endif // FEATURE_MASKED_HW_INTRINSICS - else - { - assert(varTypeUsesFloatReg(dstType)); - buildInternalFloatRegisterDefForNode(tree, dstRegMask); + else + { + assert(varTypeUsesFloatReg(dstType)); + buildInternalFloatRegisterDefForNode(tree, dstRegMask); + } } } } - } - for (int i = 0; i < srcCount; i++) - { - // We will build uses of the type of the operand registers/fields, and the codegen - // for return will move as needed. - if (!hasMismatchedRegTypes || (regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)) == - regType(retTypeDesc.GetReturnRegType(i)))) + for (int i = 0; i < srcCount; i++) { - BuildUse(op1, genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)), i); + // We will build uses of the type of the operand registers/fields, and the codegen + // for return will move as needed. + if (!hasMismatchedRegTypes || (regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)) == + regType(retTypeDesc.GetReturnRegType(i)))) + { + BuildUse(op1, genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)), i); + } + else + { + BuildUse(op1, RBM_NONE, i); + } } - else + if (hasMismatchedRegTypes) { - BuildUse(op1, RBM_NONE, i); + buildInternalRegisterUses(); } + return srcCount; } - if (hasMismatchedRegTypes) - { - buildInternalRegisterUses(); - } - return srcCount; } - } - else + else #endif // FEATURE_MULTIREG_RET - { - // Non-struct type return - determine useCandidates - switch (tree->TypeGet()) { - case TYP_VOID: - useCandidates = RBM_NONE; - break; - case TYP_FLOAT: - useCandidates = RBM_FLOATRET; - break; - case TYP_DOUBLE: - // We ONLY want the valid double register in the RBM_DOUBLERET mask. - useCandidates = (RBM_DOUBLERET & RBM_ALLDOUBLE); - break; - case TYP_LONG: - useCandidates = RBM_LNGRET; - break; - default: - useCandidates = RBM_INTRET; - break; + // Non-struct type return - determine useCandidates + switch (tree->TypeGet()) + { + case TYP_VOID: + useCandidates = RBM_NONE; + break; + case TYP_FLOAT: + useCandidates = RBM_FLOATRET; + break; + case TYP_DOUBLE: + // We ONLY want the valid double register in the RBM_DOUBLERET mask. + useCandidates = (RBM_DOUBLERET & RBM_ALLDOUBLE); + break; + case TYP_LONG: + useCandidates = RBM_LNGRET; + break; + default: + useCandidates = RBM_INTRET; + break; + } + BuildUse(op1, useCandidates); + return 1; } - BuildUse(op1, useCandidates); - return 1; } - } // No kills or defs. return 0; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index b0628d5a22dc6a..0736138a5656b0 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1186,33 +1186,33 @@ int LinearScan::BuildCall(GenTreeCall* call) else #endif // TARGET_X86 if (hasMultiRegRetVal) - { - assert(retTypeDesc != nullptr); - dstReturnCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); - assert((int)dstReturnCandidates.Count() == dstCount); - } - else if (varTypeUsesFloatReg(registerType)) - { + { + assert(retTypeDesc != nullptr); + dstReturnCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + assert((int)dstReturnCandidates.Count() == dstCount); + } + else if (varTypeUsesFloatReg(registerType)) + { #ifdef TARGET_X86 - // The return value will be on the X87 stack, and we will need to move it. - dstCandidates = allRegs(registerType); + // The return value will be on the X87 stack, and we will need to move it. + dstCandidates = allRegs(registerType); #else // !TARGET_X86 dstCandidates = RBM_FLOATRET; #endif // !TARGET_X86 - } - else - { - assert(varTypeUsesIntReg(registerType)); - - if (registerType == TYP_LONG) - { - dstCandidates = RBM_LNGRET; } else { - dstCandidates = RBM_INTRET; + assert(varTypeUsesIntReg(registerType)); + + if (registerType == TYP_LONG) + { + dstCandidates = RBM_LNGRET; + } + else + { + dstCandidates = RBM_INTRET; + } } - } // number of args to a call = // callRegArgs + (callargs - placeholders, setup, etc) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 7334b3076e38e4..077153ff2e1399 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -344,7 +344,7 @@ GenTree* Compiler::fgMorphExpandCast(GenTreeCast* tree) // x86: src = float, dst = uint32/int64/uint64 or overflow conversion. && (tree->gtOverflow() || varTypeIsLong(dstType) || (dstType == TYP_UINT)) #endif - ) + ) { oper = gtNewCastNode(TYP_DOUBLE, oper, false, TYP_DOUBLE); } @@ -436,7 +436,7 @@ GenTree* Compiler::fgMorphExpandCast(GenTreeCast* tree) #ifdef TARGET_ARM && !varTypeIsLong(oper->AsCast()->CastOp()) #endif - ) + ) { oper->gtType = TYP_FLOAT; oper->CastToType() = TYP_FLOAT; @@ -2109,8 +2109,8 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call unsigned numArgs = CountArgs(); #ifdef TARGET_X86 -// Compute the maximum number of arguments that can be passed in registers. -// For X86 we handle the varargs and unmanaged calling conventions + // Compute the maximum number of arguments that can be passed in registers. + // For X86 we handle the varargs and unmanaged calling conventions #ifndef UNIX_X86_ABI if (call->gtFlags & GTF_CALL_POP_ARGS) @@ -2513,7 +2513,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call #elif defined(TARGET_X86) || (isStructArg && comp->isTrivialPointerSizedStruct(argSigClass)) #endif - ) + ) { #ifdef TARGET_ARM if (passUsingFloatRegs) @@ -2945,7 +2945,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // we skip the corresponding floating point register argument intArgRegNum = min(intArgRegNum + size, MAX_REG_ARG); #endif // WINDOWS_AMD64_ABI - // No supported architecture supports partial structs using float registers. + // No supported architecture supports partial structs using float registers. assert(fltArgRegNum <= MAX_FLOAT_REG_ARG); } else @@ -3251,12 +3251,12 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) assert(arg.AbiInfo.GetStackSlotsNumber() == 1); makeOutArgCopy = true; #else // UNIX_AMD64_ABI - // On Unix, structs are always passed by value. - // We only need a copy if we have one of the following: - // - The sizes don't match for a non-lclVar argument. - // - We have a known struct type (e.g. SIMD) that requires multiple registers. - // TODO-Amd64-Unix-Throughput: We don't need to keep the structDesc in the argEntry if it's not - // actually passed in registers. + // On Unix, structs are always passed by value. + // We only need a copy if we have one of the following: + // - The sizes don't match for a non-lclVar argument. + // - We have a known struct type (e.g. SIMD) that requires multiple registers. + // TODO-Amd64-Unix-Throughput: We don't need to keep the structDesc in the argEntry if it's not + // actually passed in registers. if (arg.AbiInfo.IsPassedInRegisters()) { if (argObj->OperIs(GT_BLK)) @@ -3341,9 +3341,9 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) } #ifdef TARGET_AMD64 else if (!argObj->OperIs(GT_LCL_VAR) || !argObj->TypeIs(TYP_SIMD8)) // Handled by lowering. -#else // !TARGET_ARM64 +#else // !TARGET_ARM64 else -#endif // !TARGET_ARM64 +#endif // !TARGET_ARM64 { // TODO-CQ: perform this transformation in lowering instead of here and // avoid marking enregisterable structs DNER. @@ -3974,18 +3974,18 @@ void Compiler::fgMakeOutgoingStructArgCopy(GenTreeCall* call, CallArg* arg) if (!opts.MinOpts()) { found = ForEachHbvBitSet(*fgAvailableOutgoingArgTemps, [&](indexType lclNum) { - LclVarDsc* varDsc = lvaGetDesc((unsigned)lclNum); - ClassLayout* layout = varDsc->GetLayout(); - if (!layout->IsBlockLayout() && (layout->GetClassHandle() == copyBlkClass)) - { - tmp = (unsigned)lclNum; - JITDUMP("reusing outgoing struct arg V%02u\n", tmp); - fgAvailableOutgoingArgTemps->clearBit(lclNum); - return HbvWalk::Abort; - } + LclVarDsc* varDsc = lvaGetDesc((unsigned)lclNum); + ClassLayout* layout = varDsc->GetLayout(); + if (!layout->IsBlockLayout() && (layout->GetClassHandle() == copyBlkClass)) + { + tmp = (unsigned)lclNum; + JITDUMP("reusing outgoing struct arg V%02u\n", tmp); + fgAvailableOutgoingArgTemps->clearBit(lclNum); + return HbvWalk::Abort; + } - return HbvWalk::Continue; - }) == HbvWalk::Abort; + return HbvWalk::Continue; + }) == HbvWalk::Abort; } // Create the CopyBlk tree and insert it. @@ -4028,7 +4028,7 @@ void Compiler::fgMakeOutgoingStructArgCopy(GenTreeCall* call, CallArg* arg) GenTree* argNode = call->gtArgs.MakeTmpArgNode(this, arg); // Change the expression to "(tmp=val),tmp" - argNode = gtNewOperNode(GT_COMMA, argNode->TypeGet(), copyBlk, argNode); + argNode = gtNewOperNode(GT_COMMA, argNode->TypeGet(), copyBlk, argNode); #endif // !FEATURE_FIXED_OUT_ARGS @@ -4529,7 +4529,7 @@ GenTree* Compiler::fgMorphLeafLocal(GenTreeLclVarCommon* lclNode) #if FEATURE_IMPLICIT_BYREFS || varDsc->lvIsLastUseCopyOmissionCandidate #endif - ) + ) { lclNode->gtFlags |= GTF_GLOB_REF; } @@ -4603,7 +4603,7 @@ GenTree* Compiler::fgMorphExpandStackArgForVarArgs(GenTreeLclVarCommon* lclNode) { GenTree* data = lclNode->Data(); argNode = lclNode->TypeIs(TYP_STRUCT) ? gtNewStoreBlkNode(lclNode->GetLayout(this), argAddr, data) - : gtNewStoreIndNode(lclNode->TypeGet(), argAddr, data)->AsIndir(); + : gtNewStoreIndNode(lclNode->TypeGet(), argAddr, data)->AsIndir(); } else if (lclNode->OperIsLocalRead()) { @@ -6326,7 +6326,10 @@ void Compiler::fgValidateIRForTailCall(GenTreeCall* call) }; TailCallIRValidatorVisitor(Compiler* comp, GenTreeCall* tailcall) - : GenTreeVisitor(comp), m_tailcall(tailcall), m_lclNum(BAD_VAR_NUM), m_active(false) + : GenTreeVisitor(comp) + , m_tailcall(tailcall) + , m_lclNum(BAD_VAR_NUM) + , m_active(false) { } @@ -7912,7 +7915,7 @@ GenTree* Compiler::fgExpandVirtualVtableCallTarget(GenTreeCall* call) // [tmp + vtabOffsOfIndirection] GenTree* tmpTree1 = gtNewOperNode(GT_ADD, TYP_I_IMPL, gtNewLclvNode(varNum1, TYP_I_IMPL), gtNewIconNode(vtabOffsOfIndirection, TYP_I_IMPL)); - tmpTree1 = gtNewIndir(TYP_I_IMPL, tmpTree1, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); + tmpTree1 = gtNewIndir(TYP_I_IMPL, tmpTree1, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); // var1 + vtabOffsOfIndirection + vtabOffsAfterIndirection GenTree* tmpTree2 = @@ -8330,7 +8333,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA #if FEATURE_IMPLICIT_BYREFS || lclDsc->lvIsLastUseCopyOmissionCandidate #endif - ) + ) { tree->AddAllEffectsFlags(GTF_GLOB_REF); } @@ -8575,8 +8578,8 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA // Note for TARGET_ARMARCH we don't have a remainder instruction, so we don't do this optimization // #else // TARGET_XARCH - // If this is an unsigned long mod with a constant divisor, - // then don't morph to a helper call - it can be done faster inline using idiv. + // If this is an unsigned long mod with a constant divisor, + // then don't morph to a helper call - it can be done faster inline using idiv. noway_assert(op2); if ((typ == TYP_LONG) && opts.OptimizationEnabled()) @@ -11554,8 +11557,8 @@ GenTree* Compiler::fgMorphRetInd(GenTreeUnOp* ret) #if defined(TARGET_64BIT) bool canFold = (indSize == lclVarSize); #else // !TARGET_64BIT - // TODO: improve 32 bit targets handling for LONG returns if necessary, nowadays we do not support `BITCAST - // long<->double` there. + // TODO: improve 32 bit targets handling for LONG returns if necessary, nowadays we do not support `BITCAST + // long<->double` there. bool canFold = (indSize == lclVarSize) && (lclVarSize <= REGSIZE_BYTES); #endif @@ -12580,12 +12583,12 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) bool optAssertionPropDone = false; -/*------------------------------------------------------------------------- - * fgMorphTree() can potentially replace a tree with another, and the - * caller has to store the return value correctly. - * Turn this on to always make copy of "tree" here to shake out - * hidden/unupdated references. - */ + /*------------------------------------------------------------------------- + * fgMorphTree() can potentially replace a tree with another, and the + * caller has to store the return value correctly. + * Turn this on to always make copy of "tree" here to shake out + * hidden/unupdated references. + */ #ifdef DEBUG @@ -13508,7 +13511,8 @@ void Compiler::fgMorphStmtBlockOps(BasicBlock* block, Statement* stmt) DoPostOrder = true, }; - Visitor(Compiler* comp) : GenTreeVisitor(comp) + Visitor(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -14984,7 +14988,8 @@ PhaseStatus Compiler::fgMarkImplicitByRefCopyOmissionCandidates() UseExecutionOrder = true, }; - Visitor(Compiler* comp) : GenTreeVisitor(comp) + Visitor(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -15196,9 +15201,9 @@ PhaseStatus Compiler::fgRetypeImplicitByRefArgs() { // Insert IR that initializes the temp from the parameter. fgEnsureFirstBBisScratch(); - GenTree* addr = gtNewLclvNode(lclNum, TYP_BYREF); - GenTree* data = (varDsc->TypeGet() == TYP_STRUCT) ? gtNewBlkIndir(varDsc->GetLayout(), addr) - : gtNewIndir(varDsc->TypeGet(), addr); + GenTree* addr = gtNewLclvNode(lclNum, TYP_BYREF); + GenTree* data = (varDsc->TypeGet() == TYP_STRUCT) ? gtNewBlkIndir(varDsc->GetLayout(), addr) + : gtNewIndir(varDsc->TypeGet(), addr); GenTree* store = gtNewStoreLclVarNode(newLclNum, data); fgNewStmtAtBeg(fgFirstBB, store); } @@ -15569,7 +15574,10 @@ bool Compiler::fgMorphArrayOpsStmt(MorphMDArrayTempCache* pTempCache, BasicBlock }; MorphMDArrayVisitor(Compiler* compiler, BasicBlock* block, MorphMDArrayTempCache* pTempCache) - : GenTreeVisitor(compiler), m_changed(false), m_block(block), m_pTempCache(pTempCache) + : GenTreeVisitor(compiler) + , m_changed(false) + , m_block(block) + , m_pTempCache(pTempCache) { } diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index d7fa5821eb9dbc..84c30e64621a2f 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -92,7 +92,8 @@ GenTree* MorphInitBlockHelper::MorphInitBlock(Compiler* comp, GenTree* tree) // Most class members are initialized via in-class member initializers. // MorphInitBlockHelper::MorphInitBlockHelper(Compiler* comp, GenTree* store, bool initBlock = true) - : m_comp(comp), m_initBlock(initBlock) + : m_comp(comp) + , m_initBlock(initBlock) { assert(store->OperIsStore()); assert((m_initBlock == store->OperIsInitBlkOp()) && (!m_initBlock == store->OperIsCopyBlkOp())); @@ -530,8 +531,8 @@ GenTree* MorphInitBlockHelper::EliminateCommas(GenTree** commaPool) { *commaPool = nullptr; - GenTree* sideEffects = nullptr; - auto addSideEffect = [&sideEffects](GenTree* sideEff) { + GenTree* sideEffects = nullptr; + auto addSideEffect = [&sideEffects](GenTree* sideEff) { sideEff->gtNext = sideEffects; sideEffects = sideEff; }; @@ -645,7 +646,8 @@ GenTree* MorphCopyBlockHelper::MorphCopyBlock(Compiler* comp, GenTree* tree) // Notes: // Most class members are initialized via in-class member initializers. // -MorphCopyBlockHelper::MorphCopyBlockHelper(Compiler* comp, GenTree* store) : MorphInitBlockHelper(comp, store, false) +MorphCopyBlockHelper::MorphCopyBlockHelper(Compiler* comp, GenTree* store) + : MorphInitBlockHelper(comp, store, false) { } diff --git a/src/coreclr/jit/objectalloc.cpp b/src/coreclr/jit/objectalloc.cpp index a86039dc333840..0af5f4ba7a9929 100644 --- a/src/coreclr/jit/objectalloc.cpp +++ b/src/coreclr/jit/objectalloc.cpp @@ -163,7 +163,8 @@ void ObjectAllocator::MarkEscapingVarsAndBuildConnGraph() }; BuildConnGraphVisitor(ObjectAllocator* allocator) - : GenTreeVisitor(allocator->comp), m_allocator(allocator) + : GenTreeVisitor(allocator->comp) + , m_allocator(allocator) { } @@ -504,8 +505,8 @@ unsigned int ObjectAllocator::MorphAllocObjNodeIntoStackAlloc(GenTreeAllocObj* a assert(m_AnalysisDone); const bool shortLifetime = false; - const unsigned int lclNum = comp->lvaGrabTemp(shortLifetime DEBUGARG("MorphAllocObjNodeIntoStackAlloc temp")); - const int unsafeValueClsCheck = true; + const unsigned int lclNum = comp->lvaGrabTemp(shortLifetime DEBUGARG("MorphAllocObjNodeIntoStackAlloc temp")); + const int unsafeValueClsCheck = true; comp->lvaSetStruct(lclNum, allocObj->gtAllocObjClsHnd, unsafeValueClsCheck); // Initialize the object memory if necessary. @@ -766,7 +767,8 @@ void ObjectAllocator::RewriteUses() }; RewriteUsesVisitor(ObjectAllocator* allocator) - : GenTreeVisitor(allocator->comp), m_allocator(allocator) + : GenTreeVisitor(allocator->comp) + , m_allocator(allocator) { } diff --git a/src/coreclr/jit/objectalloc.h b/src/coreclr/jit/objectalloc.h index f4a56cb4ca39d9..07307161da002b 100644 --- a/src/coreclr/jit/objectalloc.h +++ b/src/coreclr/jit/objectalloc.h @@ -47,21 +47,21 @@ class ObjectAllocator final : public Phase virtual PhaseStatus DoPhase() override; private: - bool CanAllocateLclVarOnStack(unsigned int lclNum, CORINFO_CLASS_HANDLE clsHnd); - bool CanLclVarEscape(unsigned int lclNum); - void MarkLclVarAsPossiblyStackPointing(unsigned int lclNum); - void MarkLclVarAsDefinitelyStackPointing(unsigned int lclNum); - bool MayLclVarPointToStack(unsigned int lclNum); - bool DoesLclVarPointToStack(unsigned int lclNum); - void DoAnalysis(); - void MarkLclVarAsEscaping(unsigned int lclNum); - void MarkEscapingVarsAndBuildConnGraph(); - void AddConnGraphEdge(unsigned int sourceLclNum, unsigned int targetLclNum); - void ComputeEscapingNodes(BitVecTraits* bitVecTraits, BitVec& escapingNodes); - void ComputeStackObjectPointers(BitVecTraits* bitVecTraits); - bool MorphAllocObjNodes(); - void RewriteUses(); - GenTree* MorphAllocObjNodeIntoHelperCall(GenTreeAllocObj* allocObj); + bool CanAllocateLclVarOnStack(unsigned int lclNum, CORINFO_CLASS_HANDLE clsHnd); + bool CanLclVarEscape(unsigned int lclNum); + void MarkLclVarAsPossiblyStackPointing(unsigned int lclNum); + void MarkLclVarAsDefinitelyStackPointing(unsigned int lclNum); + bool MayLclVarPointToStack(unsigned int lclNum); + bool DoesLclVarPointToStack(unsigned int lclNum); + void DoAnalysis(); + void MarkLclVarAsEscaping(unsigned int lclNum); + void MarkEscapingVarsAndBuildConnGraph(); + void AddConnGraphEdge(unsigned int sourceLclNum, unsigned int targetLclNum); + void ComputeEscapingNodes(BitVecTraits* bitVecTraits, BitVec& escapingNodes); + void ComputeStackObjectPointers(BitVecTraits* bitVecTraits); + bool MorphAllocObjNodes(); + void RewriteUses(); + GenTree* MorphAllocObjNodeIntoHelperCall(GenTreeAllocObj* allocObj); unsigned int MorphAllocObjNodeIntoStackAlloc(GenTreeAllocObj* allocObj, BasicBlock* block, Statement* stmt); struct BuildConnGraphVisitorCallbackData; bool CanLclVarEscapeViaParentStack(ArrayStack* parentStack, unsigned int lclNum); diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index cb17b65035cd5f..acaed299aad42a 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -204,7 +204,9 @@ void Compiler::optCSE_GetMaskData(GenTree* tree, optCSE_MaskData* pMaskData) DoPreOrder = true, }; - MaskDataWalker(Compiler* comp, optCSE_MaskData* maskData) : GenTreeVisitor(comp), m_maskData(maskData) + MaskDataWalker(Compiler* comp, optCSE_MaskData* maskData) + : GenTreeVisitor(comp) + , m_maskData(maskData) { } @@ -396,7 +398,9 @@ void CSEdsc::ComputeNumLocals(Compiler* compiler) }; LocalCountingVisitor(Compiler* compiler) - : GenTreeVisitor(compiler), m_count(0), m_occurrences(0) + : GenTreeVisitor(compiler) + , m_count(0) + , m_occurrences(0) { } @@ -1186,7 +1190,9 @@ class CSE_DataFlow EXPSET_TP m_preMergeOut; public: - CSE_DataFlow(Compiler* pCompiler) : m_comp(pCompiler), m_preMergeOut(BitVecOps::UninitVal()) + CSE_DataFlow(Compiler* pCompiler) + : m_comp(pCompiler) + , m_preMergeOut(BitVecOps::UninitVal()) { } @@ -1742,7 +1748,8 @@ void Compiler::optValnumCSE_Availability() // Notes: // This creates the basic CSE heuristic. It never does any CSEs. // -CSE_HeuristicCommon::CSE_HeuristicCommon(Compiler* pCompiler) : m_pCompiler(pCompiler) +CSE_HeuristicCommon::CSE_HeuristicCommon(Compiler* pCompiler) + : m_pCompiler(pCompiler) { m_addCSEcount = 0; /* Count of the number of LclVars for CSEs that we added */ sortTab = nullptr; @@ -2074,7 +2081,8 @@ void CSE_HeuristicCommon::DumpMetrics() // This creates the random CSE heuristic. It does CSEs randomly, with some // predetermined likelihood (set by config or by stress). // -CSE_HeuristicRandom::CSE_HeuristicRandom(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) +CSE_HeuristicRandom::CSE_HeuristicRandom(Compiler* pCompiler) + : CSE_HeuristicCommon(pCompiler) { m_cseRNG.Init(m_pCompiler->info.compMethodHash() ^ JitConfig.JitRandomCSE()); } @@ -2200,7 +2208,8 @@ void CSE_HeuristicRandom::ConsiderCandidates() // This creates the replay CSE heuristic. It does CSEs specifed by // the ArrayConfig parsing of JitReplayCSE. // -CSE_HeuristicReplay::CSE_HeuristicReplay(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) +CSE_HeuristicReplay::CSE_HeuristicReplay(Compiler* pCompiler) + : CSE_HeuristicCommon(pCompiler) { } @@ -2292,7 +2301,8 @@ double CSE_HeuristicParameterized::s_defaultParameters[CSE_HeuristicParameterize // Arguments; // pCompiler - compiler instance // -CSE_HeuristicParameterized::CSE_HeuristicParameterized(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) +CSE_HeuristicParameterized::CSE_HeuristicParameterized(Compiler* pCompiler) + : CSE_HeuristicCommon(pCompiler) { // Default parameter values... // @@ -2605,7 +2615,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) if (!isLiveAcrossCallLSRA) { unsigned count = 0; - for (BasicBlock *block = minPostorderBlock; + for (BasicBlock* block = minPostorderBlock; block != nullptr && block != maxPostorderBlock && count < blockSpread; block = block->Next(), count++) { if (block->HasFlag(BBF_HAS_CALL)) @@ -2986,7 +2996,10 @@ void CSE_HeuristicParameterized::DumpChoices(ArrayStack& choices, CSEdsc // Uses parameters from JitRLCSE to drive a deterministic greedy policy // CSE_HeuristicRL::CSE_HeuristicRL(Compiler* pCompiler) - : CSE_HeuristicParameterized(pCompiler), m_alpha(0.0), m_updateParameters(false), m_greedy(false) + : CSE_HeuristicParameterized(pCompiler) + , m_alpha(0.0) + , m_updateParameters(false) + , m_greedy(false) { // Set up the random state // @@ -3656,7 +3669,8 @@ CSE_HeuristicRL::Choice* CSE_HeuristicRL::FindChoice(CSEdsc* dsc, ArrayStack& choices); + void BuildChoices(ArrayStack& choices); Choice& ChooseGreedy(ArrayStack& choices, bool recompute); @@ -227,12 +231,12 @@ class CSE_HeuristicRL : public CSE_HeuristicParameterized bool m_updateParameters; bool m_greedy; - Choice& ChooseSoftmax(ArrayStack& choices); - void Softmax(ArrayStack& choices); - void SoftmaxPolicy(); - void UpdateParametersStep(CSEdsc* dsc, ArrayStack& choices, double reward, double* delta); - void UpdateParameters(); - Choice* FindChoice(CSEdsc* dsc, ArrayStack& choices); + Choice& ChooseSoftmax(ArrayStack& choices); + void Softmax(ArrayStack& choices); + void SoftmaxPolicy(); + void UpdateParametersStep(CSEdsc* dsc, ArrayStack& choices, double reward, double* delta); + void UpdateParameters(); + Choice* FindChoice(CSEdsc* dsc, ArrayStack& choices); const char* Name() const; public: diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index d456cb3793f5ef..1e5d5a00b107c0 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -74,10 +74,10 @@ class OptBoolsDsc private: Statement* optOptimizeBoolsChkBlkCond(); - GenTree* optIsBoolComp(OptTestInfo* pOptTest); - bool optOptimizeBoolsChkTypeCostCond(); - void optOptimizeBoolsUpdateTrees(); - bool FindCompareChain(GenTree* condition, bool* isTestCondition); + GenTree* optIsBoolComp(OptTestInfo* pOptTest); + bool optOptimizeBoolsChkTypeCostCond(); + void optOptimizeBoolsUpdateTrees(); + bool FindCompareChain(GenTree* condition, bool* isTestCondition); }; //----------------------------------------------------------------------------- diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 3e77b92a9497ea..4de1a55e63cbd1 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -37,7 +37,8 @@ void Compiler::optInit() optCSEunmarks = 0; } -DataFlow::DataFlow(Compiler* pCompiler) : m_pCompiler(pCompiler) +DataFlow::DataFlow(Compiler* pCompiler) + : m_pCompiler(pCompiler) { } @@ -891,7 +892,7 @@ bool Compiler::optComputeLoopRep(int constInit, switch (iterOperType) { -// For small types, the iteration operator will narrow these values if big + // For small types, the iteration operator will narrow these values if big #define INIT_ITER_BY_TYPE(type) \ constInitX = (type)constInit; \ @@ -910,7 +911,7 @@ bool Compiler::optComputeLoopRep(int constInit, INIT_ITER_BY_TYPE(unsigned short); break; - // For the big types, 32 bit arithmetic is performed + // For the big types, 32 bit arithmetic is performed case TYP_INT: if (unsTest) @@ -1797,7 +1798,9 @@ void Compiler::optReplaceScalarUsesWithConst(BasicBlock* block, unsigned lclNum, bool MadeChanges = false; ReplaceVisitor(Compiler* comp, unsigned lclNum, ssize_t cnsVal) - : GenTreeVisitor(comp), m_lclNum(lclNum), m_cnsVal(cnsVal) + : GenTreeVisitor(comp) + , m_lclNum(lclNum) + , m_cnsVal(cnsVal) { } @@ -1843,7 +1846,8 @@ Compiler::OptInvertCountTreeInfoType Compiler::optInvertCountTreeInfo(GenTree* t Compiler::OptInvertCountTreeInfoType Result = {}; - CountTreeInfoVisitor(Compiler* comp) : GenTreeVisitor(comp) + CountTreeInfoVisitor(Compiler* comp) + : GenTreeVisitor(comp) { } @@ -3518,8 +3522,8 @@ bool Compiler::optNarrowTree(GenTree* tree, var_types srct, var_types dstt, Valu return true; - /* Operands that are in memory can usually be narrowed - simply by changing their gtType */ + /* Operands that are in memory can usually be narrowed + simply by changing their gtType */ case GT_LCL_VAR: /* We only allow narrowing long -> int for a GT_LCL_VAR */ @@ -3777,7 +3781,8 @@ void Compiler::optRecordSsaUses(GenTree* tree, BasicBlock* block) }; SsaRecordingVisitor(Compiler* compiler, BasicBlock* block) - : GenTreeVisitor(compiler), m_block(block) + : GenTreeVisitor(compiler) + , m_block(block) { } @@ -4614,7 +4619,11 @@ void Compiler::optHoistLoopBlocks(FlowGraphNaturalLoop* loop, const char* m_failReason; #endif - Value(GenTree* node) : m_node(node), m_hoistable(false), m_cctorDependent(false), m_invariant(false) + Value(GenTree* node) + : m_node(node) + , m_hoistable(false) + , m_cctorDependent(false) + , m_invariant(false) { #ifdef DEBUG m_failReason = "unset"; @@ -4814,9 +4823,9 @@ void Compiler::optHoistLoopBlocks(FlowGraphNaturalLoop* loop, // To be invariant the variable must be in SSA ... bool isInvariant = lclVar->HasSsaName(); // and the SSA definition must be outside the loop we're hoisting from ... - isInvariant = isInvariant && - !m_loop->ContainsBlock( - m_compiler->lvaGetDesc(lclNum)->GetPerSsaData(lclVar->GetSsaNum())->GetBlock()); + isInvariant = + isInvariant && !m_loop->ContainsBlock( + m_compiler->lvaGetDesc(lclNum)->GetPerSsaData(lclVar->GetSsaNum())->GetBlock()); // and the VN of the tree is considered invariant as well. // @@ -5469,7 +5478,9 @@ PhaseStatus Compiler::fgCanonicalizeFirstBB() return PhaseStatus::MODIFIED_EVERYTHING; } -LoopSideEffects::LoopSideEffects() : VarInOut(VarSetOps::UninitVal()), VarUseDef(VarSetOps::UninitVal()) +LoopSideEffects::LoopSideEffects() + : VarInOut(VarSetOps::UninitVal()) + , VarUseDef(VarSetOps::UninitVal()) { for (MemoryKind mk : allMemoryKinds()) { diff --git a/src/coreclr/jit/patchpoint.cpp b/src/coreclr/jit/patchpoint.cpp index 5a273679067748..71622ecfc3d759 100644 --- a/src/coreclr/jit/patchpoint.cpp +++ b/src/coreclr/jit/patchpoint.cpp @@ -34,7 +34,9 @@ class PatchpointTransformer Compiler* compiler; public: - PatchpointTransformer(Compiler* compiler) : ppCounterLclNum(BAD_VAR_NUM), compiler(compiler) + PatchpointTransformer(Compiler* compiler) + : ppCounterLclNum(BAD_VAR_NUM) + , compiler(compiler) { } diff --git a/src/coreclr/jit/phase.h b/src/coreclr/jit/phase.h index 6288d596729daf..0f3d461c2b13f1 100644 --- a/src/coreclr/jit/phase.h +++ b/src/coreclr/jit/phase.h @@ -34,14 +34,17 @@ class Phase virtual void Run(); protected: - Phase(Compiler* _compiler, Phases _phase) : comp(_compiler), m_name(nullptr), m_phase(_phase) + Phase(Compiler* _compiler, Phases _phase) + : comp(_compiler) + , m_name(nullptr) + , m_phase(_phase) { m_name = PhaseNames[_phase]; } virtual void PrePhase(); virtual PhaseStatus DoPhase() = 0; - virtual void PostPhase(PhaseStatus status); + virtual void PostPhase(PhaseStatus status); Compiler* comp; const char* m_name; @@ -54,7 +57,9 @@ template class ActionPhase final : public Phase { public: - ActionPhase(Compiler* _compiler, Phases _phase, A _action) : Phase(_compiler, _phase), action(_action) + ActionPhase(Compiler* _compiler, Phases _phase, A _action) + : Phase(_compiler, _phase) + , action(_action) { } @@ -84,7 +89,8 @@ class CompilerPhase final : public Phase { public: CompilerPhase(Compiler* _compiler, Phases _phase, void (Compiler::*_action)()) - : Phase(_compiler, _phase), action(_action) + : Phase(_compiler, _phase) + , action(_action) { } @@ -114,7 +120,8 @@ class CompilerPhaseWithStatus final : public Phase { public: CompilerPhaseWithStatus(Compiler* _compiler, Phases _phase, PhaseStatus (Compiler::*_action)()) - : Phase(_compiler, _phase), action(_action) + : Phase(_compiler, _phase) + , action(_action) { } diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 2f7b1e0b31372c..e02a5f0e06bab3 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -80,7 +80,9 @@ struct Access #endif Access(unsigned offset, var_types accessType, ClassLayout* layout) - : Layout(layout), Offset(offset), AccessType(accessType) + : Layout(layout) + , Offset(offset) + , AccessType(accessType) { } @@ -220,7 +222,8 @@ bool AggregateInfo::OverlappingReplacements(unsigned offset, // numLocals - Number of locals to support in the map // AggregateInfoMap::AggregateInfoMap(CompAllocator allocator, unsigned numLocals) - : m_aggregates(allocator), m_numLocals(numLocals) + : m_aggregates(allocator) + , m_numLocals(numLocals) { m_lclNumToAggregateIndex = new (allocator) unsigned[numLocals]; for (unsigned i = 0; i < numLocals; i++) @@ -277,7 +280,9 @@ struct PrimitiveAccess unsigned Offset; var_types AccessType; - PrimitiveAccess(unsigned offset, var_types accessType) : Offset(offset), AccessType(accessType) + PrimitiveAccess(unsigned offset, var_types accessType) + : Offset(offset) + , AccessType(accessType) { } }; @@ -290,7 +295,8 @@ class LocalUses public: LocalUses(Compiler* comp) - : m_accesses(comp->getAllocator(CMK_Promotion)), m_inducedAccesses(comp->getAllocator(CMK_Promotion)) + : m_accesses(comp->getAllocator(CMK_Promotion)) + , m_inducedAccesses(comp->getAllocator(CMK_Promotion)) { } @@ -973,7 +979,7 @@ class LocalsUseVisitor : public GenTreeVisitor , m_prom(prom) , m_candidateStores(prom->m_compiler->getAllocator(CMK_Promotion)) { - m_uses = new (prom->m_compiler, CMK_Promotion) LocalUses*[prom->m_compiler->lvaCount]{}; + m_uses = new (prom->m_compiler, CMK_Promotion) LocalUses* [prom->m_compiler->lvaCount] {}; } //------------------------------------------------------------------------ @@ -2269,7 +2275,9 @@ void ReplaceVisitor::InsertPreStatementWriteBacks() DoPreOrder = true, }; - Visitor(Compiler* comp, ReplaceVisitor* replacer) : GenTreeVisitor(comp), m_replacer(replacer) + Visitor(Compiler* comp, ReplaceVisitor* replacer) + : GenTreeVisitor(comp) + , m_replacer(replacer) { } @@ -2716,8 +2724,8 @@ void ReplaceVisitor::WriteBackBeforeUse(GenTree** use, unsigned lcl, unsigned of GenTreeOp* comma = m_compiler->gtNewOperNode(GT_COMMA, (*use)->TypeGet(), Promotion::CreateWriteBack(m_compiler, lcl, rep), *use); - *use = comma; - use = &comma->gtOp2; + *use = comma; + use = &comma->gtOp2; ClearNeedsWriteBack(rep); m_madeChanges = true; diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index c421b019bc8f99..89097d78cd1061 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -31,7 +31,9 @@ struct Replacement const char* Description = ""; #endif - Replacement(unsigned offset, var_types accessType) : Offset(offset), AccessType(accessType) + Replacement(unsigned offset, var_types accessType) + : Offset(offset) + , AccessType(accessType) { } @@ -55,7 +57,9 @@ class StructSegments { } - Segment(unsigned start, unsigned end) : Start(start), End(end) + Segment(unsigned start, unsigned end) + : Start(start) + , End(end) { } @@ -69,7 +73,8 @@ class StructSegments jitstd::vector m_segments; public: - explicit StructSegments(CompAllocator allocator) : m_segments(allocator) + explicit StructSegments(CompAllocator allocator) + : m_segments(allocator) { } @@ -96,7 +101,10 @@ struct AggregateInfo // Max offset in the struct local of the unpromoted part. unsigned UnpromotedMax = 0; - AggregateInfo(CompAllocator alloc, unsigned lclNum) : Replacements(alloc), LclNum(lclNum), Unpromoted(alloc) + AggregateInfo(CompAllocator alloc, unsigned lclNum) + : Replacements(alloc) + , LclNum(lclNum) + , Unpromoted(alloc) { } @@ -115,7 +123,7 @@ class AggregateInfoMap public: AggregateInfoMap(CompAllocator allocator, unsigned numLocals); - void Add(AggregateInfo* agg); + void Add(AggregateInfo* agg); AggregateInfo* Lookup(unsigned lclNum); jitstd::vector::iterator begin() @@ -146,10 +154,10 @@ class Promotion StructSegments SignificantSegments(ClassLayout* layout); - void ExplicitlyZeroInitReplacementLocals(unsigned lclNum, - const jitstd::vector& replacements, - Statement** prevStmt); - void InsertInitStatement(Statement** prevStmt, GenTree* tree); + void ExplicitlyZeroInitReplacementLocals(unsigned lclNum, + const jitstd::vector& replacements, + Statement** prevStmt); + void InsertInitStatement(Statement** prevStmt, GenTree* tree); static GenTree* CreateWriteBack(Compiler* compiler, unsigned structLclNum, const Replacement& replacement); static GenTree* CreateReadBack(Compiler* compiler, unsigned structLclNum, const Replacement& replacement); @@ -198,11 +206,12 @@ class Promotion bool HaveCandidateLocals(); - static bool IsCandidateForPhysicalPromotion(LclVarDsc* dsc); + static bool IsCandidateForPhysicalPromotion(LclVarDsc* dsc); static GenTree* EffectiveUser(Compiler::GenTreeStack& ancestors); public: - explicit Promotion(Compiler* compiler) : m_compiler(compiler) + explicit Promotion(Compiler* compiler) + : m_compiler(compiler) { } @@ -218,12 +227,15 @@ class StructDeaths friend class PromotionLiveness; private: - StructDeaths(BitVec deaths, AggregateInfo* agg) : m_deaths(deaths), m_aggregate(agg) + StructDeaths(BitVec deaths, AggregateInfo* agg) + : m_deaths(deaths) + , m_aggregate(agg) { } public: - StructDeaths() : m_deaths(BitVecOps::UninitVal()) + StructDeaths() + : m_deaths(BitVecOps::UninitVal()) { } @@ -236,26 +248,28 @@ struct BasicBlockLiveness; // Class to compute and track liveness information pertaining promoted structs. class PromotionLiveness { - Compiler* m_compiler; - AggregateInfoMap& m_aggregates; - BitVecTraits* m_bvTraits = nullptr; - unsigned* m_structLclToTrackedIndex = nullptr; - unsigned m_numVars = 0; - BasicBlockLiveness* m_bbInfo = nullptr; - bool m_hasPossibleBackEdge = false; - BitVec m_liveIn; - BitVec m_ehLiveVars; + Compiler* m_compiler; + AggregateInfoMap& m_aggregates; + BitVecTraits* m_bvTraits = nullptr; + unsigned* m_structLclToTrackedIndex = nullptr; + unsigned m_numVars = 0; + BasicBlockLiveness* m_bbInfo = nullptr; + bool m_hasPossibleBackEdge = false; + BitVec m_liveIn; + BitVec m_ehLiveVars; JitHashTable, BitVec> m_aggDeaths; public: PromotionLiveness(Compiler* compiler, AggregateInfoMap& aggregates) - : m_compiler(compiler), m_aggregates(aggregates), m_aggDeaths(compiler->getAllocator(CMK_Promotion)) + : m_compiler(compiler) + , m_aggregates(aggregates) + , m_aggDeaths(compiler->getAllocator(CMK_Promotion)) { } - void Run(); - bool IsReplacementLiveIn(BasicBlock* bb, unsigned structLcl, unsigned replacement); - bool IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacement); + void Run(); + bool IsReplacementLiveIn(BasicBlock* bb, unsigned structLcl, unsigned replacement); + bool IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacement); StructDeaths GetDeathsForStructLocal(GenTreeLclVarCommon* use); private: @@ -297,7 +311,10 @@ class ReplaceVisitor : public GenTreeVisitor }; ReplaceVisitor(Promotion* prom, AggregateInfoMap& aggregates, PromotionLiveness* liveness) - : GenTreeVisitor(prom->m_compiler), m_promotion(prom), m_aggregates(aggregates), m_liveness(liveness) + : GenTreeVisitor(prom->m_compiler) + , m_promotion(prom) + , m_aggregates(aggregates) + , m_liveness(liveness) { } diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index 18ac84c58e4f2b..d4f71b99835208 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -275,7 +275,9 @@ class DecompositionPlan var_types PrimitiveType; RemainderStrategy(int type, unsigned primitiveOffset = 0, var_types primitiveType = TYP_UNDEF) - : Type(type), PrimitiveOffset(primitiveOffset), PrimitiveType(primitiveType) + : Type(type) + , PrimitiveOffset(primitiveOffset) + , PrimitiveType(primitiveType) { } }; @@ -727,8 +729,8 @@ class DecompositionPlan // remainderStrategy - The strategy we are using for the remainder // dump - Whether to JITDUMP decisions made // - bool CanSkipEntry(const Entry& entry, - const StructDeaths& deaths, + bool CanSkipEntry(const Entry& entry, + const StructDeaths& deaths, const RemainderStrategy& remainderStrategy DEBUGARG(bool dump = false)) { if (entry.ToReplacement != nullptr) diff --git a/src/coreclr/jit/rangecheck.cpp b/src/coreclr/jit/rangecheck.cpp index 475df2d659cabc..eae6d627935837 100644 --- a/src/coreclr/jit/rangecheck.cpp +++ b/src/coreclr/jit/rangecheck.cpp @@ -461,7 +461,9 @@ bool RangeCheck::IsMonotonicallyIncreasing(GenTree* expr, bool rejectNegativeCon } // Remove hashtable entry for expr when we exit the present scope. - auto code = [this, expr] { m_pSearchPath->Remove(expr); }; + auto code = [this, expr] { + m_pSearchPath->Remove(expr); + }; jitstd::utility::scoped_code finally(code); if (m_pSearchPath->GetCount() > MAX_SEARCH_DEPTH) @@ -1123,7 +1125,7 @@ Range RangeCheck::GetRangeFromType(var_types type) // Compute the range for a local var definition. Range RangeCheck::ComputeRangeForLocalDef(BasicBlock* block, GenTreeLclVarCommon* lcl, - bool monIncreasing DEBUGARG(int indent)) + bool monIncreasing DEBUGARG(int indent)) { LclSsaVarDsc* ssaDef = GetSsaDefStore(lcl); if (ssaDef == nullptr) @@ -1566,7 +1568,10 @@ struct MapMethodDefsData BasicBlock* block; Statement* stmt; - MapMethodDefsData(RangeCheck* rc, BasicBlock* block, Statement* stmt) : rc(rc), block(block), stmt(stmt) + MapMethodDefsData(RangeCheck* rc, BasicBlock* block, Statement* stmt) + : rc(rc) + , block(block) + , stmt(stmt) { } }; diff --git a/src/coreclr/jit/rangecheck.h b/src/coreclr/jit/rangecheck.h index 098e1cc62b0d7a..cd4193f1e2fb73 100644 --- a/src/coreclr/jit/rangecheck.h +++ b/src/coreclr/jit/rangecheck.h @@ -83,20 +83,28 @@ struct Limit keUnknown, // The limit could not be determined. }; - Limit() : type(keUndef) + Limit() + : type(keUndef) { } - Limit(LimitType type) : type(type) + Limit(LimitType type) + : type(type) { } - Limit(LimitType type, int cns) : cns(cns), vn(ValueNumStore::NoVN), type(type) + Limit(LimitType type, int cns) + : cns(cns) + , vn(ValueNumStore::NoVN) + , type(type) { assert(type == keConstant); } - Limit(LimitType type, ValueNum vn, int cns) : cns(cns), vn(vn), type(type) + Limit(LimitType type, ValueNum vn, int cns) + : cns(cns) + , vn(vn) + , type(type) { assert(type == keBinOpArray); } @@ -242,11 +250,15 @@ struct Range Limit uLimit; Limit lLimit; - Range(const Limit& limit) : uLimit(limit), lLimit(limit) + Range(const Limit& limit) + : uLimit(limit) + , lLimit(limit) { } - Range(const Limit& lLimit, const Limit& uLimit) : uLimit(uLimit), lLimit(lLimit) + Range(const Limit& lLimit, const Limit& uLimit) + : uLimit(uLimit) + , lLimit(lLimit) { } @@ -586,7 +598,10 @@ class RangeCheck BasicBlock* block; Statement* stmt; GenTreeLclVarCommon* tree; - Location(BasicBlock* block, Statement* stmt, GenTreeLclVarCommon* tree) : block(block), stmt(stmt), tree(tree) + Location(BasicBlock* block, Statement* stmt, GenTreeLclVarCommon* tree) + : block(block) + , stmt(stmt) + , tree(tree) { } diff --git a/src/coreclr/jit/rationalize.cpp b/src/coreclr/jit/rationalize.cpp index cb54b617a6a9c8..d9b69b8df5aa27 100644 --- a/src/coreclr/jit/rationalize.cpp +++ b/src/coreclr/jit/rationalize.cpp @@ -383,7 +383,8 @@ PhaseStatus Rationalizer::DoPhase() }; RationalizeVisitor(Rationalizer& rationalizer) - : GenTreeVisitor(rationalizer.comp), m_rationalizer(rationalizer) + : GenTreeVisitor(rationalizer.comp) + , m_rationalizer(rationalizer) { } diff --git a/src/coreclr/jit/rationalize.h b/src/coreclr/jit/rationalize.h index 65264f8294582c..a8651b2e5b8c79 100644 --- a/src/coreclr/jit/rationalize.h +++ b/src/coreclr/jit/rationalize.h @@ -55,7 +55,8 @@ class Rationalizer final : public Phase Compiler::fgWalkResult RewriteNode(GenTree** useEdge, Compiler::GenTreeStack& parents); }; -inline Rationalizer::Rationalizer(Compiler* _comp) : Phase(_comp, PHASE_RATIONALIZE) +inline Rationalizer::Rationalizer(Compiler* _comp) + : Phase(_comp, PHASE_RATIONALIZE) { } diff --git a/src/coreclr/jit/redundantbranchopts.cpp b/src/coreclr/jit/redundantbranchopts.cpp index e8b346faccc376..e7569e86c2ed33 100644 --- a/src/coreclr/jit/redundantbranchopts.cpp +++ b/src/coreclr/jit/redundantbranchopts.cpp @@ -24,7 +24,9 @@ PhaseStatus Compiler::optRedundantBranches() public: bool madeChanges; - OptRedundantBranchesDomTreeVisitor(Compiler* compiler) : DomTreeVisitor(compiler), madeChanges(false) + OptRedundantBranchesDomTreeVisitor(Compiler* compiler) + : DomTreeVisitor(compiler) + , madeChanges(false) { } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 39fba1e888d3f9..79b5d9ab449aad 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -150,7 +150,7 @@ void RegSet::rsClearRegsModified() } #ifdef DEBUG -void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, +void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump) DEBUGARG(regMaskOnlyOne calleeSaveMask)) const { @@ -203,7 +203,8 @@ void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG - printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); + printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), + mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_FLT_CALLEE_SAVED)); #endif rsModifiedRegsMask.AddFloatRegMask(mask); @@ -344,7 +345,9 @@ void RegSet::SetMaskVars(AllRegsMask newMaskVars) /*****************************************************************************/ -RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) : m_rsCompiler(compiler), m_rsGCInfo(gcInfo) +RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) + : m_rsCompiler(compiler) + , m_rsGCInfo(gcInfo) { /* Initialize the spill logic */ @@ -537,9 +540,9 @@ void RegSet::rsSpillTree(regNumber reg, GenTree* tree, unsigned regIdx /* =0 */) #if defined(TARGET_X86) /***************************************************************************** -* -* Spill the top of the FP x87 stack. -*/ + * + * Spill the top of the FP x87 stack. + */ void RegSet::rsSpillFPStack(GenTreeCall* call) { SpillDsc* spill; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 90549b9e333509..4d9166a7cea869 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -58,7 +58,7 @@ class RegSet TempDsc* spillTemp; // the temp holding the spilled value static SpillDsc* alloc(Compiler* pComp, RegSet* regSet, var_types type); - static void freeDsc(RegSet* regSet, SpillDsc* spillDsc); + static void freeDsc(RegSet* regSet, SpillDsc* spillDsc); }; //------------------------------------------------------------------------- @@ -72,7 +72,7 @@ class RegSet #ifdef DEBUG bool rsModifiedRegsMaskInitialized; // Has rsModifiedRegsMask been initialized? Guards against illegal use. - void printModifiedRegsMask(regMaskOnlyOne currentMask, + void printModifiedRegsMask(regMaskOnlyOne currentMask, regMaskOnlyOne modifiedMask DEBUGARG(bool suppressDump = false) DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG @@ -239,14 +239,14 @@ class RegSet }; static var_types tmpNormalizeType(var_types type); - TempDsc* tmpGetTemp(var_types type); // get temp for the given type - void tmpRlsTemp(TempDsc* temp); - TempDsc* tmpFindNum(int temp, TEMP_USAGE_TYPE usageType = TEMP_USAGE_FREE) const; + TempDsc* tmpGetTemp(var_types type); // get temp for the given type + void tmpRlsTemp(TempDsc* temp); + TempDsc* tmpFindNum(int temp, TEMP_USAGE_TYPE usageType = TEMP_USAGE_FREE) const; void tmpEnd(); TempDsc* tmpListBeg(TEMP_USAGE_TYPE usageType = TEMP_USAGE_FREE) const; TempDsc* tmpListNxt(TempDsc* curTemp, TEMP_USAGE_TYPE usageType = TEMP_USAGE_FREE) const; - void tmpDone(); + void tmpDone(); #ifdef DEBUG bool tmpAllFree() const; diff --git a/src/coreclr/jit/scev.cpp b/src/coreclr/jit/scev.cpp index 5819b56bdfd3a1..491ee4ab06f049 100644 --- a/src/coreclr/jit/scev.cpp +++ b/src/coreclr/jit/scev.cpp @@ -206,7 +206,9 @@ void Scev::Dump(Compiler* comp) // ResetForLoop. // ScalarEvolutionContext::ScalarEvolutionContext(Compiler* comp) - : m_comp(comp), m_cache(comp->getAllocator(CMK_LoopIVOpts)), m_ephemeralCache(comp->getAllocator(CMK_LoopIVOpts)) + : m_comp(comp) + , m_cache(comp->getAllocator(CMK_LoopIVOpts)) + , m_ephemeralCache(comp->getAllocator(CMK_LoopIVOpts)) { } @@ -967,8 +969,8 @@ Scev* ScalarEvolutionContext::Simplify(Scev* scev) ScevAddRec* addRec = (ScevAddRec*)op1; Scev* newStart = Simplify(NewBinop(binop->Oper, addRec->Start, op2)); Scev* newStep = scev->OperIs(ScevOper::Mul, ScevOper::Lsh) - ? Simplify(NewBinop(binop->Oper, addRec->Step, op2)) - : addRec->Step; + ? Simplify(NewBinop(binop->Oper, addRec->Step, op2)) + : addRec->Step; return NewAddRec(newStart, newStep); } diff --git a/src/coreclr/jit/scev.h b/src/coreclr/jit/scev.h index 0800be905503a9..1aab39e3d3a5de 100644 --- a/src/coreclr/jit/scev.h +++ b/src/coreclr/jit/scev.h @@ -48,7 +48,9 @@ struct Scev const ScevOper Oper; const var_types Type; - Scev(ScevOper oper, var_types type) : Oper(oper), Type(type) + Scev(ScevOper oper, var_types type) + : Oper(oper) + , Type(type) { } @@ -74,7 +76,9 @@ struct Scev struct ScevConstant : Scev { - ScevConstant(var_types type, int64_t value) : Scev(ScevOper::Constant, type), Value(value) + ScevConstant(var_types type, int64_t value) + : Scev(ScevOper::Constant, type) + , Value(value) { } @@ -84,7 +88,9 @@ struct ScevConstant : Scev struct ScevLocal : Scev { ScevLocal(var_types type, unsigned lclNum, unsigned ssaNum) - : Scev(ScevOper::Local, type), LclNum(lclNum), SsaNum(ssaNum) + : Scev(ScevOper::Local, type) + , LclNum(lclNum) + , SsaNum(ssaNum) { } @@ -96,7 +102,9 @@ struct ScevLocal : Scev struct ScevUnop : Scev { - ScevUnop(ScevOper oper, var_types type, Scev* op1) : Scev(oper, type), Op1(op1) + ScevUnop(ScevOper oper, var_types type, Scev* op1) + : Scev(oper, type) + , Op1(op1) { } @@ -105,7 +113,9 @@ struct ScevUnop : Scev struct ScevBinop : ScevUnop { - ScevBinop(ScevOper oper, var_types type, Scev* op1, Scev* op2) : ScevUnop(oper, type, op1), Op2(op2) + ScevBinop(ScevOper oper, var_types type, Scev* op1, Scev* op2) + : ScevUnop(oper, type, op1) + , Op2(op2) { } @@ -118,7 +128,9 @@ struct ScevBinop : ScevUnop struct ScevAddRec : Scev { ScevAddRec(var_types type, Scev* start, Scev* step DEBUGARG(FlowGraphNaturalLoop* loop)) - : Scev(ScevOper::AddRec, type), Start(start), Step(step) DEBUGARG(Loop(loop)) + : Scev(ScevOper::AddRec, type) + , Start(start) + , Step(step) DEBUGARG(Loop(loop)) { } @@ -204,7 +216,7 @@ class ScalarEvolutionContext Scev* MakeAddRecFromRecursiveScev(Scev* start, Scev* scev, Scev* recursiveScev); Scev* CreateSimpleInvariantScev(GenTree* tree); Scev* CreateScevForConstant(GenTreeIntConCommon* tree); - void ExtractAddOperands(ScevBinop* add, ArrayStack& operands); + void ExtractAddOperands(ScevBinop* add, ArrayStack& operands); public: ScalarEvolutionContext(Compiler* comp); @@ -212,10 +224,10 @@ class ScalarEvolutionContext void ResetForLoop(FlowGraphNaturalLoop* loop); ScevConstant* NewConstant(var_types type, int64_t value); - ScevLocal* NewLocal(unsigned lclNum, unsigned ssaNum); - ScevUnop* NewExtension(ScevOper oper, var_types targetType, Scev* op); - ScevBinop* NewBinop(ScevOper oper, Scev* op1, Scev* op2); - ScevAddRec* NewAddRec(Scev* start, Scev* step); + ScevLocal* NewLocal(unsigned lclNum, unsigned ssaNum); + ScevUnop* NewExtension(ScevOper oper, var_types targetType, Scev* op); + ScevBinop* NewBinop(ScevOper oper, Scev* op1, Scev* op2); + ScevAddRec* NewAddRec(Scev* start, Scev* step); Scev* Analyze(BasicBlock* block, GenTree* tree); Scev* Simplify(Scev* scev); diff --git a/src/coreclr/jit/scopeinfo.cpp b/src/coreclr/jit/scopeinfo.cpp index 7a1290f9ac7858..ddb766e94a0de9 100644 --- a/src/coreclr/jit/scopeinfo.cpp +++ b/src/coreclr/jit/scopeinfo.cpp @@ -790,11 +790,9 @@ void CodeGenInterface::VariableLiveKeeper::VariableLiveDescriptor::startLiveRang else { JITDUMP("Debug: New V%02u debug range: %s\n", m_varNum, - m_VariableLiveRanges->empty() - ? "first" - : siVarLoc::Equals(&varLocation, &(m_VariableLiveRanges->back().m_VarLocation)) - ? "new var or location" - : "not adjacent"); + m_VariableLiveRanges->empty() ? "first" + : siVarLoc::Equals(&varLocation, &(m_VariableLiveRanges->back().m_VarLocation)) ? "new var or location" + : "not adjacent"); // Creates new live range with invalid end m_VariableLiveRanges->emplace_back(varLocation, emitLocation(), emitLocation()); m_VariableLiveRanges->back().m_StartEmitLocation.CaptureLocation(emit); @@ -1685,9 +1683,9 @@ NATIVE_OFFSET CodeGen::psiGetVarStackOffset(const LclVarDsc* lclVarDsc) const } /*============================================================================ -* INTERFACE (public) Functions for PrologScopeInfo -*============================================================================ -*/ + * INTERFACE (public) Functions for PrologScopeInfo + *============================================================================ + */ //------------------------------------------------------------------------ // psiBegProlog: Initializes the PrologScopeInfo creating open psiScopes or diff --git a/src/coreclr/jit/sideeffects.cpp b/src/coreclr/jit/sideeffects.cpp index e39bf596c4770b..4a9b1899b24b84 100644 --- a/src/coreclr/jit/sideeffects.cpp +++ b/src/coreclr/jit/sideeffects.cpp @@ -8,7 +8,10 @@ #include "sideeffects.h" -LclVarSet::LclVarSet() : m_bitVector(nullptr), m_hasAnyLcl(false), m_hasBitVector(false) +LclVarSet::LclVarSet() + : m_bitVector(nullptr) + , m_hasAnyLcl(false) + , m_hasBitVector(false) { } @@ -121,7 +124,10 @@ void LclVarSet::Clear() } AliasSet::AliasSet() - : m_lclVarReads(), m_lclVarWrites(), m_readsAddressableLocation(false), m_writesAddressableLocation(false) + : m_lclVarReads() + , m_lclVarWrites() + , m_readsAddressableLocation(false) + , m_writesAddressableLocation(false) { } @@ -136,7 +142,11 @@ AliasSet::AliasSet() // node - The node in question. // AliasSet::NodeInfo::NodeInfo(Compiler* compiler, GenTree* node) - : m_compiler(compiler), m_node(node), m_flags(0), m_lclNum(0), m_lclOffs(0) + : m_compiler(compiler) + , m_node(node) + , m_flags(0) + , m_lclNum(0) + , m_lclOffs(0) { if (node->IsCall()) { @@ -444,7 +454,9 @@ void AliasSet::Clear() m_lclVarWrites.Clear(); } -SideEffectSet::SideEffectSet() : m_sideEffectFlags(0), m_aliasSet() +SideEffectSet::SideEffectSet() + : m_sideEffectFlags(0) + , m_aliasSet() { } @@ -460,7 +472,9 @@ SideEffectSet::SideEffectSet() : m_sideEffectFlags(0), m_aliasSet() // compiler - The compiler context. // node - The node to use for initialization. // -SideEffectSet::SideEffectSet(Compiler* compiler, GenTree* node) : m_sideEffectFlags(0), m_aliasSet() +SideEffectSet::SideEffectSet(Compiler* compiler, GenTree* node) + : m_sideEffectFlags(0) + , m_aliasSet() { AddNode(compiler, node); } diff --git a/src/coreclr/jit/sideeffects.h b/src/coreclr/jit/sideeffects.h index d94622d9f0ca8b..0fef277532cf1a 100644 --- a/src/coreclr/jit/sideeffects.h +++ b/src/coreclr/jit/sideeffects.h @@ -13,7 +13,8 @@ // class LclVarSet final { - union { + union + { hashBv* m_bitVector; unsigned m_lclNum; }; diff --git a/src/coreclr/jit/simd.h b/src/coreclr/jit/simd.h index aec72eaab542e8..3a5311aaaa79df 100644 --- a/src/coreclr/jit/simd.h +++ b/src/coreclr/jit/simd.h @@ -6,7 +6,8 @@ struct simd8_t { - union { + union + { float f32[2]; double f64[1]; int8_t i8[8]; @@ -58,7 +59,8 @@ static_assert_no_msg(sizeof(simd8_t) == 8); #include struct simd12_t { - union { + union + { float f32[3]; int8_t i8[12]; int16_t i16[6]; @@ -116,7 +118,8 @@ static_assert_no_msg(sizeof(simd12_t) == 12); struct simd16_t { - union { + union + { float f32[4]; double f64[2]; int8_t i8[16]; @@ -170,7 +173,8 @@ static_assert_no_msg(sizeof(simd16_t) == 16); #if defined(TARGET_XARCH) struct simd32_t { - union { + union + { float f32[8]; double f64[4]; int8_t i8[32]; @@ -224,7 +228,8 @@ static_assert_no_msg(sizeof(simd32_t) == 32); struct simd64_t { - union { + union + { float f32[16]; double f64[8]; int8_t i8[64]; @@ -279,7 +284,8 @@ static_assert_no_msg(sizeof(simd64_t) == 64); struct simdmask_t { - union { + union + { int8_t i8[8]; int16_t i16[4]; int32_t i32[2]; diff --git a/src/coreclr/jit/simdashwintrinsic.cpp b/src/coreclr/jit/simdashwintrinsic.cpp index f06b38736ddadb..c22ebc7b635440 100644 --- a/src/coreclr/jit/simdashwintrinsic.cpp +++ b/src/coreclr/jit/simdashwintrinsic.cpp @@ -399,7 +399,7 @@ GenTree* Compiler::impSimdAsHWIntrinsic(NamedIntrinsic intrinsic, { argType = isInstanceMethod ? simdType : JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass))); - op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); + op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); return gtNewSimdAsHWIntrinsicNode(retType, op1, hwIntrinsic, simdBaseJitType, simdSize); } @@ -421,7 +421,7 @@ GenTree* Compiler::impSimdAsHWIntrinsic(NamedIntrinsic intrinsic, argType = isInstanceMethod ? simdType : JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass))); - op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); + op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); return gtNewSimdAsHWIntrinsicNode(retType, op1, op2, hwIntrinsic, simdBaseJitType, simdSize); } @@ -954,7 +954,7 @@ GenTree* Compiler::impSimdAsHWIntrinsicSpecial(NamedIntrinsic intrinsic, argType = isInstanceMethod ? simdType : JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass))); - op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); + op1 = getArgForHWIntrinsic(argType, argClass, isInstanceMethod); switch (intrinsic) { diff --git a/src/coreclr/jit/sm.cpp b/src/coreclr/jit/sm.cpp index 5cd6e9879c78da..5e9b97699b841f 100644 --- a/src/coreclr/jit/sm.cpp +++ b/src/coreclr/jit/sm.cpp @@ -130,8 +130,8 @@ SM_STATE_ID CodeSeqSM::GetDestState(SM_STATE_ID srcState, SM_OPCODE opcode) if (cell->srcState != srcState) { - assert(cell->srcState == 0 || - cell->srcState != srcState); // Either way means there is not outgoing edge from srcState. + assert(cell->srcState == 0 || cell->srcState != srcState); // Either way means there is not outgoing edge from + // srcState. return 0; } else diff --git a/src/coreclr/jit/smallhash.h b/src/coreclr/jit/smallhash.h index f16905c995fbd4..5bbf58e99a4bd9 100644 --- a/src/coreclr/jit/smallhash.h +++ b/src/coreclr/jit/smallhash.h @@ -338,7 +338,10 @@ class HashTableBase protected: HashTableBase(TAllocator alloc, Bucket* buckets, unsigned numBuckets) - : m_alloc(alloc), m_buckets(buckets), m_numBuckets(numBuckets), m_numFullBuckets(0) + : m_alloc(alloc) + , m_buckets(buckets) + , m_numBuckets(numBuckets) + , m_numFullBuckets(0) { if (numBuckets > 0) { @@ -359,13 +362,15 @@ class HashTableBase Bucket* m_bucket; - KeyValuePair(Bucket* bucket) : m_bucket(bucket) + KeyValuePair(Bucket* bucket) + : m_bucket(bucket) { assert(m_bucket != nullptr); } public: - KeyValuePair() : m_bucket(nullptr) + KeyValuePair() + : m_bucket(nullptr) { } @@ -392,7 +397,9 @@ class HashTableBase unsigned m_index; Iterator(Bucket* buckets, unsigned numBuckets, unsigned index) - : m_buckets(buckets), m_numBuckets(numBuckets), m_index(index) + : m_buckets(buckets) + , m_numBuckets(numBuckets) + , m_index(index) { assert((buckets != nullptr) || (numBuckets == 0)); assert(index <= numBuckets); @@ -405,7 +412,10 @@ class HashTableBase } public: - Iterator() : m_buckets(nullptr), m_numBuckets(0), m_index(0) + Iterator() + : m_buckets(nullptr) + , m_numBuckets(0) + , m_index(0) { } @@ -636,7 +646,8 @@ class HashTable final : public HashTableBase } public: - HashTable(TAllocator alloc) : TBase(alloc, nullptr, 0) + HashTable(TAllocator alloc) + : TBase(alloc, nullptr, 0) { } @@ -670,7 +681,8 @@ class SmallHashTable final : public HashTableBase::UpdateLifeBit(VARSET_TP& set, LclVarDsc* dsc, // can be dumped after potential updates. // template -void TreeLifeUpdater::StoreCurrentLifeForDump() +void TreeLifeUpdater::StoreCurrentLifeForDump() { #ifdef DEBUG if (compiler->verbose) diff --git a/src/coreclr/jit/typelist.h b/src/coreclr/jit/typelist.h index 2ef837567d670e..2f121ae84f8cc2 100644 --- a/src/coreclr/jit/typelist.h +++ b/src/coreclr/jit/typelist.h @@ -4,7 +4,7 @@ #define GCS EA_GCREF #define BRS EA_BYREF #define EPS EA_PTRSIZE -#define PS TARGET_POINTER_SIZE +#define PS TARGET_POINTER_SIZE #define PST (TARGET_POINTER_SIZE / sizeof(int)) #ifdef TARGET_64BIT diff --git a/src/coreclr/jit/unwind.cpp b/src/coreclr/jit/unwind.cpp index 51be148e38af94..3b56653ca423b4 100644 --- a/src/coreclr/jit/unwind.cpp +++ b/src/coreclr/jit/unwind.cpp @@ -128,7 +128,7 @@ void Compiler::unwindGetFuncLocations(FuncInfoDsc* func, assert(func->funKind == FUNC_HANDLER); *ppStartLoc = new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(HBtab->ebdHndBeg)); *ppEndLoc = HBtab->ebdHndLast->IsLast() ? nullptr - : new (this, CMK_UnwindInfo) + : new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(HBtab->ebdHndLast->Next())); } } diff --git a/src/coreclr/jit/unwind.h b/src/coreclr/jit/unwind.h index 4d1b540f060624..8b7fcaa5a103dc 100644 --- a/src/coreclr/jit/unwind.h +++ b/src/coreclr/jit/unwind.h @@ -21,46 +21,51 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #if defined(TARGET_ARM) const unsigned MAX_PROLOG_SIZE_BYTES = 44; const unsigned MAX_EPILOG_SIZE_BYTES = 44; -#define UWC_END 0xFF // "end" unwind code +#define UWC_END 0xFF // "end" unwind code #define UW_MAX_FRAGMENT_SIZE_BYTES (1U << 19) -#define UW_MAX_CODE_WORDS_COUNT 15 // Max number that can be encoded in the "Code Words" field of the .pdata record -#define UW_MAX_EPILOG_START_INDEX 0xFFU // Max number that can be encoded in the "Epilog Start Index" field - // of the .pdata record +#define UW_MAX_CODE_WORDS_COUNT 15 // Max number that can be encoded in the "Code Words" field of the .pdata record +#define UW_MAX_EPILOG_START_INDEX \ + 0xFFU // Max number that can be encoded in the "Epilog Start Index" field + // of the .pdata record #elif defined(TARGET_ARM64) const unsigned MAX_PROLOG_SIZE_BYTES = 100; const unsigned MAX_EPILOG_SIZE_BYTES = 100; -#define UWC_END 0xE4 // "end" unwind code -#define UWC_END_C 0xE5 // "end_c" unwind code +#define UWC_END 0xE4 // "end" unwind code +#define UWC_END_C 0xE5 // "end_c" unwind code #define UW_MAX_FRAGMENT_SIZE_BYTES (1U << 20) -#define UW_MAX_CODE_WORDS_COUNT 31 -#define UW_MAX_EPILOG_START_INDEX 0x3FFU +#define UW_MAX_CODE_WORDS_COUNT 31 +#define UW_MAX_EPILOG_START_INDEX 0x3FFU #elif defined(TARGET_LOONGARCH64) const unsigned MAX_PROLOG_SIZE_BYTES = 200; const unsigned MAX_EPILOG_SIZE_BYTES = 200; -#define UWC_END 0xE4 // "end" unwind code -#define UWC_END_C 0xE5 // "end_c" unwind code +#define UWC_END 0xE4 // "end" unwind code +#define UWC_END_C 0xE5 // "end_c" unwind code #define UW_MAX_FRAGMENT_SIZE_BYTES (1U << 20) -#define UW_MAX_CODE_WORDS_COUNT 31 -#define UW_MAX_EPILOG_START_INDEX 0x3FFU +#define UW_MAX_CODE_WORDS_COUNT 31 +#define UW_MAX_EPILOG_START_INDEX 0x3FFU #elif defined(TARGET_RISCV64) const unsigned MAX_PROLOG_SIZE_BYTES = 200; const unsigned MAX_EPILOG_SIZE_BYTES = 200; -#define UWC_END 0xE4 // "end" unwind code -#define UWC_END_C 0xE5 // "end_c" unwind code +#define UWC_END 0xE4 // "end" unwind code +#define UWC_END_C 0xE5 // "end_c" unwind code #define UW_MAX_FRAGMENT_SIZE_BYTES (1U << 20) -#define UW_MAX_CODE_WORDS_COUNT 31 -#define UW_MAX_EPILOG_START_INDEX 0x3FFU +#define UW_MAX_CODE_WORDS_COUNT 31 +#define UW_MAX_EPILOG_START_INDEX 0x3FFU #endif // TARGET_RISCV64 -#define UW_MAX_EPILOG_COUNT 31 // Max number that can be encoded in the "Epilog count" field - // of the .pdata record -#define UW_MAX_EXTENDED_CODE_WORDS_COUNT 0xFFU // Max number that can be encoded in the "Extended Code Words" - // field of the .pdata record -#define UW_MAX_EXTENDED_EPILOG_COUNT 0xFFFFU // Max number that can be encoded in the "Extended Epilog Count" - // field of the .pdata record -#define UW_MAX_EPILOG_START_OFFSET 0x3FFFFU // Max number that can be encoded in the "Epilog Start Offset" - // field of the .pdata record +#define UW_MAX_EPILOG_COUNT \ + 31 // Max number that can be encoded in the "Epilog count" field + // of the .pdata record +#define UW_MAX_EXTENDED_CODE_WORDS_COUNT \ + 0xFFU // Max number that can be encoded in the "Extended Code Words" + // field of the .pdata record +#define UW_MAX_EXTENDED_EPILOG_COUNT \ + 0xFFFFU // Max number that can be encoded in the "Extended Epilog Count" + // field of the .pdata record +#define UW_MAX_EPILOG_START_OFFSET \ + 0x3FFFFU // Max number that can be encoded in the "Epilog Start Offset" + // field of the .pdata record // // Forward declaration of class defined in emit.h @@ -85,7 +90,8 @@ class UnwindInfo; class UnwindBase { protected: - UnwindBase(Compiler* comp) : uwiComp(comp) + UnwindBase(Compiler* comp) + : uwiComp(comp) { } @@ -107,9 +113,9 @@ class UnwindCodesBase public: // Add a single unwind code. - virtual void AddCode(BYTE b1) = 0; - virtual void AddCode(BYTE b1, BYTE b2) = 0; - virtual void AddCode(BYTE b1, BYTE b2, BYTE b3) = 0; + virtual void AddCode(BYTE b1) = 0; + virtual void AddCode(BYTE b1, BYTE b2) = 0; + virtual void AddCode(BYTE b1, BYTE b2, BYTE b3) = 0; virtual void AddCode(BYTE b1, BYTE b2, BYTE b3, BYTE b4) = 0; // Get access to the unwind codes @@ -139,7 +145,9 @@ class UnwindCodesBase // information for a function, including unwind info header, the prolog codes, // and any epilog codes. -class UnwindPrologCodes : public UnwindBase, public UnwindCodesBase +class UnwindPrologCodes + : public UnwindBase + , public UnwindCodesBase { // UPC_LOCAL_COUNT is the amount of memory local to this class. For ARM CoreLib, the maximum size is 34. // Here is a histogram of other interesting sizes: @@ -303,7 +311,9 @@ class UnwindPrologCodes : public UnwindBase, public UnwindCodesBase // Epilog unwind codes arrive in the order they will be emitted. Store them as an array, // adding new ones to the end of the array. -class UnwindEpilogCodes : public UnwindBase, public UnwindCodesBase +class UnwindEpilogCodes + : public UnwindBase + , public UnwindCodesBase { // UEC_LOCAL_COUNT is the amount of memory local to this class. For ARM CoreLib, the maximum size is 6, // while 89% of epilogs fit in 4. So, set it to 4 to maintain array alignment and hit most cases. diff --git a/src/coreclr/jit/unwindamd64.cpp b/src/coreclr/jit/unwindamd64.cpp index 549c4e9910567c..e42a4368581fb7 100644 --- a/src/coreclr/jit/unwindamd64.cpp +++ b/src/coreclr/jit/unwindamd64.cpp @@ -199,7 +199,7 @@ void Compiler::unwindPushWindows(regNumber reg) // since it is pushed as a frame register. || (reg == REG_FPBASE) #endif // ETW_EBP_FRAMED - ) + ) { code->UnwindOp = UWOP_PUSH_NONVOL; code->OpInfo = (BYTE)reg; diff --git a/src/coreclr/jit/unwindarm64.cpp b/src/coreclr/jit/unwindarm64.cpp index 0725eb41dfdba5..f842737171c0b4 100644 --- a/src/coreclr/jit/unwindarm64.cpp +++ b/src/coreclr/jit/unwindarm64.cpp @@ -461,8 +461,8 @@ void Compiler::unwindSaveRegPairPreindexed(regNumber reg1, regNumber reg2, int o pu->AddCode(0x80 | (BYTE)z); } - else if ((reg1 == REG_R19) && - (-256 <= offset)) // If the offset is between -512 and -256, we use the save_regp_x unwind code. + else if ((reg1 == REG_R19) && (-256 <= offset)) // If the offset is between -512 and -256, we use the save_regp_x + // unwind code. { // save_r19r20_x: 001zzzzz: save pair at [sp-#Z*8]!, pre-indexed offset >= -248 // NOTE: I'm not sure why we allow Z==0 here; seems useless, and the calculation of offset is different from the @@ -758,7 +758,7 @@ void DumpUnwindInfo(Compiler* comp, // pHeader is not guaranteed to be aligned. We put four 0xFF end codes at the end // to provide padding, and round down to get a multiple of 4 bytes in size. DWORD UNALIGNED* pdw = (DWORD UNALIGNED*)pHeader; - DWORD dw; + DWORD dw; dw = *pdw++; diff --git a/src/coreclr/jit/unwindarmarch.cpp b/src/coreclr/jit/unwindarmarch.cpp index 7a8b51573eac71..6ba44ff128e5ca 100644 --- a/src/coreclr/jit/unwindarmarch.cpp +++ b/src/coreclr/jit/unwindarmarch.cpp @@ -243,9 +243,8 @@ void Compiler::unwindPushPopMaskInt(regMaskGpr maskInt, bool useOpsize16) } else { - assert((maskInt & - ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | - RBM_R11 | RBM_R12 | RBM_LR)) == 0); + assert((maskInt & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | + RBM_R10 | RBM_R11 | RBM_R12 | RBM_LR)) == 0); bool shortFormat = false; BYTE val = 0; @@ -323,9 +322,8 @@ void Compiler::unwindPushMaskInt(regMaskGpr maskInt) assert(IsGprRegMask(maskInt)); // Only r0-r12 and lr are supported - assert((maskInt & - ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | - RBM_R11 | RBM_R12 | RBM_LR)) == 0); + assert((maskInt & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | + RBM_R10 | RBM_R11 | RBM_R12 | RBM_LR)) == 0); #if defined(FEATURE_CFI_SUPPORT) if (generateCFIUnwindCodes()) @@ -370,9 +368,8 @@ void Compiler::unwindPopMaskInt(regMaskGpr maskInt) #endif // FEATURE_CFI_SUPPORT // Only r0-r12 and lr and pc are supported (pc is mapped to lr when encoding) - assert((maskInt & - ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | RBM_R10 | - RBM_R11 | RBM_R12 | RBM_LR | RBM_PC)) == 0); + assert((maskInt & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R9 | + RBM_R10 | RBM_R11 | RBM_R12 | RBM_LR | RBM_PC)) == 0); bool useOpsize16 = ((maskInt & (RBM_LOW_REGS | RBM_PC)) == maskInt); // Can POP use the 16-bit encoding? @@ -729,8 +726,8 @@ unsigned GetOpcodeSizeFromUnwindHeader(BYTE b1) }; BYTE opsize = s_UnwindOpsize[b1]; - assert(opsize == 2 || - opsize == 4); // We shouldn't get a code with no opsize (the 0xFF end code is handled specially) + assert(opsize == 2 || opsize == 4); // We shouldn't get a code with no opsize (the 0xFF end code is handled + // specially) return opsize; } @@ -895,9 +892,9 @@ void UnwindPrologCodes::AppendEpilog(UnwindEpilogInfo* pEpi) int epiSize = pEpi->Size(); memcpy_s(&upcMem[upcEpilogSlot], upcMemSize - upcEpilogSlot - 3, pEpi->GetCodes(), - epiSize); // -3 to avoid writing to the alignment padding - assert(pEpi->GetStartIndex() == - upcEpilogSlot - upcCodeSlot); // Make sure we copied it where we expected to copy it. + epiSize); // -3 to avoid writing to the alignment padding + assert(pEpi->GetStartIndex() == upcEpilogSlot - upcCodeSlot); // Make sure we copied it where we expected to copy + // it. upcEpilogSlot += epiSize; assert(upcEpilogSlot <= upcMemSize - 3); @@ -1463,7 +1460,7 @@ void UnwindFragmentInfo::Finalize(UNATIVE_OFFSET functionLength) } #endif -// Compute the header + // Compute the header #if defined(TARGET_ARM) noway_assert((functionLength & 1) == 0); @@ -1512,8 +1509,8 @@ void UnwindFragmentInfo::Finalize(UNATIVE_OFFSET functionLength) // Start writing the header - noway_assert(headerFunctionLength <= - 0x3FFFFU); // We create fragments to prevent this from firing, so if it hits, we have an internal error + noway_assert(headerFunctionLength <= 0x3FFFFU); // We create fragments to prevent this from firing, so if it hits, + // we have an internal error if ((headerEpilogCount > UW_MAX_EPILOG_COUNT) || (headerCodeWords > UW_MAX_CODE_WORDS_COUNT)) { @@ -1524,7 +1521,7 @@ void UnwindFragmentInfo::Finalize(UNATIVE_OFFSET functionLength) DWORD header = headerFunctionLength | (headerVers << 18) | (headerXBit << 20) | (headerEBit << 21) | (headerFBit << 22) | (headerEpilogCount << 23) | (headerCodeWords << 28); #elif defined(TARGET_ARM64) - DWORD header = headerFunctionLength | (headerVers << 18) | (headerXBit << 20) | (headerEBit << 21) | + DWORD header = headerFunctionLength | (headerVers << 18) | (headerXBit << 20) | (headerEBit << 21) | (headerEpilogCount << 22) | (headerCodeWords << 27); #endif // defined(TARGET_ARM64) @@ -2211,7 +2208,7 @@ DWORD DumpRegSetRange(const char* const rtype, DWORD start, DWORD end, DWORD lr) DWORD DumpOpsize(DWORD padding, DWORD opsize) { if (padding > 100) // underflow? - padding = 4; + padding = 4; DWORD printed = padding; for (; padding > 0; padding--) printf(" "); @@ -2239,7 +2236,7 @@ void DumpUnwindInfo(Compiler* comp, // pHeader is not guaranteed to be aligned. We put four 0xFF end codes at the end // to provide padding, and round down to get a multiple of 4 bytes in size. DWORD UNALIGNED* pdw = (DWORD UNALIGNED*)pHeader; - DWORD dw; + DWORD dw; dw = *pdw++; diff --git a/src/coreclr/jit/unwindloongarch64.cpp b/src/coreclr/jit/unwindloongarch64.cpp index 3aa5fd668d40c6..1b561eaaaae669 100644 --- a/src/coreclr/jit/unwindloongarch64.cpp +++ b/src/coreclr/jit/unwindloongarch64.cpp @@ -516,7 +516,7 @@ void DumpUnwindInfo(Compiler* comp, // pHeader is not guaranteed to be aligned. We put four 0xFF end codes at the end // to provide padding, and round down to get a multiple of 4 bytes in size. DWORD UNALIGNED* pdw = (DWORD UNALIGNED*)pHeader; - DWORD dw; + DWORD dw; dw = *pdw++; @@ -1149,9 +1149,9 @@ void UnwindPrologCodes::AppendEpilog(UnwindEpilogInfo* pEpi) int epiSize = pEpi->Size(); memcpy_s(&upcMem[upcEpilogSlot], upcMemSize - upcEpilogSlot - 3, pEpi->GetCodes(), - epiSize); // -3 to avoid writing to the alignment padding - assert(pEpi->GetStartIndex() == - upcEpilogSlot - upcCodeSlot); // Make sure we copied it where we expected to copy it. + epiSize); // -3 to avoid writing to the alignment padding + assert(pEpi->GetStartIndex() == upcEpilogSlot - upcCodeSlot); // Make sure we copied it where we expected to copy + // it. upcEpilogSlot += epiSize; assert(upcEpilogSlot <= upcMemSize - 3); @@ -1772,8 +1772,8 @@ void UnwindFragmentInfo::Finalize(UNATIVE_OFFSET functionLength) // Start writing the header - noway_assert(headerFunctionLength <= - 0x3FFFFU); // We create fragments to prevent this from firing, so if it hits, we have an internal error + noway_assert(headerFunctionLength <= 0x3FFFFU); // We create fragments to prevent this from firing, so if it hits, + // we have an internal error if ((headerEpilogCount > UW_MAX_EPILOG_COUNT) || (headerCodeWords > UW_MAX_CODE_WORDS_COUNT)) { diff --git a/src/coreclr/jit/unwindriscv64.cpp b/src/coreclr/jit/unwindriscv64.cpp index b78eb04c228e93..f9db0d433c6f13 100644 --- a/src/coreclr/jit/unwindriscv64.cpp +++ b/src/coreclr/jit/unwindriscv64.cpp @@ -327,7 +327,7 @@ void DumpUnwindInfo(Compiler* comp, // pHeader is not guaranteed to be aligned. We put four 0xFF end codes at the end // to provide padding, and round down to get a multiple of 4 bytes in size. DWORD UNALIGNED* pdw = (DWORD UNALIGNED*)pHeader; - DWORD dw; + DWORD dw; dw = *pdw++; diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 4337c42bf44b7c..6efddc85af15ae 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -1124,7 +1124,8 @@ void Counter::dump(FILE* output) * Histogram class. */ -Histogram::Histogram(const unsigned* const sizeTable) : m_sizeTable(sizeTable) +Histogram::Histogram(const unsigned* const sizeTable) + : m_sizeTable(sizeTable) { unsigned sizeCount = 0; do @@ -1856,7 +1857,8 @@ void HelperCallProperties::init() // // You must use ';' as a separator; whitespace no longer works -AssemblyNamesList2::AssemblyNamesList2(const WCHAR* list, HostAllocator alloc) : m_alloc(alloc) +AssemblyNamesList2::AssemblyNamesList2(const WCHAR* list, HostAllocator alloc) + : m_alloc(alloc) { WCHAR prevChar = '?'; // dummy LPWSTR nameStart = nullptr; // start of the name currently being processed. nullptr if no current name @@ -1943,7 +1945,9 @@ bool AssemblyNamesList2::IsInList(const char* assemblyName) // MethodSet //============================================================================= -MethodSet::MethodSet(const WCHAR* filename, HostAllocator alloc) : m_pInfos(nullptr), m_alloc(alloc) +MethodSet::MethodSet(const WCHAR* filename, HostAllocator alloc) + : m_pInfos(nullptr) + , m_alloc(alloc) { FILE* methodSetFile = _wfopen(filename, W("r")); if (methodSetFile == nullptr) @@ -2172,7 +2176,8 @@ double CachedCyclesPerSecond() } #ifdef FEATURE_JIT_METHOD_PERF -CycleCount::CycleCount() : cps(CachedCyclesPerSecond()) +CycleCount::CycleCount() + : cps(CachedCyclesPerSecond()) { } @@ -2316,7 +2321,7 @@ unsigned __int64 FloatingPointUtils::convertDoubleToUInt64(double d) u64 = UINT64(INT64(d)); #else - u64 = UINT64(d); + u64 = UINT64(d); #endif // TARGET_XARCH return u64; @@ -4116,7 +4121,7 @@ int64_t GetSigned64Magic(int64_t d, int* shift /*out*/) return GetSignedMagic(d, shift); } #endif -} +} // namespace MagicDivide namespace CheckedOps { @@ -4310,4 +4315,4 @@ bool CastFromDoubleOverflows(double fromValue, var_types toType) unreached(); } } -} +} // namespace CheckedOps diff --git a/src/coreclr/jit/utils.h b/src/coreclr/jit/utils.h index 0b1b6840be6ec3..39001f32215ef5 100644 --- a/src/coreclr/jit/utils.h +++ b/src/coreclr/jit/utils.h @@ -88,7 +88,9 @@ class IteratorPair TIterator m_end; public: - IteratorPair(TIterator begin, TIterator end) : m_begin(begin), m_end(end) + IteratorPair(TIterator begin, TIterator end) + : m_begin(begin) + , m_end(end) { } @@ -116,7 +118,8 @@ struct ConstLog2 { enum { - value = ConstLog2::value + value = ConstLog2 < val / 2, + acc + 1 > ::value }; }; @@ -246,7 +249,9 @@ class ConfigMethodRange class ConfigIntArray { public: - ConfigIntArray() : m_values(nullptr), m_length(0) + ConfigIntArray() + : m_values(nullptr) + , m_length(0) { } @@ -270,7 +275,7 @@ class ConfigIntArray } private: - void Init(const WCHAR* str); + void Init(const WCHAR* str); int* m_values; unsigned m_length; }; @@ -280,7 +285,9 @@ class ConfigIntArray class ConfigDoubleArray { public: - ConfigDoubleArray() : m_values(nullptr), m_length(0) + ConfigDoubleArray() + : m_values(nullptr) + , m_length(0) { } @@ -304,7 +311,7 @@ class ConfigDoubleArray } private: - void Init(const WCHAR* str); + void Init(const WCHAR* str); double* m_values; unsigned m_length; }; @@ -404,7 +411,8 @@ template class ScopedSetVariable { public: - ScopedSetVariable(T* pVariable, T value) : m_pVariable(pVariable) + ScopedSetVariable(T* pVariable, T value) + : m_pVariable(pVariable) { m_oldValue = *m_pVariable; *m_pVariable = value; @@ -442,7 +450,8 @@ class PhasedVar public: PhasedVar() #ifdef DEBUG - : m_initialized(false), m_writePhase(true) + : m_initialized(false) + , m_writePhase(true) #endif // DEBUG { } @@ -704,7 +713,9 @@ class MethodSet MethodInfo* m_next; MethodInfo(char* methodName, int methodHash) - : m_MethodName(methodName), m_MethodHash(methodHash), m_next(nullptr) + : m_MethodName(methodName) + , m_MethodHash(methodHash) + , m_next(nullptr) { } }; @@ -786,8 +797,8 @@ unsigned CountDigits(double num, unsigned base = 10); #endif // DEBUG /***************************************************************************** -* Floating point utility class -*/ + * Floating point utility class + */ class FloatingPointUtils { public: @@ -1019,7 +1030,7 @@ class CritSecObject CRITSEC_COOKIE m_pCs; // No copying or assignment allowed. - CritSecObject(const CritSecObject&) = delete; + CritSecObject(const CritSecObject&) = delete; CritSecObject& operator=(const CritSecObject&) = delete; }; @@ -1029,7 +1040,8 @@ class CritSecObject class CritSecHolder { public: - CritSecHolder(CritSecObject& critSec) : m_CritSec(critSec) + CritSecHolder(CritSecObject& critSec) + : m_CritSec(critSec) { ClrEnterCriticalSection(m_CritSec.Val()); } @@ -1043,7 +1055,7 @@ class CritSecHolder CritSecObject& m_CritSec; // No copying or assignment allowed. - CritSecHolder(const CritSecHolder&) = delete; + CritSecHolder(const CritSecHolder&) = delete; CritSecHolder& operator=(const CritSecHolder&) = delete; }; @@ -1059,7 +1071,7 @@ int32_t GetSigned32Magic(int32_t d, int* shift /*out*/); #ifdef TARGET_64BIT int64_t GetSigned64Magic(int64_t d, int* shift /*out*/); #endif -} +} // namespace MagicDivide // // Profiling helpers @@ -1160,6 +1172,6 @@ bool CastFromIntOverflows(int32_t fromValue, var_types toType, bool fromUnsigned bool CastFromLongOverflows(int64_t fromValue, var_types toType, bool fromUnsigned); bool CastFromFloatOverflows(float fromValue, var_types toType); bool CastFromDoubleOverflows(double fromValue, var_types toType); -} +} // namespace CheckedOps #endif // _UTILS_H_ diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 7ebde5995cf732..889e0227e992d9 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -45,7 +45,7 @@ struct FloatTraits #if defined(TARGET_XARCH) unsigned bits = 0xFFC00000u; #elif defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - unsigned bits = 0x7FC00000u; + unsigned bits = 0x7FC00000u; #else #error Unsupported or unset target architecture #endif @@ -1647,7 +1647,11 @@ bool ValueNumStore::IsSharedStatic(ValueNum vn) } ValueNumStore::Chunk::Chunk(CompAllocator alloc, ValueNum* pNextBaseVN, var_types typ, ChunkExtraAttribs attribs) - : m_defs(nullptr), m_numUsed(0), m_baseVN(*pNextBaseVN), m_typ(typ), m_attribs(attribs) + : m_defs(nullptr) + , m_numUsed(0) + , m_baseVN(*pNextBaseVN) + , m_typ(typ) + , m_attribs(attribs) { // Allocate "m_defs" here, according to the typ/attribs pair. switch (attribs) @@ -2971,7 +2975,8 @@ typedef JitHashTable, bool> ValueN class SmallValueNumSet { - union { + union + { ValueNum m_inlineElements[4]; ValueNumSet* m_set; }; @@ -3416,7 +3421,7 @@ ValueNum ValueNumStore::VNForMapSelectWork(ValueNumKind vnk, { bool usedRecursiveVN = false; ValueNum curResult = VNForMapSelectWork(vnk, type, phiArgVN, index, pBudget, - &usedRecursiveVN, recMemoryDependencies); + &usedRecursiveVN, recMemoryDependencies); *pUsedRecursiveVN |= usedRecursiveVN; if (sameSelResult == ValueNumStore::RecursiveVN) @@ -3449,8 +3454,9 @@ ValueNum ValueNumStore::VNForMapSelectWork(ValueNumKind vnk, GetMapSelectWorkCache()->Set(fstruct, entry); } - recMemoryDependencies.ForEach( - [this, &memoryDependencies](ValueNum vn) { memoryDependencies.Add(m_pComp, vn); }); + recMemoryDependencies.ForEach([this, &memoryDependencies](ValueNum vn) { + memoryDependencies.Add(m_pComp, vn); + }); return sameSelResult; } @@ -3485,7 +3491,9 @@ ValueNum ValueNumStore::VNForMapSelectWork(ValueNumKind vnk, GetMapSelectWorkCache()->Set(fstruct, entry); } - recMemoryDependencies.ForEach([this, &memoryDependencies](ValueNum vn) { memoryDependencies.Add(m_pComp, vn); }); + recMemoryDependencies.ForEach([this, &memoryDependencies](ValueNum vn) { + memoryDependencies.Add(m_pComp, vn); + }); return entry.Result; } @@ -5610,7 +5618,7 @@ ValueNum ValueNumStore::ExtendPtrVN(GenTree* opA, FieldSeq* fldSeq, ssize_t offs { fldSeq = m_pComp->GetFieldSeqStore()->Append(FieldSeqVNToFieldSeq(funcApp.m_args[1]), fldSeq); res = VNForFunc(TYP_BYREF, VNF_PtrToStatic, funcApp.m_args[0], VNForFieldSeq(fldSeq), - VNForIntPtrCon(ConstantValue(funcApp.m_args[2]) + offset)); + VNForIntPtrCon(ConstantValue(funcApp.m_args[2]) + offset)); } else if (funcApp.m_func == VNF_PtrToArrElem) { @@ -5653,7 +5661,6 @@ void Compiler::fgValueNumberLocalStore(GenTree* storeNode, auto processDef = [=](unsigned defLclNum, unsigned defSsaNum, ssize_t defOffset, unsigned defSize, ValueNumPair defValue) { - LclVarDsc* defVarDsc = lvaGetDesc(defLclNum); if (defSsaNum != SsaConfig::RESERVED_SSA_NUM) @@ -12189,8 +12196,8 @@ void Compiler::fgValueNumberCastTree(GenTree* tree) ValueNum ValueNumStore::VNForCast(ValueNum srcVN, var_types castToType, var_types castFromType, - bool srcIsUnsigned, /* = false */ - bool hasOverflowCheck) /* = false */ + bool srcIsUnsigned, /* = false */ + bool hasOverflowCheck) /* = false */ { if ((castFromType == TYP_I_IMPL) && (castToType == TYP_BYREF) && IsVNHandle(srcVN)) @@ -12235,8 +12242,8 @@ ValueNum ValueNumStore::VNForCast(ValueNum srcVN, ValueNumPair ValueNumStore::VNPairForCast(ValueNumPair srcVNPair, var_types castToType, var_types castFromType, - bool srcIsUnsigned, /* = false */ - bool hasOverflowCheck) /* = false */ + bool srcIsUnsigned, /* = false */ + bool hasOverflowCheck) /* = false */ { ValueNum srcLibVN = srcVNPair.GetLiberal(); ValueNum srcConVN = srcVNPair.GetConservative(); @@ -13742,7 +13749,6 @@ void Compiler::fgDebugCheckExceptionSets() ValueNumPair operandsExcSet = vnStore->VNPForEmptyExcSet(); tree->VisitOperands([&](GenTree* operand) -> GenTree::VisitResult { - CheckTree(operand, vnStore); ValueNumPair operandVNP = operand->gtVNPair.BothDefined() ? operand->gtVNPair : vnStore->VNPForVoid(); @@ -13796,7 +13802,7 @@ void Compiler::JitTestCheckVN() // First we have to know which nodes in the tree are reachable. typedef JitHashTable, int> NodeToIntMap; - NodeToIntMap* reachable = FindReachableNodesInNodeTestData(); + NodeToIntMap* reachable = FindReachableNodesInNodeTestData(); LabelToVNMap* labelToVN = new (getAllocatorDebugOnly()) LabelToVNMap(getAllocatorDebugOnly()); VNToLabelMap* vnToLabel = new (getAllocatorDebugOnly()) VNToLabelMap(getAllocatorDebugOnly()); @@ -13931,7 +13937,9 @@ void Compiler::vnPrint(ValueNum vn, unsigned level) #endif // DEBUG // Methods of ValueNumPair. -ValueNumPair::ValueNumPair() : m_liberal(ValueNumStore::NoVN), m_conservative(ValueNumStore::NoVN) +ValueNumPair::ValueNumPair() + : m_liberal(ValueNumStore::NoVN) + , m_conservative(ValueNumStore::NoVN) { } diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index a976893a06581f..aec401e486ff3f 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -245,7 +245,8 @@ class ValueNumStore class VNMap : public JitHashTable { public: - VNMap(CompAllocator alloc) : JitHashTable(alloc) + VNMap(CompAllocator alloc) + : JitHashTable(alloc) { } @@ -313,7 +314,7 @@ class ValueNumStore bool illegalAsVNFunc, GenTreeOperKind kind); static constexpr uint8_t GetOpAttribsForFunc(int arity, bool commute, bool knownNonNull, bool sharedStatic); - static const uint8_t s_vnfOpAttribs[]; + static const uint8_t s_vnfOpAttribs[]; // Returns "true" iff gtOper is a legal value number function. // (Requires InitValueNumStoreStatics to have been run.) @@ -362,18 +363,18 @@ class ValueNumStore public: // Given an constant value number return its value. - int GetConstantInt32(ValueNum argVN); - INT64 GetConstantInt64(ValueNum argVN); + int GetConstantInt32(ValueNum argVN); + INT64 GetConstantInt64(ValueNum argVN); double GetConstantDouble(ValueNum argVN); - float GetConstantSingle(ValueNum argVN); + float GetConstantSingle(ValueNum argVN); #if defined(FEATURE_SIMD) - simd8_t GetConstantSimd8(ValueNum argVN); + simd8_t GetConstantSimd8(ValueNum argVN); simd12_t GetConstantSimd12(ValueNum argVN); simd16_t GetConstantSimd16(ValueNum argVN); #if defined(TARGET_XARCH) - simd32_t GetConstantSimd32(ValueNum argVN); - simd64_t GetConstantSimd64(ValueNum argVN); + simd32_t GetConstantSimd32(ValueNum argVN); + simd64_t GetConstantSimd64(ValueNum argVN); simdmask_t GetConstantSimdMask(ValueNum argVN); #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -567,7 +568,7 @@ class ValueNumStore // Create or return the existimg value number representing a singleton exception set // for the exception value "x". - ValueNum VNExcSetSingleton(ValueNum x); + ValueNum VNExcSetSingleton(ValueNum x); ValueNumPair VNPExcSetSingleton(ValueNumPair x); // Returns true if the current pair of items are in ascending order and they are not duplicates. @@ -821,7 +822,7 @@ class ValueNumStore return ValueNumPair(liberalFuncVN, conservativeFuncVN); } - ValueNum VNForExpr(BasicBlock* block, var_types type = TYP_UNKNOWN); + ValueNum VNForExpr(BasicBlock* block, var_types type = TYP_UNKNOWN); ValueNumPair VNPairForExpr(BasicBlock* block, var_types type); // This controls extra tracing of the "evaluation" of "VNF_MapSelect" functions. @@ -923,7 +924,10 @@ class ValueNumStore ValueNum vnIdx; ValueNum vnBound; - UnsignedCompareCheckedBoundInfo() : cmpOper(GT_NONE), vnIdx(NoVN), vnBound(NoVN) + UnsignedCompareCheckedBoundInfo() + : cmpOper(GT_NONE) + , vnIdx(NoVN) + , vnBound(NoVN) { } }; @@ -937,7 +941,12 @@ class ValueNumStore ValueNum arrOp; unsigned cmpOper; ValueNum cmpOp; - CompareCheckedBoundArithInfo() : vnBound(NoVN), arrOper(GT_NONE), arrOp(NoVN), cmpOper(GT_NONE), cmpOp(NoVN) + CompareCheckedBoundArithInfo() + : vnBound(NoVN) + , arrOper(GT_NONE) + , arrOp(NoVN) + , cmpOper(GT_NONE) + , cmpOp(NoVN) { } #ifdef DEBUG @@ -965,7 +974,11 @@ class ValueNumStore ValueNum cmpOpVN; bool isUnsigned; - ConstantBoundInfo() : constVal(0), cmpOper(GT_NONE), cmpOpVN(NoVN), isUnsigned(false) + ConstantBoundInfo() + : constVal(0) + , cmpOper(GT_NONE) + , cmpOpVN(NoVN) + , isUnsigned(false) { } @@ -1314,7 +1327,8 @@ class ValueNumStore VNFunc m_func; ValueNum m_args[NumArgs]; - VNDefFuncApp() : m_func(VNF_COUNT) + VNDefFuncApp() + : m_func(VNF_COUNT) { for (size_t i = 0; i < NumArgs; i++) { @@ -1323,7 +1337,9 @@ class ValueNumStore } template - VNDefFuncApp(VNFunc func, VNs... vns) : m_func(func), m_args{vns...} + VNDefFuncApp(VNFunc func, VNs... vns) + : m_func(func) + , m_args{vns...} { static_assert_no_msg(NumArgs == sizeof...(VNs)); } @@ -1484,7 +1500,7 @@ class ValueNumStore static const int SmallIntConstMin = -1; static const int SmallIntConstMax = 10; static const unsigned SmallIntConstNum = SmallIntConstMax - SmallIntConstMin + 1; - static bool IsSmallIntConst(int i) + static bool IsSmallIntConst(int i) { return SmallIntConstMin <= i && i <= SmallIntConstMax; } @@ -1494,7 +1510,9 @@ class ValueNumStore { ValueNum vn; ValueNumList* next; - ValueNumList(const ValueNum& v, ValueNumList* n = nullptr) : vn(v), next(n) + ValueNumList(const ValueNum& v, ValueNumList* n = nullptr) + : vn(v) + , next(n) { } }; @@ -1525,8 +1543,8 @@ class ValueNumStore } typedef VNMap HandleToValueNumMap; - HandleToValueNumMap* m_handleMap; - HandleToValueNumMap* GetHandleMap() + HandleToValueNumMap* m_handleMap; + HandleToValueNumMap* GetHandleMap() { if (m_handleMap == nullptr) { @@ -1536,10 +1554,10 @@ class ValueNumStore } typedef SmallHashTable EmbeddedToCompileTimeHandleMap; - EmbeddedToCompileTimeHandleMap m_embeddedToCompileTimeHandleMap; + EmbeddedToCompileTimeHandleMap m_embeddedToCompileTimeHandleMap; typedef SmallHashTable FieldAddressToFieldSeqMap; - FieldAddressToFieldSeqMap m_fieldAddressToFieldSeqMap; + FieldAddressToFieldSeqMap m_fieldAddressToFieldSeqMap; struct LargePrimitiveKeyFuncsFloat : public JitLargePrimitiveKeyFuncs { @@ -1550,8 +1568,8 @@ class ValueNumStore }; typedef VNMap FloatToValueNumMap; - FloatToValueNumMap* m_floatCnsMap; - FloatToValueNumMap* GetFloatCnsMap() + FloatToValueNumMap* m_floatCnsMap; + FloatToValueNumMap* GetFloatCnsMap() { if (m_floatCnsMap == nullptr) { @@ -1570,8 +1588,8 @@ class ValueNumStore }; typedef VNMap DoubleToValueNumMap; - DoubleToValueNumMap* m_doubleCnsMap; - DoubleToValueNumMap* GetDoubleCnsMap() + DoubleToValueNumMap* m_doubleCnsMap; + DoubleToValueNumMap* GetDoubleCnsMap() { if (m_doubleCnsMap == nullptr) { @@ -1611,8 +1629,8 @@ class ValueNumStore }; typedef VNMap Simd8ToValueNumMap; - Simd8ToValueNumMap* m_simd8CnsMap; - Simd8ToValueNumMap* GetSimd8CnsMap() + Simd8ToValueNumMap* m_simd8CnsMap; + Simd8ToValueNumMap* GetSimd8CnsMap() { if (m_simd8CnsMap == nullptr) { @@ -1641,8 +1659,8 @@ class ValueNumStore }; typedef VNMap Simd12ToValueNumMap; - Simd12ToValueNumMap* m_simd12CnsMap; - Simd12ToValueNumMap* GetSimd12CnsMap() + Simd12ToValueNumMap* m_simd12CnsMap; + Simd12ToValueNumMap* GetSimd12CnsMap() { if (m_simd12CnsMap == nullptr) { @@ -1672,8 +1690,8 @@ class ValueNumStore }; typedef VNMap Simd16ToValueNumMap; - Simd16ToValueNumMap* m_simd16CnsMap; - Simd16ToValueNumMap* GetSimd16CnsMap() + Simd16ToValueNumMap* m_simd16CnsMap; + Simd16ToValueNumMap* GetSimd16CnsMap() { if (m_simd16CnsMap == nullptr) { @@ -1708,8 +1726,8 @@ class ValueNumStore }; typedef VNMap Simd32ToValueNumMap; - Simd32ToValueNumMap* m_simd32CnsMap; - Simd32ToValueNumMap* GetSimd32CnsMap() + Simd32ToValueNumMap* m_simd32CnsMap; + Simd32ToValueNumMap* GetSimd32CnsMap() { if (m_simd32CnsMap == nullptr) { @@ -1751,8 +1769,8 @@ class ValueNumStore }; typedef VNMap Simd64ToValueNumMap; - Simd64ToValueNumMap* m_simd64CnsMap; - Simd64ToValueNumMap* GetSimd64CnsMap() + Simd64ToValueNumMap* m_simd64CnsMap; + Simd64ToValueNumMap* GetSimd64CnsMap() { if (m_simd64CnsMap == nullptr) { @@ -1780,8 +1798,8 @@ class ValueNumStore }; typedef VNMap SimdMaskToValueNumMap; - SimdMaskToValueNumMap* m_simdMaskCnsMap; - SimdMaskToValueNumMap* GetSimdMaskCnsMap() + SimdMaskToValueNumMap* m_simdMaskCnsMap; + SimdMaskToValueNumMap* GetSimdMaskCnsMap() { if (m_simdMaskCnsMap == nullptr) { @@ -1820,8 +1838,8 @@ class ValueNumStore } typedef VNMap, VNDefFuncAppKeyFuncs<1>> VNFunc1ToValueNumMap; - VNFunc1ToValueNumMap* m_VNFunc1Map; - VNFunc1ToValueNumMap* GetVNFunc1Map() + VNFunc1ToValueNumMap* m_VNFunc1Map; + VNFunc1ToValueNumMap* GetVNFunc1Map() { if (m_VNFunc1Map == nullptr) { @@ -1831,8 +1849,8 @@ class ValueNumStore } typedef VNMap, VNDefFuncAppKeyFuncs<2>> VNFunc2ToValueNumMap; - VNFunc2ToValueNumMap* m_VNFunc2Map; - VNFunc2ToValueNumMap* GetVNFunc2Map() + VNFunc2ToValueNumMap* m_VNFunc2Map; + VNFunc2ToValueNumMap* GetVNFunc2Map() { if (m_VNFunc2Map == nullptr) { @@ -1842,8 +1860,8 @@ class ValueNumStore } typedef VNMap, VNDefFuncAppKeyFuncs<3>> VNFunc3ToValueNumMap; - VNFunc3ToValueNumMap* m_VNFunc3Map; - VNFunc3ToValueNumMap* GetVNFunc3Map() + VNFunc3ToValueNumMap* m_VNFunc3Map; + VNFunc3ToValueNumMap* GetVNFunc3Map() { if (m_VNFunc3Map == nullptr) { @@ -1853,8 +1871,8 @@ class ValueNumStore } typedef VNMap, VNDefFuncAppKeyFuncs<4>> VNFunc4ToValueNumMap; - VNFunc4ToValueNumMap* m_VNFunc4Map; - VNFunc4ToValueNumMap* GetVNFunc4Map() + VNFunc4ToValueNumMap* m_VNFunc4Map; + VNFunc4ToValueNumMap* GetVNFunc4Map() { if (m_VNFunc4Map == nullptr) { @@ -1865,7 +1883,8 @@ class ValueNumStore class MapSelectWorkCacheEntry { - union { + union + { ValueNum* m_memoryDependencies; ValueNum m_inlineMemoryDependencies[sizeof(ValueNum*) / sizeof(ValueNum)]; }; diff --git a/src/coreclr/jit/valuenumtype.h b/src/coreclr/jit/valuenumtype.h index 2eb3254e3e18b5..e41db972675434 100644 --- a/src/coreclr/jit/valuenumtype.h +++ b/src/coreclr/jit/valuenumtype.h @@ -115,7 +115,9 @@ struct ValueNumPair // Initializes both elements to "NoVN". Defined in ValueNum.cpp. ValueNumPair(); - ValueNumPair(ValueNum lib, ValueNum cons) : m_liberal(lib), m_conservative(cons) + ValueNumPair(ValueNum lib, ValueNum cons) + : m_liberal(lib) + , m_conservative(cons) { } diff --git a/src/coreclr/jit/varset.h b/src/coreclr/jit/varset.h index 465ab146cbaca3..b9e4cab1a0c458 100644 --- a/src/coreclr/jit/varset.h +++ b/src/coreclr/jit/varset.h @@ -108,7 +108,7 @@ typedef BitSetOpsWithCounter VarSetOps; #else -typedef VarSetOpsRaw VarSetOps; +typedef VarSetOpsRaw VarSetOps; #endif #define ALLVARSET_REP BSShortLong diff --git a/src/coreclr/jit/vartype.h b/src/coreclr/jit/vartype.h index 1623addb69b079..642ab159360350 100644 --- a/src/coreclr/jit/vartype.h +++ b/src/coreclr/jit/vartype.h @@ -225,7 +225,7 @@ inline bool varTypeIsIntOrI(T vt) #ifdef TARGET_64BIT || (TypeGet(vt) == TYP_I_IMPL) #endif // TARGET_64BIT - ); + ); } template @@ -321,11 +321,11 @@ inline bool varTypeUsesFloatReg(T vt) template inline bool varTypeUsesMaskReg(T vt) { -// The technically correct check is: -// return varTypeRegister[TypeGet(vt)] == VTR_MASK; -// -// However, we only have one type that uses VTR_MASK today -// and so its quite a bit cheaper to just check that directly + // The technically correct check is: + // return varTypeRegister[TypeGet(vt)] == VTR_MASK; + // + // However, we only have one type that uses VTR_MASK today + // and so its quite a bit cheaper to just check that directly #if defined(FEATURE_SIMD) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) assert((TypeGet(vt) == TYP_MASK) || (varTypeRegister[TypeGet(vt)] != VTR_MASK)); From 82a351873595ae111126d4b4cbb08ec03abd825a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 Apr 2024 12:20:36 -0700 Subject: [PATCH 162/201] jit format 17.0.6 --- src/coreclr/jit/block.h | 1600 ++++++++++++++--------------- src/coreclr/jit/codegenlinear.cpp | 5 +- src/coreclr/jit/emit.h | 2 +- src/coreclr/jit/lsra.h | 2 +- 4 files changed, 805 insertions(+), 804 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 14cc9b1fbb73be..aad6d8b1b676d7 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1544,1039 +1544,1039 @@ struct BasicBlock : private LIR::Range // TODO-Cleanup: Get rid of bbStkDepth and use bbStackDepthOnEntry() instead { - unsigned short bbStkDepth; // stack depth on entry + unsigned short bbStkDepth; // stack depth on entry - // Basic block predecessor lists. Predecessor lists are created by fgLinkBasicBlocks(), stored - // in 'bbPreds', and then maintained throughout compilation. 'fgPredsComputed' will be 'true' after the - // predecessor lists are created. - // - FlowEdge* bbPreds; // ptr to list of predecessors + // Basic block predecessor lists. Predecessor lists are created by fgLinkBasicBlocks(), stored + // in 'bbPreds', and then maintained throughout compilation. 'fgPredsComputed' will be 'true' after the + // predecessor lists are created. + // + FlowEdge* bbPreds; // ptr to list of predecessors - // PredEdges: convenience method for enabling range-based `for` iteration over predecessor edges, e.g.: - // for (FlowEdge* const edge : block->PredEdges()) ... - // - PredEdgeList PredEdges() const - { - return PredEdgeList(bbPreds); - } + // PredEdges: convenience method for enabling range-based `for` iteration over predecessor edges, e.g.: + // for (FlowEdge* const edge : block->PredEdges()) ... + // + PredEdgeList PredEdges() const + { + return PredEdgeList(bbPreds); + } - // PredBlocks: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: - // for (BasicBlock* const predBlock : block->PredBlocks()) ... - // - PredBlockList PredBlocks() const - { - return PredBlockList(bbPreds); - } + // PredBlocks: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: + // for (BasicBlock* const predBlock : block->PredBlocks()) ... + // + PredBlockList PredBlocks() const + { + return PredBlockList(bbPreds); + } - // PredBlocksEditing: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: - // for (BasicBlock* const predBlock : block->PredBlocksEditing()) ... - // This iterator tolerates modifications to bbPreds. - // - PredBlockList PredBlocksEditing() const - { - return PredBlockList(bbPreds); - } + // PredBlocksEditing: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: + // for (BasicBlock* const predBlock : block->PredBlocksEditing()) ... + // This iterator tolerates modifications to bbPreds. + // + PredBlockList PredBlocksEditing() const + { + return PredBlockList(bbPreds); + } - // Pred list maintenance - // - bool checkPredListOrder(); - void ensurePredListOrder(Compiler* compiler); - void reorderPredList(Compiler* compiler); + // Pred list maintenance + // + bool checkPredListOrder(); + void ensurePredListOrder(Compiler * compiler); + void reorderPredList(Compiler * compiler); - union - { - BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate - // Dominator) used to compute the dominance tree. - FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred - void* bbSparseProbeList; // Used early on by fgInstrument - }; + union + { + BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate + // Dominator) used to compute the dominance tree. + FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred + void* bbSparseProbeList; // Used early on by fgInstrument + }; - void* bbSparseCountInfo; // Used early on by fgIncorporateEdgeCounts + void* bbSparseCountInfo; // Used early on by fgIncorporateEdgeCounts - unsigned bbPreorderNum; // the block's preorder number in the graph [0...postOrderCount) - unsigned bbPostorderNum; // the block's postorder number in the graph [0...postOrderCount) + unsigned bbPreorderNum; // the block's preorder number in the graph [0...postOrderCount) + unsigned bbPostorderNum; // the block's postorder number in the graph [0...postOrderCount) - IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block - IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) - // range is not inclusive of the end offset. The count of IL bytes in the block - // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. + IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block + IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) + // range is not inclusive of the end offset. The count of IL bytes in the block + // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. #ifdef DEBUG - void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" for - // BAD_IL_OFFSET. -#endif // DEBUG - - VARSET_TP bbVarUse; // variables used by block (before a definition) - VARSET_TP bbVarDef; // variables assigned by block (before a use) + void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" + // for BAD_IL_OFFSET. +#endif // DEBUG - VARSET_TP bbLiveIn; // variables live on entry - VARSET_TP bbLiveOut; // variables live on exit + VARSET_TP bbVarUse; // variables used by block (before a definition) + VARSET_TP bbVarDef; // variables assigned by block (before a use) - // Use, def, live in/out information for the implicit memory variable. - MemoryKindSet bbMemoryUse : MemoryKindCount; // must be set for any MemoryKinds this block references - MemoryKindSet bbMemoryDef : MemoryKindCount; // must be set for any MemoryKinds this block mutates - MemoryKindSet bbMemoryLiveIn : MemoryKindCount; - MemoryKindSet bbMemoryLiveOut : MemoryKindCount; - MemoryKindSet bbMemoryHavoc : MemoryKindCount; // If true, at some point the block does an operation - // that leaves memory in an unknown state. (E.g., - // unanalyzed call, store through unknown pointer...) + VARSET_TP bbLiveIn; // variables live on entry + VARSET_TP bbLiveOut; // variables live on exit - // We want to make phi functions for the special implicit var memory. But since this is not a real - // lclVar, and thus has no local #, we can't use a GenTreePhiArg. Instead, we use this struct. - struct MemoryPhiArg - { - unsigned m_ssaNum; // SSA# for incoming value. - MemoryPhiArg* m_nextArg; // Next arg in the list, else NULL. + // Use, def, live in/out information for the implicit memory variable. + MemoryKindSet bbMemoryUse : MemoryKindCount; // must be set for any MemoryKinds this block references + MemoryKindSet bbMemoryDef : MemoryKindCount; // must be set for any MemoryKinds this block mutates + MemoryKindSet bbMemoryLiveIn : MemoryKindCount; + MemoryKindSet bbMemoryLiveOut : MemoryKindCount; + MemoryKindSet bbMemoryHavoc : MemoryKindCount; // If true, at some point the block does an operation + // that leaves memory in an unknown state. (E.g., + // unanalyzed call, store through unknown pointer...) - unsigned GetSsaNum() + // We want to make phi functions for the special implicit var memory. But since this is not a real + // lclVar, and thus has no local #, we can't use a GenTreePhiArg. Instead, we use this struct. + struct MemoryPhiArg { - return m_ssaNum; - } + unsigned m_ssaNum; // SSA# for incoming value. + MemoryPhiArg* m_nextArg; // Next arg in the list, else NULL. - MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) - : m_ssaNum(ssaNum) - , m_nextArg(nextArg) - { - } + unsigned GetSsaNum() + { + return m_ssaNum; + } - void* operator new(size_t sz, class Compiler* comp); - }; - static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list - // for Heap. - MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this value - // is NULL. - // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate - // that Heap needs a phi definition on entry, or else it is the linked list - // of the phi arguments. - unsigned bbMemorySsaNumIn[MemoryKindCount]; // The SSA # of memory on entry to the block. - unsigned bbMemorySsaNumOut[MemoryKindCount]; // The SSA # of memory on exit from the block. - - VARSET_TP bbScope; // variables in scope over the block - - void InitVarSets(class Compiler* comp); - - /* The following are the standard bit sets for dataflow analysis. - * We perform CSE and range-checks at the same time - * and assertion propagation separately, - * thus we can union them since the two operations are completely disjunct. - */ + MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) + : m_ssaNum(ssaNum) + , m_nextArg(nextArg) + { + } - union - { - EXPSET_TP bbCseGen; // CSEs computed by block - ASSERT_TP bbAssertionGen; // assertions created by block (global prop) - ASSERT_TP bbAssertionOutIfTrue; // assertions available on exit along true/jump edge (BBJ_COND, local prop) - }; + void* operator new(size_t sz, class Compiler* comp); + }; + static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list + // for Heap. + MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this + // value is NULL. + // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate + // that Heap needs a phi definition on entry, or else it is the linked list + // of the phi arguments. + unsigned bbMemorySsaNumIn[MemoryKindCount]; // The SSA # of memory on entry to the block. + unsigned bbMemorySsaNumOut[MemoryKindCount]; // The SSA # of memory on exit from the block. + + VARSET_TP bbScope; // variables in scope over the block + + void InitVarSets(class Compiler * comp); + + /* The following are the standard bit sets for dataflow analysis. + * We perform CSE and range-checks at the same time + * and assertion propagation separately, + * thus we can union them since the two operations are completely disjunct. + */ + + union + { + EXPSET_TP bbCseGen; // CSEs computed by block + ASSERT_TP bbAssertionGen; // assertions created by block (global prop) + ASSERT_TP bbAssertionOutIfTrue; // assertions available on exit along true/jump edge (BBJ_COND, local prop) + }; - union - { - EXPSET_TP bbCseIn; // CSEs available on entry - ASSERT_TP bbAssertionIn; // assertions available on entry (global prop) - }; + union + { + EXPSET_TP bbCseIn; // CSEs available on entry + ASSERT_TP bbAssertionIn; // assertions available on entry (global prop) + }; - union - { - EXPSET_TP bbCseOut; // CSEs available on exit - ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) - ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local prop) - }; + union + { + EXPSET_TP bbCseOut; // CSEs available on exit + ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) + ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local + // prop) + }; - void* bbEmitCookie; + void* bbEmitCookie; - //------------------------------------------------------------------------- + //------------------------------------------------------------------------- #if MEASURE_BLOCK_SIZE - static size_t s_Size; - static size_t s_Count; + static size_t s_Size; + static size_t s_Count; #endif // MEASURE_BLOCK_SIZE - bool bbFallsThrough() const; + bool bbFallsThrough() const; - // Our slop fraction is 1/50 of the block weight. - static weight_t GetSlopFraction(weight_t weightBlk) - { - return weightBlk / 50.0; - } + // Our slop fraction is 1/50 of the block weight. + static weight_t GetSlopFraction(weight_t weightBlk) + { + return weightBlk / 50.0; + } - // Given an the edge b1 -> b2, calculate the slop fraction by - // using the higher of the two block weights - static weight_t GetSlopFraction(BasicBlock* b1, BasicBlock* b2) - { - return GetSlopFraction(max(b1->bbWeight, b2->bbWeight)); - } + // Given an the edge b1 -> b2, calculate the slop fraction by + // using the higher of the two block weights + static weight_t GetSlopFraction(BasicBlock * b1, BasicBlock * b2) + { + return GetSlopFraction(max(b1->bbWeight, b2->bbWeight)); + } #ifdef DEBUG - unsigned bbTgtStkDepth; // Native stack depth on entry (for throw-blocks) - static unsigned s_nMaxTrees; // The max # of tree nodes in any BB + unsigned bbTgtStkDepth; // Native stack depth on entry (for throw-blocks) + static unsigned s_nMaxTrees; // The max # of tree nodes in any BB - // This is used in integrity checks. We semi-randomly pick a traversal stamp, label all blocks - // in the BB list with that stamp (in this field); then we can tell if (e.g.) predecessors are - // still in the BB list by whether they have the same stamp (with high probability). - unsigned bbTraversalStamp; + // This is used in integrity checks. We semi-randomly pick a traversal stamp, label all blocks + // in the BB list with that stamp (in this field); then we can tell if (e.g.) predecessors are + // still in the BB list by whether they have the same stamp (with high probability). + unsigned bbTraversalStamp; - // bbID is a unique block identifier number that does not change: it does not get renumbered, like bbNum. - unsigned bbID; + // bbID is a unique block identifier number that does not change: it does not get renumbered, like bbNum. + unsigned bbID; #endif // DEBUG - unsigned bbStackDepthOnEntry() const; - void bbSetStack(StackEntry* stack); - StackEntry* bbStackOnEntry() const; + unsigned bbStackDepthOnEntry() const; + void bbSetStack(StackEntry * stack); + StackEntry* bbStackOnEntry() const; - // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding - // zero-based number for use as an array index. - unsigned bbInd() const - { - assert(bbNum > 0); - return bbNum - 1; - } + // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding + // zero-based number for use as an array index. + unsigned bbInd() const + { + assert(bbNum > 0); + return bbNum - 1; + } - Statement* firstStmt() const; - Statement* lastStmt() const; - bool hasSingleStmt() const; + Statement* firstStmt() const; + Statement* lastStmt() const; + bool hasSingleStmt() const; - // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: - // for (Statement* const stmt : block->Statements()) - // - StatementList Statements() const - { - return StatementList(firstStmt()); - } + // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: + // for (Statement* const stmt : block->Statements()) + // + StatementList Statements() const + { + return StatementList(firstStmt()); + } - // NonPhiStatements: convenience method for enabling range-based `for` iteration over the statement list, - // excluding any initial PHI statements, e.g.: - // for (Statement* const stmt : block->NonPhiStatements()) - // - StatementList NonPhiStatements() const - { - return StatementList(FirstNonPhiDef()); - } + // NonPhiStatements: convenience method for enabling range-based `for` iteration over the statement list, + // excluding any initial PHI statements, e.g.: + // for (Statement* const stmt : block->NonPhiStatements()) + // + StatementList NonPhiStatements() const + { + return StatementList(FirstNonPhiDef()); + } - GenTree* lastNode() const; + GenTree* lastNode() const; - bool endsWithJmpMethod(Compiler* comp) const; + bool endsWithJmpMethod(Compiler * comp) const; - bool endsWithTailCall(Compiler* comp, - bool fastTailCallsOnly, - bool tailCallsConvertibleToLoopOnly, - GenTree** tailCall) const; + bool endsWithTailCall(Compiler * comp, bool fastTailCallsOnly, bool tailCallsConvertibleToLoopOnly, + GenTree** tailCall) const; - bool endsWithTailCallOrJmp(Compiler* comp, bool fastTailCallsOnly = false) const; + bool endsWithTailCallOrJmp(Compiler * comp, bool fastTailCallsOnly = false) const; - bool endsWithTailCallConvertibleToLoop(Compiler* comp, GenTree** tailCall) const; + bool endsWithTailCallConvertibleToLoop(Compiler * comp, GenTree * *tailCall) const; - // Returns the first statement in the statement list of "this" that is - // not an SSA definition (a lcl = phi(...) store). - Statement* FirstNonPhiDef() const; - Statement* FirstNonPhiDefOrCatchArgStore() const; + // Returns the first statement in the statement list of "this" that is + // not an SSA definition (a lcl = phi(...) store). + Statement* FirstNonPhiDef() const; + Statement* FirstNonPhiDefOrCatchArgStore() const; - BasicBlock() - : bbStmtList(nullptr) - , bbLiveIn(VarSetOps::UninitVal()) - , bbLiveOut(VarSetOps::UninitVal()) - { - } - - // Iteratable collection of successors of a block. - template - class Successors - { - Compiler* m_comp; - BasicBlock* m_block; - - public: - Successors(Compiler* comp, BasicBlock* block) - : m_comp(comp) - , m_block(block) + BasicBlock() + : bbStmtList(nullptr) + , bbLiveIn(VarSetOps::UninitVal()) + , bbLiveOut(VarSetOps::UninitVal()) { } - class iterator + // Iteratable collection of successors of a block. + template + class Successors { Compiler* m_comp; BasicBlock* m_block; - TPosition m_pos; public: - iterator(Compiler* comp, BasicBlock* block) + Successors(Compiler* comp, BasicBlock* block) : m_comp(comp) , m_block(block) - , m_pos(comp, block) { } - iterator() - : m_pos() + class iterator { - } - - void operator++(void) - { - m_pos.Advance(m_comp, m_block); - } - - BasicBlock* operator*() - { - return m_pos.Current(m_comp, m_block); - } - - bool operator==(const iterator& other) + Compiler* m_comp; + BasicBlock* m_block; + TPosition m_pos; + + public: + iterator(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) + , m_pos(comp, block) + { + } + + iterator() + : m_pos() + { + } + + void operator++(void) + { + m_pos.Advance(m_comp, m_block); + } + + BasicBlock* operator*() + { + return m_pos.Current(m_comp, m_block); + } + + bool operator==(const iterator& other) + { + return m_pos == other.m_pos; + } + + bool operator!=(const iterator& other) + { + return m_pos != other.m_pos; + } + }; + + iterator begin() { - return m_pos == other.m_pos; + return iterator(m_comp, m_block); } - bool operator!=(const iterator& other) + iterator end() { - return m_pos != other.m_pos; + return iterator(); } }; - iterator begin() - { - return iterator(m_comp, m_block); - } - - iterator end() - { - return iterator(); - } - }; - - template - BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler* comp, TFunc func); + template + BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler * comp, TFunc func); - template - BasicBlockVisit VisitAllSuccs(Compiler* comp, TFunc func); + template + BasicBlockVisit VisitAllSuccs(Compiler * comp, TFunc func); - template - BasicBlockVisit VisitEHSuccs(Compiler* comp, TFunc func); + template + BasicBlockVisit VisitEHSuccs(Compiler * comp, TFunc func); - template - BasicBlockVisit VisitRegularSuccs(Compiler* comp, TFunc func); + template + BasicBlockVisit VisitRegularSuccs(Compiler * comp, TFunc func); - bool HasPotentialEHSuccs(Compiler* comp); + bool HasPotentialEHSuccs(Compiler * comp); - // Base class for Successor block/edge iterators. - // - class SuccList - { - protected: - // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't - // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end - // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at - // `m_succs` or at the switch table successor array. - FlowEdge* m_succs[2]; - FlowEdge* const* m_begin; - FlowEdge* const* m_end; - - SuccList(const BasicBlock* block); - }; - - // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs()) ... - // - class BBSuccList : private SuccList - { - public: - BBSuccList(const BasicBlock* block) - : SuccList(block) + // Base class for Successor block/edge iterators. + // + class SuccList { - } + protected: + // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't + // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end + // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at + // `m_succs` or at the switch table successor array. + FlowEdge* m_succs[2]; + FlowEdge* const* m_begin; + FlowEdge* const* m_end; + + SuccList(const BasicBlock* block); + }; - BBArrayIterator begin() const + // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs()) ... + // + class BBSuccList : private SuccList { - return BBArrayIterator(m_begin); - } + public: + BBSuccList(const BasicBlock* block) + : SuccList(block) + { + } - BBArrayIterator end() const - { - return BBArrayIterator(m_end); - } - }; + BBArrayIterator begin() const + { + return BBArrayIterator(m_begin); + } - // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, - // normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges()) ... - // - class BBSuccEdgeList : private SuccList - { - public: - BBSuccEdgeList(const BasicBlock* block) - : SuccList(block) - { - } + BBArrayIterator end() const + { + return BBArrayIterator(m_end); + } + }; - FlowEdgeArrayIterator begin() const + // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, + // normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges()) ... + // + class BBSuccEdgeList : private SuccList { - return FlowEdgeArrayIterator(m_begin); - } + public: + BBSuccEdgeList(const BasicBlock* block) + : SuccList(block) + { + } - FlowEdgeArrayIterator end() const - { - return FlowEdgeArrayIterator(m_end); - } - }; + FlowEdgeArrayIterator begin() const + { + return FlowEdgeArrayIterator(m_begin); + } - // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs(compiler)) ... - // - // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation - // of the implications of this versus the version that does not take `Compiler*`. - class BBCompilerSuccList - { - Compiler* m_comp; - BasicBlock* m_block; + FlowEdgeArrayIterator end() const + { + return FlowEdgeArrayIterator(m_end); + } + }; - // iterator: forward iterator for an array of BasicBlock* + // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs(compiler)) ... // - class iterator + // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation + // of the implications of this versus the version that does not take `Compiler*`. + class BBCompilerSuccList { Compiler* m_comp; BasicBlock* m_block; - unsigned m_succNum; + + // iterator: forward iterator for an array of BasicBlock* + // + class iterator + { + Compiler* m_comp; + BasicBlock* m_block; + unsigned m_succNum; + + public: + iterator(Compiler* comp, BasicBlock* block, unsigned succNum) + : m_comp(comp) + , m_block(block) + , m_succNum(succNum) + { + } + + BasicBlock* operator*() const + { + assert(m_block != nullptr); + BasicBlock* bTarget = m_block->GetSucc(m_succNum, m_comp); + assert(bTarget != nullptr); + return bTarget; + } + + iterator& operator++() + { + ++m_succNum; + return *this; + } + + bool operator!=(const iterator& i) const + { + return m_succNum != i.m_succNum; + } + }; public: - iterator(Compiler* comp, BasicBlock* block, unsigned succNum) + BBCompilerSuccList(Compiler* comp, BasicBlock* block) : m_comp(comp) , m_block(block) - , m_succNum(succNum) - { - } - - BasicBlock* operator*() const { - assert(m_block != nullptr); - BasicBlock* bTarget = m_block->GetSucc(m_succNum, m_comp); - assert(bTarget != nullptr); - return bTarget; } - iterator& operator++() + iterator begin() const { - ++m_succNum; - return *this; + return iterator(m_comp, m_block, 0); } - bool operator!=(const iterator& i) const + iterator end() const { - return m_succNum != i.m_succNum; + return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); } }; - public: - BBCompilerSuccList(Compiler* comp, BasicBlock* block) - : m_comp(comp) - , m_block(block) - { - } - - iterator begin() const - { - return iterator(m_comp, m_block, 0); - } - - iterator end() const - { - return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); - } - }; - - // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, - // normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... - // - // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation - // of the implications of this versus the version that does not take `Compiler*`. - class BBCompilerSuccEdgeList - { - Compiler* m_comp; - BasicBlock* m_block; - - // iterator: forward iterator for an array of BasicBlock* + // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based + // `for`, normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... // - class iterator + // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation + // of the implications of this versus the version that does not take `Compiler*`. + class BBCompilerSuccEdgeList { Compiler* m_comp; BasicBlock* m_block; - unsigned m_succNum; + + // iterator: forward iterator for an array of BasicBlock* + // + class iterator + { + Compiler* m_comp; + BasicBlock* m_block; + unsigned m_succNum; + + public: + iterator(Compiler* comp, BasicBlock* block, unsigned succNum) + : m_comp(comp) + , m_block(block) + , m_succNum(succNum) + { + } + + FlowEdge* operator*() const + { + assert(m_block != nullptr); + FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); + assert(succEdge != nullptr); + return succEdge; + } + + iterator& operator++() + { + ++m_succNum; + return *this; + } + + bool operator!=(const iterator& i) const + { + return m_succNum != i.m_succNum; + } + }; public: - iterator(Compiler* comp, BasicBlock* block, unsigned succNum) + BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) : m_comp(comp) , m_block(block) - , m_succNum(succNum) - { - } - - FlowEdge* operator*() const { - assert(m_block != nullptr); - FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); - assert(succEdge != nullptr); - return succEdge; } - iterator& operator++() + iterator begin() const { - ++m_succNum; - return *this; + return iterator(m_comp, m_block, 0); } - bool operator!=(const iterator& i) const + iterator end() const { - return m_succNum != i.m_succNum; + return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); } }; - public: - BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) - : m_comp(comp) - , m_block(block) + // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: + // for (BasicBlock* const succ : block->Succs()) ... + // + // There are two options: one that takes a Compiler* and one that doesn't. These correspond to the + // NumSucc()/GetSucc() functions that do or do not take a Compiler*. See the comment for NumSucc()/GetSucc() + // for the distinction. + BBSuccList Succs() const { + return BBSuccList(this); } - iterator begin() const + BBCompilerSuccList Succs(Compiler * comp) { - return iterator(m_comp, m_block, 0); + return BBCompilerSuccList(comp, this); } - iterator end() const + BBSuccEdgeList SuccEdges() { - return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); + return BBSuccEdgeList(this); } - }; - // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: - // for (BasicBlock* const succ : block->Succs()) ... - // - // There are two options: one that takes a Compiler* and one that doesn't. These correspond to the - // NumSucc()/GetSucc() functions that do or do not take a Compiler*. See the comment for NumSucc()/GetSucc() - // for the distinction. - BBSuccList Succs() const - { - return BBSuccList(this); - } - - BBCompilerSuccList Succs(Compiler* comp) - { - return BBCompilerSuccList(comp, this); - } - - BBSuccEdgeList SuccEdges() - { - return BBSuccEdgeList(this); - } - - BBCompilerSuccEdgeList SuccEdges(Compiler* comp) - { - return BBCompilerSuccEdgeList(comp, this); - } + BBCompilerSuccEdgeList SuccEdges(Compiler * comp) + { + return BBCompilerSuccEdgeList(comp, this); + } - // Clone block state and statements from `from` block to `to` block (which must be new/empty) - static void CloneBlockState(Compiler* compiler, BasicBlock* to, const BasicBlock* from); + // Clone block state and statements from `from` block to `to` block (which must be new/empty) + static void CloneBlockState(Compiler * compiler, BasicBlock * to, const BasicBlock* from); - // Copy the block kind and take memory ownership of the targets. - void TransferTarget(BasicBlock* from); + // Copy the block kind and take memory ownership of the targets. + void TransferTarget(BasicBlock * from); - void MakeLIR(GenTree* firstNode, GenTree* lastNode); - bool IsLIR() const; + void MakeLIR(GenTree * firstNode, GenTree * lastNode); + bool IsLIR() const; - void SetDominatedByExceptionalEntryFlag() - { - SetFlags(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); - } + void SetDominatedByExceptionalEntryFlag() + { + SetFlags(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); + } - bool IsDominatedByExceptionalEntryFlag() const - { - return HasFlag(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); - } + bool IsDominatedByExceptionalEntryFlag() const + { + return HasFlag(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); + } #ifdef DEBUG - bool Contains(const GenTree* node) const - { - assert(IsLIR()); - for (Iterator iter = begin(); iter != end(); ++iter) + bool Contains(const GenTree* node) const { - if (*iter == node) + assert(IsLIR()); + for (Iterator iter = begin(); iter != end(); ++iter) { - return true; + if (*iter == node) + { + return true; + } } + return false; } - return false; - } #endif // DEBUG -}; - -template <> -struct JitPtrKeyFuncs : public JitKeyFuncsDefEquals -{ -public: - // Make sure hashing is deterministic and not on "ptr." - static unsigned GetHashCode(const BasicBlock* ptr); -}; + }; -// A set of blocks. -typedef JitHashTable, bool> BlkSet; + template <> + struct JitPtrKeyFuncs : public JitKeyFuncsDefEquals + { + public: + // Make sure hashing is deterministic and not on "ptr." + static unsigned GetHashCode(const BasicBlock* ptr); + }; -// A vector of blocks. -typedef jitstd::vector BlkVector; + // A set of blocks. + typedef JitHashTable, bool> BlkSet; -// A map of block -> set of blocks, can be used as sparse block trees. -typedef JitHashTable, BlkSet*> BlkToBlkSetMap; + // A vector of blocks. + typedef jitstd::vector BlkVector; -// A map of block -> vector of blocks, can be used as sparse block trees. -typedef JitHashTable, BlkVector> BlkToBlkVectorMap; + // A map of block -> set of blocks, can be used as sparse block trees. + typedef JitHashTable, BlkSet*> BlkToBlkSetMap; -// Map from Block to Block. Used for a variety of purposes. -typedef JitHashTable, BasicBlock*> BlockToBlockMap; + // A map of block -> vector of blocks, can be used as sparse block trees. + typedef JitHashTable, BlkVector> BlkToBlkVectorMap; -// BasicBlockIterator: forward iterator for the BasicBlock linked list. -// It is allowed to make changes to the BasicBlock list as long as the current block remains in the list. -// E.g., the current block `m_bbNext` pointer can be altered (such as when inserting a following block), -// as long as the current block is still in the list. -// The block list is expected to be properly doubly-linked. -// -class BasicBlockIterator -{ - BasicBlock* m_block; + // Map from Block to Block. Used for a variety of purposes. + typedef JitHashTable, BasicBlock*> BlockToBlockMap; -public: - BasicBlockIterator(BasicBlock* block) - : m_block(block) + // BasicBlockIterator: forward iterator for the BasicBlock linked list. + // It is allowed to make changes to the BasicBlock list as long as the current block remains in the list. + // E.g., the current block `m_bbNext` pointer can be altered (such as when inserting a following block), + // as long as the current block is still in the list. + // The block list is expected to be properly doubly-linked. + // + class BasicBlockIterator { - } + BasicBlock* m_block; - BasicBlock* operator*() const - { - return m_block; - } + public: + BasicBlockIterator(BasicBlock* block) + : m_block(block) + { + } - BasicBlockIterator& operator++() - { - assert(m_block != nullptr); - // Check that we haven't been spliced out of the list. - assert(m_block->IsLast() || m_block->Next()->PrevIs(m_block)); - assert(m_block->IsFirst() || m_block->Prev()->NextIs(m_block)); + BasicBlock* operator*() const + { + return m_block; + } - m_block = m_block->Next(); - return *this; - } + BasicBlockIterator& operator++() + { + assert(m_block != nullptr); + // Check that we haven't been spliced out of the list. + assert(m_block->IsLast() || m_block->Next()->PrevIs(m_block)); + assert(m_block->IsFirst() || m_block->Prev()->NextIs(m_block)); - bool operator!=(const BasicBlockIterator& i) const - { - return m_block != i.m_block; - } -}; + m_block = m_block->Next(); + return *this; + } -// BasicBlockSimpleList: adapter class for forward iteration of a lexically contiguous range of -// BasicBlock, starting at `begin` and going to the end of the function, using range-based `for`, -// normally used via Compiler::Blocks(), e.g.: -// for (BasicBlock* const block : Blocks()) ... -// -class BasicBlockSimpleList -{ - BasicBlock* m_begin; + bool operator!=(const BasicBlockIterator& i) const + { + return m_block != i.m_block; + } + }; -public: - BasicBlockSimpleList(BasicBlock* begin) - : m_begin(begin) + // BasicBlockSimpleList: adapter class for forward iteration of a lexically contiguous range of + // BasicBlock, starting at `begin` and going to the end of the function, using range-based `for`, + // normally used via Compiler::Blocks(), e.g.: + // for (BasicBlock* const block : Blocks()) ... + // + class BasicBlockSimpleList { - } + BasicBlock* m_begin; - BasicBlockIterator begin() const - { - return BasicBlockIterator(m_begin); - } + public: + BasicBlockSimpleList(BasicBlock* begin) + : m_begin(begin) + { + } - BasicBlockIterator end() const - { - return BasicBlockIterator(nullptr); - } -}; + BasicBlockIterator begin() const + { + return BasicBlockIterator(m_begin); + } -// BasicBlockRangeList: adapter class for forward iteration of a lexically contiguous range of -// BasicBlock specified with both `begin` and `end` blocks. `begin` and `end` are *inclusive* -// and must be non-null. E.g., -// for (BasicBlock* const block : BasicBlockRangeList(startBlock, endBlock)) ... -// -// Note that endBlock->bbNext is captured at the beginning of the iteration. Thus, any blocks -// inserted before that will continue the iteration. In particular, inserting blocks between endBlock -// and endBlock->bbNext will yield unexpected results, as the iteration will continue longer than desired. -// -class BasicBlockRangeList -{ - BasicBlock* m_begin; - BasicBlock* m_end; + BasicBlockIterator end() const + { + return BasicBlockIterator(nullptr); + } + }; -public: - BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) - : m_begin(begin) - , m_end(end) + // BasicBlockRangeList: adapter class for forward iteration of a lexically contiguous range of + // BasicBlock specified with both `begin` and `end` blocks. `begin` and `end` are *inclusive* + // and must be non-null. E.g., + // for (BasicBlock* const block : BasicBlockRangeList(startBlock, endBlock)) ... + // + // Note that endBlock->bbNext is captured at the beginning of the iteration. Thus, any blocks + // inserted before that will continue the iteration. In particular, inserting blocks between endBlock + // and endBlock->bbNext will yield unexpected results, as the iteration will continue longer than desired. + // + class BasicBlockRangeList { - assert(begin != nullptr); - assert(end != nullptr); - } + BasicBlock* m_begin; + BasicBlock* m_end; - BasicBlockIterator begin() const - { - return BasicBlockIterator(m_begin); - } + public: + BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) + : m_begin(begin) + , m_end(end) + { + assert(begin != nullptr); + assert(end != nullptr); + } + + BasicBlockIterator begin() const + { + return BasicBlockIterator(m_begin); + } + + BasicBlockIterator end() const + { + return BasicBlockIterator(m_end->Next()); // walk until we see the block *following* the `m_end` block + } + }; - BasicBlockIterator end() const + // BBswtDesc -- descriptor for a switch block + // + // Things to know: + // 1. If bbsHasDefault is true, the default case is the last one in the array of basic block addresses + // namely bbsDstTab[bbsCount - 1]. + // 2. bbsCount must be at least 1, for the default case. bbsCount cannot be zero. It appears that the ECMA spec + // allows for a degenerate switch with zero cases. Normally, the optimizer will optimize degenerate + // switches with just a default case to a BBJ_ALWAYS branch, and a switch with just two cases to a BBJ_COND. + // However, in debuggable code, we might not do that, so bbsCount might be 1. + // + struct BBswtDesc { - return BasicBlockIterator(m_end->Next()); // walk until we see the block *following* the `m_end` block - } -}; + FlowEdge** bbsDstTab; // case label table address + unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) -// BBswtDesc -- descriptor for a switch block -// -// Things to know: -// 1. If bbsHasDefault is true, the default case is the last one in the array of basic block addresses -// namely bbsDstTab[bbsCount - 1]. -// 2. bbsCount must be at least 1, for the default case. bbsCount cannot be zero. It appears that the ECMA spec -// allows for a degenerate switch with zero cases. Normally, the optimizer will optimize degenerate -// switches with just a default case to a BBJ_ALWAYS branch, and a switch with just two cases to a BBJ_COND. -// However, in debuggable code, we might not do that, so bbsCount might be 1. -// -struct BBswtDesc -{ - FlowEdge** bbsDstTab; // case label table address - unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) + // Case number and likelihood of most likely case + // (only known with PGO, only valid if bbsHasDominantCase is true) + unsigned bbsDominantCase; + weight_t bbsDominantFraction; - // Case number and likelihood of most likely case - // (only known with PGO, only valid if bbsHasDominantCase is true) - unsigned bbsDominantCase; - weight_t bbsDominantFraction; + bool bbsHasDefault; // true if last switch case is a default case + bool bbsHasDominantCase; // true if switch has a dominant case - bool bbsHasDefault; // true if last switch case is a default case - bool bbsHasDominantCase; // true if switch has a dominant case + BBswtDesc() + : bbsHasDefault(true) + , bbsHasDominantCase(false) + { + } - BBswtDesc() - : bbsHasDefault(true) - , bbsHasDominantCase(false) - { - } + BBswtDesc(const BBswtDesc* other); - BBswtDesc(const BBswtDesc* other); + BBswtDesc(Compiler* comp, const BBswtDesc* other); - BBswtDesc(Compiler* comp, const BBswtDesc* other); + void removeDefault() + { + assert(bbsHasDefault); + assert(bbsCount > 0); + bbsHasDefault = false; + bbsCount--; + } - void removeDefault() + FlowEdge* getDefault() + { + assert(bbsHasDefault); + assert(bbsCount > 0); + return bbsDstTab[bbsCount - 1]; + } + }; + + // BBSwitchTargetList out-of-class-declaration implementations (here due to C++ ordering requirements). + // + + inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) + : m_bbsDesc(bbsDesc) { - assert(bbsHasDefault); - assert(bbsCount > 0); - bbsHasDefault = false; - bbsCount--; + assert(m_bbsDesc != nullptr); + assert(m_bbsDesc->bbsDstTab != nullptr); } - FlowEdge* getDefault() + inline BBArrayIterator BBSwitchTargetList::begin() const { - assert(bbsHasDefault); - assert(bbsCount > 0); - return bbsDstTab[bbsCount - 1]; + return BBArrayIterator(m_bbsDesc->bbsDstTab); } -}; -// BBSwitchTargetList out-of-class-declaration implementations (here due to C++ ordering requirements). -// + inline BBArrayIterator BBSwitchTargetList::end() const + { + return BBArrayIterator(m_bbsDesc->bbsDstTab + m_bbsDesc->bbsCount); + } -inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) - : m_bbsDesc(bbsDesc) -{ - assert(m_bbsDesc != nullptr); - assert(m_bbsDesc->bbsDstTab != nullptr); -} + // BBehfDesc -- descriptor for a BBJ_EHFINALLYRET block + // + struct BBehfDesc + { + FlowEdge** bbeSuccs; // array of `FlowEdge*` pointing to BBJ_EHFINALLYRET block successors + unsigned bbeCount; // size of `bbeSuccs` array -inline BBArrayIterator BBSwitchTargetList::begin() const -{ - return BBArrayIterator(m_bbsDesc->bbsDstTab); -} + BBehfDesc() + : bbeSuccs(nullptr) + , bbeCount(0) + { + } -inline BBArrayIterator BBSwitchTargetList::end() const -{ - return BBArrayIterator(m_bbsDesc->bbsDstTab + m_bbsDesc->bbsCount); -} + BBehfDesc(Compiler* comp, const BBehfDesc* other); + }; -// BBehfDesc -- descriptor for a BBJ_EHFINALLYRET block -// -struct BBehfDesc -{ - FlowEdge** bbeSuccs; // array of `FlowEdge*` pointing to BBJ_EHFINALLYRET block successors - unsigned bbeCount; // size of `bbeSuccs` array + // BBEhfSuccList out-of-class-declaration implementations (here due to C++ ordering requirements). + // - BBehfDesc() - : bbeSuccs(nullptr) - , bbeCount(0) + inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) + : m_bbeDesc(bbeDesc) { + assert(m_bbeDesc != nullptr); + assert((m_bbeDesc->bbeSuccs != nullptr) || (m_bbeDesc->bbeCount == 0)); } - BBehfDesc(Compiler* comp, const BBehfDesc* other); -}; - -// BBEhfSuccList out-of-class-declaration implementations (here due to C++ ordering requirements). -// - -inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) - : m_bbeDesc(bbeDesc) -{ - assert(m_bbeDesc != nullptr); - assert((m_bbeDesc->bbeSuccs != nullptr) || (m_bbeDesc->bbeCount == 0)); -} - -inline BBArrayIterator BBEhfSuccList::begin() const -{ - return BBArrayIterator(m_bbeDesc->bbeSuccs); -} + inline BBArrayIterator BBEhfSuccList::begin() const + { + return BBArrayIterator(m_bbeDesc->bbeSuccs); + } -inline BBArrayIterator BBEhfSuccList::end() const -{ - return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); -} + inline BBArrayIterator BBEhfSuccList::end() const + { + return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); + } -// SuccList out-of-class-declaration implementations -// -inline BasicBlock::SuccList::SuccList(const BasicBlock* block) -{ - assert(block != nullptr); - - switch (block->bbKind) - { - case BBJ_THROW: - case BBJ_RETURN: - case BBJ_EHFAULTRET: - // We don't need m_succs. - m_begin = nullptr; - m_end = nullptr; - break; - - case BBJ_CALLFINALLY: - case BBJ_CALLFINALLYRET: - case BBJ_ALWAYS: - case BBJ_EHCATCHRET: - case BBJ_EHFILTERRET: - case BBJ_LEAVE: - m_succs[0] = block->GetTargetEdge(); - m_begin = &m_succs[0]; - m_end = &m_succs[1]; - break; - - case BBJ_COND: - m_succs[0] = block->GetFalseEdge(); - m_begin = &m_succs[0]; - - // If both fall-through and branch successors are identical, then only include - // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). - if (block->TrueEdgeIs(block->GetFalseEdge())) - { - m_end = &m_succs[1]; - } - else - { - m_succs[1] = block->GetTrueEdge(); - m_end = &m_succs[2]; - } - break; + // SuccList out-of-class-declaration implementations + // + inline BasicBlock::SuccList::SuccList(const BasicBlock* block) + { + assert(block != nullptr); - case BBJ_EHFINALLYRET: - // We don't use the m_succs in-line data; use the existing successor table in the block. - // We must tolerate iterating successors early in the system, before EH_FINALLYRET successors have - // been computed. - if (block->GetEhfTargets() == nullptr) - { + switch (block->bbKind) + { + case BBJ_THROW: + case BBJ_RETURN: + case BBJ_EHFAULTRET: + // We don't need m_succs. m_begin = nullptr; m_end = nullptr; - } - else - { - m_begin = block->GetEhfTargets()->bbeSuccs; - m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; - } - break; - - case BBJ_SWITCH: - // We don't use the m_succs in-line data for switches; use the existing jump table in the block. - assert(block->bbSwtTargets != nullptr); - assert(block->bbSwtTargets->bbsDstTab != nullptr); - m_begin = block->bbSwtTargets->bbsDstTab; - m_end = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; - break; + break; + + case BBJ_CALLFINALLY: + case BBJ_CALLFINALLYRET: + case BBJ_ALWAYS: + case BBJ_EHCATCHRET: + case BBJ_EHFILTERRET: + case BBJ_LEAVE: + m_succs[0] = block->GetTargetEdge(); + m_begin = &m_succs[0]; + m_end = &m_succs[1]; + break; + + case BBJ_COND: + m_succs[0] = block->GetFalseEdge(); + m_begin = &m_succs[0]; + + // If both fall-through and branch successors are identical, then only include + // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). + if (block->TrueEdgeIs(block->GetFalseEdge())) + { + m_end = &m_succs[1]; + } + else + { + m_succs[1] = block->GetTrueEdge(); + m_end = &m_succs[2]; + } + break; + + case BBJ_EHFINALLYRET: + // We don't use the m_succs in-line data; use the existing successor table in the block. + // We must tolerate iterating successors early in the system, before EH_FINALLYRET successors have + // been computed. + if (block->GetEhfTargets() == nullptr) + { + m_begin = nullptr; + m_end = nullptr; + } + else + { + m_begin = block->GetEhfTargets()->bbeSuccs; + m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; + } + break; + + case BBJ_SWITCH: + // We don't use the m_succs in-line data for switches; use the existing jump table in the block. + assert(block->bbSwtTargets != nullptr); + assert(block->bbSwtTargets->bbsDstTab != nullptr); + m_begin = block->bbSwtTargets->bbsDstTab; + m_end = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; + break; + + default: + unreached(); + } - default: - unreached(); + assert(m_end >= m_begin); } - assert(m_end >= m_begin); -} + // We have a simpler struct, BasicBlockList, which is simply a singly-linked + // list of blocks. -// We have a simpler struct, BasicBlockList, which is simply a singly-linked -// list of blocks. + struct BasicBlockList + { + BasicBlockList* next; // The next BasicBlock in the list, nullptr for end of list. + BasicBlock* block; // The BasicBlock of interest. -struct BasicBlockList -{ - BasicBlockList* next; // The next BasicBlock in the list, nullptr for end of list. - BasicBlock* block; // The BasicBlock of interest. + BasicBlockList() + : next(nullptr) + , block(nullptr) + { + } - BasicBlockList() - : next(nullptr) - , block(nullptr) - { - } + BasicBlockList(BasicBlock* blk, BasicBlockList* rest) + : next(rest) + , block(blk) + { + } + }; + + // FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) - BasicBlockList(BasicBlock* blk, BasicBlockList* rest) - : next(rest) - , block(blk) + inline weight_t FlowEdge::getLikelyWeight() const { + assert(m_likelihoodSet); + return m_likelihood * m_sourceBlock->bbWeight; } -}; - -// FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) -inline weight_t FlowEdge::getLikelyWeight() const -{ - assert(m_likelihoodSet); - return m_likelihood * m_sourceBlock->bbWeight; -} + // BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) -// BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) - -inline BasicBlock* BBArrayIterator::operator*() const -{ - assert(m_edgeEntry != nullptr); - FlowEdge* edgeTarget = *m_edgeEntry; - assert(edgeTarget != nullptr); - assert(edgeTarget->getDestinationBlock() != nullptr); - return edgeTarget->getDestinationBlock(); -} + inline BasicBlock* BBArrayIterator::operator*() const + { + assert(m_edgeEntry != nullptr); + FlowEdge* edgeTarget = *m_edgeEntry; + assert(edgeTarget != nullptr); + assert(edgeTarget->getDestinationBlock() != nullptr); + return edgeTarget->getDestinationBlock(); + } -// Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and FlowEdge) + // Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and + // FlowEdge) -inline PredEdgeList::iterator::iterator(FlowEdge* pred) - : m_pred(pred) -{ + inline PredEdgeList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) + { #ifdef DEBUG - m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); + m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); #endif -} + } -inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() -{ - FlowEdge* next = m_pred->getNextPredEdge(); + inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() + { + FlowEdge* next = m_pred->getNextPredEdge(); #ifdef DEBUG - // Check that the next block is the one we expect to see. - assert(next == m_next); - m_next = (next == nullptr) ? nullptr : next->getNextPredEdge(); + // Check that the next block is the one we expect to see. + assert(next == m_next); + m_next = (next == nullptr) ? nullptr : next->getNextPredEdge(); #endif // DEBUG - m_pred = next; - return *this; -} + m_pred = next; + return *this; + } -template -inline PredBlockList::iterator::iterator(FlowEdge* pred) - : m_pred(pred) -{ - bool initNextPointer = allowEdits; - INDEBUG(initNextPointer = true); - if (initNextPointer) + template + inline PredBlockList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) { - m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); + bool initNextPointer = allowEdits; + INDEBUG(initNextPointer = true); + if (initNextPointer) + { + m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); + } } -} - -template -inline BasicBlock* PredBlockList::iterator::operator*() const -{ - return m_pred->getSourceBlock(); -} -template -inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() -{ - if (allowEdits) + template + inline BasicBlock* PredBlockList::iterator::operator*() const { - // For editing iterators, m_next is always used and maintained - m_pred = m_next; - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); + return m_pred->getSourceBlock(); } - else + + template + inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() { - FlowEdge* next = m_pred->getNextPredEdge(); + if (allowEdits) + { + // For editing iterators, m_next is always used and maintained + m_pred = m_next; + m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); + } + else + { + FlowEdge* next = m_pred->getNextPredEdge(); #ifdef DEBUG - // If allowEdits=false, check that the next block is the one we expect to see. - assert(next == m_next); - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); + // If allowEdits=false, check that the next block is the one we expect to see. + assert(next == m_next); + m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); #endif // DEBUG - m_pred = next; - } + m_pred = next; + } - return *this; -} + return *this; + } -/***************************************************************************** - * - * The following call-backs supplied by the client; it's used by the code - * emitter to convert a basic block to its corresponding emitter cookie. - */ + /***************************************************************************** + * + * The following call-backs supplied by the client; it's used by the code + * emitter to convert a basic block to its corresponding emitter cookie. + */ -void* emitCodeGetCookie(const BasicBlock* block); + void* emitCodeGetCookie(const BasicBlock* block); -// An enumerator of a block's all successors. In some cases (e.g. SsaBuilder::TopologicalSort) -// using iterators is not exactly efficient, at least because they contain an unnecessary -// member - a pointer to the Compiler object. -class AllSuccessorEnumerator -{ - BasicBlock* m_block; - union + // An enumerator of a block's all successors. In some cases (e.g. SsaBuilder::TopologicalSort) + // using iterators is not exactly efficient, at least because they contain an unnecessary + // member - a pointer to the Compiler object. + class AllSuccessorEnumerator { - // We store up to 4 successors inline in the enumerator. For ASP.NET - // and libraries.pmi this is enough in 99.7% of cases. - BasicBlock* m_successors[4]; - BasicBlock** m_pSuccessors; - }; - - unsigned m_numSuccs; - unsigned m_curSucc = UINT_MAX; + BasicBlock* m_block; + union + { + // We store up to 4 successors inline in the enumerator. For ASP.NET + // and libraries.pmi this is enough in 99.7% of cases. + BasicBlock* m_successors[4]; + BasicBlock** m_pSuccessors; + }; -public: - // Constructs an enumerator of all `block`'s successors. - AllSuccessorEnumerator(Compiler* comp, BasicBlock* block); + unsigned m_numSuccs; + unsigned m_curSucc = UINT_MAX; - // Gets the block whose successors are enumerated. - BasicBlock* Block() - { - return m_block; - } + public: + // Constructs an enumerator of all `block`'s successors. + AllSuccessorEnumerator(Compiler* comp, BasicBlock* block); - // Returns the next available successor or `nullptr` if there are no more successors. - BasicBlock* NextSuccessor() - { - m_curSucc++; - if (m_curSucc >= m_numSuccs) + // Gets the block whose successors are enumerated. + BasicBlock* Block() { - return nullptr; + return m_block; } - if (m_numSuccs <= ArrLen(m_successors)) + // Returns the next available successor or `nullptr` if there are no more successors. + BasicBlock* NextSuccessor() { - return m_successors[m_curSucc]; - } + m_curSucc++; + if (m_curSucc >= m_numSuccs) + { + return nullptr; + } - return m_pSuccessors[m_curSucc]; - } -}; + if (m_numSuccs <= ArrLen(m_successors)) + { + return m_successors[m_curSucc]; + } -// Simple dominator tree node that keeps track of a node's first child and next sibling. -// The parent is provided by BasicBlock::bbIDom. -struct DomTreeNode -{ - BasicBlock* firstChild; - BasicBlock* nextSibling; -}; + return m_pSuccessors[m_curSucc]; + } + }; + + // Simple dominator tree node that keeps track of a node's first child and next sibling. + // The parent is provided by BasicBlock::bbIDom. + struct DomTreeNode + { + BasicBlock* firstChild; + BasicBlock* nextSibling; + }; /*****************************************************************************/ #endif // _BLOCK_H_ -/*****************************************************************************/ + /*****************************************************************************/ diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 213dd75fc6dcb5..61f23d80d20aa6 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -156,8 +156,9 @@ void CodeGen::genCodeForBBlist() genMarkLabelsForCodegen(); - assert(!compiler->fgFirstBBScratch || compiler->fgFirstBB == compiler->fgFirstBBScratch); // compiler->fgFirstBBScratch - // has to be first. + assert(!compiler->fgFirstBBScratch || + compiler->fgFirstBB == compiler->fgFirstBBScratch); // compiler->fgFirstBBScratch + // has to be first. /* Initialize structures used in the block list iteration */ genInitialize(); diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index eb4c4c377b5639..37755723ebf88a 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3233,7 +3233,7 @@ class emitter void emitDispMaskRegSet(regMaskPredicate regs); #endif // FEATURE_MASKED_HW_INTRINSICS void emitDispRegSet(AllRegsMask regs); - void emitDispVarSet(); + void emitDispVarSet(); #endif void emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 17612ba69bf003..ad1d7447475ef7 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1883,7 +1883,7 @@ class LinearScan : public LinearScanInterface AllRegsMask m_RegistersWithConstants; AllRegsMask fixedRegs; - void clearConstantReg(regNumber reg, var_types regType) + void clearConstantReg(regNumber reg, var_types regType) { // TODO: If we decide to have curr_RegistersWithConstants, then here we will // just operate on curr_RegistersWithConstants and assert From e43bd80b99667ecbfee36fddf1b852e2e36bd1e2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 Apr 2024 15:37:09 -0700 Subject: [PATCH 163/201] jit format --- src/coreclr/jit/block.h | 2 +- src/coreclr/jit/codegenarmarch.cpp | 9 ++++----- src/coreclr/jit/codegenxarch.cpp | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index aad6d8b1b676d7..e606e41c91ac28 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1604,7 +1604,7 @@ struct BasicBlock : private LIR::Range #ifdef DEBUG void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" // for BAD_IL_OFFSET. -#endif // DEBUG +#endif // DEBUG VARSET_TP bbVarUse; // variables used by block (before a definition) VARSET_TP bbVarDef; // variables assigned by block (before a use) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index ef3988285038bd..b2222a90fa1d73 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -5577,11 +5577,10 @@ void CodeGen::genFnEpilog(BasicBlock* block) compiler->unwindSetFrameReg(REG_SAVED_LOCALLOC_SP, 0); } - if (jmpEpilog || - regSet.rsGetModifiedRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) - { - genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); - } + if (jmpEpilog || regSet.rsGetModifiedRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) + { + genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); + } if (!unwindStarted) { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ee8e6c7bbcce1c..ca4c82984d1383 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -10111,7 +10111,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); + regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); regMaskGpr const tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; From 4f546c650246b37ebcfe0e42442127e3fb5fcbbc Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 Apr 2024 10:58:36 -0700 Subject: [PATCH 164/201] some fixes + formatting --- src/coreclr/jit/block.h | 835 ++++++++++++++--------------- src/coreclr/jit/codegenarm64.cpp | 13 +- src/coreclr/jit/codegenarmarch.cpp | 3 +- src/coreclr/jit/codegenxarch.cpp | 13 +- src/coreclr/jit/compiler.hpp | 4 +- src/coreclr/jit/lsra.cpp | 6 +- src/coreclr/jit/lsra.h | 6 +- src/coreclr/jit/lsraarm64.cpp | 7 +- src/coreclr/jit/lsrabuild.cpp | 2 +- src/coreclr/jit/regset.cpp | 3 + src/coreclr/jit/regset.h | 26 - src/coreclr/jit/target.h | 6 +- src/coreclr/jit/targetamd64.h | 23 +- src/coreclr/jit/targetarm.h | 1 + 14 files changed, 461 insertions(+), 487 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index e606e41c91ac28..ab7a28c0c94303 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1543,567 +1543,566 @@ struct BasicBlock : private LIR::Range #define handlerGetsXcptnObj(hndTyp) ((hndTyp) != BBCT_NONE && (hndTyp) != BBCT_FAULT && (hndTyp) != BBCT_FINALLY) // TODO-Cleanup: Get rid of bbStkDepth and use bbStackDepthOnEntry() instead - { - unsigned short bbStkDepth; // stack depth on entry + unsigned short bbStkDepth; // stack depth on entry - // Basic block predecessor lists. Predecessor lists are created by fgLinkBasicBlocks(), stored - // in 'bbPreds', and then maintained throughout compilation. 'fgPredsComputed' will be 'true' after the - // predecessor lists are created. - // - FlowEdge* bbPreds; // ptr to list of predecessors + // Basic block predecessor lists. Predecessor lists are created by fgLinkBasicBlocks(), stored + // in 'bbPreds', and then maintained throughout compilation. 'fgPredsComputed' will be 'true' after the + // predecessor lists are created. + // + FlowEdge* bbPreds; // ptr to list of predecessors - // PredEdges: convenience method for enabling range-based `for` iteration over predecessor edges, e.g.: - // for (FlowEdge* const edge : block->PredEdges()) ... - // - PredEdgeList PredEdges() const - { - return PredEdgeList(bbPreds); - } + // PredEdges: convenience method for enabling range-based `for` iteration over predecessor edges, e.g.: + // for (FlowEdge* const edge : block->PredEdges()) ... + // + PredEdgeList PredEdges() const + { + return PredEdgeList(bbPreds); + } - // PredBlocks: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: - // for (BasicBlock* const predBlock : block->PredBlocks()) ... - // - PredBlockList PredBlocks() const - { - return PredBlockList(bbPreds); - } + // PredBlocks: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: + // for (BasicBlock* const predBlock : block->PredBlocks()) ... + // + PredBlockList PredBlocks() const + { + return PredBlockList(bbPreds); + } - // PredBlocksEditing: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: - // for (BasicBlock* const predBlock : block->PredBlocksEditing()) ... - // This iterator tolerates modifications to bbPreds. - // - PredBlockList PredBlocksEditing() const - { - return PredBlockList(bbPreds); - } + // PredBlocksEditing: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: + // for (BasicBlock* const predBlock : block->PredBlocksEditing()) ... + // This iterator tolerates modifications to bbPreds. + // + PredBlockList PredBlocksEditing() const + { + return PredBlockList(bbPreds); + } - // Pred list maintenance - // - bool checkPredListOrder(); - void ensurePredListOrder(Compiler * compiler); - void reorderPredList(Compiler * compiler); + // Pred list maintenance + // + bool checkPredListOrder(); + void ensurePredListOrder(Compiler * compiler); + void reorderPredList(Compiler * compiler); - union - { - BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate - // Dominator) used to compute the dominance tree. - FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred - void* bbSparseProbeList; // Used early on by fgInstrument - }; + union + { + BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate + // Dominator) used to compute the dominance tree. + FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred + void* bbSparseProbeList; // Used early on by fgInstrument + }; - void* bbSparseCountInfo; // Used early on by fgIncorporateEdgeCounts + void* bbSparseCountInfo; // Used early on by fgIncorporateEdgeCounts - unsigned bbPreorderNum; // the block's preorder number in the graph [0...postOrderCount) - unsigned bbPostorderNum; // the block's postorder number in the graph [0...postOrderCount) + unsigned bbPreorderNum; // the block's preorder number in the graph [0...postOrderCount) + unsigned bbPostorderNum; // the block's postorder number in the graph [0...postOrderCount) - IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block - IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) - // range is not inclusive of the end offset. The count of IL bytes in the block - // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. + IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block + IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) + // range is not inclusive of the end offset. The count of IL bytes in the block + // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. #ifdef DEBUG - void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" - // for BAD_IL_OFFSET. + void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" + // for BAD_IL_OFFSET. #endif // DEBUG - VARSET_TP bbVarUse; // variables used by block (before a definition) - VARSET_TP bbVarDef; // variables assigned by block (before a use) + VARSET_TP bbVarUse; // variables used by block (before a definition) + VARSET_TP bbVarDef; // variables assigned by block (before a use) - VARSET_TP bbLiveIn; // variables live on entry - VARSET_TP bbLiveOut; // variables live on exit + VARSET_TP bbLiveIn; // variables live on entry + VARSET_TP bbLiveOut; // variables live on exit - // Use, def, live in/out information for the implicit memory variable. - MemoryKindSet bbMemoryUse : MemoryKindCount; // must be set for any MemoryKinds this block references - MemoryKindSet bbMemoryDef : MemoryKindCount; // must be set for any MemoryKinds this block mutates - MemoryKindSet bbMemoryLiveIn : MemoryKindCount; - MemoryKindSet bbMemoryLiveOut : MemoryKindCount; - MemoryKindSet bbMemoryHavoc : MemoryKindCount; // If true, at some point the block does an operation - // that leaves memory in an unknown state. (E.g., - // unanalyzed call, store through unknown pointer...) + // Use, def, live in/out information for the implicit memory variable. + MemoryKindSet bbMemoryUse : MemoryKindCount; // must be set for any MemoryKinds this block references + MemoryKindSet bbMemoryDef : MemoryKindCount; // must be set for any MemoryKinds this block mutates + MemoryKindSet bbMemoryLiveIn : MemoryKindCount; + MemoryKindSet bbMemoryLiveOut : MemoryKindCount; + MemoryKindSet bbMemoryHavoc : MemoryKindCount; // If true, at some point the block does an operation + // that leaves memory in an unknown state. (E.g., + // unanalyzed call, store through unknown pointer...) - // We want to make phi functions for the special implicit var memory. But since this is not a real - // lclVar, and thus has no local #, we can't use a GenTreePhiArg. Instead, we use this struct. - struct MemoryPhiArg - { - unsigned m_ssaNum; // SSA# for incoming value. - MemoryPhiArg* m_nextArg; // Next arg in the list, else NULL. + // We want to make phi functions for the special implicit var memory. But since this is not a real + // lclVar, and thus has no local #, we can't use a GenTreePhiArg. Instead, we use this struct. + struct MemoryPhiArg + { + unsigned m_ssaNum; // SSA# for incoming value. + MemoryPhiArg* m_nextArg; // Next arg in the list, else NULL. - unsigned GetSsaNum() - { - return m_ssaNum; - } + unsigned GetSsaNum() + { + return m_ssaNum; + } - MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) - : m_ssaNum(ssaNum) - , m_nextArg(nextArg) - { - } + MemoryPhiArg(unsigned ssaNum, MemoryPhiArg* nextArg = nullptr) + : m_ssaNum(ssaNum) + , m_nextArg(nextArg) + { + } - void* operator new(size_t sz, class Compiler* comp); - }; - static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list - // for Heap. - MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this - // value is NULL. - // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate - // that Heap needs a phi definition on entry, or else it is the linked list - // of the phi arguments. - unsigned bbMemorySsaNumIn[MemoryKindCount]; // The SSA # of memory on entry to the block. - unsigned bbMemorySsaNumOut[MemoryKindCount]; // The SSA # of memory on exit from the block. - - VARSET_TP bbScope; // variables in scope over the block - - void InitVarSets(class Compiler * comp); - - /* The following are the standard bit sets for dataflow analysis. - * We perform CSE and range-checks at the same time - * and assertion propagation separately, - * thus we can union them since the two operations are completely disjunct. - */ + void* operator new(size_t sz, class Compiler* comp); + }; + static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list + // for Heap. + MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this + // value is NULL. + // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate + // that Heap needs a phi definition on entry, or else it is the linked list + // of the phi arguments. + unsigned bbMemorySsaNumIn[MemoryKindCount]; // The SSA # of memory on entry to the block. + unsigned bbMemorySsaNumOut[MemoryKindCount]; // The SSA # of memory on exit from the block. + + VARSET_TP bbScope; // variables in scope over the block + + void InitVarSets(class Compiler * comp); + + /* The following are the standard bit sets for dataflow analysis. + * We perform CSE and range-checks at the same time + * and assertion propagation separately, + * thus we can union them since the two operations are completely disjunct. + */ - union - { - EXPSET_TP bbCseGen; // CSEs computed by block - ASSERT_TP bbAssertionGen; // assertions created by block (global prop) - ASSERT_TP bbAssertionOutIfTrue; // assertions available on exit along true/jump edge (BBJ_COND, local prop) - }; + union + { + EXPSET_TP bbCseGen; // CSEs computed by block + ASSERT_TP bbAssertionGen; // assertions created by block (global prop) + ASSERT_TP bbAssertionOutIfTrue; // assertions available on exit along true/jump edge (BBJ_COND, local prop) + }; - union - { - EXPSET_TP bbCseIn; // CSEs available on entry - ASSERT_TP bbAssertionIn; // assertions available on entry (global prop) - }; + union + { + EXPSET_TP bbCseIn; // CSEs available on entry + ASSERT_TP bbAssertionIn; // assertions available on entry (global prop) + }; - union - { - EXPSET_TP bbCseOut; // CSEs available on exit - ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) - ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local - // prop) - }; + union + { + EXPSET_TP bbCseOut; // CSEs available on exit + ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) + ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local + // prop) + }; - void* bbEmitCookie; + void* bbEmitCookie; - //------------------------------------------------------------------------- + //------------------------------------------------------------------------- #if MEASURE_BLOCK_SIZE - static size_t s_Size; - static size_t s_Count; + static size_t s_Size; + static size_t s_Count; #endif // MEASURE_BLOCK_SIZE - bool bbFallsThrough() const; + bool bbFallsThrough() const; - // Our slop fraction is 1/50 of the block weight. - static weight_t GetSlopFraction(weight_t weightBlk) - { - return weightBlk / 50.0; - } + // Our slop fraction is 1/50 of the block weight. + static weight_t GetSlopFraction(weight_t weightBlk) + { + return weightBlk / 50.0; + } - // Given an the edge b1 -> b2, calculate the slop fraction by - // using the higher of the two block weights - static weight_t GetSlopFraction(BasicBlock * b1, BasicBlock * b2) - { - return GetSlopFraction(max(b1->bbWeight, b2->bbWeight)); - } + // Given an the edge b1 -> b2, calculate the slop fraction by + // using the higher of the two block weights + static weight_t GetSlopFraction(BasicBlock * b1, BasicBlock * b2) + { + return GetSlopFraction(max(b1->bbWeight, b2->bbWeight)); + } #ifdef DEBUG - unsigned bbTgtStkDepth; // Native stack depth on entry (for throw-blocks) - static unsigned s_nMaxTrees; // The max # of tree nodes in any BB + unsigned bbTgtStkDepth; // Native stack depth on entry (for throw-blocks) + static unsigned s_nMaxTrees; // The max # of tree nodes in any BB - // This is used in integrity checks. We semi-randomly pick a traversal stamp, label all blocks - // in the BB list with that stamp (in this field); then we can tell if (e.g.) predecessors are - // still in the BB list by whether they have the same stamp (with high probability). - unsigned bbTraversalStamp; + // This is used in integrity checks. We semi-randomly pick a traversal stamp, label all blocks + // in the BB list with that stamp (in this field); then we can tell if (e.g.) predecessors are + // still in the BB list by whether they have the same stamp (with high probability). + unsigned bbTraversalStamp; - // bbID is a unique block identifier number that does not change: it does not get renumbered, like bbNum. - unsigned bbID; + // bbID is a unique block identifier number that does not change: it does not get renumbered, like bbNum. + unsigned bbID; #endif // DEBUG - unsigned bbStackDepthOnEntry() const; - void bbSetStack(StackEntry * stack); - StackEntry* bbStackOnEntry() const; + unsigned bbStackDepthOnEntry() const; + void bbSetStack(StackEntry * stack); + StackEntry* bbStackOnEntry() const; - // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding - // zero-based number for use as an array index. - unsigned bbInd() const - { - assert(bbNum > 0); - return bbNum - 1; - } + // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding + // zero-based number for use as an array index. + unsigned bbInd() const + { + assert(bbNum > 0); + return bbNum - 1; + } - Statement* firstStmt() const; - Statement* lastStmt() const; - bool hasSingleStmt() const; + Statement* firstStmt() const; + Statement* lastStmt() const; + bool hasSingleStmt() const; - // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: - // for (Statement* const stmt : block->Statements()) - // - StatementList Statements() const - { - return StatementList(firstStmt()); - } + // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: + // for (Statement* const stmt : block->Statements()) + // + StatementList Statements() const + { + return StatementList(firstStmt()); + } - // NonPhiStatements: convenience method for enabling range-based `for` iteration over the statement list, - // excluding any initial PHI statements, e.g.: - // for (Statement* const stmt : block->NonPhiStatements()) - // - StatementList NonPhiStatements() const - { - return StatementList(FirstNonPhiDef()); - } + // NonPhiStatements: convenience method for enabling range-based `for` iteration over the statement list, + // excluding any initial PHI statements, e.g.: + // for (Statement* const stmt : block->NonPhiStatements()) + // + StatementList NonPhiStatements() const + { + return StatementList(FirstNonPhiDef()); + } + + GenTree* lastNode() const; - GenTree* lastNode() const; + bool endsWithJmpMethod(Compiler * comp) const; - bool endsWithJmpMethod(Compiler * comp) const; + bool endsWithTailCall(Compiler * comp, bool fastTailCallsOnly, bool tailCallsConvertibleToLoopOnly, + GenTree** tailCall) const; - bool endsWithTailCall(Compiler * comp, bool fastTailCallsOnly, bool tailCallsConvertibleToLoopOnly, - GenTree** tailCall) const; + bool endsWithTailCallOrJmp(Compiler * comp, bool fastTailCallsOnly = false) const; - bool endsWithTailCallOrJmp(Compiler * comp, bool fastTailCallsOnly = false) const; + bool endsWithTailCallConvertibleToLoop(Compiler * comp, GenTree * *tailCall) const; - bool endsWithTailCallConvertibleToLoop(Compiler * comp, GenTree * *tailCall) const; + // Returns the first statement in the statement list of "this" that is + // not an SSA definition (a lcl = phi(...) store). + Statement* FirstNonPhiDef() const; + Statement* FirstNonPhiDefOrCatchArgStore() const; - // Returns the first statement in the statement list of "this" that is - // not an SSA definition (a lcl = phi(...) store). - Statement* FirstNonPhiDef() const; - Statement* FirstNonPhiDefOrCatchArgStore() const; + BasicBlock() + : bbStmtList(nullptr) + , bbLiveIn(VarSetOps::UninitVal()) + , bbLiveOut(VarSetOps::UninitVal()) + { + } + + // Iteratable collection of successors of a block. + template + class Successors + { + Compiler* m_comp; + BasicBlock* m_block; - BasicBlock() - : bbStmtList(nullptr) - , bbLiveIn(VarSetOps::UninitVal()) - , bbLiveOut(VarSetOps::UninitVal()) + public: + Successors(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { } - // Iteratable collection of successors of a block. - template - class Successors + class iterator { Compiler* m_comp; BasicBlock* m_block; + TPosition m_pos; public: - Successors(Compiler* comp, BasicBlock* block) + iterator(Compiler* comp, BasicBlock* block) : m_comp(comp) , m_block(block) + , m_pos(comp, block) { } - class iterator + iterator() + : m_pos() { - Compiler* m_comp; - BasicBlock* m_block; - TPosition m_pos; - - public: - iterator(Compiler* comp, BasicBlock* block) - : m_comp(comp) - , m_block(block) - , m_pos(comp, block) - { - } - - iterator() - : m_pos() - { - } - - void operator++(void) - { - m_pos.Advance(m_comp, m_block); - } - - BasicBlock* operator*() - { - return m_pos.Current(m_comp, m_block); - } + } - bool operator==(const iterator& other) - { - return m_pos == other.m_pos; - } + void operator++(void) + { + m_pos.Advance(m_comp, m_block); + } - bool operator!=(const iterator& other) - { - return m_pos != other.m_pos; - } - }; + BasicBlock* operator*() + { + return m_pos.Current(m_comp, m_block); + } - iterator begin() + bool operator==(const iterator& other) { - return iterator(m_comp, m_block); + return m_pos == other.m_pos; } - iterator end() + bool operator!=(const iterator& other) { - return iterator(); + return m_pos != other.m_pos; } }; - template - BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler * comp, TFunc func); + iterator begin() + { + return iterator(m_comp, m_block); + } + + iterator end() + { + return iterator(); + } + }; - template - BasicBlockVisit VisitAllSuccs(Compiler * comp, TFunc func); + template + BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler * comp, TFunc func); - template - BasicBlockVisit VisitEHSuccs(Compiler * comp, TFunc func); + template + BasicBlockVisit VisitAllSuccs(Compiler * comp, TFunc func); - template - BasicBlockVisit VisitRegularSuccs(Compiler * comp, TFunc func); + template + BasicBlockVisit VisitEHSuccs(Compiler * comp, TFunc func); - bool HasPotentialEHSuccs(Compiler * comp); + template + BasicBlockVisit VisitRegularSuccs(Compiler * comp, TFunc func); - // Base class for Successor block/edge iterators. - // - class SuccList + bool HasPotentialEHSuccs(Compiler * comp); + + // Base class for Successor block/edge iterators. + // + class SuccList + { + protected: + // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't + // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end + // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at + // `m_succs` or at the switch table successor array. + FlowEdge* m_succs[2]; + FlowEdge* const* m_begin; + FlowEdge* const* m_end; + + SuccList(const BasicBlock* block); + }; + + // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs()) ... + // + class BBSuccList : private SuccList + { + public: + BBSuccList(const BasicBlock* block) + : SuccList(block) { - protected: - // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't - // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end - // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at - // `m_succs` or at the switch table successor array. - FlowEdge* m_succs[2]; - FlowEdge* const* m_begin; - FlowEdge* const* m_end; - - SuccList(const BasicBlock* block); - }; + } - // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs()) ... - // - class BBSuccList : private SuccList + BBArrayIterator begin() const { - public: - BBSuccList(const BasicBlock* block) - : SuccList(block) - { - } + return BBArrayIterator(m_begin); + } - BBArrayIterator begin() const - { - return BBArrayIterator(m_begin); - } + BBArrayIterator end() const + { + return BBArrayIterator(m_end); + } + }; - BBArrayIterator end() const - { - return BBArrayIterator(m_end); - } - }; + // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, + // normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges()) ... + // + class BBSuccEdgeList : private SuccList + { + public: + BBSuccEdgeList(const BasicBlock* block) + : SuccList(block) + { + } - // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, - // normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges()) ... - // - class BBSuccEdgeList : private SuccList + FlowEdgeArrayIterator begin() const { - public: - BBSuccEdgeList(const BasicBlock* block) - : SuccList(block) - { - } + return FlowEdgeArrayIterator(m_begin); + } - FlowEdgeArrayIterator begin() const - { - return FlowEdgeArrayIterator(m_begin); - } + FlowEdgeArrayIterator end() const + { + return FlowEdgeArrayIterator(m_end); + } + }; - FlowEdgeArrayIterator end() const - { - return FlowEdgeArrayIterator(m_end); - } - }; + // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs(compiler)) ... + // + // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation + // of the implications of this versus the version that does not take `Compiler*`. + class BBCompilerSuccList + { + Compiler* m_comp; + BasicBlock* m_block; - // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs(compiler)) ... + // iterator: forward iterator for an array of BasicBlock* // - // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation - // of the implications of this versus the version that does not take `Compiler*`. - class BBCompilerSuccList + class iterator { Compiler* m_comp; BasicBlock* m_block; - - // iterator: forward iterator for an array of BasicBlock* - // - class iterator - { - Compiler* m_comp; - BasicBlock* m_block; - unsigned m_succNum; - - public: - iterator(Compiler* comp, BasicBlock* block, unsigned succNum) - : m_comp(comp) - , m_block(block) - , m_succNum(succNum) - { - } - - BasicBlock* operator*() const - { - assert(m_block != nullptr); - BasicBlock* bTarget = m_block->GetSucc(m_succNum, m_comp); - assert(bTarget != nullptr); - return bTarget; - } - - iterator& operator++() - { - ++m_succNum; - return *this; - } - - bool operator!=(const iterator& i) const - { - return m_succNum != i.m_succNum; - } - }; + unsigned m_succNum; public: - BBCompilerSuccList(Compiler* comp, BasicBlock* block) + iterator(Compiler* comp, BasicBlock* block, unsigned succNum) : m_comp(comp) , m_block(block) + , m_succNum(succNum) { } - iterator begin() const + BasicBlock* operator*() const { - return iterator(m_comp, m_block, 0); + assert(m_block != nullptr); + BasicBlock* bTarget = m_block->GetSucc(m_succNum, m_comp); + assert(bTarget != nullptr); + return bTarget; } - iterator end() const + iterator& operator++() { - return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); + ++m_succNum; + return *this; + } + + bool operator!=(const iterator& i) const + { + return m_succNum != i.m_succNum; } }; - // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based - // `for`, normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... - // - // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation - // of the implications of this versus the version that does not take `Compiler*`. - class BBCompilerSuccEdgeList + public: + BBCompilerSuccList(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { - Compiler* m_comp; - BasicBlock* m_block; + } - // iterator: forward iterator for an array of BasicBlock* - // - class iterator - { - Compiler* m_comp; - BasicBlock* m_block; - unsigned m_succNum; - - public: - iterator(Compiler* comp, BasicBlock* block, unsigned succNum) - : m_comp(comp) - , m_block(block) - , m_succNum(succNum) - { - } + iterator begin() const + { + return iterator(m_comp, m_block, 0); + } - FlowEdge* operator*() const - { - assert(m_block != nullptr); - FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); - assert(succEdge != nullptr); - return succEdge; - } + iterator end() const + { + return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); + } + }; - iterator& operator++() - { - ++m_succNum; - return *this; - } + // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based + // `for`, normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... + // + // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation + // of the implications of this versus the version that does not take `Compiler*`. + class BBCompilerSuccEdgeList + { + Compiler* m_comp; + BasicBlock* m_block; - bool operator!=(const iterator& i) const - { - return m_succNum != i.m_succNum; - } - }; + // iterator: forward iterator for an array of BasicBlock* + // + class iterator + { + Compiler* m_comp; + BasicBlock* m_block; + unsigned m_succNum; public: - BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) + iterator(Compiler* comp, BasicBlock* block, unsigned succNum) : m_comp(comp) , m_block(block) + , m_succNum(succNum) { } - iterator begin() const + FlowEdge* operator*() const { - return iterator(m_comp, m_block, 0); + assert(m_block != nullptr); + FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); + assert(succEdge != nullptr); + return succEdge; } - iterator end() const + iterator& operator++() { - return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); + ++m_succNum; + return *this; + } + + bool operator!=(const iterator& i) const + { + return m_succNum != i.m_succNum; } }; - // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: - // for (BasicBlock* const succ : block->Succs()) ... - // - // There are two options: one that takes a Compiler* and one that doesn't. These correspond to the - // NumSucc()/GetSucc() functions that do or do not take a Compiler*. See the comment for NumSucc()/GetSucc() - // for the distinction. - BBSuccList Succs() const + public: + BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) + : m_comp(comp) + , m_block(block) { - return BBSuccList(this); } - BBCompilerSuccList Succs(Compiler * comp) + iterator begin() const { - return BBCompilerSuccList(comp, this); + return iterator(m_comp, m_block, 0); } - BBSuccEdgeList SuccEdges() + iterator end() const { - return BBSuccEdgeList(this); + return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); } + }; - BBCompilerSuccEdgeList SuccEdges(Compiler * comp) - { - return BBCompilerSuccEdgeList(comp, this); - } + // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: + // for (BasicBlock* const succ : block->Succs()) ... + // + // There are two options: one that takes a Compiler* and one that doesn't. These correspond to the + // NumSucc()/GetSucc() functions that do or do not take a Compiler*. See the comment for NumSucc()/GetSucc() + // for the distinction. + BBSuccList Succs() const + { + return BBSuccList(this); + } - // Clone block state and statements from `from` block to `to` block (which must be new/empty) - static void CloneBlockState(Compiler * compiler, BasicBlock * to, const BasicBlock* from); + BBCompilerSuccList Succs(Compiler * comp) + { + return BBCompilerSuccList(comp, this); + } - // Copy the block kind and take memory ownership of the targets. - void TransferTarget(BasicBlock * from); + BBSuccEdgeList SuccEdges() + { + return BBSuccEdgeList(this); + } - void MakeLIR(GenTree * firstNode, GenTree * lastNode); - bool IsLIR() const; + BBCompilerSuccEdgeList SuccEdges(Compiler * comp) + { + return BBCompilerSuccEdgeList(comp, this); + } - void SetDominatedByExceptionalEntryFlag() - { - SetFlags(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); - } + // Clone block state and statements from `from` block to `to` block (which must be new/empty) + static void CloneBlockState(Compiler * compiler, BasicBlock * to, const BasicBlock* from); - bool IsDominatedByExceptionalEntryFlag() const - { - return HasFlag(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); - } + // Copy the block kind and take memory ownership of the targets. + void TransferTarget(BasicBlock * from); + + void MakeLIR(GenTree * firstNode, GenTree * lastNode); + bool IsLIR() const; + + void SetDominatedByExceptionalEntryFlag() + { + SetFlags(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); + } + + bool IsDominatedByExceptionalEntryFlag() const + { + return HasFlag(BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY); + } #ifdef DEBUG - bool Contains(const GenTree* node) const + bool Contains(const GenTree* node) const + { + assert(IsLIR()); + for (Iterator iter = begin(); iter != end(); ++iter) { - assert(IsLIR()); - for (Iterator iter = begin(); iter != end(); ++iter) + if (*iter == node) { - if (*iter == node) - { - return true; - } + return true; } - return false; } + return false; + } #endif // DEBUG }; diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index d98ed868cffc6b..007571a4040ab7 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -921,8 +921,8 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); - regMaskTP maskSaveRegsInt = regsToSaveMask.gprRegs(); - regMaskTP maskSaveRegsFloat = regsToSaveMask.floatRegs(compiler); + regMaskGpr maskSaveRegsInt = regsToSaveMask.gprRegs(); + regMaskFloat maskSaveRegsFloat = regsToSaveMask.floatRegs(compiler); assert(compiler->IsGprRegMask(maskSaveRegsInt)); assert(compiler->IsFloatRegMask(maskSaveRegsFloat)); @@ -930,7 +930,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lo unsigned regsToSaveCount = genCountBits(maskSaveRegsFloat) + genCountBits(maskSaveRegsInt); #ifdef FEATURE_MASKED_HW_INTRINSICS - regMaskTP maskSaveRegsPredicate = regsToSaveMask.predicateRegs(compiler); + regMaskPredicate maskSaveRegsPredicate = regsToSaveMask.predicateRegs(compiler); assert(compiler->IsPredicateRegMask(maskSaveRegsPredicate)); regsToSaveCount += genCountBits(maskSaveRegsPredicate); #endif @@ -1070,7 +1070,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, unsigned regsToRestoreCount = genCountBits(maskRestoreRegsInt) + genCountBits(maskRestoreRegsFloat); #ifdef FEATURE_MASKED_HW_INTRINSICS - regMaskTP maskRestoreRegsPredicate = regsToRestoreMask.predicateRegs(compiler); + regMaskPredicate maskRestoreRegsPredicate = regsToRestoreMask.predicateRegs(compiler); assert(compiler->IsPredicateRegMask(maskRestoreRegsPredicate)); regsToRestoreCount += genCountBits(maskRestoreRegsPredicate); #endif @@ -5557,8 +5557,9 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - if (((RBM_PROFILER_ENTER_TRASH | RBM_PROFILER_ENTER_ARG_FUNC_ID | RBM_PROFILER_ENTER_ARG_CALLER_SP) & - genRegMask(initReg)) != RBM_NONE) + AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + profileEnterTrash.AddRegMaskForType(RBM_PROFILER_ENTER_ARG_FUNC_ID | RBM_PROFILER_ENTER_ARG_CALLER_SP, TYP_INT); + if (profileEnterTrash.IsRegNumInMask(initReg)) { *pInitRegZeroed = false; } diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index b2222a90fa1d73..59dd71d8b300fb 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -5577,7 +5577,8 @@ void CodeGen::genFnEpilog(BasicBlock* block) compiler->unwindSetFrameReg(REG_SAVED_LOCALLOC_SP, 0); } - if (jmpEpilog || regSet.rsGetModifiedRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) + if (jmpEpilog || genStackAllocRegisterMask(compiler->compLclFrameSize, + regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) { genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); } diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ca4c82984d1383..a3ce2319a6f4fd 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9504,7 +9504,9 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - if (((RBM_PROFILER_ENTER_TRASH | RBM_ARG_0 | RBM_ARG_1) & genRegMask(initReg)) != 0) + AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + profileEnterTrash.AddRegMaskForType((RBM_ARG_0 | RBM_ARG_1), TYP_INT); + if (profileEnterTrash.IsRegNumInMask(initReg)) { *pInitRegZeroed = false; } @@ -9543,7 +9545,9 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - if (((RBM_PROFILER_ENTER_TRASH | RBM_PROFILER_ENTER_ARG_0 | RBM_PROFILER_ENTER_ARG_1) & genRegMask(initReg)) != 0) + AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + profileEnterTrash.AddRegMaskForType((RBM_PROFILER_ENTER_ARG_0 | RBM_PROFILER_ENTER_ARG_1), TYP_INT); + if (profileEnterTrash.IsRegNumInMask(initReg)) { *pInitRegZeroed = false; } @@ -9583,8 +9587,8 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // registers that profiler callback kills. if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { - regMaskOnlyOne thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum()); - noway_assert((RBM_PROFILER_LEAVE_TRASH & thisPtrMask) == 0); + regNumber thisPtrReg = compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum(); + noway_assert(!AllRegsMask_PROFILER_LEAVE_TRASH.IsRegNumInMask(thisPtrReg)); } // At this point return value is computed and stored in RAX or XMM0. @@ -10111,7 +10115,6 @@ void CodeGen::genFnEpilog(BasicBlock* block) // PatchpointInfo* const patchpointInfo = compiler->info.compPatchpointInfo; - regMaskTP const tier0CalleeSaves = (regMaskTP)patchpointInfo->CalleeSaveRegisters(); regMaskGpr const tier0IntCalleeSaves = patchpointInfo->CalleeSaveGprRegisters() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const osrIntCalleeSaves = regSet.rsGetModifiedGprRegsMask() & RBM_OSR_INT_CALLEE_SAVED; regMaskGpr const allIntCalleeSaves = osrIntCalleeSaves | tier0IntCalleeSaves; diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index f823a0f763bbb7..2877e0bd369f23 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -977,7 +977,7 @@ inline regNumber genFirstRegNumFromMask(AllRegsMask& mask) /* Convert the mask to a register number */ regNumber regNum; - regMaskTP gprOrFloatMask = mask.GetGprFloatCombinedMask(); + RegBitSet64 gprOrFloatMask = mask.GetGprFloatCombinedMask(); #ifdef HAS_MORE_THAN_64_REGISTERS // Only check this condition if there are predicate register support @@ -5347,7 +5347,7 @@ regMaskOnlyOne AllRegsMask::GetRegMaskForType(var_types type) const #endif } -regMaskTP AllRegsMask::GetGprFloatCombinedMask() const +RegBitSet64 AllRegsMask::GetGprFloatCombinedMask() const { return _combinedRegisters; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index d474d3df846580..1729f4b8d1a457 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4418,7 +4418,7 @@ void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarT } #else -void LinearScan::updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap) +void LinearScan::updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarToRegMap inVarToRegMap) { while (deadRegMask != RBM_NONE) { @@ -4892,7 +4892,7 @@ void LinearScan::inActivateRegisters(AllRegsMask& inactiveMask) } } #else -void LinearScan::inActivateRegisters(regMaskTP inactiveMask) +void LinearScan::inActivateRegisters(RegBitSet64 inactiveMask) { while (inactiveMask != RBM_NONE) { @@ -5001,7 +5001,7 @@ void LinearScan::freeRegisterMask(AllRegsMask& freeMask) } #else -void LinearScan::freeRegisterMask(regMaskTP freeMask) +void LinearScan::freeRegisterMask(RegBitSet64 freeMask) { while (freeMask != RBM_NONE) { diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 750f9cc4a047ab..291903e0249269 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1058,7 +1058,7 @@ class LinearScan : public LinearScanInterface #ifdef HAS_MORE_THAN_64_REGISTERS FORCEINLINE void updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap); #else - FORCEINLINE void updateDeadCandidatesAtBlockStart(regMaskTP deadRegMask, VarToRegMap inVarToRegMap); + FORCEINLINE void updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarToRegMap inVarToRegMap); #endif #ifdef TARGET_ARM @@ -1152,14 +1152,14 @@ class LinearScan : public LinearScanInterface #ifdef HAS_MORE_THAN_64_REGISTERS FORCEINLINE void inActivateRegisters(AllRegsMask& inactiveMask); #else - FORCEINLINE void inActivateRegisters(regMaskTP inactiveMask); + FORCEINLINE void inActivateRegisters(RegBitSet64 inactiveMask); #endif void freeRegister(RegRecord* physRegRecord); void freeRegisters(AllRegsMask regsToFree); #ifdef HAS_MORE_THAN_64_REGISTERS FORCEINLINE void freeRegisterMask(AllRegsMask& freeMask); #else - FORCEINLINE void freeRegisterMask(regMaskTP freeMask); + FORCEINLINE void freeRegisterMask(RegBitSet64 freeMask); #endif // Get the type that this tree defines. diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index c5bc3b385a4f24..13b6a543c1999b 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1337,7 +1337,6 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou int srcCount = 0; int dstCount = 0; - regMaskTP dstCandidates = RBM_NONE; if (HWIntrinsicInfo::IsMultiReg(intrin.id)) { @@ -1553,7 +1552,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } else if (HWIntrinsicInfo::IsMaskedOperation(intrin.id)) { - regMaskTP predMask = HWIntrinsicInfo::IsLowMaskedOperation(intrin.id) ? RBM_LOWMASK : RBM_ALLMASK; + regMaskPredicate predMask = HWIntrinsicInfo::IsLowMaskedOperation(intrin.id) ? RBM_LOWMASK : RBM_ALLMASK; srcCount += BuildOperandUses(intrin.op1, predMask); } else if (intrinsicTree->OperIsMemoryLoadOrStore()) @@ -1810,11 +1809,11 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if ((dstCount == 1) || (dstCount == 2)) { - BuildDef(intrinsicTree, dstCandidates); + BuildDef(intrinsicTree); if (dstCount == 2) { - BuildDef(intrinsicTree, dstCandidates, 1); + BuildDef(intrinsicTree, RBM_NONE, 1); } } else diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 462025d6a787f5..e3a84893dc4df6 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2827,7 +2827,7 @@ void LinearScan::buildIntervals() #ifdef HAS_MORE_THAN_64_REGISTERS actualRegistersMask = AllRegsMask(availableIntRegs, availableFloatRegs, availableMaskRegs); #else - if (availableRegCount < (sizeof(regMaskTP) * 8)) + if (availableRegCount < (sizeof(RegBitSet64) * 8)) { // Mask out the bits that are between 64 ~ availableRegCount actualRegistersMask = AllRegsMask((1ULL << availableRegCount) - 1); diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 258de9fa9e1dce..e2b41212cedf8a 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -144,6 +144,9 @@ void RegSet::rsClearRegsModified() #endif // DEBUG rsModifiedRegsMask.Clear(); + +#ifdef SWIFT_SUPPORT + // If this method has a SwiftError* parameter, we will return SwiftError::Value in REG_SWIFT_ERROR, // so don't treat it as callee-save. if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) { diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 82bbcea29fcdbd..3e5af05cdc5c22 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -95,32 +95,6 @@ class RegSet assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; } - - regMaskTP rsGetModifiedCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & rsAllCalleeSavedMask); - } - - regMaskTP rsGetModifiedIntCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & rsIntCalleeSavedMask); - } - -#ifdef TARGET_AMD64 - regMaskTP rsGetModifiedOsrIntCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & (rsIntCalleeSavedMask | RBM_EBP)); - } -#endif // TARGET_AMD64 - - regMaskTP rsGetModifiedFltCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & RBM_FLT_CALLEE_SAVED); - } #endif regMaskGpr rsGetModifiedGprRegsMask() const diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index c9e78b3eb70b2b..230dd3275ea1d8 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -398,9 +398,9 @@ typedef struct _regMaskAll // TODO: this might go away once we have just `regMaskTP` gpr_float field FORCEINLINE bool IsGprOrFloatPresent() const; - FORCEINLINE regMaskTP GetGprFloatCombinedMask() const; + FORCEINLINE RegBitSet64 GetGprFloatCombinedMask() const; #ifndef HAS_MORE_THAN_64_REGISTERS - FORCEINLINE regMaskTP GetAllRegistersMask() const; + FORCEINLINE RegBitSet64 GetAllRegistersMask() const; #endif // !HAS_MORE_THAN_64_REGISTERS FORCEINLINE regMaskOnlyOne operator[](int index) const; @@ -689,7 +689,7 @@ inline unsigned theFixedRetBuffArgNum(CorInfoCallConvExtension callConv) // inline regMaskGpr fullIntArgRegMask(CorInfoCallConvExtension callConv) { - regMaskTP result = RBM_ARG_REGS; + regMaskGpr result = RBM_ARG_REGS; if (hasFixedRetBuffReg(callConv)) { result |= theFixedRetBuffMask(callConv); diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index d6a88137fb0dd7..2116c7ea3e5a5d 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -528,25 +528,18 @@ // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. // STOP_FOR_GC helper preserves all the 4 possible return registers. - - // The registers trashed by profiler enter/leave/tailcall hook - // See vm\amd64\asmhelpers.S for more details. - #define RBM_PROFILER_ENTER_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_FLTARG_REGS)) -#endif // HAS_MORE_THAN_64_REGISTERS - #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) - #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH) + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) #else // See vm\amd64\asmhelpers.asm for more details. -#ifdef HAS_MORE_THAN_64_REGISTERS - - #define RBM_PROFILER_ENTER_TRASH RBM_CALLEE_TRASH - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET) | RBM_MSK_CALLEE_TRASH) -#endif // HAS_MORE_THAN_64_REGISTERS - #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) - #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH) + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_CALLEE_TRASH + +#endif // UNIX_AMD64_ABI - #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH + #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH + #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_LEAVE_TRASH // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 7e301add395bf8..80e3bbee72e1ae 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -226,6 +226,7 @@ // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. + #define RBM_PROFILER_ENTER_TRASH RBM_NONE #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must // consider it killed by the return. From a1f157bed72f5cb5b51d1091bbc21b2191ae07a9 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 Apr 2024 13:44:59 -0700 Subject: [PATCH 165/201] Fix changes from swift --- src/coreclr/jit/abi.cpp | 4 ++-- src/coreclr/jit/abi.h | 2 +- src/coreclr/jit/codegenarm.cpp | 6 +++--- src/coreclr/jit/codegenarm64.cpp | 5 +++-- src/coreclr/jit/codegenarmarch.cpp | 5 +++-- src/coreclr/jit/codegencommon.cpp | 6 +++--- src/coreclr/jit/codegenxarch.cpp | 4 ++-- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/lclvars.cpp | 4 +++- src/coreclr/jit/lsrabuild.cpp | 3 ++- src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/regset.h | 20 ++++++++++++++++---- 13 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index 91f74fca03c8fd..002d621b5bca98 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -48,10 +48,10 @@ regNumber ABIPassingSegment::GetRegister() const // Return Value: // The register mask. // -regMaskTP ABIPassingSegment::GetRegisterMask() const +regMaskOnlyOne ABIPassingSegment::GetRegisterMask() const { assert(IsPassedInRegister()); - regMaskTP reg = genRegMask(m_register); + regMaskOnlyOne reg = genRegMask(m_register); #ifdef TARGET_ARM if (genIsValidFloatReg(m_register) && (Size == 8)) diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index 7236627d375d48..1f03a2ef5a0564 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -25,7 +25,7 @@ class ABIPassingSegment // If this segment is passed in a register, return the particular register. regNumber GetRegister() const; - regMaskTP GetRegisterMask() const; + regMaskOnlyOne GetRegisterMask() const; // If this segment is passed on the stack then return the particular stack // offset, relative to the first stack argument's offset. diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 8e97199ac8a4a1..cfb15c7ffa4841 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2158,9 +2158,9 @@ bool CodeGen::genCanUsePopToReturn(bool jmpEpilog) void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - - regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; - regMaskGpr maskPopRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + + regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_ALLFLOAT; + regMaskGpr maskPopRegsInt = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); // First, pop float registers diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 007571a4040ab7..eec21fb00acfe8 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -36,8 +36,9 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; - regMaskFloat rsRestoreFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; + regMaskFloat rsRestoreFloatRegs = + regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_ALLFLOAT; + regMaskGpr rsRestoreGprRegs = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); #ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate rsRestorePredicateRegs = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; #endif diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 59dd71d8b300fb..16a0f41d650647 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4915,8 +4915,9 @@ void CodeGen::genPushCalleeSavedRegisters() intRegState.rsCalleeRegArgMaskLiveIn); #endif - regMaskGpr rsPushGprRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; - regMaskFloat rsPushFloatRegs = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; + regMaskGpr rsPushGprRegs = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs() & RBM_INT_CALLEE_SAVED; + regMaskFloat rsPushFloatRegs = + regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_FLT_CALLEE_SAVED; #ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate rsPushPredicateRegs = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; #endif diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 0f9a9b82bca8cd..bc10e3d64deaad 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5010,7 +5010,7 @@ void CodeGen::genHomeSwiftStructParameters(bool handleStack) if (seg.IsPassedInRegister()) { RegState* regState = genIsValidFloatReg(seg.GetRegister()) ? &floatRegState : &intRegState; - regMaskTP regs = seg.GetRegisterMask(); + regMaskOnlyOne regs = seg.GetRegisterMask(); if ((regState->rsCalleeRegArgMaskLiveIn & regs) != RBM_NONE) { @@ -5503,8 +5503,8 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - regMaskFloat maskPushRegsInt = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; - regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED; + regMaskFloat maskPushRegsInt = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); + regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_FLT_CALLEE_SAVED; #ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate maskPushRegsPredicate = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; #endif diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ff54be16cb9936..18cc49f796c9dd 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9853,7 +9853,7 @@ void CodeGen::genPushCalleeSavedRegisters() // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not // here. - regMaskGpr rsPushRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPushRegs = regSet.rsGetModifiedIntCalleeSavedRegsMask(); #if ETW_EBP_FRAMED if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE)) @@ -9931,7 +9931,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) // Registers saved by a normal prolog // - regMaskGpr rsPopRegs = regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPopRegs = regSet.rsGetModifiedIntCalleeSavedRegsMask(); const unsigned popCount = genPopCalleeSavedRegistersFromMask(rsPopRegs); noway_assert(compiler->compCalleeRegsPushed == popCount); } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 782a2c7c1fc82c..e65b921af6a5dd 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5938,7 +5938,7 @@ void Compiler::generatePatchpointInfo() // Record callee save registers. // Currently only needed for x64. // - regMaskGpr rsPushGprRegs = codeGen->regSet.rsGetModifiedGprRegsMask() & RBM_INT_CALLEE_SAVED; + regMaskGpr rsPushGprRegs = codeGen->regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); rsPushGprRegs |= RBM_FPBASE; patchpointInfo->SetCalleeSaveGprRegisters(rsPushGprRegs); JITDUMP("--OSR-- Tier0 callee saves: "); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 2877e0bd369f23..39a90ce64a94b4 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5386,7 +5386,7 @@ regMaskPredicate AllRegsMask::predicateRegs(const Compiler* compiler) const return _predicateRegs; #else regMaskOnlyOne allMask = compiler != nullptr ? compiler->get_RBM_ALLMASK() : (RBM_ALLMASK_EVEX); - return _combinedRegisters & compiler->get_RBM_ALLMASK(); + return _combinedRegisters & allMask; #endif } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 73b3eedc9d5bde..4e3c7b6d004e3a 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1753,7 +1753,9 @@ void Compiler::lvaClassifyParameterABI() SwiftABIClassifier classifier(cInfo); lvaClassifyParameterABI(classifier); - regMaskTP argRegs = RBM_NONE; + // There is an assumption that args never has predicate registers in case + // of OSX/arm64. + RegBitSet64 argRegs = RBM_NONE; // The calling convention details computed by the old ABI classifier // are wrong since it does not handle the Swift ABI for structs diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ecb594cf59ed58..acf3ee269fe2f1 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2558,8 +2558,9 @@ void LinearScan::buildIntervals() // handling clobbers REG_SCRATCH, so kill it here. if ((block == compiler->fgFirstBB) && compiler->lvaHasAnySwiftStackParamToReassemble()) { + AllRegsMask scratchMask = AllRegsMask(genRegMask(REG_SCRATCH)); assert(compiler->fgFirstBBisScratch()); - addRefsForPhysRegMask(genRegMask(REG_SCRATCH), currentLoc + 1, RefTypeKill, true); + addRefsForPhysRegMask(scratchMask, currentLoc + 1, RefTypeKill, true); currentLoc += 2; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index e2b41212cedf8a..22081b1b2bd67a 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -150,7 +150,7 @@ void RegSet::rsClearRegsModified() // so don't treat it as callee-save. if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) { - rsAllCalleeSavedMask &= ~RBM_SWIFT_ERROR; + rsAllCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); rsIntCalleeSavedMask &= ~RBM_SWIFT_ERROR; } #endif // SWIFT_SUPPORT diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 3e5af05cdc5c22..bc978e1328a32f 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -78,11 +78,11 @@ class RegSet #endif // DEBUG #ifdef SWIFT_SUPPORT - regMaskTP rsAllCalleeSavedMask; - regMaskTP rsIntCalleeSavedMask; + AllRegsMask rsAllCalleeSavedMask; + regMaskGpr rsIntCalleeSavedMask; #else // !SWIFT_SUPPORT - static constexpr regMaskTP rsAllCalleeSavedMask = RBM_CALLEE_SAVED; - static constexpr regMaskTP rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; + AllRegsMask rsAllCalleeSavedMask = AllRegsMask_CALLEE_SAVED; + regMaskGpr rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; #endif // !SWIFT_SUPPORT public: @@ -97,6 +97,18 @@ class RegSet } #endif + AllRegsMask rsGetModifiedCalleeSavedRegsMask() const + { + assert(rsModifiedRegsMaskInitialized); + return (rsModifiedRegsMask & rsAllCalleeSavedMask); + } + + regMaskGpr rsGetModifiedIntCalleeSavedRegsMask() const + { + assert(rsModifiedRegsMaskInitialized); + return (rsGetModifiedGprRegsMask() & rsIntCalleeSavedMask); + } + regMaskGpr rsGetModifiedGprRegsMask() const { assert(rsModifiedRegsMaskInitialized); From 40bb3b1f057c57a129d648b68b363e45cf4862c6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 Apr 2024 14:30:53 -0700 Subject: [PATCH 166/201] Fix the argument for AllRegsMask() --- src/coreclr/jit/gentree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index d57be895bff7f1..11930820ff4efd 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1085,7 +1085,7 @@ AllRegsMask GenTree::gtGetRegMask() const if (IsMultiRegCall()) { - resultMask = AllRegsMask(GetRegNum()); + resultMask = AllRegsMask(genRegMask(GetRegNum())); resultMask |= AsCall()->GetOtherRegMask(); } else if (IsCopyOrReloadOfMultiRegCall()) @@ -1123,7 +1123,7 @@ AllRegsMask GenTree::gtGetRegMask() const #endif // FEATURE_ARG_SPLIT else { - resultMask = AllRegsMask(GetRegNum()); + resultMask = AllRegsMask(genRegMask(GetRegNum())); } return resultMask; From 506dc9fcd3fa01fad78c1e73c42f8d82d1a2feba Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Sat, 6 Apr 2024 18:58:20 -0700 Subject: [PATCH 167/201] Remove TODO #1 --- src/coreclr/jit/compiler.hpp | 1 - src/coreclr/jit/emitarm64.h | 1 - src/coreclr/jit/emitxarch.h | 4 ---- src/coreclr/jit/lsra.cpp | 4 +--- src/coreclr/jit/lsra.h | 14 ++------------ src/coreclr/jit/regset.cpp | 9 --------- src/coreclr/jit/regset.h | 2 +- src/coreclr/jit/target.h | 3 --- src/coreclr/jit/utils.cpp | 3 --- 9 files changed, 4 insertions(+), 37 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 39a90ce64a94b4..f6db15704caa1f 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5045,7 +5045,6 @@ BasicBlockVisit FlowGraphNaturalLoop::VisitRegularExitBlocks(TFunc func) return BasicBlockVisit::Continue; } -// TODO: If there are lot of callers of RegisterType, simplify it. template FORCEINLINE int regIndexForType(T vt) { diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index eda020eb002ce5..68ec1621485770 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1178,7 +1178,6 @@ inline static bool isGeneralRegisterOrSP(regNumber reg) #ifdef FEATURE_MASKED_HW_INTRINSICS inline static bool isMaskReg(regNumber reg) { - // TODO: This assert will no longer be true return (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST); } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 820e211ff25b89..fa48eeb53f3a2c 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -10,26 +10,22 @@ public: inline static bool isGeneralRegister(regNumber reg) { - // TODO: This assert will no longer be true return (reg <= REG_INT_LAST); } inline static bool isFloatReg(regNumber reg) { - // TODO: This assert will no longer be true return (reg >= REG_FP_FIRST && reg <= REG_FP_LAST); } inline static bool isDoubleReg(regNumber reg) { - // TODO: This assert will no longer be true return isFloatReg(reg); } #ifdef FEATURE_MASKED_HW_INTRINSICS inline static bool isMaskReg(regNumber reg) { - // TODO: This assert will no longer be true return (reg >= REG_MASK_FIRST && reg <= REG_MASK_LAST); } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 1729f4b8d1a457..a12ef06ba54899 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4084,7 +4084,6 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) template void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { - // TODO: Save current_ConstantRegs and current_AvailableRegs assert(currentBlock != nullptr); markBlockVisited(currentBlock); @@ -4106,7 +4105,6 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) { resetAllRegistersState(); } - // TODO: Restore current_ConstantRegs and current_AvailableRegs } //------------------------------------------------------------------------ @@ -5781,7 +5779,7 @@ void LinearScan::allocateRegisters() // TODO: Can we combine this with the freeing of registers below? It might // mess with the dump, since this was previously being done before the call below - // to dumpRegRecords. TODO: Do not do anything if tempRegsToMakeInactive == RBM_NONE + // to dumpRegRecords. AllRegsMask tempRegsToMakeInactive = (regsToMakeInactive | delayRegsToMakeInactive); #ifdef HAS_MORE_THAN_64_REGISTERS inActivateRegisters(tempRegsToMakeInactive); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 291903e0249269..81c2751da69643 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1756,8 +1756,7 @@ class LinearScan : public LinearScanInterface #ifdef FEATURE_MASKED_HW_INTRINSICS PhasedVar availableMaskRegs; #endif - PhasedVar* availableRegs[TYP_COUNT]; // TODO: probably separate this out based on gpr, vector, - // predicate + PhasedVar* availableRegs[TYP_COUNT]; #ifdef FEATURE_MASKED_HW_INTRINSICS #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) @@ -1818,10 +1817,6 @@ class LinearScan : public LinearScanInterface // Register status //----------------------------------------------------------------------- - // TODO: One option is to also just have another current_AvailableREgs that - // gets reset for every refposition we are processing depending on the - // register type. That wawy we do not have to query and fetch the appropriate - // entry again and agin. AllRegsMask m_AvailableRegs; void resetAvailableRegs() @@ -1884,10 +1879,6 @@ class LinearScan : public LinearScanInterface void clearConstantReg(regNumber reg, var_types regType) { - // TODO: If we decide to have curr_RegistersWithConstants, then here we will - // just operate on curr_RegistersWithConstants and assert - // assert(m_RegistersWithConstants[regIndexForType(regType)] == curr_RegistersWithConstants); - // but we will have to make sure that we save it back too?? m_RegistersWithConstants.RemoveRegNum(reg, regType); } void setConstantReg(regNumber reg, var_types regType) @@ -1929,8 +1920,7 @@ class LinearScan : public LinearScanInterface } weight_t spillCost[REG_COUNT]; - AllRegsMask regsBusyUntilKill; // TODO: Likewise, probably have this global 32-bit and set it point to the specific - // version like gpr, vector, etc. + AllRegsMask regsBusyUntilKill; AllRegsMask regsInUseThisLocation; AllRegsMask regsInUseNextLocation; #ifdef TARGET_ARM64 diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 22081b1b2bd67a..6cd11f33a71aff 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -191,9 +191,6 @@ void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsGprRegMask(mask)); - // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE - // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask - // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG printModifiedRegsMask(rsModifiedRegsMask.gprRegs(), mask DEBUG_ARG(suppressDump) DEBUG_ARG(RBM_INT_CALLEE_SAVED)); @@ -205,9 +202,6 @@ void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsFloatRegMask(mask)); - // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE - // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask - // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG printModifiedRegsMask(rsModifiedRegsMask.floatRegs(m_rsCompiler), @@ -219,9 +213,6 @@ void RegSet::rsSetFloatRegsModified(regMaskFloat mask DEBUGARG(bool suppressDump void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) { - // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE - // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask - // assert(modifiedMask != RBM_NONE); assert(rsModifiedRegsMaskInitialized); #ifdef DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index bc978e1328a32f..7f40ad277e4f79 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -197,7 +197,7 @@ class RegSet AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property #if defined(TARGET_ARMARCH) - regMaskGpr rsGprMaskCalleeSaved; // TODO: Can use AllRegsMask here as well + regMaskGpr rsGprMaskCalleeSaved; regMaskFloat rsFloatMaskCalleeSaved; #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) regMaskMixed rsMaskCalleeSaved; // mask of the registers pushed/popped in the prolog/epilog diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 230dd3275ea1d8..5f43e09059d96c 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -249,8 +249,6 @@ typedef unsigned __int64 regMaskOnlyOne; // 2. Have a data structure like struct to pass all these together typedef unsigned __int64 regMaskMixed; // TODO: Rename this to regMaskMixed -// TODO: For LSRA, the select() method should be regMaskOnlyOne because we will be -// allocating either GPR or Vector or Mask but not all typedef unsigned __int64 singleRegMask; #else @@ -396,7 +394,6 @@ typedef struct _regMaskAll FORCEINLINE bool IsFloatMaskPresent(Compiler* compiler, regMaskFloat maskToCheck) const; FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; - // TODO: this might go away once we have just `regMaskTP` gpr_float field FORCEINLINE bool IsGprOrFloatPresent() const; FORCEINLINE RegBitSet64 GetGprFloatCombinedMask() const; #ifndef HAS_MORE_THAN_64_REGISTERS diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index af34116c104fa7..373532702d6346 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -449,9 +449,6 @@ const char* dspRegRange(regMaskOnlyOne regMask, size_t& minSiz, const char* sep, */ void dspRegMask(AllRegsMask mask, size_t minSiz) { - // TODO: Need to fix all the callers where we don't know if the input is gpr/float but is of type `regMaskOnlyOne`. - // For now, I am just making `floatMask` as optional and default to RBM_NONE so we don't have to deal with - // lot of build errors. const char* sep = ""; printf("["); From 847701c47f97b216f63d594c207d49c9f6b477a3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 09:26:49 -0700 Subject: [PATCH 168/201] Remove TODO #2 --- src/coreclr/jit/gentree.cpp | 4 +--- src/coreclr/jit/target.h | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 11930820ff4efd..05e0698bd98cdb 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1077,8 +1077,6 @@ AllRegsMask GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -// TODO: All the callers of gtGetRegMask() are just interested in gpr -// so, this can just return gpr mask and if none exist, just return RBM_NONE AllRegsMask GenTree::gtGetRegMask() const { AllRegsMask resultMask; @@ -1130,7 +1128,7 @@ AllRegsMask GenTree::gtGetRegMask() const } //--------------------------------------------------------------- -// gtGetRegMask: Get the gpr reg mask of the node. +// gtGetGprRegMask: Get the gpr reg mask of the node. // // Arguments: // None diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 5f43e09059d96c..6a460e135b1ec8 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -247,7 +247,7 @@ typedef unsigned __int64 regMaskOnlyOne; // 0. Revisit regMaskMixed and see if they should be "regMaskMixed" // 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. // 2. Have a data structure like struct to pass all these together -typedef unsigned __int64 regMaskMixed; // TODO: Rename this to regMaskMixed +typedef unsigned __int64 regMaskMixed; typedef unsigned __int64 singleRegMask; From 1289175d0500868a14362f5041520ff13076e7f7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 09:27:03 -0700 Subject: [PATCH 169/201] Refactor freeRegsiterMask() --- src/coreclr/jit/lsra.cpp | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index a12ef06ba54899..ce762dceae0756 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4979,29 +4979,16 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) } #ifdef HAS_MORE_THAN_64_REGISTERS -// TODO: Can we just if-def the method signature and `IsEmpty()`? void LinearScan::freeRegisterMask(AllRegsMask& freeMask) -{ - while (!freeMask.IsEmpty()) - { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); - - RegRecord* regRecord = getRegisterRecord(nextReg); -#ifdef TARGET_ARM - if (regRecord->assignedInterval != nullptr && (regRecord->assignedInterval->registerType == TYP_DOUBLE)) - { - assert(genIsValidDoubleReg(nextReg)); - freeMask ^= genRegMask(regNumber(nextReg + 1)); - } -#endif - freeRegister(regRecord); - } -} - #else void LinearScan::freeRegisterMask(RegBitSet64 freeMask) +#endif // HAS_MORE_THAN_64_REGISTERS { +#ifdef HAS_MORE_THAN_64_REGISTERS + while (!freeMask.IsEmpty()) +#else while (freeMask != RBM_NONE) +#endif // HAS_MORE_THAN_64_REGISTERS { regNumber nextReg = genFirstRegNumFromMaskAndToggle(freeMask); @@ -5016,7 +5003,6 @@ void LinearScan::freeRegisterMask(RegBitSet64 freeMask) freeRegister(regRecord); } } -#endif // HAS_MORE_THAN_64_REGISTERS //------------------------------------------------------------------------ // LinearScan::allocateRegistersMinimal: Perform the actual register allocation when localVars From 5ba1773a1dc7e27a81842b2a89920c8340542710 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 09:58:57 -0700 Subject: [PATCH 170/201] Return `RegBit64` for gtGetContainedRegMask(), gtGetRegMask() --- src/coreclr/jit/gentree.cpp | 16 ++++++++-------- src/coreclr/jit/gentree.h | 4 ++-- src/coreclr/jit/lsra.h | 3 +-- src/coreclr/jit/lsrabuild.cpp | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 05e0698bd98cdb..81640053cd74b9 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1053,14 +1053,14 @@ bool GenTree::NeedsConsecutiveRegisters() const // Return Value: // Reg Mask of GenTree node. // -AllRegsMask GenTree::gtGetContainedRegMask() +regMaskGpr GenTree::gtGetContainedRegMask() { if (!isContained()) { - return isUsedFromReg() ? gtGetRegMask() : AllRegsMask(); + return isUsedFromReg() ? gtGetRegMask() : RBM_NONE; } - AllRegsMask mask; + regMaskGpr mask = RBM_NONE; for (GenTree* operand : Operands()) { mask |= operand->gtGetContainedRegMask(); @@ -1077,14 +1077,14 @@ AllRegsMask GenTree::gtGetContainedRegMask() // Return Value: // Reg Mask of GenTree node. // -AllRegsMask GenTree::gtGetRegMask() const +RegBitSet64 GenTree::gtGetRegMask() const { - AllRegsMask resultMask; + RegBitSet64 resultMask; if (IsMultiRegCall()) { - resultMask = AllRegsMask(genRegMask(GetRegNum())); - resultMask |= AsCall()->GetOtherRegMask(); + resultMask = genRegMask(GetRegNum()); + resultMask |= AsCall()->GetOtherRegMask().GetGprFloatCombinedMask(); } else if (IsCopyOrReloadOfMultiRegCall()) { @@ -1121,7 +1121,7 @@ AllRegsMask GenTree::gtGetRegMask() const #endif // FEATURE_ARG_SPLIT else { - resultMask = AllRegsMask(genRegMask(GetRegNum())); + resultMask = genRegMask(GetRegNum()); } return resultMask; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index f905b61269e660..1685cdb18a978a 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -946,9 +946,9 @@ struct GenTree int GetRegisterDstCount(Compiler* compiler) const; - AllRegsMask gtGetRegMask() const; + RegBitSet64 gtGetRegMask() const; regMaskGpr gtGetGprRegMask() const; - AllRegsMask gtGetContainedRegMask(); + regMaskGpr gtGetContainedRegMask(); GenTreeFlags gtFlags; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 81c2751da69643..1369ea0c984fa9 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -814,7 +814,6 @@ class LinearScan : public LinearScanInterface return (LsraStressLimitRegs)(lsraStressMask & LSRA_LIMIT_MASK); } - // TODO: Can have separate methods for each type regMaskOnlyOne getConstrainedRegMask(RefPosition* refPosition, RegisterType regType, regMaskOnlyOne regMaskActual, @@ -2518,7 +2517,7 @@ class RefPosition // Prior to the allocation pass, registerAssignment captures the valid registers // for this RefPosition. // After the allocation pass, this contains the actual assignment - // TODO: This should really be a union, where before allocation-pass it has `mask` and + // TODO-future: This should really be a union, where before allocation-pass it has `mask` and // after allocation-pass, it has regNumber directly, to avoid calling assignedReg(); regMaskOnlyOne registerAssignment; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index acf3ee269fe2f1..2e15de729db80c 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2853,7 +2853,7 @@ void LinearScan::buildIntervals() RefPosition* pos = newRefPosition((Interval*)nullptr, currentLoc, RefTypeBB, nullptr, RBM_NONE); } - needNonIntegerRegisters |= compiler->compFloatingPointUsed; // TODO: Also track about mask registers + needNonIntegerRegisters |= compiler->compFloatingPointUsed; if (!needNonIntegerRegisters) { availableRegCount = REG_INT_COUNT; From 4f4c31749e63020a6559190fb0034fef0a9e4843 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 10:02:28 -0700 Subject: [PATCH 171/201] Remove another TODO --- src/coreclr/jit/lsra.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ce762dceae0756..b2a4010a904ff4 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -3505,9 +3505,6 @@ void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { regNumber reg = regRec->regNum; - // TODO: genIsValid* is heavy operation because it does 2 comparisons, but unfortunately - // we cannot use `varTypeRegister[interval->registerType] == VTR_INT` because sometimes - // we do `vmovd xmm1, rcx` where we assign gpr for interval of type TYP_SIMD8 compiler->codeGen->regSet.rsSetRegModified(reg DEBUGARG(true)); interval->assignedReg = regRec; From dc6ff1f416db8fabcfbaea9aa786de99241331c4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 10:11:23 -0700 Subject: [PATCH 172/201] Remaining gtGetContainedRegMask() fix --- src/coreclr/jit/codegenxarch.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 18cc49f796c9dd..4d07a4fa6220b0 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1584,7 +1584,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // If there is a conflict then swap the condition anyway. LSRA should have // ensured the other way around has no conflict. - if (trueVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)) + if ((trueVal->gtGetContainedRegMask() & genRegMask(dstReg)) != 0) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1595,7 +1595,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) // There may also be a conflict with the falseVal in case this is an AND // condition. Once again, after swapping there should be no conflict as // ensured by LSRA. - if ((desc.oper == GT_AND) && falseVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)) + if ((desc.oper == GT_AND) && (falseVal->gtGetContainedRegMask() & genRegMask(dstReg)) != 0) { std::swap(trueVal, falseVal); cc = GenCondition::Reverse(cc); @@ -1605,13 +1605,13 @@ void CodeGen::genCodeForSelect(GenTreeOp* select) inst_RV_TT(INS_mov, emitTypeSize(select), dstReg, falseVal); assert(!trueVal->isContained() || trueVal->isUsedFromMemory()); - assert(!trueVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)); + assert((trueVal->gtGetContainedRegMask() & genRegMask(dstReg)) == 0); inst_RV_TT(JumpKindToCmov(desc.jumpKind1), emitTypeSize(select), dstReg, trueVal); if (desc.oper == GT_AND) { assert(falseVal->isUsedFromReg()); - assert(!falseVal->gtGetContainedRegMask().IsRegNumInMask(dstReg)); + assert((falseVal->gtGetContainedRegMask() & genRegMask(dstReg)) == 0); inst_RV_TT(JumpKindToCmov(emitter::emitReverseJumpKind(desc.jumpKind2)), emitTypeSize(select), dstReg, falseVal); } From 80faa407427f384568c322fee899a729d4e05d0b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 10:13:04 -0700 Subject: [PATCH 173/201] Move registerType to Referencable --- src/coreclr/jit/lsra.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 1369ea0c984fa9..4956f4fe6fd44f 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -451,11 +451,12 @@ typedef jitstd::list::reverse_iterator RefPositionReverseIterator; class Referenceable { public: - Referenceable() + Referenceable(RegisterType registerType) { firstRefPosition = nullptr; recentRefPosition = nullptr; lastRefPosition = nullptr; + registerType = registerType; } // A linked list of RefPositions. These are only traversed in the forward @@ -466,6 +467,8 @@ class Referenceable RefPosition* recentRefPosition; RefPosition* lastRefPosition; + RegisterType registerType; + // Get the position of the next reference which is at or greater than // the current location (relies upon recentRefPosition being updated // during traversal). @@ -476,13 +479,12 @@ class Referenceable class RegRecord : public Referenceable { public: - RegRecord() + RegRecord() : Referenceable(IntRegisterType) { assignedInterval = nullptr; previousInterval = nullptr; regNum = REG_NA; isCalleeSave = false; - registerType = IntRegisterType; } void init(regNumber reg) @@ -2210,13 +2212,14 @@ class Interval : public Referenceable { public: Interval(RegisterType registerType, regMaskOnlyOne registerPreferences) - : registerPreferences(registerPreferences) + : Referenceable(registerType) + , registerPreferences(registerPreferences) , registerAversion(RBM_NONE) , relatedInterval(nullptr) , assignedReg(nullptr) , varNum(0) , physReg(REG_COUNT) - , registerType(registerType) + //, registerType(registerType) , isActive(false) , isLocalVar(false) , isSplit(false) @@ -2238,6 +2241,7 @@ class Interval : public Referenceable #ifdef DEBUG , intervalIndex(0) #endif + { } @@ -2273,8 +2277,6 @@ class Interval : public Referenceable // The register to which it is currently assigned. regNumber physReg; - RegisterType registerType; - // Is this Interval currently in a register and live? bool isActive; @@ -2695,9 +2697,7 @@ class RefPosition RegisterType getRegisterType() { - // TODO: The type should be just on `Referenceable` that refposition can extract - // without having to deal with the check. - return isIntervalRef() ? getInterval()->registerType : getReg()->registerType; + return referent->registerType; } // Returns true if it is a reference on a GenTree node. From 204d17b4778bce859fd94f79a0526f273cee311a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 11:59:27 -0700 Subject: [PATCH 174/201] Mark some of the AllRegsMask as const --- src/coreclr/jit/compiler.hpp | 14 +++++++------- src/coreclr/jit/target.h | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index f6db15704caa1f..209a6f98579de9 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5107,7 +5107,7 @@ AllRegsMask AllRegsMask::operator~() return result; } -bool AllRegsMask::operator==(const AllRegsMask& other) +bool AllRegsMask::operator==(const AllRegsMask& other) const { return (_combinedRegisters == other._combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5116,7 +5116,7 @@ bool AllRegsMask::operator==(const AllRegsMask& other) ; } -bool AllRegsMask::operator!=(const AllRegsMask& other) +bool AllRegsMask::operator!=(const AllRegsMask& other) const { return !(*this == other); } @@ -5149,7 +5149,7 @@ void AllRegsMask::Clear() #endif } -bool AllRegsMask::IsEmpty() +bool AllRegsMask::IsEmpty() const { #ifdef HAS_MORE_THAN_64_REGISTERS return ((_combinedRegisters | _predicateRegs) == RBM_NONE); @@ -5158,7 +5158,7 @@ bool AllRegsMask::IsEmpty() #endif } -unsigned AllRegsMask::Count() +unsigned AllRegsMask::Count() const { return genCountBits(_combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5263,7 +5263,7 @@ void AllRegsMask::RemoveRegNum(regNumber reg, var_types type) // ---------------------------------------------------------- // IsRegNumInMask: Checks if `reg` is present in the mask. // -bool AllRegsMask::IsRegNumInMask(regNumber reg) +bool AllRegsMask::IsRegNumInMask(regNumber reg) const { RegBitSet64 value = genRegMask(reg); #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5277,7 +5277,7 @@ bool AllRegsMask::IsRegNumInMask(regNumber reg) // This is similar to IsRegNumInMask(reg, regType) for all platforms // except Arm. For Arm, it calls getRegMask() instead of genRegMask() // to create a mask that needs to be added. -bool AllRegsMask::IsRegNumPresent(regNumber reg, var_types type) +bool AllRegsMask::IsRegNumPresent(regNumber reg, var_types type) const { #ifdef TARGET_ARM return (_combinedRegisters & getRegMask(reg, type)) != RBM_NONE; @@ -5309,7 +5309,7 @@ void AllRegsMask::RemoveRegNumFromMask(regNumber reg, var_types type) // IsRegNumInMask: Removes `reg` from the mask. It is same as IsRegNumInMask(reg) except // that it takes `type` as an argument and adds `reg` to the mask for that type. // -bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) +bool AllRegsMask::IsRegNumInMask(regNumber reg, var_types type) const { return (_combinedRegisters & genRegMask(reg, type)) != RBM_NONE; } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6a460e135b1ec8..f4d0168893cfa3 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -367,9 +367,9 @@ typedef struct _regMaskAll } FORCEINLINE void Clear(); - FORCEINLINE bool IsEmpty(); - FORCEINLINE unsigned Count(); - FORCEINLINE void Create(regNumber reg); + FORCEINLINE bool IsEmpty() const; + FORCEINLINE unsigned Count() const; + //FORCEINLINE void Create(regNumber reg); // Rename this to AddRegNum FORCEINLINE void AddGprRegInMask(regNumber reg); FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); @@ -380,13 +380,13 @@ typedef struct _regMaskAll FORCEINLINE void AddRegNum(regNumber reg, var_types type); FORCEINLINE void RemoveRegNumFromMask(regNumber reg); FORCEINLINE void RemoveRegNum(regNumber reg, var_types type); - FORCEINLINE bool IsRegNumInMask(regNumber reg); - FORCEINLINE bool IsRegNumPresent(regNumber reg, var_types type); + FORCEINLINE bool IsRegNumInMask(regNumber reg) const; + FORCEINLINE bool IsRegNumPresent(regNumber reg, var_types type) const; #ifdef TARGET_ARM FORCEINLINE void AddRegNumInMask(regNumber reg, var_types type); FORCEINLINE void RemoveRegNumFromMask(regNumber reg, var_types type); - FORCEINLINE bool IsRegNumInMask(regNumber reg, var_types type); + FORCEINLINE bool IsRegNumInMask(regNumber reg, var_types type) const; #endif FORCEINLINE void RemoveRegTypeFromMask(regMaskOnlyOne regMaskToRemove, var_types type); @@ -406,8 +406,8 @@ typedef struct _regMaskAll FORCEINLINE void operator|=(const regNumber reg); FORCEINLINE void operator^=(const regNumber reg); FORCEINLINE _regMaskAll operator~(); - FORCEINLINE bool operator==(const _regMaskAll& other); - FORCEINLINE bool operator!=(const _regMaskAll& other); + FORCEINLINE bool operator==(const _regMaskAll& other) const; + FORCEINLINE bool operator!=(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator&(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator|(const _regMaskAll& other) const; From 5d16126e7bc1009c25fbcbbfc45ca5a441868ef5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 12:31:38 -0700 Subject: [PATCH 175/201] Replace AllRegsMask() with AllRegsMask_NONE --- src/coreclr/jit/lsra.cpp | 46 ++++++++++++++++----------------- src/coreclr/jit/lsra.h | 3 +-- src/coreclr/jit/lsraarm.cpp | 2 +- src/coreclr/jit/lsraarm64.cpp | 2 +- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 10 +++---- src/coreclr/jit/lsraxarch.cpp | 4 +-- src/coreclr/jit/regset.h | 2 +- src/coreclr/jit/target.h | 1 + src/coreclr/jit/targetarm.h | 4 +-- 10 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b2a4010a904ff4..5140ef57575504 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5056,8 +5056,8 @@ void LinearScan::allocateRegistersMinimal() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = AllRegsMask(); - regsInUseNextLocation = AllRegsMask(); + regsInUseThisLocation = AllRegsMask_NONE; + regsInUseNextLocation = AllRegsMask_NONE; // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5086,7 +5086,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = AllRegsMask(); + delayRegsToMakeInactive = AllRegsMask_NONE; } #ifdef DEBUG @@ -5143,9 +5143,9 @@ void LinearScan::allocateRegistersMinimal() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = AllRegsMask(); + copyRegsToFree = AllRegsMask_NONE; regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask_NONE; if (!((regsToFree | delayRegsToFree).IsEmpty())) { freeRegisters(regsToFree); @@ -5156,11 +5156,11 @@ void LinearScan::allocateRegistersMinimal() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = AllRegsMask(); - regsInUseThisLocation = AllRegsMask(); + delayRegsToFree = AllRegsMask_NONE; + regsInUseThisLocation = AllRegsMask_NONE; } regsToFree = delayRegsToFree; - delayRegsToFree = AllRegsMask(); + delayRegsToFree = AllRegsMask_NONE; #ifdef DEBUG verifyFreeRegisters(regsToFree); @@ -5202,9 +5202,9 @@ void LinearScan::allocateRegistersMinimal() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = AllRegsMask(); - regsInUseThisLocation = AllRegsMask(); - regsInUseNextLocation = AllRegsMask(); + regsToFree = AllRegsMask_NONE; + regsInUseThisLocation = AllRegsMask_NONE; + regsInUseNextLocation = AllRegsMask_NONE; handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -5746,8 +5746,8 @@ void LinearScan::allocateRegisters() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = AllRegsMask(); - regsInUseNextLocation = AllRegsMask(); + regsInUseThisLocation = AllRegsMask_NONE; + regsInUseNextLocation = AllRegsMask_NONE; // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5776,7 +5776,7 @@ void LinearScan::allocateRegisters() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = AllRegsMask(); + delayRegsToMakeInactive = AllRegsMask_NONE; } #ifdef DEBUG @@ -5831,9 +5831,9 @@ void LinearScan::allocateRegisters() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = AllRegsMask(); + copyRegsToFree = AllRegsMask_NONE; regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = AllRegsMask(); + regsInUseNextLocation = AllRegsMask_NONE; #ifdef TARGET_ARM64 if (hasConsecutiveRegister) { @@ -5850,11 +5850,11 @@ void LinearScan::allocateRegisters() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = AllRegsMask(); - regsInUseThisLocation = AllRegsMask(); + delayRegsToFree = AllRegsMask_NONE; + regsInUseThisLocation = AllRegsMask_NONE; } regsToFree = delayRegsToFree; - delayRegsToFree = AllRegsMask(); + delayRegsToFree = AllRegsMask_NONE; #ifdef DEBUG verifyFreeRegisters(regsToFree); #endif @@ -5913,9 +5913,9 @@ void LinearScan::allocateRegisters() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = AllRegsMask(); - regsInUseThisLocation = AllRegsMask(); - regsInUseNextLocation = AllRegsMask(); + regsToFree = AllRegsMask_NONE; + regsInUseThisLocation = AllRegsMask_NONE; + regsInUseNextLocation = AllRegsMask_NONE; handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -11515,7 +11515,7 @@ void LinearScan::dumpRegRecordHeader() regColumnWidth + 1); // Print a "title row" including the legend and the reg names. - lastDumpedRegisters = AllRegsMask(); + lastDumpedRegisters = AllRegsMask_NONE; dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 4956f4fe6fd44f..7cb8ca7d65c6d4 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1949,8 +1949,7 @@ class LinearScan : public LinearScanInterface void resetRegState() { resetAvailableRegs(); - regsBusyUntilKill = AllRegsMask(); - ; + regsBusyUntilKill = AllRegsMask_NONE; } bool conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition); diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index aea80f705aede2..5d8e5c2f04f854 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -422,7 +422,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask()); + BuildKills(tree, AllRegsMask_NONE); break; case GT_LONG: diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 13b6a543c1999b..87278edfc989aa 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -675,7 +675,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask()); + BuildKills(tree, AllRegsMask_NONE); break; case GT_CNS_DBL: diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index b451f6062ceb03..1c0a03776f81eb 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -434,7 +434,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = AllRegsMask(); + placedArgRegs = AllRegsMask_NONE; numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 2e15de729db80c..ab121276878c2d 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -998,7 +998,7 @@ regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) AllRegsMask LinearScan::getKillSetForReturn() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_LEAVE) - : AllRegsMask(); + : AllRegsMask_NONE; } //------------------------------------------------------------------------ @@ -1012,7 +1012,7 @@ AllRegsMask LinearScan::getKillSetForReturn() AllRegsMask LinearScan::getKillSetForProfilerHook() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_TAILCALL) - : AllRegsMask(); + : AllRegsMask_NONE; } #ifdef DEBUG @@ -2304,8 +2304,8 @@ void LinearScan::buildIntervals() RegState* floatRegState = &compiler->codeGen->floatRegState; intRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; floatRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; - regsInUseThisLocation = AllRegsMask(); - regsInUseNextLocation = AllRegsMask(); + regsInUseThisLocation = AllRegsMask_NONE; + regsInUseNextLocation = AllRegsMask_NONE; #ifdef SWIFT_SUPPORT if (compiler->info.compCallConv == CorInfoCallConvExtension::Swift) @@ -2437,7 +2437,7 @@ void LinearScan::buildIntervals() } numPlacedArgLocals = 0; - placedArgRegs = AllRegsMask(); + placedArgRegs = AllRegsMask_NONE; BasicBlock* predBlock = nullptr; BasicBlock* prevBlock = nullptr; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 1acffbb6af780b..7174f4485aa635 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -137,7 +137,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask()); + BuildKills(tree, AllRegsMask_NONE); break; case GT_PROF_HOOK: @@ -1411,7 +1411,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = AllRegsMask(); + placedArgRegs = AllRegsMask_NONE; numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 7f40ad277e4f79..33699817ba35d1 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -190,7 +190,7 @@ class RegSet void ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. { - _rsAllMaskVars = AllRegsMask(); + _rsAllMaskVars = AllRegsMask_NONE; } private: diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index f4d0168893cfa3..eabec42aa833d0 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -458,6 +458,7 @@ typedef unsigned __int64 regMaskSmall; #error Unsupported or unset target architecture #endif +#define AllRegsMask_NONE RBM_NONE #ifdef HAS_MORE_THAN_64_REGISTERS #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 80e3bbee72e1ae..3f3bb4840b8c1b 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -227,11 +227,11 @@ // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. #define RBM_PROFILER_ENTER_TRASH RBM_NONE - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() + #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_NONE // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must // consider it killed by the return. #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_RET_SCRATCH - #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask() + #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_NONE // Which register are int and long values returned in ? #define REG_INTRET REG_R0 From 99de237f36ce4e5fa9d8ca11bc87b1a269387ca5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 14:26:10 -0700 Subject: [PATCH 176/201] Added CONSTREF and REF to pass AllRegsMask by (const) reference --- src/coreclr/jit/codegen.h | 6 ++++-- src/coreclr/jit/codegenarm64.cpp | 6 ++++-- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/emit.cpp | 2 +- src/coreclr/jit/emit.h | 2 +- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/lsra.cpp | 16 +++++++------- src/coreclr/jit/lsra.h | 36 ++++++++++++++++---------------- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 15 +++++++------ src/coreclr/jit/lsraxarch.cpp | 8 ++++--- src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/regset.h | 4 ++-- src/coreclr/jit/target.h | 5 ++++- 16 files changed, 63 insertions(+), 49 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 0cf102c4851c46..bf8528e6ba5c27 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -348,8 +348,10 @@ class CodeGen final : public CodeGenInterface int spDelta, int spOffset MORE_THAN_64_REG_ARG(var_types type)); - void genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); - void genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); + void genSaveCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); + void genRestoreCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToRestoreMask, + int lowestCalleeSavedOffset, + int spDelta); void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index eec21fb00acfe8..877726ce2dd8ce 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -917,7 +917,9 @@ void CodeGen::genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, // The save set can contain LR in which case LR is saved along with the other callee-saved registers. // But currently Jit doesn't use frames without frame pointer on arm64. // -void CodeGen::genSaveCalleeSavedRegistersHelp(AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta) +void CodeGen::genSaveCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToSaveMask, + int lowestCalleeSavedOffset, + int spDelta) { assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); @@ -1056,7 +1058,7 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(AllRegsMask regsToRestoreMask, +void CodeGen::genRestoreCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta) { diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index e65b921af6a5dd..a00a3f7b40fd3b 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3755,7 +3755,7 @@ void Compiler::dumpRegMask(regMaskOnlyOne mask, var_types type) const // Arguments: // regs - The set of registers to display // -void Compiler::dumpRegMask(AllRegsMask mask) const +void Compiler::dumpRegMask(CONSTREF_AllRegsMask mask) const { if (mask.gprRegs() == RBM_ALLINT) { diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 0e1604dc3a3bdc..4b396b66cfbf8f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10937,7 +10937,7 @@ class Compiler bool compJitHaltMethod(); void dumpRegMask(regMaskOnlyOne mask, var_types type) const; - void dumpRegMask(AllRegsMask mask) const; + void dumpRegMask(CONSTREF_AllRegsMask mask) const; #endif diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 209a6f98579de9..f2baff6f1dcfaa 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5097,7 +5097,7 @@ void AllRegsMask::operator^=(const regNumber reg) #endif } -AllRegsMask AllRegsMask::operator~() +AllRegsMask AllRegsMask::operator~() const { AllRegsMask result; result._combinedRegisters = ~_combinedRegisters; diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index a7c384f1c7ddea..cdda2154b7e16e 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3526,7 +3526,7 @@ void emitter::emitDispMaskRegSet(regMaskPredicate regs) } #endif // FEATURE_MASKED_HW_INTRINSICS -void emitter::emitDispRegSet(AllRegsMask regs) +void emitter::emitDispRegSet(CONSTREF_AllRegsMask regs) { emitDispGprRegSet(regs.gprRegs()); emitDispFloatRegSet(regs.floatRegs(nullptr)); diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index f88a21584be0c1..d6117fd633702a 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3223,7 +3223,7 @@ class emitter #ifdef FEATURE_MASKED_HW_INTRINSICS void emitDispMaskRegSet(regMaskPredicate regs); #endif // FEATURE_MASKED_HW_INTRINSICS - void emitDispRegSet(AllRegsMask regs); + void emitDispRegSet(CONSTREF_AllRegsMask regs); void emitDispVarSet(); #endif diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 81640053cd74b9..6c497d34614bf6 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1079,7 +1079,7 @@ regMaskGpr GenTree::gtGetContainedRegMask() // RegBitSet64 GenTree::gtGetRegMask() const { - RegBitSet64 resultMask; + RegBitSet64 resultMask = RBM_NONE; if (IsMultiRegCall()) { diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 5140ef57575504..445a64719b3f73 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4118,7 +4118,9 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // The new register to use. // #ifdef DEBUG -regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber targetReg, AllRegsMask availableRegs) +regNumber LinearScan::rotateBlockStartLocation(Interval* interval, + regNumber targetReg, + CONSTREF_AllRegsMask availableRegs) { if (targetReg != REG_STK && getLsraBlockBoundaryLocations() == LSRA_BLOCK_BOUNDARY_ROTATE) { @@ -4376,7 +4378,7 @@ void LinearScan::resetAllRegistersState() #ifdef HAS_MORE_THAN_64_REGISTERS -void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap) +void LinearScan::updateDeadCandidatesAtBlockStart(REF_AllRegsMask deadRegMask, VarToRegMap inVarToRegMap) { while (!deadRegMask.IsEmpty()) { @@ -4412,7 +4414,7 @@ void LinearScan::updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarT } } #else - +// TODO: Merge this with MORE_THAN_64 version updating the != RBM_NONE with IsEmpty() check void LinearScan::updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarToRegMap inVarToRegMap) { while (deadRegMask != RBM_NONE) @@ -4876,7 +4878,7 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) } #ifdef HAS_MORE_THAN_64_REGISTERS -void LinearScan::inActivateRegisters(AllRegsMask& inactiveMask) +void LinearScan::inActivateRegisters(REF_AllRegsMask inactiveMask) { while (!inactiveMask.IsEmpty()) { @@ -4959,7 +4961,7 @@ void LinearScan::freeRegister(RegRecord* physRegRecord) // Arguments: // regsToFree - the mask of registers to free // -void LinearScan::freeRegisters(AllRegsMask regsToFree) +void LinearScan::freeRegisters(REF_AllRegsMask regsToFree) { if (regsToFree.IsEmpty()) { @@ -4976,7 +4978,7 @@ void LinearScan::freeRegisters(AllRegsMask regsToFree) } #ifdef HAS_MORE_THAN_64_REGISTERS -void LinearScan::freeRegisterMask(AllRegsMask& freeMask) +void LinearScan::freeRegisterMask(REF_AllRegsMask freeMask) #else void LinearScan::freeRegisterMask(RegBitSet64 freeMask) #endif // HAS_MORE_THAN_64_REGISTERS @@ -11841,7 +11843,7 @@ bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) // Arguments: // regsToFree - Registers that were just freed. // -void LinearScan::verifyFreeRegisters(AllRegsMask regsToFree) +void LinearScan::verifyFreeRegisters(CONSTREF_AllRegsMask regsToFree) { regMaskOnlyOne regsMaskToFree = regsToFree.gprRegs(); regMaskOnlyOne availableRegsMask = availableIntRegs; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 7cb8ca7d65c6d4..a212ef9e8b48c5 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -451,12 +451,12 @@ typedef jitstd::list::reverse_iterator RefPositionReverseIterator; class Referenceable { public: - Referenceable(RegisterType registerType) + Referenceable(RegisterType _registerType) { firstRefPosition = nullptr; recentRefPosition = nullptr; lastRefPosition = nullptr; - registerType = registerType; + registerType = _registerType; } // A linked list of RefPositions. These are only traversed in the forward @@ -501,7 +501,7 @@ class RegRecord : public Referenceable #endif if (emitter::isGeneralRegister(reg)) { - assert(registerType == IntRegisterType); + registerType = IntRegisterType; } else if (emitter::isFloatReg(reg)) { @@ -907,7 +907,7 @@ class LinearScan : public LinearScanInterface { return (LsraBlockBoundaryLocations)(lsraStressMask & LSRA_BLOCK_BOUNDARY_MASK); } - regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, AllRegsMask availableRegs); + regNumber rotateBlockStartLocation(Interval* interval, regNumber targetReg, CONSTREF_AllRegsMask availableRegs); // This controls whether we always insert a GT_RELOAD instruction after a spill // Note that this can be combined with LSRA_SPILL_ALWAYS (or not) @@ -984,7 +984,7 @@ class LinearScan : public LinearScanInterface static bool IsResolutionMove(GenTree* node); static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); - void verifyFreeRegisters(AllRegsMask regsToFree); + void verifyFreeRegisters(CONSTREF_AllRegsMask regsToFree); void verifyFinalAllocation(); void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG @@ -1057,7 +1057,7 @@ class LinearScan : public LinearScanInterface void processBlockEndLocations(BasicBlock* current); void resetAllRegistersState(); #ifdef HAS_MORE_THAN_64_REGISTERS - FORCEINLINE void updateDeadCandidatesAtBlockStart(AllRegsMask& deadRegMask, VarToRegMap inVarToRegMap); + FORCEINLINE void updateDeadCandidatesAtBlockStart(REF_AllRegsMask deadRegMask, VarToRegMap inVarToRegMap); #else FORCEINLINE void updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarToRegMap inVarToRegMap); #endif @@ -1083,7 +1083,7 @@ class LinearScan : public LinearScanInterface void insertZeroInitRefPositions(); // add physreg refpositions for a tree node, based on calling convention and instruction selection predictions - void addRefsForPhysRegMask(AllRegsMask& mask, LsraLocation currentLoc, RefType refType, bool isLastUse); + void addRefsForPhysRegMask(CONSTREF_AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse); void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition); @@ -1141,7 +1141,7 @@ class LinearScan : public LinearScanInterface #endif // Given some tree node add refpositions for all the registers this node kills - bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask); + bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, CONSTREF_AllRegsMask killMask); regMaskOnlyOne allRegs(RegisterType rt); regMaskGpr allByteRegs(); @@ -1151,14 +1151,14 @@ class LinearScan : public LinearScanInterface void makeRegisterInactive(RegRecord* physRegRecord); #ifdef HAS_MORE_THAN_64_REGISTERS - FORCEINLINE void inActivateRegisters(AllRegsMask& inactiveMask); + FORCEINLINE void inActivateRegisters(REF_AllRegsMask inactiveMask); #else FORCEINLINE void inActivateRegisters(RegBitSet64 inactiveMask); #endif void freeRegister(RegRecord* physRegRecord); - void freeRegisters(AllRegsMask regsToFree); + void freeRegisters(REF_AllRegsMask regsToFree); #ifdef HAS_MORE_THAN_64_REGISTERS - FORCEINLINE void freeRegisterMask(AllRegsMask& freeMask); + FORCEINLINE void freeRegisterMask(REF_AllRegsMask freeMask); #else FORCEINLINE void freeRegisterMask(RegBitSet64 freeMask); #endif @@ -1835,7 +1835,7 @@ class LinearScan : public LinearScanInterface return m_AvailableRegs.IsRegNumPresent(reg, regType); } - void setRegsInUse(AllRegsMask regMask) + void setRegsInUse(CONSTREF_AllRegsMask regMask) { m_AvailableRegs &= ~regMask; } @@ -1845,7 +1845,7 @@ class LinearScan : public LinearScanInterface m_AvailableRegs.RemoveRegNum(reg, regType); } - void makeRegsAvailable(AllRegsMask regMask) + void makeRegsAvailable(CONSTREF_AllRegsMask regMask) { m_AvailableRegs |= regMask; #ifdef HAS_MORE_THAN_64_REGISTERS @@ -2038,14 +2038,14 @@ class LinearScan : public LinearScanInterface void HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* callHasFloatRegArgs); RefPosition* BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates = RBM_NONE, int multiRegIdx = 0); void BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates = RBM_NONE); - void BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates); - void BuildKills(GenTree* tree, AllRegsMask killMask); + void BuildCallDefs(GenTree* tree, int dstCount, REF_AllRegsMask dstCandidates); + void BuildKills(GenTree* tree, CONSTREF_AllRegsMask killMask); #ifdef TARGET_ARMARCH - void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask); + void BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, REF_AllRegsMask killMask); #else - void BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask); + void BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, REF_AllRegsMask killMask); #endif - void BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask); + void BuildCallDefsWithKills(GenTree* tree, int dstCount, REF_AllRegsMask dstCandidates, REF_AllRegsMask killMask); int BuildReturn(GenTree* tree); #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 1c0a03776f81eb..205e6df4f0f1db 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -453,7 +453,7 @@ int LinearScan::BuildCall(GenTreeCall* call) // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, AllRegsMask killMask) +void LinearScan::BuildDefWithKills(GenTree* tree, regMaskOnlyOne dstCandidates, REF_AllRegsMask killMask) { assert(!tree->AsCall()->HasMultiRegRetVal()); assert((int)genCountBits(dstCandidates) == 1); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ab121276878c2d..f531f595391d7a 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -701,7 +701,7 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(AllRegsMask& mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(CONSTREF_AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse) { assert(refType == RefTypeKill); @@ -1123,7 +1123,7 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) // This method can add kills even if killMask is RBM_NONE, if this tree is one of the // special cases that signals that we can't permit callee save registers to hold GC refs. -bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, AllRegsMask killMask) +bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, CONSTREF_AllRegsMask killMask) { assert(compiler->IsGprRegMask(killMask.gprRegs())); assert(compiler->IsFloatRegMask(killMask.floatRegs(compiler))); @@ -3104,7 +3104,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, regMaskOnlyOne dstCandidates, i // Notes: // Adds the RefInfo for the definitions to the defList. // -void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, AllRegsMask dstCandidates) +void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, REF_AllRegsMask dstCandidates) { assert(dstCount > 0); assert((int)dstCandidates.Count() == dstCount); @@ -3170,7 +3170,7 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskOnlyOne dstCandid // tree - The node that defines a register // killMask - The mask of registers killed by this node // -void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) +void LinearScan::BuildKills(GenTree* tree, CONSTREF_AllRegsMask killMask) { assert(killMask == getKillSetForNode(tree)); @@ -3215,7 +3215,7 @@ void LinearScan::BuildKills(GenTree* tree, AllRegsMask killMask) // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, AllRegsMask killMask) +void LinearScan::BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne dstCandidates, REF_AllRegsMask killMask) { assert(compiler->IsOnlyOneRegMask(dstCandidates)); @@ -3255,7 +3255,10 @@ void LinearScan::BuildDefWithKills(GenTree* tree, int dstCount, regMaskOnlyOne d // The def and kill functionality is folded into a single method so that the // save and restores of upper vector registers can be bracketed around the def. // -void LinearScan::BuildCallDefsWithKills(GenTree* tree, int dstCount, AllRegsMask dstCandidates, AllRegsMask killMask) +void LinearScan::BuildCallDefsWithKills(GenTree* tree, + int dstCount, + REF_AllRegsMask dstCandidates, + REF_AllRegsMask killMask) { assert(dstCount > 0); assert(!dstCandidates.IsEmpty()); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 7174f4485aa635..c660cfc1b8e962 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1369,6 +1369,11 @@ int LinearScan::BuildCall(GenTreeCall* call) // Now generate defs and kills. AllRegsMask killMask = getKillSetForCall(call); +#ifdef SWIFT_SUPPORT + // Tree is a Swift call with error handling; error register should have been killed + assert(!call->HasSwiftErrorHandling() || ((killMask.gprRegs() & RBM_SWIFT_ERROR) != 0)); +#endif // SWIFT_SUPPORT + if (dstCount > 0) { if (hasMultiRegRetVal) @@ -1390,9 +1395,6 @@ int LinearScan::BuildCall(GenTreeCall* call) #ifdef SWIFT_SUPPORT if (call->HasSwiftErrorHandling()) { - // Tree is a Swift call with error handling; error register should have been killed - assert((killMask.gprRegs() & RBM_SWIFT_ERROR) != 0); - // After a Swift call that might throw returns, we expect the error register to be consumed // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed // before GT_SWIFT_ERROR can consume it. diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 6cd11f33a71aff..fc6ecea228e218 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -239,7 +239,7 @@ void RegSet::rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump)) rsModifiedRegsMask.AddRegNumInMask(reg); } -void RegSet::rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump)) +void RegSet::rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump)) { // TODO: Commented this, so that caller don't have to check if modifiedMask is not RBM_NONE // It doesn't harm if this was RBM_NONE, as it will not modify the trackingMask diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 33699817ba35d1..44c91cc3a0551d 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -86,11 +86,11 @@ class RegSet #endif // !SWIFT_SUPPORT public: - void rsSetRegsModified(AllRegsMask& modifiedMask DEBUGARG(bool suppressDump = false)); + void rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump = false)); void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); #ifdef DEBUG - AllRegsMask rsGetModifiedRegsMask() const + const AllRegsMask& rsGetModifiedRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index eabec42aa833d0..a85ccc2232a5c1 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -405,7 +405,7 @@ typedef struct _regMaskAll FORCEINLINE void operator&=(const _regMaskAll& other); FORCEINLINE void operator|=(const regNumber reg); FORCEINLINE void operator^=(const regNumber reg); - FORCEINLINE _regMaskAll operator~(); + FORCEINLINE _regMaskAll operator~() const; FORCEINLINE bool operator==(const _regMaskAll& other) const; FORCEINLINE bool operator!=(const _regMaskAll& other) const; FORCEINLINE _regMaskAll operator&(const _regMaskAll& other) const; @@ -413,6 +413,9 @@ typedef struct _regMaskAll } AllRegsMask; +#define CONSTREF_AllRegsMask const AllRegsMask& +#define REF_AllRegsMask AllRegsMask + #define GprRegsMask(gprRegs) AllRegsMask(gprRegs) #if REGMASK_BITS == 32 From baf4e36db40b285085f346978db8d89f95893b68 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 15:02:40 -0700 Subject: [PATCH 177/201] fix the merge conflicts from parameter homing --- src/coreclr/jit/codegen.h | 2 +- src/coreclr/jit/codegencommon.cpp | 22 +++++++++++----------- src/coreclr/jit/lsrabuild.cpp | 10 +++++----- src/coreclr/jit/regset.h | 2 -- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index c4ded5825fbfd7..64810c896fc45e 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -263,7 +263,7 @@ class CodeGen final : public CodeGenInterface void genEstablishFramePointer(int delta, bool reportUnwindData); void genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed); - regMaskTP genGetParameterHomingTempRegisterCandidates(); + RegBitSet64 genGetParameterHomingTempRegisterCandidates(); var_types genParamStackStoreType(LclVarDsc* dsc, const ABIPassingSegment& seg); void genSpillOrAddRegisterParam(unsigned lclNum, class RegGraph* graph); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index c7b72ae5cf9d19..f6649e0e736b28 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -3272,16 +3272,16 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed) if ((node->outgoing != nullptr) && (node->copiedReg == REG_NA)) { var_types copyType = node->outgoing->type; - regMaskTP tempRegCandidates = genGetParameterHomingTempRegisterCandidates(); + RegBitSet64 tempRegCandidates = genGetParameterHomingTempRegisterCandidates(); tempRegCandidates &= ~busyRegs; - regMaskTP regTypeMask = varTypeUsesFloatReg(copyType) ? RBM_ALLFLOAT : RBM_ALLINT; - regMaskTP availRegs = tempRegCandidates & regTypeMask; + regMaskOnlyOne regTypeMask = varTypeUsesFloatReg(copyType) ? RBM_ALLFLOAT : RBM_ALLINT; + regMaskOnlyOne availRegs = tempRegCandidates & regTypeMask; // We should have ensured temporary registers are available in // genFinalizeFrame. noway_assert(availRegs != RBM_NONE); - node->copiedReg = genFirstRegNumFromMask(availRegs); + node->copiedReg = genFirstRegNumFromMask(availRegs MORE_THAN_64_REG_ARG(copyType)); busyRegs |= genRegMask(node->copiedReg); instruction ins = ins_Copy(node->reg, copyType); @@ -3360,10 +3360,10 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed) // destination register of a parameter, or because a value passed in one of // these registers is still needed. // -regMaskTP CodeGen::genGetParameterHomingTempRegisterCandidates() +RegBitSet64 CodeGen::genGetParameterHomingTempRegisterCandidates() { return RBM_CALLEE_TRASH | intRegState.rsCalleeRegArgMaskLiveIn | floatRegState.rsCalleeRegArgMaskLiveIn | - regSet.rsGetModifiedRegsMask(); + regSet.rsGetModifiedRegsMask().GetGprFloatCombinedMask(); } /***************************************************************************** @@ -4643,21 +4643,21 @@ void CodeGen::genFinalizeFrame() // Parameter homing may need an additional register to handle conflicts if // all callee trash registers are used by parameters. - regMaskTP homingCandidates = genGetParameterHomingTempRegisterCandidates(); + RegBitSet64 homingCandidates = genGetParameterHomingTempRegisterCandidates(); if (((homingCandidates & ~intRegState.rsCalleeRegArgMaskLiveIn) & RBM_ALLINT) == RBM_NONE) { - regMaskTP extraRegMask = RBM_ALLINT & ~homingCandidates; + regMaskGpr extraRegMask = RBM_ALLINT & ~homingCandidates; assert(extraRegMask != RBM_NONE); - regNumber extraReg = genFirstRegNumFromMask(extraRegMask); + regNumber extraReg = genFirstRegNumFromMask(extraRegMask MORE_THAN_64_REG_ARG(TYP_INT)); JITDUMP("No temporary registers are available for integer parameter homing. Adding %s\n", getRegName(extraReg)); regSet.rsSetRegsModified(genRegMask(extraReg)); } if (((homingCandidates & ~floatRegState.rsCalleeRegArgMaskLiveIn) & RBM_ALLFLOAT) == RBM_NONE) { - regMaskTP extraRegMask = RBM_ALLFLOAT & ~homingCandidates; + regMaskFloat extraRegMask = RBM_ALLFLOAT & ~homingCandidates; assert(extraRegMask != RBM_NONE); - regNumber extraReg = genFirstRegNumFromMask(extraRegMask); + regNumber extraReg = genFirstRegNumFromMask(extraRegMask MORE_THAN_64_REG_ARG(TYP_FLOAT)); JITDUMP("No temporary registers are available for float parameter homing. Adding %s\n", getRegName(extraReg)); regSet.rsSetRegsModified(genRegMask(extraReg)); } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 88dd14bc12b9f9..758f2b81bc30d6 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2916,8 +2916,8 @@ void LinearScan::stressSetRandomParameterPreferences() { CLRRandom rng; rng.Init(compiler->info.compMethodHash()); - regMaskTP intRegs = compiler->codeGen->intRegState.rsCalleeRegArgMaskLiveIn; - regMaskTP floatRegs = compiler->codeGen->floatRegState.rsCalleeRegArgMaskLiveIn; + regMaskGpr intRegs = compiler->codeGen->intRegState.rsCalleeRegArgMaskLiveIn; + regMaskFloat floatRegs = compiler->codeGen->floatRegState.rsCalleeRegArgMaskLiveIn; for (unsigned int varIndex = 0; varIndex < compiler->lvaTrackedCount; varIndex++) { @@ -2930,7 +2930,7 @@ void LinearScan::stressSetRandomParameterPreferences() Interval* interval = getIntervalForLocalVar(varIndex); - regMaskTP* regs; + regMaskOnlyOne* regs; if (interval->registerType == FloatRegisterType) { regs = &floatRegs; @@ -2950,10 +2950,10 @@ void LinearScan::stressSetRandomParameterPreferences() int bitIndex = rng.Next((int)numBits); regNumber prefReg = REG_NA; - regMaskTP regsLeft = *regs; + regMaskOnlyOne regsLeft = *regs; for (int i = 0; i <= bitIndex; i++) { - prefReg = genFirstRegNumFromMaskAndToggle(regsLeft); + prefReg = genFirstRegNumFromMaskAndToggle(regsLeft MORE_THAN_64_REG_ARG(interval->registerType)); } *regs &= ~genRegMask(prefReg); diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 44c91cc3a0551d..7074bda2915f01 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -89,13 +89,11 @@ class RegSet void rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump = false)); void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); -#ifdef DEBUG const AllRegsMask& rsGetModifiedRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; } -#endif AllRegsMask rsGetModifiedCalleeSavedRegsMask() const { From 78614faf36dc8096d19ffca44787e62075894b78 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 15:03:17 -0700 Subject: [PATCH 178/201] jit format --- src/coreclr/jit/block.h | 824 +++++++++++++++-------------- src/coreclr/jit/codegen.h | 4 +- src/coreclr/jit/codegenarm.cpp | 2 +- src/coreclr/jit/codegenarm64.cpp | 6 +- src/coreclr/jit/codegenarmarch.cpp | 6 +- src/coreclr/jit/codegencommon.cpp | 9 +- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/lclvars.cpp | 2 +- src/coreclr/jit/lsra.h | 3 +- src/coreclr/jit/lsraarm64.cpp | 4 +- src/coreclr/jit/lsrabuild.cpp | 11 +- src/coreclr/jit/target.h | 8 +- 13 files changed, 445 insertions(+), 438 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index f8cd96297ff6ad..5c19022df38a2f 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1579,13 +1579,13 @@ struct BasicBlock : private LIR::Range // Pred list maintenance // bool checkPredListOrder(); - void ensurePredListOrder(Compiler * compiler); - void reorderPredList(Compiler * compiler); + void ensurePredListOrder(Compiler* compiler); + void reorderPredList(Compiler* compiler); union { BasicBlock* bbIDom; // Represent the closest dominator to this block (called the Immediate - // Dominator) used to compute the dominance tree. + // Dominator) used to compute the dominance tree. FlowEdge* bbLastPred; // Used early on by fgLinkBasicBlock/fgAddRefPred void* bbSparseProbeList; // Used early on by fgInstrument }; @@ -1597,13 +1597,13 @@ struct BasicBlock : private LIR::Range IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) - // range is not inclusive of the end offset. The count of IL bytes in the block - // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. + // range is not inclusive of the end offset. The count of IL bytes in the block + // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET. #ifdef DEBUG void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" - // for BAD_IL_OFFSET. -#endif // DEBUG + // for BAD_IL_OFFSET. +#endif // DEBUG VARSET_TP bbVarUse; // variables used by block (before a definition) VARSET_TP bbVarDef; // variables assigned by block (before a use) @@ -1617,8 +1617,8 @@ struct BasicBlock : private LIR::Range MemoryKindSet bbMemoryLiveIn : MemoryKindCount; MemoryKindSet bbMemoryLiveOut : MemoryKindCount; MemoryKindSet bbMemoryHavoc : MemoryKindCount; // If true, at some point the block does an operation - // that leaves memory in an unknown state. (E.g., - // unanalyzed call, store through unknown pointer...) + // that leaves memory in an unknown state. (E.g., + // unanalyzed call, store through unknown pointer...) // We want to make phi functions for the special implicit var memory. But since this is not a real // lclVar, and thus has no local #, we can't use a GenTreePhiArg. Instead, we use this struct. @@ -1643,7 +1643,7 @@ struct BasicBlock : private LIR::Range static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list // for Heap. MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this - // value is NULL. + // value is NULL. // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate // that Heap needs a phi definition on entry, or else it is the linked list // of the phi arguments. @@ -1652,13 +1652,13 @@ struct BasicBlock : private LIR::Range VARSET_TP bbScope; // variables in scope over the block - void InitVarSets(class Compiler * comp); + void InitVarSets(class Compiler* comp); /* The following are the standard bit sets for dataflow analysis. - * We perform CSE and range-checks at the same time - * and assertion propagation separately, - * thus we can union them since the two operations are completely disjunct. - */ + * We perform CSE and range-checks at the same time + * and assertion propagation separately, + * thus we can union them since the two operations are completely disjunct. + */ union { @@ -1678,7 +1678,7 @@ struct BasicBlock : private LIR::Range EXPSET_TP bbCseOut; // CSEs available on exit ASSERT_TP bbAssertionOut; // assertions available on exit (global prop, local prop & !BBJ_COND) ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local - // prop) + // prop) }; void* bbEmitCookie; @@ -1700,7 +1700,7 @@ struct BasicBlock : private LIR::Range // Given an the edge b1 -> b2, calculate the slop fraction by // using the higher of the two block weights - static weight_t GetSlopFraction(BasicBlock * b1, BasicBlock * b2) + static weight_t GetSlopFraction(BasicBlock* b1, BasicBlock* b2) { return GetSlopFraction(max(b1->bbWeight, b2->bbWeight)); } @@ -1719,7 +1719,7 @@ struct BasicBlock : private LIR::Range #endif // DEBUG unsigned bbStackDepthOnEntry() const; - void bbSetStack(StackEntry * stack); + void bbSetStack(StackEntry* stack); StackEntry* bbStackOnEntry() const; // "bbNum" is one-based (for unknown reasons); it is sometimes useful to have the corresponding @@ -1753,14 +1753,16 @@ struct BasicBlock : private LIR::Range GenTree* lastNode() const; - bool endsWithJmpMethod(Compiler * comp) const; + bool endsWithJmpMethod(Compiler* comp) const; - bool endsWithTailCall(Compiler * comp, bool fastTailCallsOnly, bool tailCallsConvertibleToLoopOnly, - GenTree** tailCall) const; + bool endsWithTailCall(Compiler* comp, + bool fastTailCallsOnly, + bool tailCallsConvertibleToLoopOnly, + GenTree** tailCall) const; - bool endsWithTailCallOrJmp(Compiler * comp, bool fastTailCallsOnly = false) const; + bool endsWithTailCallOrJmp(Compiler* comp, bool fastTailCallsOnly = false) const; - bool endsWithTailCallConvertibleToLoop(Compiler * comp, GenTree * *tailCall) const; + bool endsWithTailCallConvertibleToLoop(Compiler* comp, GenTree** tailCall) const; // Returns the first statement in the statement list of "this" that is // not an SSA definition (a lcl = phi(...) store). @@ -1840,18 +1842,18 @@ struct BasicBlock : private LIR::Range }; template - BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler * comp, TFunc func); + BasicBlockVisit VisitEHEnclosedHandlerSecondPassSuccs(Compiler* comp, TFunc func); template - BasicBlockVisit VisitAllSuccs(Compiler * comp, TFunc func); + BasicBlockVisit VisitAllSuccs(Compiler* comp, TFunc func); template - BasicBlockVisit VisitEHSuccs(Compiler * comp, TFunc func); + BasicBlockVisit VisitEHSuccs(Compiler* comp, TFunc func); template - BasicBlockVisit VisitRegularSuccs(Compiler * comp, TFunc func); + BasicBlockVisit VisitRegularSuccs(Compiler* comp, TFunc func); - bool HasPotentialEHSuccs(Compiler * comp); + bool HasPotentialEHSuccs(Compiler* comp); // Base class for Successor block/edge iterators. // @@ -2056,7 +2058,7 @@ struct BasicBlock : private LIR::Range return BBSuccList(this); } - BBCompilerSuccList Succs(Compiler * comp) + BBCompilerSuccList Succs(Compiler* comp) { return BBCompilerSuccList(comp, this); } @@ -2066,18 +2068,18 @@ struct BasicBlock : private LIR::Range return BBSuccEdgeList(this); } - BBCompilerSuccEdgeList SuccEdges(Compiler * comp) + BBCompilerSuccEdgeList SuccEdges(Compiler* comp) { return BBCompilerSuccEdgeList(comp, this); } // Clone block state and statements from `from` block to `to` block (which must be new/empty) - static void CloneBlockState(Compiler * compiler, BasicBlock * to, const BasicBlock* from); + static void CloneBlockState(Compiler* compiler, BasicBlock* to, const BasicBlock* from); // Copy the block kind and take memory ownership of the targets. - void TransferTarget(BasicBlock * from); + void TransferTarget(BasicBlock* from); - void MakeLIR(GenTree * firstNode, GenTree * lastNode); + void MakeLIR(GenTree* firstNode, GenTree* lastNode); bool IsLIR() const; void SetDominatedByExceptionalEntryFlag() @@ -2104,478 +2106,478 @@ struct BasicBlock : private LIR::Range return false; } #endif // DEBUG - }; - - template <> - struct JitPtrKeyFuncs : public JitKeyFuncsDefEquals - { - public: - // Make sure hashing is deterministic and not on "ptr." - static unsigned GetHashCode(const BasicBlock* ptr); - }; +}; - // A set of blocks. - typedef JitHashTable, bool> BlkSet; +template <> +struct JitPtrKeyFuncs : public JitKeyFuncsDefEquals +{ +public: + // Make sure hashing is deterministic and not on "ptr." + static unsigned GetHashCode(const BasicBlock* ptr); +}; - // A vector of blocks. - typedef jitstd::vector BlkVector; +// A set of blocks. +typedef JitHashTable, bool> BlkSet; - // A map of block -> set of blocks, can be used as sparse block trees. - typedef JitHashTable, BlkSet*> BlkToBlkSetMap; +// A vector of blocks. +typedef jitstd::vector BlkVector; - // A map of block -> vector of blocks, can be used as sparse block trees. - typedef JitHashTable, BlkVector> BlkToBlkVectorMap; +// A map of block -> set of blocks, can be used as sparse block trees. +typedef JitHashTable, BlkSet*> BlkToBlkSetMap; - // Map from Block to Block. Used for a variety of purposes. - typedef JitHashTable, BasicBlock*> BlockToBlockMap; +// A map of block -> vector of blocks, can be used as sparse block trees. +typedef JitHashTable, BlkVector> BlkToBlkVectorMap; - // BasicBlockIterator: forward iterator for the BasicBlock linked list. - // It is allowed to make changes to the BasicBlock list as long as the current block remains in the list. - // E.g., the current block `m_bbNext` pointer can be altered (such as when inserting a following block), - // as long as the current block is still in the list. - // The block list is expected to be properly doubly-linked. - // - class BasicBlockIterator - { - BasicBlock* m_block; +// Map from Block to Block. Used for a variety of purposes. +typedef JitHashTable, BasicBlock*> BlockToBlockMap; - public: - BasicBlockIterator(BasicBlock* block) - : m_block(block) - { - } - - BasicBlock* operator*() const - { - return m_block; - } - - BasicBlockIterator& operator++() - { - assert(m_block != nullptr); - // Check that we haven't been spliced out of the list. - assert(m_block->IsLast() || m_block->Next()->PrevIs(m_block)); - assert(m_block->IsFirst() || m_block->Prev()->NextIs(m_block)); +// BasicBlockIterator: forward iterator for the BasicBlock linked list. +// It is allowed to make changes to the BasicBlock list as long as the current block remains in the list. +// E.g., the current block `m_bbNext` pointer can be altered (such as when inserting a following block), +// as long as the current block is still in the list. +// The block list is expected to be properly doubly-linked. +// +class BasicBlockIterator +{ + BasicBlock* m_block; - m_block = m_block->Next(); - return *this; - } +public: + BasicBlockIterator(BasicBlock* block) + : m_block(block) + { + } - bool operator!=(const BasicBlockIterator& i) const - { - return m_block != i.m_block; - } - }; + BasicBlock* operator*() const + { + return m_block; + } - // BasicBlockSimpleList: adapter class for forward iteration of a lexically contiguous range of - // BasicBlock, starting at `begin` and going to the end of the function, using range-based `for`, - // normally used via Compiler::Blocks(), e.g.: - // for (BasicBlock* const block : Blocks()) ... - // - class BasicBlockSimpleList + BasicBlockIterator& operator++() { - BasicBlock* m_begin; + assert(m_block != nullptr); + // Check that we haven't been spliced out of the list. + assert(m_block->IsLast() || m_block->Next()->PrevIs(m_block)); + assert(m_block->IsFirst() || m_block->Prev()->NextIs(m_block)); - public: - BasicBlockSimpleList(BasicBlock* begin) - : m_begin(begin) - { - } + m_block = m_block->Next(); + return *this; + } - BasicBlockIterator begin() const - { - return BasicBlockIterator(m_begin); - } + bool operator!=(const BasicBlockIterator& i) const + { + return m_block != i.m_block; + } +}; - BasicBlockIterator end() const - { - return BasicBlockIterator(nullptr); - } - }; +// BasicBlockSimpleList: adapter class for forward iteration of a lexically contiguous range of +// BasicBlock, starting at `begin` and going to the end of the function, using range-based `for`, +// normally used via Compiler::Blocks(), e.g.: +// for (BasicBlock* const block : Blocks()) ... +// +class BasicBlockSimpleList +{ + BasicBlock* m_begin; - // BasicBlockRangeList: adapter class for forward iteration of a lexically contiguous range of - // BasicBlock specified with both `begin` and `end` blocks. `begin` and `end` are *inclusive* - // and must be non-null. E.g., - // for (BasicBlock* const block : BasicBlockRangeList(startBlock, endBlock)) ... - // - // Note that endBlock->bbNext is captured at the beginning of the iteration. Thus, any blocks - // inserted before that will continue the iteration. In particular, inserting blocks between endBlock - // and endBlock->bbNext will yield unexpected results, as the iteration will continue longer than desired. - // - class BasicBlockRangeList +public: + BasicBlockSimpleList(BasicBlock* begin) + : m_begin(begin) { - BasicBlock* m_begin; - BasicBlock* m_end; + } - public: - BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) - : m_begin(begin) - , m_end(end) - { - assert(begin != nullptr); - assert(end != nullptr); - } + BasicBlockIterator begin() const + { + return BasicBlockIterator(m_begin); + } - BasicBlockIterator begin() const - { - return BasicBlockIterator(m_begin); - } + BasicBlockIterator end() const + { + return BasicBlockIterator(nullptr); + } +}; - BasicBlockIterator end() const - { - return BasicBlockIterator(m_end->Next()); // walk until we see the block *following* the `m_end` block - } - }; +// BasicBlockRangeList: adapter class for forward iteration of a lexically contiguous range of +// BasicBlock specified with both `begin` and `end` blocks. `begin` and `end` are *inclusive* +// and must be non-null. E.g., +// for (BasicBlock* const block : BasicBlockRangeList(startBlock, endBlock)) ... +// +// Note that endBlock->bbNext is captured at the beginning of the iteration. Thus, any blocks +// inserted before that will continue the iteration. In particular, inserting blocks between endBlock +// and endBlock->bbNext will yield unexpected results, as the iteration will continue longer than desired. +// +class BasicBlockRangeList +{ + BasicBlock* m_begin; + BasicBlock* m_end; - // BBswtDesc -- descriptor for a switch block - // - // Things to know: - // 1. If bbsHasDefault is true, the default case is the last one in the array of basic block addresses - // namely bbsDstTab[bbsCount - 1]. - // 2. bbsCount must be at least 1, for the default case. bbsCount cannot be zero. It appears that the ECMA spec - // allows for a degenerate switch with zero cases. Normally, the optimizer will optimize degenerate - // switches with just a default case to a BBJ_ALWAYS branch, and a switch with just two cases to a BBJ_COND. - // However, in debuggable code, we might not do that, so bbsCount might be 1. - // - struct BBswtDesc +public: + BasicBlockRangeList(BasicBlock* begin, BasicBlock* end) + : m_begin(begin) + , m_end(end) { - FlowEdge** bbsDstTab; // case label table address - unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) + assert(begin != nullptr); + assert(end != nullptr); + } - // Case number and likelihood of most likely case - // (only known with PGO, only valid if bbsHasDominantCase is true) - unsigned bbsDominantCase; - weight_t bbsDominantFraction; + BasicBlockIterator begin() const + { + return BasicBlockIterator(m_begin); + } - bool bbsHasDefault; // true if last switch case is a default case - bool bbsHasDominantCase; // true if switch has a dominant case + BasicBlockIterator end() const + { + return BasicBlockIterator(m_end->Next()); // walk until we see the block *following* the `m_end` block + } +}; - BBswtDesc() - : bbsHasDefault(true) - , bbsHasDominantCase(false) - { - } +// BBswtDesc -- descriptor for a switch block +// +// Things to know: +// 1. If bbsHasDefault is true, the default case is the last one in the array of basic block addresses +// namely bbsDstTab[bbsCount - 1]. +// 2. bbsCount must be at least 1, for the default case. bbsCount cannot be zero. It appears that the ECMA spec +// allows for a degenerate switch with zero cases. Normally, the optimizer will optimize degenerate +// switches with just a default case to a BBJ_ALWAYS branch, and a switch with just two cases to a BBJ_COND. +// However, in debuggable code, we might not do that, so bbsCount might be 1. +// +struct BBswtDesc +{ + FlowEdge** bbsDstTab; // case label table address + unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) - BBswtDesc(const BBswtDesc* other); + // Case number and likelihood of most likely case + // (only known with PGO, only valid if bbsHasDominantCase is true) + unsigned bbsDominantCase; + weight_t bbsDominantFraction; - BBswtDesc(Compiler* comp, const BBswtDesc* other); + bool bbsHasDefault; // true if last switch case is a default case + bool bbsHasDominantCase; // true if switch has a dominant case - void removeDefault() - { - assert(bbsHasDefault); - assert(bbsCount > 0); - bbsHasDefault = false; - bbsCount--; - } + BBswtDesc() + : bbsHasDefault(true) + , bbsHasDominantCase(false) + { + } - FlowEdge* getDefault() - { - assert(bbsHasDefault); - assert(bbsCount > 0); - return bbsDstTab[bbsCount - 1]; - } - }; + BBswtDesc(const BBswtDesc* other); - // BBSwitchTargetList out-of-class-declaration implementations (here due to C++ ordering requirements). - // + BBswtDesc(Compiler* comp, const BBswtDesc* other); - inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) - : m_bbsDesc(bbsDesc) + void removeDefault() { - assert(m_bbsDesc != nullptr); - assert(m_bbsDesc->bbsDstTab != nullptr); + assert(bbsHasDefault); + assert(bbsCount > 0); + bbsHasDefault = false; + bbsCount--; } - inline BBArrayIterator BBSwitchTargetList::begin() const + FlowEdge* getDefault() { - return BBArrayIterator(m_bbsDesc->bbsDstTab); + assert(bbsHasDefault); + assert(bbsCount > 0); + return bbsDstTab[bbsCount - 1]; } +}; - inline BBArrayIterator BBSwitchTargetList::end() const - { - return BBArrayIterator(m_bbsDesc->bbsDstTab + m_bbsDesc->bbsCount); - } +// BBSwitchTargetList out-of-class-declaration implementations (here due to C++ ordering requirements). +// - // BBehfDesc -- descriptor for a BBJ_EHFINALLYRET block - // - struct BBehfDesc - { - FlowEdge** bbeSuccs; // array of `FlowEdge*` pointing to BBJ_EHFINALLYRET block successors - unsigned bbeCount; // size of `bbeSuccs` array +inline BBSwitchTargetList::BBSwitchTargetList(BBswtDesc* bbsDesc) + : m_bbsDesc(bbsDesc) +{ + assert(m_bbsDesc != nullptr); + assert(m_bbsDesc->bbsDstTab != nullptr); +} - BBehfDesc() - : bbeSuccs(nullptr) - , bbeCount(0) - { - } +inline BBArrayIterator BBSwitchTargetList::begin() const +{ + return BBArrayIterator(m_bbsDesc->bbsDstTab); +} - BBehfDesc(Compiler* comp, const BBehfDesc* other); - }; +inline BBArrayIterator BBSwitchTargetList::end() const +{ + return BBArrayIterator(m_bbsDesc->bbsDstTab + m_bbsDesc->bbsCount); +} - // BBEhfSuccList out-of-class-declaration implementations (here due to C++ ordering requirements). - // +// BBehfDesc -- descriptor for a BBJ_EHFINALLYRET block +// +struct BBehfDesc +{ + FlowEdge** bbeSuccs; // array of `FlowEdge*` pointing to BBJ_EHFINALLYRET block successors + unsigned bbeCount; // size of `bbeSuccs` array - inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) - : m_bbeDesc(bbeDesc) + BBehfDesc() + : bbeSuccs(nullptr) + , bbeCount(0) { - assert(m_bbeDesc != nullptr); - assert((m_bbeDesc->bbeSuccs != nullptr) || (m_bbeDesc->bbeCount == 0)); } - inline BBArrayIterator BBEhfSuccList::begin() const - { - return BBArrayIterator(m_bbeDesc->bbeSuccs); - } + BBehfDesc(Compiler* comp, const BBehfDesc* other); +}; - inline BBArrayIterator BBEhfSuccList::end() const - { - return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); - } +// BBEhfSuccList out-of-class-declaration implementations (here due to C++ ordering requirements). +// - // SuccList out-of-class-declaration implementations - // - inline BasicBlock::SuccList::SuccList(const BasicBlock* block) - { - assert(block != nullptr); +inline BBEhfSuccList::BBEhfSuccList(BBehfDesc* bbeDesc) + : m_bbeDesc(bbeDesc) +{ + assert(m_bbeDesc != nullptr); + assert((m_bbeDesc->bbeSuccs != nullptr) || (m_bbeDesc->bbeCount == 0)); +} - switch (block->bbKind) - { - case BBJ_THROW: - case BBJ_RETURN: - case BBJ_EHFAULTRET: - // We don't need m_succs. +inline BBArrayIterator BBEhfSuccList::begin() const +{ + return BBArrayIterator(m_bbeDesc->bbeSuccs); +} + +inline BBArrayIterator BBEhfSuccList::end() const +{ + return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); +} + +// SuccList out-of-class-declaration implementations +// +inline BasicBlock::SuccList::SuccList(const BasicBlock* block) +{ + assert(block != nullptr); + + switch (block->bbKind) + { + case BBJ_THROW: + case BBJ_RETURN: + case BBJ_EHFAULTRET: + // We don't need m_succs. + m_begin = nullptr; + m_end = nullptr; + break; + + case BBJ_CALLFINALLY: + case BBJ_CALLFINALLYRET: + case BBJ_ALWAYS: + case BBJ_EHCATCHRET: + case BBJ_EHFILTERRET: + case BBJ_LEAVE: + m_succs[0] = block->GetTargetEdge(); + m_begin = &m_succs[0]; + m_end = &m_succs[1]; + break; + + case BBJ_COND: + m_succs[0] = block->GetFalseEdge(); + m_begin = &m_succs[0]; + + // If both fall-through and branch successors are identical, then only include + // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). + if (block->TrueEdgeIs(block->GetFalseEdge())) + { + m_end = &m_succs[1]; + } + else + { + m_succs[1] = block->GetTrueEdge(); + m_end = &m_succs[2]; + } + break; + + case BBJ_EHFINALLYRET: + // We don't use the m_succs in-line data; use the existing successor table in the block. + // We must tolerate iterating successors early in the system, before EH_FINALLYRET successors have + // been computed. + if (block->GetEhfTargets() == nullptr) + { m_begin = nullptr; m_end = nullptr; - break; - - case BBJ_CALLFINALLY: - case BBJ_CALLFINALLYRET: - case BBJ_ALWAYS: - case BBJ_EHCATCHRET: - case BBJ_EHFILTERRET: - case BBJ_LEAVE: - m_succs[0] = block->GetTargetEdge(); - m_begin = &m_succs[0]; - m_end = &m_succs[1]; - break; - - case BBJ_COND: - m_succs[0] = block->GetFalseEdge(); - m_begin = &m_succs[0]; - - // If both fall-through and branch successors are identical, then only include - // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). - if (block->TrueEdgeIs(block->GetFalseEdge())) - { - m_end = &m_succs[1]; - } - else - { - m_succs[1] = block->GetTrueEdge(); - m_end = &m_succs[2]; - } - break; - - case BBJ_EHFINALLYRET: - // We don't use the m_succs in-line data; use the existing successor table in the block. - // We must tolerate iterating successors early in the system, before EH_FINALLYRET successors have - // been computed. - if (block->GetEhfTargets() == nullptr) - { - m_begin = nullptr; - m_end = nullptr; - } - else - { - m_begin = block->GetEhfTargets()->bbeSuccs; - m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; - } - break; - - case BBJ_SWITCH: - // We don't use the m_succs in-line data for switches; use the existing jump table in the block. - assert(block->bbSwtTargets != nullptr); - assert(block->bbSwtTargets->bbsDstTab != nullptr); - m_begin = block->bbSwtTargets->bbsDstTab; - m_end = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; - break; - - default: - unreached(); - } + } + else + { + m_begin = block->GetEhfTargets()->bbeSuccs; + m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; + } + break; - assert(m_end >= m_begin); + case BBJ_SWITCH: + // We don't use the m_succs in-line data for switches; use the existing jump table in the block. + assert(block->bbSwtTargets != nullptr); + assert(block->bbSwtTargets->bbsDstTab != nullptr); + m_begin = block->bbSwtTargets->bbsDstTab; + m_end = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; + break; + + default: + unreached(); } - // We have a simpler struct, BasicBlockList, which is simply a singly-linked - // list of blocks. + assert(m_end >= m_begin); +} - struct BasicBlockList - { - BasicBlockList* next; // The next BasicBlock in the list, nullptr for end of list. - BasicBlock* block; // The BasicBlock of interest. +// We have a simpler struct, BasicBlockList, which is simply a singly-linked +// list of blocks. - BasicBlockList() - : next(nullptr) - , block(nullptr) - { - } - - BasicBlockList(BasicBlock* blk, BasicBlockList* rest) - : next(rest) - , block(blk) - { - } - }; - - // FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) +struct BasicBlockList +{ + BasicBlockList* next; // The next BasicBlock in the list, nullptr for end of list. + BasicBlock* block; // The BasicBlock of interest. - inline weight_t FlowEdge::getLikelyWeight() const + BasicBlockList() + : next(nullptr) + , block(nullptr) { - assert(m_likelihoodSet); - return m_likelihood * m_sourceBlock->bbWeight; } - // BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) - - inline BasicBlock* BBArrayIterator::operator*() const + BasicBlockList(BasicBlock* blk, BasicBlockList* rest) + : next(rest) + , block(blk) { - assert(m_edgeEntry != nullptr); - FlowEdge* edgeTarget = *m_edgeEntry; - assert(edgeTarget != nullptr); - assert(edgeTarget->getDestinationBlock() != nullptr); - return edgeTarget->getDestinationBlock(); } +}; - // Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and - // FlowEdge) +// FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) - inline PredEdgeList::iterator::iterator(FlowEdge* pred) - : m_pred(pred) - { +inline weight_t FlowEdge::getLikelyWeight() const +{ + assert(m_likelihoodSet); + return m_likelihood * m_sourceBlock->bbWeight; +} + +// BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) + +inline BasicBlock* BBArrayIterator::operator*() const +{ + assert(m_edgeEntry != nullptr); + FlowEdge* edgeTarget = *m_edgeEntry; + assert(edgeTarget != nullptr); + assert(edgeTarget->getDestinationBlock() != nullptr); + return edgeTarget->getDestinationBlock(); +} + +// Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and +// FlowEdge) + +inline PredEdgeList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) +{ #ifdef DEBUG - m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); + m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); #endif - } +} - inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() - { - FlowEdge* next = m_pred->getNextPredEdge(); +inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() +{ + FlowEdge* next = m_pred->getNextPredEdge(); #ifdef DEBUG - // Check that the next block is the one we expect to see. - assert(next == m_next); - m_next = (next == nullptr) ? nullptr : next->getNextPredEdge(); + // Check that the next block is the one we expect to see. + assert(next == m_next); + m_next = (next == nullptr) ? nullptr : next->getNextPredEdge(); #endif // DEBUG - m_pred = next; - return *this; - } + m_pred = next; + return *this; +} - template - inline PredBlockList::iterator::iterator(FlowEdge* pred) - : m_pred(pred) +template +inline PredBlockList::iterator::iterator(FlowEdge* pred) + : m_pred(pred) +{ + bool initNextPointer = allowEdits; + INDEBUG(initNextPointer = true); + if (initNextPointer) { - bool initNextPointer = allowEdits; - INDEBUG(initNextPointer = true); - if (initNextPointer) - { - m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); - } + m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); } +} - template - inline BasicBlock* PredBlockList::iterator::operator*() const +template +inline BasicBlock* PredBlockList::iterator::operator*() const +{ + return m_pred->getSourceBlock(); +} + +template +inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() +{ + if (allowEdits) { - return m_pred->getSourceBlock(); + // For editing iterators, m_next is always used and maintained + m_pred = m_next; + m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); } - - template - inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() + else { - if (allowEdits) - { - // For editing iterators, m_next is always used and maintained - m_pred = m_next; - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); - } - else - { - FlowEdge* next = m_pred->getNextPredEdge(); + FlowEdge* next = m_pred->getNextPredEdge(); #ifdef DEBUG - // If allowEdits=false, check that the next block is the one we expect to see. - assert(next == m_next); - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); + // If allowEdits=false, check that the next block is the one we expect to see. + assert(next == m_next); + m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); #endif // DEBUG - m_pred = next; - } - - return *this; + m_pred = next; } - /***************************************************************************** - * - * The following call-backs supplied by the client; it's used by the code - * emitter to convert a basic block to its corresponding emitter cookie. - */ + return *this; +} - void* emitCodeGetCookie(const BasicBlock* block); +/***************************************************************************** + * + * The following call-backs supplied by the client; it's used by the code + * emitter to convert a basic block to its corresponding emitter cookie. + */ - // An enumerator of a block's all successors. In some cases (e.g. SsaBuilder::TopologicalSort) - // using iterators is not exactly efficient, at least because they contain an unnecessary - // member - a pointer to the Compiler object. - class AllSuccessorEnumerator +void* emitCodeGetCookie(const BasicBlock* block); + +// An enumerator of a block's all successors. In some cases (e.g. SsaBuilder::TopologicalSort) +// using iterators is not exactly efficient, at least because they contain an unnecessary +// member - a pointer to the Compiler object. +class AllSuccessorEnumerator +{ + BasicBlock* m_block; + union { - BasicBlock* m_block; - union - { - // We store up to 4 successors inline in the enumerator. For ASP.NET - // and libraries.pmi this is enough in 99.7% of cases. - BasicBlock* m_successors[4]; - BasicBlock** m_pSuccessors; - }; + // We store up to 4 successors inline in the enumerator. For ASP.NET + // and libraries.pmi this is enough in 99.7% of cases. + BasicBlock* m_successors[4]; + BasicBlock** m_pSuccessors; + }; - unsigned m_numSuccs; - unsigned m_curSucc = UINT_MAX; + unsigned m_numSuccs; + unsigned m_curSucc = UINT_MAX; - public: - // Constructs an enumerator of all `block`'s successors. - AllSuccessorEnumerator(Compiler* comp, BasicBlock* block); +public: + // Constructs an enumerator of all `block`'s successors. + AllSuccessorEnumerator(Compiler* comp, BasicBlock* block); - // Gets the block whose successors are enumerated. - BasicBlock* Block() + // Gets the block whose successors are enumerated. + BasicBlock* Block() + { + return m_block; + } + + // Returns the next available successor or `nullptr` if there are no more successors. + BasicBlock* NextSuccessor() + { + m_curSucc++; + if (m_curSucc >= m_numSuccs) { - return m_block; + return nullptr; } - // Returns the next available successor or `nullptr` if there are no more successors. - BasicBlock* NextSuccessor() + if (m_numSuccs <= ArrLen(m_successors)) { - m_curSucc++; - if (m_curSucc >= m_numSuccs) - { - return nullptr; - } - - if (m_numSuccs <= ArrLen(m_successors)) - { - return m_successors[m_curSucc]; - } - - return m_pSuccessors[m_curSucc]; + return m_successors[m_curSucc]; } - }; - // Simple dominator tree node that keeps track of a node's first child and next sibling. - // The parent is provided by BasicBlock::bbIDom. - struct DomTreeNode - { - BasicBlock* firstChild; - BasicBlock* nextSibling; - }; + return m_pSuccessors[m_curSucc]; + } +}; + +// Simple dominator tree node that keeps track of a node's first child and next sibling. +// The parent is provided by BasicBlock::bbIDom. +struct DomTreeNode +{ + BasicBlock* firstChild; + BasicBlock* nextSibling; +}; /*****************************************************************************/ #endif // _BLOCK_H_ - /*****************************************************************************/ +/*****************************************************************************/ diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 64810c896fc45e..433af3b2fe89fa 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -261,8 +261,8 @@ class CodeGen final : public CodeGenInterface // Prolog functions and data (there are a few exceptions for more generally used things) // - void genEstablishFramePointer(int delta, bool reportUnwindData); - void genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed); + void genEstablishFramePointer(int delta, bool reportUnwindData); + void genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed); RegBitSet64 genGetParameterHomingTempRegisterCandidates(); var_types genParamStackStoreType(LclVarDsc* dsc, const ABIPassingSegment& seg); diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index cfb15c7ffa4841..e725fe0397afcc 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -2158,7 +2158,7 @@ bool CodeGen::genCanUsePopToReturn(bool jmpEpilog) void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { assert(compiler->compGeneratingEpilog); - + regMaskFloat maskPopRegsFloat = regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_ALLFLOAT; regMaskGpr maskPopRegsInt = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 877726ce2dd8ce..b3358065e889e6 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -924,7 +924,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToSaveMas assert(spDelta <= 0); assert(-spDelta <= STACK_PROBE_BOUNDARY_THRESHOLD_BYTES); - regMaskGpr maskSaveRegsInt = regsToSaveMask.gprRegs(); + regMaskGpr maskSaveRegsInt = regsToSaveMask.gprRegs(); regMaskFloat maskSaveRegsFloat = regsToSaveMask.floatRegs(compiler); assert(compiler->IsGprRegMask(maskSaveRegsInt)); @@ -1059,8 +1059,8 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask, // None. void CodeGen::genRestoreCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToRestoreMask, - int lowestCalleeSavedOffset, - int spDelta) + int lowestCalleeSavedOffset, + int spDelta) { assert(spDelta >= 0); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 16a0f41d650647..631a8ed35b8eb1 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -5580,9 +5580,9 @@ void CodeGen::genFnEpilog(BasicBlock* block) if (jmpEpilog || genStackAllocRegisterMask(compiler->compLclFrameSize, regSet.rsGetModifiedFloatRegsMask() & RBM_FLT_CALLEE_SAVED) == RBM_NONE) - { - genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); - } + { + genFreeLclFrame(compiler->compLclFrameSize, &unwindStarted); + } if (!unwindStarted) { diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index f6649e0e736b28..30626226826090 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -3271,7 +3271,7 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed) if ((node->outgoing != nullptr) && (node->copiedReg == REG_NA)) { - var_types copyType = node->outgoing->type; + var_types copyType = node->outgoing->type; RegBitSet64 tempRegCandidates = genGetParameterHomingTempRegisterCandidates(); tempRegCandidates &= ~busyRegs; @@ -4167,7 +4167,7 @@ void CodeGen::genHomeSwiftStructParameters(bool handleStack) if (seg.IsPassedInRegister()) { - RegState* regState = genIsValidFloatReg(seg.GetRegister()) ? &floatRegState : &intRegState; + RegState* regState = genIsValidFloatReg(seg.GetRegister()) ? &floatRegState : &intRegState; regMaskOnlyOne regs = seg.GetRegisterMask(); if ((regState->rsCalleeRegArgMaskLiveIn & regs) != RBM_NONE) @@ -4680,8 +4680,9 @@ void CodeGen::genFinalizeFrame() noway_assert(!regSet.rsRegsModified(RBM_FPBASE)); #endif - regMaskFloat maskPushRegsInt = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); - regMaskGpr maskPushRegsFloat = regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_FLT_CALLEE_SAVED; + regMaskFloat maskPushRegsInt = regSet.rsGetModifiedCalleeSavedRegsMask().gprRegs(); + regMaskGpr maskPushRegsFloat = + regSet.rsGetModifiedCalleeSavedRegsMask().GetGprFloatCombinedMask() & RBM_FLT_CALLEE_SAVED; #ifdef FEATURE_MASKED_HW_INTRINSICS regMaskPredicate maskPushRegsPredicate = regSet.rsGetModifiedPredicateRegsMask() & RBM_MSK_CALLEE_SAVED; #endif diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ed809b969340bf..16cbf4087910f2 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9598,7 +9598,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) // registers that profiler callback kills. if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { - regNumber thisPtrReg = compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum(); + regNumber thisPtrReg = compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum(); noway_assert(!AllRegsMask_PROFILER_LEAVE_TRASH.IsRegNumInMask(thisPtrReg)); } diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5b165699a75658..65114ca89c2488 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -947,7 +947,7 @@ struct GenTree RegBitSet64 gtGetRegMask() const; regMaskGpr gtGetGprRegMask() const; - regMaskGpr gtGetContainedRegMask(); + regMaskGpr gtGetContainedRegMask(); GenTreeFlags gtFlags; diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index d3a876234b2dfe..705c57158d242e 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1752,7 +1752,7 @@ void Compiler::lvaClassifyParameterABI() // There is an assumption that args never has predicate registers in case // of OSX/arm64. - RegBitSet64 argRegs = RBM_NONE; + RegBitSet64 argRegs = RBM_NONE; // The calling convention details computed by the old ABI classifier // are wrong since it does not handle the Swift ABI for structs diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index e93129044530f3..47e8c1effce811 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -479,7 +479,8 @@ class Referenceable class RegRecord : public Referenceable { public: - RegRecord() : Referenceable(IntRegisterType) + RegRecord() + : Referenceable(IntRegisterType) { assignedInterval = nullptr; previousInterval = nullptr; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 87278edfc989aa..98e9aa32901d43 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1335,8 +1335,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou const HWIntrinsic intrin(intrinsicTree); - int srcCount = 0; - int dstCount = 0; + int srcCount = 0; + int dstCount = 0; if (HWIntrinsicInfo::IsMultiReg(intrin.id)) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 758f2b81bc30d6..34c1497c42ebdf 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -701,7 +701,10 @@ bool LinearScan::isContainableMemoryOp(GenTree* node) // refType - the type of refposition // isLastUse - true IFF this is a last use of the register // -void LinearScan::addRefsForPhysRegMask(CONSTREF_AllRegsMask mask, LsraLocation currentLoc, RefType refType, bool isLastUse) +void LinearScan::addRefsForPhysRegMask(CONSTREF_AllRegsMask mask, + LsraLocation currentLoc, + RefType refType, + bool isLastUse) { assert(refType == RefTypeKill); @@ -2916,7 +2919,7 @@ void LinearScan::stressSetRandomParameterPreferences() { CLRRandom rng; rng.Init(compiler->info.compMethodHash()); - regMaskGpr intRegs = compiler->codeGen->intRegState.rsCalleeRegArgMaskLiveIn; + regMaskGpr intRegs = compiler->codeGen->intRegState.rsCalleeRegArgMaskLiveIn; regMaskFloat floatRegs = compiler->codeGen->floatRegState.rsCalleeRegArgMaskLiveIn; for (unsigned int varIndex = 0; varIndex < compiler->lvaTrackedCount; varIndex++) @@ -2948,8 +2951,8 @@ void LinearScan::stressSetRandomParameterPreferences() continue; } - int bitIndex = rng.Next((int)numBits); - regNumber prefReg = REG_NA; + int bitIndex = rng.Next((int)numBits); + regNumber prefReg = REG_NA; regMaskOnlyOne regsLeft = *regs; for (int i = 0; i <= bitIndex; i++) { diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index a85ccc2232a5c1..d4baa0e1030ee3 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -369,8 +369,8 @@ typedef struct _regMaskAll FORCEINLINE void Clear(); FORCEINLINE bool IsEmpty() const; FORCEINLINE unsigned Count() const; - //FORCEINLINE void Create(regNumber reg); - // Rename this to AddRegNum + // FORCEINLINE void Create(regNumber reg); + // Rename this to AddRegNum FORCEINLINE void AddGprRegInMask(regNumber reg); FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); @@ -394,7 +394,7 @@ typedef struct _regMaskAll FORCEINLINE bool IsFloatMaskPresent(Compiler* compiler, regMaskFloat maskToCheck) const; FORCEINLINE regMaskOnlyOne GetRegMaskForType(var_types type) const; - FORCEINLINE bool IsGprOrFloatPresent() const; + FORCEINLINE bool IsGprOrFloatPresent() const; FORCEINLINE RegBitSet64 GetGprFloatCombinedMask() const; #ifndef HAS_MORE_THAN_64_REGISTERS FORCEINLINE RegBitSet64 GetAllRegistersMask() const; @@ -414,7 +414,7 @@ typedef struct _regMaskAll } AllRegsMask; #define CONSTREF_AllRegsMask const AllRegsMask& -#define REF_AllRegsMask AllRegsMask +#define REF_AllRegsMask AllRegsMask #define GprRegsMask(gprRegs) AllRegsMask(gprRegs) From ad513dc8cd0055e446069d3b651d08dce86828ea Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 21:27:00 -0700 Subject: [PATCH 179/201] Move from target definition to compiler object --- src/coreclr/jit/compiler.h | 155 ++++++++++++++++++++++++++++++++++ src/coreclr/jit/target.h | 14 +-- src/coreclr/jit/targetamd64.h | 28 +++--- src/coreclr/jit/targetarm.h | 36 ++++---- src/coreclr/jit/targetarm64.h | 40 ++++----- src/coreclr/jit/targetx86.h | 44 +++++----- 6 files changed, 236 insertions(+), 81 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 57c99b4a18b55d..70a2077054b948 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11236,6 +11236,161 @@ class Compiler } #endif // TARGET_XARCH +#ifdef HAS_MORE_THAN_64_REGISTERS + const AllRegsMask AllRegsMask_CALLEE_SAVED = + AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); + const AllRegsMask AllRegsMask_CALLEE_TRASH = + AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH); +#else + const AllRegsMask AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_CALLEE_SAVED); + const AllRegsMask AllRegsMask_CALLEE_TRASH = AllRegsMask(RBM_CALLEE_TRASH); +#endif + +#if defined(TARGET_X86) + + const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_EDX); + // The registers trashed by profiler enter/leave/tailcall hook + // See vm\i386\asmhelpers.asm for more details. + const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; + const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask(); + const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)); + +#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_EAX | RBM_EDX); +#else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH; +#endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = GprRegsMask(RBM_EDX); + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ECX); + + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; + + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling + // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. + const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB); + + const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX); + +#elif defined(TARGET_AMD64) + const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF); + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF); + +#ifdef UNIX_AMD64_ABI + + // See vm\amd64\unixasmhelpers.S for more details. + // + // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. + // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. + // STOP_FOR_GC helper preserves all the 4 possible return registers. + const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), + (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), + RBM_MSK_CALLEE_TRASH); + const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), + (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), + RBM_MSK_CALLEE_TRASH); +#else + // See vm\amd64\asmhelpers.asm for more details. + const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH); + const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_CALLEE_TRASH; +#endif // UNIX_AMD64_ABI + + const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_STOP_FOR_GC_TRASH; + const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_LEAVE_TRASH; + + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; + const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); + +#elif defined(TARGET_ARM) + + const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = + GprRegsMask(RBM_R0 | RBM_R3 | RBM_LR | RBM_DEFAULT_HELPER_CALL_TARGET); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_WRITEBARRIER; + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; + const AllRegsMask AllRegsMask_PROFILER_RET_SCRATCH = GprRegsMask(RBM_R2); + // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must + // consider it killed by the return. + const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_RET_SCRATCH; + const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_NONE; + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). + // See vm\arm\amshelpers.asm for more details. + const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET | RBM_R7 | RBM_R8 | RBM_R11)), + (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET | RBM_F2 | RBM_F3 | RBM_F4 | RBM_F5 | RBM_F6 | RBM_F7))); + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = + (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)); + + const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH); + +#elif defined(TARGET_ARM64) + + const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), + (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), + RBM_MSK_CALLEE_TRASH); + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. + const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; + + const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; + const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; + + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; + const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); +#endif + +#if defined(TARGET_ARM) + // profiler scratch remains gc live + const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; +#else + const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; +#endif + }; // end of class Compiler //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index d4baa0e1030ee3..252de27821df70 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -462,13 +462,13 @@ typedef unsigned __int64 regMaskSmall; #endif #define AllRegsMask_NONE RBM_NONE -#ifdef HAS_MORE_THAN_64_REGISTERS - #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) - #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) -#else - #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_CALLEE_SAVED) - #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_CALLEE_TRASH) -#endif +// #ifdef HAS_MORE_THAN_64_REGISTERS +// #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) +// #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) +// #else +// #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_CALLEE_SAVED) +// #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_CALLEE_TRASH) +// #endif #ifdef TARGET_XARCH diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index bbae933b7490db..d6d8ee73a83d27 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -200,25 +200,25 @@ #define REG_WRITE_BARRIER_SRC REG_ARG_1 #define RBM_WRITE_BARRIER_SRC RBM_ARG_1 - #define AllRegsMask_CALLEE_TRASH_NOGC AllRegsMask_CALLEE_TRASH + // #define AllRegsMask_CALLEE_TRASH_NOGC AllRegsMask_CALLEE_TRASH // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_RAX | RBM_RCX) - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) // Registers killed by CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF) + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF) // We have two register classifications // * callee trash: aka volatile or caller saved @@ -527,24 +527,24 @@ // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. // STOP_FOR_GC helper preserves all the 4 possible return registers. - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH) - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) + // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH) + // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) #else // See vm\amd64\asmhelpers.asm for more details. - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH) - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_CALLEE_TRASH + // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH) + // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_CALLEE_TRASH #endif // UNIX_AMD64_ABI - #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH - #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_LEAVE_TRASH + // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH + // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_LEAVE_TRASH - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH + // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX)) - #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) + // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_RAX diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 6cb5d902f95188..edce679e2a03ad 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -167,20 +167,20 @@ #define RBM_WRITE_BARRIER_SRC_BYREF RBM_ARG_1 #define _RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) - #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) + // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + // // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_WRITEBARRIER + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_WRITEBARRIER - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) + // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R4 @@ -221,16 +221,16 @@ #define REG_PROFILER_ENTER_ARG REG_R0 #define RBM_PROFILER_ENTER_ARG RBM_R0 #define REG_PROFILER_RET_SCRATCH REG_R2 - #define AllRegsMask_PROFILER_RET_SCRATCH GprRegsMask(RBM_R2) + // #define AllRegsMask_PROFILER_RET_SCRATCH GprRegsMask(RBM_R2) // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. #define RBM_PROFILER_ENTER_TRASH RBM_NONE - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_NONE + // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_NONE // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must // consider it killed by the return. - #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_RET_SCRATCH - #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_NONE + // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_RET_SCRATCH + // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_NONE // Which register are int and long values returned in ? #define REG_INTRET REG_R0 @@ -247,12 +247,12 @@ // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). // See vm\arm\amshelpers.asm for more details. -#define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) +// #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) + // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) - #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH) + // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R0 #define REG_FPBASE REG_R11 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 80e909b5668f3b..60da68e3e558de 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -192,20 +192,20 @@ #define RBM_WRITE_BARRIER_SRC_BYREF RBM_R13 #define _RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) - #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) + // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC) + // // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC) - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) + // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R15 @@ -253,13 +253,13 @@ #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 // The registers trashed by profiler enter/leave/tailcall hook -#ifdef HAS_MORE_THAN_64_REGISTERS - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) -#else - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS) | RBM_MSK_CALLEE_TRASH) -#endif - #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH - #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH +// #ifdef HAS_MORE_THAN_64_REGISTERS +// #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) +// #else +// #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS) | RBM_MSK_CALLEE_TRASH) +// #endif + // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH + // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH // Which register are int and long values returned in ? #define REG_INTRET REG_R0 @@ -274,13 +274,13 @@ #define RBM_DOUBLERET RBM_V0 // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH + // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH + // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15)) - #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) + // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15 #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index 42dec2bec5aea3..40fb5cb1889b6f 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -225,27 +225,27 @@ #define RBM_OPTIMIZED_WRITE_BARRIER_SRC (RBM_EAX|RBM_ECX|RBM_EBX|RBM_ESI|RBM_EDI) #endif // NOGC_WRITE_BARRIERS - #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(RBM_EDX) + // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(RBM_EDX) // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. // Note that x86 normally emits an optimized (source-register-specific) write barrier, but can emit // a call to a "general" write barrier. -#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_EAX | RBM_EDX) -#else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH -#endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS +// #ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS +// #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_EAX | RBM_EDX) +// #else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS +// #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH +// #endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER GprRegsMask(RBM_EDX) + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER GprRegsMask(RBM_EDX) - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX) + // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX) - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. - #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ECX) + // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. + // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ECX) // GenericPInvokeCalliHelper unmanaged target parameter #define REG_PINVOKE_TARGET_PARAM REG_EAX @@ -292,14 +292,14 @@ #define RBM_FLOATRET RBM_NONE #define RBM_DOUBLERET RBM_NONE - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH + // // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling - // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. - #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) + // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling + // // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. + // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) - #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX) + // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_ECX #define REG_FPBASE REG_EBP @@ -331,9 +331,9 @@ // The registers trashed by profiler enter/leave/tailcall hook // See vm\i386\asmhelpers.asm for more details. - #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() - #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask() - #define AllRegsMask_PROFILER_TAILCALL_TRASH (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)) + // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() + // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask() + // #define AllRegsMask_PROFILER_TAILCALL_TRASH (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)) // What sort of reloc do we use for [disp32] address mode #define IMAGE_REL_BASED_DISP32 IMAGE_REL_BASED_HIGHLOW From c38cfdbdaaac13899a3bd8d80f047b5a4e10884b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 21:35:10 -0700 Subject: [PATCH 180/201] Use AllRegsMask_* from compiler object --- src/coreclr/jit/codegenarm.cpp | 8 ++++---- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/codegenarmarch.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 10 +++++----- src/coreclr/jit/compiler.h | 8 ++++---- src/coreclr/jit/emit.cpp | 23 +++++++++-------------- src/coreclr/jit/lsrabuild.cpp | 6 +++--- src/coreclr/jit/regset.cpp | 4 +--- src/coreclr/jit/regset.h | 5 ----- 9 files changed, 28 insertions(+), 40 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index e725fe0397afcc..63287bacebcba8 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1663,7 +1663,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, ); } - regSet.verifyRegistersUsed(AllRegsMask_CALLEE_TRASH); + regSet.verifyRegistersUsed(compiler->AllRegsMask_CALLEE_TRASH); } #ifdef PROFILING_SUPPORTED @@ -1825,9 +1825,9 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) { GetEmitter()->emitIns_Mov(INS_mov, attr, REG_R0, REG_PROFILER_RET_SCRATCH, /* canSkip */ false); genTransferRegGCState(REG_R0, REG_PROFILER_RET_SCRATCH); - assert(compiler->IsGprRegMask(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs())); - assert(AllRegsMask_PROFILER_RET_SCRATCH.floatRegs(compiler) == RBM_NONE); - gcInfo.gcMarkRegSetNpt(AllRegsMask_PROFILER_RET_SCRATCH.gprRegs()); + assert(compiler->IsGprRegMask(compiler->AllRegsMask_PROFILER_RET_SCRATCH.gprRegs())); + assert(compiler->AllRegsMask_PROFILER_RET_SCRATCH.floatRegs(compiler) == RBM_NONE); + gcInfo.gcMarkRegSetNpt(compiler->AllRegsMask_PROFILER_RET_SCRATCH.gprRegs()); } } diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index b3358065e889e6..a22b6a8c1af98a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5560,7 +5560,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + AllRegsMask profileEnterTrash = compiler->AllRegsMask_PROFILER_ENTER_TRASH; profileEnterTrash.AddRegMaskForType(RBM_PROFILER_ENTER_ARG_FUNC_ID | RBM_PROFILER_ENTER_ARG_CALLER_SP, TYP_INT); if (profileEnterTrash.IsRegNumInMask(initReg)) { diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 631a8ed35b8eb1..2dfc9d791a6a99 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3394,7 +3394,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + AllRegsMask killMask = compiler->AllRegsMask_CALLEE_TRASH; if (call->IsHelperCall()) { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 16cbf4087910f2..322a97db56b085 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6030,7 +6030,7 @@ void CodeGen::genCall(GenTreeCall* call) // We should not have GC pointers in killed registers live around the call. // GC info for arg registers were cleared when consuming arg nodes above // and LSRA should ensure it for other trashed registers. - AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + AllRegsMask killMask = compiler->AllRegsMask_CALLEE_TRASH; if (call->IsHelperCall()) { @@ -9515,7 +9515,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + AllRegsMask profileEnterTrash = compiler->AllRegsMask_PROFILER_ENTER_TRASH; profileEnterTrash.AddRegMaskForType((RBM_ARG_0 | RBM_ARG_1), TYP_INT); if (profileEnterTrash.IsRegNumInMask(initReg)) { @@ -9556,7 +9556,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed) // If initReg is trashed, either because it was an arg to the enter // callback, or because the enter callback itself trashes it, then it needs // to be zero'ed again before using. - AllRegsMask profileEnterTrash = AllRegsMask_PROFILER_ENTER_TRASH; + AllRegsMask profileEnterTrash = compiler->AllRegsMask_PROFILER_ENTER_TRASH; profileEnterTrash.AddRegMaskForType((RBM_PROFILER_ENTER_ARG_0 | RBM_PROFILER_ENTER_ARG_1), TYP_INT); if (profileEnterTrash.IsRegNumInMask(initReg)) { @@ -9599,7 +9599,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg()) { regNumber thisPtrReg = compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum(); - noway_assert(!AllRegsMask_PROFILER_LEAVE_TRASH.IsRegNumInMask(thisPtrReg)); + noway_assert(!compiler->AllRegsMask_PROFILER_LEAVE_TRASH.IsRegNumInMask(thisPtrReg)); } // At this point return value is computed and stored in RAX or XMM0. @@ -10219,7 +10219,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) // lea esp, [ebp - compiler->compCalleeRegsPushed * REGSIZE_BYTES] needLea = true; } - else if (!regSet.rsRegsModified(AllRegsMask_CALLEE_SAVED)) + else if (!regSet.rsRegsModified(compiler->AllRegsMask_CALLEE_SAVED)) { if (compiler->compLclFrameSize != 0) { diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 70a2077054b948..b4d4218dd4c67e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11281,7 +11281,7 @@ class Compiler const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX); #elif defined(TARGET_AMD64) - const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; + const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. @@ -11385,10 +11385,10 @@ class Compiler #endif #if defined(TARGET_ARM) - // profiler scratch remains gc live - const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; + // profiler scratch remains gc live + const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; #else - const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; + const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; #endif }; // end of class Compiler diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 2af6b4ea9c2a7c..4e47fc8969e6dc 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -10384,7 +10384,7 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) else { // This is the saved set of registers after a normal call. - return AllRegsMask_CALLEE_SAVED; // TODO: We should have this as a constant. + return emitComp->AllRegsMask_CALLEE_SAVED; } } @@ -10413,44 +10413,39 @@ AllRegsMask emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: - result = AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; + result = emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; break; case CORINFO_HELP_ASSIGN_BYREF: - result = AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; + result = emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; break; #if !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) case CORINFO_HELP_PROF_FCN_ENTER: - result = AllRegsMask_PROFILER_ENTER_TRASH; + result = emitComp->AllRegsMask_PROFILER_ENTER_TRASH; break; case CORINFO_HELP_PROF_FCN_LEAVE: -#if defined(TARGET_ARM) - // profiler scratch remains gc live - result = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; -#else - result = AllRegsMask_PROFILER_LEAVE_TRASH; -#endif + result = emitComp->AllRegsMask_PROF_FNC_LEAVE; break; case CORINFO_HELP_PROF_FCN_TAILCALL: - result = AllRegsMask_PROFILER_TAILCALL_TRASH; + result = emitComp->AllRegsMask_PROFILER_TAILCALL_TRASH; break; #endif // !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) #if defined(TARGET_X86) case CORINFO_HELP_INIT_PINVOKE_FRAME: - result = AllRegsMask_INIT_PINVOKE_FRAME_TRASH; + result = emitComp->AllRegsMask_INIT_PINVOKE_FRAME_TRASH; break; #endif // defined(TARGET_X86) case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - result = AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; + result = emitComp->AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; break; default: - result = AllRegsMask_CALLEE_TRASH_NOGC; + result = emitComp->AllRegsMask_CALLEE_TRASH_NOGC; break; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 34c1497c42ebdf..73af96d489c9da 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -761,7 +761,7 @@ AllRegsMask LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) // the allocated register for the `data` operand. However, all the (x86) optimized // helpers have the same kill set: EDX. And note that currently, only x86 can return // `true` for genUseOptimizedWriteBarriers(). - killMask = AllRegsMask_CALLEE_TRASH_NOGC; + killMask = compiler->AllRegsMask_CALLEE_TRASH_NOGC; } else { @@ -848,7 +848,7 @@ regMaskGpr LinearScan::getKillSetForModDiv(GenTreeOp* node) // AllRegsMask LinearScan::getKillSetForCall(GenTreeCall* call) { - AllRegsMask killMask = AllRegsMask_CALLEE_TRASH; + AllRegsMask killMask = compiler->AllRegsMask_CALLEE_TRASH; #ifdef TARGET_X86 if (compiler->compFloatingPointUsed) @@ -1173,7 +1173,7 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo Interval* interval = getIntervalForLocalVar(varIndex); regMaskOnlyOne regsKillMask = killMask.GetRegMaskForType(interval->registerType); const bool isCallKill = - (killMask.gprRegs() == RBM_INT_CALLEE_TRASH) || (killMask == AllRegsMask_CALLEE_TRASH); + (killMask.gprRegs() == RBM_INT_CALLEE_TRASH) || (killMask == compiler->AllRegsMask_CALLEE_TRASH); if (isCallKill) { diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index fc6ecea228e218..01a34c5d398204 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -375,10 +375,8 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) rsMaskPreSpillAlign = RBM_NONE; #endif -#ifdef SWIFT_SUPPORT - rsAllCalleeSavedMask = RBM_CALLEE_SAVED; rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; -#endif // SWIFT_SUPPORT + rsAllCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; #ifdef DEBUG rsModifiedRegsMaskInitialized = false; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 7074bda2915f01..a450b559cf4790 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -77,13 +77,8 @@ class RegSet DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG -#ifdef SWIFT_SUPPORT AllRegsMask rsAllCalleeSavedMask; regMaskGpr rsIntCalleeSavedMask; -#else // !SWIFT_SUPPORT - AllRegsMask rsAllCalleeSavedMask = AllRegsMask_CALLEE_SAVED; - regMaskGpr rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; -#endif // !SWIFT_SUPPORT public: void rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump = false)); From 3b1eeed29108e92c5ac01b3fcd6689fc42adf6ac Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 22:08:44 -0700 Subject: [PATCH 181/201] WIP: Make methods return CONSTREF --- src/coreclr/jit/codegenarm64.cpp | 4 +-- src/coreclr/jit/codegencommon.cpp | 4 +-- src/coreclr/jit/codegenxarch.cpp | 2 +- src/coreclr/jit/compiler.h | 4 ++- src/coreclr/jit/lsra.cpp | 46 +++++++++++++++---------------- src/coreclr/jit/lsra.h | 8 +++--- src/coreclr/jit/lsraarm.cpp | 7 ++--- src/coreclr/jit/lsraarm64.cpp | 10 +++---- src/coreclr/jit/lsraarmarch.cpp | 2 +- src/coreclr/jit/lsrabuild.cpp | 24 ++++++++-------- src/coreclr/jit/lsraxarch.cpp | 12 ++++---- src/coreclr/jit/regset.cpp | 7 ++++- src/coreclr/jit/regset.h | 7 ++--- src/coreclr/jit/target.h | 2 +- 14 files changed, 68 insertions(+), 71 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index a22b6a8c1af98a..a75d5056973ed6 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5211,7 +5211,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, } regMaskGpr callTargetMask = genRegMask(callTargetReg); - AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + CONSTREF_AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet noway_assert((callTargetMask & callKillSet.gprRegs()) == callTargetMask); @@ -5233,7 +5233,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, false /* isJump */ ); - AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); regSet.verifyRegistersUsed(killMask); } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 30626226826090..231e7332d53fcf 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -577,7 +577,7 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // Return Value: // Mask of register kills -- registers whose values are no longer guaranteed to be the same. // -AllRegsMask Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) +CONSTREF_AllRegsMask Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) { switch (helper) { @@ -617,7 +617,7 @@ AllRegsMask Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) case CORINFO_HELP_CHECKED_ASSIGN_REF_EBP: case CORINFO_HELP_CHECKED_ASSIGN_REF_ESI: case CORINFO_HELP_CHECKED_ASSIGN_REF_EDI: - return GprRegsMask(RBM_EDX); + return AllRegsMask_EDX; #endif case CORINFO_HELP_STOP_FOR_GC: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 322a97db56b085..8c2a6a00c95194 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9063,7 +9063,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); regNumber callTarget = REG_NA; - AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index b4d4218dd4c67e..1ea072d03e91bd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8503,7 +8503,7 @@ class Compiler // Gets a register mask that represent the kill set for a helper call since // not all JIT Helper calls follow the standard ABI on the target architecture. - AllRegsMask compHelperCallKillSet(CorInfoHelpFunc helper); + CONSTREF_AllRegsMask compHelperCallKillSet(CorInfoHelpFunc helper); /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -11236,6 +11236,7 @@ class Compiler } #endif // TARGET_XARCH + const AllRegsMask AllRegsMask_NONE = AllRegsMask(); #ifdef HAS_MORE_THAN_64_REGISTERS const AllRegsMask AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); @@ -11279,6 +11280,7 @@ class Compiler const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB); const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX); + const AllRegsMask AllRegsMask_EDX = GprRegsMask(RBM_EDX); #elif defined(TARGET_AMD64) const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 0aef9d70c57c78..aeb904b02b7a84 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5040,8 +5040,8 @@ void LinearScan::allocateRegistersMinimal() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = AllRegsMask_NONE; - regsInUseNextLocation = AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5070,7 +5070,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = AllRegsMask_NONE; + delayRegsToMakeInactive = compiler->AllRegsMask_NONE; } #ifdef DEBUG @@ -5127,9 +5127,9 @@ void LinearScan::allocateRegistersMinimal() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = AllRegsMask_NONE; + copyRegsToFree = compiler->AllRegsMask_NONE; regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; if (!((regsToFree | delayRegsToFree).IsEmpty())) { freeRegisters(regsToFree); @@ -5140,11 +5140,11 @@ void LinearScan::allocateRegistersMinimal() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = AllRegsMask_NONE; - regsInUseThisLocation = AllRegsMask_NONE; + delayRegsToFree = compiler->AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; } regsToFree = delayRegsToFree; - delayRegsToFree = AllRegsMask_NONE; + delayRegsToFree = compiler->AllRegsMask_NONE; #ifdef DEBUG verifyFreeRegisters(regsToFree); @@ -5186,9 +5186,9 @@ void LinearScan::allocateRegistersMinimal() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = AllRegsMask_NONE; - regsInUseThisLocation = AllRegsMask_NONE; - regsInUseNextLocation = AllRegsMask_NONE; + regsToFree = compiler->AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -5730,8 +5730,8 @@ void LinearScan::allocateRegisters() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = AllRegsMask_NONE; - regsInUseNextLocation = AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5760,7 +5760,7 @@ void LinearScan::allocateRegisters() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = AllRegsMask_NONE; + delayRegsToMakeInactive = compiler->AllRegsMask_NONE; } #ifdef DEBUG @@ -5815,9 +5815,9 @@ void LinearScan::allocateRegisters() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = AllRegsMask_NONE; + copyRegsToFree = compiler->AllRegsMask_NONE; regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; #ifdef TARGET_ARM64 if (hasConsecutiveRegister) { @@ -5834,11 +5834,11 @@ void LinearScan::allocateRegisters() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = AllRegsMask_NONE; - regsInUseThisLocation = AllRegsMask_NONE; + delayRegsToFree = compiler->AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; } regsToFree = delayRegsToFree; - delayRegsToFree = AllRegsMask_NONE; + delayRegsToFree = compiler->AllRegsMask_NONE; #ifdef DEBUG verifyFreeRegisters(regsToFree); #endif @@ -5897,9 +5897,9 @@ void LinearScan::allocateRegisters() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = AllRegsMask_NONE; - regsInUseThisLocation = AllRegsMask_NONE; - regsInUseNextLocation = AllRegsMask_NONE; + regsToFree = compiler->AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -11499,7 +11499,7 @@ void LinearScan::dumpRegRecordHeader() regColumnWidth + 1); // Print a "title row" including the legend and the reg names. - lastDumpedRegisters = AllRegsMask_NONE; + lastDumpedRegisters = compiler->AllRegsMask_NONE; dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 47e8c1effce811..57165170a3a5db 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1131,14 +1131,14 @@ class LinearScan : public LinearScanInterface } // Helpers for getKillSetForNode(). - AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); + CONSTREF_AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); regMaskGpr getKillSetForMul(GenTreeOp* tree); AllRegsMask getKillSetForCall(GenTreeCall* call); regMaskGpr getKillSetForModDiv(GenTreeOp* tree); AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); - AllRegsMask getKillSetForReturn(); - AllRegsMask getKillSetForProfilerHook(); + CONSTREF_AllRegsMask getKillSetForReturn(); + CONSTREF_AllRegsMask getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS regMaskGpr getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node); #endif // FEATURE_HW_INTRINSICS @@ -1961,7 +1961,7 @@ class LinearScan : public LinearScanInterface void resetRegState() { resetAvailableRegs(); - regsBusyUntilKill = AllRegsMask_NONE; + regsBusyUntilKill = compiler->AllRegsMask_NONE; } bool conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition); diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 5d8e5c2f04f854..362b9df1ec0c97 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -371,7 +371,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 1; assert(dstCount == 0); BuildUse(tree->gtGetOp1()); - AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildKills(tree, killMask); break; } @@ -422,7 +422,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask_NONE); + BuildKills(tree, compiler->AllRegsMask_NONE); break; case GT_LONG: @@ -469,8 +469,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { srcCount = BuildReturn(tree); - AllRegsMask killMask = getKillSetForReturn(); - BuildKills(tree, killMask); + BuildKills(tree, getKillSetForReturn()); break; } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 98e9aa32901d43..d99e0605ffd0ee 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -666,8 +666,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = 0; assert(dstCount == 0); - AllRegsMask killMask = getKillSetForProfilerHook(); - BuildKills(tree, killMask); + BuildKills(tree, getKillSetForProfilerHook()); break; } @@ -675,7 +674,7 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask_NONE); + BuildKills(tree, compiler->AllRegsMask_NONE); break; case GT_CNS_DBL: @@ -740,8 +739,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { srcCount = BuildReturn(tree); - AllRegsMask killMask = getKillSetForReturn(); - BuildKills(tree, killMask); + BuildKills(tree, getKillSetForReturn()); break; } @@ -833,7 +831,7 @@ int LinearScan::BuildNode(GenTree* tree) BuildUse(tree->gtGetOp1()); srcCount = 1; assert(dstCount == 0); - AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildKills(tree, killMask); break; } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 205e6df4f0f1db..07fa7af7ba8b0c 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -434,7 +434,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = AllRegsMask_NONE; + placedArgRegs = compiler->AllRegsMask_NONE; numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 73af96d489c9da..76ac303729332b 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -746,12 +746,10 @@ void LinearScan::addRefsForPhysRegMask(CONSTREF_AllRegsMask mask, // // Return Value: a register mask of the registers killed // -AllRegsMask LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) +CONSTREF_AllRegsMask LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) { assert(tree->OperIs(GT_STOREIND)); - AllRegsMask killMask; - GCInfo::WriteBarrierForm writeBarrierForm = compiler->codeGen->gcInfo.gcIsWriteBarrierCandidate(tree); if (writeBarrierForm != GCInfo::WBF_NoBarrier) { @@ -761,16 +759,16 @@ AllRegsMask LinearScan::getKillSetForStoreInd(GenTreeStoreInd* tree) // the allocated register for the `data` operand. However, all the (x86) optimized // helpers have the same kill set: EDX. And note that currently, only x86 can return // `true` for genUseOptimizedWriteBarriers(). - killMask = compiler->AllRegsMask_CALLEE_TRASH_NOGC; + return compiler->AllRegsMask_CALLEE_TRASH_NOGC; } else { // Figure out which helper we're going to use, and then get the kill set for that helper. CorInfoHelpFunc helper = compiler->codeGen->genWriteBarrierHelperForWriteBarrierForm(writeBarrierForm); - killMask = compiler->compHelperCallKillSet(helper); + return compiler->compHelperCallKillSet(helper); } } - return killMask; + return compiler->AllRegsMask_NONE; } //------------------------------------------------------------------------ @@ -998,10 +996,10 @@ regMaskGpr LinearScan::getKillSetForHWIntrinsic(GenTreeHWIntrinsic* node) // // Return Value: a register mask of the registers killed // -AllRegsMask LinearScan::getKillSetForReturn() +CONSTREF_AllRegsMask LinearScan::getKillSetForReturn() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_LEAVE) - : AllRegsMask_NONE; + : compiler->AllRegsMask_NONE; } //------------------------------------------------------------------------ @@ -1012,10 +1010,10 @@ AllRegsMask LinearScan::getKillSetForReturn() // // Return Value: a register mask of the registers killed // -AllRegsMask LinearScan::getKillSetForProfilerHook() +CONSTREF_AllRegsMask LinearScan::getKillSetForProfilerHook() { return compiler->compIsProfilerHookNeeded() ? compiler->compHelperCallKillSet(CORINFO_HELP_PROF_FCN_TAILCALL) - : AllRegsMask_NONE; + : compiler->AllRegsMask_NONE; } #ifdef DEBUG @@ -2307,8 +2305,8 @@ void LinearScan::buildIntervals() RegState* floatRegState = &compiler->codeGen->floatRegState; intRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; floatRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; - regsInUseThisLocation = AllRegsMask_NONE; - regsInUseNextLocation = AllRegsMask_NONE; + regsInUseThisLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation = compiler->AllRegsMask_NONE; #ifdef SWIFT_SUPPORT if (compiler->info.compCallConv == CorInfoCallConvExtension::Swift) @@ -2447,7 +2445,7 @@ void LinearScan::buildIntervals() #endif numPlacedArgLocals = 0; - placedArgRegs = AllRegsMask_NONE; + placedArgRegs = compiler->AllRegsMask_NONE; BasicBlock* predBlock = nullptr; BasicBlock* prevBlock = nullptr; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index ae9e9a561e391f..45c8ba6cfa08dc 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -137,15 +137,14 @@ int LinearScan::BuildNode(GenTree* tree) // This kills GC refs in callee save regs srcCount = 0; assert(dstCount == 0); - BuildKills(tree, AllRegsMask_NONE); + BuildKills(tree, compiler->AllRegsMask_NONE); break; case GT_PROF_HOOK: { srcCount = 0; assert(dstCount == 0); - AllRegsMask killMask = getKillSetForProfilerHook(); - BuildKills(tree, killMask); + BuildKills(tree, getKillSetForProfilerHook()); break; } @@ -191,8 +190,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { srcCount = BuildReturn(tree); - AllRegsMask killMask = getKillSetForReturn(); - BuildKills(tree, killMask); + BuildKills(tree, getKillSetForReturn()); break; } @@ -298,7 +296,7 @@ int LinearScan::BuildNode(GenTree* tree) RefPosition* internalDef = buildInternalIntRegisterDefForNode(tree); srcCount = BuildOperandUses(tree->gtGetOp1()); buildInternalRegisterUses(); - AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); BuildKills(tree, killMask); } break; @@ -1413,7 +1411,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = AllRegsMask_NONE; + placedArgRegs = compiler->AllRegsMask_NONE; numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 01a34c5d398204..f4aba2c3db13ba 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -108,7 +108,7 @@ void RegSet::verifyRegUsed(regNumber reg) // should simply validate that the register (or registers) have // already been added to the modified set. -void RegSet::verifyRegistersUsed(AllRegsMask regs) +void RegSet::verifyRegistersUsed(CONSTREF_AllRegsMask regs) { if (m_rsCompiler->opts.OptimizationDisabled()) { @@ -297,6 +297,11 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) rsModifiedRegsMask.RemoveRegTypeFromMask(mask, TYP_INT); } +void RegSet::ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. +{ + _rsAllMaskVars = m_rsCompiler->AllRegsMask_NONE; +} + void RegSet::SetMaskVars(AllRegsMask newMaskVars) { #ifdef DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index a450b559cf4790..61df5db7bbf8f0 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -149,7 +149,7 @@ class RegSet void verifyRegUsed(regNumber reg); void verifyGprRegUsed(regNumber reg); - void verifyRegistersUsed(AllRegsMask mask); + void verifyRegistersUsed(CONSTREF_AllRegsMask mask); public: regMaskOnlyOne GetMaskVars(var_types type) const // 'get' property function for rsMaskVars property @@ -181,10 +181,7 @@ class RegSet SetMaskVars(newMask); } - void ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. - { - _rsAllMaskVars = AllRegsMask_NONE; - } + void ClearMaskVars(); // Like SetMaskVars(RBM_NONE), but without any debug output. private: AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 252de27821df70..1f2f9d2bb1ad33 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -461,7 +461,7 @@ typedef unsigned __int64 regMaskSmall; #error Unsupported or unset target architecture #endif -#define AllRegsMask_NONE RBM_NONE +//#define AllRegsMask_NONE RBM_NONE // #ifdef HAS_MORE_THAN_64_REGISTERS // #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) // #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) From d8ba17473ad0b7bb75a7bcabca6282ee12f4b11b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 23:36:20 -0700 Subject: [PATCH 182/201] Introduce compInitAllRegsMask() --- src/coreclr/jit/compiler.cpp | 160 +++++++++++++++++++++++++++++++- src/coreclr/jit/compiler.h | 174 ++++------------------------------- 2 files changed, 177 insertions(+), 157 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 02b8c39c96a167..54e2ba03b1da60 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2019,6 +2019,163 @@ void Compiler::compInit(ArenaAllocator* pAlloc, new (&Metrics, jitstd::placement_t()) JitMetrics(); } +void Compiler::compInitAllRegsMask() +{ + AllRegsMask_NONE = AllRegsMask(); +#ifdef HAS_MORE_THAN_64_REGISTERS + AllRegsMask_CALLEE_SAVED = + AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); + AllRegsMask_CALLEE_TRASH = + AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH); +#else + AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_CALLEE_SAVED); + AllRegsMask_CALLEE_TRASH = AllRegsMask(RBM_CALLEE_TRASH); +#endif + +#if defined(TARGET_X86) + + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_EDX); + // The registers trashed by profiler enter/leave/tailcall hook + // See vm\i386\asmhelpers.asm for more details. + AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; + AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask(); + AllRegsMask_PROFILER_TAILCALL_TRASH = (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)); + +#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_EAX | RBM_EDX); +#else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH; +#endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = GprRegsMask(RBM_EDX); + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ECX); + + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; + + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling + // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. + AllRegsMask_INIT_PINVOKE_FRAME_TRASH = GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB); + + AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX); + AllRegsMask_EDX = GprRegsMask(RBM_EDX); + +#elif defined(TARGET_AMD64) + AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF); + AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF); + +#ifdef UNIX_AMD64_ABI + + // See vm\amd64\unixasmhelpers.S for more details. + // + // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. + // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. + // STOP_FOR_GC helper preserves all the 4 possible return registers. + AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), + (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH); + AllRegsMask_PROFILER_ENTER_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), + RBM_MSK_CALLEE_TRASH); +#else + // See vm\amd64\asmhelpers.asm for more details. + AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH); + AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_CALLEE_TRASH; +#endif // UNIX_AMD64_ABI + + AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_STOP_FOR_GC_TRASH; + AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_LEAVE_TRASH; + + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; + AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); + +#elif defined(TARGET_ARM) + + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = + GprRegsMask(RBM_R0 | RBM_R3 | RBM_LR | RBM_DEFAULT_HELPER_CALL_TARGET); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_WRITEBARRIER; + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; + AllRegsMask_PROFILER_RET_SCRATCH = GprRegsMask(RBM_R2); + // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must + // consider it killed by the return. + AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_RET_SCRATCH; + AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_NONE; + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). + // See vm\arm\amshelpers.asm for more details. + AllRegsMask_STOP_FOR_GC_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET | RBM_R7 | RBM_R8 | RBM_R11)), + (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET | RBM_F2 | RBM_F3 | RBM_F4 | RBM_F5 | RBM_F6 | RBM_F7))); + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + AllRegsMask_INIT_PINVOKE_FRAME_TRASH = + (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)); + + AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH); + +#elif defined(TARGET_ARM64) + + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + AllRegsMask_PROFILER_ENTER_TRASH = + AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), + (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH); + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; + + AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; + AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; + + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. + AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; + AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); +#endif + +#if defined(TARGET_ARM) + // profiler scratch remains gc live + AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; +#else + AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; +#endif +} /***************************************************************************** * * Destructor @@ -2564,7 +2721,7 @@ void Compiler::compInitOptions(JitFlags* jitFlags) } else { - verbose = false; + verbose = false; codeGen->setVerbose(false); } verboseTrees = verbose && shouldUseVerboseTrees(); @@ -6979,6 +7136,7 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr, // compInitOptions will set the correct verbose flag. compInitOptions(compileFlags); + compInitAllRegsMask(); if (!compIsForInlining() && !opts.altJit && opts.jitFlags->IsSet(JitFlags::JIT_FLAG_ALT_JIT)) { diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 1ea072d03e91bd..3d28411dbc812d 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10571,6 +10571,7 @@ class Compiler COMP_HANDLE compHnd, CORINFO_METHOD_INFO* methodInfo, InlineInfo* inlineInfo); + void compInitAllRegsMask(); void compDone(); static void compDisplayStaticSizes(); @@ -11236,162 +11237,23 @@ class Compiler } #endif // TARGET_XARCH - const AllRegsMask AllRegsMask_NONE = AllRegsMask(); -#ifdef HAS_MORE_THAN_64_REGISTERS - const AllRegsMask AllRegsMask_CALLEE_SAVED = - AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); - const AllRegsMask AllRegsMask_CALLEE_TRASH = - AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH); -#else - const AllRegsMask AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_CALLEE_SAVED); - const AllRegsMask AllRegsMask_CALLEE_TRASH = AllRegsMask(RBM_CALLEE_TRASH); -#endif - -#if defined(TARGET_X86) - - const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_EDX); - // The registers trashed by profiler enter/leave/tailcall hook - // See vm\i386\asmhelpers.asm for more details. - const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; - const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask(); - const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)); - -#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_EAX | RBM_EDX); -#else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH; -#endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = GprRegsMask(RBM_EDX); - - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX); - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_ECX); - - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; - - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling - // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. - const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB); - - const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX); - const AllRegsMask AllRegsMask_EDX = GprRegsMask(RBM_EDX); - -#elif defined(TARGET_AMD64) - const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = AllRegsMask_CALLEE_TRASH; - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF); - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF); - -#ifdef UNIX_AMD64_ABI - - // See vm\amd64\unixasmhelpers.S for more details. - // - // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. - // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. - // STOP_FOR_GC helper preserves all the 4 possible return registers. - const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = - AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), - (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), - RBM_MSK_CALLEE_TRASH); - const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), - (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), - RBM_MSK_CALLEE_TRASH); -#else - // See vm\amd64\asmhelpers.asm for more details. - const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = - AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH); - const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_CALLEE_TRASH; -#endif // UNIX_AMD64_ABI - - const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_STOP_FOR_GC_TRASH; - const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_LEAVE_TRASH; - - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; - const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); - -#elif defined(TARGET_ARM) - - const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); - const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; - - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = - GprRegsMask(RBM_R0 | RBM_R3 | RBM_LR | RBM_DEFAULT_HELPER_CALL_TARGET); - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_WRITEBARRIER; - - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; - const AllRegsMask AllRegsMask_PROFILER_RET_SCRATCH = GprRegsMask(RBM_R2); - // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must - // consider it killed by the return. - const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_RET_SCRATCH; - const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_NONE; - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). - // See vm\arm\amshelpers.asm for more details. - const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = - AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET | RBM_R7 | RBM_R8 | RBM_R11)), - (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET | RBM_F2 | RBM_F3 | RBM_F4 | RBM_F5 | RBM_F6 | RBM_F7))); - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = - (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)); - - const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_INT_CALLEE_TRASH); - -#elif defined(TARGET_ARM64) - - const AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); - const AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH = - AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), - (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), - RBM_MSK_CALLEE_TRASH); - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC); - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; - - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - const AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. - const AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = AllRegsMask_CALLEE_TRASH_NOGC; - - const AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; - const AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH = AllRegsMask_PROFILER_ENTER_TRASH; - - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - const AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask_CALLEE_TRASH; - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - const AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH = AllRegsMask_CALLEE_TRASH; - const AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH = GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH); -#endif - -#if defined(TARGET_ARM) - // profiler scratch remains gc live - const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; -#else - const AllRegsMask AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; -#endif +AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; +AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; +AllRegsMask AllRegsMask_CALLEE_SAVED; +AllRegsMask AllRegsMask_CALLEE_TRASH; +AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC; +AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER; +AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF; +AllRegsMask AllRegsMask_EDX; +AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH; +AllRegsMask AllRegsMask_NONE; +AllRegsMask AllRegsMask_PROF_FNC_LEAVE; +AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH; +AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH; +AllRegsMask AllRegsMask_PROFILER_RET_SCRATCH; +AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH; +AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH; +AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; }; // end of class Compiler From 0ccb59a400084bf46af3f6e6968f8f6c01c1db83 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 23:36:40 -0700 Subject: [PATCH 183/201] Remove rsAllCalleeSavedMask from RegSet The reason being, it gets initialized as part of codeGen initialization which happens way before AllRegsMask on compiler object is initialized. We cannot initialize --- src/coreclr/jit/regset.cpp | 17 +++++++++++++++-- src/coreclr/jit/regset.h | 7 +------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index f4aba2c3db13ba..919c352eaf8810 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -150,7 +150,6 @@ void RegSet::rsClearRegsModified() // so don't treat it as callee-save. if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) { - rsAllCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); rsIntCalleeSavedMask &= ~RBM_SWIFT_ERROR; } #endif // SWIFT_SUPPORT @@ -188,6 +187,21 @@ void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, } #endif +AllRegsMask RegSet::rsGetModifiedCalleeSavedRegsMask() const +{ + assert(rsModifiedRegsMaskInitialized); + AllRegsMask allCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; +#ifdef SWIFT_SUPPORT + // If this method has a SwiftError* parameter, we will return SwiftError::Value in REG_SWIFT_ERROR, + // so don't treat it as callee-save. + if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) + { + allCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); + } +#endif // SWIFT_SUPPORT + return (rsModifiedRegsMask & allCalleeSavedMask); +} + void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsGprRegMask(mask)); @@ -381,7 +395,6 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) #endif rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; - rsAllCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; #ifdef DEBUG rsModifiedRegsMaskInitialized = false; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 61df5db7bbf8f0..6c6c1ac02589e4 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -77,7 +77,6 @@ class RegSet DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG - AllRegsMask rsAllCalleeSavedMask; regMaskGpr rsIntCalleeSavedMask; public: @@ -90,11 +89,7 @@ class RegSet return rsModifiedRegsMask; } - AllRegsMask rsGetModifiedCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & rsAllCalleeSavedMask); - } + AllRegsMask rsGetModifiedCalleeSavedRegsMask() const; regMaskGpr rsGetModifiedIntCalleeSavedRegsMask() const { From e27691c7edcad204a56bf96f746ef0ac319e8404 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 23:40:31 -0700 Subject: [PATCH 184/201] Revert "Remove rsAllCalleeSavedMask from RegSet" This reverts commit 0ccb59a400084bf46af3f6e6968f8f6c01c1db83. --- src/coreclr/jit/regset.cpp | 17 ++--------------- src/coreclr/jit/regset.h | 7 ++++++- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 919c352eaf8810..f4aba2c3db13ba 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -150,6 +150,7 @@ void RegSet::rsClearRegsModified() // so don't treat it as callee-save. if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) { + rsAllCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); rsIntCalleeSavedMask &= ~RBM_SWIFT_ERROR; } #endif // SWIFT_SUPPORT @@ -187,21 +188,6 @@ void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, } #endif -AllRegsMask RegSet::rsGetModifiedCalleeSavedRegsMask() const -{ - assert(rsModifiedRegsMaskInitialized); - AllRegsMask allCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; -#ifdef SWIFT_SUPPORT - // If this method has a SwiftError* parameter, we will return SwiftError::Value in REG_SWIFT_ERROR, - // so don't treat it as callee-save. - if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) - { - allCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); - } -#endif // SWIFT_SUPPORT - return (rsModifiedRegsMask & allCalleeSavedMask); -} - void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsGprRegMask(mask)); @@ -395,6 +381,7 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) #endif rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; + rsAllCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; #ifdef DEBUG rsModifiedRegsMaskInitialized = false; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 6c6c1ac02589e4..61df5db7bbf8f0 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -77,6 +77,7 @@ class RegSet DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG + AllRegsMask rsAllCalleeSavedMask; regMaskGpr rsIntCalleeSavedMask; public: @@ -89,7 +90,11 @@ class RegSet return rsModifiedRegsMask; } - AllRegsMask rsGetModifiedCalleeSavedRegsMask() const; + AllRegsMask rsGetModifiedCalleeSavedRegsMask() const + { + assert(rsModifiedRegsMaskInitialized); + return (rsModifiedRegsMask & rsAllCalleeSavedMask); + } regMaskGpr rsGetModifiedIntCalleeSavedRegsMask() const { From dc96a996f99300b98d1eea1b1b0c013d270e2286 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 Apr 2024 23:40:55 -0700 Subject: [PATCH 185/201] Move the initialization of rbmAllFloat inside compInitAllRegsMask() --- src/coreclr/jit/compiler.cpp | 50 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 54e2ba03b1da60..650177b049fa42 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2021,6 +2021,32 @@ void Compiler::compInit(ArenaAllocator* pAlloc, void Compiler::compInitAllRegsMask() { +#if defined(TARGET_AMD64) + rbmAllFloat = RBM_ALLFLOAT_INIT; + rbmFltCalleeTrash = RBM_FLT_CALLEE_TRASH_INIT; + cntCalleeTrashFloat = CNT_CALLEE_TRASH_FLOAT_INIT; + + if (canUseEvexEncoding()) + { + rbmAllFloat |= RBM_HIGHFLOAT; + rbmFltCalleeTrash |= RBM_HIGHFLOAT; + cntCalleeTrashFloat += CNT_CALLEE_TRASH_HIGHFLOAT; + } +#endif // TARGET_AMD64 + +#if defined(TARGET_XARCH) + rbmAllMask = RBM_ALLMASK_INIT; + rbmMskCalleeTrash = RBM_MSK_CALLEE_TRASH_INIT; + cntCalleeTrashMask = CNT_CALLEE_TRASH_MASK_INIT; + + if (canUseEvexEncoding()) + { + rbmAllMask |= RBM_ALLMASK_EVEX; + rbmMskCalleeTrash |= RBM_MSK_CALLEE_TRASH_EVEX; + cntCalleeTrashMask += CNT_CALLEE_TRASH_MASK_EVEX; + } +#endif // TARGET_XARCH + AllRegsMask_NONE = AllRegsMask(); #ifdef HAS_MORE_THAN_64_REGISTERS AllRegsMask_CALLEE_SAVED = @@ -3614,31 +3640,7 @@ void Compiler::compInitOptions(JitFlags* jitFlags) } #endif // defined(DEBUG) && defined(TARGET_ARM64) -#if defined(TARGET_AMD64) - rbmAllFloat = RBM_ALLFLOAT_INIT; - rbmFltCalleeTrash = RBM_FLT_CALLEE_TRASH_INIT; - cntCalleeTrashFloat = CNT_CALLEE_TRASH_FLOAT_INIT; - - if (canUseEvexEncoding()) - { - rbmAllFloat |= RBM_HIGHFLOAT; - rbmFltCalleeTrash |= RBM_HIGHFLOAT; - cntCalleeTrashFloat += CNT_CALLEE_TRASH_HIGHFLOAT; - } -#endif // TARGET_AMD64 - #if defined(TARGET_XARCH) - rbmAllMask = RBM_ALLMASK_INIT; - rbmMskCalleeTrash = RBM_MSK_CALLEE_TRASH_INIT; - cntCalleeTrashMask = CNT_CALLEE_TRASH_MASK_INIT; - - if (canUseEvexEncoding()) - { - rbmAllMask |= RBM_ALLMASK_EVEX; - rbmMskCalleeTrash |= RBM_MSK_CALLEE_TRASH_EVEX; - cntCalleeTrashMask += CNT_CALLEE_TRASH_MASK_EVEX; - } - // Make sure we copy the register info and initialize the // trash regs after the underlying fields are initialized From cba4510792e141fed9decf4a82bb4ad0361977fb Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 00:04:05 -0700 Subject: [PATCH 186/201] Reapply "Remove rsAllCalleeSavedMask from RegSet" This reverts commit e27691c7edcad204a56bf96f746ef0ac319e8404. --- src/coreclr/jit/regset.cpp | 17 +++++++++++++++-- src/coreclr/jit/regset.h | 7 +------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index f4aba2c3db13ba..919c352eaf8810 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -150,7 +150,6 @@ void RegSet::rsClearRegsModified() // so don't treat it as callee-save. if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) { - rsAllCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); rsIntCalleeSavedMask &= ~RBM_SWIFT_ERROR; } #endif // SWIFT_SUPPORT @@ -188,6 +187,21 @@ void RegSet::printModifiedRegsMask(regMaskOnlyOne currentMask, } #endif +AllRegsMask RegSet::rsGetModifiedCalleeSavedRegsMask() const +{ + assert(rsModifiedRegsMaskInitialized); + AllRegsMask allCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; +#ifdef SWIFT_SUPPORT + // If this method has a SwiftError* parameter, we will return SwiftError::Value in REG_SWIFT_ERROR, + // so don't treat it as callee-save. + if (m_rsCompiler->lvaSwiftErrorArg != BAD_VAR_NUM) + { + allCalleeSavedMask.RemoveRegNum(REG_SWIFT_ERROR, TYP_INT); + } +#endif // SWIFT_SUPPORT + return (rsModifiedRegsMask & allCalleeSavedMask); +} + void RegSet::rsSetGprRegsModified(regMaskGpr mask DEBUGARG(bool suppressDump)) { assert(m_rsCompiler->IsGprRegMask(mask)); @@ -381,7 +395,6 @@ RegSet::RegSet(Compiler* compiler, GCInfo& gcInfo) #endif rsIntCalleeSavedMask = RBM_INT_CALLEE_SAVED; - rsAllCalleeSavedMask = m_rsCompiler->AllRegsMask_CALLEE_SAVED; #ifdef DEBUG rsModifiedRegsMaskInitialized = false; diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 61df5db7bbf8f0..6c6c1ac02589e4 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -77,7 +77,6 @@ class RegSet DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG - AllRegsMask rsAllCalleeSavedMask; regMaskGpr rsIntCalleeSavedMask; public: @@ -90,11 +89,7 @@ class RegSet return rsModifiedRegsMask; } - AllRegsMask rsGetModifiedCalleeSavedRegsMask() const - { - assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & rsAllCalleeSavedMask); - } + AllRegsMask rsGetModifiedCalleeSavedRegsMask() const; regMaskGpr rsGetModifiedIntCalleeSavedRegsMask() const { From fc6b76f93a5046baf1b0cec023471fe3775a1ccd Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 00:12:27 -0700 Subject: [PATCH 187/201] Move codeGen->CopyRegisterInfo() in compInitAllRegsMask() --- src/coreclr/jit/compiler.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 650177b049fa42..823e54d8dedc09 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2200,7 +2200,11 @@ void Compiler::compInitAllRegsMask() AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH & ~AllRegsMask_PROFILER_RET_SCRATCH; #else AllRegsMask_PROF_FNC_LEAVE = AllRegsMask_PROFILER_LEAVE_TRASH; -#endif +#endif // TARGET_ARM + +#ifdef TARGET_XARCH + codeGen->CopyRegisterInfo(); +#endif // TARGET_XARCH } /***************************************************************************** * @@ -3650,8 +3654,6 @@ void Compiler::compInitOptions(JitFlags* jitFlags) #undef DEF_TP }; memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskOnlyOne) * TYP_COUNT); - - codeGen->CopyRegisterInfo(); #endif // TARGET_XARCH } From 3df5a5d95e3de792842ea7541f016809dbc52b10 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 00:30:07 -0700 Subject: [PATCH 188/201] Copy calleeTrashRegs info as well --- src/coreclr/jit/compiler.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 823e54d8dedc09..c69a97c8d194fc 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2203,7 +2203,21 @@ void Compiler::compInitAllRegsMask() #endif // TARGET_ARM #ifdef TARGET_XARCH - codeGen->CopyRegisterInfo(); + + // Make sure we copy the register info and initialize the + // trash regs after the underlying fields are initialized + + const regMaskTP vtCalleeTrashRegs[TYP_COUNT]{ +#define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, +#include "typelist.h" +#undef DEF_TP + }; + memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskTP) * TYP_COUNT); + + if (codeGen != nullptr) + { + codeGen->CopyRegisterInfo(); + } #endif // TARGET_XARCH } /***************************************************************************** @@ -3643,18 +3657,6 @@ void Compiler::compInitOptions(JitFlags* jitFlags) opts.compJitSaveFpLrWithCalleeSavedRegisters = JitConfig.JitSaveFpLrWithCalleeSavedRegisters(); } #endif // defined(DEBUG) && defined(TARGET_ARM64) - -#if defined(TARGET_XARCH) - // Make sure we copy the register info and initialize the - // trash regs after the underlying fields are initialized - - const regMaskOnlyOne vtCalleeTrashRegs[TYP_COUNT]{ -#define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, -#include "typelist.h" -#undef DEF_TP - }; - memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskOnlyOne) * TYP_COUNT); -#endif // TARGET_XARCH } #ifdef DEBUG From c591106580cb00bdca90e742940041ce494761b8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 00:51:06 -0700 Subject: [PATCH 189/201] Make emitGetGCRegsKilledByNoGCCall() return CONSTREF_ --- src/coreclr/jit/emit.cpp | 45 ++++++++++++++++++++-------------------- src/coreclr/jit/emit.h | 2 +- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 4e47fc8969e6dc..7ee4051fbd8aec 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -10405,55 +10405,54 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) // Return Value: // Mask of GC register kills // -AllRegsMask emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) +CONSTREF_AllRegsMask emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); - AllRegsMask result; + +#ifdef DEBUG + // compHelperCallKillSet returns a superset of the registers which values are not guaranteed to be the same + // after the call, if a register loses its GC or byref it has to be in the compHelperCallKillSet set as well. + AllRegsMask expectedKillSet = emitComp->compHelperCallKillSet(helper); +#endif // DEBUG + +#define CHECK_AND_RETURN(killSet) \ + assert((expectedKillSet & killSet) == killSet); \ + return killSet; + switch (helper) { case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: - result = emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER); case CORINFO_HELP_ASSIGN_BYREF: - result = emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF); #if !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) case CORINFO_HELP_PROF_FCN_ENTER: - result = emitComp->AllRegsMask_PROFILER_ENTER_TRASH; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_PROFILER_ENTER_TRASH); case CORINFO_HELP_PROF_FCN_LEAVE: - result = emitComp->AllRegsMask_PROF_FNC_LEAVE; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_PROF_FNC_LEAVE); case CORINFO_HELP_PROF_FCN_TAILCALL: - result = emitComp->AllRegsMask_PROFILER_TAILCALL_TRASH; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_PROFILER_TAILCALL_TRASH); + #endif // !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) #if defined(TARGET_X86) case CORINFO_HELP_INIT_PINVOKE_FRAME: - result = emitComp->AllRegsMask_INIT_PINVOKE_FRAME_TRASH; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_INIT_PINVOKE_FRAME_TRASH); #endif // defined(TARGET_X86) case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - result = emitComp->AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH); default: - result = emitComp->AllRegsMask_CALLEE_TRASH_NOGC; - break; + CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_TRASH_NOGC); } - // compHelperCallKillSet returns a superset of the registers which values are not guaranteed to be the same - // after the call, if a register loses its GC or byref it has to be in the compHelperCallKillSet set as well. - assert((result & emitComp->compHelperCallKillSet(helper)) == result); - - return result; + return emitComp->AllRegsMask_NONE; } #if !defined(JIT32_GCENCODER) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 87493a830dd5a1..7406fe311d6f7e 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3143,7 +3143,7 @@ class emitter AllRegsMask emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); // Gets a register mask that represent the kill set for a NoGC helper call. - AllRegsMask emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); + CONSTREF_AllRegsMask emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); #if EMIT_TRACK_STACK_DEPTH unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere From 72f27e8b5a4f2419e4bd53d6f89b47ff4f34a286 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 10:27:24 -0700 Subject: [PATCH 190/201] fix some more CONSTREF --- src/coreclr/jit/compiler.hpp | 12 ++++++------ src/coreclr/jit/lsrabuild.cpp | 8 ++++---- src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/regset.h | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 05e2874abb0497..5a6e7f75a41a03 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -5038,7 +5038,7 @@ FORCEINLINE int regIndexForType(T vt) return (type - 1); } -void AllRegsMask::operator|=(const AllRegsMask& other) +void AllRegsMask::operator|=(CONSTREF_AllRegsMask other) { _combinedRegisters |= other._combinedRegisters; #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5046,7 +5046,7 @@ void AllRegsMask::operator|=(const AllRegsMask& other) #endif } -void AllRegsMask::operator&=(const AllRegsMask& other) +void AllRegsMask::operator&=(CONSTREF_AllRegsMask other) { _combinedRegisters &= other._combinedRegisters; #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5086,7 +5086,7 @@ AllRegsMask AllRegsMask::operator~() const return result; } -bool AllRegsMask::operator==(const AllRegsMask& other) const +bool AllRegsMask::operator==(CONSTREF_AllRegsMask other) const { return (_combinedRegisters == other._combinedRegisters) #ifdef HAS_MORE_THAN_64_REGISTERS @@ -5095,12 +5095,12 @@ bool AllRegsMask::operator==(const AllRegsMask& other) const ; } -bool AllRegsMask::operator!=(const AllRegsMask& other) const +bool AllRegsMask::operator!=(CONSTREF_AllRegsMask other) const { return !(*this == other); } -AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const +AllRegsMask AllRegsMask::operator&(CONSTREF_AllRegsMask other) const { AllRegsMask result; result._combinedRegisters = _combinedRegisters & other._combinedRegisters; @@ -5110,7 +5110,7 @@ AllRegsMask AllRegsMask::operator&(const AllRegsMask& other) const return result; } -AllRegsMask AllRegsMask::operator|(const AllRegsMask& other) const +AllRegsMask AllRegsMask::operator|(CONSTREF_AllRegsMask other) const { AllRegsMask result; result._combinedRegisters = _combinedRegisters | other._combinedRegisters; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 76ac303729332b..8f55e68dc95d5f 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1039,7 +1039,7 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) case GT_LSH_HI: case GT_RSH_LO: #endif - killMask.AddGprRegMask(getKillSetForShiftRotate(tree->AsOp())); + killMask = GprRegsMask(getKillSetForShiftRotate(tree->AsOp())); break; case GT_MUL: @@ -1047,14 +1047,14 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) #if !defined(TARGET_64BIT) || defined(TARGET_ARM64) case GT_MUL_LONG: #endif - killMask.AddGprRegMask(getKillSetForMul(tree->AsOp())); + killMask = GprRegsMask(getKillSetForMul(tree->AsOp())); break; case GT_MOD: case GT_DIV: case GT_UMOD: case GT_UDIV: - killMask.AddGprRegMask(getKillSetForModDiv(tree->AsOp())); + killMask = GprRegsMask(getKillSetForModDiv(tree->AsOp())); break; case GT_STORE_BLK: @@ -1089,7 +1089,7 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: - killMask.AddGprRegMask(getKillSetForHWIntrinsic(tree->AsHWIntrinsic())); + killMask = GprRegsMask(getKillSetForHWIntrinsic(tree->AsHWIntrinsic())); break; #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index 919c352eaf8810..e69374561f1bbb 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -316,7 +316,7 @@ void RegSet::ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debu _rsAllMaskVars = m_rsCompiler->AllRegsMask_NONE; } -void RegSet::SetMaskVars(AllRegsMask newMaskVars) +void RegSet::SetMaskVars(CONSTREF_AllRegsMask newMaskVars) { #ifdef DEBUG if (m_rsCompiler->verbose) diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 6c6c1ac02589e4..e6b006291ec769 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -83,7 +83,7 @@ class RegSet void rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump = false)); void rsSetRegModified(regNumber reg DEBUGARG(bool suppressDump = false)); - const AllRegsMask& rsGetModifiedRegsMask() const + CONSTREF_AllRegsMask rsGetModifiedRegsMask() const { assert(rsModifiedRegsMaskInitialized); return rsModifiedRegsMask; @@ -135,7 +135,7 @@ class RegSet // return (rsModifiedGprRegsMask & mask) != 0; } - bool rsRegsModified(const AllRegsMask& mask) const + bool rsRegsModified(CONSTREF_AllRegsMask mask) const { assert(rsModifiedRegsMaskInitialized); return !((rsModifiedRegsMask & mask).IsEmpty()); @@ -157,7 +157,7 @@ class RegSet return _rsAllMaskVars.gprRegs(); } - void SetMaskVars(AllRegsMask newMaskVars); // 'put' property function for rsMaskVars property + void SetMaskVars(CONSTREF_AllRegsMask newMaskVars); // 'put' property function for rsMaskVars property void AddMaskVars(var_types type, regMaskOnlyOne addMaskVars) // union 'addMaskVars' with the rsMaskVars set { From e1823d40c01cecf81dd174f5e90976bec3889b70 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 13:55:14 -0700 Subject: [PATCH 191/201] Consolidate updateDeadCandidates and inActivateRegisters --- src/coreclr/jit/lsra.cpp | 60 ++++++++-------------------------------- 1 file changed, 12 insertions(+), 48 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index aeb904b02b7a84..7452b70ac30b64 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4358,48 +4358,18 @@ void LinearScan::resetAllRegistersState() } } -#ifdef HAS_MORE_THAN_64_REGISTERS +#ifdef HAS_MORE_THAN_64_REGISTERS void LinearScan::updateDeadCandidatesAtBlockStart(REF_AllRegsMask deadRegMask, VarToRegMap inVarToRegMap) -{ - while (!deadRegMask.IsEmpty()) - { - regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); - RegRecord* physRegRecord = getRegisterRecord(reg); - - makeRegAvailable(reg, physRegRecord->registerType); - Interval* assignedInterval = physRegRecord->assignedInterval; - - if (assignedInterval != nullptr) - { - assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); - - if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) - { - assignedInterval->isActive = false; - if (assignedInterval->getNextRefPosition() == nullptr) - { - unassignPhysReg(physRegRecord, nullptr); - } - if (!assignedInterval->IsUpperVector()) - { - inVarToRegMap[assignedInterval->getVarIndex(compiler)] = REG_STK; - } - } - else - { - // This interval may still be active, but was in another register in an - // intervening block. - clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); - } - } - } -} #else -// TODO: Merge this with MORE_THAN_64 version updating the != RBM_NONE with IsEmpty() check void LinearScan::updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarToRegMap inVarToRegMap) +#endif // HAS_MORE_THAN_64_REGISTERS { +#ifdef HAS_MORE_THAN_64_REGISTERS + while (!deadRegMask.IsEmpty()) +#else while (deadRegMask != RBM_NONE) +#endif { regNumber reg = genFirstRegNumFromMaskAndToggle(deadRegMask); RegRecord* physRegRecord = getRegisterRecord(reg); @@ -4432,7 +4402,6 @@ void LinearScan::updateDeadCandidatesAtBlockStart(RegBitSet64 deadRegMask, VarTo } } } -#endif // HAS_MORE_THAN_64_REGISTERS //------------------------------------------------------------------------ // processBlockStartLocations: Update var locations on entry to 'currentBlock' and clear constant @@ -4861,19 +4830,15 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) #ifdef HAS_MORE_THAN_64_REGISTERS void LinearScan::inActivateRegisters(REF_AllRegsMask inactiveMask) -{ - while (!inactiveMask.IsEmpty()) - { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(inactiveMask); - RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); - makeRegisterInactive(regRecord); - } -} #else void LinearScan::inActivateRegisters(RegBitSet64 inactiveMask) +#endif // HAS_MORE_THAN_64_REGISTERS { +#ifdef HAS_MORE_THAN_64_REGISTERS + while (!inactiveMask.IsEmpty()) +#else while (inactiveMask != RBM_NONE) +#endif // HAS_MORE_THAN_64_REGISTERS { regNumber nextReg = genFirstRegNumFromMaskAndToggle(inactiveMask); RegRecord* regRecord = getRegisterRecord(nextReg); @@ -4881,9 +4846,8 @@ void LinearScan::inActivateRegisters(RegBitSet64 inactiveMask) makeRegisterInactive(regRecord); } } -#endif -//------------------------------------------------------------------------ + //------------------------------------------------------------------------ // LinearScan::freeRegister: Make a register available for use // // Arguments: From 41d49afe1b8233d0867e4d916df027960ba618bf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 14:00:46 -0700 Subject: [PATCH 192/201] Replace AllRegsMask_NONE with .Clear() --- src/coreclr/jit/lsra.cpp | 46 +++++++++++++++++------------------ src/coreclr/jit/lsra.h | 2 +- src/coreclr/jit/lsrabuild.cpp | 6 ++--- src/coreclr/jit/lsraxarch.cpp | 2 +- src/coreclr/jit/regset.cpp | 2 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 7452b70ac30b64..b8b28c0825671d 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5004,8 +5004,8 @@ void LinearScan::allocateRegistersMinimal() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = compiler->AllRegsMask_NONE; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsInUseThisLocation.Clear(); + regsInUseNextLocation.Clear(); // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5034,7 +5034,7 @@ void LinearScan::allocateRegistersMinimal() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = compiler->AllRegsMask_NONE; + delayRegsToMakeInactive.Clear(); } #ifdef DEBUG @@ -5091,9 +5091,9 @@ void LinearScan::allocateRegistersMinimal() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = compiler->AllRegsMask_NONE; + copyRegsToFree.Clear(); regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation.Clear(); if (!((regsToFree | delayRegsToFree).IsEmpty())) { freeRegisters(regsToFree); @@ -5104,11 +5104,11 @@ void LinearScan::allocateRegistersMinimal() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = compiler->AllRegsMask_NONE; - regsInUseThisLocation = compiler->AllRegsMask_NONE; + delayRegsToFree.Clear(); + regsInUseThisLocation.Clear(); } regsToFree = delayRegsToFree; - delayRegsToFree = compiler->AllRegsMask_NONE; + delayRegsToFree.Clear(); #ifdef DEBUG verifyFreeRegisters(regsToFree); @@ -5150,9 +5150,9 @@ void LinearScan::allocateRegistersMinimal() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = compiler->AllRegsMask_NONE; - regsInUseThisLocation = compiler->AllRegsMask_NONE; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsToFree.Clear(); + regsInUseThisLocation.Clear(); + regsInUseNextLocation.Clear(); handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -5694,8 +5694,8 @@ void LinearScan::allocateRegisters() AllRegsMask regsToMakeInactive; AllRegsMask delayRegsToMakeInactive; AllRegsMask copyRegsToFree; - regsInUseThisLocation = compiler->AllRegsMask_NONE; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsInUseThisLocation.Clear(); + regsInUseNextLocation.Clear(); // This is the most recent RefPosition for which a register was allocated // - currently only used for DEBUG but maintained in non-debug, for clarity of code @@ -5724,7 +5724,7 @@ void LinearScan::allocateRegisters() // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. regsToMakeInactive = delayRegsToMakeInactive; - delayRegsToMakeInactive = compiler->AllRegsMask_NONE; + delayRegsToMakeInactive.Clear(); } #ifdef DEBUG @@ -5779,9 +5779,9 @@ void LinearScan::allocateRegisters() { // CopyRegs are simply made available - we don't want to make the associated interval inactive. makeRegsAvailable(copyRegsToFree); - copyRegsToFree = compiler->AllRegsMask_NONE; + copyRegsToFree.Clear(); regsInUseThisLocation = regsInUseNextLocation; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsInUseNextLocation.Clear(); #ifdef TARGET_ARM64 if (hasConsecutiveRegister) { @@ -5798,11 +5798,11 @@ void LinearScan::allocateRegisters() assert(!"Found a delayRegFree associated with Location with no reference"); // However, to be cautious for the Release build case, we will free them. freeRegisters(delayRegsToFree); - delayRegsToFree = compiler->AllRegsMask_NONE; - regsInUseThisLocation = compiler->AllRegsMask_NONE; + delayRegsToFree.Clear(); + regsInUseThisLocation.Clear(); } regsToFree = delayRegsToFree; - delayRegsToFree = compiler->AllRegsMask_NONE; + delayRegsToFree.Clear(); #ifdef DEBUG verifyFreeRegisters(regsToFree); #endif @@ -5861,9 +5861,9 @@ void LinearScan::allocateRegisters() { // Free any delayed regs (now in regsToFree) before processing the block boundary freeRegisters(regsToFree); - regsToFree = compiler->AllRegsMask_NONE; - regsInUseThisLocation = compiler->AllRegsMask_NONE; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsToFree.Clear(); + regsInUseThisLocation.Clear(); + regsInUseNextLocation.Clear(); handledBlockEnd = true; curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) @@ -11463,7 +11463,7 @@ void LinearScan::dumpRegRecordHeader() regColumnWidth + 1); // Print a "title row" including the legend and the reg names. - lastDumpedRegisters = compiler->AllRegsMask_NONE; + lastDumpedRegisters.Clear(); dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 57165170a3a5db..b8041431629266 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1961,7 +1961,7 @@ class LinearScan : public LinearScanInterface void resetRegState() { resetAvailableRegs(); - regsBusyUntilKill = compiler->AllRegsMask_NONE; + regsBusyUntilKill.Clear(); } bool conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 8f55e68dc95d5f..13203dc99c20e6 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2305,8 +2305,8 @@ void LinearScan::buildIntervals() RegState* floatRegState = &compiler->codeGen->floatRegState; intRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; floatRegState->rsCalleeRegArgMaskLiveIn = RBM_NONE; - regsInUseThisLocation = compiler->AllRegsMask_NONE; - regsInUseNextLocation = compiler->AllRegsMask_NONE; + regsInUseThisLocation.Clear(); + regsInUseNextLocation.Clear(); #ifdef SWIFT_SUPPORT if (compiler->info.compCallConv == CorInfoCallConvExtension::Swift) @@ -2445,7 +2445,7 @@ void LinearScan::buildIntervals() #endif numPlacedArgLocals = 0; - placedArgRegs = compiler->AllRegsMask_NONE; + placedArgRegs.Clear(); BasicBlock* predBlock = nullptr; BasicBlock* prevBlock = nullptr; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 45c8ba6cfa08dc..a5b42b58e8658e 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1411,7 +1411,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = compiler->AllRegsMask_NONE; + placedArgRegs.Clear(); numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index e69374561f1bbb..df614451201b82 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -313,7 +313,7 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) void RegSet::ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. { - _rsAllMaskVars = m_rsCompiler->AllRegsMask_NONE; + _rsAllMaskVars.Clear(); } void RegSet::SetMaskVars(CONSTREF_AllRegsMask newMaskVars) From 8e448c5f562b1b644cb018a8c5dda4a2815a16bc Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 14:18:25 -0700 Subject: [PATCH 193/201] Add AllRegsMask in clrjit.natvis: --- src/coreclr/jit/clrjit.natvis | 45 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/clrjit.natvis b/src/coreclr/jit/clrjit.natvis index 3408fd822b185e..1b751541aec7e7 100644 --- a/src/coreclr/jit/clrjit.natvis +++ b/src/coreclr/jit/clrjit.natvis @@ -137,19 +137,9 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- block = block->Next() - this->m_AvailableRegs[0] + this->m_AvailableRegs - - - - ((regNumber)regIndex),en - regIndex++ - reg = reg >> 1 - - - this->m_AvailableRegs[1] - - + ((regNumber)regIndex),en @@ -157,19 +147,9 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- reg = reg >> 1 - this->m_RegistersWithConstants[0] - - - - - ((regNumber)regIndex),en - regIndex++ - reg = reg >> 1 - - - this->m_RegistersWithConstants[1] + this->m_RegistersWithConstants - + ((regNumber)regIndex),en @@ -178,7 +158,7 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- - + [#{rpNum,d} - {refType,en}] @@ -197,6 +177,21 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- + + + + + + + + ((regNumber)regIndex),en + regIndex++ + reg = reg >> 1 + + + + + [U{this->relatedInterval->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}] [V{this->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}] From e5eb8e4fc8a3c21a393f7062a723b0deae81ec68 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 14:21:16 -0700 Subject: [PATCH 194/201] Remove the commented out #define --- src/coreclr/jit/target.h | 9 -------- src/coreclr/jit/targetamd64.h | 39 ----------------------------------- src/coreclr/jit/targetarm.h | 28 ------------------------- src/coreclr/jit/targetarm64.h | 30 --------------------------- src/coreclr/jit/targetx86.h | 36 -------------------------------- 5 files changed, 142 deletions(-) diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 1f2f9d2bb1ad33..de3d5e70fd521d 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -461,15 +461,6 @@ typedef unsigned __int64 regMaskSmall; #error Unsupported or unset target architecture #endif -//#define AllRegsMask_NONE RBM_NONE -// #ifdef HAS_MORE_THAN_64_REGISTERS -// #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED) -// #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH) -// #else -// #define AllRegsMask_CALLEE_SAVED AllRegsMask(RBM_CALLEE_SAVED) -// #define AllRegsMask_CALLEE_TRASH AllRegsMask(RBM_CALLEE_TRASH) -// #endif - #ifdef TARGET_XARCH #define JMP_DIST_SMALL_MAX_NEG (-128) diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index d6d8ee73a83d27..d805c9dd911185 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -200,26 +200,12 @@ #define REG_WRITE_BARRIER_SRC REG_ARG_1 #define RBM_WRITE_BARRIER_SRC RBM_ARG_1 - // #define AllRegsMask_CALLEE_TRASH_NOGC AllRegsMask_CALLEE_TRASH - - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC - - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_RAX | RBM_RCX) - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) - - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) -// Registers killed by CORINFO_HELP_ASSIGN_BYREF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF) - // We have two register classifications // * callee trash: aka volatile or caller saved // * callee saved: aka non-volatile @@ -519,32 +505,7 @@ #define RBM_FLTARG_REGS (RBM_FLTARG_0|RBM_FLTARG_1|RBM_FLTARG_2|RBM_FLTARG_3) #endif // !UNIX_AMD64_ABI - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper. -#ifdef UNIX_AMD64_ABI - // See vm\amd64\unixasmhelpers.S for more details. - // - // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. - // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. - // STOP_FOR_GC helper preserves all the 4 possible return registers. - - // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH) - // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) - -#else - // See vm\amd64\asmhelpers.asm for more details. - // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_INTRET), (RBM_FLT_CALLEE_TRASH & ~RBM_FLOATRET), RBM_MSK_CALLEE_TRASH) - // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_CALLEE_TRASH - -#endif // UNIX_AMD64_ABI - - // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_STOP_FOR_GC_TRASH - // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_LEAVE_TRASH - - // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH - #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX)) - // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_RAX diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index edce679e2a03ad..56f1a8bec91563 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -167,20 +167,6 @@ #define RBM_WRITE_BARRIER_SRC_BYREF RBM_ARG_1 #define _RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) - // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) - - // // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_WRITEBARRIER - - // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R4 @@ -221,16 +207,10 @@ #define REG_PROFILER_ENTER_ARG REG_R0 #define RBM_PROFILER_ENTER_ARG RBM_R0 #define REG_PROFILER_RET_SCRATCH REG_R2 - // #define AllRegsMask_PROFILER_RET_SCRATCH GprRegsMask(RBM_R2) // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. #define RBM_PROFILER_ENTER_TRASH RBM_NONE - // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask_NONE - // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must - // consider it killed by the return. - // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_RET_SCRATCH - // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_NONE // Which register are int and long values returned in ? #define REG_INTRET REG_R0 @@ -245,14 +225,6 @@ #define RBM_FLOATRET RBM_F0 #define RBM_DOUBLERET (RBM_F0|RBM_F1) - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper (JIT_RareDisableHelper). - // See vm\arm\amshelpers.asm for more details. -// #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_LNGRET|RBM_R7|RBM_R8|RBM_R11)), (RBM_FLT_CALLEE_TRASH & ~(RBM_DOUBLERET|RBM_F2|RBM_F3|RBM_F4|RBM_F5|RBM_F6|RBM_F7))) - - // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH (AllRegsMask_CALLEE_TRASH | GprRegsMask(RBM_PINVOKE_TCB | RBM_PINVOKE_SCRATCH)) - - // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R0 #define REG_FPBASE REG_R11 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 60da68e3e558de..eaedc94ff38180 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -192,20 +192,6 @@ #define RBM_WRITE_BARRIER_SRC_BYREF RBM_R13 #define _RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) - // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(_RBM_CALLEE_TRASH_NOGC) - - // // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC) - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH_NOGC - - // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC) - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF AllRegsMask_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R15 @@ -252,15 +238,6 @@ #define REG_PROFILER_LEAVE_ARG_CALLER_SP REG_R11 #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 - // The registers trashed by profiler enter/leave/tailcall hook -// #ifdef HAS_MORE_THAN_64_REGISTERS -// #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH) -// #else -// #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS) | RBM_MSK_CALLEE_TRASH) -// #endif - // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask_PROFILER_ENTER_TRASH - // #define AllRegsMask_PROFILER_TAILCALL_TRASH AllRegsMask_PROFILER_ENTER_TRASH - // Which register are int and long values returned in ? #define REG_INTRET REG_R0 #define RBM_INTRET RBM_R0 @@ -273,14 +250,7 @@ #define RBM_FLOATRET RBM_V0 #define RBM_DOUBLERET RBM_V0 - // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH - - // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. - // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH AllRegsMask_CALLEE_TRASH - #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15)) - // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_VALIDATE_INDIRECT_CALL_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15 #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index 40fb5cb1889b6f..c74aa61869eed0 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -225,28 +225,6 @@ #define RBM_OPTIMIZED_WRITE_BARRIER_SRC (RBM_EAX|RBM_ECX|RBM_EBX|RBM_ESI|RBM_EDI) #endif // NOGC_WRITE_BARRIERS - // #define AllRegsMask_CALLEE_TRASH_NOGC GprRegsMask(RBM_EDX) - - // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // Note that x86 normally emits an optimized (source-register-specific) write barrier, but can emit - // a call to a "general" write barrier. - -// #ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS -// #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER GprRegsMask(RBM_EAX | RBM_EDX) -// #else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS -// #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER AllRegsMask_CALLEE_TRASH -// #endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER GprRegsMask(RBM_EDX) - - // // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - // #define AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ESI | RBM_EDI | RBM_ECX) - - // // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - // // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. - // #define AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF GprRegsMask(RBM_ECX) - // GenericPInvokeCalliHelper unmanaged target parameter #define REG_PINVOKE_TARGET_PARAM REG_EAX #define RBM_PINVOKE_TARGET_PARAM RBM_EAX @@ -292,14 +270,6 @@ #define RBM_FLOATRET RBM_NONE #define RBM_DOUBLERET RBM_NONE - // // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper - // #define AllRegsMask_STOP_FOR_GC_TRASH AllRegsMask_CALLEE_TRASH - - // // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling - // // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. - // #define AllRegsMask_INIT_PINVOKE_FRAME_TRASH GprRegsMask(RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) - - // #define AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH GprRegsMask(RBM_INT_CALLEE_TRASH & ~RBM_ECX) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_ECX #define REG_FPBASE REG_EBP @@ -329,12 +299,6 @@ #define RBM_ARG_REGS (RBM_ARG_0|RBM_ARG_1) - // The registers trashed by profiler enter/leave/tailcall hook - // See vm\i386\asmhelpers.asm for more details. - // #define AllRegsMask_PROFILER_ENTER_TRASH AllRegsMask() - // #define AllRegsMask_PROFILER_LEAVE_TRASH AllRegsMask() - // #define AllRegsMask_PROFILER_TAILCALL_TRASH (AllRegsMask_CALLEE_TRASH & GprRegsMask(~RBM_ARG_REGS)) - // What sort of reloc do we use for [disp32] address mode #define IMAGE_REL_BASED_DISP32 IMAGE_REL_BASED_HIGHLOW From 7e522f15f41d97ad71e12215ce43ef04f669c9a1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 14:51:28 -0700 Subject: [PATCH 195/201] Make emitGetGCRegsKilledByNoGCCall() return RegbitSet64 instead of AllRegsMask --- src/coreclr/jit/compiler.cpp | 10 +++--- src/coreclr/jit/emit.cpp | 60 +++++++++++++++++++---------------- src/coreclr/jit/emit.h | 4 +-- src/coreclr/jit/morph.cpp | 9 ------ src/coreclr/jit/targetamd64.h | 32 +++++++++++++++++++ src/coreclr/jit/targetarm.h | 20 +++++++++++- src/coreclr/jit/targetarm64.h | 20 +++++++++++- src/coreclr/jit/targetx86.h | 28 ++++++++++++++++ 8 files changed, 138 insertions(+), 45 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index c69a97c8d194fc..7843c212437a6b 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2132,7 +2132,7 @@ void Compiler::compInitAllRegsMask() #elif defined(TARGET_ARM) - AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_CALLEE_TRASH_NOGC); AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. @@ -2144,7 +2144,7 @@ void Compiler::compInitAllRegsMask() // Registers killed by CORINFO_HELP_ASSIGN_BYREF. AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC); // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. @@ -2167,19 +2167,19 @@ void Compiler::compInitAllRegsMask() #elif defined(TARGET_ARM64) - AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(_RBM_CALLEE_TRASH_NOGC); + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_CALLEE_TRASH_NOGC); AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH); // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | _RBM_CALLEE_TRASH_NOGC); + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | RBM_CALLEE_TRASH_NOGC); // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; // Registers killed by CORINFO_HELP_ASSIGN_BYREF. AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | _RBM_CALLEE_TRASH_NOGC); + GprRegsMask(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC); // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 7ee4051fbd8aec..6e0e326fbde496 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -10359,7 +10359,7 @@ const char* emitter::emitOffsetToLabel(unsigned offs) // Return value: // the saved set of registers. // -AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) +regMaskGpr emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? bool isNoGCHelper = emitNoGChelper(methHnd); @@ -10368,7 +10368,7 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd); // Get the set of registers that this call kills and remove it from the saved set. - regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc).gprRegs(); + regMaskGpr savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc); #ifdef DEBUG if (emitComp->verbose) @@ -10379,12 +10379,12 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) printf("\n"); } #endif - return GprRegsMask(savedSet); + return savedSet; } else { // This is the saved set of registers after a normal call. - return emitComp->AllRegsMask_CALLEE_SAVED; + return RBM_INT_CALLEE_SAVED; } } @@ -10397,7 +10397,7 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) // address registers as killed for liveness purposes, since their values change. However, they still are // valid byref pointers after the call, so the dst/src address registers are NOT reported as killed here. // -// Note: This list may not be complete and defaults to the default AllRegsMask_CALLEE_TRASH_NOGC registers. +// Note: This list may not be complete and defaults to the default RBM_CALLEE_TRASH_NOGC registers. // // Arguments: // helper - The helper being inquired about @@ -10405,54 +10405,60 @@ AllRegsMask emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) // Return Value: // Mask of GC register kills // -CONSTREF_AllRegsMask emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) +RegBitSet64 emitter::emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper) { assert(emitNoGChelper(helper)); - -#ifdef DEBUG - // compHelperCallKillSet returns a superset of the registers which values are not guaranteed to be the same - // after the call, if a register loses its GC or byref it has to be in the compHelperCallKillSet set as well. - AllRegsMask expectedKillSet = emitComp->compHelperCallKillSet(helper); -#endif // DEBUG - -#define CHECK_AND_RETURN(killSet) \ - assert((expectedKillSet & killSet) == killSet); \ - return killSet; - + regMaskTP result; switch (helper) { case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: - CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER); + result = RBM_CALLEE_GCTRASH_WRITEBARRIER; + break; case CORINFO_HELP_ASSIGN_BYREF: - CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF); + result = RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF; + break; #if !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) case CORINFO_HELP_PROF_FCN_ENTER: - CHECK_AND_RETURN(emitComp->AllRegsMask_PROFILER_ENTER_TRASH); + result = RBM_PROFILER_ENTER_TRASH; + break; case CORINFO_HELP_PROF_FCN_LEAVE: - CHECK_AND_RETURN(emitComp->AllRegsMask_PROF_FNC_LEAVE); +#if defined(TARGET_ARM) + // profiler scratch remains gc live + result = RBM_PROFILER_LEAVE_TRASH & ~RBM_PROFILER_RET_SCRATCH; +#else + result = RBM_PROFILER_LEAVE_TRASH; +#endif + break; case CORINFO_HELP_PROF_FCN_TAILCALL: - CHECK_AND_RETURN(emitComp->AllRegsMask_PROFILER_TAILCALL_TRASH); - + result = RBM_PROFILER_TAILCALL_TRASH; + break; #endif // !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) #if defined(TARGET_X86) case CORINFO_HELP_INIT_PINVOKE_FRAME: - CHECK_AND_RETURN(emitComp->AllRegsMask_INIT_PINVOKE_FRAME_TRASH); + result = RBM_INIT_PINVOKE_FRAME_TRASH; + break; #endif // defined(TARGET_X86) case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - CHECK_AND_RETURN(emitComp->AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH); + result = RBM_VALIDATE_INDIRECT_CALL_TRASH; + break; default: - CHECK_AND_RETURN(emitComp->AllRegsMask_CALLEE_TRASH_NOGC); + result = RBM_CALLEE_TRASH_NOGC; + break; } - return emitComp->AllRegsMask_NONE; + // compHelperCallKillSet returns a superset of the registers which values are not guaranteed to be the same + // after the call, if a register loses its GC or byref it has to be in the compHelperCallKillSet set as well. + assert((result & emitComp->compHelperCallKillSet(helper).GetGprFloatCombinedMask()) == result); + + return result; } #if !defined(JIT32_GCENCODER) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 7406fe311d6f7e..d9a4b276118d50 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3140,10 +3140,10 @@ class emitter bool emitFullGCinfo; // full GC pointer maps? bool emitFullyInt; // fully interruptible code? - AllRegsMask emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); + regMaskGpr emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd); // Gets a register mask that represent the kill set for a NoGC helper call. - CONSTREF_AllRegsMask emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); + RegBitSet64 emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper); #if EMIT_TRACK_STACK_DEPTH unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 0c588043316ef5..b8be505076d655 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2699,11 +2699,6 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call } } - // if ((structIntRegs > 0) && (structFloatRegs > 0)) - //{ - // assert(!"Different numbers"); - //} - isRegArg = ((nextFltArgRegNum + structFloatRegs) <= MAX_FLOAT_REG_ARG) && ((intArgRegNum + structIntRegs) <= MAX_REG_ARG); } @@ -2815,10 +2810,6 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call ++structFloatRegs; } } - // if ((structIntRegs > 0) && (structFloatRegs > 0)) - //{ - // assert(!"Different numbers"); - //} } #endif // defined(UNIX_AMD64_ABI) else diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index d805c9dd911185..00c4b29e0a95ad 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -200,6 +200,14 @@ #define REG_WRITE_BARRIER_SRC REG_ARG_1 #define RBM_WRITE_BARRIER_SRC RBM_ARG_1 + #define RBM_CALLEE_TRASH_NOGC RBM_CALLEE_TRASH + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_TRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_RAX | RBM_RCX) @@ -505,6 +513,30 @@ #define RBM_FLTARG_REGS (RBM_FLTARG_0|RBM_FLTARG_1|RBM_FLTARG_2|RBM_FLTARG_3) #endif // !UNIX_AMD64_ABI + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper. +#ifdef UNIX_AMD64_ABI + // See vm\amd64\unixasmhelpers.S for more details. + // + // On Unix a struct of size >=9 and <=16 bytes in size is returned in two return registers. + // The return registers could be any two from the set { RAX, RDX, XMM0, XMM1 }. + // STOP_FOR_GC helper preserves all the 4 possible return registers. + #define RBM_STOP_FOR_GC_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) + + // The registers trashed by profiler enter/leave/tailcall hook + // See vm\amd64\asmhelpers.S for more details. + #define RBM_PROFILER_ENTER_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_FLTARG_REGS)) + #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET | RBM_FLOATRET_1 | RBM_INTRET_1)) + #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + +#else + // See vm\amd64\asmhelpers.asm for more details. + #define RBM_STOP_FOR_GC_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) + + #define RBM_PROFILER_ENTER_TRASH RBM_CALLEE_TRASH + #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_INTRET)) + #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH +#endif + #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX)) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_RAX diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 56f1a8bec91563..83f9380db1ef5c 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -166,7 +166,20 @@ #define REG_WRITE_BARRIER_SRC_BYREF REG_ARG_1 #define RBM_WRITE_BARRIER_SRC_BYREF RBM_ARG_1 - #define _RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + #define RBM_CALLEE_TRASH_NOGC (RBM_R2|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_R0|RBM_R3|RBM_LR|RBM_DEFAULT_HELPER_CALL_TARGET) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_WRITEBARRIER + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that r0 and r1 are still valid byref pointers after this helper call, despite their value being changed. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R4 @@ -207,10 +220,15 @@ #define REG_PROFILER_ENTER_ARG REG_R0 #define RBM_PROFILER_ENTER_ARG RBM_R0 #define REG_PROFILER_RET_SCRATCH REG_R2 + #define RBM_PROFILER_RET_SCRATCH RBM_R2 // The registers trashed by profiler enter/leave/tailcall hook // See vm\arm\asmhelpers.asm for more details. #define RBM_PROFILER_ENTER_TRASH RBM_NONE + // While REG_PROFILER_RET_SCRATCH is not trashed by the method, the register allocator must + // consider it killed by the return. + #define RBM_PROFILER_LEAVE_TRASH RBM_PROFILER_RET_SCRATCH + #define RBM_PROFILER_TAILCALL_TRASH RBM_NONE // Which register are int and long values returned in ? #define REG_INTRET REG_R0 diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index eaedc94ff38180..1cde57bc59ab8a 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -191,7 +191,20 @@ #define REG_WRITE_BARRIER_SRC_BYREF REG_R13 #define RBM_WRITE_BARRIER_SRC_BYREF RBM_R13 - #define _RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) + #define RBM_CALLEE_TRASH_NOGC (RBM_R12|RBM_R15|RBM_IP0|RBM_IP1|RBM_DEFAULT_HELPER_CALL_TARGET) + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_R14|RBM_CALLEE_TRASH_NOGC) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF | RBM_CALLEE_TRASH_NOGC) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that x13 and x14 are still valid byref pointers after this helper call, despite their value being changed. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_CALLEE_TRASH_NOGC // GenericPInvokeCalliHelper VASigCookie Parameter #define REG_PINVOKE_COOKIE_PARAM REG_R15 @@ -238,6 +251,11 @@ #define REG_PROFILER_LEAVE_ARG_CALLER_SP REG_R11 #define RBM_PROFILER_LEAVE_ARG_CALLER_SP RBM_R11 + // The registers trashed by profiler enter/leave/tailcall hook + #define RBM_PROFILER_ENTER_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FLTARG_REGS|RBM_FP)) + #define RBM_PROFILER_LEAVE_TRASH (RBM_CALLEE_TRASH & ~(RBM_ARG_REGS|RBM_ARG_RET_BUFF|RBM_FLTARG_REGS|RBM_FP)) + #define RBM_PROFILER_TAILCALL_TRASH RBM_PROFILER_LEAVE_TRASH + // Which register are int and long values returned in ? #define REG_INTRET REG_R0 #define RBM_INTRET RBM_R0 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index c74aa61869eed0..ebef74b1acd29c 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -225,6 +225,28 @@ #define RBM_OPTIMIZED_WRITE_BARRIER_SRC (RBM_EAX|RBM_ECX|RBM_EBX|RBM_ESI|RBM_EDI) #endif // NOGC_WRITE_BARRIERS + #define RBM_CALLEE_TRASH_NOGC RBM_EDX + + // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + // Note that x86 normally emits an optimized (source-register-specific) write barrier, but can emit + // a call to a "general" write barrier. + +#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS + #define RBM_CALLEE_TRASH_WRITEBARRIER (RBM_EAX | RBM_EDX) +#else // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + #define RBM_CALLEE_TRASH_WRITEBARRIER RBM_CALLEE_TRASH +#endif // !FEATURE_USE_ASM_GC_WRITE_BARRIERS + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_EDX + + // Registers killed by CORINFO_HELP_ASSIGN_BYREF. + #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_ESI | RBM_EDI | RBM_ECX) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + // Note that RDI and RSI are still valid byref pointers after this helper call, despite their value being changed. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF RBM_ECX + // GenericPInvokeCalliHelper unmanaged target parameter #define REG_PINVOKE_TARGET_PARAM REG_EAX #define RBM_PINVOKE_TARGET_PARAM RBM_EAX @@ -299,6 +321,12 @@ #define RBM_ARG_REGS (RBM_ARG_0|RBM_ARG_1) + // The registers trashed by profiler enter/leave/tailcall hook + // See vm\i386\asmhelpers.asm for more details. + #define RBM_PROFILER_ENTER_TRASH RBM_NONE + #define RBM_PROFILER_LEAVE_TRASH RBM_NONE + #define RBM_PROFILER_TAILCALL_TRASH (RBM_CALLEE_TRASH & ~RBM_ARG_REGS) + // What sort of reloc do we use for [disp32] address mode #define IMAGE_REL_BASED_DISP32 IMAGE_REL_BASED_HIGHLOW From 3eb1237ccef3159b6c7cb5bf75c991deb98a8f25 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 15:10:43 -0700 Subject: [PATCH 196/201] emitGetGCRegsSaveOrModified() returns RegBitSet64 --- src/coreclr/jit/emitarm64.cpp | 6 +++--- src/coreclr/jit/emitxarch.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 5b8f7edc8ee01d..b327bccab2972c 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -8968,9 +8968,9 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)std::abs(argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet.gprRegs(); - byrefRegs &= savedSet.gprRegs(); + regMaskGpr savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet; + byrefRegs &= savedSet; #ifdef DEBUG if (EMIT_GC_VERBOSE) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 0518248a6332a3..321b2ffb9dd1bb 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9518,9 +9518,9 @@ void emitter::emitIns_Call(EmitCallType callType, assert((unsigned)abs((signed)argSize) <= codeGen->genStackLevel); // Trim out any callee-trashed registers from the live set. - AllRegsMask savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet.gprRegs(); - byrefRegs &= savedSet.gprRegs(); + regMaskGpr savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet; + byrefRegs &= savedSet; #ifdef DEBUG if (EMIT_GC_VERBOSE) From 335b2b589a3178419153ee756b52266ca0e5d8d3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 15:11:15 -0700 Subject: [PATCH 197/201] add back some RBM_* in target** --- src/coreclr/jit/targetarm.h | 1 + src/coreclr/jit/targetx86.h | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 83f9380db1ef5c..f9d486a5e7f6c5 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -243,6 +243,7 @@ #define RBM_FLOATRET RBM_F0 #define RBM_DOUBLERET (RBM_F0|RBM_F1) + #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R0 #define REG_FPBASE REG_R11 diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h index ebef74b1acd29c..8a19781ed67e4a 100644 --- a/src/coreclr/jit/targetx86.h +++ b/src/coreclr/jit/targetx86.h @@ -292,6 +292,14 @@ #define RBM_FLOATRET RBM_NONE #define RBM_DOUBLERET RBM_NONE + // The registers trashed by the CORINFO_HELP_STOP_FOR_GC helper + #define RBM_STOP_FOR_GC_TRASH RBM_CALLEE_TRASH + + // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. On x86, this helper has a custom calling + // convention that takes EDI as argument (but doesn't trash it), trashes EAX, and returns ESI. + #define RBM_INIT_PINVOKE_FRAME_TRASH (RBM_PINVOKE_SCRATCH | RBM_PINVOKE_TCB) + + #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~RBM_ECX) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_ECX #define REG_FPBASE REG_EBP From c60f3b112a55d4cc713e91b1910dcae0224754df Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 16:15:43 -0700 Subject: [PATCH 198/201] misc. cleanup --- src/coreclr/jit/lsra.h | 6 ------ src/coreclr/jit/lsraarm.cpp | 5 ++--- src/coreclr/jit/lsraarm64.cpp | 3 +-- src/coreclr/jit/lsraarmarch.cpp | 7 +++---- src/coreclr/jit/lsrabuild.cpp | 6 ------ src/coreclr/jit/lsraxarch.cpp | 6 ++---- src/coreclr/jit/regset.cpp | 2 +- src/coreclr/jit/regset.h | 3 +-- src/coreclr/jit/target.h | 28 ++-------------------------- src/coreclr/jit/targetamd64.h | 2 +- 10 files changed, 13 insertions(+), 55 deletions(-) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index b8041431629266..09b33cd873002a 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1860,11 +1860,6 @@ class LinearScan : public LinearScanInterface void makeRegsAvailable(CONSTREF_AllRegsMask regMask) { m_AvailableRegs |= regMask; -#ifdef HAS_MORE_THAN_64_REGISTERS - assert(compiler->IsGprRegMask(m_AvailableRegs.gprRegs())); - assert(compiler->IsFloatRegMask(m_AvailableRegs.floatRegs(compiler))); - assert(compiler->IsPredicateRegMask(m_AvailableRegs.predicateRegs(compiler))); -#endif } void makeRegAvailable(regNumber reg, var_types regType) @@ -2230,7 +2225,6 @@ class Interval : public Referenceable , assignedReg(nullptr) , varNum(0) , physReg(REG_COUNT) - //, registerType(registerType) , isActive(false) , isLocalVar(false) , isSplit(false) diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 362b9df1ec0c97..bfe382d38d82e4 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -371,8 +371,7 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 1; assert(dstCount == 0); BuildUse(tree->gtGetOp1()); - CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildKills(tree, killMask); + BuildKills(tree, compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC)); break; } @@ -692,7 +691,7 @@ int LinearScan::BuildNode(GenTree* tree) { srcCount = 0; } - BuildDefs(tree, dstCount, argMask); // This is regMaskGpr + BuildDefs(tree, dstCount, argMask); } break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index d99e0605ffd0ee..ffda059d3c09d0 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -831,8 +831,7 @@ int LinearScan::BuildNode(GenTree* tree) BuildUse(tree->gtGetOp1()); srcCount = 1; assert(dstCount == 0); - CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildKills(tree, killMask); + BuildKills(tree, compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC)); break; } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 07fa7af7ba8b0c..249d24d424722e 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -434,7 +434,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #endif // SWIFT_SUPPORT // No args are placed in registers anymore. - placedArgRegs = compiler->AllRegsMask_NONE; + placedArgRegs.Clear(); numPlacedArgLocals = 0; return srcCount; } @@ -649,7 +649,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) } } buildInternalRegisterUses(); - BuildDefs(argNode, dstCount, argMask); // this should only be regMaskGpr + BuildDefs(argNode, dstCount, argMask); return srcCount; } @@ -892,8 +892,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) assert(compiler->IsGprRegMask(sizeRegMask)); buildInternalRegisterUses(); - AllRegsMask killMask = getKillSetForBlockStore(blkNode); - BuildKills(blkNode, killMask); + BuildKills(blkNode, getKillSetForBlockStore(blkNode)); return useCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 13203dc99c20e6..585badc93eea8e 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1126,12 +1126,6 @@ AllRegsMask LinearScan::getKillSetForNode(GenTree* tree) bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc, CONSTREF_AllRegsMask killMask) { - assert(compiler->IsGprRegMask(killMask.gprRegs())); - assert(compiler->IsFloatRegMask(killMask.floatRegs(compiler))); -#ifdef FEATURE_MASKED_HW_INTRINSICS - assert(compiler->IsPredicateRegMask(killMask.predicateRegs(compiler))); -#endif // FEATURE_MASKED_HW_INTRINSICS - bool insertedKills = false; if (!killMask.IsEmpty()) diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index a5b42b58e8658e..ef6df2b83dd254 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -296,8 +296,7 @@ int LinearScan::BuildNode(GenTree* tree) RefPosition* internalDef = buildInternalIntRegisterDefForNode(tree); srcCount = BuildOperandUses(tree->gtGetOp1()); buildInternalRegisterUses(); - CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC); - BuildKills(tree, killMask); + BuildKills(tree, compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC)); } break; @@ -1677,8 +1676,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) #endif buildInternalRegisterUses(); - AllRegsMask killMask = getKillSetForBlockStore(blkNode); - BuildKills(blkNode, killMask); + BuildKills(blkNode, getKillSetForBlockStore(blkNode)); return useCount; } diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index df614451201b82..09b892377ae61f 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -311,7 +311,7 @@ void RegSet::rsRemoveRegsModified(regMaskGpr mask) rsModifiedRegsMask.RemoveRegTypeFromMask(mask, TYP_INT); } -void RegSet::ClearMaskVars() // Like SetMaskVars(RBM_NONE), but without any debug output. +void RegSet::ClearMaskVars() // Like SetMaskVars(AllRegsMask), but without any debug output. { _rsAllMaskVars.Clear(); } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index e6b006291ec769..1aaed64acd50ce 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -132,7 +132,6 @@ class RegSet { assert(rsModifiedRegsMaskInitialized); return (rsModifiedRegsMask.gprRegs() & mask) != 0; - // return (rsModifiedGprRegsMask & mask) != 0; } bool rsRegsModified(CONSTREF_AllRegsMask mask) const @@ -179,7 +178,7 @@ class RegSet void ClearMaskVars(); // Like SetMaskVars(RBM_NONE), but without any debug output. private: - AllRegsMask _rsAllMaskVars; // backing store for rsGprMaskVars property + AllRegsMask _rsAllMaskVars; // backing store for rsMaskVars property #if defined(TARGET_ARMARCH) regMaskGpr rsGprMaskCalleeSaved; diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index de3d5e70fd521d..6f63ba3252ec75 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -224,31 +224,9 @@ typedef unsigned __int64 regMaskFloat; typedef unsigned __int64 regMaskPredicate; typedef unsigned __int64 RegBitSet64; -// Design: -// 1. Reduce regMaskGpr to 32-bit -// 2. singleRegMask > regMaskOnlyOne > (regMaskGpr, regMaskFloat, regMaskPredicate) > regMaskMixed -// 3. Revisit `regMaskMixed` and see how we can separate them into `regMaskGpr`, `regMaskFloat`, `regMaskPredicate`. -// 4. Once #3 is solved, `regMaskOnlyOne` should be reduced to 32-bit and so can `regMaskFloat` and `regMaskPredicate` -// - -// -// We will add a "//TODO: regMaskOnlyOne" through out the code -// that we know that the existing `regMaskTP` contains data that -// can be represented by `regMaskOnlyOne`. In other words, at -// those places, we never pass both gpr and vector registers -// together as part of `regMaskTP`. This will be eventually -// converted to "unsigned" +// Represents that the mask in this type is from one of the register type - gpr/float/predicate +// but not more than 1. typedef unsigned __int64 regMaskOnlyOne; - -// `regMaskMixed` tells that the mask can contain any of the gpr/vector -// registers. Once we identify all the places with `regMaskOnlyOne`, -// `regMaskFloat`, `regMaskGpr`, we will revisit `regMaskMixed` and try -// to either: -// 0. Revisit regMaskMixed and see if they should be "regMaskMixed" -// 1. Send separate parameter for `regMaskGpr` and `regMaskFloat`, etc. -// 2. Have a data structure like struct to pass all these together -typedef unsigned __int64 regMaskMixed; - typedef unsigned __int64 singleRegMask; #else @@ -369,8 +347,6 @@ typedef struct _regMaskAll FORCEINLINE void Clear(); FORCEINLINE bool IsEmpty() const; FORCEINLINE unsigned Count() const; - // FORCEINLINE void Create(regNumber reg); - // Rename this to AddRegNum FORCEINLINE void AddGprRegInMask(regNumber reg); FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 00c4b29e0a95ad..6fbc6a81d0eecd 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -171,7 +171,7 @@ #define REG_FLT_CALLEE_SAVED_LAST REG_XMM15 #define RBM_CALLEE_TRASH (RBM_INT_CALLEE_TRASH | RBM_FLT_CALLEE_TRASH | RBM_MSK_CALLEE_TRASH) - #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) // This will need to change once floatRegs are changed to 4-bytes + #define RBM_CALLEE_SAVED (RBM_INT_CALLEE_SAVED | RBM_FLT_CALLEE_SAVED | RBM_MSK_CALLEE_SAVED) #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) #define RBM_ALLGPR (RBM_ALLINT) From 7042510a9bae4acf095d55f613ef9076ed85cc5b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 16:16:45 -0700 Subject: [PATCH 199/201] jit format --- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/codegenxarch.cpp | 8 ++++---- src/coreclr/jit/compiler.cpp | 30 +++++++++++----------------- src/coreclr/jit/compiler.h | 34 ++++++++++++++++---------------- src/coreclr/jit/lsra.cpp | 19 +++++++++--------- src/coreclr/jit/lsra.h | 10 +++++----- src/coreclr/jit/lsraarm.cpp | 2 +- src/coreclr/jit/lsraarm64.cpp | 2 +- src/coreclr/jit/lsraxarch.cpp | 2 +- src/coreclr/jit/regset.h | 2 +- src/coreclr/jit/target.h | 8 ++++---- 11 files changed, 56 insertions(+), 63 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index a75d5056973ed6..11c468fb6f3322 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5210,7 +5210,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, callTargetReg = REG_DEFAULT_HELPER_CALL_TARGET; } - regMaskGpr callTargetMask = genRegMask(callTargetReg); + regMaskGpr callTargetMask = genRegMask(callTargetReg); CONSTREF_AllRegsMask callKillSet = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); // assert that all registers in callTargetMask are in the callKillSet diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 8c2a6a00c95194..dda69cff0f874c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9060,10 +9060,10 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, void* addr = nullptr; void* pAddr = nullptr; - emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; - addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); - regNumber callTarget = REG_NA; - CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); + emitter::EmitCallType callType = emitter::EC_FUNC_TOKEN; + addr = compiler->compGetHelperFtn((CorInfoHelpFunc)helper, &pAddr); + regNumber callTarget = REG_NA; + CONSTREF_AllRegsMask killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper); if (!addr) { diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 7843c212437a6b..9a850538649b07 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2049,10 +2049,8 @@ void Compiler::compInitAllRegsMask() AllRegsMask_NONE = AllRegsMask(); #ifdef HAS_MORE_THAN_64_REGISTERS - AllRegsMask_CALLEE_SAVED = - AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); - AllRegsMask_CALLEE_TRASH = - AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH); + AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_INT_CALLEE_SAVED, RBM_FLT_CALLEE_SAVED, RBM_MSK_CALLEE_SAVED); + AllRegsMask_CALLEE_TRASH = AllRegsMask(RBM_INT_CALLEE_TRASH, RBM_FLT_CALLEE_TRASH, RBM_MSK_CALLEE_TRASH); #else AllRegsMask_CALLEE_SAVED = AllRegsMask(RBM_CALLEE_SAVED); AllRegsMask_CALLEE_TRASH = AllRegsMask(RBM_CALLEE_TRASH); @@ -2098,10 +2096,9 @@ void Compiler::compInitAllRegsMask() // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. AllRegsMask_CALLEE_TRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; - AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = - GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF); - AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF); + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_NOGC; + AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF); + AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF = GprRegsMask(RBM_CALLEE_TRASH_WRITEBARRIER_BYREF); #ifdef UNIX_AMD64_ABI @@ -2113,9 +2110,8 @@ void Compiler::compInitAllRegsMask() AllRegsMask_STOP_FOR_GC_TRASH = AllRegsMask(RBM_INT_CALLEE_TRASH & ~(RBM_INTRET | RBM_INTRET_1), (RBM_FLT_CALLEE_TRASH & ~(RBM_FLOATRET | RBM_FLOATRET_1)), RBM_MSK_CALLEE_TRASH); - AllRegsMask_PROFILER_ENTER_TRASH = - AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), - RBM_MSK_CALLEE_TRASH); + AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask((RBM_INT_CALLEE_TRASH & ~RBM_ARG_REGS), + (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH); #else // See vm\amd64\asmhelpers.asm for more details. AllRegsMask_STOP_FOR_GC_TRASH = @@ -2136,8 +2132,7 @@ void Compiler::compInitAllRegsMask() AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask_NONE; // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. - AllRegsMask_CALLEE_TRASH_WRITEBARRIER = - GprRegsMask(RBM_R0 | RBM_R3 | RBM_LR | RBM_DEFAULT_HELPER_CALL_TARGET); + AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R0 | RBM_R3 | RBM_LR | RBM_DEFAULT_HELPER_CALL_TARGET); // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER = AllRegsMask_CALLEE_TRASH_WRITEBARRIER; @@ -2167,10 +2162,9 @@ void Compiler::compInitAllRegsMask() #elif defined(TARGET_ARM64) - AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_CALLEE_TRASH_NOGC); - AllRegsMask_PROFILER_ENTER_TRASH = - AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), - (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH); + AllRegsMask_CALLEE_TRASH_NOGC = GprRegsMask(RBM_CALLEE_TRASH_NOGC); + AllRegsMask_PROFILER_ENTER_TRASH = AllRegsMask((RBM_INT_CALLEE_TRASH & ~(RBM_ARG_REGS | RBM_ARG_RET_BUFF | RBM_FP)), + (RBM_FLT_CALLEE_TRASH & ~RBM_FLTARG_REGS), RBM_MSK_CALLEE_TRASH); // Registers killed by CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. AllRegsMask_CALLEE_TRASH_WRITEBARRIER = GprRegsMask(RBM_R14 | RBM_CALLEE_TRASH_NOGC); @@ -2765,7 +2759,7 @@ void Compiler::compInitOptions(JitFlags* jitFlags) } else { - verbose = false; + verbose = false; codeGen->setVerbose(false); } verboseTrees = verbose && shouldUseVerboseTrees(); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3d28411dbc812d..188cd2339c558e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11237,23 +11237,23 @@ class Compiler } #endif // TARGET_XARCH -AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; -AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; -AllRegsMask AllRegsMask_CALLEE_SAVED; -AllRegsMask AllRegsMask_CALLEE_TRASH; -AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC; -AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER; -AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF; -AllRegsMask AllRegsMask_EDX; -AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH; -AllRegsMask AllRegsMask_NONE; -AllRegsMask AllRegsMask_PROF_FNC_LEAVE; -AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH; -AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH; -AllRegsMask AllRegsMask_PROFILER_RET_SCRATCH; -AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH; -AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH; -AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; + AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER; + AllRegsMask AllRegsMask_CALLEE_GCTRASH_WRITEBARRIER_BYREF; + AllRegsMask AllRegsMask_CALLEE_SAVED; + AllRegsMask AllRegsMask_CALLEE_TRASH; + AllRegsMask AllRegsMask_CALLEE_TRASH_NOGC; + AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER; + AllRegsMask AllRegsMask_CALLEE_TRASH_WRITEBARRIER_BYREF; + AllRegsMask AllRegsMask_EDX; + AllRegsMask AllRegsMask_INIT_PINVOKE_FRAME_TRASH; + AllRegsMask AllRegsMask_NONE; + AllRegsMask AllRegsMask_PROF_FNC_LEAVE; + AllRegsMask AllRegsMask_PROFILER_ENTER_TRASH; + AllRegsMask AllRegsMask_PROFILER_LEAVE_TRASH; + AllRegsMask AllRegsMask_PROFILER_RET_SCRATCH; + AllRegsMask AllRegsMask_PROFILER_TAILCALL_TRASH; + AllRegsMask AllRegsMask_STOP_FOR_GC_TRASH; + AllRegsMask AllRegsMask_VALIDATE_INDIRECT_CALL_TRASH; }; // end of class Compiler diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b8b28c0825671d..055db0d1e443ba 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4358,7 +4358,6 @@ void LinearScan::resetAllRegistersState() } } - #ifdef HAS_MORE_THAN_64_REGISTERS void LinearScan::updateDeadCandidatesAtBlockStart(REF_AllRegsMask deadRegMask, VarToRegMap inVarToRegMap) #else @@ -4847,7 +4846,7 @@ void LinearScan::inActivateRegisters(RegBitSet64 inactiveMask) } } - //------------------------------------------------------------------------ +//------------------------------------------------------------------------ // LinearScan::freeRegister: Make a register available for use // // Arguments: @@ -5033,7 +5032,7 @@ void LinearScan::allocateRegistersMinimal() makeRegsAvailable(regsToMakeInactive); // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. - regsToMakeInactive = delayRegsToMakeInactive; + regsToMakeInactive = delayRegsToMakeInactive; delayRegsToMakeInactive.Clear(); } @@ -5107,7 +5106,7 @@ void LinearScan::allocateRegistersMinimal() delayRegsToFree.Clear(); regsInUseThisLocation.Clear(); } - regsToFree = delayRegsToFree; + regsToFree = delayRegsToFree; delayRegsToFree.Clear(); #ifdef DEBUG @@ -5153,8 +5152,8 @@ void LinearScan::allocateRegistersMinimal() regsToFree.Clear(); regsInUseThisLocation.Clear(); regsInUseNextLocation.Clear(); - handledBlockEnd = true; - curBBStartLocation = currentRefPosition.nodeLocation; + handledBlockEnd = true; + curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) { currentBlock = startBlockSequence(); @@ -5723,7 +5722,7 @@ void LinearScan::allocateRegisters() makeRegsAvailable(regsToMakeInactive); // TODO: Clean this up. We need to make the delayRegs inactive as well, but don't want // to mark them as free yet. - regsToMakeInactive = delayRegsToMakeInactive; + regsToMakeInactive = delayRegsToMakeInactive; delayRegsToMakeInactive.Clear(); } @@ -5801,7 +5800,7 @@ void LinearScan::allocateRegisters() delayRegsToFree.Clear(); regsInUseThisLocation.Clear(); } - regsToFree = delayRegsToFree; + regsToFree = delayRegsToFree; delayRegsToFree.Clear(); #ifdef DEBUG verifyFreeRegisters(regsToFree); @@ -5864,8 +5863,8 @@ void LinearScan::allocateRegisters() regsToFree.Clear(); regsInUseThisLocation.Clear(); regsInUseNextLocation.Clear(); - handledBlockEnd = true; - curBBStartLocation = currentRefPosition.nodeLocation; + handledBlockEnd = true; + curBBStartLocation = currentRefPosition.nodeLocation; if (currentBlock == nullptr) { currentBlock = startBlockSequence(); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 09b33cd873002a..f932c68c0f6fd4 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1132,11 +1132,11 @@ class LinearScan : public LinearScanInterface // Helpers for getKillSetForNode(). CONSTREF_AllRegsMask getKillSetForStoreInd(GenTreeStoreInd* tree); - regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); - regMaskGpr getKillSetForMul(GenTreeOp* tree); - AllRegsMask getKillSetForCall(GenTreeCall* call); - regMaskGpr getKillSetForModDiv(GenTreeOp* tree); - AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); + regMaskGpr getKillSetForShiftRotate(GenTreeOp* tree); + regMaskGpr getKillSetForMul(GenTreeOp* tree); + AllRegsMask getKillSetForCall(GenTreeCall* call); + regMaskGpr getKillSetForModDiv(GenTreeOp* tree); + AllRegsMask getKillSetForBlockStore(GenTreeBlk* blkNode); CONSTREF_AllRegsMask getKillSetForReturn(); CONSTREF_AllRegsMask getKillSetForProfilerHook(); #ifdef FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index bfe382d38d82e4..7fbc07adcab4be 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -467,7 +467,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { - srcCount = BuildReturn(tree); + srcCount = BuildReturn(tree); BuildKills(tree, getKillSetForReturn()); break; } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index ffda059d3c09d0..00f6429d6ecd74 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -738,7 +738,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { - srcCount = BuildReturn(tree); + srcCount = BuildReturn(tree); BuildKills(tree, getKillSetForReturn()); break; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index ef6df2b83dd254..d1aee6faa45183 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -189,7 +189,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RETURN: { - srcCount = BuildReturn(tree); + srcCount = BuildReturn(tree); BuildKills(tree, getKillSetForReturn()); break; } diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 1aaed64acd50ce..2709e3898ecaaa 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -77,7 +77,7 @@ class RegSet DEBUGARG(regMaskOnlyOne calleeSaveMask = RBM_NONE)) const; #endif // DEBUG - regMaskGpr rsIntCalleeSavedMask; + regMaskGpr rsIntCalleeSavedMask; public: void rsSetRegsModified(CONSTREF_AllRegsMask modifiedMask DEBUGARG(bool suppressDump = false)); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 6f63ba3252ec75..a8773eb818c80e 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -347,10 +347,10 @@ typedef struct _regMaskAll FORCEINLINE void Clear(); FORCEINLINE bool IsEmpty() const; FORCEINLINE unsigned Count() const; - FORCEINLINE void AddGprRegInMask(regNumber reg); - FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); - FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); - FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); + FORCEINLINE void AddGprRegInMask(regNumber reg); + FORCEINLINE void AddRegMaskForType(regMaskOnlyOne maskToAdd, var_types type); + FORCEINLINE void AddGprRegMask(regMaskGpr maskToAdd); + FORCEINLINE void AddFloatRegMask(regMaskFloat maskToAdd); FORCEINLINE void AddRegNumInMask(regNumber reg); FORCEINLINE void AddRegNum(regNumber reg, var_types type); From 22077eb01de7c1832568f3dee880f30fe11f6544 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 16:20:43 -0700 Subject: [PATCH 200/201] fix linux-x86 build --- src/coreclr/jit/optimizer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index a87a9a58f85059..9ac37de2d8eb3b 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -4189,7 +4189,9 @@ bool Compiler::optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* ho { hoistCtxt->m_loopVarMskCount = 0; hoistCtxt->m_loopVarInOutMskCount = 0; +#ifdef FEATURE_MASKED_HW_INTRINSICS hoistCtxt->m_hoistedMskExprCount = 0; +#endif } #endif // TARGET_XARCH @@ -4313,9 +4315,9 @@ bool Compiler::optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* ho optHoistLoopBlocks(loop, &defExec, hoistCtxt); unsigned numHoisted = hoistCtxt->m_hoistedFPExprCount + hoistCtxt->m_hoistedExprCount; -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS numHoisted += hoistCtxt->m_hoistedMskExprCount; -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS return numHoisted > 0; } From 893f6be8935514fb4facc4eb1c44aa5368aedb45 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 10 Apr 2024 16:23:24 -0700 Subject: [PATCH 201/201] fix linux-x64 Native_GCC build error --- src/coreclr/jit/register.h | 2 +- src/coreclr/jit/registerarm.h | 2 +- src/coreclr/jit/registerarm64.h | 4 ++-- src/coreclr/jit/registerloongarch64.h | 2 +- src/coreclr/jit/registerriscv64.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/register.h b/src/coreclr/jit/register.h index 2f84c8c40acbf6..e784f18a167fba 100644 --- a/src/coreclr/jit/register.h +++ b/src/coreclr/jit/register.h @@ -131,7 +131,7 @@ REGDEF(K5, 5+KBASE, KMASK(5), "k5", 2) REGDEF(K6, 6+KBASE, KMASK(6), "k6", 2) REGDEF(K7, 7+KBASE, KMASK(7), "k7", 2) -REGDEF(STK, 8+KBASE, 0x0000, "STK", -1) +REGDEF(STK, 8+KBASE, 0x0000, "STK", 3) #elif defined(TARGET_ARM) #include "registerarm.h" diff --git a/src/coreclr/jit/registerarm.h b/src/coreclr/jit/registerarm.h index 678c0ea3fb0035..c8e4e06947d4fc 100644 --- a/src/coreclr/jit/registerarm.h +++ b/src/coreclr/jit/registerarm.h @@ -75,7 +75,7 @@ REGALIAS(R14, LR) REGALIAS(R15, PC) // This must be last! -REGDEF(STK, 32+FPBASE, 0x0000, "STK", -1) +REGDEF(STK, 32+FPBASE, 0x0000, "STK", 2) /*****************************************************************************/ #undef REGDEF diff --git a/src/coreclr/jit/registerarm64.h b/src/coreclr/jit/registerarm64.h index b6da3ff3d4548a..e754add488404b 100644 --- a/src/coreclr/jit/registerarm64.h +++ b/src/coreclr/jit/registerarm64.h @@ -120,9 +120,9 @@ REGDEF(P15, 15+PBASE, PMASK(15), "p15", "na", 2) // The registers with values 80 (NBASE) and above are not real register numbers #define NBASE 80 -REGDEF(SP, 0+NBASE, 0x0000, "sp", "wsp?", -1) +REGDEF(SP, 0+NBASE, 0x0000, "sp", "wsp?", 3) // This must be last! -REGDEF(STK, 1+NBASE, 0x0000, "STK", "STK", -1) +REGDEF(STK, 1+NBASE, 0x0000, "STK", "STK", 3) /*****************************************************************************/ #undef RMASK diff --git a/src/coreclr/jit/registerloongarch64.h b/src/coreclr/jit/registerloongarch64.h index 2c063b8a76985f..288bb020a93ad0 100644 --- a/src/coreclr/jit/registerloongarch64.h +++ b/src/coreclr/jit/registerloongarch64.h @@ -101,7 +101,7 @@ REGDEF(F31, 31+FBASE, FMASK(31), "f31", 1) #define NBASE 64 // This must be last! -REGDEF(STK, 0+NBASE, 0x0000, "STK", -1) +REGDEF(STK, 0+NBASE, 0x0000, "STK", 2) /*****************************************************************************/ #undef RMASK diff --git a/src/coreclr/jit/registerriscv64.h b/src/coreclr/jit/registerriscv64.h index 96c4a7505424b8..2522fd3f5f9e91 100644 --- a/src/coreclr/jit/registerriscv64.h +++ b/src/coreclr/jit/registerriscv64.h @@ -93,7 +93,7 @@ REGDEF(F31, 31+FBASE, FMASK(31), "f31", 1) // The registers with values 64 (NBASE) and above are not real register numbers #define NBASE 64 -REGDEF(STK, 0+NBASE, 0x0000, "STK", -1) +REGDEF(STK, 0+NBASE, 0x0000, "STK", 2) /*****************************************************************************/ #undef RMASK