From fea5284c31a8409a2e7ac04cdb66f35c4fd099cc Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Mar 2024 20:03:53 +0100 Subject: [PATCH 01/15] JIT: Split ABI classification by ABI --- src/coreclr/jit/abi.cpp | 374 ++++++++++++++++++++++++ src/coreclr/jit/abi.h | 117 +++++++- src/coreclr/jit/compiler.h | 5 + src/coreclr/jit/compmemkind.h | 1 + src/coreclr/jit/lclvars.cpp | 219 +++++++------- src/coreclr/jit/registerargconvention.h | 13 +- 6 files changed, 598 insertions(+), 131 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index d28e11f6129b3f..dc5f2b9e43d10a 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -120,3 +120,377 @@ bool ABIPassingInformation::IsSplitAcrossRegistersAndStack() const } return anyReg && anyStack; } + +ABIPassingInformation ABIPassingInformation::FromSegment(Compiler* comp, const ABIPassingSegment& segment) +{ + ABIPassingInformation info; + info.NumSegments = 1; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment(segment); + return info; +} + +regNumber RegisterQueue::Dequeue() +{ + assert(Count() > 0); + return static_cast(m_regs[m_index++]); +} + +regNumber RegisterQueue::Peek() +{ + assert(Count() > 0); + return static_cast(m_regs[m_index]); +} + +void RegisterQueue::Clear() +{ + m_index = m_numRegs; +} + +static unsigned TypeSize(var_types type, ClassLayout* structLayout) +{ + return type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); +} + +#ifdef TARGET_X86 +ABIPassingInformation X86Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + unsigned size = TypeSize(type, structLayout); + unsigned numSlots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + + bool canEnreg = false; + if (m_regs.Count() >= numSlots) + { + switch (type) + { + case TYP_BYTE: + case TYP_UBYTE: + case TYP_SHORT: + case TYP_USHORT: + case TYP_INT: + case TYP_REF: + case TYP_BYREF: + canEnreg = true; + break; + case TYP_STRUCT: + canEnreg = comp->isTrivialPointerSizedStruct(structLayout->GetClassHandle()); + default: + break; + } + } + + ABIPassingSegment segment; + if (canEnreg) + { + assert(numSlots == 1); + segment = ABIPassingSegment::InRegister(m_regs.Dequeue(), 0, size); + } + else + { + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, size); + m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); + } + + return ABIPassingInformation::FromSegment(comp, segment); +} +#endif + +#ifdef WINDOWS_AMD64_ABI +static const regNumberSmall WinX64IntArgRegs[] = {REG_RCX, REG_RDX, REG_R8, REG_R9}; +static const regNumberSmall WinX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3}; + +WinX64Classifier::WinX64Classifier() + : m_intRegs(WinX64IntArgRegs, ArrLen(WinX64IntArgRegs)), m_floatRegs(WinX64FloatArgRegs, ArrLen(WinX64FloatArgRegs)) +{ +} + +ABIPassingInformation WinX64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + // On windows-x64 ABI all parameters take exactly 1 stack slot (structs + // that do not fit are passed implicitly by reference). Passing a parameter + // in an int register also consumes the corresponding float register and + // vice versa. + assert(m_intRegs.Count() == m_floatRegs.Count()); + + unsigned typeSize = TypeSize(type, structLayout); + if ((typeSize > TARGET_POINTER_SIZE) || !isPow2(typeSize)) + { + typeSize = TARGET_POINTER_SIZE; // Passed by implicit byref + } + + ABIPassingSegment segment; + if (m_intRegs.Count() > 0) + { + regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Peek() : m_intRegs.Peek(); + segment = ABIPassingSegment::InRegister(reg, 0, typeSize); + m_intRegs.Dequeue(); + m_floatRegs.Dequeue(); + } + else + { + segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, typeSize); + m_stackArgSize += TARGET_POINTER_SIZE; + } + + return ABIPassingInformation::FromSegment(comp, segment); +} +#endif + +#ifdef UNIX_AMD64_ABI +static const regNumberSmall SysVX64IntArgRegs[] = {REG_EDI, REG_ESI, REG_EDX, REG_ECX, REG_R8, REG_R9}; +static const regNumberSmall SysVX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, + REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7}; + +SysVX64Classifier::SysVX64Classifier() + : m_intRegs(SysVX64IntArgRegs, ArrLen(SysVX64IntArgRegs)) + , m_floatRegs(SysVX64FloatArgRegs, ArrLen(SysVX64FloatArgRegs)) +{ +} + +ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + bool canEnreg = false; + SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; + if (varTypeIsStruct(type)) + { + comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(structLayout->GetClassHandle(), &structDesc); + + if (structDesc.passedInRegisters) + { + unsigned intRegCount = 0; + unsigned floatRegCount = 0; + + for (unsigned int i = 0; i < structDesc.eightByteCount; i++) + { + if (structDesc.IsIntegralSlot(i)) + { + intRegCount++; + } + else if (structDesc.IsSseSlot(i)) + { + floatRegCount++; + } + else + { + assert(!"Invalid eightbyte classification type."); + break; + } + } + + canEnreg = (m_intRegs.Count() <= intRegCount) && (m_floatRegs.Count() <= floatRegCount); + } + } + else + { + unsigned availRegs = varTypeUsesFloatArgReg(type) ? m_floatRegs.Count() : m_intRegs.Count(); + canEnreg = availRegs > 0; + } + + ABIPassingInformation info; + if (canEnreg) + { + if (varTypeIsStruct(type)) + { + info.NumSegments = structDesc.eightByteCount; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[structDesc.eightByteCount]; + + for (unsigned i = 0; i < structDesc.eightByteCount; i++) + { + regNumber reg = structDesc.IsIntegralSlot(i) ? m_intRegs.Dequeue() : m_floatRegs.Dequeue(); + info.Segments[i] = + ABIPassingSegment::InRegister(reg, structDesc.eightByteOffsets[i], structDesc.eightByteSizes[i]); + } + } + else + { + regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Dequeue() : m_intRegs.Dequeue(); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(reg, 0, genTypeSize(type))); + } + } + else + { + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + unsigned size = TypeSize(type, structLayout); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, size)); + m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); + } + + return info; +} +#endif + +#ifdef TARGET_ARM64 +static const regNumberSmall Arm64IntArgRegs[] = {REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, REG_R6, REG_R7}; +static const regNumberSmall Arm64FloatArgRegs[] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, REG_V6, REG_V7}; + +Arm64Classifier::Arm64Classifier() + : m_intRegs(Arm64IntArgRegs, ArrLen(Arm64IntArgRegs)), m_floatRegs(Arm64FloatArgRegs, ArrLen(Arm64FloatArgRegs)) +{ +} + +ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + if (wellKnownParam == WellKnownArg::RetBuffer) + { + return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_ARG_RET_BUFF, 0, + TARGET_POINTER_SIZE)); + } + + if (varTypeIsStruct(type) && !comp->info.compIsVarArgs) + { + var_types hfaType = comp->GetHfaType(structLayout->GetClassHandle()); + + if (hfaType != TYP_UNDEF) + { + unsigned elemSize = genTypeSize(hfaType); + unsigned slots = structLayout->GetSize() / elemSize; + ABIPassingInformation info; + if (m_floatRegs.Count() >= slots) + { + info.NumSegments = slots; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; + + for (unsigned i = 0; i < slots; i++) + { + info.Segments[i] = ABIPassingSegment::InRegister(m_floatRegs.Dequeue(), i * elemSize, elemSize); + } + } + else + { + unsigned alignment = compAppleArm64Abi() ? elemSize : TARGET_POINTER_SIZE; + m_stackArgSize = roundUp(m_stackArgSize, alignment); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, + structLayout->GetSize())); + m_stackArgSize += structLayout->GetSize(); + // After passing any float value on the stack, we should not enregister more float values. + m_floatRegs.Clear(); + } + + return info; + } + } + + unsigned slots; + if (varTypeIsStruct(type)) + { + unsigned size = structLayout->GetSize(); + if (size > 16) + { + slots = 1; // Passed by implicit byref + } + else + { + slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + } + } + else + { + assert(genTypeSize(type) <= TARGET_POINTER_SIZE); + slots = 1; + } + + assert((slots == 1) || (slots == 2)); + + ABIPassingInformation info; + if (comp->info.compIsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) + { + // On varargs we split structs between register and stack in this case. + // Normally a struct that does not fit in registers will always be + // passed on stack. + assert(compFeatureArgSplit()); + info.NumSegments = 2; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]; + info.Segments[0] = ABIPassingSegment::InRegister(m_intRegs.Dequeue(), 0, TARGET_POINTER_SIZE); + info.Segments[1] = ABIPassingSegment::OnStack(m_stackArgSize, TARGET_POINTER_SIZE, + structLayout->GetSize() - TARGET_POINTER_SIZE); + m_stackArgSize += TARGET_POINTER_SIZE; + } + else + { + RegisterQueue& regs = varTypeUsesFloatArgReg(type) ? m_floatRegs : m_intRegs; + + if (regs.Count() >= slots) + { + info.NumSegments = slots; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; + unsigned slotSize = varTypeIsStruct(type) ? TARGET_POINTER_SIZE : genTypeSize(type); + info.Segments[0] = ABIPassingSegment::InRegister(regs.Dequeue(), 0, slotSize); + if (slots == 2) + { + assert(varTypeIsStruct(type)); + unsigned tailSize = structLayout->GetSize() - slotSize; + info.Segments[1] = ABIPassingSegment::InRegister(regs.Dequeue(), slotSize, tailSize); + } + } + else + { + unsigned alignment; + if (compAppleArm64Abi()) + { + if (varTypeIsStruct(type)) + { + alignment = TARGET_POINTER_SIZE; + } + else + { + alignment = genTypeSize(type); + } + + m_stackArgSize = roundUp(m_stackArgSize, alignment); + } + else + { + alignment = TARGET_POINTER_SIZE; + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + } + + unsigned size = TypeSize(type, structLayout); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, size)); + + m_stackArgSize += roundUp(size, alignment); + + // As soon as we pass something on stack we cannot go back and + // enregister something else. + regs.Clear(); + } + } + + return info; +} +#endif + +#ifdef SWIFT_SUPPORT +ABIPassingInformation SwiftABIClassifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ +#ifdef TARGET_AMD64 + if (wellKnownParam == WellKnownArg::RetBuffer) + { + return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_SWIFT_ARG_RET_BUFF, 0, + TARGET_POINTER_SIZE)); + } +#endif + + if (wellKnownParam == WellKnownArg::SwiftSelf) + { + return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_SWIFT_SELF, 0, + TARGET_POINTER_SIZE)); + } + + return m_classifier.Classify(comp, type, structLayout, wellKnownParam); +} +#endif diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index 4d50c46851fa9c..9bfa0b94863116 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -36,18 +36,127 @@ class ABIPassingSegment struct ABIPassingInformation { // The number of segments used to pass the value. Examples: - // - On x86, TYP_LONG can be passed in two registers, resulting in two - // register segments // - On SysV x64, structs can be passed in two registers, resulting in two // register segments // - On arm64/arm32, HFAs can be passed in up to four registers, giving // four register segments // - On arm32, structs can be split out over register and stack, giving // multiple register segments and a struct segment. - // - On Windows x64, all parameters always belong into one stack slot or register, - // and thus always have NumSegments == 1 + // - On Windows x64, all parameters always fit into one stack slot or + // register, and thus always have NumSegments == 1 unsigned NumSegments = 0; ABIPassingSegment* Segments = nullptr; bool IsSplitAcrossRegistersAndStack() const; + + static ABIPassingInformation FromSegment(Compiler* comp, const ABIPassingSegment& segment); +}; + +class RegisterQueue +{ + const regNumberSmall* m_regs; + unsigned int m_numRegs; + unsigned int m_index = 0; + +public: + RegisterQueue(const regNumberSmall* regs, unsigned int numRegs) : m_regs(regs), m_numRegs(numRegs) + { + } + + unsigned Count() + { + return m_numRegs - m_index; + } + + regNumber Dequeue(); + regNumber Peek(); + void Clear(); +}; + +class X86Classifier +{ + RegisterQueue m_regs; + unsigned m_stackArgSize = 0; + +public: + X86Classifier(const regNumberSmall* regs, unsigned int numRegs) : m_regs(regs, numRegs) + { + } + + ABIPassingInformation Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam); +}; + +class WinX64Classifier +{ + RegisterQueue m_intRegs; + RegisterQueue m_floatRegs; + unsigned m_stackArgSize = 0; + +public: + WinX64Classifier(); + + ABIPassingInformation Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam); +}; + +class SysVX64Classifier +{ + RegisterQueue m_intRegs; + RegisterQueue m_floatRegs; + unsigned m_stackArgSize = 0; + +public: + SysVX64Classifier(); + + ABIPassingInformation Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam); +}; + +class Arm64Classifier +{ + RegisterQueue m_intRegs; + RegisterQueue m_floatRegs; + unsigned m_stackArgSize = 0; + +public: + Arm64Classifier(); + + ABIPassingInformation Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam); +}; + +#if defined(TARGET_X86) +typedef X86Classifier PlatformClassifier; +#elif defined(WINDOWS_AMD64_ABI) +typedef WinX64Classifier PlatformClassifier; +#elif defined(UNIX_AMD64_ABI) +typedef SysVX64Classifier PlatformClassifier; +#elif defined(TARGET_ARM64) +typedef Arm64Classifier PlatformClassifier; +#endif + +#ifdef SWIFT_SUPPORT +class SwiftABIClassifier +{ + PlatformClassifier m_classifier; + +public: + SwiftABIClassifier() + { + } + + ABIPassingInformation Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam); }; +#endif diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 4e9ec67b31a467..5d85ec124c3f6c 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3994,6 +3994,11 @@ class Compiler CORINFO_ARG_LIST_HANDLE varList, CORINFO_SIG_INFO* varSig); + template + void lvaClassifyParameterABI(Classifier& classifier); + + void lvaClassifyParameterABI(); + bool lvaInitSpecialSwiftParam(InitVarDscInfo* varDscInfo, CorInfoType type, CORINFO_CLASS_HANDLE typeHnd); var_types lvaGetActualType(unsigned lclNum); diff --git a/src/coreclr/jit/compmemkind.h b/src/coreclr/jit/compmemkind.h index e986682894c3b6..0221eadb067492 100644 --- a/src/coreclr/jit/compmemkind.h +++ b/src/coreclr/jit/compmemkind.h @@ -10,6 +10,7 @@ // and the corresponding array of string names for these enum members. // clang-format off +CompMemKindMacro(ABI) CompMemKindMacro(AssertionProp) CompMemKindMacro(ASTNode) CompMemKindMacro(InstDesc) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 3630020fc12873..2503642a42c44d 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -35,6 +35,8 @@ unsigned Compiler::s_lvaDoubleAlignedProcsCount = 0; void Compiler::lvaInit() { + lvaParameterPassingInfo = nullptr; + /* We haven't allocated stack variables yet */ lvaRefCountState = RCS_INVALID; @@ -258,23 +260,23 @@ void Compiler::lvaInitTypeRef() { case CorInfoCallConvExtension::Thiscall: // In thiscall the this parameter goes into a register. - varDscInfo.Init(lvaTable, lvaParameterPassingInfo, hasRetBuffArg, 1, 0); + varDscInfo.Init(lvaTable, hasRetBuffArg, 1, 0); break; case CorInfoCallConvExtension::C: case CorInfoCallConvExtension::Stdcall: case CorInfoCallConvExtension::CMemberFunction: case CorInfoCallConvExtension::StdcallMemberFunction: - varDscInfo.Init(lvaTable, lvaParameterPassingInfo, hasRetBuffArg, 0, 0); + varDscInfo.Init(lvaTable, hasRetBuffArg, 0, 0); break; case CorInfoCallConvExtension::Managed: case CorInfoCallConvExtension::Fastcall: case CorInfoCallConvExtension::FastcallMemberFunction: default: - varDscInfo.Init(lvaTable, lvaParameterPassingInfo, hasRetBuffArg, MAX_REG_ARG, MAX_FLOAT_REG_ARG); + varDscInfo.Init(lvaTable, hasRetBuffArg, MAX_REG_ARG, MAX_FLOAT_REG_ARG); break; } #else - varDscInfo.Init(lvaTable, lvaParameterPassingInfo, hasRetBuffArg, MAX_REG_ARG, MAX_FLOAT_REG_ARG); + varDscInfo.Init(lvaTable, hasRetBuffArg, MAX_REG_ARG, MAX_FLOAT_REG_ARG); #endif lvaInitArgs(&varDscInfo); @@ -415,7 +417,7 @@ void Compiler::lvaInitArgs(InitVarDscInfo* varDscInfo) //---------------------------------------------------------------------- - /* Is there a "this" pointer ? */ + // Is there a "this" pointer ? lvaInitThisPtr(varDscInfo); unsigned numUserArgsToSkip = 0; @@ -468,6 +470,10 @@ void Compiler::lvaInitArgs(InitVarDscInfo* varDscInfo) // We have set info.compArgsCount in compCompile() noway_assert(varDscInfo->varNum == info.compArgsCount); + + // Now we have parameters created in the right order. Figure out how they're passed. + lvaClassifyParameterABI(); + assert(varDscInfo->intRegArgNum <= MAX_REG_ARG); codeGen->intRegState.rsCalleeRegArgCount = varDscInfo->intRegArgNum; @@ -515,16 +521,6 @@ void Compiler::lvaInitThisPtr(InitVarDscInfo* varDscInfo) lvaSetClass(varDscInfo->varNum, info.compClassHnd); } - varDsc->lvIsRegArg = 1; - noway_assert(varDscInfo->intRegArgNum == 0); - - varDsc->SetArgReg( - genMapRegArgNumToRegNum(varDscInfo->allocRegArg(TYP_INT), varDsc->TypeGet(), info.compCallConv)); -#if FEATURE_MULTIREG_ARGS - varDsc->SetOtherArgReg(REG_NA); -#endif - varDsc->lvOnFrame = true; // The final home for this incoming register might be our local stack frame - #ifdef DEBUG if (verbose) { @@ -581,19 +577,6 @@ void Compiler::lvaInitRetBuffArg(InitVarDscInfo* varDscInfo, bool useFixedRetBuf } #endif - ABIPassingInformation* abiInfo = varDscInfo->abiInfo; - abiInfo->NumSegments = 1; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[1]; - - if (varDsc->lvIsRegArg) - { - abiInfo->Segments[0] = ABIPassingSegment::InRegister(varDsc->GetArgReg(), 0, TARGET_POINTER_SIZE); - } - else - { - abiInfo->Segments[0] = ABIPassingSegment::OnStack(varDsc->GetStackOffset(), 0, TARGET_POINTER_SIZE); - } - compArgSize += TARGET_POINTER_SIZE; varDscInfo->nextParam(); @@ -677,7 +660,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un } #endif - ABIPassingInformation* abiInfo = varDscInfo->abiInfo; // For ARM, ARM64, LOONGARCH64, RISCV64 and AMD64 varargs, all arguments go in integer registers var_types argType = mangleVarArgsType(varDsc->TypeGet()); @@ -1053,16 +1035,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un genMapRegArgNumToRegNum(firstAllocatedRegArgNum + 1, TYP_I_IMPL, info.compCallConv)); varDsc->lvIsMultiRegArg = true; } - - abiInfo->NumSegments = cSlots; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[cSlots]; - for (unsigned i = 0; i < cSlots; i++) - { - abiInfo->Segments[i] = - ABIPassingSegment::InRegister(genMapRegArgNumToRegNum(firstAllocatedRegArgNum + i, TYP_I_IMPL, - info.compCallConv), - i * TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - } } #elif defined(UNIX_AMD64_ABI) if (varTypeIsStruct(argType)) @@ -1085,14 +1057,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un } assert(structDesc.eightByteCount <= 2); - abiInfo->NumSegments = structDesc.eightByteCount; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[structDesc.eightByteCount]; - for (int i = 0; i < structDesc.eightByteCount; i++) - { - regNumber reg = i == 0 ? varDsc->GetArgReg() : varDsc->GetOtherArgReg(); - abiInfo->Segments[i] = ABIPassingSegment::InRegister(reg, structDesc.eightByteOffsets[i], - structDesc.eightByteSizes[i]); - } } #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) if (argType == TYP_STRUCT) @@ -1125,14 +1089,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un varDsc->lvIsSplit = 1; varDsc->SetOtherArgReg(REG_STK); varDscInfo->setAllRegArgUsed(argRegTypeInStruct1); - - abiInfo->NumSegments = 2; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[2]; - abiInfo->Segments[0] = - ABIPassingSegment::InRegister(varDsc->GetArgReg(), 0, genTypeSize(argRegTypeInStruct1)); - abiInfo->Segments[1] = ABIPassingSegment::OnStack(varDscInfo->stackArgSize, TARGET_POINTER_SIZE, - TARGET_POINTER_SIZE); - varDscInfo->stackArgSize += TARGET_POINTER_SIZE; #ifdef TARGET_RISCV64 varDscInfo->hasSplitParam = true; @@ -1204,44 +1160,8 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un { numEnregistered = cSlots; } - - abiInfo->NumSegments = numEnregistered + (stackSize > 0 ? 1 : 0); - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[abiInfo->NumSegments]; - for (unsigned i = 0; i < numEnregistered; i++) - { - abiInfo->Segments[i] = - ABIPassingSegment::InRegister(genMapRegArgNumToRegNum(firstAllocatedRegArgNum + i, argType, - info.compCallConv), - i * TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - } - - if (stackSize > 0) - { - abiInfo->Segments[numEnregistered] = - ABIPassingSegment::OnStack(0, numEnregistered * TARGET_POINTER_SIZE, stackSize); - } - #endif // TARGET_ARM - if (abiInfo->NumSegments == 0) - { - // Structs at this point are always passed in 1 slot (either - // because it is small enough, or because it is an implicit - // byref). HFA's are also handled here, but they were retyped - // to have argType equal to their element type. - assert((argType != TYP_STRUCT) || (cSlots == 1)); - abiInfo->NumSegments = cSlots; - unsigned size = argType == TYP_STRUCT ? TARGET_POINTER_SIZE : genTypeSize(argType); - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[cSlots]; - for (unsigned i = 0; i < cSlots; i++) - { - abiInfo->Segments[i] = - ABIPassingSegment::InRegister(genMapRegArgNumToRegNum(firstAllocatedRegArgNum + i, argType, - info.compCallConv), - i * size, size); - } - } - #ifdef DEBUG if (verbose) { @@ -1391,11 +1311,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un JITDUMP("set user arg V%02u offset to %u\n", varDscInfo->varNum, varDscInfo->stackArgSize); varDsc->SetStackOffset(varDscInfo->stackArgSize); - - abiInfo->NumSegments = 1; - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::OnStack(varDscInfo->stackArgSize, 0, argSize)); - varDscInfo->stackArgSize += argSize; } @@ -1486,15 +1401,11 @@ bool Compiler::lvaInitSpecialSwiftParam(InitVarDscInfo* varDscInfo, CorInfoType const char* className = info.compCompHnd->getClassNameFromMetadata(typeHnd, &namespaceName); if ((strcmp(className, "SwiftSelf") == 0) && (strcmp(namespaceName, "System.Runtime.InteropServices.Swift") == 0)) { - LclVarDsc* varDsc = varDscInfo->varDsc; - ABIPassingInformation* abiInfo = varDscInfo->abiInfo; + LclVarDsc* varDsc = varDscInfo->varDsc; varDsc->SetArgReg(REG_SWIFT_SELF); varDsc->SetOtherArgReg(REG_NA); varDsc->lvIsRegArg = true; - abiInfo->NumSegments = 1; - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::InRegister(REG_SWIFT_SELF, 0, TARGET_POINTER_SIZE)); compArgSize += TARGET_POINTER_SIZE; lvaSwiftSelfArg = varDscInfo->varNum; @@ -1515,11 +1426,9 @@ void Compiler::lvaInitGenericsCtxt(InitVarDscInfo* varDscInfo) { info.compTypeCtxtArg = varDscInfo->varNum; - LclVarDsc* varDsc = varDscInfo->varDsc; - varDsc->lvIsParam = 1; - varDsc->lvType = TYP_I_IMPL; - ABIPassingInformation* abiInfo = varDscInfo->abiInfo; - abiInfo->NumSegments = 1; + LclVarDsc* varDsc = varDscInfo->varDsc; + varDsc->lvIsParam = 1; + varDsc->lvType = TYP_I_IMPL; if (varDscInfo->canEnreg(TYP_I_IMPL)) { @@ -1531,8 +1440,6 @@ void Compiler::lvaInitGenericsCtxt(InitVarDscInfo* varDscInfo) #if FEATURE_MULTIREG_ARGS varDsc->SetOtherArgReg(REG_NA); #endif - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::InRegister(varDsc->GetArgReg(), 0, TARGET_POINTER_SIZE)); varDsc->lvOnFrame = true; // The final home for this incoming register might be our local stack frame varDscInfo->intRegArgNum++; @@ -1549,8 +1456,6 @@ void Compiler::lvaInitGenericsCtxt(InitVarDscInfo* varDscInfo) // We need to mark these as being on the stack, as this is not done elsewhere in the case that canEnreg // returns false. varDsc->lvOnFrame = true; - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::OnStack(varDscInfo->stackArgSize, 0, TARGET_POINTER_SIZE)); varDsc->SetStackOffset(varDscInfo->stackArgSize); varDscInfo->stackArgSize += TARGET_POINTER_SIZE; } @@ -1582,9 +1487,6 @@ void Compiler::lvaInitVarArgsHandle(InitVarDscInfo* varDscInfo) #endif // TARGET_X86 varDsc->lvHasLdAddrOp = 1; - ABIPassingInformation* abiInfo = varDscInfo->abiInfo; - abiInfo->NumSegments = 1; - lvaSetVarDoNotEnregister(lvaVarargsHandleArg DEBUGARG(DoNotEnregisterReason::VMNeedsStackAddr)); assert(mostRecentlyActivePhase == PHASE_PRE_IMPORT); @@ -1597,8 +1499,6 @@ void Compiler::lvaInitVarArgsHandle(InitVarDscInfo* varDscInfo) varDsc->lvIsRegArg = 1; varDsc->SetArgReg(genMapRegArgNumToRegNum(varArgHndArgNum, TYP_I_IMPL, info.compCallConv)); - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::InRegister(varDsc->GetArgReg(), 0, TARGET_POINTER_SIZE)); #if FEATURE_MULTIREG_ARGS varDsc->SetOtherArgReg(REG_NA); #endif @@ -1625,8 +1525,6 @@ void Compiler::lvaInitVarArgsHandle(InitVarDscInfo* varDscInfo) // We need to mark these as being on the stack, as this is not done elsewhere in the case that canEnreg // returns false. varDsc->lvOnFrame = true; - abiInfo->Segments = new (this, CMK_LvaTable) - ABIPassingSegment(ABIPassingSegment::OnStack(varDscInfo->stackArgSize, 0, TARGET_POINTER_SIZE)); varDsc->SetStackOffset(varDscInfo->stackArgSize); varDscInfo->stackArgSize += TARGET_POINTER_SIZE; } @@ -1724,6 +1622,93 @@ void Compiler::lvaInitVarDsc(LclVarDsc* varDsc, #endif // FEATURE_MULTIREG_ARGS } +template +void Compiler::lvaClassifyParameterABI(Classifier& classifier) +{ + lvaParameterPassingInfo = new (this, CMK_LvaTable) ABIPassingInformation[info.compArgsCount]; + + for (unsigned i = 0; i < info.compArgsCount; i++) + { + LclVarDsc* dsc = lvaGetDesc(i); + ClassLayout* structLayout = varTypeIsStruct(dsc) ? dsc->GetLayout() : nullptr; + + WellKnownArg wellKnownArg = WellKnownArg::None; + if (i == info.compRetBuffArg) + { + wellKnownArg = WellKnownArg::RetBuffer; + } +#ifdef SWIFT_SUPPORT + else if (i == lvaSwiftSelfArg) + { + wellKnownArg = WellKnownArg::SwiftSelf; + } +#endif + + lvaParameterPassingInfo[i] = classifier.Classify(this, dsc->TypeGet(), structLayout, wellKnownArg); + } +} + +void Compiler::lvaClassifyParameterABI() +{ + if (info.compArgsCount == 0) + { + return; + } + +#if defined(TARGET_X86) + switch (info.compCallConv) + { + case CorInfoCallConvExtension::Thiscall: + { + static const regNumberSmall thiscallRegs[] = {REG_ECX}; + X86Classifier classifier(thiscallRegs, ArrLen(thiscallRegs)); + lvaClassifyParameterABI(classifier); + break; + } + case CorInfoCallConvExtension::C: + case CorInfoCallConvExtension::Stdcall: + case CorInfoCallConvExtension::CMemberFunction: + case CorInfoCallConvExtension::StdcallMemberFunction: + { + X86Classifier classifier(nullptr, 0); + lvaClassifyParameterABI(classifier); + break; + } + default: + { + static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; + unsigned numRegs = ArrLen(regs); + if (info.compIsVarArgs) + { + // In varargs methods we only enregister the this pointer (if there is one). + numRegs = info.compThisArg == BAD_VAR_NUM ? 0 : 1; + } + X86Classifier classifier(regs, numRegs); + lvaClassifyParameterABI(classifier); + break; + } + } +#elif !defined(TARGET_ARM) +#ifdef SWIFT_SUPPORT + if (info.compCallConv == CorInfoCallConvExtension::Swift) + { + SwiftABIClassifier classifier; + lvaClassifyParameterABI(classifier); + } + else +#endif + { + PlatformClassifier classifier; + lvaClassifyParameterABI(classifier); + } +#endif + + if (lvaParameterPassingInfo == nullptr) + { + return; + } +} + /***************************************************************************** * Returns our internal varNum for a given IL variable. * Asserts assume it is called after lvaTable[] has been set up. diff --git a/src/coreclr/jit/registerargconvention.h b/src/coreclr/jit/registerargconvention.h index bddb5ae1477df1..840f7adc4fcebd 100644 --- a/src/coreclr/jit/registerargconvention.h +++ b/src/coreclr/jit/registerargconvention.h @@ -8,9 +8,8 @@ class LclVarDsc; struct InitVarDscInfo { - LclVarDsc* varDsc; - unsigned varNum; - ABIPassingInformation* abiInfo; + LclVarDsc* varDsc; + unsigned varNum; unsigned intRegArgNum; unsigned floatRegArgNum; @@ -35,16 +34,11 @@ struct InitVarDscInfo public: // set to initial values - void Init(LclVarDsc* lvaTable, - ABIPassingInformation* abiInfoTable, - bool _hasRetBufArg, - unsigned _maxIntRegArgNum, - unsigned _maxFloatRegArgNum) + void Init(LclVarDsc* lvaTable, bool _hasRetBufArg, unsigned _maxIntRegArgNum, unsigned _maxFloatRegArgNum) { hasRetBufArg = _hasRetBufArg; varDsc = lvaTable; // the first argument LclVar 0 varNum = 0; // the first argument varNum 0 - this->abiInfo = abiInfoTable; intRegArgNum = 0; floatRegArgNum = 0; maxIntRegArgNum = _maxIntRegArgNum; @@ -117,7 +111,6 @@ struct InitVarDscInfo { varDsc++; varNum++; - abiInfo++; } private: From 150cb283854fb06683a93a86f1182b06007f3e01 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Mar 2024 22:21:23 +0100 Subject: [PATCH 02/15] Refactor, fix arm64 --- src/coreclr/jit/abi.cpp | 77 +++++++++++++++++++++------ src/coreclr/jit/abi.h | 21 +++++--- src/coreclr/jit/compiler.hpp | 4 +- src/coreclr/jit/gentree.cpp | 6 +-- src/coreclr/jit/lclvars.cpp | 100 ++++++++++++++++++++++------------- src/coreclr/jit/target.h | 2 +- 6 files changed, 143 insertions(+), 67 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index dc5f2b9e43d10a..c3949071c4ecc5 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -152,6 +152,39 @@ static unsigned TypeSize(var_types type, ClassLayout* structLayout) } #ifdef TARGET_X86 +X86Classifier::X86Classifier(const ClassifierInfo& info) + : m_regs(nullptr, 0) +{ + switch (info.CallConv) + { + case CorInfoCallConvExtension::Thiscall: + { + static const regNumberSmall thiscallRegs[] = {REG_ECX}; + m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); + break; + } + case CorInfoCallConvExtension::C: + case CorInfoCallConvExtension::Stdcall: + case CorInfoCallConvExtension::CMemberFunction: + case CorInfoCallConvExtension::StdcallMemberFunction: + { + break; + } + default: + { + static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; + unsigned numRegs = ArrLen(regs); + if (info.IsVarArgs) + { + // In varargs methods we only enregister the this pointer (if there is one). + numRegs = info.HasThis ? 1 : 0; + } + m_regs = RegisterQueue(regs, numRegs); + break; + } + } +} + ABIPassingInformation X86Classifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, @@ -202,7 +235,7 @@ ABIPassingInformation X86Classifier::Classify(Compiler* comp, static const regNumberSmall WinX64IntArgRegs[] = {REG_RCX, REG_RDX, REG_R8, REG_R9}; static const regNumberSmall WinX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3}; -WinX64Classifier::WinX64Classifier() +WinX64Classifier::WinX64Classifier(const ClassifierInfo& info) : m_intRegs(WinX64IntArgRegs, ArrLen(WinX64IntArgRegs)), m_floatRegs(WinX64FloatArgRegs, ArrLen(WinX64FloatArgRegs)) { } @@ -247,7 +280,7 @@ static const regNumberSmall SysVX64IntArgRegs[] = {REG_EDI, REG_ESI, REG_EDX, static const regNumberSmall SysVX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7}; -SysVX64Classifier::SysVX64Classifier() +SysVX64Classifier::SysVX64Classifier(const ClassifierInfo& info) : m_intRegs(SysVX64IntArgRegs, ArrLen(SysVX64IntArgRegs)) , m_floatRegs(SysVX64FloatArgRegs, ArrLen(SysVX64FloatArgRegs)) { @@ -332,8 +365,10 @@ ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, static const regNumberSmall Arm64IntArgRegs[] = {REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, REG_R6, REG_R7}; static const regNumberSmall Arm64FloatArgRegs[] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, REG_V6, REG_V7}; -Arm64Classifier::Arm64Classifier() - : m_intRegs(Arm64IntArgRegs, ArrLen(Arm64IntArgRegs)), m_floatRegs(Arm64FloatArgRegs, ArrLen(Arm64FloatArgRegs)) +Arm64Classifier::Arm64Classifier(const ClassifierInfo& info) + : m_info(info) + , m_intRegs(Arm64IntArgRegs, ArrLen(Arm64IntArgRegs)) + , m_floatRegs(Arm64FloatArgRegs, ArrLen(Arm64FloatArgRegs)) { } @@ -342,13 +377,13 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, ClassLayout* structLayout, WellKnownArg wellKnownParam) { - if (wellKnownParam == WellKnownArg::RetBuffer) + if ((wellKnownParam == WellKnownArg::RetBuffer) && hasFixedRetBuffReg(m_info.CallConv)) { return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_ARG_RET_BUFF, 0, TARGET_POINTER_SIZE)); } - if (varTypeIsStruct(type) && !comp->info.compIsVarArgs) + if (varTypeIsStruct(type) && !m_info.IsVarArgs) { var_types hfaType = comp->GetHfaType(structLayout->GetClassHandle()); @@ -373,7 +408,7 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, m_stackArgSize = roundUp(m_stackArgSize, alignment); info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, structLayout->GetSize())); - m_stackArgSize += structLayout->GetSize(); + m_stackArgSize += roundUp(structLayout->GetSize(), TARGET_POINTER_SIZE); // After passing any float value on the stack, we should not enregister more float values. m_floatRegs.Clear(); } @@ -383,28 +418,32 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, } unsigned slots; + unsigned passedSize; if (varTypeIsStruct(type)) { unsigned size = structLayout->GetSize(); if (size > 16) { slots = 1; // Passed by implicit byref + passedSize = TARGET_POINTER_SIZE; } else { slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + passedSize = size; } } else { assert(genTypeSize(type) <= TARGET_POINTER_SIZE); slots = 1; + passedSize = genTypeSize(type); } assert((slots == 1) || (slots == 2)); ABIPassingInformation info; - if (comp->info.compIsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) + if (m_info.IsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) { // On varargs we split structs between register and stack in this case. // Normally a struct that does not fit in registers will always be @@ -419,19 +458,26 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, } else { - RegisterQueue& regs = varTypeUsesFloatArgReg(type) ? m_floatRegs : m_intRegs; + RegisterQueue* regs = &m_intRegs; + + // In varargs methods (only supported on Windows) all arguments go in + // integer registers. + if (varTypeUsesFloatArgReg(type) && !m_info.IsVarArgs) + { + regs = &m_floatRegs; + } - if (regs.Count() >= slots) + if (regs->Count() >= slots) { info.NumSegments = slots; info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; unsigned slotSize = varTypeIsStruct(type) ? TARGET_POINTER_SIZE : genTypeSize(type); - info.Segments[0] = ABIPassingSegment::InRegister(regs.Dequeue(), 0, slotSize); + info.Segments[0] = ABIPassingSegment::InRegister(regs->Dequeue(), 0, slotSize); if (slots == 2) { assert(varTypeIsStruct(type)); unsigned tailSize = structLayout->GetSize() - slotSize; - info.Segments[1] = ABIPassingSegment::InRegister(regs.Dequeue(), slotSize, tailSize); + info.Segments[1] = ABIPassingSegment::InRegister(regs->Dequeue(), slotSize, tailSize); } } else @@ -456,14 +502,13 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); } - unsigned size = TypeSize(type, structLayout); - info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, size)); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, passedSize)); - m_stackArgSize += roundUp(size, alignment); + m_stackArgSize += roundUp(passedSize, alignment); // As soon as we pass something on stack we cannot go back and // enregister something else. - regs.Clear(); + regs->Clear(); } } diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index 9bfa0b94863116..200eb9c6c1a207 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -73,15 +73,20 @@ class RegisterQueue void Clear(); }; +struct ClassifierInfo +{ + CorInfoCallConvExtension CallConv; + bool IsVarArgs; + bool HasThis; +}; + class X86Classifier { RegisterQueue m_regs; unsigned m_stackArgSize = 0; public: - X86Classifier(const regNumberSmall* regs, unsigned int numRegs) : m_regs(regs, numRegs) - { - } + X86Classifier(const ClassifierInfo& info); ABIPassingInformation Classify(Compiler* comp, var_types type, @@ -96,7 +101,7 @@ class WinX64Classifier unsigned m_stackArgSize = 0; public: - WinX64Classifier(); + WinX64Classifier(const ClassifierInfo& info); ABIPassingInformation Classify(Compiler* comp, var_types type, @@ -111,7 +116,7 @@ class SysVX64Classifier unsigned m_stackArgSize = 0; public: - SysVX64Classifier(); + SysVX64Classifier(const ClassifierInfo& info); ABIPassingInformation Classify(Compiler* comp, var_types type, @@ -121,12 +126,13 @@ class SysVX64Classifier class Arm64Classifier { + const ClassifierInfo& m_info; RegisterQueue m_intRegs; RegisterQueue m_floatRegs; unsigned m_stackArgSize = 0; public: - Arm64Classifier(); + Arm64Classifier(const ClassifierInfo& info); ABIPassingInformation Classify(Compiler* comp, var_types type, @@ -150,7 +156,8 @@ class SwiftABIClassifier PlatformClassifier m_classifier; public: - SwiftABIClassifier() + SwiftABIClassifier(const ClassifierInfo& info) + : m_classifier(info) { } diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 432a435c98919d..b6ba06ba7a1986 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -2817,8 +2817,8 @@ inline var_types Compiler::mangleVarArgsType(var_types type) if (varTypeIsSIMD(type)) { - // Vectors also get passed in int registers. Use TYP_INT. - return TYP_INT; + // Vectors should be considered like passing a struct + return TYP_STRUCT; } } #endif // defined(TARGET_ARMARCH) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index a8374adbe0a22c..c6d87d6141a747 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1810,11 +1810,7 @@ regNumber CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension c case WellKnownArg::RetBuffer: if (hasFixedRetBuffReg(cc)) { - // Windows does not use fixed ret buff arg for instance calls, but does otherwise. - if (!TargetOS::IsWindows || !callConvIsInstanceMethodCallConv(cc)) - { - return theFixedRetBuffReg(cc); - } + return theFixedRetBuffReg(cc); } break; diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 2503642a42c44d..41607909e79398 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -521,6 +521,16 @@ void Compiler::lvaInitThisPtr(InitVarDscInfo* varDscInfo) lvaSetClass(varDscInfo->varNum, info.compClassHnd); } + varDsc->lvIsRegArg = 1; + noway_assert(varDscInfo->intRegArgNum == 0); + + varDsc->SetArgReg( + genMapRegArgNumToRegNum(varDscInfo->allocRegArg(TYP_INT), varDsc->TypeGet(), info.compCallConv)); +#if FEATURE_MULTIREG_ARGS + varDsc->SetOtherArgReg(REG_NA); +#endif + varDsc->lvOnFrame = true; // The final home for this incoming register might be our local stack frame + #ifdef DEBUG if (verbose) { @@ -733,6 +743,7 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un !varDscInfo->canEnreg(TYP_INT, cSlots)) // The end of the struct can't fit in a register { cSlotsToEnregister = 1; // Force the split + varDscInfo->stackArgSize += TARGET_POINTER_SIZE; } } } @@ -1655,50 +1666,22 @@ void Compiler::lvaClassifyParameterABI() return; } -#if defined(TARGET_X86) - switch (info.compCallConv) - { - case CorInfoCallConvExtension::Thiscall: - { - static const regNumberSmall thiscallRegs[] = {REG_ECX}; - X86Classifier classifier(thiscallRegs, ArrLen(thiscallRegs)); - lvaClassifyParameterABI(classifier); - break; - } - case CorInfoCallConvExtension::C: - case CorInfoCallConvExtension::Stdcall: - case CorInfoCallConvExtension::CMemberFunction: - case CorInfoCallConvExtension::StdcallMemberFunction: - { - X86Classifier classifier(nullptr, 0); - lvaClassifyParameterABI(classifier); - break; - } - default: - { - static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; - unsigned numRegs = ArrLen(regs); - if (info.compIsVarArgs) - { - // In varargs methods we only enregister the this pointer (if there is one). - numRegs = info.compThisArg == BAD_VAR_NUM ? 0 : 1; - } - X86Classifier classifier(regs, numRegs); - lvaClassifyParameterABI(classifier); - break; - } - } -#elif !defined(TARGET_ARM) + ClassifierInfo cInfo; + cInfo.CallConv = info.compCallConv; + cInfo.IsVarArgs = info.compIsVarArgs; + cInfo.HasThis = info.compThisArg != BAD_VAR_NUM; + #ifdef SWIFT_SUPPORT if (info.compCallConv == CorInfoCallConvExtension::Swift) { - SwiftABIClassifier classifier; + SwiftABIClassifier classifier(cInfo); lvaClassifyParameterABI(classifier); } else #endif +#ifndef TARGET_ARM { - PlatformClassifier classifier; + PlatformClassifier classifier(cInfo); lvaClassifyParameterABI(classifier); } #endif @@ -1707,6 +1690,51 @@ void Compiler::lvaClassifyParameterABI() { return; } + + for (unsigned lclNum = 0; lclNum < info.compArgsCount; lclNum++) + { + LclVarDsc* dsc = lvaGetDesc(lclNum); + const ABIPassingInformation& abiInfo = lvaParameterPassingInfo[lclNum]; + + assert(abiInfo.NumSegments > 0); + + unsigned numSegmentsToCompare = abiInfo.NumSegments; + if (dsc->lvIsHfa()) + { + assert(abiInfo.NumSegments >= 1); + // LclVarDsc only has one register set for HFAs + numSegmentsToCompare = 1; + } + + for (unsigned i = 0; i < numSegmentsToCompare; i++) + { + const ABIPassingSegment& expected = abiInfo.Segments[i]; + regNumber reg = REG_NA; + if (i == 0) + { + reg = dsc->GetArgReg(); + } +#if FEATURE_MULTIREG_ARGS + else if (i == 1) + { + reg = dsc->GetOtherArgReg(); + } +#endif + + if (expected.IsPassedOnStack()) + { + if (i == 0) + { + assert(reg == REG_STK); + assert((unsigned)dsc->GetStackOffset() == expected.GetStackOffset()); + } + } + else + { + assert(reg == expected.GetRegister()); + } + } + } } /***************************************************************************** diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 94a4810c561238..38fa7add49dfae 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -422,7 +422,7 @@ inline bool genIsValidDoubleReg(regNumber reg) inline bool hasFixedRetBuffReg(CorInfoCallConvExtension callConv) { #if defined(TARGET_ARM64) - return true; + return !TargetOS::IsWindows || !callConvIsInstanceMethodCallConv(callConv); #elif defined(TARGET_AMD64) && defined(SWIFT_SUPPORT) return callConv == CorInfoCallConvExtension::Swift; #else From af2fc728c62613de9287a28766c4b11b7c729d8e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Mar 2024 22:22:31 +0100 Subject: [PATCH 03/15] Run jit-format --- src/coreclr/jit/abi.cpp | 11 +++++------ src/coreclr/jit/abi.h | 13 ++++++------- src/coreclr/jit/lclvars.cpp | 8 ++++---- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index c3949071c4ecc5..447b0e6f3bc8b1 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -152,15 +152,14 @@ static unsigned TypeSize(var_types type, ClassLayout* structLayout) } #ifdef TARGET_X86 -X86Classifier::X86Classifier(const ClassifierInfo& info) - : m_regs(nullptr, 0) +X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) { switch (info.CallConv) { case CorInfoCallConvExtension::Thiscall: { static const regNumberSmall thiscallRegs[] = {REG_ECX}; - m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); + m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); break; } case CorInfoCallConvExtension::C: @@ -424,19 +423,19 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, unsigned size = structLayout->GetSize(); if (size > 16) { - slots = 1; // Passed by implicit byref + slots = 1; // Passed by implicit byref passedSize = TARGET_POINTER_SIZE; } else { - slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; passedSize = size; } } else { assert(genTypeSize(type) <= TARGET_POINTER_SIZE); - slots = 1; + slots = 1; passedSize = genTypeSize(type); } diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index 200eb9c6c1a207..ef5e966f66fbbc 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -76,8 +76,8 @@ class RegisterQueue struct ClassifierInfo { CorInfoCallConvExtension CallConv; - bool IsVarArgs; - bool HasThis; + bool IsVarArgs; + bool HasThis; }; class X86Classifier @@ -127,9 +127,9 @@ class SysVX64Classifier class Arm64Classifier { const ClassifierInfo& m_info; - RegisterQueue m_intRegs; - RegisterQueue m_floatRegs; - unsigned m_stackArgSize = 0; + RegisterQueue m_intRegs; + RegisterQueue m_floatRegs; + unsigned m_stackArgSize = 0; public: Arm64Classifier(const ClassifierInfo& info); @@ -156,8 +156,7 @@ class SwiftABIClassifier PlatformClassifier m_classifier; public: - SwiftABIClassifier(const ClassifierInfo& info) - : m_classifier(info) + SwiftABIClassifier(const ClassifierInfo& info) : m_classifier(info) { } diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 41607909e79398..8dec5bbcb2b303 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1667,9 +1667,9 @@ void Compiler::lvaClassifyParameterABI() } ClassifierInfo cInfo; - cInfo.CallConv = info.compCallConv; + cInfo.CallConv = info.compCallConv; cInfo.IsVarArgs = info.compIsVarArgs; - cInfo.HasThis = info.compThisArg != BAD_VAR_NUM; + cInfo.HasThis = info.compThisArg != BAD_VAR_NUM; #ifdef SWIFT_SUPPORT if (info.compCallConv == CorInfoCallConvExtension::Swift) @@ -1693,7 +1693,7 @@ void Compiler::lvaClassifyParameterABI() for (unsigned lclNum = 0; lclNum < info.compArgsCount; lclNum++) { - LclVarDsc* dsc = lvaGetDesc(lclNum); + LclVarDsc* dsc = lvaGetDesc(lclNum); const ABIPassingInformation& abiInfo = lvaParameterPassingInfo[lclNum]; assert(abiInfo.NumSegments > 0); @@ -1709,7 +1709,7 @@ void Compiler::lvaClassifyParameterABI() for (unsigned i = 0; i < numSegmentsToCompare; i++) { const ABIPassingSegment& expected = abiInfo.Segments[i]; - regNumber reg = REG_NA; + regNumber reg = REG_NA; if (i == 0) { reg = dsc->GetArgReg(); From 1d09c9f5985050902c0a19fcb1a782fc1df084c3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 09:53:51 +0100 Subject: [PATCH 04/15] Comments --- src/coreclr/jit/abi.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index 447b0e6f3bc8b1..b54f139853154d 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -382,6 +382,8 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, TARGET_POINTER_SIZE)); } + // First handle HFA/HVAs. These are allowed to be passed in more registers + // than other structures. if (varTypeIsStruct(type) && !m_info.IsVarArgs) { var_types hfaType = comp->GetHfaType(structLayout->GetClassHandle()); @@ -444,9 +446,9 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, ABIPassingInformation info; if (m_info.IsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) { - // On varargs we split structs between register and stack in this case. - // Normally a struct that does not fit in registers will always be - // passed on stack. + // For varargs we split structs between register and stack in this + // case. Normally a struct that does not fit in registers will always + // be passed on stack. assert(compFeatureArgSplit()); info.NumSegments = 2; info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]; @@ -459,7 +461,7 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, { RegisterQueue* regs = &m_intRegs; - // In varargs methods (only supported on Windows) all arguments go in + // In varargs methods (only supported on Windows) all parameters go in // integer registers. if (varTypeUsesFloatArgReg(type) && !m_info.IsVarArgs) { From dcccedd293dcf93f889ad597ea7cc1030bf98aa4 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 11:55:38 +0100 Subject: [PATCH 05/15] Fix varargs retbuf case --- src/coreclr/jit/abi.cpp | 4 ++-- src/coreclr/jit/abi.h | 1 + src/coreclr/jit/lclvars.cpp | 14 +++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index b54f139853154d..598dd55c04859a 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -175,8 +175,8 @@ X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) unsigned numRegs = ArrLen(regs); if (info.IsVarArgs) { - // In varargs methods we only enregister the this pointer (if there is one). - numRegs = info.HasThis ? 1 : 0; + // In varargs methods we only enregister the this pointer or retbuff. + numRegs = info.HasThis || info.HasRetBuff ? 1 : 0; } m_regs = RegisterQueue(regs, numRegs); break; diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index ef5e966f66fbbc..c7027057ea2444 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -78,6 +78,7 @@ struct ClassifierInfo CorInfoCallConvExtension CallConv; bool IsVarArgs; bool HasThis; + bool HasRetBuff; }; class X86Classifier diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 8dec5bbcb2b303..38d760eb05d886 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1667,9 +1667,10 @@ void Compiler::lvaClassifyParameterABI() } ClassifierInfo cInfo; - cInfo.CallConv = info.compCallConv; - cInfo.IsVarArgs = info.compIsVarArgs; - cInfo.HasThis = info.compThisArg != BAD_VAR_NUM; + cInfo.CallConv = info.compCallConv; + cInfo.IsVarArgs = info.compIsVarArgs; + cInfo.HasThis = info.compThisArg != BAD_VAR_NUM; + cInfo.HasRetBuff = info.compRetBuffArg != BAD_VAR_NUM; #ifdef SWIFT_SUPPORT if (info.compCallConv == CorInfoCallConvExtension::Swift) @@ -1726,7 +1727,14 @@ void Compiler::lvaClassifyParameterABI() if (i == 0) { assert(reg == REG_STK); + +// On x86, varargs methods access stack args off of a base pointer, and the +// first stack arg is not considered to be at offset 0. +// TODO-Cleanup: Unify things so that x86 is consistent with other platforms +// here and change fgMorphExpandStackArgForVarArgs to account for that. +#ifndef TARGET_X86 assert((unsigned)dsc->GetStackOffset() == expected.GetStackOffset()); +#endif } } else From 7fe878bf47d1e100f936de52d8cc50d1bd20df25 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:18:57 +0100 Subject: [PATCH 06/15] Fix SysV --- src/coreclr/jit/abi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index 598dd55c04859a..38e6ca70e5d12e 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -318,7 +318,7 @@ ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, } } - canEnreg = (m_intRegs.Count() <= intRegCount) && (m_floatRegs.Count() <= floatRegCount); + canEnreg = (intRegCount <= m_intRegs.Count()) && (floatRegCount <= m_floatRegs.Count()); } } else From 9297865298c4af7a242c9e3136a8dc442f8b2517 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:33:21 +0100 Subject: [PATCH 07/15] Add function headers --- src/coreclr/jit/abi.cpp | 139 +++++++++++++++++++++++++++++++++++- src/coreclr/jit/abi.h | 8 +-- src/coreclr/jit/lclvars.cpp | 16 +++++ 3 files changed, 158 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index 38e6ca70e5d12e..fc552e10d66e0c 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -121,6 +121,17 @@ bool ABIPassingInformation::IsSplitAcrossRegistersAndStack() const return anyReg && anyStack; } +//----------------------------------------------------------------------------- +// FromSegment: +// Create ABIPassingInformation from a single segment. +// +// Parameters: +// comp - Compiler instance +// segment - The single segment that represents the passing information +// +// Return Value: +// An instance of ABIPassingInformation. +// ABIPassingInformation ABIPassingInformation::FromSegment(Compiler* comp, const ABIPassingSegment& segment) { ABIPassingInformation info; @@ -129,29 +140,65 @@ ABIPassingInformation ABIPassingInformation::FromSegment(Compiler* comp, const A return info; } +//----------------------------------------------------------------------------- +// RegisterQueue::Dequeue: +// Dequeue a register from the queue. +// +// Return Value: +// The dequeued register. +// regNumber RegisterQueue::Dequeue() { assert(Count() > 0); return static_cast(m_regs[m_index++]); } +//----------------------------------------------------------------------------- +// RegisterQueue::Peek: +// Peek at the head of the queue. +// +// Return Value: +// The head register in the queue. +// regNumber RegisterQueue::Peek() { assert(Count() > 0); return static_cast(m_regs[m_index]); } +//----------------------------------------------------------------------------- +// RegisterQueue::Clear: +// Clear the register queue. +// void RegisterQueue::Clear() { m_index = m_numRegs; } +//----------------------------------------------------------------------------- +// TypeSize: +// Get the size of a JIT type or struct. +// +// Parameters: +// type - The type +// structLayout - The layout to get the size if type is TYP_STRUCT +// +// Returns: +// The type size. +// static unsigned TypeSize(var_types type, ClassLayout* structLayout) { return type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); } #ifdef TARGET_X86 +//----------------------------------------------------------------------------- +// X86Classifier: +// Construct a new instance of the x86 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) { switch (info.CallConv) @@ -184,6 +231,20 @@ X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) } } +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the x86 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// ABIPassingInformation X86Classifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, @@ -234,11 +295,32 @@ ABIPassingInformation X86Classifier::Classify(Compiler* comp, static const regNumberSmall WinX64IntArgRegs[] = {REG_RCX, REG_RDX, REG_R8, REG_R9}; static const regNumberSmall WinX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3}; +//----------------------------------------------------------------------------- +// WinX64Classifier: +// Construct a new instance of the Windows x64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// WinX64Classifier::WinX64Classifier(const ClassifierInfo& info) : m_intRegs(WinX64IntArgRegs, ArrLen(WinX64IntArgRegs)), m_floatRegs(WinX64FloatArgRegs, ArrLen(WinX64FloatArgRegs)) { } +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the Windows x64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// ABIPassingInformation WinX64Classifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, @@ -278,13 +360,33 @@ ABIPassingInformation WinX64Classifier::Classify(Compiler* comp, static const regNumberSmall SysVX64IntArgRegs[] = {REG_EDI, REG_ESI, REG_EDX, REG_ECX, REG_R8, REG_R9}; static const regNumberSmall SysVX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7}; - +//----------------------------------------------------------------------------- +// SysVX64Classifier: +// Construct a new instance of the SysV x64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// SysVX64Classifier::SysVX64Classifier(const ClassifierInfo& info) : m_intRegs(SysVX64IntArgRegs, ArrLen(SysVX64IntArgRegs)) , m_floatRegs(SysVX64FloatArgRegs, ArrLen(SysVX64FloatArgRegs)) { } +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the SysV x64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, @@ -364,6 +466,13 @@ ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, static const regNumberSmall Arm64IntArgRegs[] = {REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, REG_R6, REG_R7}; static const regNumberSmall Arm64FloatArgRegs[] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, REG_V6, REG_V7}; +//----------------------------------------------------------------------------- +// Arm64Classifier: +// Construct a new instance of the ARM64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// Arm64Classifier::Arm64Classifier(const ClassifierInfo& info) : m_info(info) , m_intRegs(Arm64IntArgRegs, ArrLen(Arm64IntArgRegs)) @@ -371,6 +480,20 @@ Arm64Classifier::Arm64Classifier(const ClassifierInfo& info) { } +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the ARM64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, @@ -518,6 +641,20 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, #endif #ifdef SWIFT_SUPPORT +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the Swift ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// ABIPassingInformation SwiftABIClassifier::Classify(Compiler* comp, var_types type, ClassLayout* structLayout, diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index c7027057ea2444..bfc76236501a9c 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -75,10 +75,10 @@ class RegisterQueue struct ClassifierInfo { - CorInfoCallConvExtension CallConv; - bool IsVarArgs; - bool HasThis; - bool HasRetBuff; + CorInfoCallConvExtension CallConv = CorInfoCallConvExtension::Managed; + bool IsVarArgs = false; + bool HasThis = false; + bool HasRetBuff = false; }; class X86Classifier diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 38d760eb05d886..63e94b11280fcb 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1633,6 +1633,16 @@ void Compiler::lvaInitVarDsc(LclVarDsc* varDsc, #endif // FEATURE_MULTIREG_ARGS } +//----------------------------------------------------------------------------- +// lvaClassifyParameterABI: +// Classify the ABI information for all parameters. +// +// Type parameters: +// Classifier - The type of classifier to use. +// +// Parameters: +// classifier - The classifier to use +// template void Compiler::lvaClassifyParameterABI(Classifier& classifier) { @@ -1659,6 +1669,10 @@ void Compiler::lvaClassifyParameterABI(Classifier& classifier) } } +//----------------------------------------------------------------------------- +// lvaClassifyParameterABI: +// Classify the ABI information for all parameters. +// void Compiler::lvaClassifyParameterABI() { if (info.compArgsCount == 0) @@ -1687,6 +1701,7 @@ void Compiler::lvaClassifyParameterABI() } #endif +#ifdef DEBUG if (lvaParameterPassingInfo == nullptr) { return; @@ -1743,6 +1758,7 @@ void Compiler::lvaClassifyParameterABI() } } } +#endif } /***************************************************************************** From 25e2c49282cbd6bcd26d26aa9d2e74930817b88d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:35:15 +0100 Subject: [PATCH 08/15] Nit --- src/coreclr/jit/abi.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index fc552e10d66e0c..f553968c699fad 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -269,6 +269,7 @@ ABIPassingInformation X86Classifier::Classify(Compiler* comp, break; case TYP_STRUCT: canEnreg = comp->isTrivialPointerSizedStruct(structLayout->GetClassHandle()); + break; default: break; } From 99540622675f177083a0a358740ddb20df1db347 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:40:38 +0100 Subject: [PATCH 09/15] Fix macOS arm64 ABI bug --- src/coreclr/jit/abi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index f553968c699fad..fcf01d132bd3ff 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -533,7 +533,7 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, m_stackArgSize = roundUp(m_stackArgSize, alignment); info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, structLayout->GetSize())); - m_stackArgSize += roundUp(structLayout->GetSize(), TARGET_POINTER_SIZE); + m_stackArgSize += roundUp(structLayout->GetSize(), alignment); // After passing any float value on the stack, we should not enregister more float values. m_floatRegs.Clear(); } From 5fab654d85e8e7eac5bf2b22a41da2be9e00117b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:54:00 +0100 Subject: [PATCH 10/15] Preemptively fix build --- src/coreclr/jit/lclvars.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 63e94b11280fcb..9d96d8643ce0cb 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1694,7 +1694,7 @@ void Compiler::lvaClassifyParameterABI() } else #endif -#ifndef TARGET_ARM +#if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) { PlatformClassifier classifier(cInfo); lvaClassifyParameterABI(classifier); From 3f1c738a8709193e1eb0a19fa047442588097def Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 12:56:42 +0100 Subject: [PATCH 11/15] More build fixes --- src/coreclr/jit/lclvars.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 9d96d8643ce0cb..8e85840909c95f 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1083,15 +1083,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un varDsc->SetOtherArgReg( genMapRegArgNumToRegNum(secondAllocatedRegArgNum, argRegTypeInStruct2, info.compCallConv)); varDsc->lvIs4Field2 = (genTypeSize(argRegTypeInStruct2) == 4) ? 1 : 0; - - abiInfo->NumSegments = 2; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[2]; - abiInfo->Segments[0] = - ABIPassingSegment::InRegister(varDsc->GetArgReg(), 0, genTypeSize(argRegTypeInStruct1)); - abiInfo->Segments[1] = ABIPassingSegment::InRegister(varDsc->GetOtherArgReg(), - roundUp(genTypeSize(argRegTypeInStruct1), - genTypeSize(argRegTypeInStruct2)), - genTypeSize(argRegTypeInStruct2)); } else if (cSlots > 1) { @@ -1116,14 +1107,6 @@ void Compiler::lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, un } assert(cSlots <= 2); - abiInfo->NumSegments = cSlots; - abiInfo->Segments = new (this, CMK_LvaTable) ABIPassingSegment[cSlots]; - for (unsigned i = 0; i < cSlots; i++) - { - regNumber reg = i == 0 ? varDsc->GetArgReg() : varDsc->GetOtherArgReg(); - abiInfo->Segments[i] = - ABIPassingSegment::InRegister(reg, TARGET_POINTER_SIZE * i, argRegTypeInStruct1); - } } } #else // ARM32 From b8d9334c43eab7355600969f1c66d70846a40ae5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 13:21:53 +0100 Subject: [PATCH 12/15] Further fix build --- src/coreclr/jit/lclvars.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 8e85840909c95f..978fff461e146a 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -243,8 +243,6 @@ void Compiler::lvaInitTypeRef() new (&lvaTable[i], jitstd::placement_t()) LclVarDsc(); // call the constructor. } - lvaParameterPassingInfo = new (this, CMK_LvaTable) ABIPassingInformation[max(info.compArgsCount, 1)]{}; - //------------------------------------------------------------------------- // Count the arguments and initialize the respective lvaTable[] entries // From f12a9d74e55fbabf39c8ee07fa389cb5bd27f4ca Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 13:27:42 +0100 Subject: [PATCH 13/15] Move classifiers into target specific files --- src/coreclr/jit/abi.cpp | 466 -------------------------------- src/coreclr/jit/abi.h | 8 +- src/coreclr/jit/targetamd64.cpp | 164 +++++++++++ src/coreclr/jit/targetarm64.cpp | 171 ++++++++++++ src/coreclr/jit/targetx86.cpp | 99 +++++++ 5 files changed, 438 insertions(+), 470 deletions(-) diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index fcf01d132bd3ff..c52c13273c63cc 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -175,472 +175,6 @@ void RegisterQueue::Clear() m_index = m_numRegs; } -//----------------------------------------------------------------------------- -// TypeSize: -// Get the size of a JIT type or struct. -// -// Parameters: -// type - The type -// structLayout - The layout to get the size if type is TYP_STRUCT -// -// Returns: -// The type size. -// -static unsigned TypeSize(var_types type, ClassLayout* structLayout) -{ - return type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); -} - -#ifdef TARGET_X86 -//----------------------------------------------------------------------------- -// X86Classifier: -// Construct a new instance of the x86 ABI classifier. -// -// Parameters: -// info - Info about the method being classified. -// -X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) -{ - switch (info.CallConv) - { - case CorInfoCallConvExtension::Thiscall: - { - static const regNumberSmall thiscallRegs[] = {REG_ECX}; - m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); - break; - } - case CorInfoCallConvExtension::C: - case CorInfoCallConvExtension::Stdcall: - case CorInfoCallConvExtension::CMemberFunction: - case CorInfoCallConvExtension::StdcallMemberFunction: - { - break; - } - default: - { - static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; - unsigned numRegs = ArrLen(regs); - if (info.IsVarArgs) - { - // In varargs methods we only enregister the this pointer or retbuff. - numRegs = info.HasThis || info.HasRetBuff ? 1 : 0; - } - m_regs = RegisterQueue(regs, numRegs); - break; - } - } -} - -//----------------------------------------------------------------------------- -// Classify: -// Classify a parameter for the x86 ABI. -// -// Parameters: -// comp - Compiler instance -// type - The type of the parameter -// structLayout - The layout of the struct. Expected to be non-null if -// varTypeIsStruct(type) is true. -// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) -// -// Returns: -// Classification information for the parameter. -// -ABIPassingInformation X86Classifier::Classify(Compiler* comp, - var_types type, - ClassLayout* structLayout, - WellKnownArg wellKnownParam) -{ - unsigned size = TypeSize(type, structLayout); - unsigned numSlots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; - - bool canEnreg = false; - if (m_regs.Count() >= numSlots) - { - switch (type) - { - case TYP_BYTE: - case TYP_UBYTE: - case TYP_SHORT: - case TYP_USHORT: - case TYP_INT: - case TYP_REF: - case TYP_BYREF: - canEnreg = true; - break; - case TYP_STRUCT: - canEnreg = comp->isTrivialPointerSizedStruct(structLayout->GetClassHandle()); - break; - default: - break; - } - } - - ABIPassingSegment segment; - if (canEnreg) - { - assert(numSlots == 1); - segment = ABIPassingSegment::InRegister(m_regs.Dequeue(), 0, size); - } - else - { - assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); - segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, size); - m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); - } - - return ABIPassingInformation::FromSegment(comp, segment); -} -#endif - -#ifdef WINDOWS_AMD64_ABI -static const regNumberSmall WinX64IntArgRegs[] = {REG_RCX, REG_RDX, REG_R8, REG_R9}; -static const regNumberSmall WinX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3}; - -//----------------------------------------------------------------------------- -// WinX64Classifier: -// Construct a new instance of the Windows x64 ABI classifier. -// -// Parameters: -// info - Info about the method being classified. -// -WinX64Classifier::WinX64Classifier(const ClassifierInfo& info) - : m_intRegs(WinX64IntArgRegs, ArrLen(WinX64IntArgRegs)), m_floatRegs(WinX64FloatArgRegs, ArrLen(WinX64FloatArgRegs)) -{ -} - -//----------------------------------------------------------------------------- -// Classify: -// Classify a parameter for the Windows x64 ABI. -// -// Parameters: -// comp - Compiler instance -// type - The type of the parameter -// structLayout - The layout of the struct. Expected to be non-null if -// varTypeIsStruct(type) is true. -// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) -// -// Returns: -// Classification information for the parameter. -// -ABIPassingInformation WinX64Classifier::Classify(Compiler* comp, - var_types type, - ClassLayout* structLayout, - WellKnownArg wellKnownParam) -{ - // On windows-x64 ABI all parameters take exactly 1 stack slot (structs - // that do not fit are passed implicitly by reference). Passing a parameter - // in an int register also consumes the corresponding float register and - // vice versa. - assert(m_intRegs.Count() == m_floatRegs.Count()); - - unsigned typeSize = TypeSize(type, structLayout); - if ((typeSize > TARGET_POINTER_SIZE) || !isPow2(typeSize)) - { - typeSize = TARGET_POINTER_SIZE; // Passed by implicit byref - } - - ABIPassingSegment segment; - if (m_intRegs.Count() > 0) - { - regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Peek() : m_intRegs.Peek(); - segment = ABIPassingSegment::InRegister(reg, 0, typeSize); - m_intRegs.Dequeue(); - m_floatRegs.Dequeue(); - } - else - { - segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, typeSize); - m_stackArgSize += TARGET_POINTER_SIZE; - } - - return ABIPassingInformation::FromSegment(comp, segment); -} -#endif - -#ifdef UNIX_AMD64_ABI -static const regNumberSmall SysVX64IntArgRegs[] = {REG_EDI, REG_ESI, REG_EDX, REG_ECX, REG_R8, REG_R9}; -static const regNumberSmall SysVX64FloatArgRegs[] = {REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, - REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7}; -//----------------------------------------------------------------------------- -// SysVX64Classifier: -// Construct a new instance of the SysV x64 ABI classifier. -// -// Parameters: -// info - Info about the method being classified. -// -SysVX64Classifier::SysVX64Classifier(const ClassifierInfo& info) - : m_intRegs(SysVX64IntArgRegs, ArrLen(SysVX64IntArgRegs)) - , m_floatRegs(SysVX64FloatArgRegs, ArrLen(SysVX64FloatArgRegs)) -{ -} - -//----------------------------------------------------------------------------- -// Classify: -// Classify a parameter for the SysV x64 ABI. -// -// Parameters: -// comp - Compiler instance -// type - The type of the parameter -// structLayout - The layout of the struct. Expected to be non-null if -// varTypeIsStruct(type) is true. -// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) -// -// Returns: -// Classification information for the parameter. -// -ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, - var_types type, - ClassLayout* structLayout, - WellKnownArg wellKnownParam) -{ - bool canEnreg = false; - SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; - if (varTypeIsStruct(type)) - { - comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(structLayout->GetClassHandle(), &structDesc); - - if (structDesc.passedInRegisters) - { - unsigned intRegCount = 0; - unsigned floatRegCount = 0; - - for (unsigned int i = 0; i < structDesc.eightByteCount; i++) - { - if (structDesc.IsIntegralSlot(i)) - { - intRegCount++; - } - else if (structDesc.IsSseSlot(i)) - { - floatRegCount++; - } - else - { - assert(!"Invalid eightbyte classification type."); - break; - } - } - - canEnreg = (intRegCount <= m_intRegs.Count()) && (floatRegCount <= m_floatRegs.Count()); - } - } - else - { - unsigned availRegs = varTypeUsesFloatArgReg(type) ? m_floatRegs.Count() : m_intRegs.Count(); - canEnreg = availRegs > 0; - } - - ABIPassingInformation info; - if (canEnreg) - { - if (varTypeIsStruct(type)) - { - info.NumSegments = structDesc.eightByteCount; - info.Segments = new (comp, CMK_ABI) ABIPassingSegment[structDesc.eightByteCount]; - - for (unsigned i = 0; i < structDesc.eightByteCount; i++) - { - regNumber reg = structDesc.IsIntegralSlot(i) ? m_intRegs.Dequeue() : m_floatRegs.Dequeue(); - info.Segments[i] = - ABIPassingSegment::InRegister(reg, structDesc.eightByteOffsets[i], structDesc.eightByteSizes[i]); - } - } - else - { - regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Dequeue() : m_intRegs.Dequeue(); - info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(reg, 0, genTypeSize(type))); - } - } - else - { - assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); - unsigned size = TypeSize(type, structLayout); - info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, size)); - m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); - } - - return info; -} -#endif - -#ifdef TARGET_ARM64 -static const regNumberSmall Arm64IntArgRegs[] = {REG_R0, REG_R1, REG_R2, REG_R3, REG_R4, REG_R5, REG_R6, REG_R7}; -static const regNumberSmall Arm64FloatArgRegs[] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, REG_V6, REG_V7}; - -//----------------------------------------------------------------------------- -// Arm64Classifier: -// Construct a new instance of the ARM64 ABI classifier. -// -// Parameters: -// info - Info about the method being classified. -// -Arm64Classifier::Arm64Classifier(const ClassifierInfo& info) - : m_info(info) - , m_intRegs(Arm64IntArgRegs, ArrLen(Arm64IntArgRegs)) - , m_floatRegs(Arm64FloatArgRegs, ArrLen(Arm64FloatArgRegs)) -{ -} - -//----------------------------------------------------------------------------- -// Classify: -// Classify a parameter for the ARM64 ABI. -// -// Parameters: -// comp - Compiler instance -// type - The type of the parameter -// structLayout - The layout of the struct. Expected to be non-null if -// varTypeIsStruct(type) is true. -// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) -// -// Returns: -// Classification information for the parameter. -// -ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, - var_types type, - ClassLayout* structLayout, - WellKnownArg wellKnownParam) -{ - if ((wellKnownParam == WellKnownArg::RetBuffer) && hasFixedRetBuffReg(m_info.CallConv)) - { - return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_ARG_RET_BUFF, 0, - TARGET_POINTER_SIZE)); - } - - // First handle HFA/HVAs. These are allowed to be passed in more registers - // than other structures. - if (varTypeIsStruct(type) && !m_info.IsVarArgs) - { - var_types hfaType = comp->GetHfaType(structLayout->GetClassHandle()); - - if (hfaType != TYP_UNDEF) - { - unsigned elemSize = genTypeSize(hfaType); - unsigned slots = structLayout->GetSize() / elemSize; - ABIPassingInformation info; - if (m_floatRegs.Count() >= slots) - { - info.NumSegments = slots; - info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; - - for (unsigned i = 0; i < slots; i++) - { - info.Segments[i] = ABIPassingSegment::InRegister(m_floatRegs.Dequeue(), i * elemSize, elemSize); - } - } - else - { - unsigned alignment = compAppleArm64Abi() ? elemSize : TARGET_POINTER_SIZE; - m_stackArgSize = roundUp(m_stackArgSize, alignment); - info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, - structLayout->GetSize())); - m_stackArgSize += roundUp(structLayout->GetSize(), alignment); - // After passing any float value on the stack, we should not enregister more float values. - m_floatRegs.Clear(); - } - - return info; - } - } - - unsigned slots; - unsigned passedSize; - if (varTypeIsStruct(type)) - { - unsigned size = structLayout->GetSize(); - if (size > 16) - { - slots = 1; // Passed by implicit byref - passedSize = TARGET_POINTER_SIZE; - } - else - { - slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; - passedSize = size; - } - } - else - { - assert(genTypeSize(type) <= TARGET_POINTER_SIZE); - slots = 1; - passedSize = genTypeSize(type); - } - - assert((slots == 1) || (slots == 2)); - - ABIPassingInformation info; - if (m_info.IsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) - { - // For varargs we split structs between register and stack in this - // case. Normally a struct that does not fit in registers will always - // be passed on stack. - assert(compFeatureArgSplit()); - info.NumSegments = 2; - info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]; - info.Segments[0] = ABIPassingSegment::InRegister(m_intRegs.Dequeue(), 0, TARGET_POINTER_SIZE); - info.Segments[1] = ABIPassingSegment::OnStack(m_stackArgSize, TARGET_POINTER_SIZE, - structLayout->GetSize() - TARGET_POINTER_SIZE); - m_stackArgSize += TARGET_POINTER_SIZE; - } - else - { - RegisterQueue* regs = &m_intRegs; - - // In varargs methods (only supported on Windows) all parameters go in - // integer registers. - if (varTypeUsesFloatArgReg(type) && !m_info.IsVarArgs) - { - regs = &m_floatRegs; - } - - if (regs->Count() >= slots) - { - info.NumSegments = slots; - info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; - unsigned slotSize = varTypeIsStruct(type) ? TARGET_POINTER_SIZE : genTypeSize(type); - info.Segments[0] = ABIPassingSegment::InRegister(regs->Dequeue(), 0, slotSize); - if (slots == 2) - { - assert(varTypeIsStruct(type)); - unsigned tailSize = structLayout->GetSize() - slotSize; - info.Segments[1] = ABIPassingSegment::InRegister(regs->Dequeue(), slotSize, tailSize); - } - } - else - { - unsigned alignment; - if (compAppleArm64Abi()) - { - if (varTypeIsStruct(type)) - { - alignment = TARGET_POINTER_SIZE; - } - else - { - alignment = genTypeSize(type); - } - - m_stackArgSize = roundUp(m_stackArgSize, alignment); - } - else - { - alignment = TARGET_POINTER_SIZE; - assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); - } - - info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, passedSize)); - - m_stackArgSize += roundUp(passedSize, alignment); - - // As soon as we pass something on stack we cannot go back and - // enregister something else. - regs->Clear(); - } - } - - return info; -} -#endif - #ifdef SWIFT_SUPPORT //----------------------------------------------------------------------------- // Classify: diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index bfc76236501a9c..f6303899b2509a 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -54,12 +54,12 @@ struct ABIPassingInformation class RegisterQueue { - const regNumberSmall* m_regs; - unsigned int m_numRegs; - unsigned int m_index = 0; + const regNumber* m_regs; + unsigned int m_numRegs; + unsigned int m_index = 0; public: - RegisterQueue(const regNumberSmall* regs, unsigned int numRegs) : m_regs(regs), m_numRegs(numRegs) + RegisterQueue(const regNumber* regs, unsigned int numRegs) : m_regs(regs), m_numRegs(numRegs) { } diff --git a/src/coreclr/jit/targetamd64.cpp b/src/coreclr/jit/targetamd64.cpp index 4ac48cb229fbed..576a152d4b8161 100644 --- a/src/coreclr/jit/targetamd64.cpp +++ b/src/coreclr/jit/targetamd64.cpp @@ -30,4 +30,168 @@ const regMaskTP fltArgMasks[] = { RBM_XMM0, RBM_XMM1, RBM_XMM2, RBM_XMM3 }; #endif // !UNIX_AMD64_ABI // clang-format on +#ifdef UNIX_AMD64_ABI +//----------------------------------------------------------------------------- +// SysVX64Classifier: +// Construct a new instance of the SysV x64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// +SysVX64Classifier::SysVX64Classifier(const ClassifierInfo& info) + : m_intRegs(intArgRegs, ArrLen(intArgRegs)), m_floatRegs(fltArgRegs, ArrLen(fltArgRegs)) +{ +} + +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the SysV x64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// +ABIPassingInformation SysVX64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + bool canEnreg = false; + SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; + if (varTypeIsStruct(type)) + { + comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(structLayout->GetClassHandle(), &structDesc); + + if (structDesc.passedInRegisters) + { + unsigned intRegCount = 0; + unsigned floatRegCount = 0; + + for (unsigned int i = 0; i < structDesc.eightByteCount; i++) + { + if (structDesc.IsIntegralSlot(i)) + { + intRegCount++; + } + else if (structDesc.IsSseSlot(i)) + { + floatRegCount++; + } + else + { + assert(!"Invalid eightbyte classification type."); + break; + } + } + + canEnreg = (intRegCount <= m_intRegs.Count()) && (floatRegCount <= m_floatRegs.Count()); + } + } + else + { + unsigned availRegs = varTypeUsesFloatArgReg(type) ? m_floatRegs.Count() : m_intRegs.Count(); + canEnreg = availRegs > 0; + } + + ABIPassingInformation info; + if (canEnreg) + { + if (varTypeIsStruct(type)) + { + info.NumSegments = structDesc.eightByteCount; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[structDesc.eightByteCount]; + + for (unsigned i = 0; i < structDesc.eightByteCount; i++) + { + regNumber reg = structDesc.IsIntegralSlot(i) ? m_intRegs.Dequeue() : m_floatRegs.Dequeue(); + info.Segments[i] = + ABIPassingSegment::InRegister(reg, structDesc.eightByteOffsets[i], structDesc.eightByteSizes[i]); + } + } + else + { + regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Dequeue() : m_intRegs.Dequeue(); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(reg, 0, genTypeSize(type))); + } + } + else + { + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + unsigned size = type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, size)); + m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); + } + + return info; +} + +#else // !UNIX_AMD64_ABI + +//----------------------------------------------------------------------------- +// WinX64Classifier: +// Construct a new instance of the Windows x64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// +WinX64Classifier::WinX64Classifier(const ClassifierInfo& info) + : m_intRegs(intArgRegs, ArrLen(intArgRegs)), m_floatRegs(fltArgRegs, ArrLen(fltArgRegs)) +{ +} + +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the Windows x64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// +ABIPassingInformation WinX64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + // On windows-x64 ABI all parameters take exactly 1 stack slot (structs + // that do not fit are passed implicitly by reference). Passing a parameter + // in an int register also consumes the corresponding float register and + // vice versa. + assert(m_intRegs.Count() == m_floatRegs.Count()); + + unsigned typeSize = type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); + if ((typeSize > TARGET_POINTER_SIZE) || !isPow2(typeSize)) + { + typeSize = TARGET_POINTER_SIZE; // Passed by implicit byref + } + + ABIPassingSegment segment; + if (m_intRegs.Count() > 0) + { + regNumber reg = varTypeUsesFloatArgReg(type) ? m_floatRegs.Peek() : m_intRegs.Peek(); + segment = ABIPassingSegment::InRegister(reg, 0, typeSize); + m_intRegs.Dequeue(); + m_floatRegs.Dequeue(); + } + else + { + segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, typeSize); + m_stackArgSize += TARGET_POINTER_SIZE; + } + + return ABIPassingInformation::FromSegment(comp, segment); +} +#endif + #endif // TARGET_AMD64 diff --git a/src/coreclr/jit/targetarm64.cpp b/src/coreclr/jit/targetarm64.cpp index dcec1db6c5229f..befe688247c09f 100644 --- a/src/coreclr/jit/targetarm64.cpp +++ b/src/coreclr/jit/targetarm64.cpp @@ -24,4 +24,175 @@ const regNumber fltArgRegs [] = {REG_V0, REG_V1, REG_V2, REG_V3, REG_V4, REG_V5, const regMaskTP fltArgMasks[] = {RBM_V0, RBM_V1, RBM_V2, RBM_V3, RBM_V4, RBM_V5, RBM_V6, RBM_V7 }; // clang-format on +//----------------------------------------------------------------------------- +// Arm64Classifier: +// Construct a new instance of the ARM64 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// +Arm64Classifier::Arm64Classifier(const ClassifierInfo& info) + : m_info(info), m_intRegs(intArgRegs, ArrLen(intArgRegs)), m_floatRegs(fltArgRegs, ArrLen(fltArgRegs)) +{ +} + +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the ARM64 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// +ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + if ((wellKnownParam == WellKnownArg::RetBuffer) && hasFixedRetBuffReg(m_info.CallConv)) + { + return ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_ARG_RET_BUFF, 0, + TARGET_POINTER_SIZE)); + } + + // First handle HFA/HVAs. These are allowed to be passed in more registers + // than other structures. + if (varTypeIsStruct(type) && !m_info.IsVarArgs) + { + var_types hfaType = comp->GetHfaType(structLayout->GetClassHandle()); + + if (hfaType != TYP_UNDEF) + { + unsigned elemSize = genTypeSize(hfaType); + unsigned slots = structLayout->GetSize() / elemSize; + ABIPassingInformation info; + if (m_floatRegs.Count() >= slots) + { + info.NumSegments = slots; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; + + for (unsigned i = 0; i < slots; i++) + { + info.Segments[i] = ABIPassingSegment::InRegister(m_floatRegs.Dequeue(), i * elemSize, elemSize); + } + } + else + { + unsigned alignment = compAppleArm64Abi() ? elemSize : TARGET_POINTER_SIZE; + m_stackArgSize = roundUp(m_stackArgSize, alignment); + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, + structLayout->GetSize())); + m_stackArgSize += roundUp(structLayout->GetSize(), alignment); + // After passing any float value on the stack, we should not enregister more float values. + m_floatRegs.Clear(); + } + + return info; + } + } + + unsigned slots; + unsigned passedSize; + if (varTypeIsStruct(type)) + { + unsigned size = structLayout->GetSize(); + if (size > 16) + { + slots = 1; // Passed by implicit byref + passedSize = TARGET_POINTER_SIZE; + } + else + { + slots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + passedSize = size; + } + } + else + { + assert(genTypeSize(type) <= TARGET_POINTER_SIZE); + slots = 1; + passedSize = genTypeSize(type); + } + + assert((slots == 1) || (slots == 2)); + + ABIPassingInformation info; + if (m_info.IsVarArgs && (slots == 2) && (m_intRegs.Count() == 1)) + { + // For varargs we split structs between register and stack in this + // case. Normally a struct that does not fit in registers will always + // be passed on stack. + assert(compFeatureArgSplit()); + info.NumSegments = 2; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]; + info.Segments[0] = ABIPassingSegment::InRegister(m_intRegs.Dequeue(), 0, TARGET_POINTER_SIZE); + info.Segments[1] = ABIPassingSegment::OnStack(m_stackArgSize, TARGET_POINTER_SIZE, + structLayout->GetSize() - TARGET_POINTER_SIZE); + m_stackArgSize += TARGET_POINTER_SIZE; + } + else + { + RegisterQueue* regs = &m_intRegs; + + // In varargs methods (only supported on Windows) all parameters go in + // integer registers. + if (varTypeUsesFloatArgReg(type) && !m_info.IsVarArgs) + { + regs = &m_floatRegs; + } + + if (regs->Count() >= slots) + { + info.NumSegments = slots; + info.Segments = new (comp, CMK_ABI) ABIPassingSegment[slots]; + unsigned slotSize = varTypeIsStruct(type) ? TARGET_POINTER_SIZE : genTypeSize(type); + info.Segments[0] = ABIPassingSegment::InRegister(regs->Dequeue(), 0, slotSize); + if (slots == 2) + { + assert(varTypeIsStruct(type)); + unsigned tailSize = structLayout->GetSize() - slotSize; + info.Segments[1] = ABIPassingSegment::InRegister(regs->Dequeue(), slotSize, tailSize); + } + } + else + { + unsigned alignment; + if (compAppleArm64Abi()) + { + if (varTypeIsStruct(type)) + { + alignment = TARGET_POINTER_SIZE; + } + else + { + alignment = genTypeSize(type); + } + + m_stackArgSize = roundUp(m_stackArgSize, alignment); + } + else + { + alignment = TARGET_POINTER_SIZE; + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + } + + info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, passedSize)); + + m_stackArgSize += roundUp(passedSize, alignment); + + // As soon as we pass something on stack we cannot go back and + // enregister something else. + regs->Clear(); + } + } + + return info; +} + #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/targetx86.cpp b/src/coreclr/jit/targetx86.cpp index d5ed8b0bbf606f..36044df9a981ad 100644 --- a/src/coreclr/jit/targetx86.cpp +++ b/src/coreclr/jit/targetx86.cpp @@ -21,4 +21,103 @@ const regNumber intArgRegs [] = {REG_ECX, REG_EDX}; const regMaskTP intArgMasks[] = {RBM_ECX, RBM_EDX}; // clang-format on +//----------------------------------------------------------------------------- +// X86Classifier: +// Construct a new instance of the x86 ABI classifier. +// +// Parameters: +// info - Info about the method being classified. +// +X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) +{ + switch (info.CallConv) + { + case CorInfoCallConvExtension::Thiscall: + { + static const regNumberSmall thiscallRegs[] = {REG_ECX}; + m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); + break; + } + case CorInfoCallConvExtension::C: + case CorInfoCallConvExtension::Stdcall: + case CorInfoCallConvExtension::CMemberFunction: + case CorInfoCallConvExtension::StdcallMemberFunction: + { + break; + } + default: + { + static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; + unsigned numRegs = ArrLen(regs); + if (info.IsVarArgs) + { + // In varargs methods we only enregister the this pointer or retbuff. + numRegs = info.HasThis || info.HasRetBuff ? 1 : 0; + } + m_regs = RegisterQueue(regs, numRegs); + break; + } + } +} + +//----------------------------------------------------------------------------- +// Classify: +// Classify a parameter for the x86 ABI. +// +// Parameters: +// comp - Compiler instance +// type - The type of the parameter +// structLayout - The layout of the struct. Expected to be non-null if +// varTypeIsStruct(type) is true. +// wellKnownParam - Well known type of the parameter (if it may affect its ABI classification) +// +// Returns: +// Classification information for the parameter. +// +ABIPassingInformation X86Classifier::Classify(Compiler* comp, + var_types type, + ClassLayout* structLayout, + WellKnownArg wellKnownParam) +{ + unsigned size = type == TYP_STRUCT ? structLayout->GetSize() : genTypeSize(type); + unsigned numSlots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; + + bool canEnreg = false; + if (m_regs.Count() >= numSlots) + { + switch (type) + { + case TYP_BYTE: + case TYP_UBYTE: + case TYP_SHORT: + case TYP_USHORT: + case TYP_INT: + case TYP_REF: + case TYP_BYREF: + canEnreg = true; + break; + case TYP_STRUCT: + canEnreg = comp->isTrivialPointerSizedStruct(structLayout->GetClassHandle()); + break; + default: + break; + } + } + + ABIPassingSegment segment; + if (canEnreg) + { + assert(numSlots == 1); + segment = ABIPassingSegment::InRegister(m_regs.Dequeue(), 0, size); + } + else + { + assert((m_stackArgSize % TARGET_POINTER_SIZE) == 0); + segment = ABIPassingSegment::OnStack(m_stackArgSize, 0, size); + m_stackArgSize += roundUp(size, TARGET_POINTER_SIZE); + } + + return ABIPassingInformation::FromSegment(comp, segment); +} + #endif // TARGET_X86 From 70a0affa1230335ed41de0e92ec9fac423abc6fb Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 13:29:30 +0100 Subject: [PATCH 14/15] Fix x86 --- src/coreclr/jit/targetx86.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/targetx86.cpp b/src/coreclr/jit/targetx86.cpp index 36044df9a981ad..1c3e91be1bd2ff 100644 --- a/src/coreclr/jit/targetx86.cpp +++ b/src/coreclr/jit/targetx86.cpp @@ -34,8 +34,8 @@ X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) { case CorInfoCallConvExtension::Thiscall: { - static const regNumberSmall thiscallRegs[] = {REG_ECX}; - m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); + static const regNumber thiscallRegs[] = {REG_ECX}; + m_regs = RegisterQueue(thiscallRegs, ArrLen(thiscallRegs)); break; } case CorInfoCallConvExtension::C: @@ -47,14 +47,13 @@ X86Classifier::X86Classifier(const ClassifierInfo& info) : m_regs(nullptr, 0) } default: { - static const regNumberSmall regs[] = {REG_ECX, REG_EDX}; - unsigned numRegs = ArrLen(regs); + unsigned numRegs = ArrLen(intArgRegs); if (info.IsVarArgs) { // In varargs methods we only enregister the this pointer or retbuff. numRegs = info.HasThis || info.HasRetBuff ? 1 : 0; } - m_regs = RegisterQueue(regs, numRegs); + m_regs = RegisterQueue(intArgRegs, numRegs); break; } } From 0e82029fb9609347f33a83d599f18d2624ac4eb2 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 26 Mar 2024 14:18:01 +0100 Subject: [PATCH 15/15] Fix Apple ARM64 ABI for HVAs For HVAs the element size can be > 8, but should at most be 8-byte aligned. --- src/coreclr/jit/targetarm64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/targetarm64.cpp b/src/coreclr/jit/targetarm64.cpp index befe688247c09f..f48cfae542cd34 100644 --- a/src/coreclr/jit/targetarm64.cpp +++ b/src/coreclr/jit/targetarm64.cpp @@ -84,7 +84,7 @@ ABIPassingInformation Arm64Classifier::Classify(Compiler* comp, } else { - unsigned alignment = compAppleArm64Abi() ? elemSize : TARGET_POINTER_SIZE; + unsigned alignment = compAppleArm64Abi() ? min(elemSize, TARGET_POINTER_SIZE) : TARGET_POINTER_SIZE; m_stackArgSize = roundUp(m_stackArgSize, alignment); info = ABIPassingInformation::FromSegment(comp, ABIPassingSegment::OnStack(m_stackArgSize, 0, structLayout->GetSize()));