diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 1271e594777a6..428f030571499 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -3829,9 +3829,9 @@ void CodeGen::genZeroInitFltRegs(const regMaskTP& initFltRegs, const regMaskTP& // Iterate through float/double registers and initialize them to 0 or // copy from already initialized register of the same type. - regMaskTP regMask = genRegMask(REG_FP_FIRST); - for (regNumber reg = REG_FP_FIRST; reg <= REG_FP_LAST; reg = REG_NEXT(reg), regMask <<= 1) + for (regNumber reg = REG_FP_FIRST; reg <= REG_FP_LAST; reg = REG_NEXT(reg)) { + regMaskTP regMask = genRegMask(reg); if (regMask & initFltRegs) { // Do we have a float register already set to 0? @@ -5732,10 +5732,9 @@ void CodeGen::genFnProlog() if (initRegs) { - regMaskTP regMask = 0x1; - - for (regNumber reg = REG_INT_FIRST; reg <= REG_INT_LAST; reg = REG_NEXT(reg), regMask <<= 1) + for (regNumber reg = REG_INT_FIRST; reg <= REG_INT_LAST; reg = REG_NEXT(reg)) { + regMaskTP regMask = genRegMask(reg); if (regMask & initRegs) { // Check if we have already zeroed this register diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index d8844c2e35afb..f9c30f43612f7 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -99,6 +99,33 @@ inline bool genExactlyOneBit(T value) return ((value != 0) && genMaxOneBit(value)); } +#ifdef TARGET_ARM64 +inline regMaskTP genFindLowestBit(regMaskTP value) +{ + return regMaskTP(genFindLowestBit(value.getLow())); +} + +/***************************************************************************** + * + * Return true if the given value has exactly zero or one bits set. + */ + +inline bool genMaxOneBit(regMaskTP value) +{ + return genMaxOneBit(value.getLow()); +} + +/***************************************************************************** + * + * Return true if the given value has exactly one bit set. + */ + +inline bool genExactlyOneBit(regMaskTP value) +{ + return genExactlyOneBit(value.getLow()); +} +#endif + /***************************************************************************** * * Given a value that has exactly one bit set, return the position of that @@ -147,6 +174,13 @@ inline unsigned genCountBits(uint64_t bits) return BitOperations::PopCount(bits); } +#ifdef TARGET_ARM64 +inline unsigned genCountBits(regMaskTP mask) +{ + return BitOperations::PopCount(mask.getLow()); +} +#endif + /***************************************************************************** * * A rather simple routine that counts the number of bits in a given number. @@ -914,11 +948,18 @@ inline regNumber genRegNumFromMask(regMaskTP mask) /* Convert the mask to a register number */ - regNumber regNum = (regNumber)genLog2(mask); +#ifdef TARGET_ARM64 + regNumber regNum = (regNumber)genLog2(mask.getLow()); /* Make sure we got it right */ + assert(genRegMask(regNum) == mask.getLow()); +#else + regNumber regNum = (regNumber)genLog2(mask); + + /* Make sure we got it right */ assert(genRegMask(regNum) == mask); +#endif return regNum; } @@ -940,7 +981,8 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask) /* Convert the mask to a register number */ - regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); + regNumber regNum = (regNumber)BitScanForward(mask); + mask ^= genRegMask(regNum); return regNum; @@ -962,7 +1004,7 @@ inline regNumber genFirstRegNumFromMask(regMaskTP mask) /* Convert the mask to a register number */ - regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); + regNumber regNum = (regNumber)BitScanForward(mask); return regNum; } @@ -4463,7 +4505,11 @@ inline void* operator new[](size_t sz, Compiler* compiler, CompMemKind cmk) inline void printRegMask(regMaskTP mask) { +#ifdef TARGET_ARM64 + printf(REG_MASK_ALL_FMT, mask.getLow()); +#else printf(REG_MASK_ALL_FMT, mask); +#endif } inline char* regMaskToString(regMaskTP mask, Compiler* context) @@ -4471,14 +4517,22 @@ inline char* regMaskToString(regMaskTP mask, Compiler* context) const size_t cchRegMask = 24; char* regmask = new (context, CMK_Unknown) char[cchRegMask]; +#ifdef TARGET_ARM64 + sprintf_s(regmask, cchRegMask, REG_MASK_ALL_FMT, mask.getLow()); +#else sprintf_s(regmask, cchRegMask, REG_MASK_ALL_FMT, mask); +#endif return regmask; } inline void printRegMaskInt(regMaskTP mask) { +#ifdef TARGET_ARM64 + printf(REG_MASK_INT_FMT, (mask & RBM_ALLINT).getLow()); +#else printf(REG_MASK_INT_FMT, (mask & RBM_ALLINT)); +#endif } inline char* regMaskIntToString(regMaskTP mask, Compiler* context) @@ -4486,7 +4540,11 @@ inline char* regMaskIntToString(regMaskTP mask, Compiler* context) const size_t cchRegMask = 24; char* regmask = new (context, CMK_Unknown) char[cchRegMask]; +#ifdef TARGET_ARM64 + sprintf_s(regmask, cchRegMask, REG_MASK_INT_FMT, (mask & RBM_ALLINT).getLow()); +#else sprintf_s(regmask, cchRegMask, REG_MASK_INT_FMT, (mask & RBM_ALLINT)); +#endif return regmask; } diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 61fcf9447137d..3e003a14af83e 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -3510,7 +3510,7 @@ void emitter::emitDispRegSet(regMaskTP regs) continue; } - regs -= curReg; + regs ^= curReg; if (sp) { @@ -3870,8 +3870,8 @@ void emitter::emitDispGCRegDelta(const char* title, regMaskTP prevRegs, regMaskT { emitDispGCDeltaTitle(title); regMaskTP sameRegs = prevRegs & curRegs; - regMaskTP removedRegs = prevRegs - sameRegs; - regMaskTP addedRegs = curRegs - sameRegs; + regMaskTP removedRegs = prevRegs ^ sameRegs; + regMaskTP addedRegs = curRegs ^ sameRegs; if (removedRegs != RBM_NONE) { printf(" -"); @@ -8972,7 +8972,7 @@ void emitter::emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr) emitGCregDeadUpd(reg, addr); } - chg -= bit; + chg ^= bit; } while (chg); assert(emitThisXXrefRegs == regs); diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 6e1fa8bab9b7c..d592187ed55e9 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -4667,7 +4667,7 @@ void GCInfo::gcInfoRecordGCRegStateChange(GcInfoEncoder* gcInfoEncoder, } // Turn the bit we've just generated off and continue. - regMask -= tmpMask; // EAX,ECX,EDX,EBX,---,EBP,ESI,EDI + regMask ^= tmpMask; // EAX,ECX,EDX,EBX,---,EBP,ESI,EDI } } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index cac4c845a2cdc..ccffeae49afd0 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -13614,7 +13614,7 @@ regMaskTP LinearScan::RegisterSelection::select(Interval* current &overallLimitCandidates); assert(limitConsecutiveResult != RBM_NONE); - unsigned startRegister = BitOperations::BitScanForward(limitConsecutiveResult); + unsigned startRegister = BitScanForward(limitConsecutiveResult); regMaskTP registersNeededMask = (1ULL << refPosition->regCount) - 1; candidates |= (registersNeededMask << startRegister); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 09f8b8d63e581..9f6db34034800 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -794,8 +794,8 @@ class LinearScan : public LinearScanInterface static const regMaskTP LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5); static const regMaskTP LsraLimitSmallFPSet = (RBM_F0 | RBM_F1 | RBM_F2 | RBM_F16 | RBM_F17); #elif defined(TARGET_ARM64) - static const regMaskTP LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); - static const regMaskTP LsraLimitSmallFPSet = (RBM_V0 | RBM_V1 | RBM_V2 | RBM_V8 | RBM_V9); + static constexpr regMaskTP LsraLimitSmallIntSet = (RBM_R0 | RBM_R1 | RBM_R2 | RBM_R19 | RBM_R20); + static constexpr regMaskTP LsraLimitSmallFPSet = (RBM_V0 | RBM_V1 | RBM_V2 | RBM_V8 | RBM_V9); #elif defined(TARGET_X86) static const regMaskTP LsraLimitSmallIntSet = (RBM_EAX | RBM_ECX | RBM_EDI); static const regMaskTP LsraLimitSmallFPSet = (RBM_XMM0 | RBM_XMM1 | RBM_XMM2 | RBM_XMM6 | RBM_XMM7); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index f6f2cb2a2021c..e548fc3feec66 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -180,7 +180,7 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, unsigned int registersNeeded, regMaskTP* allConsecutiveCandidates) { - if (BitOperations::PopCount(candidates) < registersNeeded) + if (PopCount(candidates) < registersNeeded) { // There is no way the register demanded can be satisfied for this RefPosition // based on the candidates from which it can allocate a register. @@ -205,7 +205,7 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, do { // From LSB, find the first available register (bit `1`) - regAvailableStartIndex = BitOperations::BitScanForward(static_cast(currAvailableRegs)); + regAvailableStartIndex = BitScanForward(currAvailableRegs); regMaskTP startMask = (1ULL << regAvailableStartIndex) - 1; // Mask all the bits that are processed from LSB thru regAvailableStart until the last `1`. @@ -223,7 +223,7 @@ regMaskTP LinearScan::filterConsecutiveCandidates(regMaskTP candidates, } else { - regAvailableEndIndex = BitOperations::BitScanForward(static_cast(maskProcessed)); + regAvailableEndIndex = BitScanForward(maskProcessed); } regMaskTP endMask = (1ULL << regAvailableEndIndex) - 1; @@ -335,7 +335,7 @@ regMaskTP LinearScan::filterConsecutiveCandidatesForSpill(regMaskTP consecutiveC do { // From LSB, find the first available register (bit `1`) - regAvailableStartIndex = BitOperations::BitScanForward(static_cast(unprocessedRegs)); + regAvailableStartIndex = BitScanForward(unprocessedRegs); // For the current range, find how many registers are free vs. busy regMaskTP maskForCurRange = RBM_NONE; @@ -370,7 +370,7 @@ regMaskTP LinearScan::filterConsecutiveCandidatesForSpill(regMaskTP consecutiveC // In the given range, there are some free registers available. Calculate how many registers // will need spilling if this range is picked. - int curSpillRegs = registersNeeded - BitOperations::PopCount(maskForCurRange); + int curSpillRegs = registersNeeded - PopCount(maskForCurRange); if (curSpillRegs < maxSpillRegs) { consecutiveResultForBusy = 1ULL << regAvailableStartIndex; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index a05e880659db9..b36695280809f 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2898,7 +2898,7 @@ void LinearScan::stressSetRandomParameterPreferences() // Select a random register from all possible parameter registers // (of the right type). Preference this parameter to that register. - unsigned numBits = BitOperations::PopCount(*regs); + unsigned numBits = PopCount(*regs); if (numBits == 0) { continue; diff --git a/src/coreclr/jit/regset.cpp b/src/coreclr/jit/regset.cpp index dfc6df891bcf8..2ef2f9c1e17f1 100644 --- a/src/coreclr/jit/regset.cpp +++ b/src/coreclr/jit/regset.cpp @@ -957,7 +957,7 @@ regMaskSmall genRegMaskFromCalleeSavedMask(unsigned short calleeSaveMask) regMaskSmall res = 0; for (int i = 0; i < CNT_CALL_GC_REGS; i++) { - if ((calleeSaveMask & ((regMaskTP)1 << i)) != 0) + if ((calleeSaveMask & (1 << i)) != 0) { res |= raRbmCalleeSaveOrder[i]; } diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index d57d9baade310..6ebe5a5ea5002 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -208,12 +208,159 @@ enum _regMask_enum : unsigned // In any case, we believe that is OK to freely cast between these types; no information will // be lost. -#if defined(TARGET_AMD64) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) +typedef _regNumber_enum regNumber; +typedef unsigned char regNumberSmall; + +#if defined(TARGET_AMD64) || defined(TARGET_ARM) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) typedef unsigned __int64 regMaskTP; +#elif defined(TARGET_ARM64) +typedef unsigned __int64 regMaskSmall; +struct regMaskTP +{ +private: + uint64_t low; + +public: + constexpr regMaskTP(uint64_t regMask) + : low(regMask) + { + } + + regMaskTP() + { + } + + explicit operator bool() const + { + return low != RBM_NONE; + } + + explicit operator regMaskSmall() const + { + return (regMaskSmall)low; + } + + explicit operator unsigned int() const + { + return (unsigned int)low; + } + + uint64_t getLow() const + { + return low; + } +}; + +static regMaskTP operator^(regMaskTP first, regMaskTP second) +{ + regMaskTP result(first.getLow() ^ second.getLow()); + return result; +} + +static regMaskTP operator&(regMaskTP first, regMaskTP second) +{ + regMaskTP result(first.getLow() & second.getLow()); + return result; +} + +static regMaskTP operator|(regMaskTP first, regMaskTP second) +{ + regMaskTP result(first.getLow() | second.getLow()); + return result; +} + +static regMaskTP operator<<(regMaskTP first, const int b) +{ + regMaskTP result(first.getLow() << b); + return result; +} + +static regMaskTP operator>>(regMaskTP first, const int b) +{ + regMaskTP result(first.getLow() >> b); + return result; +} + +static regMaskTP& operator>>=(regMaskTP& first, const int b) +{ + first = first >> b; + return first; +} + +static regMaskTP& operator|=(regMaskTP& first, regMaskTP second) +{ + first = first | second; + return first; +} + +static regMaskTP& operator^=(regMaskTP& first, regMaskTP second) +{ + first = first ^ second; + return first; +} + +static regMaskSmall operator^=(regMaskSmall& first, regMaskTP second) +{ + first ^= second.getLow(); + return first; +} + +static regMaskSmall operator&=(regMaskSmall& first, regMaskTP second) +{ + first &= second.getLow(); + return first; +} + +static regMaskSmall operator|=(regMaskSmall& first, regMaskTP second) +{ + first |= second.getLow(); + return first; +} + +static regMaskTP& operator&=(regMaskTP& first, regMaskTP second) +{ + first = first & second; + return first; +} + +static bool operator==(regMaskTP first, regMaskTP second) +{ + return (first.getLow() == second.getLow()); +} + +static bool operator!=(regMaskTP first, regMaskTP second) +{ + return (first.getLow() != second.getLow()); +} + +static regMaskTP operator~(regMaskTP first) +{ + regMaskTP result(~first.getLow()); + return result; +} + #else typedef unsigned regMaskTP; #endif +static uint32_t PopCount(regMaskTP value) +{ +#ifdef TARGET_ARM64 + return BitOperations::PopCount(value.getLow()); +#else + return BitOperations::PopCount(value); +#endif +} + +static uint32_t BitScanForward(regMaskTP mask) +{ +#ifdef TARGET_ARM64 + return BitOperations::BitScanForward(mask.getLow()); +#else + return BitOperations::BitScanForward(mask); +#endif +} + #if REGMASK_BITS == 8 typedef unsigned char regMaskSmall; #define REG_MASK_INT_FMT "%02X" @@ -232,9 +379,6 @@ typedef unsigned __int64 regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif -typedef _regNumber_enum regNumber; -typedef unsigned char regNumberSmall; - /*****************************************************************************/ #ifdef DEBUG diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 07ca1fb6dc09c..ed06b863657ef 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -417,11 +417,6 @@ const char* dspRegRange(regMaskTP regMask, size_t& minSiz, const char* sep, regN sep = " "; } - if (regBit > regMask) - { - break; - } - regPrev = regNum; }