Merge pull request #18226 from unknownbrackets/x86-ir-breakpoints

x86jit: Improve memory breakpoint speed

Core/MIPS/ARM64/Arm64IRCompSystem.cpp

@@ -21,9 +21,11 @@
 
 #include "Common/Profiler/Profiler.h"
 #include "Core/Core.h"
+#include "Core/Debugger/Breakpoints.h"
 #include "Core/HLE/HLE.h"
 #include "Core/HLE/ReplaceTables.h"
 #include "Core/MemMap.h"
+#include "Core/MIPS/MIPSAnalyst.h"
 #include "Core/MIPS/IR/IRInterpreter.h"
 #include "Core/MIPS/ARM64/Arm64IRJit.h"
 #include "Core/MIPS/ARM64/Arm64IRRegCache.h"
@@ -70,6 +72,7 @@ void Arm64JitBackend::CompIR_Basic(IRInst inst) {
 		break;
 
 	case IROp::SetPCConst:
+		lastConstPC_ = inst.constant;
 		MOVI2R(SCRATCH1, inst.constant);
 		MovToPC(SCRATCH1);
 		break;
@@ -85,37 +88,118 @@ void Arm64JitBackend::CompIR_Breakpoint(IRInst inst) {
 
 	switch (inst.op) {
 	case IROp::Breakpoint:
+	{
 		FlushAll();
 		// Note: the constant could be a delay slot.
 		MOVI2R(W0, inst.constant);
 		QuickCallFunction(SCRATCH2_64, &IRRunBreakpoint);
-		break;
 
-	case IROp::MemoryCheck:
-	{
-		ARM64Reg addrBase = regs_.MapGPR(inst.src1);
-		FlushAll();
-		ADDI2R(W1, addrBase, inst.constant, SCRATCH1);
-		MovFromPC(W0);
-		ADDI2R(W0, W0, inst.dest, SCRATCH1);
-		QuickCallFunction(SCRATCH2_64, &IRRunMemCheck);
+		ptrdiff_t distance = dispatcherCheckCoreState_ - GetCodePointer();
+		if (distance >= -0x100000 && distance < 0x100000) {
+			CBNZ(W0, dispatcherCheckCoreState_);
+		} else {
+			FixupBranch keepOnKeepingOn = CBZ(W0);
+			B(dispatcherCheckCoreState_);
+			SetJumpTarget(keepOnKeepingOn);
+		}
 		break;
 	}
 
+	case IROp::MemoryCheck:
+		if (regs_.IsGPRImm(inst.src1)) {
+			uint32_t iaddr = regs_.GetGPRImm(inst.src1) + inst.constant;
+			uint32_t checkedPC = lastConstPC_ + inst.dest;
+			int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			if (size == 0) {
+				checkedPC += 4;
+				size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			}
+			bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC);
+
+			MemCheck check;
+			if (CBreakPoints::GetMemCheckInRange(iaddr, size, &check)) {
+				if (!(check.cond & MEMCHECK_READ) && !isWrite)
+					break;
+				if (!(check.cond & (MEMCHECK_WRITE | MEMCHECK_WRITE_ONCHANGE)) && isWrite)
+					break;
+
+				// We need to flush, or conditions and log expressions will see old register values.
+				FlushAll();
+
+				MOVI2R(W0, checkedPC);
+				MOVI2R(W1, iaddr);
+				QuickCallFunction(SCRATCH2_64, &IRRunMemCheck);
+
+				ptrdiff_t distance = dispatcherCheckCoreState_ - GetCodePointer();
+				if (distance >= -0x100000 && distance < 0x100000) {
+					CBNZ(W0, dispatcherCheckCoreState_);
+				} else {
+					FixupBranch keepOnKeepingOn = CBZ(W0);
+					B(dispatcherCheckCoreState_);
+					SetJumpTarget(keepOnKeepingOn);
+				}
+			}
+		} else {
+			uint32_t checkedPC = lastConstPC_ + inst.dest;
+			int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			if (size == 0) {
+				checkedPC += 4;
+				size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			}
+			bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC);
+
+			const auto memchecks = CBreakPoints::GetMemCheckRanges(isWrite);
+			// We can trivially skip if there are no checks for this type (i.e. read vs write.)
+			if (memchecks.empty())
+				break;
+
+			ARM64Reg addrBase = regs_.MapGPR(inst.src1);
+			ADDI2R(SCRATCH1, addrBase, inst.constant, SCRATCH2);
+
+			// We need to flush, or conditions and log expressions will see old register values.
+			FlushAll();
+
+			std::vector<FixupBranch> hitChecks;
+			for (auto it : memchecks) {
+				if (it.end != 0) {
+					CMPI2R(SCRATCH1, it.start - size, SCRATCH2);
+					MOVI2R(SCRATCH2, it.end);
+					CCMP(SCRATCH1, SCRATCH2, 0xF, CC_HI);
+					hitChecks.push_back(B(CC_LO));
+				} else {
+					CMPI2R(SCRATCH1, it.start, SCRATCH2);
+					hitChecks.push_back(B(CC_EQ));
+				}
+			}
+
+			FixupBranch noHits = B();
+
+			// Okay, now land any hit here.
+			for (auto &fixup : hitChecks)
+				SetJumpTarget(fixup);
+			hitChecks.clear();
+
+			MOVI2R(W0, checkedPC);
+			MOV(W1, SCRATCH1);
+			QuickCallFunction(SCRATCH2_64, &IRRunMemCheck);
+
+			ptrdiff_t distance = dispatcherCheckCoreState_ - GetCodePointer();
+			if (distance >= -0x100000 && distance < 0x100000) {
+				CBNZ(W0, dispatcherCheckCoreState_);
+			} else {
+				FixupBranch keepOnKeepingOn = CBZ(W0);
+				B(dispatcherCheckCoreState_);
+				SetJumpTarget(keepOnKeepingOn);
+			}
+
+			SetJumpTarget(noHits);
+		}
+		break;
+
 	default:
 		INVALIDOP;
 		break;
 	}
-
-	// Both return a flag on whether to bail out.
-	ptrdiff_t distance = dispatcherCheckCoreState_ - GetCodePointer();
-	if (distance >= -0x100000 && distance < 0x100000) {
-		CBNZ(W0, dispatcherCheckCoreState_);
-	} else {
-		FixupBranch keepOnKeepingOn = CBZ(W0);
-		B(dispatcherCheckCoreState_);
-		SetJumpTarget(keepOnKeepingOn);
-	}
 }
 
 void Arm64JitBackend::CompIR_System(IRInst inst) {
@@ -274,6 +358,66 @@ void Arm64JitBackend::CompIR_ValidateAddress(IRInst inst) {
 		INVALIDOP;
 		break;
 	}
+
+	if (regs_.IsGPRMappedAsPointer(inst.src1)) {
+		if (!jo.enablePointerify) {
+			SUB(SCRATCH1_64, regs_.RPtr(inst.src1), MEMBASEREG);
+			ADDI2R(SCRATCH1, SCRATCH1, inst.constant, SCRATCH2);
+		} else {
+			ADDI2R(SCRATCH1, regs_.R(inst.src1), inst.constant, SCRATCH2);
+		}
+	} else {
+		regs_.Map(inst);
+		ADDI2R(SCRATCH1, regs_.R(inst.src1), inst.constant, SCRATCH2);
+	}
+	ANDI2R(SCRATCH1, SCRATCH1, 0x3FFFFFFF, SCRATCH2);
+
+	std::vector<FixupBranch> validJumps;
+
+	FixupBranch unaligned;
+	if (alignment == 2) {
+		unaligned = TBNZ(SCRATCH1, 0);
+	} else if (alignment != 1) {
+		TSTI2R(SCRATCH1, alignment - 1, SCRATCH2);
+		unaligned = B(CC_NEQ);
+	}
+
+	CMPI2R(SCRATCH1, PSP_GetUserMemoryEnd() - alignment, SCRATCH2);
+	FixupBranch tooHighRAM = B(CC_HI);
+	CMPI2R(SCRATCH1, PSP_GetKernelMemoryBase(), SCRATCH2);
+	validJumps.push_back(B(CC_HS));
+
+	CMPI2R(SCRATCH1, PSP_GetVidMemEnd() - alignment, SCRATCH2);
+	FixupBranch tooHighVid = B(CC_HI);
+	CMPI2R(SCRATCH1, PSP_GetVidMemBase(), SCRATCH2);
+	validJumps.push_back(B(CC_HS));
+
+	CMPI2R(SCRATCH1, PSP_GetScratchpadMemoryEnd() - alignment, SCRATCH2);
+	FixupBranch tooHighScratch = B(CC_HI);
+	CMPI2R(SCRATCH1, PSP_GetScratchpadMemoryBase(), SCRATCH2);
+	validJumps.push_back(B(CC_HS));
+
+	if (alignment != 1)
+		SetJumpTarget(unaligned);
+	SetJumpTarget(tooHighRAM);
+	SetJumpTarget(tooHighVid);
+	SetJumpTarget(tooHighScratch);
+
+	// If we got here, something unusual and bad happened, so we'll always go back to the dispatcher.
+	// Because of that, we can avoid flushing outside this case.
+	auto regsCopy = regs_;
+	regsCopy.FlushAll();
+
+	// Ignores the return value, always returns to the dispatcher.
+	// Otherwise would need a thunk to restore regs.
+	MOV(W0, SCRATCH1);
+	MOVI2R(W1, alignment);
+	MOVI2R(W2, isWrite ? 1 : 0);
+	QuickCallFunction(SCRATCH2, &ReportBadAddress);
+	B(dispatcherCheckCoreState_);
+
+	for (FixupBranch &b : validJumps)
+		SetJumpTarget(b);
 }
 
 } // namespace MIPSComp

Core/MIPS/ARM64/Arm64IRJit.cpp

@@ -87,6 +87,7 @@ bool Arm64JitBackend::CompileBlock(IRBlock *block, int block_num, bool preload)
 	const u8 *blockStart = GetCodePointer();
 	block->SetTargetOffset((int)GetOffset(blockStart));
 	compilingBlockNum_ = block_num;
+	lastConstPC_ = 0;
 
 	regs_.Start(block);
 

Core/MIPS/ARM64/Arm64IRJit.h

@@ -145,6 +145,8 @@ private:
 	int jitStartOffset_ = 0;
 	int compilingBlockNum_ = -1;
 	int logBlocks_ = 0;
+	// Only useful in breakpoints, where it's set immediately prior.
+	uint32_t lastConstPC_ = 0;
 };
 
 class Arm64IRJit : public IRNativeJit {

Core/MIPS/IR/IRNativeCommon.cpp

@@ -19,7 +19,9 @@
 #include "Common/Profiler/Profiler.h"
 #include "Common/StringUtils.h"
 #include "Common/TimeUtil.h"
+#include "Core/Core.h"
 #include "Core/Debugger/SymbolMap.h"
+#include "Core/MemMap.h"
 #include "Core/MIPS/MIPSTables.h"
 #include "Core/MIPS/IR/IRNativeCommon.h"
 
@@ -99,6 +101,23 @@ uint32_t IRNativeBackend::DoIRInst(uint64_t value) {
 	return IRInterpret(currentMIPS, &inst, 1);
 }
 
+int IRNativeBackend::ReportBadAddress(uint32_t addr, uint32_t alignment, uint32_t isWrite) {
+	const auto toss = [&](MemoryExceptionType t) {
+		Core_MemoryException(addr, alignment, currentMIPS->pc, t);
+		return coreState != CORE_RUNNING ? 1 : 0;
+	};
+
+	if (!Memory::IsValidRange(addr, alignment)) {
+		MemoryExceptionType t = isWrite == 1 ? MemoryExceptionType::WRITE_WORD : MemoryExceptionType::READ_WORD;
+		if (alignment > 4)
+			t = isWrite ? MemoryExceptionType::WRITE_BLOCK : MemoryExceptionType::READ_BLOCK;
+		return toss(t);
+	} else if (alignment > 1 && (addr & (alignment - 1)) != 0) {
+		return toss(MemoryExceptionType::ALIGNMENT);
+	}
+	return 0;
+}
+
 IRNativeBackend::IRNativeBackend(IRBlockCache &blocks) : blocks_(blocks) {}
 
 void IRNativeBackend::CompileIRInst(IRInst inst) {

Core/MIPS/IR/IRNativeCommon.h

@@ -131,6 +131,8 @@ protected:
 	// Callback to log AND perform an IR interpreter inst.  Returns 0 or a PC to jump to.
 	static uint32_t DoIRInst(uint64_t inst);
 
+	static int ReportBadAddress(uint32_t addr, uint32_t alignment, uint32_t isWrite);
+
 	void AddLinkableExit(int block_num, uint32_t pc, int exitStartOffset, int exitLen);
 	void EraseAllLinks(int block_num);
 

Core/MIPS/x86/X64IRCompSystem.cpp

@@ -20,9 +20,11 @@
 
 #include "Common/Profiler/Profiler.h"
 #include "Core/Core.h"
+#include "Core/Debugger/Breakpoints.h"
 #include "Core/HLE/HLE.h"
 #include "Core/HLE/ReplaceTables.h"
 #include "Core/MemMap.h"
+#include "Core/MIPS/MIPSAnalyst.h"
 #include "Core/MIPS/IR/IRInterpreter.h"
 #include "Core/MIPS/x86/X64IRJit.h"
 #include "Core/MIPS/x86/X64IRRegCache.h"
@@ -88,6 +90,7 @@ void X64JitBackend::CompIR_Basic(IRInst inst) {
 		break;
 
 	case IROp::SetPCConst:
+		lastConstPC_ = inst.constant;
 		MOV(32, R(SCRATCH1), Imm32(inst.constant));
 		MovToPC(SCRATCH1);
 		break;
@@ -111,17 +114,80 @@ void X64JitBackend::CompIR_Breakpoint(IRInst inst) {
 		break;
 
 	case IROp::MemoryCheck:
-	{
-		X64Reg addrBase = regs_.MapGPR(inst.src1);
-		FlushAll();
-		LEA(32, addrBase, MDisp(addrBase, inst.constant));
-		MovFromPC(SCRATCH1);
-		LEA(32, SCRATCH1, MDisp(SCRATCH1, inst.dest));
-		ABI_CallFunctionRR((const void *)&IRRunMemCheck, SCRATCH1, addrBase);
-		TEST(32, R(EAX), R(EAX));
-		J_CC(CC_NZ, dispatcherCheckCoreState_, true);
+		if (regs_.IsGPRImm(inst.src1)) {
+			uint32_t iaddr = regs_.GetGPRImm(inst.src1) + inst.constant;
+			uint32_t checkedPC = lastConstPC_ + inst.dest;
+			int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			if (size == 0) {
+				checkedPC += 4;
+				size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			}
+			bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC);
+
+			MemCheck check;
+			if (CBreakPoints::GetMemCheckInRange(iaddr, size, &check)) {
+				if (!(check.cond & MEMCHECK_READ) && !isWrite)
+					break;
+				if (!(check.cond & (MEMCHECK_WRITE | MEMCHECK_WRITE_ONCHANGE)) && isWrite)
+					break;
+
+				// We need to flush, or conditions and log expressions will see old register values.
+				FlushAll();
+
+				ABI_CallFunctionCC((const void *)&IRRunMemCheck, checkedPC, iaddr);
+				TEST(32, R(EAX), R(EAX));
+				J_CC(CC_NZ, dispatcherCheckCoreState_, true);
+			}
+		} else {
+			uint32_t checkedPC = lastConstPC_ + inst.dest;
+			int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			if (size == 0) {
+				checkedPC += 4;
+				size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);
+			}
+			bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC);
+
+			const auto memchecks = CBreakPoints::GetMemCheckRanges(isWrite);
+			// We can trivially skip if there are no checks for this type (i.e. read vs write.)
+			if (memchecks.empty())
+				break;
+
+			X64Reg addrBase = regs_.MapGPR(inst.src1);
+			LEA(32, SCRATCH1, MDisp(addrBase, inst.constant));
+
+			// We need to flush, or conditions and log expressions will see old register values.
+			FlushAll();
+
+			std::vector<FixupBranch> hitChecks;
+			for (auto it : memchecks) {
+				if (it.end != 0) {
+					CMP(32, R(SCRATCH1), Imm32(it.start - size));
+					FixupBranch skipNext = J_CC(CC_BE);
+
+					CMP(32, R(SCRATCH1), Imm32(it.end));
+					hitChecks.push_back(J_CC(CC_B, true));
+
+					SetJumpTarget(skipNext);
+				} else {
+					CMP(32, R(SCRATCH1), Imm32(it.start));
+					hitChecks.push_back(J_CC(CC_E, true));
+				}
+			}
+
+			FixupBranch noHits = J(true);
+
+			// Okay, now land any hit here.
+			for (auto &fixup : hitChecks)
+				SetJumpTarget(fixup);
+			hitChecks.clear();
+
+			ABI_CallFunctionAA((const void *)&IRRunMemCheck, Imm32(checkedPC), R(SCRATCH1));
+			TEST(32, R(EAX), R(EAX));
+			J_CC(CC_NZ, dispatcherCheckCoreState_, true);
+
+			SetJumpTarget(noHits);
+		}
 		break;
-	}
 
 	default:
 		INVALIDOP;
@@ -271,23 +337,6 @@ void X64JitBackend::CompIR_Transfer(IRInst inst) {
 	}
 }
 
-int ReportBadAddress(uint32_t addr, uint32_t alignment, uint32_t isWrite) {
-	const auto toss = [&](MemoryExceptionType t) {
-		Core_MemoryException(addr, alignment, currentMIPS->pc, t);
-		return coreState != CORE_RUNNING ? 1 : 0;
-	};
-
-	if (!Memory::IsValidRange(addr, alignment)) {
-		MemoryExceptionType t = isWrite == 1 ? MemoryExceptionType::WRITE_WORD : MemoryExceptionType::READ_WORD;
-		if (alignment > 4)
-			t = isWrite ? MemoryExceptionType::WRITE_BLOCK : MemoryExceptionType::READ_BLOCK;
-		return toss(t);
-	} else if (alignment > 1 && (addr & (alignment - 1)) != 0) {
-		return toss(MemoryExceptionType::ALIGNMENT);
-	}
-	return 0;
-}
-
 void X64JitBackend::CompIR_ValidateAddress(IRInst inst) {
 	CONDITIONAL_DISABLE;
 
@@ -315,10 +364,17 @@ void X64JitBackend::CompIR_ValidateAddress(IRInst inst) {
 		break;
 	}
 
-	// This is unfortunate...
-	FlushAll();
-	regs_.Map(inst);
-	LEA(PTRBITS, SCRATCH1, MDisp(regs_.RX(inst.src1), inst.constant));
+	if (regs_.IsGPRMappedAsPointer(inst.src1)) {
+		LEA(PTRBITS, SCRATCH1, MDisp(regs_.RXPtr(inst.src1), inst.constant));
+#if defined(MASKED_PSP_MEMORY)
+		SUB(PTRBITS, R(SCRATCH1), ImmPtr(Memory::base));
+#else
+		SUB(PTRBITS, R(SCRATCH1), R(MEMBASEREG));
+#endif
+	} else {
+		regs_.Map(inst);
+		LEA(PTRBITS, SCRATCH1, MDisp(regs_.RX(inst.src1), inst.constant));
+	}
 	AND(32, R(SCRATCH1), Imm32(0x3FFFFFFF));
 
 	std::vector<FixupBranch> validJumps;
@@ -332,25 +388,32 @@ void X64JitBackend::CompIR_ValidateAddress(IRInst inst) {
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetUserMemoryEnd() - alignment));
 	FixupBranch tooHighRAM = J_CC(CC_A);
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetKernelMemoryBase()));
-	validJumps.push_back(J_CC(CC_AE));
+	validJumps.push_back(J_CC(CC_AE, true));
 
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetVidMemEnd() - alignment));
 	FixupBranch tooHighVid = J_CC(CC_A);
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetVidMemBase()));
-	validJumps.push_back(J_CC(CC_AE));
+	validJumps.push_back(J_CC(CC_AE, true));
 
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetScratchpadMemoryEnd() - alignment));
 	FixupBranch tooHighScratch = J_CC(CC_A);
 	CMP(32, R(SCRATCH1), Imm32(PSP_GetScratchpadMemoryBase()));
-	validJumps.push_back(J_CC(CC_AE));
+	validJumps.push_back(J_CC(CC_AE, true));
 
+	if (alignment != 1)
+		SetJumpTarget(unaligned);
 	SetJumpTarget(tooHighRAM);
 	SetJumpTarget(tooHighVid);
 	SetJumpTarget(tooHighScratch);
 
+	// If we got here, something unusual and bad happened, so we'll always go back to the dispatcher.
+	// Because of that, we can avoid flushing outside this case.
+	auto regsCopy = regs_;
+	regsCopy.FlushAll();
+
+	// Ignores the return value, always returns to the dispatcher.
+	// Otherwise would need a thunk to restore regs.
 	ABI_CallFunctionACC((const void *)&ReportBadAddress, R(SCRATCH1), alignment, isWrite);
-	TEST(32, R(EAX), R(EAX));
-	validJumps.push_back(J_CC(CC_Z));
 	JMP(dispatcherCheckCoreState_, true);
 
 	for (FixupBranch &b : validJumps)

Core/MIPS/x86/X64IRJit.cpp

@@ -80,6 +80,7 @@ bool X64JitBackend::CompileBlock(IRBlock *block, int block_num, bool preload) {
 	const u8 *blockStart = GetCodePointer();
 	block->SetTargetOffset((int)GetOffset(blockStart));
 	compilingBlockNum_ = block_num;
+	lastConstPC_ = 0;
 
 	regs_.Start(block);
 

Core/MIPS/x86/X64IRJit.h

@@ -159,6 +159,8 @@ private:
 	int jitStartOffset_ = 0;
 	int compilingBlockNum_ = -1;
 	int logBlocks_ = 0;
+	// Only useful in breakpoints, where it's set immediately prior.
+	uint32_t lastConstPC_ = 0;
 };
 
 class X64IRJit : public IRNativeJit {