Merge pull request #17893 from unknownbrackets/riscv-blocklink

riscv: Enable block linking

Common/RiscVEmitter.cpp

@@ -1182,14 +1182,19 @@ bool RiscVEmitter::CJInRange(const void *src, const void *dst) const {
 
 void RiscVEmitter::QuickJAL(RiscVReg scratchreg, RiscVReg rd, const u8 *dst) {
 	if (!JInRange(GetCodePointer(), dst)) {
+		int32_t lower = 0;
 		static_assert(sizeof(intptr_t) <= sizeof(int64_t));
+		// If it's near PC, we're better off shooting for AUIPC.  Should take 8 bytes.
 		int64_t pcdelta = (int64_t)dst - (int64_t)GetCodePointer();
-		int32_t lower = (int32_t)SignReduce64(pcdelta, 12);
-		uintptr_t upper = ((pcdelta - lower) >> 12) << 12;
-		if (scratchreg != rd)
-			LI(scratchreg, (uintptr_t)GetCodePointer() + upper, rd);
-		else
+		if (pcdelta < 0x100000000LL && pcdelta >= -0x100000000LL) {
+			lower = (int32_t)SignReduce64(pcdelta, 12);
+			uintptr_t upper = ((pcdelta - lower) >> 12) << 12;
 			LI(scratchreg, (uintptr_t)GetCodePointer() + upper);
+		} else {
+			lower = (int32_t)SignReduce64((int64_t)dst, 12);
+			// Abuse rd as a temporary if we need to.
+			LI(scratchreg, dst - lower, rd == scratchreg ? R_ZERO : rd);
+		}
 		JALR(rd, scratchreg, lower);
 	} else {
 		JAL(rd, dst);
@@ -1258,8 +1263,12 @@ void RiscVEmitter::SetRegToImmediate(RiscVReg rd, uint64_t value, RiscVReg temp)
 	// If this is just a 32-bit unsigned value, use a wall to mask.
 	if ((svalue >> 32) == 0) {
 		LI(rd, (int32_t)(svalue & 0xFFFFFFFF));
-		SLLI(rd, rd, BitsSupported() - 32);
-		SRLI(rd, rd, BitsSupported() - 32);
+		if (SupportsBitmanip('a')) {
+			ZEXT_W(rd, rd);
+		} else {
+			SLLI(rd, rd, BitsSupported() - 32);
+			SRLI(rd, rd, BitsSupported() - 32);
+		}
 		return;
 	}
 
@@ -1277,7 +1286,7 @@ void RiscVEmitter::SetRegToImmediate(RiscVReg rd, uint64_t value, RiscVReg temp)
 		return;
 	}
 
-	// Okay, let's just start with the upper 32 bits and add the rest via ORI.
+	// Okay, let's just start with the upper 32 bits and add the rest via ADDI.
 	int64_t upper = svalue >> 32;
 	LI(rd, upper);
 
@@ -1293,7 +1302,7 @@ void RiscVEmitter::SetRegToImmediate(RiscVReg rd, uint64_t value, RiscVReg temp)
 		int32_t chunk = (remaining >> sourceShift) & 0x07FF;
 
 		SLLI(rd, rd, targetShift - shifted);
-		ORI(rd, rd, chunk);
+		ADDI(rd, rd, chunk);
 
 		// Okay, increase shift and clear the bits we've deposited.
 		shifted = targetShift;

Core/MIPS/IR/IRJit.cpp

@@ -35,8 +35,9 @@
 #include "Core/MIPS/MIPSInt.h"
 #include "Core/MIPS/MIPSTables.h"
 #include "Core/MIPS/IR/IRRegCache.h"
-#include "Core/MIPS/IR/IRJit.h"
 #include "Core/MIPS/IR/IRInterpreter.h"
+#include "Core/MIPS/IR/IRJit.h"
+#include "Core/MIPS/IR/IRNativeCommon.h"
 #include "Core/MIPS/JitCommon/JitCommon.h"
 #include "Core/Reporting.h"
 
@@ -72,7 +73,12 @@ void IRJit::ClearCache() {
 }
 
 void IRJit::InvalidateCacheAt(u32 em_address, int length) {
-	blocks_.InvalidateICache(em_address, length);
+	std::vector<int> numbers = blocks_.FindInvalidatedBlockNumbers(em_address, length);
+	for (int block_num : numbers) {
+		auto block = blocks_.GetBlock(block_num);
+		int cookie = block->GetTargetOffset() < 0 ? block_num : block->GetTargetOffset();
+		block->Destroy(cookie);
+	}
 }
 
 void IRJit::Compile(u32 em_address) {
@@ -88,6 +94,7 @@ void IRJit::Compile(u32 em_address) {
 			b->Finalize(cookie);
 			if (b->IsValid()) {
 				// Success, we're done.
+				FinalizeTargetBlock(b, block_num);
 				return;
 			}
 		}
@@ -135,6 +142,8 @@ bool IRJit::CompileBlock(u32 em_address, std::vector<IRInst> &instructions, u32
 		return false;
 	// Overwrites the first instruction, and also updates stats.
 	blocks_.FinalizeBlock(block_num, preload);
+	if (!preload)
+		FinalizeTargetBlock(b, block_num);
 
 	return true;
 }
@@ -271,10 +280,11 @@ void IRBlockCache::Clear() {
 	byPage_.clear();
 }
 
-void IRBlockCache::InvalidateICache(u32 address, u32 length) {
+std::vector<int> IRBlockCache::FindInvalidatedBlockNumbers(u32 address, u32 length) {
 	u32 startPage = AddressToPage(address);
 	u32 endPage = AddressToPage(address + length);
 
+	std::vector<int> found;
 	for (u32 page = startPage; page <= endPage; ++page) {
 		const auto iter = byPage_.find(page);
 		if (iter == byPage_.end())
@@ -284,11 +294,12 @@ void IRBlockCache::InvalidateICache(u32 address, u32 length) {
 		for (int i : blocksInPage) {
 			if (blocks_[i].OverlapsRange(address, length)) {
 				// Not removing from the page, hopefully doesn't build up with small recompiles.
-				int cookie = blocks_[i].GetTargetOffset() < 0 ? i : blocks_[i].GetTargetOffset();
-				blocks_[i].Destroy(cookie);
+				found.push_back(i);
 			}
 		}
 	}
+
+	return found;
 }
 
 void IRBlockCache::FinalizeBlock(int i, bool preload) {
@@ -321,10 +332,7 @@ int IRBlockCache::FindPreloadBlock(u32 em_address) {
 
 	const std::vector<int> &blocksInPage = iter->second;
 	for (int i : blocksInPage) {
-		u32 start, mipsBytes;
-		blocks_[i].GetRange(start, mipsBytes);
-
-		if (start == em_address) {
+		if (blocks_[i].GetOriginalStart() == em_address) {
 			if (blocks_[i].HashMatches()) {
 				return i;
 			}
@@ -446,9 +454,7 @@ int IRBlockCache::GetBlockNumberFromStartAddress(u32 em_address, bool realBlocks
 	const std::vector<int> &blocksInPage = iter->second;
 	int best = -1;
 	for (int i : blocksInPage) {
-		uint32_t start, size;
-		blocks_[i].GetRange(start, size);
-		if (start == em_address) {
+		if (blocks_[i].GetOriginalStart() == em_address) {
 			best = i;
 			if (blocks_[i].IsValid()) {
 				return i;

Core/MIPS/IR/IRJit.h

@@ -90,6 +90,9 @@ public:
 		start = origAddr_;
 		size = origSize_;
 	}
+	u32 GetOriginalStart() const {
+		return origAddr_;
+	}
 
 	void Finalize(int number);
 	void Destroy(int number);
@@ -110,7 +113,7 @@ class IRBlockCache : public JitBlockCacheDebugInterface {
 public:
 	IRBlockCache() {}
 	void Clear();
-	void InvalidateICache(u32 address, u32 length);
+	std::vector<int> FindInvalidatedBlockNumbers(u32 address, u32 length);
 	void FinalizeBlock(int i, bool preload = false);
 	int GetNumBlocks() const override { return (int)blocks_.size(); }
 	int AllocateBlock(int emAddr) {
@@ -189,6 +192,7 @@ public:
 protected:
 	bool CompileBlock(u32 em_address, std::vector<IRInst> &instructions, u32 &mipsBytes, bool preload);
 	virtual bool CompileTargetBlock(IRBlock *block, int block_num, bool preload) { return true; }
+	virtual void FinalizeTargetBlock(IRBlock *block, int block_num) {}
 
 	JitOptions jo;
 

Core/MIPS/IR/IRNativeCommon.cpp

@@ -97,6 +97,8 @@ uint32_t IRNativeBackend::DoIRInst(uint64_t value) {
 	return IRInterpret(currentMIPS, &inst, 1);
 }
 
+IRNativeBackend::IRNativeBackend(IRBlockCache &blocks) : blocks_(blocks) {}
+
 void IRNativeBackend::CompileIRInst(IRInst inst) {
 	switch (inst.op) {
 	case IROp::Nop:
@@ -393,7 +395,7 @@ IRNativeJit::IRNativeJit(MIPSState *mipsState)
 
 void IRNativeJit::Init(IRNativeBackend &backend) {
 	backend_ = &backend;
-	debugInterface_.Init(&backend_->CodeBlock());
+	debugInterface_.Init(backend_);
 	backend_->GenerateFixedCode(mips_);
 
 	// Wanted this to be a reference, but vtbls get in the way.  Shouldn't change.
@@ -404,6 +406,10 @@ bool IRNativeJit::CompileTargetBlock(IRBlock *block, int block_num, bool preload
 	return backend_->CompileBlock(block, block_num, preload);
 }
 
+void IRNativeJit::FinalizeTargetBlock(IRBlock *block, int block_num) {
+	backend_->FinalizeBlock(block, block_num, jo);
+}
+
 void IRNativeJit::RunLoopUntil(u64 globalticks) {
 	if constexpr (enableDebugStats) {
 		LogDebugStats();
@@ -418,6 +424,21 @@ void IRNativeJit::ClearCache() {
 	backend_->ClearAllBlocks();
 }
 
+void IRNativeJit::InvalidateCacheAt(u32 em_address, int length) {
+	std::vector<int> numbers = blocks_.FindInvalidatedBlockNumbers(em_address, length);
+	for (int block_num : numbers) {
+		auto block = blocks_.GetBlock(block_num);
+		if (em_address != 0 || length < 0x1FFFFFFF) {
+			backend_->InvalidateBlock(block, block_num);
+		}
+		block->Destroy(block->GetTargetOffset());
+	}
+
+	if (em_address == 0 && length >= 0x1FFFFFFF) {
+		backend_->ClearAllBlocks();
+	}
+}
+
 bool IRNativeJit::DescribeCodePtr(const u8 *ptr, std::string &name) {
 	if (ptr != nullptr && backend_->DescribeCodePtr(ptr, name))
 		return true;
@@ -501,13 +522,73 @@ int IRNativeBackend::OffsetFromCodePtr(const u8 *ptr) {
 	return (int)codeBlock.GetOffset(ptr);
 }
 
-} // namespace MIPSComp
+void IRNativeBackend::FinalizeBlock(IRBlock *block, int block_num, const JitOptions &jo) {
+	if (jo.enableBlocklink) {
+		uint32_t pc = block->GetOriginalStart();
+
+		// First, link other blocks to this one now that it's finalized.
+		auto incoming = linksTo_.equal_range(pc);
+		for (auto it = incoming.first; it != incoming.second; ++it) {
+			auto &exits = nativeBlocks_[it->second].exits;
+			for (auto &blockExit : exits) {
+				if (blockExit.dest == pc)
+					OverwriteExit(blockExit.offset, blockExit.len, block_num);
+			}
+		}
+
+		// And also any blocks from this one, in case we're finalizing it later.
+		auto &outgoing = nativeBlocks_[block_num].exits;
+		for (auto &blockExit : outgoing) {
+			int dstBlockNum = blocks_.GetBlockNumberFromStartAddress(blockExit.dest);
+			const IRNativeBlock *nativeBlock = GetNativeBlock(dstBlockNum);
+			if (nativeBlock)
+				OverwriteExit(blockExit.offset, blockExit.len, dstBlockNum);
+		}
+	}
+}
+
+const IRNativeBlock *IRNativeBackend::GetNativeBlock(int block_num) const {
+	if (block_num < 0 || block_num >= (int)nativeBlocks_.size())
+		return nullptr;
+	return &nativeBlocks_[block_num];
+}
+
+void IRNativeBackend::SetBlockCheckedOffset(int block_num, int offset) {
+	if (block_num >= (int)nativeBlocks_.size())
+		nativeBlocks_.resize(block_num + 1);
+
+	nativeBlocks_[block_num].checkedOffset = offset;
+}
+
+void IRNativeBackend::AddLinkableExit(int block_num, uint32_t pc, int exitStartOffset, int exitLen) {
+	linksTo_.insert(std::make_pair(pc, block_num));
+
+	if (block_num >= (int)nativeBlocks_.size())
+		nativeBlocks_.resize(block_num + 1);
+	IRNativeBlockExit blockExit;
+	blockExit.offset = exitStartOffset;
+	blockExit.len = exitLen;
+	blockExit.dest = pc;
+	nativeBlocks_[block_num].exits.push_back(blockExit);
+}
+
+void IRNativeBackend::EraseAllLinks(int block_num) {
+	if (block_num == -1) {
+		linksTo_.clear();
+		nativeBlocks_.clear();
+	} else {
+		linksTo_.erase(block_num);
+		if (block_num < (int)nativeBlocks_.size())
+			nativeBlocks_[block_num].exits.clear();
+	}
+}
 
 IRNativeBlockCacheDebugInterface::IRNativeBlockCacheDebugInterface(const IRBlockCache &irBlocks)
 	: irBlocks_(irBlocks) {}
 
-void IRNativeBlockCacheDebugInterface::Init(const CodeBlockCommon *codeBlock) {
-	codeBlock_ = codeBlock;
+void IRNativeBlockCacheDebugInterface::Init(const IRNativeBackend *backend) {
+	codeBlock_ = &backend->CodeBlock();
+	backend_ = backend;
 }
 
 int IRNativeBlockCacheDebugInterface::GetNumBlocks() const {
@@ -520,14 +601,18 @@ int IRNativeBlockCacheDebugInterface::GetBlockNumberFromStartAddress(u32 em_addr
 
 void IRNativeBlockCacheDebugInterface::GetBlockCodeRange(int blockNum, int *startOffset, int *size) const {
 	int blockOffset = irBlocks_.GetBlock(blockNum)->GetTargetOffset();
-	int endOffset;
-	// We assume linear allocation.  Maybe a bit dangerous, should always be right.
-	if (blockNum + 1 >= GetNumBlocks()) {
-		// Last block, get from current code pointer.
-		endOffset = (int)codeBlock_->GetOffset(codeBlock_->GetCodePtr());
-	} else {
-		endOffset = irBlocks_.GetBlock(blockNum + 1)->GetTargetOffset();
-		_assert_msg_(endOffset >= blockOffset, "Next block not sequential, block=%d/%08x, next=%d/%08x", blockNum, blockOffset, blockNum + 1, endOffset);
+	int endOffset = backend_->GetNativeBlock(blockNum)->checkedOffset;
+
+	// If endOffset is before, the checked entry is before the block start.
+	if (endOffset < blockOffset) {
+		// We assume linear allocation.  Maybe a bit dangerous, should always be right.
+		if (blockNum + 1 >= GetNumBlocks()) {
+			// Last block, get from current code pointer.
+			endOffset = (int)codeBlock_->GetOffset(codeBlock_->GetCodePtr());
+		} else {
+			endOffset = irBlocks_.GetBlock(blockNum + 1)->GetTargetOffset();
+			_assert_msg_(endOffset >= blockOffset, "Next block not sequential, block=%d/%08x, next=%d/%08x", blockNum, blockOffset, blockNum + 1, endOffset);
+		}
 	}
 
 	*startOffset = blockOffset;
@@ -540,7 +625,6 @@ JitBlockDebugInfo IRNativeBlockCacheDebugInterface::GetBlockDebugInfo(int blockN
 	int blockOffset, codeSize;
 	GetBlockCodeRange(blockNum, &blockOffset, &codeSize);
 
-	// TODO: Normal entry?
 	const u8 *blockStart = codeBlock_->GetBasePtr() + blockOffset;
 #if PPSSPP_ARCH(ARM)
 	debugInfo.targetDisasm = DisassembleArm2(blockStart, codeSize);
@@ -589,3 +673,5 @@ void IRNativeBlockCacheDebugInterface::ComputeStats(BlockCacheStats &bcStats) co
 	bcStats.maxBloat = (float)maxBloat;
 	bcStats.avgBloat = (float)(totalBloat / (double)numBlocks);
 }
+
+} // namespace MIPSComp

Core/MIPS/IR/IRNativeCommon.h

@@ -15,25 +15,10 @@
 // Official git repository and contact information can be found at
 // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
 
+#include <unordered_map>
 #include "Core/MIPS/IR/IRJit.h"
 #include "Core/MIPS/JitCommon/JitBlockCache.h"
 
-class IRNativeBlockCacheDebugInterface : public JitBlockCacheDebugInterface {
-public:
-	IRNativeBlockCacheDebugInterface(const MIPSComp::IRBlockCache &irBlocks);
-	void Init(const CodeBlockCommon *codeBlock);
-	int GetNumBlocks() const;
-	int GetBlockNumberFromStartAddress(u32 em_address, bool realBlocksOnly = true) const;
-	JitBlockDebugInfo GetBlockDebugInfo(int blockNum) const;
-	void ComputeStats(BlockCacheStats &bcStats) const;
-
-private:
-	void GetBlockCodeRange(int blockNum, int *startOffset, int *size) const;
-
-	const MIPSComp::IRBlockCache &irBlocks_;
-	const CodeBlockCommon *codeBlock_ = nullptr;
-};
-
 namespace MIPSComp {
 
 typedef void (*IRNativeFuncNoArg)();
@@ -46,8 +31,20 @@ struct IRNativeHooks {
 	const uint8_t *crashHandler = nullptr;
 };
 
+struct IRNativeBlockExit {
+	int offset;
+	int len;
+	uint32_t dest;
+};
+
+struct IRNativeBlock {
+	int checkedOffset = 0;
+	std::vector<IRNativeBlockExit> exits;
+};
+
 class IRNativeBackend {
 public:
+	IRNativeBackend(IRBlockCache &blocks);
 	virtual ~IRNativeBackend() {}
 
 	void CompileIRInst(IRInst inst);
@@ -59,11 +56,16 @@ public:
 	virtual void GenerateFixedCode(MIPSState *mipsState) = 0;
 	virtual bool CompileBlock(IRBlock *block, int block_num, bool preload) = 0;
 	virtual void ClearAllBlocks() = 0;
+	virtual void InvalidateBlock(IRBlock *block, int block_num) = 0;
+	void FinalizeBlock(IRBlock *block, int block_num, const JitOptions &jo);
 
 	const IRNativeHooks &GetNativeHooks() const {
 		return hooks_;
 	}
 
+	const IRNativeBlock *GetNativeBlock(int block_num) const;
+	void SetBlockCheckedOffset(int block_num, int offset);
+
 	virtual const CodeBlockCommon &CodeBlock() const = 0;
 
 protected:
@@ -110,6 +112,8 @@ protected:
 	virtual void CompIR_VecStore(IRInst inst) = 0;
 	virtual void CompIR_ValidateAddress(IRInst inst) = 0;
 
+	virtual void OverwriteExit(int srcOffset, int len, int block_num) = 0;
+
 	// Returns true when debugging statistics should be compiled in.
 	bool DebugStatsEnabled() const;
 
@@ -123,7 +127,30 @@ protected:
 	// Callback to log AND perform an IR interpreter inst.  Returns 0 or a PC to jump to.
 	static uint32_t DoIRInst(uint64_t inst);
 
+	void AddLinkableExit(int block_num, uint32_t pc, int exitStartOffset, int exitLen);
+	void EraseAllLinks(int block_num);
+
 	IRNativeHooks hooks_;
+	IRBlockCache &blocks_;
+	std::vector<IRNativeBlock> nativeBlocks_;
+	std::unordered_multimap<uint32_t, int> linksTo_;
+};
+
+class IRNativeBlockCacheDebugInterface : public JitBlockCacheDebugInterface {
+public:
+	IRNativeBlockCacheDebugInterface(const MIPSComp::IRBlockCache &irBlocks);
+	void Init(const IRNativeBackend *backend);
+	int GetNumBlocks() const;
+	int GetBlockNumberFromStartAddress(u32 em_address, bool realBlocksOnly = true) const;
+	JitBlockDebugInfo GetBlockDebugInfo(int blockNum) const;
+	void ComputeStats(BlockCacheStats &bcStats) const;
+
+private:
+	void GetBlockCodeRange(int blockNum, int *startOffset, int *size) const;
+
+	const MIPSComp::IRBlockCache &irBlocks_;
+	const CodeBlockCommon *codeBlock_ = nullptr;
+	const IRNativeBackend *backend_ = nullptr;
 };
 
 class IRNativeJit : public IRJit {
@@ -133,6 +160,7 @@ public:
 	void RunLoopUntil(u64 globalticks) override;
 
 	void ClearCache() override;
+	void InvalidateCacheAt(u32 em_address, int length = 4) override;
 
 	bool DescribeCodePtr(const u8 *ptr, std::string &name) override;
 	bool CodeInRange(const u8 *ptr) const override;
@@ -145,6 +173,7 @@ public:
 protected:
 	void Init(IRNativeBackend &backend);
 	bool CompileTargetBlock(IRBlock *block, int block_num, bool preload) override;
+	void FinalizeTargetBlock(IRBlock *block, int block_num) override;
 
 	IRNativeBackend *backend_ = nullptr;
 	IRNativeHooks hooks_;

Core/MIPS/JitCommon/JitState.cpp

@@ -64,6 +64,10 @@ namespace MIPSComp {
 		useStaticAlloc = !Disabled(JitDisable::STATIC_ALLOC);
 		// iOS/etc. may disable at runtime if Memory::base is not nicely aligned.
 		enablePointerify = !Disabled(JitDisable::POINTERIFY);
+#endif
+#if PPSSPP_ARCH(RISCV64)
+		// Seems to perform slightly better than a checked entry at the start.
+		useBackJump = true;
 #endif
 	}
 

Core/MIPS/RiscV/RiscVCompBranch.cpp

@@ -41,8 +41,7 @@ void RiscVJitBackend::CompIR_Exit(IRInst inst) {
 	switch (inst.op) {
 	case IROp::ExitToConst:
 		FlushAll();
-		LI(SCRATCH1, inst.constant);
-		QuickJ(R_RA, dispatcherPCInSCRATCH1_);
+		WriteConstExit(inst.constant);
 		break;
 
 	case IROp::ExitToReg:
@@ -92,8 +91,7 @@ void RiscVJitBackend::CompIR_ExitIf(IRInst inst) {
 			break;
 		}
 
-		LI(SCRATCH1, inst.constant);
-		QuickJ(R_RA, dispatcherPCInSCRATCH1_);
+		WriteConstExit(inst.constant);
 		SetJumpTarget(fixup);
 		break;
 
@@ -127,8 +125,7 @@ void RiscVJitBackend::CompIR_ExitIf(IRInst inst) {
 			break;
 		}
 
-		LI(SCRATCH1, inst.constant);
-		QuickJ(R_RA, dispatcherPCInSCRATCH1_);
+		WriteConstExit(inst.constant);
 		SetJumpTarget(fixup);
 		break;
 

Core/MIPS/RiscV/RiscVJit.cpp

@@ -25,8 +25,12 @@ namespace MIPSComp {
 using namespace RiscVGen;
 using namespace RiscVJitConstants;
 
-RiscVJitBackend::RiscVJitBackend(MIPSState *mipsState, JitOptions &jitopt)
-	: jo(jitopt), gpr(mipsState, &jo), fpr(mipsState, &jo) {
+// Needs space for a LI and J which might both be 32-bit offsets.
+static constexpr int MIN_BLOCK_NORMAL_LEN = 16;
+static constexpr int MIN_BLOCK_EXIT_LEN = 8;
+
+RiscVJitBackend::RiscVJitBackend(MIPSState *mipsState, JitOptions &jitopt, IRBlockCache &blocks)
+	: IRNativeBackend(blocks), jo(jitopt), gpr(mipsState, &jo), fpr(mipsState, &jo) {
 	// Automatically disable incompatible options.
 	if (((intptr_t)Memory::base & 0x00000000FFFFFFFFUL) != 0) {
 		jo.enablePointerify = false;
@@ -52,10 +56,23 @@ bool RiscVJitBackend::CompileBlock(IRBlock *block, int block_num, bool preload)
 	if (GetSpaceLeft() < 0x800)
 		return false;
 
-	// Don't worry, the codespace isn't large enough to overflow offsets.
-	block->SetTargetOffset((int)GetOffset(GetCodePointer()));
+	u32 startPC = block->GetOriginalStart();
+	bool wroteCheckedOffset = false;
+	FixupBranch lateCheckFail;
+	if (jo.enableBlocklink && !jo.useBackJump) {
+		SetBlockCheckedOffset(block_num, (int)GetOffset(GetCodePointer()));
+		wroteCheckedOffset = true;
 
-	// TODO: Block linking, checked entries and such.
+		FixupBranch normalEntry = BGE(DOWNCOUNTREG, R_ZERO);
+		LI(SCRATCH1, startPC);
+		QuickJ(R_RA, outerLoopPCInSCRATCH1_);
+		SetJumpTarget(normalEntry);
+	}
+
+	// Don't worry, the codespace isn't large enough to overflow offsets.
+	const u8 *blockStart = GetCodePointer();
+	block->SetTargetOffset((int)GetOffset(blockStart));
+	compilingBlockNum_ = block_num;
 
 	gpr.Start(block);
 	fpr.Start(block);
@@ -74,6 +91,7 @@ bool RiscVJitBackend::CompileBlock(IRBlock *block, int block_num, bool preload)
 
 		// Safety check, in case we get a bunch of really large jit ops without a lot of branching.
 		if (GetSpaceLeft() < 0x800) {
+			compilingBlockNum_ = -1;
 			return false;
 		}
 	}
@@ -85,11 +103,85 @@ bool RiscVJitBackend::CompileBlock(IRBlock *block, int block_num, bool preload)
 		QuickJ(R_RA, hooks_.crashHandler);
 	}
 
+	int len = (int)GetOffset(GetCodePointer()) - block->GetTargetOffset();
+	if (len < MIN_BLOCK_NORMAL_LEN) {
+		// We need at least 16 bytes to invalidate blocks with, but larger doesn't need to align.
+		ReserveCodeSpace(MIN_BLOCK_NORMAL_LEN - len);
+	}
+
+	if (!wroteCheckedOffset) {
+		// Always record this, even if block link disabled - it's used for size calc.
+		SetBlockCheckedOffset(block_num, (int)GetOffset(GetCodePointer()));
+	}
+
+	if (jo.enableBlocklink && jo.useBackJump) {
+		// Most blocks shouldn't be >= 4KB, so usually we can just BGE.
+		if (BInRange(blockStart)) {
+			BGE(DOWNCOUNTREG, R_ZERO, blockStart);
+		} else {
+			FixupBranch skip = BLT(DOWNCOUNTREG, R_ZERO);
+			J(blockStart);
+			SetJumpTarget(skip);
+		}
+		LI(SCRATCH1, startPC);
+		QuickJ(R_RA, outerLoopPCInSCRATCH1_);
+	}
+
 	FlushIcache();
+	compilingBlockNum_ = -1;
 
 	return true;
 }
 
+void RiscVJitBackend::WriteConstExit(uint32_t pc) {
+	int block_num = blocks_.GetBlockNumberFromStartAddress(pc);
+	const IRNativeBlock *nativeBlock = GetNativeBlock(block_num);
+
+	int exitStart = (int)GetOffset(GetCodePointer());
+	if (block_num >= 0 && jo.enableBlocklink && nativeBlock && nativeBlock->checkedOffset != 0) {
+		// Don't bother recording, we don't every overwrite to "unlink".
+		// Instead, we would mark the target block to jump to the dispatcher.
+		QuickJ(SCRATCH1, GetBasePtr() + nativeBlock->checkedOffset);
+	} else {
+		LI(SCRATCH1, pc);
+		QuickJ(R_RA, dispatcherPCInSCRATCH1_);
+	}
+
+	if (jo.enableBlocklink) {
+		// In case of compression or early link, make sure it's large enough.
+		int len = (int)GetOffset(GetCodePointer()) - exitStart;
+		if (len < MIN_BLOCK_EXIT_LEN) {
+			ReserveCodeSpace(MIN_BLOCK_EXIT_LEN - len);
+			len = MIN_BLOCK_EXIT_LEN;
+		}
+
+		AddLinkableExit(compilingBlockNum_, pc, exitStart, len);
+	}
+}
+
+void RiscVJitBackend::OverwriteExit(int srcOffset, int len, int block_num) {
+	_dbg_assert_(len >= MIN_BLOCK_EXIT_LEN);
+
+	const IRNativeBlock *nativeBlock = GetNativeBlock(block_num);
+	if (nativeBlock) {
+		u8 *writable = GetWritablePtrFromCodePtr(GetBasePtr()) + srcOffset;
+		if (PlatformIsWXExclusive()) {
+			ProtectMemoryPages(writable, len, MEM_PROT_READ | MEM_PROT_WRITE);
+		}
+
+		RiscVEmitter emitter(GetBasePtr() + srcOffset, writable);
+		emitter.QuickJ(SCRATCH1, GetBasePtr() + nativeBlock->checkedOffset);
+		int bytesWritten = (int)(emitter.GetWritableCodePtr() - writable);
+		if (bytesWritten < len)
+			emitter.ReserveCodeSpace(len - bytesWritten);
+		emitter.FlushIcache();
+
+		if (PlatformIsWXExclusive()) {
+			ProtectMemoryPages(writable, 16, MEM_PROT_READ | MEM_PROT_EXEC);
+		}
+	}
+}
+
 void RiscVJitBackend::CompIR_Generic(IRInst inst) {
 	// If we got here, we're going the slow way.
 	uint64_t value;
@@ -156,6 +248,37 @@ bool RiscVJitBackend::DescribeCodePtr(const u8 *ptr, std::string &name) const {
 void RiscVJitBackend::ClearAllBlocks() {
 	ClearCodeSpace(jitStartOffset_);
 	FlushIcacheSection(region + jitStartOffset_, region + region_size - jitStartOffset_);
+	EraseAllLinks(-1);
+}
+
+void RiscVJitBackend::InvalidateBlock(IRBlock *block, int block_num) {
+	int offset = block->GetTargetOffset();
+	u8 *writable = GetWritablePtrFromCodePtr(GetBasePtr()) + offset;
+
+	// Overwrite the block with a jump to compile it again.
+	u32 pc = block->GetOriginalStart();
+	if (pc != 0) {
+		// Hopefully we always have at least 16 bytes, which should be all we need.
+		if (PlatformIsWXExclusive()) {
+			ProtectMemoryPages(writable, 16, MEM_PROT_READ | MEM_PROT_WRITE);
+		}
+
+		RiscVEmitter emitter(GetBasePtr() + offset, writable);
+		// We sign extend to ensure it will fit in 32-bit and 8 bytes LI.
+		// TODO: May need to change if dispatcher doesn't reload PC.
+		emitter.LI(SCRATCH1, (int32_t)pc);
+		emitter.QuickJ(R_RA, dispatcherPCInSCRATCH1_);
+		int bytesWritten = (int)(emitter.GetWritableCodePtr() - writable);
+		if (bytesWritten < MIN_BLOCK_NORMAL_LEN)
+			emitter.ReserveCodeSpace(MIN_BLOCK_NORMAL_LEN - bytesWritten);
+		emitter.FlushIcache();
+
+		if (PlatformIsWXExclusive()) {
+			ProtectMemoryPages(writable, 16, MEM_PROT_READ | MEM_PROT_EXEC);
+		}
+	}
+
+	EraseAllLinks(block_num);
 }
 
 void RiscVJitBackend::RestoreRoundingMode(bool force) {

Core/MIPS/RiscV/RiscVJit.h

@@ -31,7 +31,7 @@ namespace MIPSComp {
 
 class RiscVJitBackend : public RiscVGen::RiscVCodeBlock, public IRNativeBackend {
 public:
-	RiscVJitBackend(MIPSState *mipsState, JitOptions &jo);
+	RiscVJitBackend(MIPSState *mipsState, JitOptions &jo, IRBlockCache &blocks);
 	~RiscVJitBackend();
 
 	bool DescribeCodePtr(const u8 *ptr, std::string &name) const override;
@@ -39,6 +39,7 @@ public:
 	void GenerateFixedCode(MIPSState *mipsState) override;
 	bool CompileBlock(IRBlock *block, int block_num, bool preload) override;
 	void ClearAllBlocks() override;
+	void InvalidateBlock(IRBlock *block, int block_num) override;
 
 protected:
 	const CodeBlockCommon &CodeBlock() const override {
@@ -57,6 +58,9 @@ private:
 	// Note: destroys SCRATCH1.
 	void FlushAll();
 
+	void WriteConstExit(uint32_t pc);
+	void OverwriteExit(int srcOffset, int len, int block_num) override;
+
 	void CompIR_Arith(IRInst inst) override;
 	void CompIR_Assign(IRInst inst) override;
 	void CompIR_Basic(IRInst inst) override;
@@ -122,12 +126,13 @@ private:
 	const u8 *loadStaticRegisters_ = nullptr;
 
 	int jitStartOffset_ = 0;
+	int compilingBlockNum_ = -1;
 };
 
 class RiscVJit : public IRNativeJit {
 public:
 	RiscVJit(MIPSState *mipsState)
-		: IRNativeJit(mipsState), rvBackend_(mipsState, jo) {
+		: IRNativeJit(mipsState), rvBackend_(mipsState, jo, blocks_) {
 		Init(rvBackend_);
 	}
 

UI/DevScreens.cpp

@@ -1216,7 +1216,7 @@ void JitCompareScreen::OnRandomBlock(int flag) {
 			currentBlock_ = rand() % numBlocks;
 			JitBlockDebugInfo b = blockCache->GetBlockDebugInfo(currentBlock_);
 			u32 mipsBytes = (u32)b.origDisasm.size() * 4;
-			for (u32 addr = b.originalAddress; addr <= b.originalAddress + mipsBytes; addr += 4) {
+			for (u32 addr = b.originalAddress; addr < b.originalAddress + mipsBytes; addr += 4) {
 				MIPSOpcode opcode = Memory::Read_Instruction(addr);
 				if (MIPSGetInfo(opcode) & flag) {
 					char temp[256];

ext/riscv-disas.cpp

@@ -119,7 +119,7 @@ static const rvc_constraint rvcc_jal_j[] = { rvc_rd_eq_x0, rvc_end };
 static const rvc_constraint rvcc_jal_jal[] = { rvc_rd_eq_ra, rvc_end };
 static const rvc_constraint rvcc_jalr_jr[] = { rvc_rd_eq_x0, rvc_imm_eq_zero, rvc_end };
 static const rvc_constraint rvcc_jalr_jalr[] = { rvc_rd_eq_ra, rvc_imm_eq_zero, rvc_end };
-static const rvc_constraint rvcc_jalr_ret[] = { rvc_rd_eq_x0, rvc_rs1_eq_ra, rvc_end };
+static const rvc_constraint rvcc_jalr_ret[] = { rvc_rd_eq_x0, rvc_rs1_eq_ra, rvc_imm_eq_zero, rvc_end };
 static const rvc_constraint rvcc_addi_nop[] = { rvc_rd_eq_x0, rvc_rs1_eq_x0, rvc_imm_eq_zero, rvc_end };
 static const rvc_constraint rvcc_rdcycle[] = { rvc_rs1_eq_x0, rvc_csr_eq_0xc00, rvc_end };
 static const rvc_constraint rvcc_rdtime[] = { rvc_rs1_eq_x0, rvc_csr_eq_0xc01, rvc_end };