Debugger: Fixed Lua-related multithreading bugs

Core/Console.cpp

@@ -349,6 +349,8 @@ void Console::Run()
 	_runLock.Acquire();
 	_stopLock.Acquire();
 
+	_emulationThreadId = std::this_thread::get_id();
+
 	targetTime = GetFrameDelay();
 
 	VideoDecoder::GetInstance()->StartThread();
@@ -487,6 +489,8 @@ void Console::Run()
 	_stopLock.Release();
 	_runLock.Release();
 
+	_emulationThreadId = std::thread::id();
+
 	MessageManager::SendNotification(ConsoleNotificationType::GameStopped);
 	MessageManager::SendNotification(ConsoleNotificationType::EmulationStopped);
 }
@@ -636,6 +640,11 @@ uint32_t Console::GetLagCounter()
 	return Instance->_lagCounter;
 }
 
+std::thread::id Console::GetEmulationThreadId()
+{
+	return Instance->_emulationThreadId;
+}
+
 void Console::ResetLagCounter()
 {
 	Instance->_lagCounter = 0;

Core/Console.h

@@ -58,6 +58,7 @@ class Console
 		atomic<uint32_t> _lagCounter;
 		
 		bool _initialized = false;
+		std::thread::id _emulationThreadId;
 
 		void LoadHdPack(VirtualFile &romFile, VirtualFile &patchFile);
 
@@ -71,6 +72,9 @@ class Console
 		~Console();
 		void Run();
 		void Stop();
+
+		static std::thread::id GetEmulationThreadId();
+
 		static void RequestReset();
 		static void Reset(bool softReset = true);
 		static void PowerCycle();

Core/DebugBreakHelper.h

@@ -1,31 +1,40 @@
 #pragma once
 #include "stdafx.h"
 #include "Debugger.h"
+#include "Console.h"
 
 class DebugBreakHelper
 {
 private:
 	Debugger* _debugger;
 	bool _needResume = false;
+	bool _isEmulationThread = false;
 
 public:
 	DebugBreakHelper(Debugger* debugger)
 	{
 		_debugger = debugger;
 
-		if(!debugger->IsExecutionStopped()) {
-			debugger->SetSendNotificationFlag(false);
-			debugger->Step(1);
-			while(!debugger->IsExecutionStopped()) {}
-			_needResume = true;
+		_isEmulationThread = Console::GetEmulationThreadId() == std::this_thread::get_id();
+
+		if(!_isEmulationThread) {
+			//Only attempt to break if this is done in a thread other than the main emulation thread
+			debugger->PreventResume();
+			if(!debugger->IsExecutionStopped()) {
+				debugger->Step(1);
+				while(!debugger->IsExecutionStopped()) {}
+				_needResume = true;
+			}
 		}
 	}
 
 	~DebugBreakHelper()
 	{
-		if(_needResume) {
-			_debugger->Run();
-			_debugger->SetSendNotificationFlag(true);
+		if(!_isEmulationThread) {
+			if(_needResume) {
+				_debugger->Run();
+			}
+			_debugger->AllowResume();
 		}
 	}
 };

Core/Debugger.cpp

@@ -54,8 +54,8 @@ Debugger::Debugger(shared_ptr<Console> console, shared_ptr<CPU> cpu, shared_ptr<
 	_stepOverAddr = -1;
 	_stepCycleCount = -1;
 	_ppuStepCount = -1;
-	_sendNotification = true;
 
+	_preventResume = 0;
 	_stopFlag = false;
 	_suspendCount = 0;
 
@@ -578,7 +578,7 @@ bool Debugger::SleepUntilResume()
 		auto lock = _breakLock.AcquireSafe();
 		_executionStopped = true;
 
-		if(_sendNotification) {
+		if(_preventResume == 0) {
 			SoundMixer::StopAudio();
 			MessageManager::SendNotification(ConsoleNotificationType::CodeBreak);
 			ProcessEvent(EventType::CodeBreak);
@@ -588,7 +588,7 @@ bool Debugger::SleepUntilResume()
 			}
 		}
 
-		while(stepCount == 0 && !_stopFlag && _suspendCount == 0) {
+		while((stepCount == 0 && !_stopFlag && _suspendCount == 0) || _preventResume > 0) {
 			std::this_thread::sleep_for(std::chrono::duration<int, std::milli>(10));
 			stepCount = _stepCount.load();
 		}
@@ -657,7 +657,7 @@ void Debugger::PpuStep(uint32_t count)
 	_stepCount = -1;
 }
 
-void Debugger::Step(uint32_t count, bool sendNotification)
+void Debugger::Step(uint32_t count)
 {
 	//Run CPU for [count] INSTRUCTIONS before breaking again
 	_stepOut = false;
@@ -703,11 +703,6 @@ void Debugger::StepBack()
 	}
 }
 
-void Debugger::SetSendNotificationFlag(bool enabled)
-{
-	_sendNotification = enabled;
-}
-
 void Debugger::Run()
 {
 	//Resume execution after a breakpoint has been hit
@@ -891,6 +886,16 @@ bool Debugger::IsExecutionStopped()
 	return _executionStopped || _console->IsPaused();
 }
 
+void Debugger::PreventResume()
+{
+	_preventResume++;
+}
+
+void Debugger::AllowResume()
+{
+	_preventResume--;
+}
+
 void Debugger::GetAbsoluteAddressAndType(uint32_t relativeAddr, AddressTypeInfo* info)
 {
 	if(relativeAddr < 0x2000) {
@@ -1054,12 +1059,17 @@ int Debugger::LoadScript(string content, int32_t scriptId)
 		_hasScript = true;
 		return script->GetScriptId();
 	} else {
-		auto result = std::find_if(_scripts.begin(), _scripts.end(), [=](shared_ptr<ScriptHost> &script) {
-			return script->GetScriptId() == scriptId;
-		});
-		if(result != _scripts.end()) {
-			(*result)->LoadScript(content, this);
-			return scriptId;
+		if(content.empty()) {
+			RemoveScript(scriptId);
+			return 0;
+		} else {
+			auto result = std::find_if(_scripts.begin(), _scripts.end(), [=](shared_ptr<ScriptHost> &script) {
+				return script->GetScriptId() == scriptId;
+			});
+			if(result != _scripts.end()) {
+				(*result)->LoadScript(content, this);
+				return scriptId;
+			}
 		}
 	}
 

Core/Debugger.h

@@ -65,6 +65,7 @@ private:
 	bool _bpUpdateNeeded;
 	SimpleLock _bpUpdateLock;
 
+	atomic<int32_t> _preventResume;
 	atomic<bool> _stopFlag;
 	atomic<bool> _executionStopped;
 	atomic<int32_t> _suspendCount;
@@ -102,7 +103,6 @@ private:
 	atomic<uint8_t> _lastInstruction;
 	atomic<bool> _stepOut;
 	atomic<int32_t> _stepOverAddr;
-	atomic<bool> _sendNotification;
 
 	int32_t _ppuViewerScanline;
 	int32_t _ppuViewerCycle;
@@ -158,13 +158,12 @@ public:
 	void Resume();
 
 	void PpuStep(uint32_t count = 1);
-	void Step(uint32_t count = 1, bool sendNotification = true);
+	void Step(uint32_t count = 1);
 	void StepCycles(uint32_t cycleCount = 1);
 	void StepOver();
 	void StepOut();
 	void StepBack();
 	void Run();
-	void SetSendNotificationFlag(bool enabled);
 
 	bool LoadCdlFile(string cdlFilepath);
 	void ResetCdl();
@@ -177,6 +176,9 @@ public:
 
 	bool IsExecutionStopped();
 
+	void PreventResume();
+	void AllowResume();
+
 	void GenerateCodeOutput();
 	const char* GetCode(uint32_t &length);
 	

GUI.NET/Debugger/frmScript.cs

@@ -232,11 +232,13 @@ namespace Mesen.GUI.Debugger
 
 		private void StopScript()
 		{
-			_scriptId = InteropEmu.DebugLoadScript(string.Empty, _scriptId);
-			if(_scriptId < 0) {
-				MessageBox.Show("Error while stopping script.");
-			} else {
-				lblScriptActive.Visible = false;
+			if(_scriptId >= 0) {
+				if(InteropEmu.DebugLoadScript(string.Empty, _scriptId) == 0) {
+					lblScriptActive.Visible = false;
+					_scriptId = -1;
+				} else {
+					MessageBox.Show("Error while stopping script.");
+				}
 			}
 		}