diff --git a/Common/GPU/D3D9/thin3d_d3d9.cpp b/Common/GPU/D3D9/thin3d_d3d9.cpp
index 7b2fa934b0..845fa45d5f 100644
--- a/Common/GPU/D3D9/thin3d_d3d9.cpp
+++ b/Common/GPU/D3D9/thin3d_d3d9.cpp
@@ -259,17 +259,23 @@ class D3D9Pipeline : public Pipeline {
 public:
 	D3D9Pipeline() {}
 	~D3D9Pipeline() {
+		if (vshader) {
+			vshader->Release();
+		}
+		if (pshader) {
+			pshader->Release();
+		}
 	}
 
-	D3D9ShaderModule *vshader;
-	D3D9ShaderModule *pshader;
+	D3D9ShaderModule *vshader = nullptr;
+	D3D9ShaderModule *pshader = nullptr;
 
-	D3DPRIMITIVETYPE prim;
+	D3DPRIMITIVETYPE prim{};
 	AutoRef<D3D9InputLayout> inputLayout;
 	AutoRef<D3D9DepthStencilState> depthStencil;
 	AutoRef<D3D9BlendState> blend;
 	AutoRef<D3D9RasterState> raster;
-	UniformBufferDesc dynamicUniforms;
+	UniformBufferDesc dynamicUniforms{};
 
 	void Apply(LPDIRECT3DDEVICE9 device, uint8_t stencilRef, uint8_t stencilWriteMask, uint8_t stencilCompareMask);
 };
@@ -713,9 +719,11 @@ Pipeline *D3D9Context::CreateGraphicsPipeline(const PipelineDesc &desc) {
 		}
 		if (iter->GetStage() == ShaderStage::Fragment) {
 			pipeline->pshader = static_cast<D3D9ShaderModule *>(iter);
+			pipeline->pshader->AddRef();
 		}
 		else if (iter->GetStage() == ShaderStage::Vertex) {
 			pipeline->vshader = static_cast<D3D9ShaderModule *>(iter);
+			pipeline->vshader->AddRef();
 		}
 	}
 	pipeline->prim = primToD3D9[(int)desc.prim];
diff --git a/Common/GPU/thin3d.cpp b/Common/GPU/thin3d.cpp
index b5a2b89c2c..9dee5898fb 100644
--- a/Common/GPU/thin3d.cpp
+++ b/Common/GPU/thin3d.cpp
@@ -66,10 +66,15 @@ bool DataFormatIsDepthStencil(DataFormat fmt) {
 	}
 }
 
+RefCountedObject::~RefCountedObject() {
+	_dbg_assert_(refcount_ == 0xDEDEDE);
+}
 
 bool RefCountedObject::Release() {
 	if (refcount_ > 0 && refcount_ < 10000) {
 		if (--refcount_ == 0) {
+			// Make it very obvious if we try to free this again.
+			refcount_ = 0xDEDEDE;
 			delete this;
 			return true;
 		}
diff --git a/Common/GPU/thin3d.h b/Common/GPU/thin3d.h
index 7dfa09fc7e..264e240fa9 100644
--- a/Common/GPU/thin3d.h
+++ b/Common/GPU/thin3d.h
@@ -341,7 +341,9 @@ public:
 	RefCountedObject() {
 		refcount_ = 1;
 	}
-	virtual ~RefCountedObject() {}
+	RefCountedObject(const RefCountedObject &other) = delete;
+	RefCountedObject& operator=(RefCountedObject const&) = delete;
+	virtual ~RefCountedObject();
 
 	void AddRef() { refcount_++; }
 	bool Release();
diff --git a/GPU/Common/Draw2D.cpp b/GPU/Common/Draw2D.cpp
index 1a89c45fd4..a137379b59 100644
--- a/GPU/Common/Draw2D.cpp
+++ b/GPU/Common/Draw2D.cpp
@@ -110,19 +110,20 @@ void FramebufferManagerCommon::DrawStrip2D(Draw::Texture *tex, Draw2DVertex *ver
 
 	if (!draw2DPipelineColor_) {
 		char *fsCode = new char[4000];
-		char *fsDepthCode = new char[4000];
 		GenerateDraw2DFs(fsCode, shaderLanguageDesc);
-		GenerateDraw2DDepthFs(fsDepthCode, shaderLanguageDesc);
 
-		draw2DFs_ = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsCode, strlen(fsCode), "draw2d_fs");
+		ShaderModule *draw2DFs = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsCode, strlen(fsCode), "draw2d_fs");
+		delete[] fsCode;
 
-		_assert_(draw2DFs_);
+		_assert_(draw2DFs);
 
+		ShaderModule *draw2DFsDepth = nullptr;
 		if (draw_->GetDeviceCaps().fragmentShaderDepthWriteSupported) {
-			draw2DFsDepth_ = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsDepthCode, strlen(fsDepthCode), "draw2d_depth_fs");
-			_assert_(draw2DFsDepth_);
-		} else {
-			draw2DFsDepth_ = nullptr;
+			char *fsDepthCode = new char[4000];
+			GenerateDraw2DDepthFs(fsDepthCode, shaderLanguageDesc);
+			draw2DFsDepth = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsDepthCode, strlen(fsDepthCode), "draw2d_depth_fs");
+			delete[] fsDepthCode;
+			_assert_(draw2DFsDepth);
 		}
 
 		// verts have positions in 2D clip coordinates.
@@ -151,7 +152,7 @@ void FramebufferManagerCommon::DrawStrip2D(Draw::Texture *tex, Draw2DVertex *ver
 
 		PipelineDesc draw2DColorPipelineDesc{
 			Primitive::TRIANGLE_STRIP,
-			{ draw2DVs_, draw2DFs_ },
+			{ draw2DVs_, draw2DFs },
 			inputLayout, noDepthStencil, blendOff, rasterNoCull, nullptr,
 		};
 
@@ -159,7 +160,7 @@ void FramebufferManagerCommon::DrawStrip2D(Draw::Texture *tex, Draw2DVertex *ver
 
 		PipelineDesc draw2DDepthPipelineDesc{
 			Primitive::TRIANGLE_STRIP,
-			{ draw2DVs_, draw2DFsDepth_ },
+			{ draw2DVs_, draw2DFsDepth },
 			inputLayout, depthWriteAlways, blendDiscard, rasterNoCull, nullptr,
 		};
 
@@ -170,7 +171,10 @@ void FramebufferManagerCommon::DrawStrip2D(Draw::Texture *tex, Draw2DVertex *ver
 			draw2DPipelineDepth_ = nullptr;
 		}
 
-		delete[] fsCode;
+		draw2DFs->Release();
+		if (draw2DFsDepth) {
+			draw2DFsDepth->Release();
+		}
 
 		rasterNoCull->Release();
 		blendOff->Release();
diff --git a/GPU/Common/FramebufferManagerCommon.cpp b/GPU/Common/FramebufferManagerCommon.cpp
index a755a5aab1..b402cff8e0 100644
--- a/GPU/Common/FramebufferManagerCommon.cpp
+++ b/GPU/Common/FramebufferManagerCommon.cpp
@@ -2351,6 +2351,9 @@ static void DoRelease(T *&obj) {
 
 void FramebufferManagerCommon::DeviceLost() {
 	DestroyAllFBOs();
+
+	presentation_->DeviceLost();
+
 	for (int i = 0; i < 3; i++) {
 		for (int j = 0; j < 3; j++) {
 			DoRelease(reinterpretFromTo_[i][j]);
@@ -2363,14 +2366,12 @@ void FramebufferManagerCommon::DeviceLost() {
 	DoRelease(stencilUploadVs_);
 	DoRelease(stencilUploadSampler_);
 	DoRelease(stencilUploadPipeline_);
-	DoRelease(draw2DPipelineColor_);
-	DoRelease(draw2DPipelineDepth_);
 	DoRelease(draw2DSamplerNearest_);
 	DoRelease(draw2DSamplerLinear_);
 	DoRelease(draw2DVs_);
-	DoRelease(draw2DFs_);
-	DoRelease(draw2DFsDepth_);
-	presentation_->DeviceLost();
+	DoRelease(draw2DPipelineColor_);
+	DoRelease(draw2DPipelineDepth_);
+
 	draw_ = nullptr;
 }
 
diff --git a/GPU/Common/FramebufferManagerCommon.h b/GPU/Common/FramebufferManagerCommon.h
index 624d20a3d0..df35a139b6 100644
--- a/GPU/Common/FramebufferManagerCommon.h
+++ b/GPU/Common/FramebufferManagerCommon.h
@@ -492,6 +492,5 @@ protected:
 	Draw::SamplerState *draw2DSamplerLinear_ = nullptr;
 	Draw::SamplerState *draw2DSamplerNearest_ = nullptr;
 	Draw::ShaderModule *draw2DVs_ = nullptr;
-	Draw::ShaderModule *draw2DFs_ = nullptr;
-	Draw::ShaderModule *draw2DFsDepth_ = nullptr;
+	// The fragment shaders are "owned" by the pipelines since they're 1:1.
 };