From 061685678d45cb2ad7ac6c44f1ef5cde389960b8 Mon Sep 17 00:00:00 2001
From: Henrik Rydgard <hrydgard@gmail.com>
Date: Fri, 17 Feb 2017 17:34:21 +0100
Subject: [PATCH] State management fixes. DBZ toon shader effect now works.

---
 GPU/D3D11/DepalettizeShaderD3D11.cpp |  8 --------
 GPU/D3D11/DepalettizeShaderD3D11.h   |  2 --
 GPU/D3D11/DrawEngineD3D11.h          |  2 ++
 GPU/D3D11/StateMappingD3D11.cpp      | 16 ++++++++++------
 GPU/D3D11/TextureCacheD3D11.cpp      |  1 -
 5 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/GPU/D3D11/DepalettizeShaderD3D11.cpp b/GPU/D3D11/DepalettizeShaderD3D11.cpp
index 216dd7e05e..34f8a6409c 100644
--- a/GPU/D3D11/DepalettizeShaderD3D11.cpp
+++ b/GPU/D3D11/DepalettizeShaderD3D11.cpp
@@ -60,13 +60,6 @@ DepalShaderCacheD3D11::DepalShaderCacheD3D11(ID3D11Device *device, ID3D11DeviceC
 	std::vector<uint8_t> vsByteCode;
 	vertexShader_ = CreateVertexShaderD3D11(device, depalVShaderHLSL, strlen(depalVShaderHLSL), &vsByteCode);
 	device_->CreateInputLayout(g_DepalVertexElements, ARRAY_SIZE(g_DepalVertexElements), vsByteCode.data(), vsByteCode.size(), &inputLayout_);
-
-	D3D11_SAMPLER_DESC sampDesc{};
-	sampDesc.AddressU = D3D11_TEXTURE_ADDRESS_WRAP;
-	sampDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP;
-	sampDesc.AddressW = D3D11_TEXTURE_ADDRESS_WRAP;
-	sampDesc.Filter = D3D11_FILTER_MIN_MAG_MIP_POINT;
-	device_->CreateSamplerState(&sampDesc, &clutSampler);
 }
 
 DepalShaderCacheD3D11::~DepalShaderCacheD3D11() {
@@ -74,7 +67,6 @@ DepalShaderCacheD3D11::~DepalShaderCacheD3D11() {
 	if (vertexShader_) {
 		vertexShader_->Release();
 	}
-	clutSampler->Release();
 }
 
 u32 DepalShaderCacheD3D11::GenerateShaderID(GEPaletteFormat clutFormat, GEBufferFormat pixelFormat) {
diff --git a/GPU/D3D11/DepalettizeShaderD3D11.h b/GPU/D3D11/DepalettizeShaderD3D11.h
index c4889b1669..5d3db8aac1 100644
--- a/GPU/D3D11/DepalettizeShaderD3D11.h
+++ b/GPU/D3D11/DepalettizeShaderD3D11.h
@@ -51,7 +51,6 @@ public:
 	ID3D11VertexShader *GetDepalettizeVertexShader() { return vertexShader_; }
 	ID3D11InputLayout *GetInputLayout() { return inputLayout_; }
 	ID3D11ShaderResourceView *GetClutTexture(GEPaletteFormat clutFormat, const u32 clutHash, u32 *rawClut);
-	ID3D11SamplerState *GetClutSampler() { return clutSampler; }
 	void Clear();
 	void Decimate();
 
@@ -62,7 +61,6 @@ private:
 	ID3D11DeviceContext *context_;
 	ID3D11VertexShader *vertexShader_ = nullptr;
 	ID3D11InputLayout *inputLayout_ = nullptr;
-	ID3D11SamplerState *clutSampler = nullptr;
 
 	std::map<u32, DepalShaderD3D11 *> cache_;
 	std::map<u32, DepalTextureD3D11 *> texCache_;
diff --git a/GPU/D3D11/DrawEngineD3D11.h b/GPU/D3D11/DrawEngineD3D11.h
index 4a94a307df..c6031ae324 100644
--- a/GPU/D3D11/DrawEngineD3D11.h
+++ b/GPU/D3D11/DrawEngineD3D11.h
@@ -240,6 +240,8 @@ private:
 	std::map<uint32_t, ID3D11RasterizerState *> rasterCache_;
 
 	// Keep the depth state between ApplyDrawState and ApplyDrawStateLate
+	ID3D11RasterizerState *rasterState_;
+	ID3D11BlendState *blendState_;
 	ID3D11DepthStencilState *depthStencilState_;
 
 	// State keys
diff --git a/GPU/D3D11/StateMappingD3D11.cpp b/GPU/D3D11/StateMappingD3D11.cpp
index b8e2888a96..cd038e3459 100644
--- a/GPU/D3D11/StateMappingD3D11.cpp
+++ b/GPU/D3D11/StateMappingD3D11.cpp
@@ -329,7 +329,6 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 	if (vpAndScissor.dirtyProj) {
 		gstate_c.Dirty(DIRTY_PROJMATRIX);
 	}
-	context_->RSSetViewports(1, &vp);
 
 	D3D11_RECT &scissor = dynState_.scissor;
 	if (vpAndScissor.scissorEnable) {
@@ -344,7 +343,6 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 		scissor.right = framebufferManager_->GetRenderWidth();
 		scissor.bottom = framebufferManager_->GetRenderHeight();
 	}
-	context_->RSSetScissorRects(1, &scissor);
 
 	if (gstate_c.IsDirty(DIRTY_TEXTURE_IMAGE | DIRTY_TEXTURE_PARAMS) && !gstate.isModeClear() && gstate.isTextureMapEnabled()) {
 		textureCache_->SetTexture();
@@ -378,9 +376,7 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 		bs = blendIter->second;
 	}
 
-	float blendColor[4];
-	Uint8x4ToFloat4(blendColor, dynState_.blendColor);
-	context_->OMSetBlendState(bs, blendColor, 0xFFFFFFFF);
+	blendState_ = bs;
 
 	auto depthIter = depthStencilCache_.find(keys_.depthStencil.value);
 	if (depthIter == depthStencilCache_.end()) {
@@ -415,7 +411,7 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 	} else {
 		rs = rasterIter->second;
 	}
-	context_->RSSetState(rs);
+	rasterState_ = rs;
 }
 
 void DrawEngineD3D11::ApplyDrawStateLate(bool applyStencilRef, uint8_t stencilRef) {
@@ -429,5 +425,13 @@ void DrawEngineD3D11::ApplyDrawStateLate(bool applyStencilRef, uint8_t stencilRe
 		}
 		textureCache_->ApplyTexture();
 	}
+
+	// Need to do this AFTER ApplyTexture because the process of depalettization can ruin the blend state.
+	float blendColor[4];
+	Uint8x4ToFloat4(blendColor, dynState_.blendColor);
+	context_->RSSetViewports(1, &dynState_.viewport);
+	context_->RSSetScissorRects(1, &dynState_.scissor);
+	context_->RSSetState(rasterState_);
+	context_->OMSetBlendState(blendState_, blendColor, 0xFFFFFFFF);
 	context_->OMSetDepthStencilState(depthStencilState_, applyStencilRef ? stencilRef : dynState_.stencilRef);
 }
\ No newline at end of file
diff --git a/GPU/D3D11/TextureCacheD3D11.cpp b/GPU/D3D11/TextureCacheD3D11.cpp
index dfcb52ded6..a296231e5e 100644
--- a/GPU/D3D11/TextureCacheD3D11.cpp
+++ b/GPU/D3D11/TextureCacheD3D11.cpp
@@ -548,7 +548,6 @@ void TextureCacheD3D11::ApplyTextureFramebuffer(TexCacheEntry *entry, VirtualFra
 		shaderApply.Use(depalShaderCache_->GetDepalettizeVertexShader(), depalShaderCache_->GetInputLayout());
 
 		context_->PSSetShaderResources(1, 1, &clutTexture);
-		context_->PSSetSamplers(1, 1, &stockD3D11.samplerPoint2DWrap);
 		framebufferManagerD3D11_->BindFramebufferAsColorTexture(0, framebuffer, BINDFBCOLOR_SKIP_COPY);
 		context_->PSSetSamplers(0, 1, &stockD3D11.samplerPoint2DWrap);
 		draw_->BindFramebufferAsRenderTarget(depalFBO);