Correct NaN handling in fpu comparisons.

Core/MIPS/MIPSInt.cpp

@@ -27,6 +27,13 @@
 #include "../HLE/HLE.h"
 #include "../System.h"
 
+#ifdef __APPLE__
+using std::isnan;
+#endif
+#ifdef _MSC_VER
+#define isnan _isnan
+#endif
+
 #define R(i) (currentMIPS->r[i])
 #define RF(i) (*(float*)(&(currentMIPS->r[i])))
 #define F(i) (currentMIPS->f[i])
@@ -826,40 +833,51 @@ namespace MIPSInt
 
 	void Int_FPUComp(u32 op)
 	{
-		// TODO: NaN handling seems wrong.
 		int fs = _FS;
 		int ft = _FT;
 		bool cond;
 		switch (op & 0xf)
 		{
 		case 0: //f
-		case 1: //un
 		case 8: //sf
-		case 9: //ngle
 			cond = false;
 			break;
 
+		case 1: //un
+		case 9: //ngle
+			cond = isnan(F(fs)) || isnan(F(ft));
+			break;
+
 		case 2: //eq
 		case 10: //seq
-		case 3: //ueq
-		case 11: //ngl
 			cond = (F(fs) == F(ft));
 			break;
 
+		case 3: //ueq
+		case 11: //ngl
+			cond = (F(fs) == F(ft)) || isnan(F(fs)) || isnan(F(ft));
+			break;
+
 		case 4: //olt
-		case 5: //ult
 		case 12: //lt
-		case 13: //nge
 			cond = (F(fs) < F(ft));
 			break;
 
+		case 5: //ult
+		case 13: //nge
+			cond = (F(fs) < F(ft)) || isnan(F(fs)) || isnan(F(ft));
+			break;
+
 		case 6: //ole
-		case 7: //ule
 		case 14: //le
-		case 15: //ngt
 			cond = (F(fs) <= F(ft));
 			break;
 
+		case 7: //ule
+		case 15: //ngt
+			cond = (F(fs) <= F(ft)) || isnan(F(fs)) || isnan(F(ft));
+			break;
+
 		default:
 			_dbg_assert_msg_(CPU,0,"Trying to interpret FPUComp instruction that can't be interpreted");
 			cond = false;

Core/MIPS/x86/CompFPU.cpp

@@ -150,47 +150,87 @@ void Jit::Comp_FPULS(u32 op)
 static const u64 GC_ALIGNED16(ssSignBits2[2])	= {0x8000000080000000ULL, 0x8000000080000000ULL};
 static const u64 GC_ALIGNED16(ssNoSignMask[2]) = {0x7FFFFFFF7FFFFFFFULL, 0x7FFFFFFF7FFFFFFFULL};
 
-void Jit::Comp_FPUComp(u32 op) {
+static u32 ssCompareTemp;
+
+enum
+{
+	CMPEQSS = 0,
+	CMPLTSS = 1,
+	CMPLESS = 2,
+	CMPUNORDSS = 3,
+	CMPNEQSS = 4,
+	CMPNLTSS = 5,
+	CMPNLESS = 6,
+	CMPORDSS = 7,
+};
+
+void Jit::CompFPComp(int lhs, int rhs, u8 compare, bool allowNaN)
+{
+	CONDITIONAL_DISABLE;
+
+	MOVSS(XMM0, fpr.R(lhs));
+	CMPSS(XMM0, fpr.R(rhs), compare);
+	MOVSS(M((void *) &currentMIPS->fpcond), XMM0);
+
+	// This means that NaN also means true, e.g. !<> or !>, etc.
+	if (allowNaN)
+	{
+		MOVSS(XMM0, fpr.R(lhs));
+		CMPSS(XMM0, fpr.R(rhs), CMPUNORDSS);
+		MOVSS(M((void *) &ssCompareTemp), XMM0);
+
+		MOV(32, R(EAX), M((void *) &ssCompareTemp));
+		OR(32, M((void *) &currentMIPS->fpcond), R(EAX));
+	}
+}
+
+void Jit::Comp_FPUComp(u32 op)
+{
 	CONDITIONAL_DISABLE;
 
 	int fs = _FS;
 	int ft = _FT;
 
-	// TODO: NaN handling seems wrong.
 	switch (op & 0xf)
 	{
 	case 0: //f
-	case 1: //un
 	case 8: //sf
-	case 9: //ngle
 		MOV(32, M((void *) &currentMIPS->fpcond), Imm32(0));
 		break;
 
+	case 1: //un
+	case 9: //ngle
+		CompFPComp(fs, ft, CMPUNORDSS);
+		break;
+
 	case 2: //eq
-	case 3: //ueq
 	case 10: //seq
+		CompFPComp(fs, ft, CMPEQSS);
+		break;
+
+	case 3: //ueq
 	case 11: //ngl
-		MOVSS(XMM0, fpr.R(fs));
+		CompFPComp(fs, ft, CMPEQSS, true);
-		CMPSS(XMM0, fpr.R(ft), 0);
-		MOVSS(M((void *) &currentMIPS->fpcond), XMM0);
 		break;
 
 	case 4: //olt
-	case 5: //ult
 	case 12: //lt
+		CompFPComp(fs, ft, CMPLTSS);
+		break;
+
+	case 5: //ult
 	case 13: //nge
-		MOVSS(XMM0, fpr.R(fs));
+		CompFPComp(fs, ft, CMPLTSS, true);
-		CMPSS(XMM0, fpr.R(ft), 1);
-		MOVSS(M((void *) &currentMIPS->fpcond), XMM0);
 		break;
 
 	case 6: //ole
-	case 7: //ule
 	case 14: //le
+		CompFPComp(fs, ft, CMPLESS);
+		break;
+
+	case 7: //ule
 	case 15: //ngt
-		MOVSS(XMM0, fpr.R(fs));
+		CompFPComp(fs, ft, CMPLESS, true);
-		CMPSS(XMM0, fpr.R(ft), 2);
-		MOVSS(M((void *) &currentMIPS->fpcond), XMM0);
 		break;
 
 	default:

Core/MIPS/x86/Jit.h

@@ -183,6 +183,7 @@ private:
 	void CompITypeMemWrite(u32 op, u32 bits, void *safeFunc);
 
 	void CompFPTriArith(u32 op, void (XEmitter::*arith)(X64Reg reg, OpArg), bool orderMatters);
+	void CompFPComp(int lhs, int rhs, u8 compare, bool allowNaN = false);
 
 	JitBlockCache blocks;
 	JitOptions jo;