diff --git a/Core/HW/SimpleAudioDec.cpp b/Core/HW/SimpleAudioDec.cpp
index ed897dcbc9..472011f224 100644
--- a/Core/HW/SimpleAudioDec.cpp
+++ b/Core/HW/SimpleAudioDec.cpp
@@ -351,7 +351,8 @@ u32 AuCtx::AuDecode(u32 pcmAddr) {
 			// get consumed source length
 			int srcPos = decoder->GetSourcePos() + nextSync;
 			// remove the consumed source
-			sourcebuff.erase(0, srcPos);
+			if (srcPos > 0)
+				sourcebuff.erase(sourcebuff.begin(), sourcebuff.begin() + srcPos);
 			// reduce the available Aubuff size
 			// (the available buff size is now used to know if we can read again from file and how many to read)
 			AuBufAvailable -= srcPos;
@@ -430,7 +431,8 @@ u32 AuCtx::AuNotifyAddStreamData(int size) {
 	}
 
 	if (Memory::IsValidRange(AuBuf, size)) {
-		sourcebuff.append((const char *)Memory::GetPointer(AuBuf + offset), size);
+		sourcebuff.resize(sourcebuff.size() + size);
+		Memory::MemcpyUnchecked(&sourcebuff[sourcebuff.size() - size], AuBuf + offset, size);
 	}
 
 	return 0;
diff --git a/Core/HW/SimpleAudioDec.h b/Core/HW/SimpleAudioDec.h
index 1adad1c2cf..1dc8aa28e5 100644
--- a/Core/HW/SimpleAudioDec.h
+++ b/Core/HW/SimpleAudioDec.h
@@ -119,7 +119,11 @@ public:
 	void DoState(PointerWrap &p);
 
 	void EatSourceBuff(int amount) {
-		sourcebuff.erase(0, amount);
+		if (amount > sourcebuff.size()) {
+			amount = (int)sourcebuff.size();
+		}
+		if (amount > 0)
+			sourcebuff.erase(sourcebuff.begin(), sourcebuff.begin() + amount);
 		AuBufAvailable -= amount;
 	}
 	// Au source information. Written to from for example sceAacInit so public for now.
@@ -155,7 +159,7 @@ public:
 private:
 	size_t FindNextMp3Sync();
 
-	std::string sourcebuff; // source buffer
+	std::vector<u8> sourcebuff; // source buffer
 };