From 1e9a391f650ee7b5fec654678a486e8f79f49e48 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Fri, 6 Aug 2021 23:14:37 -0700 Subject: [PATCH 1/3] SaveState: Add const for save compression. --- Common/Serialize/Serializer.cpp | 55 +++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/Common/Serialize/Serializer.cpp b/Common/Serialize/Serializer.cpp index a04448cc8a..7ba0ad2183 100644 --- a/Common/Serialize/Serializer.cpp +++ b/Common/Serialize/Serializer.cpp @@ -25,6 +25,14 @@ #include "Common/File/FileUtil.h" #include "Common/StringUtils.h" +enum class SerializeCompressType { + NONE = 0, + SNAPPY = 1, + ZSTD = 2, +}; + +static constexpr SerializeCompressType SAVE_TYPE = SerializeCompressType::SNAPPY; + PointerWrapSection PointerWrap::Section(const char *title, int ver) { return Section(title, ver, ver); } @@ -289,12 +297,17 @@ CChunkFileReader::Error CChunkFileReader::LoadFile(const Path &filename, std::st u8 *uncomp_buffer = new u8[header.UncompressedSize]; size_t uncomp_size = header.UncompressedSize; bool success = false; - if (header.Compress == 1) { + if (SerializeCompressType(header.Compress) == SerializeCompressType::SNAPPY) { auto status = snappy_uncompress((const char *)buffer, sz, (char *)uncomp_buffer, &uncomp_size); success = status == SNAPPY_OK; - } else { - auto status = ZSTD_decompress((char *)uncomp_buffer, uncomp_size, (const char *)buffer, sz); + } else if (SerializeCompressType(header.Compress) == SerializeCompressType::ZSTD) { + size_t status = ZSTD_decompress((char *)uncomp_buffer, uncomp_size, (const char *)buffer, sz); success = !ZSTD_isError(status); + if (success) { + uncomp_size = status; + } + } else { + ERROR_LOG(SAVESTATE, "ChunkReader: Unexpected compression type %d", header.Compress); } if (!success) { ERROR_LOG(SAVESTATE, "ChunkReader: Failed to decompress file"); @@ -336,16 +349,40 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const Path &filename, const s } // Make sure we can allocate a buffer to compress before compressing. - size_t write_len = ZSTD_compressBound(sz); - u8 *compressed_buffer = (u8 *)malloc(write_len); + size_t write_len; + SerializeCompressType usedType = SAVE_TYPE; + switch (usedType) { + case SerializeCompressType::NONE: + write_len = 0; + break; + case SerializeCompressType::SNAPPY: + write_len = snappy_max_compressed_length(sz); + break; + case SerializeCompressType::ZSTD: + write_len = ZSTD_compressBound(sz); + break; + } + u8 *compressed_buffer = write_len == 0 ? nullptr : (u8 *)malloc(write_len); u8 *write_buffer = buffer; if (!compressed_buffer) { - ERROR_LOG(SAVESTATE, "ChunkReader: Unable to allocate compressed buffer"); + if (write_len != 0) + ERROR_LOG(SAVESTATE, "ChunkReader: Unable to allocate compressed buffer"); // We'll save uncompressed. Better than not saving... write_len = sz; + usedType = SerializeCompressType::NONE; } else { - // TODO: If free disk space is low, we could max this out to 22? - write_len = ZSTD_compress(compressed_buffer, write_len, buffer, sz, 1); + switch (usedType) { + case SerializeCompressType::NONE: + _assert_(false); + break; + case SerializeCompressType::SNAPPY: + snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &write_len); + break; + case SerializeCompressType::ZSTD: + // TODO: If free disk space is low, we could max this out to 22? + write_len = ZSTD_compress(compressed_buffer, write_len, buffer, sz, 1); + break; + } free(buffer); write_buffer = compressed_buffer; @@ -353,7 +390,7 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const Path &filename, const s // Create header SChunkHeader header{}; - header.Compress = compressed_buffer ? 2 : 0; + header.Compress = (int)usedType; header.Revision = REVISION_CURRENT; header.ExpectedSize = (u32)write_len; header.UncompressedSize = (u32)sz; From 0097772d3c01c0ac604c047a1baa7da9e93ec2ad Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Fri, 6 Aug 2021 23:42:43 -0700 Subject: [PATCH 2/3] SaveState: Ask to include xxhash in frames. Hopefully this should detect any decompression errors better. --- Common/Serialize/Serializer.cpp | 34 +++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/Common/Serialize/Serializer.cpp b/Common/Serialize/Serializer.cpp index 7ba0ad2183..78f4ac1342 100644 --- a/Common/Serialize/Serializer.cpp +++ b/Common/Serialize/Serializer.cpp @@ -31,7 +31,7 @@ enum class SerializeCompressType { ZSTD = 2, }; -static constexpr SerializeCompressType SAVE_TYPE = SerializeCompressType::SNAPPY; +static constexpr SerializeCompressType SAVE_TYPE = SerializeCompressType::ZSTD; PointerWrapSection PointerWrap::Section(const char *title, int ver) { return Section(title, ver, ver); @@ -371,21 +371,43 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const Path &filename, const s write_len = sz; usedType = SerializeCompressType::NONE; } else { + bool success = true; switch (usedType) { case SerializeCompressType::NONE: _assert_(false); break; case SerializeCompressType::SNAPPY: - snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &write_len); + success = snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &write_len) == SNAPPY_OK; break; case SerializeCompressType::ZSTD: - // TODO: If free disk space is low, we could max this out to 22? - write_len = ZSTD_compress(compressed_buffer, write_len, buffer, sz, 1); + { + auto ctx = ZSTD_createCCtx(); + if (!ctx) { + success = false; + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, 1); + ZSTD_CCtx_setParameter(ctx, ZSTD_c_checksumFlag, 1); + ZSTD_CCtx_setPledgedSrcSize(ctx, sz); + // TODO: If free disk space is low, we could max this out to 22? + write_len = ZSTD_compress2(ctx, compressed_buffer, write_len, buffer, sz); + success = !ZSTD_isError(write_len); + } + ZSTD_freeCCtx(ctx); + } break; } - free(buffer); - write_buffer = compressed_buffer; + if (success) { + free(buffer); + write_buffer = compressed_buffer; + } else { + ERROR_LOG(SAVESTATE, "ChunkReader: Compression failed"); + free(compressed_buffer); + + // We can still save uncompressed. + write_len = sz; + usedType = SerializeCompressType::NONE; + } } // Create header From dc12b3a7c8839c8e1dc1f331c8070ed38cba9458 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Fri, 6 Aug 2021 23:46:07 -0700 Subject: [PATCH 3/3] SaveState: Use default ZSTD compress level. --- Common/Serialize/Serializer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/Serialize/Serializer.cpp b/Common/Serialize/Serializer.cpp index 78f4ac1342..6000328a75 100644 --- a/Common/Serialize/Serializer.cpp +++ b/Common/Serialize/Serializer.cpp @@ -385,10 +385,10 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const Path &filename, const s if (!ctx) { success = false; } else { - ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, 1); + // TODO: If free disk space is low, we could max this out to 22? + ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT); ZSTD_CCtx_setParameter(ctx, ZSTD_c_checksumFlag, 1); ZSTD_CCtx_setPledgedSrcSize(ctx, sz); - // TODO: If free disk space is low, we could max this out to 22? write_len = ZSTD_compress2(ctx, compressed_buffer, write_len, buffer, sz); success = !ZSTD_isError(write_len); }