From cec7da37b90804d60dd508dcb9aac648a0f5a590 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 7 Feb 2019 19:40:39 +0100 Subject: [PATCH 01/10] core: Do not link LZ4 to core. Use common/data_compression for nso segment decompression instead. --- src/core/loader/nso.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp index df2fb7b7f1..2ca9bf63c2 100644 --- a/src/core/loader/nso.cpp +++ b/src/core/loader/nso.cpp @@ -5,6 +5,7 @@ #include #include +#include "common/data_compression.h" #include "common/common_funcs.h" #include "common/file_util.h" #include "common/hex_util.h" From a05f94dcc8001cc2b90da89f34db5adfb3356f04 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Feb 2019 06:45:50 +0100 Subject: [PATCH 02/10] Addressed feedback --- src/core/loader/nso.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp index 2ca9bf63c2..df2fb7b7f1 100644 --- a/src/core/loader/nso.cpp +++ b/src/core/loader/nso.cpp @@ -5,7 +5,6 @@ #include #include -#include "common/data_compression.h" #include "common/common_funcs.h" #include "common/file_util.h" #include "common/hex_util.h" From 93de7a7b40114259c4e3255d6567c519c8300fa1 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Feb 2019 22:08:48 +0100 Subject: [PATCH 03/10] externals: Add Zstandard v1.3.8 --- .gitmodules | 3 +++ externals/zstd | 1 + 2 files changed, 4 insertions(+) create mode 160000 externals/zstd diff --git a/.gitmodules b/.gitmodules index 2558a5ebc1..b891805de2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -40,3 +40,6 @@ [submodule "Vulkan-Headers"] path = externals/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "externals/zstd"] + path = externals/zstd + url = https://github.com/facebook/zstd diff --git a/externals/zstd b/externals/zstd new file mode 160000 index 0000000000..470344d33e --- /dev/null +++ b/externals/zstd @@ -0,0 +1 @@ +Subproject commit 470344d33e1d52a2ada75d278466da8d4ee2faf6 From d85c1141b9fd0133bb2d988ad3c7b5369a6fac3a Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Feb 2019 23:03:28 +0100 Subject: [PATCH 04/10] externals: Add libzstd_static to externals CMakeLists.txt --- externals/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index e156bbece4..aa3319eb11 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -49,6 +49,10 @@ add_subdirectory(open_source_archives EXCLUDE_FROM_ALL) add_library(unicorn-headers INTERFACE) target_include_directories(unicorn-headers INTERFACE ./unicorn/include) +# Zstandard +add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL) +target_include_directories(libzstd_static INTERFACE ./zstd/lib) + # SoundTouch add_subdirectory(soundtouch) From ca82589350970effc06d5aa494e6fee865c36c8a Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Feb 2019 23:03:53 +0100 Subject: [PATCH 05/10] common: Link libzstd_static --- src/common/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 5639021d3e..b1c3b9b520 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -138,4 +138,4 @@ endif() create_target_directory_groups(common) target_link_libraries(common PUBLIC Boost::boost fmt microprofile) -target_link_libraries(common PRIVATE lz4_static) +target_link_libraries(common PRIVATE lz4_static libzstd_static) From 72477731ed20c56a4d6f18a22f43224fab667cef Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 8 Feb 2019 23:11:00 +0100 Subject: [PATCH 06/10] common/zstd_compression: Add Zstandard wrapper --- src/common/CMakeLists.txt | 2 ++ src/common/zstd_compression.cpp | 52 +++++++++++++++++++++++++++++++++ src/common/zstd_compression.h | 44 ++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 src/common/zstd_compression.cpp create mode 100644 src/common/zstd_compression.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index b1c3b9b520..1e8e1b215b 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -125,6 +125,8 @@ add_library(common STATIC uint128.h vector_math.h web_result.h + zstd_compression.cpp + zstd_compression.h ) if(ARCHITECTURE_x86_64) diff --git a/src/common/zstd_compression.cpp b/src/common/zstd_compression.cpp new file mode 100644 index 0000000000..f6cf7a791a --- /dev/null +++ b/src/common/zstd_compression.cpp @@ -0,0 +1,52 @@ +// Copyright 2019 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "common/assert.h" +#include "common/zstd_compression.h" + +namespace Common::Compression { + +std::vector CompressDataZSTD(const u8* source, std::size_t source_size, s32 compression_level) { + + compression_level = std::clamp(compression_level, 1, ZSTD_maxCLevel()); + + const std::size_t max_compressed_size = ZSTD_compressBound(source_size); + std::vector compressed(max_compressed_size); + + const std::size_t compressed_size = + ZSTD_compress(compressed.data(), compressed.size(), source, source_size, compression_level); + + if (ZSTD_isError(compressed_size)) { + // Compression failed + return {}; + } + + compressed.resize(compressed_size); + + return compressed; +} + +std::vector CompressDataZSTDDefault(const u8* source, std::size_t source_size) { + return CompressDataZSTD(source, source_size, ZSTD_CLEVEL_DEFAULT); +} + +std::vector DecompressDataZSTD(const std::vector& compressed, + std::size_t uncompressed_size) { + std::vector uncompressed(uncompressed_size); + const std::size_t uncompressed_result_size = ZSTD_decompress( + uncompressed.data(), uncompressed.size(), compressed.data(), compressed.size()); + + if (uncompressed_size != uncompressed_result_size || ZSTD_isError(uncompressed_result_size)) { + // Decompression failed + return {}; + } + return uncompressed; +} + +} // namespace Common::Compression diff --git a/src/common/zstd_compression.h b/src/common/zstd_compression.h new file mode 100644 index 0000000000..c011ac34bf --- /dev/null +++ b/src/common/zstd_compression.h @@ -0,0 +1,44 @@ +// Copyright 2019 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include + +#include "common/common_types.h" + +namespace Common::Compression { + +/** + * Compresses a source memory region with Zstandard and returns the compressed data in a vector. + * + * @param source the uncompressed source memory region. + * @param source_size the size in bytes of the uncompressed source memory region. + * @param compression_level the used compression level. Should be between 1 and 22. + * + * @return the compressed data. + */ +std::vector CompressDataZSTD(const u8* source, std::size_t source_size, s32 compression_level); + +/** + * Compresses a source memory region with Zstandard with the default compression level and returns + * the compressed data in a vector. + * + * @param source the uncompressed source memory region. + * @param source_size the size in bytes of the uncompressed source memory region. + * + * @return the compressed data. + */ +std::vector CompressDataZSTDDefault(const u8* source, std::size_t source_size); + +/** + * Decompresses a source memory region with Zstandard and returns the uncompressed data in a vector. + * + * @param compressed the compressed source memory region. + * @param uncompressed_size the size in bytes of the uncompressed data. + * + * @return the decompressed data. + */ +std::vector DecompressDataZSTD(const std::vector& compressed, + std::size_t uncompressed_size); + +} // namespace Common::Compression \ No newline at end of file From 35ebbbc167986552f5fb4784cc53214ddcca505a Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 9 Feb 2019 09:05:47 +0100 Subject: [PATCH 07/10] gl_shader_disk_cache: Use Zstandard for compression --- .../renderer_opengl/gl_shader_disk_cache.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index d2d9799975..e74ffd2915 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -10,7 +10,7 @@ #include "common/common_types.h" #include "common/file_util.h" #include "common/logging/log.h" -#include "common/lz4_compression.h" +#include "common/zstd_compression.h" #include "common/scm_rev.h" #include "core/core.h" @@ -259,7 +259,7 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { return {}; } - dump.binary = Common::Compression::DecompressDataLZ4(compressed_binary, binary_length); + dump.binary = Common::Compression::DecompressDataZSTD(compressed_binary, binary_length); if (dump.binary.empty()) { return {}; } @@ -288,7 +288,7 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn return {}; } - const std::vector code = Common::Compression::DecompressDataLZ4(compressed_code, code_size); + const std::vector code = Common::Compression::DecompressDataZSTD(compressed_code, code_size); if (code.empty()) { return {}; } @@ -474,8 +474,8 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str if (!IsUsable()) return; - const std::vector compressed_code{Common::Compression::CompressDataLZ4HC( - reinterpret_cast(code.data()), code.size(), 9)}; + const std::vector compressed_code{Common::Compression::CompressDataZSTDDefault( + reinterpret_cast(code.data()), code.size())}; if (compressed_code.empty()) { LOG_ERROR(Render_OpenGL, "Failed to compress GLSL code - skipping shader {:016x}", unique_identifier); @@ -506,7 +506,7 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); const std::vector compressed_binary = - Common::Compression::CompressDataLZ4HC(binary.data(), binary.size(), 9); + Common::Compression::CompressDataZSTDDefault(binary.data(), binary.size()); if (compressed_binary.empty()) { LOG_ERROR(Render_OpenGL, "Failed to compress binary program in shader={:016x}", From aa92da205e3f975dffc3c706fb0a176f13649492 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 9 Feb 2019 10:20:52 +0100 Subject: [PATCH 08/10] gl_shader_disk_cache: Fixup clang format --- src/video_core/renderer_opengl/gl_shader_disk_cache.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index e74ffd2915..ec32602adb 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -10,8 +10,8 @@ #include "common/common_types.h" #include "common/file_util.h" #include "common/logging/log.h" -#include "common/zstd_compression.h" #include "common/scm_rev.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/hle/kernel/process.h" @@ -288,7 +288,8 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn return {}; } - const std::vector code = Common::Compression::DecompressDataZSTD(compressed_code, code_size); + const std::vector code = + Common::Compression::DecompressDataZSTD(compressed_code, code_size); if (code.empty()) { return {}; } From b4857e326fdf4c980c88ca399e7d12f99253777e Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 10 Feb 2019 10:28:04 +0100 Subject: [PATCH 09/10] common/zstd_compression: simplify decompression interface --- src/common/zstd_compression.cpp | 17 +++++++++-------- src/common/zstd_compression.h | 4 +--- .../renderer_opengl/gl_shader_disk_cache.cpp | 5 ++--- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/common/zstd_compression.cpp b/src/common/zstd_compression.cpp index f6cf7a791a..60a35c67cc 100644 --- a/src/common/zstd_compression.cpp +++ b/src/common/zstd_compression.cpp @@ -13,7 +13,6 @@ namespace Common::Compression { std::vector CompressDataZSTD(const u8* source, std::size_t source_size, s32 compression_level) { - compression_level = std::clamp(compression_level, 1, ZSTD_maxCLevel()); const std::size_t max_compressed_size = ZSTD_compressBound(source_size); @@ -36,17 +35,19 @@ std::vector CompressDataZSTDDefault(const u8* source, std::size_t source_siz return CompressDataZSTD(source, source_size, ZSTD_CLEVEL_DEFAULT); } -std::vector DecompressDataZSTD(const std::vector& compressed, - std::size_t uncompressed_size) { - std::vector uncompressed(uncompressed_size); - const std::size_t uncompressed_result_size = ZSTD_decompress( - uncompressed.data(), uncompressed.size(), compressed.data(), compressed.size()); +std::vector DecompressDataZSTD(const std::vector& compressed) { + const std::size_t decompressed_size = + ZSTD_getDecompressedSize(compressed.data(), compressed.size()); + std::vector decompressed(decompressed_size); - if (uncompressed_size != uncompressed_result_size || ZSTD_isError(uncompressed_result_size)) { + const std::size_t uncompressed_result_size = ZSTD_decompress( + decompressed.data(), decompressed.size(), compressed.data(), compressed.size()); + + if (decompressed_size != uncompressed_result_size || ZSTD_isError(uncompressed_result_size)) { // Decompression failed return {}; } - return uncompressed; + return decompressed; } } // namespace Common::Compression diff --git a/src/common/zstd_compression.h b/src/common/zstd_compression.h index c011ac34bf..e0a64b0355 100644 --- a/src/common/zstd_compression.h +++ b/src/common/zstd_compression.h @@ -34,11 +34,9 @@ std::vector CompressDataZSTDDefault(const u8* source, std::size_t source_siz * Decompresses a source memory region with Zstandard and returns the uncompressed data in a vector. * * @param compressed the compressed source memory region. - * @param uncompressed_size the size in bytes of the uncompressed data. * * @return the decompressed data. */ -std::vector DecompressDataZSTD(const std::vector& compressed, - std::size_t uncompressed_size); +std::vector DecompressDataZSTD(const std::vector& compressed); } // namespace Common::Compression \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index ec32602adb..8a43eb1576 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -259,7 +259,7 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { return {}; } - dump.binary = Common::Compression::DecompressDataZSTD(compressed_binary, binary_length); + dump.binary = Common::Compression::DecompressDataZSTD(compressed_binary); if (dump.binary.empty()) { return {}; } @@ -288,8 +288,7 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn return {}; } - const std::vector code = - Common::Compression::DecompressDataZSTD(compressed_code, code_size); + const std::vector code = Common::Compression::DecompressDataZSTD(compressed_code); if (code.empty()) { return {}; } From eadc834bb307381aa6bcdc11bde678f027a05ff8 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 13 Feb 2019 15:53:57 +0100 Subject: [PATCH 10/10] gitmodules: Replace taps with spaces --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index b891805de2..26b4e5272b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -41,5 +41,5 @@ path = externals/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git [submodule "externals/zstd"] - path = externals/zstd - url = https://github.com/facebook/zstd + path = externals/zstd + url = https://github.com/facebook/zstd