From 3fe542cf6072614a9201d9d4dcc0b47c815f6275 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 9 Feb 2019 15:36:50 +0100 Subject: [PATCH 1/4] gl_shader_disk_cache: Remove per shader compression --- .../renderer_opengl/gl_shader_disk_cache.cpp | 55 ++++--------------- .../renderer_opengl/gl_shader_disk_cache.h | 1 - 2 files changed, 11 insertions(+), 45 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 53752b38d1..170d53e832 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -247,20 +247,12 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { return {}; u32 binary_length{}; - u32 compressed_size{}; - if (file.ReadBytes(&binary_length, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&compressed_size, sizeof(u32)) != sizeof(u32)) { + if (file.ReadBytes(&binary_length, sizeof(u32)) != sizeof(u32)) { return {}; } - std::vector compressed_binary(compressed_size); - if (file.ReadArray(compressed_binary.data(), compressed_binary.size()) != - compressed_binary.size()) { - return {}; - } - - dump.binary = Common::Compression::DecompressDataZSTD(compressed_binary); - if (dump.binary.empty()) { + dump.binary.resize(binary_length); + if (file.ReadArray(dump.binary.data(), binary_length) != binary_length) { return {}; } @@ -277,21 +269,15 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { std::optional ShaderDiskCacheOpenGL::LoadDecompiledEntry( FileUtil::IOFile& file) { u32 code_size{}; - u32 compressed_code_size{}; - if (file.ReadBytes(&code_size, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&compressed_code_size, sizeof(u32)) != sizeof(u32)) { + if (file.ReadBytes(&code_size, sizeof(u32)) != sizeof(u32)) { return {}; } - std::vector compressed_code(compressed_code_size); - if (file.ReadArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) { + std::vector code(code_size); + if (file.ReadArray(code.data(), code.size()) != code_size) { return {}; } - const std::vector code = Common::Compression::DecompressDataZSTD(compressed_code); - if (code.empty()) { - return {}; - } ShaderDiskCacheDecompiled entry; entry.code = std::string(reinterpret_cast(code.data()), code_size); @@ -369,13 +355,11 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn bool ShaderDiskCacheOpenGL::SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, const std::string& code, - const std::vector& compressed_code, const GLShader::ShaderEntries& entries) { if (file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)) != 1 || file.WriteObject(unique_identifier) != 1 || file.WriteObject(static_cast(code.size())) != 1 || - file.WriteObject(static_cast(compressed_code.size())) != 1 || - file.WriteArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) { + file.WriteArray(code.data(), code.size()) != code.size()) { return false; } @@ -485,19 +469,12 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str if (!IsUsable()) return; - const std::vector compressed_code{Common::Compression::CompressDataZSTDDefault( - reinterpret_cast(code.data()), code.size())}; - if (compressed_code.empty()) { - LOG_ERROR(Render_OpenGL, "Failed to compress GLSL code - skipping shader {:016x}", - unique_identifier); - return; - } - FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) return; - if (!SaveDecompiledFile(file, unique_identifier, code, compressed_code, entries)) { + if (!SaveDecompiledFile(file, unique_identifier, code, entries)) { LOG_ERROR(Render_OpenGL, "Failed to save decompiled entry to the precompiled file - removing"); file.Close(); @@ -516,25 +493,15 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p std::vector binary(binary_length); glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); - const std::vector compressed_binary = - Common::Compression::CompressDataZSTDDefault(binary.data(), binary.size()); - - if (compressed_binary.empty()) { - LOG_ERROR(Render_OpenGL, "Failed to compress binary program in shader={:016x}", - usage.unique_identifier); - return; - } - FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) return; if (file.WriteObject(static_cast(PrecompiledEntryKind::Dump)) != 1 || file.WriteObject(usage) != 1 || file.WriteObject(static_cast(binary_format)) != 1 || file.WriteObject(static_cast(binary_length)) != 1 || - file.WriteObject(static_cast(compressed_binary.size())) != 1 || - file.WriteArray(compressed_binary.data(), compressed_binary.size()) != - compressed_binary.size()) { + file.WriteArray(binary.data(), binary.size()) != binary_length) { LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing", usage.unique_identifier); file.Close(); diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 6be0c0547b..a2864c5e29 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -201,7 +201,6 @@ private: /// Saves a decompiled entry to the passed file. Returns true on success. bool SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, const std::string& code, - const std::vector& compressed_code, const GLShader::ShaderEntries& entries); /// Returns if the cache can be used From 9db2c734c95d1c5c99d6ead95673de72e9389d8d Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 25 Feb 2019 11:47:23 +0100 Subject: [PATCH 2/4] gl_shader_disk_cache: Use VectorVfsFile for the virtual precompiled shader cache file --- .../renderer_opengl/gl_shader_cache.cpp | 12 +- .../renderer_opengl/gl_shader_disk_cache.cpp | 204 ++++++++++-------- .../renderer_opengl/gl_shader_disk_cache.h | 53 ++++- 3 files changed, 168 insertions(+), 101 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 2a81b11690..dd18052dbe 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -363,6 +363,10 @@ void ShaderCacheOpenGL::LoadDiskCache(const std::atomic_bool& stop_loading, if (stop_loading) return; + // Track if precompiled cache was altered during loading to know if we have to serialize the + // virtual precompiled cache file back to the hard drive + bool precompiled_cache_altered = false; + // Build shaders if (callback) callback(VideoCore::LoadCallbackStage::Build, 0, usages.size()); @@ -384,6 +388,7 @@ void ShaderCacheOpenGL::LoadDiskCache(const std::atomic_bool& stop_loading, if (!shader) { // Invalidate the precompiled cache if a shader dumped shader was rejected disk_cache.InvalidatePrecompiled(); + precompiled_cache_altered = true; dumps.clear(); } } @@ -405,8 +410,13 @@ void ShaderCacheOpenGL::LoadDiskCache(const std::atomic_bool& stop_loading, if (dumps.find(usage) == dumps.end()) { const auto& program = precompiled_programs.at(usage); disk_cache.SaveDump(usage, program->handle); + precompiled_cache_altered = true; } } + + if (precompiled_cache_altered) { + disk_cache.SaveVirtualPrecompiledFile(); + } } CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( @@ -524,4 +534,4 @@ Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { return last_shaders[static_cast(program)] = shader; } -} // namespace OpenGL +} // namespace OpenGL \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 170d53e832..4c76f943f2 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -11,7 +11,6 @@ #include "common/file_util.h" #include "common/logging/log.h" #include "common/scm_rev.h" -#include "common/zstd_compression.h" #include "core/core.h" #include "core/hle/kernel/process.h" @@ -104,7 +103,8 @@ bool ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { return true; } -ShaderDiskCacheOpenGL::ShaderDiskCacheOpenGL(Core::System& system) : system{system} {} +ShaderDiskCacheOpenGL::ShaderDiskCacheOpenGL(Core::System& system) + : system{system}, precompiled_cache_virtual_file_offset{0} {} std::optional, std::vector>> ShaderDiskCacheOpenGL::LoadTransferable() { @@ -177,6 +177,7 @@ ShaderDiskCacheOpenGL::LoadTransferable() { return {}; } } + return {{raws, usages}}; } @@ -208,51 +209,62 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { std::optional, std::unordered_map>> ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { + std::vector precompiled_cache(file.GetSize()); + file.ReadBytes(precompiled_cache.data(), precompiled_cache.size()); + SaveArrayToPrecompiled(precompiled_cache.data(), precompiled_cache.size()); + precompiled_cache_virtual_file_offset = 0; + ShaderCacheVersionHash file_hash{}; - if (file.ReadArray(file_hash.data(), file_hash.size()) != file_hash.size()) { + if (!LoadArrayFromPrecompiled(file_hash.data(), file_hash.size())) { + precompiled_cache_virtual_file_offset = 0; return {}; } if (GetShaderCacheVersionHash() != file_hash) { LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator"); + precompiled_cache_virtual_file_offset = 0; return {}; } std::unordered_map decompiled; std::unordered_map dumps; - while (file.Tell() < file.GetSize()) { + while (precompiled_cache_virtual_file_offset < precompiled_cache_virtual_file.GetSize()) { PrecompiledEntryKind kind{}; - if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) { + if (!LoadObjectFromPrecompiled(kind)) { return {}; } switch (kind) { case PrecompiledEntryKind::Decompiled: { u64 unique_identifier{}; - if (file.ReadBytes(&unique_identifier, sizeof(u64)) != sizeof(u64)) + if (!LoadObjectFromPrecompiled(unique_identifier)) { return {}; + } - const auto entry = LoadDecompiledEntry(file); - if (!entry) + const auto entry = LoadDecompiledEntry(); + if (!entry) { return {}; + } decompiled.insert({unique_identifier, std::move(*entry)}); break; } case PrecompiledEntryKind::Dump: { ShaderDiskCacheUsage usage; - if (file.ReadBytes(&usage, sizeof(usage)) != sizeof(usage)) + if (!LoadObjectFromPrecompiled(usage)) { return {}; + } ShaderDiskCacheDump dump; - if (file.ReadBytes(&dump.binary_format, sizeof(u32)) != sizeof(u32)) + if (!LoadObjectFromPrecompiled(dump.binary_format)) { return {}; + } u32 binary_length{}; - if (file.ReadBytes(&binary_length, sizeof(u32)) != sizeof(u32)) { + if (!LoadObjectFromPrecompiled(binary_length)) { return {}; } dump.binary.resize(binary_length); - if (file.ReadArray(dump.binary.data(), binary_length) != binary_length) { + if (!LoadArrayFromPrecompiled(dump.binary.data(), dump.binary.size())) { return {}; } @@ -266,15 +278,14 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { return {{decompiled, dumps}}; } -std::optional ShaderDiskCacheOpenGL::LoadDecompiledEntry( - FileUtil::IOFile& file) { +std::optional ShaderDiskCacheOpenGL::LoadDecompiledEntry() { u32 code_size{}; - if (file.ReadBytes(&code_size, sizeof(u32)) != sizeof(u32)) { + if (!LoadObjectFromPrecompiled(code_size)) { return {}; } std::vector code(code_size); - if (file.ReadArray(code.data(), code.size()) != code_size) { + if (!LoadArrayFromPrecompiled(code.data(), code.size())) { return {}; } @@ -282,23 +293,26 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn entry.code = std::string(reinterpret_cast(code.data()), code_size); u32 const_buffers_count{}; - if (file.ReadBytes(&const_buffers_count, sizeof(u32)) != sizeof(u32)) + if (!LoadObjectFromPrecompiled(const_buffers_count)) { return {}; + } + for (u32 i = 0; i < const_buffers_count; ++i) { u32 max_offset{}; u32 index{}; u8 is_indirect{}; - if (file.ReadBytes(&max_offset, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&index, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&is_indirect, sizeof(u8)) != sizeof(u8)) { + if (!LoadObjectFromPrecompiled(max_offset) || !LoadObjectFromPrecompiled(index) || + !LoadObjectFromPrecompiled(is_indirect)) { return {}; } entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index); } u32 samplers_count{}; - if (file.ReadBytes(&samplers_count, sizeof(u32)) != sizeof(u32)) + if (!LoadObjectFromPrecompiled(samplers_count)) { return {}; + } + for (u32 i = 0; i < samplers_count; ++i) { u64 offset{}; u64 index{}; @@ -306,12 +320,9 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn u8 is_array{}; u8 is_shadow{}; u8 is_bindless{}; - if (file.ReadBytes(&offset, sizeof(u64)) != sizeof(u64) || - file.ReadBytes(&index, sizeof(u64)) != sizeof(u64) || - file.ReadBytes(&type, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&is_array, sizeof(u8)) != sizeof(u8) || - file.ReadBytes(&is_shadow, sizeof(u8)) != sizeof(u8) || - file.ReadBytes(&is_bindless, sizeof(u8)) != sizeof(u8)) { + if (!LoadObjectFromPrecompiled(offset) || !LoadObjectFromPrecompiled(index) || + !LoadObjectFromPrecompiled(type) || !LoadObjectFromPrecompiled(is_array) || + !LoadObjectFromPrecompiled(is_shadow) || !LoadObjectFromPrecompiled(is_bindless)) { return {}; } entry.entries.samplers.emplace_back(static_cast(offset), @@ -321,17 +332,17 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn } u32 global_memory_count{}; - if (file.ReadBytes(&global_memory_count, sizeof(u32)) != sizeof(u32)) + if (!LoadObjectFromPrecompiled(global_memory_count)) { return {}; + } + for (u32 i = 0; i < global_memory_count; ++i) { u32 cbuf_index{}; u32 cbuf_offset{}; u8 is_read{}; u8 is_written{}; - if (file.ReadBytes(&cbuf_index, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&cbuf_offset, sizeof(u32)) != sizeof(u32) || - file.ReadBytes(&is_read, sizeof(u8)) != sizeof(u8) || - file.ReadBytes(&is_written, sizeof(u8)) != sizeof(u8)) { + if (!LoadObjectFromPrecompiled(cbuf_index) || !LoadObjectFromPrecompiled(cbuf_offset) || + !LoadObjectFromPrecompiled(is_read) || !LoadObjectFromPrecompiled(is_written)) { return {}; } entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset, is_read != 0, @@ -340,72 +351,81 @@ std::optional ShaderDiskCacheOpenGL::LoadDecompiledEn for (auto& clip_distance : entry.entries.clip_distances) { u8 clip_distance_raw{}; - if (file.ReadBytes(&clip_distance_raw, sizeof(u8)) != sizeof(u8)) + if (!LoadObjectFromPrecompiled(clip_distance_raw)) return {}; clip_distance = clip_distance_raw != 0; } u64 shader_length{}; - if (file.ReadBytes(&shader_length, sizeof(u64)) != sizeof(u64)) + if (!LoadObjectFromPrecompiled(shader_length)) { return {}; + } + entry.entries.shader_length = static_cast(shader_length); return entry; } -bool ShaderDiskCacheOpenGL::SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, - const std::string& code, +bool ShaderDiskCacheOpenGL::SaveDecompiledFile(u64 unique_identifier, const std::string& code, const GLShader::ShaderEntries& entries) { - if (file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)) != 1 || - file.WriteObject(unique_identifier) != 1 || - file.WriteObject(static_cast(code.size())) != 1 || - file.WriteArray(code.data(), code.size()) != code.size()) { + if (!SaveObjectToPrecompiled(static_cast(PrecompiledEntryKind::Decompiled)) || + !SaveObjectToPrecompiled(unique_identifier) || + !SaveObjectToPrecompiled(static_cast(code.size())) || + !SaveArrayToPrecompiled(code.data(), code.size())) { return false; } - if (file.WriteObject(static_cast(entries.const_buffers.size())) != 1) + if (!SaveObjectToPrecompiled(static_cast(entries.const_buffers.size()))) { return false; + } for (const auto& cbuf : entries.const_buffers) { - if (file.WriteObject(static_cast(cbuf.GetMaxOffset())) != 1 || - file.WriteObject(static_cast(cbuf.GetIndex())) != 1 || - file.WriteObject(static_cast(cbuf.IsIndirect() ? 1 : 0)) != 1) { + if (!SaveObjectToPrecompiled(static_cast(cbuf.GetMaxOffset())) || + !SaveObjectToPrecompiled(static_cast(cbuf.GetIndex())) || + !SaveObjectToPrecompiled(static_cast(cbuf.IsIndirect() ? 1 : 0))) { return false; } } - if (file.WriteObject(static_cast(entries.samplers.size())) != 1) + if (!SaveObjectToPrecompiled(static_cast(entries.samplers.size()))) { return false; + } for (const auto& sampler : entries.samplers) { - if (file.WriteObject(static_cast(sampler.GetOffset())) != 1 || - file.WriteObject(static_cast(sampler.GetIndex())) != 1 || - file.WriteObject(static_cast(sampler.GetType())) != 1 || - file.WriteObject(static_cast(sampler.IsArray() ? 1 : 0)) != 1 || - file.WriteObject(static_cast(sampler.IsShadow() ? 1 : 0)) != 1 || - file.WriteObject(static_cast(sampler.IsBindless() ? 1 : 0)) != 1) { + if (!SaveObjectToPrecompiled(static_cast(sampler.GetOffset())) || + !SaveObjectToPrecompiled(static_cast(sampler.GetIndex())) || + !SaveObjectToPrecompiled(static_cast(sampler.GetType())) || + !SaveObjectToPrecompiled(static_cast(sampler.IsArray() ? 1 : 0)) || + !SaveObjectToPrecompiled(static_cast(sampler.IsShadow() ? 1 : 0)) || + !SaveObjectToPrecompiled(static_cast(sampler.IsBindless() ? 1 : 0))) { return false; } } - if (file.WriteObject(static_cast(entries.global_memory_entries.size())) != 1) + if (!SaveObjectToPrecompiled(static_cast(entries.global_memory_entries.size()))) { return false; + } for (const auto& gmem : entries.global_memory_entries) { - if (file.WriteObject(static_cast(gmem.GetCbufIndex())) != 1 || - file.WriteObject(static_cast(gmem.GetCbufOffset())) != 1 || - file.WriteObject(static_cast(gmem.IsRead() ? 1 : 0)) != 1 || - file.WriteObject(static_cast(gmem.IsWritten() ? 1 : 0)) != 1) { + if (!SaveObjectToPrecompiled(static_cast(gmem.GetCbufIndex())) || + !SaveObjectToPrecompiled(static_cast(gmem.GetCbufOffset())) || + !SaveObjectToPrecompiled(static_cast(gmem.IsRead() ? 1 : 0)) || + !SaveObjectToPrecompiled(static_cast(gmem.IsWritten() ? 1 : 0))) { return false; } } for (const bool clip_distance : entries.clip_distances) { - if (file.WriteObject(static_cast(clip_distance ? 1 : 0)) != 1) + if (!SaveObjectToPrecompiled(static_cast(clip_distance ? 1 : 0))) { return false; + } } - return file.WriteObject(static_cast(entries.shader_length)) == 1; + if (!SaveObjectToPrecompiled(static_cast(entries.shader_length))) { + return false; + } + + return true; } -void ShaderDiskCacheOpenGL::InvalidateTransferable() const { +void ShaderDiskCacheOpenGL::InvalidateTransferable() { if (!FileUtil::Delete(GetTransferablePath())) { LOG_ERROR(Render_OpenGL, "Failed to invalidate transferable file={}", GetTransferablePath()); @@ -413,7 +433,10 @@ void ShaderDiskCacheOpenGL::InvalidateTransferable() const { InvalidatePrecompiled(); } -void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { +void ShaderDiskCacheOpenGL::InvalidatePrecompiled() { + // Clear virtaul precompiled cache file + precompiled_cache_virtual_file.Resize(0); + if (!FileUtil::Delete(GetPrecompiledPath())) { LOG_ERROR(Render_OpenGL, "Failed to invalidate precompiled file={}", GetPrecompiledPath()); } @@ -469,15 +492,13 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str if (!IsUsable()) return; - FileUtil::IOFile file = AppendPrecompiledFile(); + if (precompiled_cache_virtual_file.GetSize() == 0) { + SavePrecompiledHeaderToVirtualPrecompiledCache(); + } - if (!file.IsOpen()) - return; - - if (!SaveDecompiledFile(file, unique_identifier, code, entries)) { + if (!SaveDecompiledFile(unique_identifier, code, entries)) { LOG_ERROR(Render_OpenGL, "Failed to save decompiled entry to the precompiled file - removing"); - file.Close(); InvalidatePrecompiled(); } } @@ -493,18 +514,13 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p std::vector binary(binary_length); glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); - FileUtil::IOFile file = AppendPrecompiledFile(); - - if (!file.IsOpen()) - return; - - if (file.WriteObject(static_cast(PrecompiledEntryKind::Dump)) != 1 || - file.WriteObject(usage) != 1 || file.WriteObject(static_cast(binary_format)) != 1 || - file.WriteObject(static_cast(binary_length)) != 1 || - file.WriteArray(binary.data(), binary.size()) != binary_length) { + if (!SaveObjectToPrecompiled(static_cast(PrecompiledEntryKind::Dump)) || + !SaveObjectToPrecompiled(usage) || + !SaveObjectToPrecompiled(static_cast(binary_format)) || + !SaveObjectToPrecompiled(static_cast(binary_length)) || + !SaveArrayToPrecompiled(binary.data(), binary.size())) { LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing", usage.unique_identifier); - file.Close(); InvalidatePrecompiled(); return; } @@ -537,28 +553,32 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { return file; } -FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const { - if (!EnsureDirectories()) - return {}; +void ShaderDiskCacheOpenGL::SavePrecompiledHeaderToVirtualPrecompiledCache() { + const auto hash{GetShaderCacheVersionHash()}; + if (!SaveArrayToPrecompiled(hash.data(), hash.size())) { + LOG_ERROR( + Render_OpenGL, + "Failed to write precompiled cache version hash to virtual precompiled cache file"); + } +} + +void ShaderDiskCacheOpenGL::SaveVirtualPrecompiledFile() { + precompiled_cache_virtual_file_offset = 0; + const std::vector& precompiled_cache = precompiled_cache_virtual_file.ReadAllBytes(); const auto precompiled_path{GetPrecompiledPath()}; - const bool existed = FileUtil::Exists(precompiled_path); + FileUtil::IOFile file(precompiled_path, "wb"); - FileUtil::IOFile file(precompiled_path, "ab"); if (!file.IsOpen()) { LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path); - return {}; + return; } - - if (!existed || file.GetSize() == 0) { - const auto hash{GetShaderCacheVersionHash()}; - if (file.WriteArray(hash.data(), hash.size()) != hash.size()) { - LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version hash in path={}", - precompiled_path); - return {}; - } + if (file.WriteBytes(precompiled_cache.data(), precompiled_cache.size()) != + precompiled_cache.size()) { + LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}", + precompiled_path); + return; } - return file; } bool ShaderDiskCacheOpenGL::EnsureDirectories() const { @@ -599,4 +619,4 @@ std::string ShaderDiskCacheOpenGL::GetTitleID() const { return fmt::format("{:016X}", system.CurrentProcess()->GetTitleID()); } -} // namespace OpenGL +} // namespace OpenGL \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index a2864c5e29..0142b2e3be 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -16,6 +16,7 @@ #include "common/assert.h" #include "common/common_types.h" +#include "core/file_sys/vfs_vector.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_opengl/gl_shader_gen.h" @@ -172,10 +173,10 @@ public: LoadPrecompiled(); /// Removes the transferable (and precompiled) cache file. - void InvalidateTransferable() const; + void InvalidateTransferable(); - /// Removes the precompiled cache file. - void InvalidatePrecompiled() const; + /// Removes the precompiled cache file and clears virtual precompiled cache file. + void InvalidatePrecompiled(); /// Saves a raw dump to the transferable file. Checks for collisions. void SaveRaw(const ShaderDiskCacheRaw& entry); @@ -190,17 +191,21 @@ public: /// Saves a dump entry to the precompiled file. Does not check for collisions. void SaveDump(const ShaderDiskCacheUsage& usage, GLuint program); + /// Serializes virtual precompiled shader cache file to real file + void SaveVirtualPrecompiledFile(); + private: /// Loads the transferable cache. Returns empty on failure. std::optional, std::unordered_map>> LoadPrecompiledFile(FileUtil::IOFile& file); - /// Loads a decompiled cache entry from the passed file. Returns empty on failure. - std::optional LoadDecompiledEntry(FileUtil::IOFile& file); + /// Loads a decompiled cache entry from m_precompiled_cache_virtual_file. Returns empty on + /// failure. + std::optional LoadDecompiledEntry(); /// Saves a decompiled entry to the passed file. Returns true on success. - bool SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, const std::string& code, + bool SaveDecompiledFile(u64 unique_identifier, const std::string& code, const GLShader::ShaderEntries& entries); /// Returns if the cache can be used @@ -209,8 +214,8 @@ private: /// Opens current game's transferable file and write it's header if it doesn't exist FileUtil::IOFile AppendTransferableFile() const; - /// Opens current game's precompiled file and write it's header if it doesn't exist - FileUtil::IOFile AppendPrecompiledFile() const; + /// Save precompiled header to precompiled_cache_in_memory + void SavePrecompiledHeaderToVirtualPrecompiledCache(); /// Create shader disk cache directories. Returns true on success. bool EnsureDirectories() const; @@ -233,10 +238,42 @@ private: /// Get current game's title id std::string GetTitleID() const; + template + bool SaveArrayToPrecompiled(const T* data, std::size_t length) { + const std::size_t write_length = precompiled_cache_virtual_file.WriteArray( + data, length, precompiled_cache_virtual_file_offset); + precompiled_cache_virtual_file_offset += write_length; + return write_length == sizeof(T) * length; + } + + template + bool LoadArrayFromPrecompiled(T* data, std::size_t length) { + const std::size_t read_length = precompiled_cache_virtual_file.ReadArray( + data, length, precompiled_cache_virtual_file_offset); + precompiled_cache_virtual_file_offset += read_length; + return read_length == sizeof(T) * length; + } + + template + bool SaveObjectToPrecompiled(const T& object) { + return SaveArrayToPrecompiled(&object, 1); + } + + template + bool LoadObjectFromPrecompiled(T& object) { + return LoadArrayFromPrecompiled(&object, 1); + } + // Copre system Core::System& system; // Stored transferable shaders std::map> transferable; + // Stores whole precompiled cache which will be read from or saved to the precompiled chache + // file + FileSys::VectorVfsFile precompiled_cache_virtual_file; + // Stores the current offset of the precompiled cache file for IO purposes + std::size_t precompiled_cache_virtual_file_offset; + // The cache has been loaded at boot bool tried_to_load{}; }; From 3091b40691d767161c0265bab4bddb6687e3698d Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 10 Feb 2019 16:38:34 +0100 Subject: [PATCH 3/4] gl_shader_disk_cache: Compress precompiled shader cache file with Zstandard --- .../renderer_opengl/gl_shader_disk_cache.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 4c76f943f2..3c39e5c8cd 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -11,6 +11,7 @@ #include "common/file_util.h" #include "common/logging/log.h" #include "common/scm_rev.h" +#include "common/zstd_compression.h" #include "core/core.h" #include "core/hle/kernel/process.h" @@ -209,9 +210,11 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { std::optional, std::unordered_map>> ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { - std::vector precompiled_cache(file.GetSize()); - file.ReadBytes(precompiled_cache.data(), precompiled_cache.size()); - SaveArrayToPrecompiled(precompiled_cache.data(), precompiled_cache.size()); + // Read compressed file from disk and decompress to virtual precompiled cache file + std::vector compressed(file.GetSize()); + file.ReadBytes(compressed.data(), compressed.size()); + const std::vector decompressed = Common::Compression::DecompressDataZSTD(compressed); + SaveArrayToPrecompiled(decompressed.data(), decompressed.size()); precompiled_cache_virtual_file_offset = 0; ShaderCacheVersionHash file_hash{}; @@ -564,7 +567,9 @@ void ShaderDiskCacheOpenGL::SavePrecompiledHeaderToVirtualPrecompiledCache() { void ShaderDiskCacheOpenGL::SaveVirtualPrecompiledFile() { precompiled_cache_virtual_file_offset = 0; - const std::vector& precompiled_cache = precompiled_cache_virtual_file.ReadAllBytes(); + const std::vector& uncompressed = precompiled_cache_virtual_file.ReadAllBytes(); + const std::vector& compressed = + Common::Compression::CompressDataZSTDDefault(uncompressed.data(), uncompressed.size()); const auto precompiled_path{GetPrecompiledPath()}; FileUtil::IOFile file(precompiled_path, "wb"); @@ -573,8 +578,7 @@ void ShaderDiskCacheOpenGL::SaveVirtualPrecompiledFile() { LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path); return; } - if (file.WriteBytes(precompiled_cache.data(), precompiled_cache.size()) != - precompiled_cache.size()) { + if (file.WriteBytes(compressed.data(), compressed.size()) != compressed.size()) { LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}", precompiled_path); return; From 1a3ff252a4d41dedee69735259f3b4707347af0f Mon Sep 17 00:00:00 2001 From: FreddyFunk Date: Tue, 23 Apr 2019 23:19:28 +0200 Subject: [PATCH 4/4] Re added new lines at the end of files --- src/video_core/renderer_opengl/gl_shader_cache.cpp | 2 +- src/video_core/renderer_opengl/gl_shader_disk_cache.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index dd18052dbe..b1c8f7c358 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -534,4 +534,4 @@ Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { return last_shaders[static_cast(program)] = shader; } -} // namespace OpenGL \ No newline at end of file +} // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 3c39e5c8cd..ed7afc4a00 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -623,4 +623,4 @@ std::string ShaderDiskCacheOpenGL::GetTitleID() const { return fmt::format("{:016X}", system.CurrentProcess()->GetTitleID()); } -} // namespace OpenGL \ No newline at end of file +} // namespace OpenGL